From 7c7d0ee25e8ad3f83d635dec264965d2d216d8f0 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:32 +0200 Subject: [PATCH 01/30] Fix ReadBufferFromFileView position tracking (B115) ReadBufferFromFileView assumed the inner buffer keeps its working buffer across setReadUntilPosition; ReadBufferFromS3 resets it, so getPosition lied and seek logic could re-read a stale decompressed block. Recompute the offset from the inner buffer after every right-bound change. Includes the ReadBufferFromMemory counterpart and gtest batteries. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- src/IO/ReadBufferFromFileView.cpp | 63 +++- src/IO/ReadBufferFromMemory.cpp | 5 +- .../gtest_read_buffer_from_file_view.cpp | 280 ++++++++++++++++++ .../tests/gtest_read_buffer_from_memory.cpp | 19 ++ 4 files changed, 357 insertions(+), 10 deletions(-) create mode 100644 src/IO/tests/gtest_read_buffer_from_file_view.cpp create mode 100644 src/IO/tests/gtest_read_buffer_from_memory.cpp diff --git a/src/IO/ReadBufferFromFileView.cpp b/src/IO/ReadBufferFromFileView.cpp index 1304f372df06..22de4731673f 100644 --- a/src/IO/ReadBufferFromFileView.cpp +++ b/src/IO/ReadBufferFromFileView.cpp @@ -19,11 +19,13 @@ ReadBufferFromFileView::ReadBufferFromFileView( , file_offset_of_buffer_end(left_bound_) , original_working_buffer(working_buffer) { - /// Seek to the begin of file. + /// Seek to the begin of file. The impl still owns its native buffer state here (no swap yet), + /// so its buffer-end offset can be read directly after the seek. impl->seek(left_bound, SEEK_SET); + const size_t impl_buffer_end = impl->getPosition() + impl->available(); swap(*impl); - file_offset_of_buffer_end += available(); + file_offset_of_buffer_end = impl_buffer_end; original_working_buffer = working_buffer; resizeWorkingBuffer(); } @@ -40,14 +42,31 @@ void ReadBufferFromFileView::setReadUntilPosition(size_t position) throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Cannot read until position: {}. File size is {}", position, getFileSize()); - executeWithOriginalBuffer([&]{ impl->setReadUntilPosition(*read_until_position); }); + /// The impl is allowed to DISCARD its working buffer here (e.g. `ReadBufferFromS3` rebases its + /// offset to the consumer position and resets the buffer when the range changes), so the view's + /// buffer-end offset MUST be rebased from the impl's post-op state - keeping the stale value + /// over a replaced buffer silently shifts the reported position by the discarded bytes. + size_t impl_buffer_end = 0; + executeWithOriginalBuffer([&] + { + impl->setReadUntilPosition(*read_until_position); + impl_buffer_end = impl->getPosition() + impl->available(); + }); + file_offset_of_buffer_end = impl_buffer_end; resizeWorkingBuffer(); } void ReadBufferFromFileView::setReadUntilEnd() { read_until_position.reset(); - executeWithOriginalBuffer([&]{ impl->setReadUntilPosition(right_bound); }); + /// Same rebase contract as setReadUntilPosition. + size_t impl_buffer_end = 0; + executeWithOriginalBuffer([&] + { + impl->setReadUntilPosition(right_bound); + impl_buffer_end = impl->getPosition() + impl->available(); + }); + file_offset_of_buffer_end = impl_buffer_end; resizeWorkingBuffer(); } @@ -63,11 +82,19 @@ bool ReadBufferFromFileView::nextImpl() return false; bool result = false; - executeWithOriginalBuffer([&] { result = impl->next(); }); + size_t impl_buffer_end = 0; + executeWithOriginalBuffer([&] + { + result = impl->next(); + impl_buffer_end = impl->getPosition() + impl->available(); + }); if (result) { - file_offset_of_buffer_end += available(); + /// Rebase from the impl's own accounting instead of incrementing: the view's previous + /// buffer-end may have been clamped by resizeWorkingBuffer below the impl's real one, and + /// the impl continues from ITS position - incrementing would mislabel the new chunk. + file_offset_of_buffer_end = impl_buffer_end; resizeWorkingBuffer(); } @@ -87,7 +114,12 @@ off_t ReadBufferFromFileView::seek(off_t off, int whence) throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "ReadBufferFromFileView::seek expects SEEK_SET or SEEK_CUR as whence"); off_t result = 0; - executeWithOriginalBuffer([&] { result = impl->seek(new_pos, SEEK_SET); }); + size_t impl_buffer_end = 0; + executeWithOriginalBuffer([&] + { + result = impl->seek(new_pos, SEEK_SET); + impl_buffer_end = impl->getPosition() + impl->available(); + }); if (result < 0) throw Exception(ErrorCodes::SEEK_POSITION_OUT_OF_BOUND, "Seek position ({}) underflow", result); @@ -96,7 +128,7 @@ off_t ReadBufferFromFileView::seek(off_t off, int whence) throw Exception(ErrorCodes::SEEK_POSITION_OUT_OF_BOUND, "Seek position ({}) is out of bound. Available range: [{}, {}]", result, left_bound, right_bound); - file_offset_of_buffer_end = result + available(); + file_offset_of_buffer_end = impl_buffer_end; resizeWorkingBuffer(); return result - left_bound; @@ -110,7 +142,20 @@ void ReadBufferFromFileView::executeWithOriginalBuffer(Op && op) /// Set working buffer and other internal into impl. swap(*impl); - op(); + try + { + op(); + } + catch (...) + { + /// The swap MUST be undone even if `op` throws — otherwise `this` and `impl` are left holding + /// each other's working buffers (and a stale `original_working_buffer`), so any subsequent + /// read or seek over-reads / serves wrong bytes. `op` can throw (e.g. setReadUntilPosition / + /// seek bound checks), so restore-on-exception is required for the view to stay consistent. + swap(*impl); + original_working_buffer = working_buffer; + throw; + } swap(*impl); original_working_buffer = working_buffer; diff --git a/src/IO/ReadBufferFromMemory.cpp b/src/IO/ReadBufferFromMemory.cpp index 882f8b6a07d3..9f3c20fc51e2 100644 --- a/src/IO/ReadBufferFromMemory.cpp +++ b/src/IO/ReadBufferFromMemory.cpp @@ -76,7 +76,10 @@ ReadBufferFromMemoryFileBase::ReadBufferFromMemoryFileBase(bool owns_memory, { chassert(data.size() == internal_buffer.size()); - if (owns_memory) + /// memcpy's pointers are __attribute__((nonnull)) even when the length is 0. An empty file yields + /// data.data() == nullptr, so guard on non-empty to avoid the nonnull-attribute UB the asan_ubsan + /// lane aborts on (STID 5930-5afa). Nothing to copy when empty. + if (owns_memory && !data.empty()) std::memcpy(internal_buffer.begin(), data.data(), data.size()); working_buffer = internal_buffer; diff --git a/src/IO/tests/gtest_read_buffer_from_file_view.cpp b/src/IO/tests/gtest_read_buffer_from_file_view.cpp new file mode 100644 index 000000000000..b4a433361856 --- /dev/null +++ b/src/IO/tests/gtest_read_buffer_from_file_view.cpp @@ -0,0 +1,280 @@ +#include + +#include +#include + +#include + +using namespace DB; + +namespace +{ + +/// How the inner buffer reacts to setReadUntilPosition - the axis that broke B115. +enum class InnerMode : uint8_t +{ + /// Like local file descriptors: setReadUntilPosition is a no-op, the buffer is kept. + FileLike, + /// Like ReadBufferFromS3: a range change rebases the offset to the CONSUMER position and + /// DISCARDS the working buffer (the next nextImpl re-fetches from the consumer position). + RemoteLike, +}; + +/// A seekable ReadBufferFromFileBase over a string, reading at most `chunk` bytes per nextImpl, +/// with selectable setReadUntilPosition semantics. Mirrors the state conventions of real +/// implementations: `file_offset` is the absolute offset of working_buffer.end(). +class FakeInnerBuffer : public ReadBufferFromFileBase +{ +public: + FakeInnerBuffer(String data_, size_t chunk, InnerMode mode_) + : ReadBufferFromFileBase(chunk, nullptr, 0) + , data(std::move(data_)) + , mode(mode_) + { + } + + String getFileName() const override { return "fake_inner"; } + std::optional tryGetFileSize() override { return data.size(); } + size_t getFileOffsetOfBufferEnd() const override { return file_offset; } + off_t getPosition() override { return file_offset - available(); } + + off_t seek(off_t off, int whence) override + { + EXPECT_EQ(whence, SEEK_SET); + const size_t target = static_cast(off); + /// In-buffer seek (both real local and S3 buffers do this). + if (!working_buffer.empty() && target + working_buffer.size() >= file_offset && target < file_offset) + { + pos = working_buffer.end() - (file_offset - target); + return off; + } + resetWorkingBuffer(); + file_offset = target; + return off; + } + + void setReadUntilPosition(size_t position) override + { + if (read_until && *read_until == position) + return; + if (mode == InnerMode::RemoteLike) + { + /// ReadBufferFromS3: offset = getPosition(); resetWorkingBuffer(); impl.reset(); + file_offset = getPosition(); + resetWorkingBuffer(); + } + read_until = position; + } + + void setReadUntilEnd() override { setReadUntilPosition(data.size()); } + +private: + bool nextImpl() override + { + const size_t limit = read_until ? std::min(*read_until, data.size()) : data.size(); + if (file_offset >= limit) + return false; + const size_t to_read = std::min(limit - file_offset, internal_buffer.size()); + memcpy(internal_buffer.begin(), data.data() + file_offset, to_read); + working_buffer = Buffer(internal_buffer.begin(), internal_buffer.begin() + to_read); + file_offset += to_read; + return true; + } + + String data; + InnerMode mode; + size_t file_offset = 0; + std::optional read_until; +}; + +constexpr size_t kHeader = 256; /// the view's left bound (the CHCA envelope size in production) + +String makePayload(size_t size) +{ + String s(size, 0); + for (size_t i = 0; i < size; ++i) + s[i] = static_cast((i * 131 + 7) % 251); + return s; +} + +std::unique_ptr makeView(const String & payload, size_t chunk, InnerMode mode) +{ + String object = String(kHeader, '\xee') + payload; + auto inner = std::make_unique(std::move(object), chunk, mode); + return std::make_unique(std::move(inner), "viewed", kHeader, kHeader + payload.size()); +} + +String readExact(ReadBuffer & buf, size_t n) +{ + String out(n, 0); + buf.readStrict(out.data(), n); + return out; +} + +struct Case +{ + size_t chunk; + InnerMode mode; +}; + +class ReadBufferFromFileViewTest : public ::testing::TestWithParam +{ +}; + +} + +TEST_P(ReadBufferFromFileViewTest, SequentialReadWholeView) +{ + const auto [chunk, mode] = GetParam(); + const auto payload = makePayload(1000); + auto view = makeView(payload, chunk, mode); + + EXPECT_EQ(readExact(*view, payload.size()), payload); + EXPECT_TRUE(view->eof()); + EXPECT_EQ(view->getPosition(), static_cast(payload.size())); +} + +TEST_P(ReadBufferFromFileViewTest, SeekAndRead) +{ + const auto [chunk, mode] = GetParam(); + const auto payload = makePayload(1000); + auto view = makeView(payload, chunk, mode); + + for (size_t target : {size_t(0), size_t(700), size_t(20), size_t(21), size_t(999), size_t(5)}) + { + EXPECT_EQ(view->seek(target, SEEK_SET), static_cast(target)); + EXPECT_EQ(view->getPosition(), static_cast(target)); + EXPECT_EQ(readExact(*view, 1), payload.substr(target, 1)); + EXPECT_EQ(view->getPosition(), static_cast(target + 1)); + } +} + +/// B115 regression. The in-order MergeTree reader adjusts the right mark (setReadUntilPosition) +/// while the consumer is mid-buffer. A remote-like inner buffer legitimately discards its working +/// buffer on the range change; the view MUST keep reporting the consumer's position - before the +/// fix it teleported forward by the discarded bytes, so the next seek was treated as "already +/// there" and a stale block was re-served (duplicated + missing granules at the SQL level). +TEST_P(ReadBufferFromFileViewTest, SetReadUntilPositionMidBufferKeepsPosition) +{ + const auto [chunk, mode] = GetParam(); + const auto payload = makePayload(1000); + auto view = makeView(payload, chunk, mode); + + EXPECT_EQ(readExact(*view, 36), payload.substr(0, 36)); + EXPECT_EQ(view->getPosition(), 36); + + view->setReadUntilPosition(72); + EXPECT_EQ(view->getPosition(), 36) << "position must survive a right-bound change"; + + /// The consumer's next seek to its current position must be a no-op... + EXPECT_EQ(view->seek(36, SEEK_SET), 36); + /// ...and the bytes must continue from 36, not from a stale buffer. + EXPECT_EQ(readExact(*view, 36), payload.substr(36, 36)); +} + +/// Truncate-then-extend: the right bound shrinks below already-buffered data, the consumer reads +/// up to it, the bound is extended again. The continuation must produce the file's real bytes +/// (before the fix the view's incremental buffer-end accounting drifted from the inner buffer's). +TEST_P(ReadBufferFromFileViewTest, SetReadUntilTruncateThenExtend) +{ + const auto [chunk, mode] = GetParam(); + const auto payload = makePayload(1000); + auto view = makeView(payload, chunk, mode); + + EXPECT_EQ(readExact(*view, 10), payload.substr(0, 10)); + + view->setReadUntilPosition(30); + EXPECT_EQ(view->getPosition(), 10); + EXPECT_EQ(readExact(*view, 20), payload.substr(10, 20)); + EXPECT_TRUE(view->eof()); + EXPECT_EQ(view->getPosition(), 30); + + view->setReadUntilPosition(500); + EXPECT_EQ(view->getPosition(), 30); + EXPECT_EQ(readExact(*view, 100), payload.substr(30, 100)); + + view->setReadUntilEnd(); + EXPECT_EQ(readExact(*view, payload.size() - 130), payload.substr(130)); + EXPECT_TRUE(view->eof()); +} + +/// The exact shape of the failing compact-part in-order read: per granule, adjust the right +/// mark, seek to the granule's block, read it. Every block must contain its own bytes. +TEST_P(ReadBufferFromFileViewTest, GranulePatternRegression) +{ + const auto [chunk, mode] = GetParam(); + constexpr size_t block = 36; + constexpr size_t blocks = 20; + const auto payload = makePayload(block * blocks); + auto view = makeView(payload, chunk, mode); + + for (size_t g = 0; g < blocks; ++g) + { + view->setReadUntilPosition(std::min((g + 2) * block, payload.size())); + EXPECT_EQ(view->seek(g * block, SEEK_SET), static_cast(g * block)); + EXPECT_EQ(readExact(*view, block), payload.substr(g * block, block)) << "block " << g; + } +} + +/// Randomized conformance battery against a golden model. +TEST_P(ReadBufferFromFileViewTest, RandomizedOps) +{ + const auto [chunk, mode] = GetParam(); + const auto payload = makePayload(2000); + + for (unsigned seed = 1; seed <= 5; ++seed) + { + auto view = makeView(payload, chunk, mode); + size_t model_pos = 0; + size_t model_until = payload.size(); + unsigned rng = seed; + auto next_rand = [&rng] { rng = rng * 1103515245 + 12345; return (rng >> 8) % 1000; }; + + for (int step = 0; step < 300; ++step) + { + switch (next_rand() % 3) + { + case 0: /// read up to the current until-bound + { + const size_t want = next_rand() % 64; + const size_t n = std::min(want, model_until - model_pos); + if (n) + { + ASSERT_EQ(readExact(*view, n), payload.substr(model_pos, n)) << "seed " << seed << " step " << step; + model_pos += n; + } + break; + } + case 1: /// seek (never beyond the current until-bound - the consumer contract: + /// the right mark always covers the ranges being read) + { + const size_t target = next_rand() % (model_until + 1); + ASSERT_EQ(view->seek(target, SEEK_SET), static_cast(target)); + model_pos = target; + break; + } + case 2: /// move the right bound (never below the consumer position) + { + const size_t until = model_pos + next_rand() % (payload.size() - model_pos + 1); + view->setReadUntilPosition(until); + model_until = until; + break; + } + default: + UNREACHABLE(); + } + ASSERT_EQ(view->getPosition(), static_cast(model_pos)) << "seed " << seed << " step " << step; + } + } +} + +INSTANTIATE_TEST_SUITE_P( + ChunksAndModes, + ReadBufferFromFileViewTest, + ::testing::Values( + Case{7, InnerMode::FileLike}, + Case{7, InnerMode::RemoteLike}, + Case{108, InnerMode::FileLike}, + Case{108, InnerMode::RemoteLike}, + Case{1 << 20, InnerMode::FileLike}, + Case{1 << 20, InnerMode::RemoteLike})); diff --git a/src/IO/tests/gtest_read_buffer_from_memory.cpp b/src/IO/tests/gtest_read_buffer_from_memory.cpp new file mode 100644 index 000000000000..b7955f816e79 --- /dev/null +++ b/src/IO/tests/gtest_read_buffer_from_memory.cpp @@ -0,0 +1,19 @@ +#include + +#include + +#include + +using namespace DB; + +/// An empty file materialized into an OWNED in-memory buffer must construct without undefined +/// behaviour: std::memcpy's pointer arguments are __attribute__((nonnull)), so memcpy(dst, nullptr, 0) +/// -- which an empty std::string_view (data() == nullptr) produces -- is UB that the asan_ubsan lane +/// aborts on (STID 5930-5afa, PR #2073). The buffer must construct and be immediately at EOF. +TEST(ReadBufferFromMemoryFileBase, EmptyOwnedBufferConstructsWithoutUB) +{ + /// ReadBufferFromMemoryFileBase's constructor is protected; ReadBufferFromOwnMemoryFile is the + /// public concrete class that always passes owns_memory=true, exercising the guarded memcpy path. + ReadBufferFromOwnMemoryFile buf("empty", std::string_view{}); + EXPECT_TRUE(buf.eof()); +} From 3d86e1cee068ba60b98307bdf3969d873b890d44 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:32 +0200 Subject: [PATCH 02/30] Stop retrying S3 reads after query cancellation (B117) processException kept retrying transient errors after KILL QUERY; check CurrentThread::get().isQueryCanceled() in the outer retry loop. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- src/IO/ReadBufferFromS3.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index f2d184fd8e71..34f1f2261891 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -353,6 +354,12 @@ bool ReadBufferFromS3::processException(size_t read_offset, size_t attempt) cons bucket, key, version_id.empty() ? "Latest" : version_id, read_offset, attempt, request_settings[S3RequestSetting::max_single_read_retries].value, getCurrentExceptionMessage(/* with_stacktrace = */ false)); + /// Stop retrying once the query is cancelled (B117): otherwise a killed query's reads keep + /// retrying a transient error (e.g. a dropped connection) for many attempts with backoff, + /// zombying for minutes and adding load. The SDK's own RetryStrategy makes the same check + /// (src/IO/S3/Client.cpp), but this outer ReadBufferFromS3 retry loop did not. + if (CurrentThread::isInitialized() && CurrentThread::get().isQueryCanceled()) + return false; if (auto * s3_exception = current_exception_cast()) { From 002f387b4f6872cd8868e6bb7b8f347ca799b6f3 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:32 +0200 Subject: [PATCH 03/30] Retain parent ThreadGroup from borrowed children (B90) A borrowed child ThreadGroup parents its trackers at the parent group via raw pointers; background work outliving the query produced a use-after-free in parent counters. The child now holds a shared_ptr to the parent group. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- src/Common/ThreadStatus.h | 8 ++++++++ src/Interpreters/ThreadStatusExt.cpp | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 975349e2dde5..ef3b73048e9d 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -84,6 +84,14 @@ class ThreadGroup const Int32 os_threads_nice_value; + /// A borrowed child group (materialized view / async-insert flush) parents its `memory_tracker` + /// and `performance_counters` at the parent group's trackers via RAW pointers. Retain a shared_ptr + /// to the parent so those trackers cannot be freed while any thread is still attached to this child + /// group — otherwise a detached task (e.g. an S3 upload scheduled via `threadPoolCallbackRunnerUnsafe`) + /// that attaches the child group can walk a freed parent tracker chain (use-after-free, B90). Null for + /// a top-level query/background group, whose parent is a process-lifetime tracker (user/total/background). + ThreadGroupPtr parent_thread_group; + MemorySpillScheduler::Ptr memory_spill_scheduler; ProfileEvents::Counters performance_counters{VariableContext::Process}; MemoryTracker memory_tracker{VariableContext::Process}; diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 813254ef842b..1ac5c8aa6050 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -129,6 +129,8 @@ ThreadGroup::ThreadGroup(ThreadGroupPtr parent) , global_context(parent->global_context) , fatal_error_callback(parent->fatal_error_callback) , os_threads_nice_value(parent->os_threads_nice_value) + /// Keep the parent group alive: this child parents its trackers at the parent's via raw pointers (B90). + , parent_thread_group(parent) , memory_spill_scheduler(parent->memory_spill_scheduler) , performance_counters(VariableContext::Process, &parent->performance_counters) , memory_tracker(&parent->memory_tracker, VariableContext::Process, /*log_peak_memory_usage_in_destructor*/ false) @@ -143,6 +145,8 @@ ThreadGroup::ThreadGroup(ContextPtr query_context_, ThreadGroupPtr parent) , global_context(query_context_->getGlobalContext()) , fatal_error_callback(parent->fatal_error_callback) , os_threads_nice_value(parent->os_threads_nice_value) + /// Keep the parent group alive: this child parents its trackers at the parent's via raw pointers (B90). + , parent_thread_group(parent) , memory_spill_scheduler(parent->memory_spill_scheduler) , performance_counters(VariableContext::Process, &parent->performance_counters) , memory_tracker(&parent->memory_tracker, VariableContext::Process, /*log_peak_memory_usage_in_destructor*/ false) From a5a76b9a12daecdec7cb086c7250f7ee748466d5 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:32 +0200 Subject: [PATCH 04/30] Fail closed on a null MergeTreeDeduplicationLog writer (B37) addPart/dropPart guarded current_writer only with chassert (a release no-op), so a missing writer meant a null dereference; throw LOGICAL_ERROR instead. Also treats a missing logs_dir as normal for storages that do not materialize empty directories (carries a CAS-related hunk; wired later). Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../MergeTree/MergeTreeDeduplicationLog.cpp | 29 +++- .../gtest_deduplication_log_null_writer.cpp | 139 ++++++++++++++++++ 2 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 src/Storages/MergeTree/tests/gtest_deduplication_log_null_writer.cpp diff --git a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp index 9987e466c53b..a4ff5691bf4a 100644 --- a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp +++ b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp @@ -20,6 +20,7 @@ namespace DB namespace ErrorCodes { extern const int ABORTED; + extern const int LOGICAL_ERROR; } namespace @@ -103,8 +104,14 @@ void MergeTreeDeduplicationLog::load() { if (auto * object_storage = dynamic_cast(disk.get())) { - // MetadataStorageType::Plain does not have directory concept. When checking `logs_dir` existence, it might return false. - if (object_storage->getMetadataStorage()->getType() != MetadataStorageType::Plain) + // Plain and ContentAddressed object storages do not materialize empty directories, so a + // missing logs_dir is normal for a fresh table: fall through so the current_writer is still + // created (an INSERT must have a writer, else addPart fails closed). For these types a + // missing dir is NOT evidence of nothing to do; iterateDirectory below finds any logs that + // already exist, and rotate() creates the writer when there are none. Any other object + // storage returns here: a missing dir means there is genuinely nothing and nowhere to write. + const auto type = object_storage->getMetadataStorage()->getType(); + if (type != MetadataStorageType::Plain && type != MetadataStorageType::CAS) return; } } @@ -268,7 +275,15 @@ std::vector MergeTreeDeduplicationLog: throw Exception(ErrorCodes::ABORTED, "Storage has been shutdown when we add this part."); } - chassert(current_writer != nullptr); + /// A disk that cannot host the append-mode log leaves current_writer null; the release-build + /// chassert above is a no-op, so dereferencing it would segfault. Fail closed with a clear + /// exception instead of crashing the server (B37). + if (!current_writer) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "MergeTree deduplication log has no writer (the disk does not support the on-disk " + "deduplication log); cannot add part {}", + part_info.getPartNameAndCheckFormat(format_version)); for (const auto & block_id : block_ids) { @@ -306,7 +321,13 @@ void MergeTreeDeduplicationLog::dropPart(const MergeTreePartInfo & drop_part_inf throw Exception(ErrorCodes::ABORTED, "Storage has been shutdown when we drop this part."); } - chassert(current_writer != nullptr); + /// As in addPart: a null writer must produce a clear exception, never a segfault (B37). + if (!current_writer) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "MergeTree deduplication log has no writer (the disk does not support the on-disk " + "deduplication log); cannot drop part {}", + drop_part_info.getPartNameAndCheckFormat(format_version)); for (auto itr = deduplication_map.begin(); itr != deduplication_map.end(); /* no increment here, we erasing from map */) { diff --git a/src/Storages/MergeTree/tests/gtest_deduplication_log_null_writer.cpp b/src/Storages/MergeTree/tests/gtest_deduplication_log_null_writer.cpp new file mode 100644 index 000000000000..7a2f72313a19 --- /dev/null +++ b/src/Storages/MergeTree/tests/gtest_deduplication_log_null_writer.cpp @@ -0,0 +1,139 @@ +#include + +#include +#include +#include +#include +#include /// DEBUG_OR_SANITIZER_BUILD + +#include +#include +#include +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace +{ +constexpr auto FORMAT_VERSION = MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING; + +/// B37 regression: a `MergeTreeDeduplicationLog` whose `current_writer` is null (the disk could not +/// host the append-mode log -- see `MergeTreeDeduplicationLog::load()`'s early-return path for a +/// `DiskObjectStorage` whose metadata storage type is neither `Plain` nor `ContentAddressed`) used to +/// be dereferenced unconditionally by `addPart`/`dropPart`: a release-build `chassert` is a no-op, so +/// this was a null-pointer dereference (segfault) rather than a handled error. The fix makes both +/// throw a `LOGICAL_ERROR` `DB::Exception` instead. +/// +/// There is no way to drive this from a stateless SQL test: every disk type that reaches production +/// either materializes `logs_dir` (so `load()` takes the normal `rotate()` path and sets a writer) or +/// is one of the two types (`Plain`, `ContentAddressed`) `load()` explicitly special-cases to still get +/// a writer. So this test constructs the log directly and never calls `load()` -- `current_writer` +/// simply stays at its default-constructed null value, which is the exact precondition the guard in +/// `addPart`/`dropPart` exists for. +struct DeduplicationLogNullWriterFixture : public ::testing::Test +{ + std::filesystem::path base_path; + DiskPtr disk; + std::unique_ptr log; + + void SetUp() override + { + const auto unique = std::to_string(::getpid()) + "_" + std::to_string(reinterpret_cast(this)); + base_path = std::filesystem::temp_directory_path() / ("dedup_log_null_writer_gtest_" + unique); + std::filesystem::create_directories(base_path); + disk = std::make_shared("test_disk_" + unique, base_path.string()); + + /// deduplication_window != 0 so addPart/dropPart don't bail out on the "deduplication is off" + /// fast path before ever reaching the null-writer guard. `load()` is deliberately NOT called: + /// that is what leaves `current_writer` null. + log = std::make_unique("deduplication_logs", /*deduplication_window_=*/4, FORMAT_VERSION, disk); + } + + void TearDown() override + { + log.reset(); + std::error_code ec; + std::filesystem::remove_all(base_path, ec); + } +}; + +} + +#if defined(DEBUG_OR_SANITIZER_BUILD) +/// gtest runs *DeathTest suites before others; reuse the same fixture via an alias so the death arm +/// gets the same null-writer precondition. +using DeduplicationLogNullWriterDeathTest = DeduplicationLogNullWriterFixture; +#endif + +#ifndef DEBUG_OR_SANITIZER_BUILD +TEST_F(DeduplicationLogNullWriterFixture, AddPartThrowsLogicalErrorInsteadOfCrashing) +{ + /// LOGICAL_ERROR "no writer" is a broken-invariant guard (addPart on a null current_writer). Under + /// abort_on_logical_error it aborts at construction instead of being catchable -- the DeathTest + /// below proves the abort in those builds. + auto part_info = MergeTreePartInfo::fromPartName("all_0_0_0", FORMAT_VERSION); + + EXPECT_THROW( + { + try + { + log->addPart({"block-1"}, part_info); + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::LOGICAL_ERROR); + EXPECT_NE(e.message().find("no writer"), std::string::npos); + throw; + } + }, + Exception); + + /// The object stays alive and usable after the guard fires: it isn't left half-corrupted by the + /// failed call, and repeating the same call (still no writer) throws again, cleanly, rather than + /// crashing or behaving differently the second time. + EXPECT_THROW(log->addPart({"block-1"}, part_info), Exception); +} +#endif + +#if defined(DEBUG_OR_SANITIZER_BUILD) +TEST_F(DeduplicationLogNullWriterDeathTest, AddPartAborts) +{ + auto part_info = MergeTreePartInfo::fromPartName("all_0_0_0", FORMAT_VERSION); + EXPECT_DEATH({ log->addPart({"block-1"}, part_info); }, "no writer"); +} +#endif + +#ifndef DEBUG_OR_SANITIZER_BUILD +TEST_F(DeduplicationLogNullWriterFixture, DropPartThrowsLogicalErrorInsteadOfCrashing) +{ + auto part_info = MergeTreePartInfo::fromPartName("all_0_0_0", FORMAT_VERSION); + + EXPECT_THROW( + { + try + { + log->dropPart(part_info); + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::LOGICAL_ERROR); + EXPECT_NE(e.message().find("no writer"), std::string::npos); + throw; + } + }, + Exception); +} +#endif + +#if defined(DEBUG_OR_SANITIZER_BUILD) +TEST_F(DeduplicationLogNullWriterDeathTest, DropPartAborts) +{ + auto part_info = MergeTreePartInfo::fromPartName("all_0_0_0", FORMAT_VERSION); + EXPECT_DEATH({ log->dropPart(part_info); }, "no writer"); +} +#endif From 069f021bf71cc0859f26ed3467df3a6d372679c4 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:32 +0200 Subject: [PATCH 05/30] S3 conditional-write support and 412 no-retry policy HTTP 412 on a conditional request is deterministic: never retry it (S3Exception::isPreconditionFailed, RetryStrategy). Adds conditional PUT/COPY (If-Match / If-None-Match) through the client, WriteBufferFromS3 and copyS3File, and the token-conditional operations on S3ObjectStorage. Also carries the copyS3File message_format_string fix (PreformattedMessage instead of a preformatted string) and CAS-facing write-settings plumbing (wired later). Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../ObjectStorages/S3/S3ObjectStorage.cpp | 189 ++++++++++- .../ObjectStorages/S3/S3ObjectStorage.h | 38 +++ .../ObjectStorages/S3/diskSettings.cpp | 2 + src/IO/S3/Client.cpp | 41 ++- src/IO/S3/Client.h | 17 + src/IO/S3/Requests.h | 2 + src/IO/S3/copyS3File.cpp | 73 +++- src/IO/S3/copyS3File.h | 14 +- src/IO/S3/tests/gtest_aws_s3_client.cpp | 137 ++++++++ src/IO/S3AuthSettings.cpp | 1 + src/IO/S3Common.cpp | 45 +++ src/IO/S3Common.h | 53 ++- src/IO/S3Defines.h | 7 + src/IO/WriteBufferFromS3.cpp | 30 +- src/IO/WriteBufferFromS3.h | 8 + src/IO/tests/gtest_writebuffer_s3.cpp | 320 ++++++++++++++++++ 16 files changed, 954 insertions(+), 23 deletions(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 34eb1eaebb9d..32aa5b34e4b3 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ #include #include #include +#include #include #include @@ -68,8 +70,13 @@ namespace Setting namespace S3RequestSetting { + extern const S3RequestSettingsBool allow_native_copy; + extern const S3RequestSettingsBool check_objects_after_upload; extern const S3RequestSettingsUInt64 list_object_keys_size; extern const S3RequestSettingsUInt64 objects_chunk_size_to_delete; + extern const S3RequestSettingsUInt64 max_single_part_upload_size; + extern const S3RequestSettingsUInt64 min_upload_part_size; + extern const S3RequestSettingsUInt64 max_unexpected_write_error_retries; } @@ -77,6 +84,8 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; + extern const int S3_ERROR; } namespace @@ -227,7 +236,8 @@ class S3IteratorAsync final : public IObjectStorageIteratorAsync bool S3ObjectStorage::exists(const StoredObject & object) const { auto settings_ptr = s3_settings.get(); - return S3::objectExists(*client.get(), uri.bucket, object.remote_path, {}); + const bool e = S3::objectExists(*client.get(), uri.bucket, object.remote_path, {}); + return e; } std::unique_ptr S3ObjectStorage::readObject( /// NOLINT @@ -312,6 +322,29 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN request_settings.updateFromSettings(settings, /* if_changed */ true, settings[Setting::s3_validate_request_settings]); } + if (write_settings.s3_check_objects_after_upload_override) + request_settings[S3RequestSetting::check_objects_after_upload] = *write_settings.s3_check_objects_after_upload_override; + + if (write_settings.s3_single_part_upload_max_bytes_override) + { + /// Keep the whole body in ONE buffered part so the single-PUT path stays available up to + /// the cap (conditional writes on generation-token stores; see WriteSettings). + request_settings[S3RequestSetting::max_single_part_upload_size] + = write_settings.s3_single_part_upload_max_bytes_override; + request_settings[S3RequestSetting::min_upload_part_size] + = write_settings.s3_single_part_upload_max_bytes_override; + } + + if (write_settings.s3_max_unexpected_write_error_retries_override) + { + /// WriteBufferFromS3's OWN retry loop (makeSinglepartUpload/completeMultipartUpload) reissues + /// the identical request — WITH its If-None-Match/If-Match condition — on a NO_SUCH_KEY + /// response; this sits ABOVE the S3 client, so a client-level profile override does not bound + /// it. See WriteSettings. + request_settings[S3RequestSetting::max_unexpected_write_error_retries] + = write_settings.s3_max_unexpected_write_error_retries_override; + } + ThreadPoolCallbackRunnerUnsafe scheduler; if (write_settings.s3_allow_parallel_part_upload) scheduler = threadPoolCallbackRunnerUnsafe(getThreadPoolWriter(), ThreadName::REMOTE_FS_WRITE_THREAD_POOL); @@ -320,8 +353,18 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN if (blob_storage_log) blob_storage_log->local_path = object.local_path; + /// The SingleAttempt profile (e.g. CAS conditional writes, RFC cas-s3-timeout-retry-control) rides + /// on WriteSettings instead of changing this disk's shared client — every other write keeps using + /// client.get() and its normal retry policy unchanged. getSingleAttemptClient() is only invoked + /// when actually selected, so a plain write never pays for building/locking the clone. + std::shared_ptr used_client; + if (write_settings.object_storage_retry_profile == ObjectStorageRetryProfile::SingleAttempt) + used_client = getSingleAttemptClient(); + else + used_client = client.get(); + return std::make_unique( - client.get(), + used_client, uri.bucket, object.remote_path, write_settings.use_adaptive_write_buffer ? write_settings.adaptive_write_buffer_initial_size : buf_size, @@ -443,6 +486,65 @@ void S3ObjectStorage::removeObjectsIfExist(const StoredObjects & objects) removeObjectsImpl(objects, true); } +ConditionalRemoveResult S3ObjectStorage::removeObjectIfTokenMatches(const StoredObject & object, const std::string & etag) +{ + S3::DeleteObjectRequest request; + request.SetBucket(uri.bucket); + request.SetKey(object.remote_path); + request.SetIfMatch(etag); + + ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); + + auto outcome = client.get()->DeleteObject(request); + + /// Mirror removeObjectImpl (deleteFileFromS3): every conditional delete lands in + /// system.blob_storage_log too — GC reclaim was invisible there otherwise. TokenMismatch + /// and NotFound are routine protocol outcomes, recorded with the S3 error for filtering. + if (auto blob_storage_log = BlobStorageLogWriter::create(disk_name)) + blob_storage_log->addEvent(BlobStorageLogElement::EventType::Delete, + uri.bucket, object.remote_path, + object.local_path, object.bytes_size, + /* elapsed_microseconds */ 0, + outcome.IsSuccess() ? 0 : static_cast(outcome.GetError().GetErrorType()), + outcome.IsSuccess() ? "" : outcome.GetError().GetMessage()); + + if (outcome.IsSuccess()) + return {ConditionalRemoveOutcome::Removed, outcome.GetResult().GetDeleteMarker()}; + + const auto & err = outcome.GetError(); + + /// The token did not match the current incarnation: the conditional delete is rejected with a 412 + /// (see `S3::isPreconditionFailedError` for the one policy). Callers treat 'mismatch' and 'gone' + /// alike (re-validate); a genuine absence is disambiguated downstream by a HEAD re-check. + if (S3::isPreconditionFailedError(err)) + return {ConditionalRemoveOutcome::TokenMismatch, false}; + + /// The object no longer exists (404). Protocol callers treat 'mismatch' and 'gone' alike (re-validate). + if (S3::isNotFoundError(err.GetErrorType())) + return {ConditionalRemoveOutcome::NotFound, false}; + + throw S3Exception(err.GetErrorType(), + "{} (Code: {}, S3 exception: '{}') while conditionally removing object with path {} from S3", + err.GetMessage(), static_cast(err.GetErrorType()), err.GetExceptionName(), object.remote_path); +} + +bool S3ObjectStorage::conditionalOpsUseGenerationTokens() const +{ + return client.get()->usesGcsConditionalDialect(); +} + +std::optional S3ObjectStorage::isBucketVersioningEnabled() const +{ + S3::GetBucketVersioningRequest request; + request.SetBucket(uri.bucket); + + auto outcome = client.get()->GetBucketVersioning(request); + if (!outcome.IsSuccess()) + return std::nullopt; + + return outcome.GetResult().GetStatus() == Aws::S3::Model::BucketVersioningStatus::Enabled; +} + static void putObjectsTagOnS3( const std::shared_ptr & s3_client, const String & bucket, @@ -667,6 +769,66 @@ void S3ObjectStorage::copyObject( // NOLINT object_to_attributes); } +ConditionalCopyResult S3ObjectStorage::copyObjectConditional( // NOLINT + const StoredObject & object_from, + const StoredObject & object_to, + const ReadSettings & read_settings, + const WriteSettings &, + std::optional object_to_attributes) +{ + auto current_client = client.get(); + auto settings_ptr = s3_settings.get(); + + /// `copyS3File`'s `If-None-Match` precondition is only honored on the native server-side copy + /// path (`CopyObject` / `CompleteMultipartUpload`): if native copy is disabled it silently falls + /// back to an unconditional read-write copy (`copyDataToS3File`), which would defeat the + /// write-once guarantee the content-addressed staging promote relies on. Fail closed instead of + /// racing an unconditional overwrite onto what may already be a live blob. + if (!settings_ptr->request_settings[S3RequestSetting::allow_native_copy]) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Conditional (write-once) object copy requires the native S3 copy path, which is disabled " + "(allow_native_copy=false) for object storage {}", getName()); + + auto size = S3::getObjectSize(*current_client, uri.bucket, object_from.remote_path, {}); + auto scheduler = threadPoolCallbackRunnerUnsafe(getThreadPoolWriter(), ThreadName::S3_COPY_POOL); + const auto read_settings_to_use = patchSettings(read_settings); + + String dest_etag; + try + { + copyS3File( + /*src_s3_client=*/current_client, + /*src_bucket=*/uri.bucket, + /*src_key=*/object_from.remote_path, + /*src_offset=*/0, + /*src_size=*/size, + /*dest_s3_client=*/current_client, + /*dest_bucket=*/uri.bucket, + /*dest_key=*/object_to.remote_path, + settings_ptr->request_settings, + read_settings_to_use, + BlobStorageLogWriter::create(disk_name), + scheduler, + [&, this]{ return readObject(object_from, read_settings_to_use);}, + object_to_attributes, + /*if_none_match=*/"*", + /*out_dest_etag=*/&dest_etag); + } + catch (S3Exception & exc) + { + /// A `412 Precondition Failed` is the expected "lost the race" signal (the destination already + /// exists), not an error — see `S3Exception::isPreconditionFailed` for the one policy. + if (exc.isPreconditionFailed()) + return {.created = false, .dest_etag = {}}; + + /// Any other failure (network error, access denied, etc.) is a real error and must propagate: + /// never silently treat it as "lost the race". + throw; + } + + return {.created = true, .dest_etag = dest_etag}; +} + void S3ObjectStorage::shutdown() { /// This call stops any next retry attempts for ongoing S3 requests. @@ -756,6 +918,29 @@ std::shared_ptr S3ObjectStorage::tryGetS3StorageClient() return client.get(); } +std::shared_ptr S3ObjectStorage::getSingleAttemptClient() const +{ + auto base = client.get(); + std::lock_guard lock(single_attempt_client_mutex); + if (single_attempt_client && single_attempt_client_base == base) + return single_attempt_client; + + auto cfg = base->getClientConfiguration(); + cfg.retry_strategy.max_retries = 0; + cfg.retryStrategy = std::make_shared(); + + /// A server can reject an If-Match/If-None-Match request before accepting its body; waiting for + /// the 100-continue response avoids uploading a large body that cannot commit. Respect the + /// disk's configured expect_continue_min_bytes; if unset, use the established 1 MiB floor. + static constexpr uint64_t fallback_expect_continue_min_bytes = 1024 * 1024; + if (cfg.expect_continue_min_bytes == 0) + cfg.expect_continue_min_bytes = fallback_expect_continue_min_bytes; + + single_attempt_client = base->cloneWithConfigurationOverride(cfg); + single_attempt_client_base = base; + return single_attempt_client; +} + bool S3ObjectStorage::tryRefreshCredentialsViaCallback() { fiu_do_on(FailPoints::object_storage_force_refresh_callback_success, { return true; }); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index 1760153a1219..d357c3874e78 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -108,6 +109,9 @@ class S3ObjectStorage : public IObjectStorage /// `DeleteObjectsRequest` does not exist on GCS, see https://issuetracker.google.com/issues/162653700 . void removeObjectsIfExist(const StoredObjects & objects) override; + /// Uses `DeleteObjectRequest` with `If-Match` (token-exact removal for content-addressed disks). + ConditionalRemoveResult removeObjectIfTokenMatches(const StoredObject & object, const std::string & etag) override; + void tagObjects(const StoredObjects & objects, const std::string & tag_key, const std::string & tag_value) override; ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags) const override; @@ -121,6 +125,17 @@ class S3ObjectStorage : public IObjectStorage const WriteSettings & write_settings, std::optional object_to_attributes = {}) override; + /// Write-once conditional server-side copy (`CopyObject`/`CompleteMultipartUpload` with + /// `If-None-Match: *`). Only performed via the native copy path (`copyS3File`'s + /// `allow_native_copy` path); if native copy is not available this throws rather than silently + /// falling back to an unconditional overwrite (see `.cpp` for details). + ConditionalCopyResult copyObjectConditional( + const StoredObject & object_from, + const StoredObject & object_to, + const ReadSettings & read_settings, + const WriteSettings & write_settings, + std::optional object_to_attributes) override; + void copyObjectToAnotherObjectStorage( /// NOLINT const StoredObject & object_from, const StoredObject & object_to, @@ -149,6 +164,12 @@ class S3ObjectStorage : public IObjectStorage bool isReadOnly() const override { return s3_settings.get()->request_settings[S3RequestSetting::read_only]; } + bool conditionalOpsUseGenerationTokens() const override; + + std::optional isBucketVersioningEnabled() const override; + + bool supportsRetryProfile(ObjectStorageRetryProfile) const override { return true; } + std::shared_ptr getS3StorageClient() override; std::shared_ptr tryGetS3StorageClient() override; @@ -156,6 +177,12 @@ class S3ObjectStorage : public IObjectStorage S3::URI getURI() const { return uri; } S3Settings getS3Settings() const { return *s3_settings.get(); } + + /// Lazily-built clone of the current disk client with the single-attempt retry profile + /// (SingleAttemptRetryStrategy, max_retries=0, Expect:100-continue floor). Rebuilt whenever the + /// disk client rotates (applyNewSettings/credentials refresh) — the cached clone is keyed by the + /// base client's identity, so a stale clone can never outlive a rotation. + std::shared_ptr getSingleAttemptClient() const; private: void removeObjectImpl(const StoredObject & object, bool if_exists); void removeObjectsImpl(const StoredObjects & objects, bool if_exists); @@ -174,6 +201,17 @@ class S3ObjectStorage : public IObjectStorage const bool for_disk_s3; S3CredentialsRefreshCallback credentials_refresh_callback; + + mutable std::mutex single_attempt_client_mutex; + mutable std::shared_ptr single_attempt_client; + /// The base client the cached clone above was built from. Deliberately held as a shared_ptr (not + /// a raw pointer): a raw pointer would be compared for identity AFTER the object it once pointed + /// to could have been freed and a new client reallocated at the same address by an unrelated + /// rotation (ABA), which would false-match and serve a stale clone (e.g. built from retired + /// credentials) indefinitely. Holding the shared_ptr pins at most one retired client version — + /// released as soon as the next rotation is observed and the clone is rebuilt — which is what + /// makes the identity comparison in getSingleAttemptClient sound. + mutable std::shared_ptr single_attempt_client_base; }; } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp index f95f53763de4..a72f0eb02fed 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/diskSettings.cpp @@ -51,6 +51,7 @@ namespace S3AuthSetting extern const S3AuthSettingsString access_key_id; extern const S3AuthSettingsUInt64 connect_timeout_ms; extern const S3AuthSettingsBool disable_checksum; + extern const S3AuthSettingsUInt64 expect_continue_min_bytes; extern const S3AuthSettingsUInt64 expiration_window_seconds; extern const S3AuthSettingsBool gcs_issue_compose_request; extern const S3AuthSettingsUInt64 http_keep_alive_max_requests; @@ -178,6 +179,7 @@ getClient(const S3::URI & url, const S3Settings & settings, ContextPtr context, client_configuration.endpointOverride = url.endpoint; client_configuration.s3_use_adaptive_timeouts = auth_settings[S3AuthSetting::use_adaptive_timeouts]; + client_configuration.expect_continue_min_bytes = auth_settings[S3AuthSetting::expect_continue_min_bytes]; if (request_settings.proxy_resolver) { diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index af6f0a2e6894..d1c5c733ec9d 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -25,8 +25,10 @@ #include #include +#include #include +#include #include #include #include @@ -65,6 +67,8 @@ namespace ProfileEvents extern const Event S3Clients; extern const Event TinyS3Clients; + + extern const Event S3SingleAttemptRetryConsultations; } namespace CurrentMetrics @@ -102,6 +106,13 @@ bool Client::RetryStrategy::ShouldRetry(const Aws::Client::AWSError= config.max_retries) return false; @@ -181,6 +192,13 @@ void Client::RetryStrategy::RequestBookkeeping( RequestBookkeeping(httpResponseOutcome); } +/// NOLINTNEXTLINE(google-runtime-int) +bool SingleAttemptRetryStrategy::ShouldRetry(const Aws::Client::AWSError &, long) const +{ + ProfileEvents::increment(ProfileEvents::S3SingleAttemptRetryConsultations); + return false; +} + namespace { @@ -289,7 +307,14 @@ Client::Client( /// find credential keys we can simply behave as the underlying storage is S3 /// otherwise, we need to be aware we are making requests to GCS /// and replace all headers with a valid prefix when needed - if (credentials_provider) + if (Poco::toLower(client_configuration.http_client) == "gcs_hmac") + { + /// GOOG4-HMAC mode: all requests are re-signed with x-goog headers at the HTTP layer, + /// so the SDK-side GCS accommodations (x-amz header renames, x-amz-api-version + /// deletion) must be active even though credentials are present. + api_mode = ApiMode::GCS; + } + else if (credentials_provider) { auto credentials = credentials_provider->GetAWSCredentials(); if (credentials.IsEmpty()) @@ -506,6 +531,12 @@ Model::GetObjectTaggingOutcome Client::GetObjectTagging(GetObjectTaggingRequest doRequest(request, [this](const Model::GetObjectTaggingRequest & req) { return GetObjectTagging(req); })); } +Model::GetBucketVersioningOutcome Client::GetBucketVersioning(GetBucketVersioningRequest & request) const +{ + return processRequestResult( + doRequest(request, [this](const Model::GetBucketVersioningRequest & req) { return GetBucketVersioning(req); })); +} + Model::ListObjectsV2Outcome Client::ListObjectsV2(ListObjectsV2Request & request) const { return doRequestWithRetryNetworkErrors( @@ -1274,6 +1305,14 @@ std::unique_ptr ClientFactory::create( // NOLINT auto credentials_provider = getCredentialsProvider(client_configuration, credentials, credentials_configuration); + if (Poco::toLower(client_configuration.http_client) == "gcs_hmac") + { + client_configuration.gcs_conditional_dialect = true; + client_configuration.gcs_hmac_credentials_provider = credentials_provider; + } + else if (Poco::toLower(client_configuration.http_client) == "gcp_oauth") + client_configuration.gcs_conditional_dialect = true; + /// Disable per-thread retry loops if global retry coordination is in use. if (client_configuration.s3_slow_all_threads_after_retryable_error) { diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index ad4d685d88a2..5a6847799c29 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -208,6 +208,7 @@ class Client : private Aws::S3::S3Client Model::HeadObjectOutcome HeadObject(HeadObjectRequest & request) const; Model::GetObjectTaggingOutcome GetObjectTagging(GetObjectTaggingRequest & request) const; + Model::GetBucketVersioningOutcome GetBucketVersioning(GetBucketVersioningRequest & request) const; Model::ListObjectsV2Outcome ListObjectsV2(ListObjectsV2Request & request) const; Model::ListObjectsOutcome ListObjects(ListObjectsRequest & request) const; Model::GetObjectOutcome GetObject(GetObjectRequest & request) const; @@ -252,6 +253,10 @@ class Client : private Aws::S3::S3Client const PocoHTTPClientConfiguration & getClientConfiguration() const { return client_configuration; } + /// True when this client's HTTP layer runs the GCS conditional dialect (http_client = + /// gcs_hmac or gcp_oauth): conditional tokens are GCS generations riding the ETag plumbing. + bool usesGcsConditionalDialect() const { return client_configuration.gcs_conditional_dialect; } + /// For testing purposes only ClientCache * getRawCache() const { return cache.get(); } @@ -273,6 +278,7 @@ class Client : private Aws::S3::S3Client /// otherwise region and endpoint redirection won't work using Aws::S3::S3Client::HeadObject; using Aws::S3::S3Client::GetObjectTagging; + using Aws::S3::S3Client::GetBucketVersioning; using Aws::S3::S3Client::ListObjectsV2; using Aws::S3::S3Client::ListObjects; using Aws::S3::S3Client::GetObject; @@ -346,6 +352,17 @@ class Client : private Aws::S3::S3Client LoggerPtr log; }; +/// Refuses every SDK-transparent retry and counts each consultation. Used by the +/// ObjectStorageRetryProfile::SingleAttempt per-write profile (conditional writes whose retry +/// decisions live ABOVE the SDK: the caller must resolve an uncertain PUT before reissuing). +class SingleAttemptRetryStrategy final : public Aws::Client::RetryStrategy +{ +public: + bool ShouldRetry(const Aws::Client::AWSError &, long) const override; // NOLINT(google-runtime-int) + long CalculateDelayBeforeNextRetry(const Aws::Client::AWSError &, long) const override { return 0; } // NOLINT(google-runtime-int) + long GetMaxAttempts() const override { return 1; } // NOLINT(google-runtime-int) +}; + class ClientFactory { public: diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h index aa21602674f0..0f55ac33296e 100644 --- a/src/IO/S3/Requests.h +++ b/src/IO/S3/Requests.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -158,6 +159,7 @@ using ListObjectsV2Request = ExtendedRequest; using ListObjectsRequest = ExtendedRequest; using GetObjectRequest = ExtendedRequest; using GetObjectTaggingRequest = ExtendedRequest; +using GetBucketVersioningRequest = ExtendedRequest; class UploadPartRequest : public ExtendedRequest { diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index 4a54d7d53ec9..f0aa8cde9b18 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -81,7 +81,9 @@ namespace const std::optional & object_metadata_, ThreadPoolCallbackRunnerUnsafe schedule_, BlobStorageLogWriterPtr blob_storage_log_, - const LoggerPtr log_) + const LoggerPtr log_, + const std::optional & if_none_match_ = {}, + String * out_dest_etag_ = nullptr) : client_ptr(client_ptr_) , dest_bucket(dest_bucket_) , dest_key(dest_key_) @@ -90,6 +92,8 @@ namespace , schedule(schedule_) , blob_storage_log(blob_storage_log_) , log(log_) + , if_none_match(if_none_match_) + , out_dest_etag(out_dest_etag_) , num_parts(0) , normal_part_size(0) { @@ -107,6 +111,13 @@ namespace BlobStorageLogWriterPtr blob_storage_log; const LoggerPtr log; + /// If set, passed as the `If-None-Match` precondition on the destination write of a copy + /// (`CopyObject` and, for large objects, `CompleteMultipartUpload`), making the copy write-once + /// conditional. Only meaningful for copyS3File() (CopyFileHelper); unused by copyDataToS3File(). + const std::optional if_none_match; + /// If non-null, filled in with the destination object's ETag on a successful copy. + String * out_dest_etag; + /// Represents a task uploading a single part. /// Keep this struct small because there can be thousands of parts. /// For example, `UploadPartTask` must not contain a read buffer or `S3::UploadPartRequest` @@ -197,6 +208,9 @@ namespace request.SetMultipartUpload(multipart_upload); + if (if_none_match.has_value()) + request.SetIfNoneMatch(*if_none_match); + size_t max_retries = std::max(request_settings[S3RequestSetting::max_unexpected_write_error_retries].value, 1UL); for (size_t retries = 1;; ++retries) { @@ -216,6 +230,8 @@ namespace if (outcome.IsSuccess()) { + if (out_dest_etag) + *out_dest_etag = outcome.GetResult().GetETag(); LOG_TRACE(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", dest_bucket, dest_key, multipart_upload_id, multipart_tags.size()); break; } @@ -228,10 +244,17 @@ namespace continue; /// will retry } ProfileEvents::increment(ProfileEvents::WriteBufferFromS3RequestsErrors, 1); + /// Preserve the S3 exception name (e.g. `PreconditionFailed` for a rejected + /// `If-None-Match` on `CompleteMultipartUpload`) on the thrown exception: `GetErrorType()` + /// alone maps an unmodeled error like a 412 to `UNKNOWN`, so callers that need to detect + /// a specific condition (a write-once conditional copy losing the race) must be able to + /// read `S3Exception::getExceptionName()`, mirroring how `S3ObjectStorage::removeObjectIfTokenMatches` + /// reads `AWSError::GetExceptionName()` directly off the (not-yet-thrown) outcome. throw S3Exception( + PreformattedMessage::create("Message: {}, Key: {}, Bucket: {}, Tags: {}", + outcome.GetError().GetMessage(), dest_key, dest_bucket, fmt::join(multipart_tags.begin(), multipart_tags.end(), " ")), outcome.GetError().GetErrorType(), - "Message: {}, Key: {}, Bucket: {}, Tags: {}", - outcome.GetError().GetMessage(), dest_key, dest_bucket, fmt::join(multipart_tags.begin(), multipart_tags.end(), " ")); + outcome.GetError().GetExceptionName()); } } @@ -613,7 +636,9 @@ namespace const std::optional & object_metadata_, ThreadPoolCallbackRunnerUnsafe schedule_, BlobStorageLogWriterPtr blob_storage_log_, - std::function fallback_method_) + std::function fallback_method_, + const std::optional & if_none_match_ = {}, + String * out_dest_etag_ = nullptr) : UploadHelper( client_ptr_, dest_bucket_, @@ -622,7 +647,9 @@ namespace object_metadata_, schedule_, blob_storage_log_, - getLogger("copyS3File")) + getLogger("copyS3File"), + if_none_match_, + out_dest_etag_) , src_bucket(src_bucket_) , src_key(src_key_) , offset(src_offset_) @@ -676,6 +703,9 @@ namespace request.SetMetadataDirective(Aws::S3::Model::MetadataDirective::REPLACE); } + if (if_none_match.has_value()) + request.SetIfNoneMatch(*if_none_match); + const auto & storage_class_name = request_settings[S3RequestSetting::storage_class_name]; if (!storage_class_name.value.empty()) request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(storage_class_name)); @@ -698,6 +728,8 @@ namespace auto outcome = client_ptr->CopyObject(request); if (outcome.IsSuccess()) { + if (out_dest_etag) + *out_dest_etag = outcome.GetResult().GetCopyObjectResultDetails().GetETag(); LOG_TRACE( log, "Single operation copy has completed. Bucket: {}, Key: {}, Object size: {}", @@ -715,6 +747,12 @@ namespace { if (!supports_multipart_copy || outcome.GetError().GetExceptionName() == "AccessDenied") { + if (if_none_match.has_value()) + throw S3Exception( + outcome.GetError().GetMessage(), + outcome.GetError().GetErrorType(), + outcome.GetError().GetExceptionName()); + LOG_INFO( log, "Multipart upload using copy is not supported, will try regular upload for Bucket: {}, Key: {}, Object size: " @@ -753,13 +791,17 @@ namespace continue; /// will retry } + /// Preserve the S3 exception name for the same reason as the `CompleteMultipartUpload` + /// throw in `completeMultipartUpload()` above (a 412 on a conditional `CopyObject` maps + /// to `S3Errors::UNKNOWN`; the exception name is the only reliable discriminator). throw S3Exception( + PreformattedMessage::create("Message: {}, Key: {}, Bucket: {}, Object size: {}", + outcome.GetError().GetMessage(), + dest_key, + dest_bucket, + size), outcome.GetError().GetErrorType(), - "Message: {}, Key: {}, Bucket: {}, Object size: {}", - outcome.GetError().GetMessage(), - dest_key, - dest_bucket, - size); + outcome.GetError().GetExceptionName()); } } @@ -774,6 +816,9 @@ namespace if (e.getS3ErrorCode() != Aws::S3::S3Errors::ACCESS_DENIED) throw; + if (if_none_match.has_value()) + throw; + tryLogCurrentException(log, "Multi part copy failed, trying with regular upload"); fallback_method(); } @@ -855,7 +900,9 @@ void copyS3File( BlobStorageLogWriterPtr blob_storage_log, ThreadPoolCallbackRunnerUnsafe schedule, const CreateReadBuffer& fallback_file_reader, - const std::optional & object_metadata) + const std::optional & object_metadata, + std::optional if_none_match, + String * out_dest_etag) { if (!dest_s3_client) dest_s3_client = src_s3_client; @@ -895,7 +942,9 @@ void copyS3File( object_metadata, schedule, blob_storage_log, - std::move(fallback_method)}; + std::move(fallback_method), + if_none_match, + out_dest_etag}; helper.performCopy(); } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index c61437585d4d..a67dfaa9b4b3 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -30,6 +30,16 @@ using CreateReadBuffer = std::function()>; /// (copyDataToS3File()). /// /// read_settings - is used for throttling in case of native copy is not possible +/// +/// If `if_none_match` is set, it is passed as the `If-None-Match` precondition on the destination +/// write (both the single-operation `CopyObject` request and, for large objects, the multipart +/// `CompleteMultipartUpload` request), turning the copy into a write-once conditional copy. A `412 +/// Precondition Failed` response (destination already exists) is not swallowed: it surfaces as a +/// thrown `S3Exception`. This precondition is only honored on the native server-side copy path; it +/// is not applied if the copy falls back to the non-native read-write copy (`copyDataToS3File`). +/// +/// If `out_dest_etag` is non-null, it is filled in with the ETag of the destination object as +/// reported by the copy response, on success. void copyS3File( std::shared_ptr src_s3_client, const String & src_bucket, @@ -44,7 +54,9 @@ void copyS3File( BlobStorageLogWriterPtr blob_storage_log, ThreadPoolCallbackRunnerUnsafe schedule, const CreateReadBuffer& fallback_file_reader, - const std::optional & object_metadata = std::nullopt); + const std::optional & object_metadata = std::nullopt, + std::optional if_none_match = {}, + String * out_dest_etag = nullptr); /// Copies data from any seekable source to S3. /// The same functionality can be done by using the function copyData() and the class WriteBufferFromS3 diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index b3d8f50135bf..d86eba30b75f 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -20,13 +20,16 @@ #include #include #include +#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -42,6 +45,11 @@ namespace DB::S3RequestSetting extern const S3RequestSettingsUInt64 max_unexpected_write_error_retries; } +namespace ProfileEvents +{ + extern const Event S3SingleAttemptRetryConsultations; +} + /* * When all tests are executed together, `Context::getGlobalContextInstance()` is not null. Global context is used by * ProxyResolvers to get proxy configuration (used by S3 clients). If global context does not have a valid ConfigRef, it relies on @@ -197,6 +205,135 @@ static void testServerSideEncryption( EXPECT_EQ(content, expected_headers); } +TEST(IOTestAwsS3Client, DoesNotRetryPreconditionFailed) +{ + /// B166: a 412 Precondition Failed (conditional CAS/dedup writes of the content-addressed + /// backend) must NOT be retried, even when the SDK marks it retryable because an S3-compatible + /// server (e.g. RustFS) returned a body whose ExceptionName it could not parse. Retrying it is a + /// storm that stalls the write path. + DB::S3::Client::RetryStrategy strategy(DB::S3::PocoHTTPClientConfiguration::RetryStrategy{.max_retries = 10}); + + Aws::Client::AWSError precondition(Aws::Client::CoreErrors::UNKNOWN, /*isRetryable=*/true); + precondition.SetResponseCode(Aws::Http::HttpResponseCode::PRECONDITION_FAILED); + EXPECT_FALSE(strategy.ShouldRetry(precondition, /*attemptedRetries=*/0)); + EXPECT_TRUE(DB::S3::isPreconditionFailedError(precondition)); // one policy: agrees via response code + + /// A genuinely transient error is still retried (the guard is specific to 412). + Aws::Client::AWSError unavailable(Aws::Client::CoreErrors::SLOW_DOWN, /*isRetryable=*/true); + unavailable.SetResponseCode(Aws::Http::HttpResponseCode::SERVICE_UNAVAILABLE); + EXPECT_TRUE(strategy.ShouldRetry(unavailable, /*attemptedRetries=*/0)); + EXPECT_FALSE(DB::S3::isPreconditionFailedError(unavailable)); + + /// The one 412 policy also matches on the canonical name / raw body (the two CA conditional + /// ops see an error whose ExceptionName the SDK DID parse, or whose body carries the token). + Aws::Client::AWSError named(Aws::S3::S3Errors::UNKNOWN, "PreconditionFailed", "precondition failed", false); + EXPECT_TRUE(DB::S3::isPreconditionFailedError(named)); + + /// Typed-exception surface (the conditional copy / finalize catch an S3Exception): name and message. + EXPECT_TRUE(DB::S3Exception("boom", Aws::S3::S3Errors::UNKNOWN, "PreconditionFailed").isPreconditionFailed()); + EXPECT_FALSE(DB::S3Exception("boom", Aws::S3::S3Errors::NO_SUCH_KEY, "NoSuchKey").isPreconditionFailed()); +} + +/// Every consultation is counted, not just the first: simulating two retryable 5xx decisions in a row +/// proves the counter tracks each SDK consultation rather than being fixed/clamped at 1, which is what +/// makes it a live tripwire ("SDK-level retries must remain zero for conditional writes") rather than a +/// value nothing ever touches. +TEST(IOTestAwsS3Client, SingleAttemptRetryStrategyRefusesAndCounts) +{ + using ProfileEvents::global_counters; + const auto before = global_counters[ProfileEvents::S3SingleAttemptRetryConsultations].load(); + DB::S3::SingleAttemptRetryStrategy strategy; + const Aws::Client::AWSError retryable_5xx( + Aws::Client::CoreErrors::INTERNAL_FAILURE, /*isRetryable=*/true); + EXPECT_FALSE(strategy.ShouldRetry(retryable_5xx, /*attempted=*/0)); + EXPECT_FALSE(strategy.ShouldRetry(retryable_5xx, /*attempted=*/1)); + EXPECT_EQ(strategy.GetMaxAttempts(), 1); + EXPECT_EQ(global_counters[ProfileEvents::S3SingleAttemptRetryConsultations].load() - before, 2u); +} + +/// Drive a single-part conditional PUT (`If-None-Match: *`) with `body_size` bytes through a real +/// S3 client whose `expect_continue_min_bytes` gate is `threshold`, against the mock HTTP server, and +/// report whether the request that reached the wire carried an `Expect: 100-continue` header. +static bool conditionalPutNegotiatesExpectContinue(uint64_t threshold, size_t body_size) +{ + TestPocoHTTPServer http; + + DB::RemoteHostFilter remote_host_filter; + DB::S3::URI uri(http.getUrl() + "/IOTestAwsS3ClientExpectContinue/test.txt"); + + DB::S3::PocoHTTPClientConfiguration client_configuration = DB::S3::ClientFactory::instance().createClientConfiguration( + "us-east-1", + remote_host_filter, + /* s3_max_redirects = */ 100, + DB::S3::PocoHTTPClientConfiguration::RetryStrategy{.max_retries = 0}, + /* s3_slow_all_threads_after_network_error = */ true, + /* s3_slow_all_threads_after_retryable_error = */ true, + /* enable_s3_requests_logging = */ false, + /* for_disk_s3 = */ false, + /* opt_disk_name = */ {}, + /* request_throttler = */ {}, + uri.uri.getScheme()); + + client_configuration.endpointOverride = uri.endpoint; + client_configuration.expect_continue_min_bytes = threshold; + + DB::S3::ClientSettings client_settings{ + .use_virtual_addressing = uri.is_virtual_hosted_style, + .disable_checksum = false, + .gcs_issue_compose_request = false, + .is_s3express_bucket = false, + }; + + std::shared_ptr client = DB::S3::ClientFactory::instance().create( + client_configuration, + client_settings, + /* access_key_id = */ "ACCESS_KEY_ID", + /* secret_access_key = */ "SECRET_ACCESS_KEY", + /* server_side_encryption_customer_key_base64 = */ "", + /* sse_kms_config = */ {}, + /* headers = */ {}, + DB::S3::CredentialsConfiguration{ + .use_environment_credentials = false, + .use_insecure_imds_request = false, + }); + + DB::S3::S3RequestSettings request_settings; + request_settings[DB::S3RequestSetting::max_unexpected_write_error_retries] = 1; + + DB::WriteSettings write_settings; + write_settings.object_storage_write_if_none_match = "*"; + + DB::WriteBufferFromS3 write_buffer( + client, + uri.bucket, + uri.key, + DB::DBMS_DEFAULT_BUFFER_SIZE, + request_settings, + /* blob_log = */ nullptr, + /* object_metadata = */ std::nullopt, + /* schedule = */ {}, + write_settings); + + const std::string body(body_size, 'x'); + write_buffer.write(body.data(), body.size()); + write_buffer.finalize(); + + return http.getLastRequestHeader().has("Expect"); +} + +TEST(IOTestAwsS3Client, ExpectContinueOnlyWhenThresholdPositive) +{ + /// RExpect: `Expect: 100-continue` (B118) is scoped to CAS-owned conditional writes. A non-CAS S3 + /// client carries the default threshold 0 (disabled) and must NOT negotiate Expect on a conditional + /// PUT — that is the upstream wire behaviour a non-CAS disk (e.g. Iceberg's If-None-Match commits) + /// must keep. A CAS conditional-write client raises the threshold (see the single-attempt client in + /// ObjectStorageBackend) and DOES negotiate it for a body at least that large. + EXPECT_FALSE(conditionalPutNegotiatesExpectContinue(/*threshold=*/0, /*body_size=*/64)); + EXPECT_TRUE(conditionalPutNegotiatesExpectContinue(/*threshold=*/8, /*body_size=*/64)); + /// A positive threshold still excludes a body below it (only large bodies warrant the round-trip). + EXPECT_FALSE(conditionalPutNegotiatesExpectContinue(/*threshold=*/128, /*body_size=*/64)); +} + TEST(IOTestAwsS3Client, AppendExtraSSECHeadersRead) { /// See https://github.com/ClickHouse/ClickHouse/pull/19748 diff --git a/src/IO/S3AuthSettings.cpp b/src/IO/S3AuthSettings.cpp index 2bcfe965e84c..7946dccbeb2c 100644 --- a/src/IO/S3AuthSettings.cpp +++ b/src/IO/S3AuthSettings.cpp @@ -25,6 +25,7 @@ namespace DB DECLARE(Bool, no_sign_request, S3::DEFAULT_NO_SIGN_REQUEST, "", 0) \ DECLARE(Bool, use_insecure_imds_request, false, "", 0) \ DECLARE(Bool, use_adaptive_timeouts, S3::DEFAULT_USE_ADAPTIVE_TIMEOUTS, "", 0) \ + DECLARE(UInt64, expect_continue_min_bytes, S3::DEFAULT_EXPECT_CONTINUE_MIN_BYTES, "", 0) \ DECLARE(Bool, is_virtual_hosted_style, false, "", 0) \ DECLARE(Bool, disable_checksum, S3::DEFAULT_DISABLE_CHECKSUM, "", 0) \ DECLARE(Bool, gcs_issue_compose_request, false, "", 0) \ diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index be42b184589b..47f7c53386f9 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -43,6 +43,51 @@ bool S3Exception::isAccessTokenExpiredError() const return code == Aws::S3::S3Errors::INVALID_ACCESS_KEY_ID || code == Aws::S3::S3Errors::ACCESS_DENIED || code == Aws::S3::S3Errors::INVALID_SIGNATURE || code == Aws::S3::S3Errors::UNKNOWN; } +bool S3Exception::isPreconditionFailed() const +{ + /// See `S3::isPreconditionFailedError`. The thrown exception no longer carries the HTTP status, so + /// only the name and raw message are available here — fail-safe: matching too broadly maps a hard + /// error to a retryable re-validate, never a false success. + return exception_name == "PreconditionFailed" + || message().find("PreconditionFailed") != std::string::npos; +} + +namespace S3 +{ + +/// A synchronous rejection PROVING the request was never applied — matched by the canonical S3 error +/// code STRING (many of these are UNKNOWN in the SDK's modeled enum, mirroring +/// ObjectStorageBackend::finalizeConditionalWrite's own name-first matching) plus the modeled enum +/// value where one exists, belt-and-suspenders. +bool isMalformedRequestError(const S3Exception & e) +{ + const String & name = e.getExceptionName(); + return name == "MalformedXML" || name == "MalformedPOSTRequest" || name == "InvalidArgument" + || name == "InvalidRequest" || name == "InvalidBucketName" || name == "KeyTooLongError" + || e.getS3ErrorCode() == Aws::S3::S3Errors::INVALID_PARAMETER_VALUE + || e.getS3ErrorCode() == Aws::S3::S3Errors::INVALID_REQUEST + || e.getS3ErrorCode() == Aws::S3::S3Errors::VALIDATION; +} + +bool isEntityTooLargeError(const S3Exception & e) +{ + /// No modeled enum value for this error — name-only match, same as PreconditionFailed elsewhere. + return e.getExceptionName() == "EntityTooLarge"; +} + +bool isAccessDeniedError(const S3Exception & e) +{ + const String & name = e.getExceptionName(); + return name == "AccessDenied" || name == "InvalidAccessKeyId" || name == "SignatureDoesNotMatch" + || name == "InvalidToken" || name == "ExpiredToken" || name == "AccountProblem" + || e.getS3ErrorCode() == Aws::S3::S3Errors::ACCESS_DENIED + || e.getS3ErrorCode() == Aws::S3::S3Errors::INVALID_ACCESS_KEY_ID + || e.getS3ErrorCode() == Aws::S3::S3Errors::SIGNATURE_DOES_NOT_MATCH + || e.getS3ErrorCode() == Aws::S3::S3Errors::INVALID_CLIENT_TOKEN_ID; +} + +} + } #endif diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index 27532ca851e7..8c1a4d524294 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -37,9 +37,18 @@ class S3Exception : public Exception { } - S3Exception(const std::string & msg, Aws::S3::S3Errors code_) + S3Exception(const std::string & msg, Aws::S3::S3Errors code_, String exception_name_ = {}) : Exception(msg, ErrorCodes::S3_ERROR) , code(code_) + , exception_name(std::move(exception_name_)) + {} + + /// Preserves the static format string (system.text_log / system.errors grouping) while also + /// carrying the canonical S3 error name — build msg with PreformattedMessage::create. + S3Exception(PreformattedMessage && msg, Aws::S3::S3Errors code_, String exception_name_) + : Exception(std::move(msg), ErrorCodes::S3_ERROR) + , code(code_) + , exception_name(std::move(exception_name_)) {} Aws::S3::S3Errors getS3ErrorCode() const @@ -47,15 +56,57 @@ class S3Exception : public Exception return code; } + /// The canonical S3 error code string from the response XML `` (e.g. "PreconditionFailed", + /// "NoSuchKey") as reported by `Aws::Client::AWSError::GetExceptionName`. Errors unmodeled by the + /// SDK (a conditional-PUT 412 is one) have `getS3ErrorCode` == UNKNOWN, so this name is the only + /// machine-readable discriminator. Empty when the throw site did not attach it. + /// Not `Exception::name`; this is the AWS `` string. + const String & getExceptionName() const + { + return exception_name; + } + bool isRetryableError() const; bool isAccessTokenExpiredError() const; + /// True for a conditional-request 412 (a lost `If-Match`/`If-None-Match`). The thrown exception + /// discards the HTTP status, so it matches on the canonical `` name and the raw message — + /// see `S3::isPreconditionFailedError` for the full (response-code-aware) policy. + bool isPreconditionFailed() const; + S3Exception * clone() const override { return new S3Exception(*this); } void rethrow() const override { throw *this; } /// NOLINT(cert-err60-cpp) private: Aws::S3::S3Errors code; + String exception_name; }; + +namespace S3 +{ + +/// One policy for "is this error a conditional-request 412 (`PreconditionFailed`)?", shared by the +/// retry strategy and the CA conditional delete/copy paths. The HTTP status is authoritative — a +/// non-AWS body (e.g. RustFS) leaves the SDK-parsed `ExceptionName` empty — with the canonical `` +/// name and the raw message as fallbacks. Fail-safe by direction: over-matching only forces a caller +/// re-validate, never a false success. +template +inline bool isPreconditionFailedError(const Aws::Client::AWSError & error) +{ + return error.GetResponseCode() == Aws::Http::HttpResponseCode::PRECONDITION_FAILED + || error.GetExceptionName() == "PreconditionFailed" + || error.GetMessage().find("PreconditionFailed") != std::string::npos; +} + +/// Error-name classifiers factored out of the CAS conditional-write outcome mapping +/// (`CasRequestControl.cpp`), so the name lists live next to the other S3 error classifiers here +/// and are available for reuse. +bool isMalformedRequestError(const S3Exception & e); +bool isEntityTooLargeError(const S3Exception & e); +bool isAccessDeniedError(const S3Exception & e); + +} + } #endif diff --git a/src/IO/S3Defines.h b/src/IO/S3Defines.h index 228758e57f44..8d13dd036b9a 100644 --- a/src/IO/S3Defines.h +++ b/src/IO/S3Defines.h @@ -33,6 +33,13 @@ inline static constexpr uint64_t DEFAULT_LIST_OBJECT_KEYS_SIZE = 1000; inline static constexpr uint64_t DEFAULT_MAX_SINGLE_READ_TRIES = 4; inline static constexpr uint64_t DEFAULT_MAX_UNEXPECTED_WRITE_ERROR_RETRIES = 4; inline static constexpr uint64_t DEFAULT_MAX_REDIRECTS = 10; +/// Gate for the `Expect: 100-continue` negotiation on a conditional write (If-None-Match / If-Match): +/// `0` = disabled (never negotiate Expect); a positive `N` negotiates Expect for a conditional `PUT` +/// whose body is at least `N` bytes, so the server can reject (e.g. 412) BEFORE the body is streamed +/// (B118). The default is DISABLED: only a CAS conditional-write client raises this (see the +/// single-attempt client built in `ObjectStorageBackend`), so non-CAS S3 traffic keeps upstream +/// behaviour instead of negotiating Expect on large conditional PUTs it never negotiated before. +inline static constexpr uint64_t DEFAULT_EXPECT_CONTINUE_MIN_BYTES = 0; inline static constexpr uint64_t DEFAULT_RETRY_ATTEMPTS = 500; inline static constexpr uint64_t DEFAULT_RETRY_INITIAL_DELAY_MS = 25; inline static constexpr uint64_t DEFAULT_RETRY_MAX_DELAY_MS = 5000; diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 8021a1f6cd04..81677edf6ee5 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -64,6 +64,7 @@ namespace ErrorCodes extern const int S3_ERROR; extern const int INVALID_CONFIG_PARAMETER; extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; } struct WriteBufferFromS3::PartData @@ -405,6 +406,15 @@ void WriteBufferFromS3::writeMultipartUpload() void WriteBufferFromS3::createMultipartUpload() { + if (write_settings.s3_force_single_part_upload) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "A conditional write would start a MULTIPART upload, but the target store enforces no " + "preconditions on CompleteMultipartUpload (GCS, measured 2026-07-03) — refusing " + "(silent-data-loss risk). The single-PUT budget is governed by the disk setting " + "gcs_max_conditional_put_bytes; the production-grade path for bigger blobs " + "(unconditional multipart to a temp key + conditional Compose) is not implemented yet. {}", + getShortLogDetails()); + LOG_TEST(limited_log, "Create multipart upload. {}", getShortLogDetails()); S3::CreateMultipartUploadRequest req; @@ -678,6 +688,7 @@ bool WriteBufferFromS3::completeMultipartUpload() if (outcome.IsSuccess()) { + object_etag = outcome.GetResult().GetETag(); LOG_TRACE(limited_log, "Multipart upload has completed. {}, Parts: {}", getShortLogDetails(), multipart_tags.size()); return true; } @@ -692,10 +703,13 @@ bool WriteBufferFromS3::completeMultipartUpload() } else { + /// Pass the canonical S3 error name: a conditional-write 412 is UNMODELED for the SDK + /// (the error type is UNKNOWN), so the name is the caller's only typed signal. throw S3Exception( + PreformattedMessage::create("Message: {}, Key: {}, Bucket: {}, Tags: {}", + outcome.GetError().GetMessage(), key, bucket, fmt::join(multipart_tags.begin(), multipart_tags.end(), " ")), outcome.GetError().GetErrorType(), - "Message: {}, Key: {}, Bucket: {}, Tags: {}", - outcome.GetError().GetMessage(), key, bucket, fmt::join(multipart_tags.begin(), multipart_tags.end(), " ")); + outcome.GetError().GetExceptionName()); } } @@ -770,6 +784,7 @@ void WriteBufferFromS3::makeSinglepartUpload(WriteBufferFromS3::PartData && data if (outcome.IsSuccess()) { + object_etag = outcome.GetResult().GetETag(); LOG_TRACE(limited_log, "Single part upload has completed. {}, size {}", getShortLogDetails(), content_length); return; } @@ -785,17 +800,20 @@ void WriteBufferFromS3::makeSinglepartUpload(WriteBufferFromS3::PartData && data else { /// PreconditionFailed is an expected response for conditional writes (e.g. If-None-Match: *), - /// not a genuine error — the caller handles it. - if (outcome.GetError().GetExceptionName() == "PreconditionFailed") + /// not a genuine error — the caller handles it (see `S3::isPreconditionFailedError`). + if (S3::isPreconditionFailedError(outcome.GetError())) LOG_INFO(log, "S3Exception name {}, Message: {}, bucket {}, key {}, object size {}", outcome.GetError().GetExceptionName(), outcome.GetError().GetMessage(), bucket, key, content_length); else LOG_ERROR(log, "S3Exception name {}, Message: {}, bucket {}, key {}, object size {}", outcome.GetError().GetExceptionName(), outcome.GetError().GetMessage(), bucket, key, content_length); + /// Pass the canonical S3 error name: a conditional-write 412 is UNMODELED for the SDK + /// (the error type is UNKNOWN), so the name is the caller's only typed signal. throw S3Exception( + PreformattedMessage::create("Message: {}, bucket {}, key {}, object size {}", + outcome.GetError().GetMessage(), bucket, key, content_length), outcome.GetError().GetErrorType(), - "Message: {}, bucket {}, key {}, object size {}", - outcome.GetError().GetMessage(), bucket, key, content_length); + outcome.GetError().GetExceptionName()); } } diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 6a5e88875f27..fbce538c0a43 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -51,6 +51,10 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase void preFinalize() override; std::string getFileName() const override { return key; } void sync() override { next(); } + /// The object ETag from the PutObject / CompleteMultipartUpload response, captured on a + /// successful upload. Lets content-addressed callers record the written incarnation's token + /// without a follow-up HEAD. Valid only after a successful finalize(). + std::optional getResultObjectETag() const override { return object_etag; } private: /// Receives response from the server after sending all data. @@ -88,6 +92,10 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase const WriteSettings write_settings; const std::shared_ptr client_ptr; const std::optional object_metadata; + /// Set from the PutObject / CompleteMultipartUpload response ETag on a successful upload; read + /// by getResultObjectETag() after finalize(). Written by the upload worker, read after the + /// finalize barrier (happens-before), so no extra synchronization is needed. + std::optional object_etag; LoggerPtr log = getLogger("WriteBufferFromS3"); LogSeriesLimiterPtr limited_log = std::make_shared(log, 1, 5); diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index 997ff354a535..fb09907fe6bd 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -18,6 +18,9 @@ #include #include #include +#include +#include +#include #include #include @@ -28,12 +31,16 @@ #include #include #include +#include +#include #include #include #include +#include #include +#include #include @@ -184,6 +191,9 @@ struct EventCounts size_t multiUploadAbort = 0; size_t uploadParts = 0; size_t writtenSize = 0; + size_t copyObject = 0; + size_t deleteObject = 0; + size_t getBucketVersioning = 0; size_t totalRequestsCount() const { @@ -208,6 +218,9 @@ struct InjectionModel DeclareInjectCall(CompleteMultipartUpload) DeclareInjectCall(AbortMultipartUpload) DeclareInjectCall(UploadPart) + DeclareInjectCall(CopyObject) + DeclareInjectCall(DeleteObject) + DeclareInjectCall(GetBucketVersioning) #undef DeclareInjectCall }; @@ -277,6 +290,7 @@ struct Client : DB::S3::Client Aws::S3::Model::PutObjectOutcome outcome; Aws::S3::Model::PutObjectResult result(outcome.GetResultWithOwnership()); + result.SetETag("etag-singlepart-" + request.GetKey()); return result; } @@ -392,6 +406,7 @@ struct Client : DB::S3::Client bStore.CompleteMPU(request.GetKey(), request.GetUploadId(), etags); Aws::S3::Model::CompleteMultipartUploadResult result; + result.SetETag("etag-multipart-" + request.GetKey()); return Aws::S3::Model::CompleteMultipartUploadOutcome(result); } @@ -414,6 +429,69 @@ struct Client : DB::S3::Client return Aws::S3::Model::AbortMultipartUploadOutcome(result); } + Aws::S3::Model::CopyObjectOutcome CopyObject(const Aws::S3::Model::CopyObjectRequest & request) const override + { + ++counters.copyObject; + + if (injections) + { + if (auto opt_val = injections->call(request)) + return std::move(*opt_val); + } + + /// CopySource is "/"; parse it back apart to look the source object up + /// (both source and destination live in the same S3MemStrore in these tests). + const std::string & copy_source = request.GetCopySource(); + const size_t sep = copy_source.find('/'); + chassert(sep != std::string::npos); + const std::string src_bucket_name = copy_source.substr(0, sep); + const std::string src_key = copy_source.substr(sep + 1); + + auto & src_store = store->GetBucketStore(src_bucket_name); + const std::string data = src_store.objects.at(src_key); + + auto & dst_store = store->GetBucketStore(request.GetBucket()); + dst_store.PutObject(request.GetKey(), data); + + Aws::S3::Model::CopyObjectResult result; + Aws::S3::Model::CopyObjectResultDetails details; + details.SetETag("etag-copy-" + request.GetKey()); + result.SetCopyObjectResultDetails(details); + return Aws::S3::Model::CopyObjectOutcome(result); + } + + Aws::S3::Model::DeleteObjectOutcome DeleteObject(const Aws::S3::Model::DeleteObjectRequest & request) const override + { + ++counters.deleteObject; + + if (injections) + { + if (auto opt_val = injections->call(request)) + return std::move(*opt_val); + } + + auto & bStore = store->GetBucketStore(request.GetBucket()); + bStore.objects.erase(request.GetKey()); + + Aws::S3::Model::DeleteObjectResult result; + return Aws::S3::Model::DeleteObjectOutcome(result); + } + + Aws::S3::Model::GetBucketVersioningOutcome GetBucketVersioning(const Aws::S3::Model::GetBucketVersioningRequest & request) const override + { + ++counters.getBucketVersioning; + + if (injections) + { + if (auto opt_val = injections->call(request)) + return std::move(*opt_val); + } + + Aws::S3::Model::GetBucketVersioningResult result; + result.SetStatus(Aws::S3::Model::BucketVersioningStatus::Enabled); + return Aws::S3::Model::GetBucketVersioningOutcome(result); + } + std::shared_ptr store; mutable EventCounts counters; mutable std::shared_ptr injections; @@ -460,6 +538,39 @@ struct UploadPartFailIngection: InjectionModel } }; +/// Injects an arbitrary AWSError on DeleteObject -- used to drive the conditional-remove +/// (`removeObjectIfTokenMatches`) outcome mapping: a 412-shaped error (exception name "PreconditionFailed", +/// matched by `S3::isPreconditionFailedError`) must map to `ConditionalRemoveOutcome::TokenMismatch`, and a +/// 404-shaped error (a `NO_SUCH_KEY`/`RESOURCE_NOT_FOUND`/`NO_SUCH_BUCKET` error type, matched by +/// `S3::isNotFoundError`) must map to `ConditionalRemoveOutcome::NotFound`. +struct DeleteObjectErrorInjection: InjectionModel +{ + explicit DeleteObjectErrorInjection(Aws::Client::AWSError error_) : error(std::move(error_)) {} + + std::optional call(const Aws::S3::Model::DeleteObjectRequest & /*request*/) override + { + return error; + } + + Aws::Client::AWSError error; +}; + +/// Injects an arbitrary AWSError on CopyObject -- used to drive `copyObjectConditional` / +/// `copyS3File`'s `If-None-Match` handling: a "PreconditionFailed" error is the expected "lost the +/// race" signal, while an "AccessDenied" error must propagate as a genuine failure rather than being +/// swallowed into the unconditional-copy fallback (see `copyS3File.cpp`'s `processCopyRequest`). +struct CopyObjectErrorInjection: InjectionModel +{ + explicit CopyObjectErrorInjection(Aws::Client::AWSError error_) : error(std::move(error_)) {} + + std::optional call(const Aws::S3::Model::CopyObjectRequest & /*request*/) override + { + return error; + } + + Aws::Client::AWSError error; +}; + struct BaseSyncPolicy { virtual ~BaseSyncPolicy() = default; @@ -952,6 +1063,33 @@ TEST_F(WBS3Test, PrefinalizeCalledMultipleTimes) { #endif } +// The object ETag from the PutObject / CompleteMultipartUpload response is surfaced via +// getResultObjectETag() after a successful finalize() — lets content-addressed callers record the +// just-written incarnation's token WITHOUT a follow-up HEAD (CA head-after-put elimination). +TEST_F(WBS3Test, ResultObjectETagIsCaptured) { + // Singlepart upload: the PutObject response ETag. + { + auto buffer = getWriteBuffer("singlepart-file"); + writeAsOneBlock(*buffer, 10); + getAsyncPolicy().setAutoExecute(true); + buffer->finalize(); + ASSERT_TRUE(buffer->getResultObjectETag().has_value()); + ASSERT_EQ(*buffer->getResultObjectETag(), "etag-singlepart-singlepart-file"); + } + + // Multipart upload: the final object ETag comes from CompleteMultipartUpload, NOT a per-part tag. + { + getSettings()[Setting::s3_max_single_part_upload_size] = 0; // no single part — force multipart + getSettings()[Setting::s3_min_upload_part_size] = 1; + auto buffer = getWriteBuffer("multipart-file"); + writeAsOneBlock(*buffer, 10); + getAsyncPolicy().setAutoExecute(true); + buffer->finalize(); + ASSERT_TRUE(buffer->getResultObjectETag().has_value()); + ASSERT_EQ(*buffer->getResultObjectETag(), "etag-multipart-multipart-file"); + } +} + TEST_P(SyncAsync, EmptyFile) { getSettings()[Setting::s3_check_objects_after_upload] = true; @@ -1186,6 +1324,188 @@ TEST_P(SyncAsync, StrictUploadPartSize) { } } +/// Mock-S3 coverage for the content-addressed conditional-write primitives: `removeObjectIfTokenMatches` +/// (`If-Match` `DeleteObject`) and `copyObjectConditional` (`If-None-Match: *` `CopyObject`), plus the +/// fallback-disable guarantee in `copyS3File` when a conditional copy is requested. +class S3ObjectStorageConditionalOpsTest : public ::testing::Test +{ +public: + const String bucket = "cond-ops-bucket"; + const String disk_name = "cond-ops-disk"; + + std::shared_ptr object_storage; + MockS3::Client * mock_client = nullptr; + std::shared_ptr store; + +protected: + void SetUp() override + { + /// removeObjectIfTokenMatches()/copyObjectConditional() unconditionally call + /// BlobStorageLogWriter::create(), which falls back to Context::getGlobalContextInstance() + /// when there is no query context. Force that global context to exist (harmless -- blob + /// storage logging stays off by default) regardless of which other gtest TU ran first. + (void)getContext(); + + store = std::make_shared(); + store->CreateBucket(bucket); + + auto owned_client = std::make_unique(store); + mock_client = owned_client.get(); + + S3::URI uri; + uri.bucket = bucket; + S3Capabilities capabilities; + ObjectStorageKeyGeneratorPtr key_generator; + + object_storage = std::make_shared( + std::move(owned_client), std::make_unique(), std::move(uri), capabilities, key_generator, disk_name); + } + + void TearDown() override + { + object_storage.reset(); + mock_client = nullptr; + store.reset(); + } +}; + +TEST_F(S3ObjectStorageConditionalOpsTest, RemoveObjectIfTokenMatchesSuccess) +{ + store->GetBucketStore(bucket).PutObject("key1", "data"); + + auto result = object_storage->removeObjectIfTokenMatches(StoredObject("key1"), "etag-1"); + + ASSERT_EQ(result.outcome, ConditionalRemoveOutcome::Removed); + ASSERT_EQ(mock_client->counters.deleteObject, 1); +} + +TEST_F(S3ObjectStorageConditionalOpsTest, RemoveObjectIfTokenMatchesPreconditionFailedIsTokenMismatch) +{ + mock_client->setInjectionModel(std::make_shared( + Aws::Client::AWSError(Aws::S3::S3Errors::UNKNOWN, "PreconditionFailed", "precondition failed", false))); + + auto result = object_storage->removeObjectIfTokenMatches(StoredObject("key1"), "stale-etag"); + + ASSERT_EQ(result.outcome, ConditionalRemoveOutcome::TokenMismatch); +} + +TEST_F(S3ObjectStorageConditionalOpsTest, RemoveObjectIfTokenMatchesNotFoundIsNotFound) +{ + mock_client->setInjectionModel(std::make_shared( + Aws::Client::AWSError(Aws::S3::S3Errors::NO_SUCH_KEY, "NoSuchKey", "not found", false))); + + auto result = object_storage->removeObjectIfTokenMatches(StoredObject("missing-key"), "any-etag"); + + ASSERT_EQ(result.outcome, ConditionalRemoveOutcome::NotFound); +} + +TEST_F(S3ObjectStorageConditionalOpsTest, CopyObjectConditionalSuccess) +{ + store->GetBucketStore(bucket).PutObject("src-key", "hello-world"); + + auto result = object_storage->copyObjectConditional( + StoredObject("src-key"), StoredObject("dst-key"), ReadSettings{}, WriteSettings{}, std::nullopt); + + ASSERT_TRUE(result.created); + ASSERT_FALSE(result.dest_etag.empty()); + ASSERT_EQ(store->GetBucketStore(bucket).objects.at("dst-key"), "hello-world"); +} + +TEST_F(S3ObjectStorageConditionalOpsTest, CopyObjectConditionalPreconditionFailedIsNotCreated) +{ + store->GetBucketStore(bucket).PutObject("src-key", "hello-world"); + + mock_client->setInjectionModel(std::make_shared( + Aws::Client::AWSError(Aws::S3::S3Errors::UNKNOWN, "PreconditionFailed", "precondition failed", false))); + + auto result = object_storage->copyObjectConditional( + StoredObject("src-key"), StoredObject("dst-key"), ReadSettings{}, WriteSettings{}, std::nullopt); + + /// "Lost the race": not an error, just created == false and no destination etag. + ASSERT_FALSE(result.created); + ASSERT_TRUE(result.dest_etag.empty()); +} + +/// B166-adjacent: `copyS3File`'s `If-None-Match` conditional copy must not silently fall back to an +/// unconditional read-write copy on an `AccessDenied` `CopyObject` response -- that would defeat the +/// write-once guarantee the CA promote path relies on. The exception must propagate, and the fallback +/// (an unconditional `PutObject` upload of the source data) must never run. +TEST_F(WBS3Test, CopyS3FileConditionalAccessDeniedPropagatesWithoutFallback) +{ + client->store->GetBucketStore(bucket).PutObject("src-key", "hello"); + + setInjectionModel(std::make_shared( + Aws::Client::AWSError(Aws::S3::S3Errors::ACCESS_DENIED, "AccessDenied", "access denied", false))); + + client->resetCounters(); + + S3::S3RequestSettings request_settings; + ReadSettings read_settings; + bool fallback_called = false; + auto fallback_reader = [&]() -> std::unique_ptr + { + fallback_called = true; + return nullptr; + }; + + EXPECT_THROW({ + try + { + String dest_etag; + copyS3File(client, bucket, "src-key", 0, 5, client, bucket, "dst-key", + request_settings, read_settings, nullptr, getAsyncPolicy().getScheduler(), + fallback_reader, std::nullopt, String("*"), &dest_etag); + } + catch (const DB::S3Exception & e) + { + EXPECT_FALSE(e.isPreconditionFailed()); + EXPECT_EQ(e.getExceptionName(), "AccessDenied"); + throw; + } + }, DB::S3Exception); + + EXPECT_FALSE(fallback_called); + EXPECT_EQ(client->counters.copyObject, 1); + EXPECT_EQ(client->counters.putObject, 0); +} + +/// The other half of the same guard: a losing conditional copy (412) is a distinct, recognizable +/// outcome (`S3Exception::isPreconditionFailed()`), still without ever running the fallback. +TEST_F(WBS3Test, CopyS3FileConditionalPreconditionFailedSurfacesAsException) +{ + client->store->GetBucketStore(bucket).PutObject("src-key", "world"); + + setInjectionModel(std::make_shared( + Aws::Client::AWSError(Aws::S3::S3Errors::UNKNOWN, "PreconditionFailed", "precondition failed", false))); + + client->resetCounters(); + + S3::S3RequestSettings request_settings; + ReadSettings read_settings; + auto fallback_reader = []() -> std::unique_ptr + { + ADD_FAILURE() << "fallback must not run on a losing conditional copy"; + return nullptr; + }; + + EXPECT_THROW({ + try + { + String dest_etag; + copyS3File(client, bucket, "src-key", 0, 5, client, bucket, "dst-key", + request_settings, read_settings, nullptr, getAsyncPolicy().getScheduler(), + fallback_reader, std::nullopt, String("*"), &dest_etag); + } + catch (const DB::S3Exception & e) + { + EXPECT_TRUE(e.isPreconditionFailed()); + throw; + } + }, DB::S3Exception); + + EXPECT_EQ(client->counters.putObject, 0); +} + [[maybe_unused]] static String fillStringWithPattern(String pattern, int n) { String data; From f6816f566530d4cfa96d9c55ac985c09aeb46201 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:32 +0200 Subject: [PATCH 06/30] Use Expect: 100-continue for large conditional S3 uploads A conditional PUT that is doomed to 412 should not stream its whole body; send Expect: 100-continue above a size threshold and peek the response. Prevents mid-upload connection resets and retry storms on S3-compatible stores. Carries the GCS dialect integration points (wired later). Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- src/IO/S3/PocoHTTPClient.cpp | 93 ++++++++++++++++++++++++++++- src/IO/S3/PocoHTTPClient.h | 38 ++++++++++++ src/IO/S3/PocoHTTPClientFactory.cpp | 3 + 3 files changed, 133 insertions(+), 1 deletion(-) diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 2b76300bbbc9..319d7375f62d 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -7,6 +7,8 @@ #if USE_AWS_S3 #include +#include +#include #include #include @@ -24,6 +26,7 @@ #include #include +#include #include #include #include @@ -89,6 +92,7 @@ namespace DB::ErrorCodes extern const int DNS_ERROR; extern const int AUTHENTICATION_FAILED; extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; } namespace HistogramMetrics @@ -221,11 +225,13 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_config , remote_host_filter(client_configuration.remote_host_filter) , s3_max_redirects(client_configuration.s3_max_redirects) , s3_use_adaptive_timeouts(client_configuration.s3_use_adaptive_timeouts) + , expect_continue_min_bytes(client_configuration.expect_continue_min_bytes) , http_max_fields(client_configuration.http_max_fields) , http_max_field_name_size(client_configuration.http_max_field_name_size) , http_max_field_value_size(client_configuration.http_max_field_value_size) , enable_s3_requests_logging(client_configuration.enable_s3_requests_logging) , for_disk_s3(client_configuration.for_disk_s3) + , gcs_conditional_dialect(client_configuration.gcs_conditional_dialect) , request_throttler(client_configuration.request_throttler) , extra_headers(client_configuration.extra_headers) { @@ -617,6 +623,42 @@ void PocoHTTPClient::makeRequestInternalImpl( Stopwatch watch; + /// A conditional write (`If-None-Match` / `If-Match`) that loses its precondition can waste + /// a LARGE body: streaming multi-MB into a request the server has already decided to reject + /// makes some stores (e.g. RustFS) close mid-upload or answer a retryable 500, which the SDK + /// then RETRIES up to `s3_retry_attempts` (500) times — a ~40-min stall that hangs CA INSERTs + /// (see B118). `Expect: 100-continue` lets the server reject (e.g. 412) BEFORE the body, so we + /// skip the doomed upload. + /// + /// `expect_continue_min_bytes` is the negotiation gate: `0` (the default, carried by every + /// non-CAS S3 client) DISABLES it entirely — non-CAS conditional PUTs keep upstream wire + /// behaviour and this whole block, INCLUDING the body-size probe, is skipped. A positive value + /// negotiates Expect for a conditional PUT whose body is at least that many bytes; only a CAS + /// conditional-write client raises it (the single-attempt client built in `ObjectStorageBackend`), + /// so the scope is exactly CAS-owned conditional writes. `x-goog-if-generation-match` is the GCS + /// conditional dialect's rename of If-None-Match / If-Match (applied BEFORE this point). + bool conditional_write = false; + if (expect_continue_min_bytes > 0 + && method == Poco::Net::HTTPRequest::HTTP_PUT + && (poco_request.has("if-none-match") || poco_request.has("if-match") + || poco_request.has("x-goog-if-generation-match"))) + { + size_t content_body_size = 0; + if (const auto & content_body = request.GetContentBody()) + { + content_body->clear(); + content_body->seekg(0, std::ios_base::end); + const auto end_pos = content_body->tellg(); + content_body->clear(); + content_body->seekg(0, std::ios_base::beg); + if (end_pos > 0) + content_body_size = static_cast(end_pos); + } + conditional_write = content_body_size >= expect_continue_min_bytes; + } + if (conditional_write) + poco_request.setExpectContinue(true); + auto & request_body_stream = session->sendRequest(poco_request, &connect_time, &first_byte_time); /// We record connect time here and not earlier, so that if an exception occurs while sending a request, /// we won't record the same latency twice. @@ -624,7 +666,20 @@ void PocoHTTPClient::makeRequestInternalImpl( observeLatency(request, first_byte_latency_type, static_cast(first_byte_time)); latency_recorded = true; - if (request.GetContentBody()) + /// With `Expect: 100-continue`, peek the interim response after the headers. `true` means + /// the server sent `100 Continue` (proceed with the body); `false` means it already sent a + /// FINAL response (now in `poco_response`) and the body must NOT be sent. `receiveResponse` + /// below is still called in both cases (Poco contract) and skips re-reading the headers. + bool skip_body = false; + if (conditional_write) + { + setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ true)); + skip_body = !session->peekResponse(poco_response); + if (enable_s3_requests_logging) + LOG_TEST(log, "Expect: 100-continue peek -> {}", skip_body ? "final response, skipping body" : "100 Continue"); + } + + if (request.GetContentBody() && !skip_body) { if (enable_s3_requests_logging) LOG_TEST(log, "Writing request body."); @@ -693,6 +748,17 @@ void PocoHTTPClient::makeRequestInternalImpl( response->SetResponseCode(static_cast(status_code)); response->SetContentType(poco_response.getContentType()); + auto apply_gcs_generation_etag_override = [&] + { + if (gcs_conditional_dialect) + { + /// The generation IS the incarnation token on GCS: surface it as the ETag so the + /// entire existing ETag/token plumbing works unchanged (see GCSConditionalDialect.h). + if (auto etag_override = gcsGenerationETagOverride(poco_response)) + response->AddHeader("ETag", *etag_override); + } + }; + if (enable_s3_requests_logging) { WriteBufferFromOwnString headers_ss; @@ -701,12 +767,14 @@ void PocoHTTPClient::makeRequestInternalImpl( response->AddHeader(header_name, header_value); headers_ss << header_name << ": " << header_value << "; "; } + apply_gcs_generation_etag_override(); LOG_TEST(log, "Received headers: {}", headers_ss.str()); } else { for (const auto & [header_name, header_value] : poco_response) response->AddHeader(header_name, header_value); + apply_gcs_generation_etag_override(); } /// Request is successful but for some special requests we can have actual error message in body @@ -835,6 +903,9 @@ void PocoHTTPClientGCPOAuth::makeRequestInternal( Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const { + if (gcs_conditional_dialect) + applyGcsConditionalDialectToRequest(request); + { std::lock_guard lock(mutex); if (!bearer_token || std::chrono::system_clock::now() > bearer_token->is_valid_to) @@ -922,6 +993,26 @@ PocoHTTPClientGCPOAuth::BearerToken PocoHTTPClientGCPOAuth::requestBearerTokenFr }; } +PocoHTTPClientGCSHMAC::PocoHTTPClientGCSHMAC(const PocoHTTPClientConfiguration & client_configuration) + : PocoHTTPClient(client_configuration) + , credentials_provider(client_configuration.gcs_hmac_credentials_provider) +{ + if (!credentials_provider) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "PocoHTTPClientGCSHMAC requires a credentials provider (http_client = gcs_hmac wiring bug)"); +} + +void PocoHTTPClientGCSHMAC::makeRequestInternal( + Aws::Http::HttpRequest & request, + std::shared_ptr & response, + Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, + Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const +{ + applyGcsConditionalDialectToRequest(request); + signRequestGOOG4(request, credentials_provider->GetAWSCredentials(), std::chrono::system_clock::now()); + PocoHTTPClient::makeRequestInternal(request, response, readLimiter, writeLimiter); +} + } #endif diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index e3182a508be9..d3154892919f 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -30,6 +30,11 @@ namespace Aws::Http::Standard class StandardHttpResponse; } +namespace Aws::Auth +{ +class AWSCredentialsProvider; +} + namespace DB { class Context; @@ -73,6 +78,14 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration HTTPHeaderEntries extra_headers; String http_client; + /// GCS conditional dialect (spec 2026-07-03-cas-gcs-generation-binding-design): translate + /// AWS-style conditional headers and x-amz-* prefixes to the x-goog dialect at the wire + /// boundary, and surface x-goog-generation as the response ETag. Set for http_client values + /// `gcs_hmac` and `gcp_oauth`; never set for plain AWS-compatible endpoints. + bool gcs_conditional_dialect = false; + /// Credentials for the GOOG4-HMAC signer (http_client = gcs_hmac only): the same provider + /// chain the AWS path builds (inline keys, use_environment_credentials, ...). + std::shared_ptr gcs_hmac_credentials_provider; String service_account; String metadata_service; String request_token_path; @@ -82,6 +95,10 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration /// See PoolBase::BehaviourOnLimit bool s3_use_adaptive_timeouts = true; + /// Conditional PUT (If-None-Match / If-Match) bodies >= this negotiate Expect: 100-continue (B118). + /// `0` (the default) disables it, so non-CAS S3 clients keep upstream behaviour; only a CAS + /// conditional-write client raises it (see the single-attempt client in `ObjectStorageBackend`). + size_t expect_continue_min_bytes = DEFAULT_EXPECT_CONTINUE_MIN_BYTES; size_t http_keep_alive_timeout = DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT; size_t http_keep_alive_max_requests = DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST; @@ -224,11 +241,13 @@ class PocoHTTPClient : public Aws::Http::HttpClient const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects = DEFAULT_MAX_REDIRECTS; bool s3_use_adaptive_timeouts = true; + size_t expect_continue_min_bytes = DEFAULT_EXPECT_CONTINUE_MIN_BYTES; const UInt64 http_max_fields = 1000000; const UInt64 http_max_field_name_size = 128 * 1024; const UInt64 http_max_field_value_size = 128 * 1024; bool enable_s3_requests_logging = false; bool for_disk_s3 = false; + bool gcs_conditional_dialect = false; HTTPRequestThrottler request_throttler; @@ -268,6 +287,25 @@ class PocoHTTPClientGCPOAuth : public PocoHTTPClient BearerToken requestBearerTokenFromADC() const; }; +/// GCS with HMAC credentials over the XML API, signed with Google's native GOOG4-HMAC-SHA256 — +/// the ONLY way HMAC credentials get enforced conditional semantics on GCS (the S3-compatible +/// sigv4 surface silently ignores If-None-Match / If-Match; measured 2026-07-03). Applies the GCS +/// conditional dialect, then signs. Selected by `http_client = gcs_hmac`. +class PocoHTTPClientGCSHMAC : public PocoHTTPClient +{ +public: + explicit PocoHTTPClientGCSHMAC(const PocoHTTPClientConfiguration & client_configuration); + +private: + void makeRequestInternal( + Aws::Http::HttpRequest & request, + std::shared_ptr & response, + Aws::Utils::RateLimits::RateLimiterInterface * readLimiter, + Aws::Utils::RateLimits::RateLimiterInterface * writeLimiter) const override; + + std::shared_ptr credentials_provider; +}; + } #endif diff --git a/src/IO/S3/PocoHTTPClientFactory.cpp b/src/IO/S3/PocoHTTPClientFactory.cpp index 0fb1cf40d93c..b7599290adbf 100644 --- a/src/IO/S3/PocoHTTPClientFactory.cpp +++ b/src/IO/S3/PocoHTTPClientFactory.cpp @@ -23,6 +23,9 @@ PocoHTTPClientFactory::CreateHttpClient(const Aws::Client::ClientConfiguration & if (Poco::toLower(poco_client_configuration.http_client) == "gcp_oauth") return std::make_shared(poco_client_configuration); + if (Poco::toLower(poco_client_configuration.http_client) == "gcs_hmac") + return std::make_shared(poco_client_configuration); + return std::make_shared(poco_client_configuration); } From 57bd41968cc22743cb18f0827c3aa303e8a984f0 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:32 +0200 Subject: [PATCH 07/30] GCS conditional-write dialect and GOOG4 signer GCS XML API needs native GOOG4 signing and x-goog-if-generation-match for generation-safe conditional writes; AWS SigV4 If-Match semantics are not enough, and conditional multipart complete is silently ignored. Adds the signer, the header dialect, and fixed-vector tests. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- src/IO/S3/GCSConditionalDialect.cpp | 104 ++++++++++++ src/IO/S3/GCSConditionalDialect.h | 36 ++++ src/IO/S3/GOOG4Signer.cpp | 149 ++++++++++++++++ src/IO/S3/GOOG4Signer.h | 30 ++++ .../tests/gtest_gcs_conditional_dialect.cpp | 159 ++++++++++++++++++ src/IO/S3/tests/gtest_goog4_signer.cpp | 62 +++++++ 6 files changed, 540 insertions(+) create mode 100644 src/IO/S3/GCSConditionalDialect.cpp create mode 100644 src/IO/S3/GCSConditionalDialect.h create mode 100644 src/IO/S3/GOOG4Signer.cpp create mode 100644 src/IO/S3/GOOG4Signer.h create mode 100644 src/IO/S3/tests/gtest_gcs_conditional_dialect.cpp create mode 100644 src/IO/S3/tests/gtest_goog4_signer.cpp diff --git a/src/IO/S3/GCSConditionalDialect.cpp b/src/IO/S3/GCSConditionalDialect.cpp new file mode 100644 index 000000000000..5c153ce44c2e --- /dev/null +++ b/src/IO/S3/GCSConditionalDialect.cpp @@ -0,0 +1,104 @@ +#include + +#if USE_AWS_S3 + +#include +#include +#include + +#include +#include + +namespace DB::ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace DB::S3 +{ + +namespace +{ + +bool isAllDigits(const std::string & s) +{ + return !s.empty() && std::all_of(s.begin(), s.end(), [](char c) { return c >= '0' && c <= '9'; }); +} + +std::string stripQuotes(const std::string & s) +{ + if (s.size() >= 2 && s.front() == '"' && s.back() == '"') + return s.substr(1, s.size() - 2); + return s; +} + +} + +void applyGcsConditionalDialectToRequest(Aws::Http::HttpRequest & request) +{ + const auto query_params = request.GetUri().GetQueryStringParameters(); + const bool is_complete_multipart = request.GetMethod() == Aws::Http::HttpMethod::HTTP_POST + && query_params.contains("uploadId") && !query_params.contains("partNumber"); + + /// --- Conditional headers -> x-goog-if-generation-match --- + std::optional generation_match; + if (request.HasHeader("if-none-match")) + { + const auto value = request.GetHeaderValue("if-none-match"); + if (value != "*") + throw Exception(ErrorCodes::LOGICAL_ERROR, + "GCS conditional dialect: If-None-Match with a value other than '*' has no GCS " + "equivalent (got '{}') — refusing to silently change semantics", value); + generation_match = "0"; + request.DeleteHeader("if-none-match"); + } + if (request.HasHeader("if-match")) + { + const auto value = stripQuotes(request.GetHeaderValue("if-match")); + if (!isAllDigits(value)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "GCS conditional dialect: If-Match value '{}' is not a generation number — an " + "ETag-kind token leaked into a generation-dialect client (mixed-mode misconfiguration)", + value); + generation_match = value; + request.DeleteHeader("if-match"); + } + if (generation_match) + { + if (is_complete_multipart) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "GCS conditional dialect: a CONDITIONAL CompleteMultipartUpload was about to be sent. " + "GCS silently ignores preconditions on CompleteMultipartUpload (measured 2026-07-03) — " + "this would be silent data loss. Conditional writes must use the single-PUT path."); + request.SetHeaderValue("x-goog-if-generation-match", *generation_match); + } + + /// --- AWS auth artifacts: drop (the GCS-mode client re-authenticates after this call) --- + for (const auto * header : {"authorization", "x-amz-date", "x-amz-content-sha256", + "x-amz-security-token", "x-amz-api-version"}) + request.DeleteHeader(header); + + /// --- Rename every remaining x-amz-* header to x-goog-* (mixing is rejected by GCS) --- + std::vector> renamed; + for (const auto & [name, value] : request.GetHeaders()) + { + if (name.starts_with("x-amz-")) + renamed.emplace_back("x-goog-" + name.substr(6), value); + } + for (const auto & [goog_name, value] : renamed) + { + request.DeleteHeader(("x-amz-" + goog_name.substr(7)).c_str()); + request.SetHeaderValue(goog_name.c_str(), value); + } +} + +std::optional gcsGenerationETagOverride(const Poco::Net::HTTPResponse & response) +{ + if (!response.has("x-goog-generation")) + return std::nullopt; + return "\"" + response.get("x-goog-generation") + "\""; +} + +} + +#endif diff --git a/src/IO/S3/GCSConditionalDialect.h b/src/IO/S3/GCSConditionalDialect.h new file mode 100644 index 000000000000..f63f331ae508 --- /dev/null +++ b/src/IO/S3/GCSConditionalDialect.h @@ -0,0 +1,36 @@ +#pragma once +#include "config.h" +#if USE_AWS_S3 + +#include +#include + +namespace Aws::Http { class HttpRequest; } +namespace Poco::Net { class HTTPResponse; } + +namespace DB::S3 +{ + +/// The GCS conditional dialect, request side (spec: 2026-07-03-cas-gcs-generation-binding-design). +/// Applied at the wire boundary by the GCS-mode Poco HTTP clients, so everything above keeps +/// speaking AWS. Translations: +/// - AWS auth artifacts (`authorization`, `x-amz-date`, `x-amz-content-sha256`, +/// `x-amz-security-token`, `x-amz-api-version`) are DROPPED (the caller re-authenticates); +/// - every remaining `x-amz-*` header is renamed to `x-goog-*`; +/// - `If-None-Match: *` becomes `x-goog-if-generation-match: 0`; +/// - `If-Match: ""` (quotes optional) becomes `x-goog-if-generation-match: `. +/// Fail-close guards (throw LOGICAL_ERROR, the request never leaves the process): +/// - `If-None-Match` with any value other than `*` (no GCS equivalent); +/// - a non-numeric `If-Match` (an ETag-kind token leaked into a generation dialect); +/// - a CONDITIONAL CompleteMultipartUpload (POST with `uploadId` and no `partNumber`): GCS +/// silently ignores preconditions there (measured live 2026-07-03) — silent data loss. +void applyGcsConditionalDialectToRequest(Aws::Http::HttpRequest & request); + +/// The dialect, response side: when the response carries `x-goog-generation`, returns it QUOTED — +/// the caller substitutes it for the `ETag` response header, making the generation ride the +/// entire existing ETag/token plumbing unchanged. Returns nullopt when no generation is present. +std::optional gcsGenerationETagOverride(const Poco::Net::HTTPResponse & response); + +} + +#endif diff --git a/src/IO/S3/GOOG4Signer.cpp b/src/IO/S3/GOOG4Signer.cpp new file mode 100644 index 000000000000..740b65d85c44 --- /dev/null +++ b/src/IO/S3/GOOG4Signer.cpp @@ -0,0 +1,149 @@ +#include + +#if USE_AWS_S3 + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +namespace DB::ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace DB::S3 +{ + +namespace +{ + +constexpr auto UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; + +std::string hmacSHA256(const std::string & key, const std::string & message) +{ + unsigned char out[SHA256_DIGEST_LENGTH]; + unsigned int out_len = 0; + HMAC(EVP_sha256(), + key.data(), static_cast(key.size()), + reinterpret_cast(message.data()), message.size(), + out, &out_len); + return std::string(reinterpret_cast(out), out_len); +} + +std::string sha256Hex(const std::string & data) +{ + unsigned char out[SHA256_DIGEST_LENGTH]; + SHA256(reinterpret_cast(data.data()), data.size(), out); + return hexString(out, SHA256_DIGEST_LENGTH); +} + +} + +void signRequestGOOG4( + Aws::Http::HttpRequest & request, + const Aws::Auth::AWSCredentials & credentials, + std::chrono::system_clock::time_point now) +{ + const std::time_t now_t = std::chrono::system_clock::to_time_t(now); + std::tm tm_utc{}; + gmtime_r(&now_t, &tm_utc); + const std::string timestamp = fmt::format( + "{:04}{:02}{:02}T{:02}{:02}{:02}Z", + tm_utc.tm_year + 1900, tm_utc.tm_mon + 1, tm_utc.tm_mday, + tm_utc.tm_hour, tm_utc.tm_min, tm_utc.tm_sec); + const std::string datestamp = timestamp.substr(0, 8); + + request.SetHeaderValue("x-goog-date", timestamp); + request.SetHeaderValue("x-goog-content-sha256", UNSIGNED_PAYLOAD); + + /// Canonical headers: `host` + every x-goog-* header, lowercase names, sorted. + /// std::map keeps them sorted for us. + std::map signed_headers_map; + for (const auto & [name, value] : request.GetHeaders()) + { + std::string lower = Aws::Utils::StringUtils::ToLower(name.c_str()); + if (lower == "host" || lower.starts_with("x-goog-")) + signed_headers_map.emplace(std::move(lower), value); + } + if (!signed_headers_map.contains("host")) + throw Exception(ErrorCodes::LOGICAL_ERROR, "GOOG4 signing requires a Host header on the request"); + + std::string canonical_headers; + std::string signed_headers; + for (const auto & [name, value] : signed_headers_map) + { + canonical_headers += name + ":" + value + "\n"; + if (!signed_headers.empty()) + signed_headers += ";"; + signed_headers += name; + } + + /// Canonical query string: URL-encoded key=value pairs sorted by key; a parameter without a + /// value still gets a trailing `=` (e.g. `versioning=`). + /// + /// `Aws::Http::URI` has no ready-made helper for this: `CanonicalizeQueryString` only rewrites + /// the query string when it already contains an `=`, so a bare flag like `?versioning` (no `=`) + /// passes through unsorted and unencoded. `GetQueryStringParameters` doesn't help either — for + /// a valueless flag it has no `=` to split on, so it treats the whole `key` as the `value` too + /// (`versioning` becomes `versioning=versioning`, not `versioning=`). Parse the raw query string + /// by hand instead, splitting each `key[=value]` pair on the first `=` with an empty value when + /// absent, then URL-encode and join sorted `key=value` pairs with `&`. + std::map query_params; + { + const std::string raw_query = request.GetUri().GetQueryString(); + size_t pos = raw_query.empty() ? std::string::npos : 1; /// skip leading '?' + while (pos != std::string::npos && pos < raw_query.size()) + { + const size_t amp = raw_query.find('&', pos); + const std::string pair = raw_query.substr(pos, amp == std::string::npos ? std::string::npos : amp - pos); + const size_t eq = pair.find('='); + std::string key = eq == std::string::npos ? pair : pair.substr(0, eq); + std::string value = eq == std::string::npos ? std::string() : pair.substr(eq + 1); + query_params.emplace( + Aws::Utils::StringUtils::URLDecode(key.c_str()), + Aws::Utils::StringUtils::URLDecode(value.c_str())); + pos = amp == std::string::npos ? std::string::npos : amp + 1; + } + } + std::string canonical_query; + for (const auto & [key, value] : query_params) + { + if (!canonical_query.empty()) + canonical_query += "&"; + canonical_query += Aws::Utils::StringUtils::URLEncode(key.c_str()) + "=" + Aws::Utils::StringUtils::URLEncode(value.c_str()); + } + const std::string canonical_uri = request.GetUri().GetURLEncodedPath(); + + const std::string method = Aws::Http::HttpMethodMapper::GetNameForHttpMethod(request.GetMethod()); + + const std::string canonical_request = fmt::format( + "{}\n{}\n{}\n{}\n{}\n{}", + method, canonical_uri, canonical_query, canonical_headers, signed_headers, UNSIGNED_PAYLOAD); + + const std::string scope = fmt::format("{}/auto/storage/goog4_request", datestamp); + const std::string string_to_sign = fmt::format( + "GOOG4-HMAC-SHA256\n{}\n{}\n{}", timestamp, scope, sha256Hex(canonical_request)); + + std::string key = hmacSHA256("GOOG4" + credentials.GetAWSSecretKey(), datestamp); + key = hmacSHA256(key, "auto"); + key = hmacSHA256(key, "storage"); + key = hmacSHA256(key, "goog4_request"); + const std::string signature = hexString(hmacSHA256(key, string_to_sign).data(), SHA256_DIGEST_LENGTH); + + request.SetHeaderValue("authorization", fmt::format( + "GOOG4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}", + credentials.GetAWSAccessKeyId(), scope, signed_headers, signature)); +} + +} + +#endif diff --git a/src/IO/S3/GOOG4Signer.h b/src/IO/S3/GOOG4Signer.h new file mode 100644 index 000000000000..4b1f4c1b89c0 --- /dev/null +++ b/src/IO/S3/GOOG4Signer.h @@ -0,0 +1,30 @@ +#pragma once +#include "config.h" +#if USE_AWS_S3 + +#include + +namespace Aws::Http { class HttpRequest; } +namespace Aws::Auth { class AWSCredentials; } + +namespace DB::S3 +{ + +/// Sign `request` in place with GOOG4-HMAC-SHA256 — Google Cloud Storage's native V4 HMAC scheme +/// for the XML API. Structurally sigv4 with renamed constants: key prefix `GOOG4`, scope +/// terminator `goog4_request`, headers `x-goog-date` / `x-goog-content-sha256`. Bodies are never +/// hashed (`UNSIGNED-PAYLOAD`), so streaming uploads sign in O(1). +/// +/// Signs the `host` header plus EVERY `x-goog-*` header present on the request (GCS requires all +/// x-goog headers to be signed); other headers ride unsigned. `now` is injected so unit tests can +/// pin the timestamp to fixed vectors. +/// +/// Live-validated against GCS 2026-07-03 (see `utils/ca-soak/scripts/gcs_goog4_probe.py`, 12/12). +void signRequestGOOG4( + Aws::Http::HttpRequest & request, + const Aws::Auth::AWSCredentials & credentials, + std::chrono::system_clock::time_point now); + +} + +#endif diff --git a/src/IO/S3/tests/gtest_gcs_conditional_dialect.cpp b/src/IO/S3/tests/gtest_gcs_conditional_dialect.cpp new file mode 100644 index 000000000000..30d1117a0c60 --- /dev/null +++ b/src/IO/S3/tests/gtest_gcs_conditional_dialect.cpp @@ -0,0 +1,159 @@ +#include "config.h" +#if USE_AWS_S3 +#include +#include +#include +#include +#include +#include /// DEBUG_OR_SANITIZER_BUILD + +using namespace DB::S3; + +static Aws::Http::Standard::StandardHttpRequest makeRequest( + const char * url = "https://storage.googleapis.com/b/k", + Aws::Http::HttpMethod method = Aws::Http::HttpMethod::HTTP_PUT) +{ + Aws::Http::Standard::StandardHttpRequest request{Aws::Http::URI(url), method}; + request.SetHeaderValue("host", "storage.googleapis.com"); + return request; +} + +TEST(GCSConditionalDialect, IfNoneMatchStarBecomesGenerationZero) +{ + auto r = makeRequest(); + r.SetHeaderValue("if-none-match", "*"); + applyGcsConditionalDialectToRequest(r); + EXPECT_FALSE(r.HasHeader("if-none-match")); + EXPECT_EQ(r.GetHeaderValue("x-goog-if-generation-match"), "0"); +} + +TEST(GCSConditionalDialect, IfMatchDigitsMappedQuotesStripped) +{ + auto r = makeRequest(); + r.SetHeaderValue("if-match", "\"1783078552147137\""); + applyGcsConditionalDialectToRequest(r); + EXPECT_FALSE(r.HasHeader("if-match")); + EXPECT_EQ(r.GetHeaderValue("x-goog-if-generation-match"), "1783078552147137"); +} + +TEST(GCSConditionalDialect, IfMatchUnquotedDigitsAlsoAccepted) +{ + auto r = makeRequest(); + r.SetHeaderValue("if-match", "1783078552147137"); + applyGcsConditionalDialectToRequest(r); + EXPECT_EQ(r.GetHeaderValue("x-goog-if-generation-match"), "1783078552147137"); +} + +#ifndef DEBUG_OR_SANITIZER_BUILD +TEST(GCSConditionalDialect, NonNumericIfMatchThrows) +{ + /// The guard throws LOGICAL_ERROR (a broken-invariant signal: an S3-style ETag reached a + /// generation-dialect client). Under abort_on_logical_error that aborts at construction instead of + /// being catchable, so GCSConditionalDialectDeathTest.NonNumericIfMatchAborts proves it there. + auto r = makeRequest(); + r.SetHeaderValue("if-match", "\"6654c734ccab8f440ff0825eb443dc7f\""); /// an ETag leaked into a generation dialect + EXPECT_THROW(applyGcsConditionalDialectToRequest(r), DB::Exception); +} +#endif + +#if defined(DEBUG_OR_SANITIZER_BUILD) +TEST(GCSConditionalDialectDeathTest, NonNumericIfMatchAborts) +{ + auto r = makeRequest(); + r.SetHeaderValue("if-match", "\"6654c734ccab8f440ff0825eb443dc7f\""); + EXPECT_DEATH({ applyGcsConditionalDialectToRequest(r); }, ""); +} +#endif + +#ifndef DEBUG_OR_SANITIZER_BUILD +TEST(GCSConditionalDialect, NonStarIfNoneMatchThrows) +{ + /// LOGICAL_ERROR (broken invariant); aborts under abort_on_logical_error -- see the DeathTest below. + auto r = makeRequest(); + r.SetHeaderValue("if-none-match", "\"123\""); + EXPECT_THROW(applyGcsConditionalDialectToRequest(r), DB::Exception); +} +#endif + +#if defined(DEBUG_OR_SANITIZER_BUILD) +TEST(GCSConditionalDialectDeathTest, NonStarIfNoneMatchAborts) +{ + auto r = makeRequest(); + r.SetHeaderValue("if-none-match", "\"123\""); + EXPECT_DEATH({ applyGcsConditionalDialectToRequest(r); }, ""); +} +#endif + +TEST(GCSConditionalDialect, AmzHeadersRenamedAuthArtifactsDropped) +{ + auto r = makeRequest(); + r.SetHeaderValue("authorization", "AWS4-HMAC-SHA256 ..."); + r.SetHeaderValue("x-amz-date", "20260703T000000Z"); + r.SetHeaderValue("x-amz-content-sha256", "deadbeef"); + r.SetHeaderValue("x-amz-security-token", "tok"); + r.SetHeaderValue("x-amz-api-version", "2006-03-01"); + r.SetHeaderValue("x-amz-meta-foo", "bar"); + r.SetHeaderValue("x-amz-storage-class", "STANDARD"); + applyGcsConditionalDialectToRequest(r); + EXPECT_FALSE(r.HasHeader("authorization")); + EXPECT_FALSE(r.HasHeader("x-amz-date")); + EXPECT_FALSE(r.HasHeader("x-amz-content-sha256")); + EXPECT_FALSE(r.HasHeader("x-amz-security-token")); + EXPECT_FALSE(r.HasHeader("x-amz-api-version")); + EXPECT_FALSE(r.HasHeader("x-amz-meta-foo")); + EXPECT_FALSE(r.HasHeader("x-amz-storage-class")); + EXPECT_EQ(r.GetHeaderValue("x-goog-meta-foo"), "bar"); + EXPECT_EQ(r.GetHeaderValue("x-goog-storage-class"), "STANDARD"); +} + +#ifndef DEBUG_OR_SANITIZER_BUILD +TEST(GCSConditionalDialect, ConditionalCompleteMultipartUploadThrows) +{ + /// GCS silently IGNORES preconditions on CompleteMultipartUpload (measured live 2026-07-03) -- + /// sending one would be silent data loss, so the dialect fails closed client-side with a + /// LOGICAL_ERROR; aborts under abort_on_logical_error -- see the DeathTest below. + auto r = makeRequest("https://storage.googleapis.com/b/k?uploadId=abc", Aws::Http::HttpMethod::HTTP_POST); + r.SetHeaderValue("if-none-match", "*"); + EXPECT_THROW(applyGcsConditionalDialectToRequest(r), DB::Exception); +} +#endif + +#if defined(DEBUG_OR_SANITIZER_BUILD) +TEST(GCSConditionalDialectDeathTest, ConditionalCompleteMultipartUploadAborts) +{ + auto r = makeRequest("https://storage.googleapis.com/b/k?uploadId=abc", Aws::Http::HttpMethod::HTTP_POST); + r.SetHeaderValue("if-none-match", "*"); + EXPECT_DEATH({ applyGcsConditionalDialectToRequest(r); }, ""); +} +#endif + +TEST(GCSConditionalDialect, UnconditionalCompleteMultipartUploadPasses) +{ + auto r = makeRequest("https://storage.googleapis.com/b/k?uploadId=abc", Aws::Http::HttpMethod::HTTP_POST); + EXPECT_NO_THROW(applyGcsConditionalDialectToRequest(r)); +} + +TEST(GCSConditionalDialect, UploadPartIsNotComplete) +{ + /// PUT ?partNumber=N&uploadId=... is an UploadPart, not a Complete — must not trip the guard. + auto r = makeRequest("https://storage.googleapis.com/b/k?partNumber=1&uploadId=abc", Aws::Http::HttpMethod::HTTP_PUT); + EXPECT_NO_THROW(applyGcsConditionalDialectToRequest(r)); +} + +TEST(GCSConditionalDialect, ResponseGenerationOverridesETag) +{ + Poco::Net::HTTPResponse response; + response.set("ETag", "\"6654c734ccab8f440ff0825eb443dc7f\""); + response.set("x-goog-generation", "1783078552147137"); + auto override_etag = gcsGenerationETagOverride(response); + ASSERT_TRUE(override_etag.has_value()); + EXPECT_EQ(*override_etag, "\"1783078552147137\""); +} + +TEST(GCSConditionalDialect, ResponseWithoutGenerationNoOverride) +{ + Poco::Net::HTTPResponse response; + response.set("ETag", "\"abc\""); + EXPECT_FALSE(gcsGenerationETagOverride(response).has_value()); +} +#endif diff --git a/src/IO/S3/tests/gtest_goog4_signer.cpp b/src/IO/S3/tests/gtest_goog4_signer.cpp new file mode 100644 index 000000000000..f3e1a95aad6a --- /dev/null +++ b/src/IO/S3/tests/gtest_goog4_signer.cpp @@ -0,0 +1,62 @@ +#include "config.h" +#if USE_AWS_S3 +#include +#include +#include +#include + +using namespace DB::S3; + +static std::chrono::system_clock::time_point fixedNow() +{ + /// 2026-07-03 00:00:00 UTC + return std::chrono::system_clock::from_time_t(1783036800); +} + +TEST(GOOG4Signer, PutWithGenerationPrecondition) +{ + Aws::Http::Standard::StandardHttpRequest request( + Aws::Http::URI("https://storage.googleapis.com/test-bucket/dir/obj.txt"), Aws::Http::HttpMethod::HTTP_PUT); + request.SetHeaderValue("host", "storage.googleapis.com"); + request.SetHeaderValue("x-goog-if-generation-match", "0"); + + signRequestGOOG4(request, Aws::Auth::AWSCredentials("GOOGTESTACCESSKEY", "testsecretkey"), fixedNow()); + + EXPECT_EQ(request.GetHeaderValue("x-goog-date"), "20260703T000000Z"); + EXPECT_EQ(request.GetHeaderValue("x-goog-content-sha256"), "UNSIGNED-PAYLOAD"); + EXPECT_EQ(request.GetHeaderValue("authorization"), + "GOOG4-HMAC-SHA256 Credential=GOOGTESTACCESSKEY/20260703/auto/storage/goog4_request, " + "SignedHeaders=host;x-goog-content-sha256;x-goog-date;x-goog-if-generation-match, " + "Signature=4f82e49c69753329afd4768ccf1db6b472dbbd86d082a08b5b9f9fe368fb6ef6"); +} + +TEST(GOOG4Signer, GetWithQueryString) +{ + Aws::Http::Standard::StandardHttpRequest request( + Aws::Http::URI("https://storage.googleapis.com/test-bucket/?versioning"), Aws::Http::HttpMethod::HTTP_GET); + request.SetHeaderValue("host", "storage.googleapis.com"); + + signRequestGOOG4(request, Aws::Auth::AWSCredentials("GOOGTESTACCESSKEY", "testsecretkey"), fixedNow()); + + EXPECT_EQ(request.GetHeaderValue("authorization"), + "GOOG4-HMAC-SHA256 Credential=GOOGTESTACCESSKEY/20260703/auto/storage/goog4_request, " + "SignedHeaders=host;x-goog-content-sha256;x-goog-date, " + "Signature=28a981c32acff334738b9ea1a0f82c28c9a1ccff5b6dc8fb92a2e6622c8db73f"); +} + +TEST(GOOG4Signer, NonGoogHeadersAreNotSigned) +{ + Aws::Http::Standard::StandardHttpRequest request( + Aws::Http::URI("https://storage.googleapis.com/test-bucket/dir/obj.txt"), Aws::Http::HttpMethod::HTTP_PUT); + request.SetHeaderValue("host", "storage.googleapis.com"); + request.SetHeaderValue("x-goog-if-generation-match", "0"); + request.SetHeaderValue("content-type", "binary/octet-stream"); + request.SetHeaderValue("amz-sdk-invocation-id", "whatever"); + + signRequestGOOG4(request, Aws::Auth::AWSCredentials("GOOGTESTACCESSKEY", "testsecretkey"), fixedNow()); + + /// Unsigned headers must not perturb the signature: same vector as PutWithGenerationPrecondition. + EXPECT_NE(request.GetHeaderValue("authorization").find( + "Signature=4f82e49c69753329afd4768ccf1db6b472dbbd86d082a08b5b9f9fe368fb6ef6"), std::string::npos); +} +#endif From 4fcc40f97d983f492d1b63119f61b3084a7f764a Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:32 +0200 Subject: [PATCH 08/30] LocalObjectStorage: snapshot listing semantics and hardening Emulate object-store behavior under concurrent removal: a file vanishing between listing and stat is skipped, not a filesystem_error; non-recursive explicit-stack walk with error_code overloads and a symlink guard; fail closed on an embedded-NUL path (AST fuzzer). Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../ObjectStorages/Local/LocalObjectStorage.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp index 11a7cbf29acc..4719e6ae9237 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/Local/LocalObjectStorage.cpp @@ -408,6 +408,12 @@ std::optional LocalObjectStorage::tryGetObjectMetadata(const std throw fs::filesystem_error("Got unexpected error while getting last write time", path, error); } + /// A directory is not an object: fs::file_size would throw "Is a directory". Treat it as a + /// missing object (nullopt) so callers probing whether a path is a readable object do not get + /// a raw filesystem error (B38: system.remote_data_paths traversal on a CAS pool). + if (fs::is_directory(path, error)) + return {}; + object_metadata.size_bytes = fs::file_size(path, error); if (error) { From 93d545b744627c3b05b0cb791444f3bdbd5b90cd Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:33 +0200 Subject: [PATCH 09/30] Disk-transaction contract: one logical part = one transaction Projection sub-parts ride the parent whole-part transaction instead of committing early; read-your-writes (in-flight resolve) becomes part of the IDiskTransaction contract so staged state is visible before commit; clone/freeze/restore wrap the whole part in one transaction; staged operation order is explicit. Mixed-file note: these files also carry the content-addressed capability surface and eager-dispatch branches that are wired by the later CAS integration commits. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../DiskObjectStorageTransaction.cpp | 109 ++++++++--- .../DiskObjectStorageTransaction.h | 20 +++ src/Disks/IDiskTransaction.h | 17 ++ .../MergeTree/DataPartStorageOnDiskBase.cpp | 168 +++++++++++++++-- .../MergeTree/DataPartStorageOnDiskBase.h | 2 + .../MergeTree/DataPartStorageOnDiskFull.cpp | 170 ++++++++++++++++-- src/Storages/MergeTree/IDataPartStorage.h | 9 + src/Storages/MergeTree/IMergeTreeDataPart.cpp | 8 +- .../MergeTree/MergeProjectionPartsTask.cpp | 4 + src/Storages/MergeTree/MergeTask.cpp | 16 +- src/Storages/MergeTree/MergeTask.h | 4 + src/Storages/MergeTree/MergeTreeData.cpp | 116 +++++++++++- src/Storages/MergeTree/MergeTreeData.h | 14 +- .../MergeTree/MergeTreeDataWriter.cpp | 2 + src/Storages/MergeTree/MutateTask.cpp | 3 + .../gtest_projection_borrowed_transaction.cpp | 86 +++++++++ 16 files changed, 691 insertions(+), 57 deletions(-) create mode 100644 src/Storages/MergeTree/tests/gtest_projection_borrowed_transaction.cpp diff --git a/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp b/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp index 34d14edaffa5..6a52ca93e673 100644 --- a/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp +++ b/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp @@ -110,7 +110,7 @@ MultipleDisksObjectStorageTransaction::MultipleDisksObjectStorageTransaction( void DiskObjectStorageTransaction::createDirectory(const std::string & path) { - operations_to_execute.push_back([path](MetadataTransactionPtr tx) + dispatch([path](MetadataTransactionPtr tx) { tx->createDirectory(path); }); @@ -118,7 +118,7 @@ void DiskObjectStorageTransaction::createDirectory(const std::string & path) void DiskObjectStorageTransaction::createDirectories(const std::string & path) { - operations_to_execute.push_back([path](MetadataTransactionPtr tx) + dispatch([path](MetadataTransactionPtr tx) { tx->createDirectoryRecursive(path); }); @@ -126,7 +126,7 @@ void DiskObjectStorageTransaction::createDirectories(const std::string & path) void DiskObjectStorageTransaction::moveDirectory(const std::string & from_path, const std::string & to_path) { - operations_to_execute.push_back([from_path, to_path](MetadataTransactionPtr tx) + dispatch([from_path, to_path](MetadataTransactionPtr tx) { tx->moveDirectory(from_path, to_path); }); @@ -134,7 +134,7 @@ void DiskObjectStorageTransaction::moveDirectory(const std::string & from_path, void DiskObjectStorageTransaction::moveFile(const String & from_path, const String & to_path) { - operations_to_execute.push_back([from_path, to_path](MetadataTransactionPtr tx) + dispatch([from_path, to_path](MetadataTransactionPtr tx) { tx->moveFile(from_path, to_path); }); @@ -142,7 +142,7 @@ void DiskObjectStorageTransaction::moveFile(const String & from_path, const Stri void DiskObjectStorageTransaction::truncateFile(const String & path, size_t size) { - operations_to_execute.push_back([path, size](MetadataTransactionPtr tx) + dispatch([path, size](MetadataTransactionPtr tx) { tx->truncateFile(path, size); }); @@ -150,7 +150,7 @@ void DiskObjectStorageTransaction::truncateFile(const String & path, size_t size void DiskObjectStorageTransaction::replaceFile(const std::string & from_path, const std::string & to_path) { - operations_to_execute.push_back([from_path, to_path](MetadataTransactionPtr tx) + dispatch([from_path, to_path](MetadataTransactionPtr tx) { tx->replaceFile(from_path, to_path); }); @@ -158,7 +158,7 @@ void DiskObjectStorageTransaction::replaceFile(const std::string & from_path, co void DiskObjectStorageTransaction::removeFile(const std::string & path) { - operations_to_execute.push_back([path](MetadataTransactionPtr tx) + dispatch([path](MetadataTransactionPtr tx) { tx->unlinkFile(path, /*if_exists=*/false, /*should_remove_objects=*/true); }); @@ -166,7 +166,7 @@ void DiskObjectStorageTransaction::removeFile(const std::string & path) void DiskObjectStorageTransaction::removeSharedFile(const std::string & path, bool keep_shared_data) { - operations_to_execute.push_back([path, keep_shared_data](MetadataTransactionPtr tx) + dispatch([path, keep_shared_data](MetadataTransactionPtr tx) { tx->unlinkFile(path, /*if_exists=*/false, /*should_remove_objects=*/!keep_shared_data); }); @@ -177,14 +177,14 @@ void DiskObjectStorageTransaction::removeSharedRecursive( { if (!keep_all_shared_data && file_names_remove_metadata_only.empty()) { - operations_to_execute.push_back([path](MetadataTransactionPtr tx) + dispatch([path](MetadataTransactionPtr tx) { tx->removeRecursive(path, /*should_remove_objects=*/nullptr); }); } else { - operations_to_execute.push_back([path, keep_all_shared_data, file_names_remove_metadata_only](MetadataTransactionPtr tx) + dispatch([path, keep_all_shared_data, file_names_remove_metadata_only](MetadataTransactionPtr tx) { tx->removeRecursive(path, /*should_remove_objects=*/[keep_all_shared_data, file_names_remove_metadata_only](const std::string & relative_path) { @@ -196,7 +196,7 @@ void DiskObjectStorageTransaction::removeSharedRecursive( void DiskObjectStorageTransaction::removeSharedFileIfExists(const std::string & path, bool keep_shared_data) { - operations_to_execute.push_back([path, keep_shared_data](MetadataTransactionPtr tx) + dispatch([path, keep_shared_data](MetadataTransactionPtr tx) { tx->unlinkFile(path, /*if_exists=*/true, /*should_remove_objects=*/!keep_shared_data); }); @@ -204,7 +204,7 @@ void DiskObjectStorageTransaction::removeSharedFileIfExists(const std::string & void DiskObjectStorageTransaction::removeDirectory(const std::string & path) { - operations_to_execute.push_back([path](MetadataTransactionPtr tx) + dispatch([path](MetadataTransactionPtr tx) { tx->removeDirectory(path); }); @@ -212,7 +212,7 @@ void DiskObjectStorageTransaction::removeDirectory(const std::string & path) void DiskObjectStorageTransaction::removeRecursive(const std::string & path) { - operations_to_execute.push_back([path](MetadataTransactionPtr tx) + dispatch([path](MetadataTransactionPtr tx) { tx->removeRecursive(path, /*should_remove_objects=*/nullptr); }); @@ -220,9 +220,9 @@ void DiskObjectStorageTransaction::removeRecursive(const std::string & path) void DiskObjectStorageTransaction::removeFileIfExists(const std::string & path) { - operations_to_execute.push_back([path](MetadataTransactionPtr tx) + dispatch([path](MetadataTransactionPtr tx) { - tx->unlinkFile(path, /*if_exists=*/true, /*should_remove_objects*/true); + tx->unlinkFile(path, /*if_exists=*/true, /*should_remove_objects=*/true); }); } @@ -231,7 +231,7 @@ void DiskObjectStorageTransaction::removeSharedFiles(const RemoveBatchRequest & for (const auto & [path, if_exists] : files) { const bool should_remove_objects = !keep_all_batch_data && !file_names_remove_metadata_only.contains(fs::path(path).filename()); - operations_to_execute.push_back([path, if_exists, should_remove_objects](MetadataTransactionPtr tx) + dispatch([path, if_exists, should_remove_objects](MetadataTransactionPtr tx) { tx->unlinkFile(path, if_exists, should_remove_objects); }); @@ -267,6 +267,16 @@ std::unique_ptr DiskObjectStorageTransaction::writeFile WriteSettings enriched_settings = updateIOSchedulingSettings(settings, read_resource_name, write_resource_name); + /// [TXN-ONE-PIPELINE] Give the metadata storage a chance to own the write (e.g. a content-addressed + /// hash-on-write buffer whose blob key is known only after the last byte). It returns a fully-wrapped + /// buffer (hash-on-write + append RMW + inline/blob split + autocommit/lifetime pin using `owner`), or + /// nullptr to fall through to the generic up-front-key streaming path below. Called BEFORE the append + /// check so a CA storage (which reports no native append) can service a verbatim append via + /// read-modify-rewrite inside the hook and reject a part-file append there. + if (auto buffer = metadata_transaction->tryCreateWriteBuffer( + shared_from_this(), path, buf_size, mode, enriched_settings, autocommit)) + return buffer; + /// NOTE: We check it here and not after writing blob because in case of plain/plain-rewritable metadata storages /// undo of disk tx will actually remove existing data. if (mode == WriteMode::Append && !metadata_storage->supportWritingWithAppend()) @@ -395,7 +405,10 @@ void DiskObjectStorageTransaction::writeFileUsingBlobWritingFunction( /// We always use mode Rewrite because we simulate append using metadata and different files object.bytes_size = std::move(write_blob_function)(blob_path, WriteMode::Rewrite, /*object_attributes=*/std::nullopt); - operations_to_execute.push_back([object, mode](MetadataTransactionPtr tx) + /// [TXN-ONE-PIPELINE] Routed through dispatch for uniformity. Unreachable on CA (Audit 6): + /// generateObjectKeyForPath above throws NOT_IMPLEMENTED first, so CA never reaches this metadata + /// effect and never queues. On ordinary storage dispatch queues exactly as before. + dispatch([object, mode](MetadataTransactionPtr tx) { if (mode == WriteMode::Rewrite) { @@ -413,15 +426,47 @@ void DiskObjectStorageTransaction::writeFileUsingBlobWritingFunction( void DiskObjectStorageTransaction::createHardLink(const std::string & src_path, const std::string & dst_path) { - operations_to_execute.push_back([src_path, dst_path](MetadataTransactionPtr tx) + /// For CA `dispatch` runs eagerly (call-time), which is load-bearing for read-your-writes: a + /// carried-forward projection hardlinked into the open whole-part transaction during a mutation must + /// be visible to `loadProjections` (same finalize, before commit) via the directory overlay. Deferring + /// it to commit replay would hide it until after `loadProjections` ran (B58/B63). The metadata-level + /// `createHardLink` is an idempotent map assignment, so eager staging is equivalent to the queued + /// replay — commit publishes the manifest from the staging. + dispatch([src_path, dst_path](MetadataTransactionPtr tx) { tx->createHardLink(src_path, dst_path); }); } +std::optional DiskObjectStorageTransaction::tryGetInFlightStorageObjects(const std::string & path) const +{ + return metadata_transaction->tryGetInFlightStorageObjects(path); +} + +std::unique_ptr DiskObjectStorageTransaction::tryReadFileInFlight( + const std::string & path, const ReadSettings & settings, std::optional read_hint) const +{ + return metadata_transaction->tryReadFileInFlight(path, settings, read_hint); +} + +std::optional DiskObjectStorageTransaction::tryGetInFlightFileSize(const std::string & path) const +{ + return metadata_transaction->tryGetInFlightFileSize(path); +} + +bool DiskObjectStorageTransaction::hasInFlightDirectory(const std::string & path) const +{ + return metadata_transaction->hasInFlightDirectory(path); +} + +std::vector DiskObjectStorageTransaction::listInFlightDirectory(const std::string & path) const +{ + return metadata_transaction->listInFlightDirectory(path); +} + void DiskObjectStorageTransaction::setReadOnly(const std::string & path) { - operations_to_execute.push_back([path](MetadataTransactionPtr tx) + dispatch([path](MetadataTransactionPtr tx) { tx->setReadOnly(path); }); @@ -429,7 +474,7 @@ void DiskObjectStorageTransaction::setReadOnly(const std::string & path) void DiskObjectStorageTransaction::setLastModified(const std::string & path, const Poco::Timestamp & timestamp) { - operations_to_execute.push_back([path, timestamp](MetadataTransactionPtr tx) + dispatch([path, timestamp](MetadataTransactionPtr tx) { tx->setLastModified(path, timestamp); }); @@ -437,7 +482,7 @@ void DiskObjectStorageTransaction::setLastModified(const std::string & path, con void DiskObjectStorageTransaction::chmod(const String & path, mode_t mode) { - operations_to_execute.push_back([path, mode](MetadataTransactionPtr tx) + dispatch([path, mode](MetadataTransactionPtr tx) { tx->chmod(path, mode); }); @@ -516,7 +561,11 @@ void DiskObjectStorageTransaction::copyFileImpl( return; } - operations_to_execute.push_back([blobs_to_create, missing_locations, to_file_path](MetadataTransactionPtr tx) + /// [TXN-ONE-PIPELINE] Routed through dispatch for uniformity. Unreachable on CA (Audit 6): + /// copyFileImpl calls generateObjectKeyForPath above, which throws NOT_IMPLEMENTED on CA before this + /// point (and the empty-source case returns via the real writeFile above), so CA never queues here. + /// On ordinary storage dispatch queues exactly as before. + dispatch([blobs_to_create, missing_locations, to_file_path](MetadataTransactionPtr tx) { for (const auto & blob : blobs_to_create) tx->recordBlobsReplication(blob, missing_locations); @@ -537,6 +586,15 @@ void MultipleDisksObjectStorageTransaction::copyFile(const std::string & from_fi void DiskObjectStorageTransaction::commit() { + /// [TXN-ONE-PIPELINE] An eager staging-overlay transaction (e.g. CA) must route every mutating method + /// straight to the metadata transaction at call time and keep this queue empty. A non-empty queue here + /// means a mutating method bypassed `dispatch` — which would re-introduce the two-timeline split this + /// design eliminates. Fail closed with a real throw (NOT chassert, which is a no-op in release builds). + if (metadata_storage->transactionIsStagingOverlay() && !operations_to_execute.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "An eager staging-overlay transaction must not queue deferred operations " + "(a mutating method bypassed dispatch): {} queued", operations_to_execute.size()); + auto component_guard = Coordination::setCurrentComponent("DiskObjectStorageTransaction::commit"); for (size_t i = 0; i < operations_to_execute.size(); ++i) { @@ -581,6 +639,13 @@ void DiskObjectStorageTransaction::commit() TransactionCommitOutcomeVariant DiskObjectStorageTransaction::tryCommit(const TransactionCommitOptionsVariant & options) { + /// [TXN-ONE-PIPELINE] See commit(): an eager staging-overlay transaction must never queue deferred + /// operations. Fail closed (real throw, not chassert) if a mutating method bypassed `dispatch`. + if (metadata_storage->transactionIsStagingOverlay() && !operations_to_execute.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "An eager staging-overlay transaction must not queue deferred operations " + "(a mutating method bypassed dispatch): {} queued", operations_to_execute.size()); + for (size_t i = 0; i < operations_to_execute.size(); ++i) { try diff --git a/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.h b/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.h index 2322263a0e06..14424d480a4b 100644 --- a/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.h +++ b/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.h @@ -104,6 +104,15 @@ struct DiskObjectStorageTransaction : public IDiskTransaction, public std::enabl void setReadOnly(const std::string & path) override; void createHardLink(const std::string & src_path, const std::string & dst_path) override; + /// B59 in-flight read-your-writes: forward to the metadata transaction (e.g. a CA part-build + /// transaction resolving its own staged-but-uncommitted files). + std::optional tryGetInFlightStorageObjects(const std::string & path) const override; + std::unique_ptr tryReadFileInFlight( + const std::string & path, const ReadSettings & settings, std::optional read_hint) const override; + std::optional tryGetInFlightFileSize(const std::string & path) const override; + bool hasInFlightDirectory(const std::string & path) const override; + std::vector listInFlightDirectory(const std::string & path) const override; + protected: /// Shared between `DiskObjectStorageTransaction::copyFile` and /// `MultipleDisksObjectStorageTransaction::copyFile`. Reads source blobs from the @@ -118,6 +127,17 @@ struct DiskObjectStorageTransaction : public IDiskTransaction, public std::enabl const ReadSettings & read_settings, const WriteSettings & write_settings); + /// [TXN-ONE-PIPELINE] Route one metadata effect either into the FIFO replay queue (ordinary object + /// storage) or straight to the metadata transaction at call time (eager staging overlay, e.g. CA). + template + void dispatch(Operation && operation) + { + if (metadata_storage->transactionIsStagingOverlay()) + operation(metadata_transaction); + else + operations_to_execute.emplace_back(std::forward(operation)); + } + private: std::unique_ptr writeFileImpl( /// NOLINT bool autocommit, diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h index 720de290a4b9..db6d5b615724 100644 --- a/src/Disks/IDiskTransaction.h +++ b/src/Disks/IDiskTransaction.h @@ -138,6 +138,23 @@ struct IDiskTransaction : private boost::noncopyable /// Truncate file to the target size. virtual void truncateFile(const std::string & src_path, size_t size) = 0; + + /// In-flight read-your-writes for a part being assembled by THIS transaction (B59). Forwarded to the + /// metadata transaction by object-storage disk transactions; default (e.g. local disk) is no in-flight + /// visibility, so a reader falls through to the committed path. + virtual std::optional tryGetInFlightStorageObjects(const std::string & /*path*/) const { return {}; } + virtual std::unique_ptr tryReadFileInFlight( + const std::string & /*path*/, const ReadSettings & /*settings*/, std::optional /*read_hint*/) const { return nullptr; } + virtual std::optional tryGetInFlightFileSize(const std::string & /*path*/) const { return {}; } + /// In-flight read-your-writes at DIRECTORY granularity: true iff this transaction has STAGED at least one + /// file under `path` for `path`'s part (mirrors the file trio above). Forwarded to the metadata + /// transaction by object-storage disk transactions; default (e.g. local disk) is no in-flight directory + /// visibility, so a reader falls through to the committed path. + virtual bool hasInFlightDirectory(const std::string & /*path*/) const { return false; } + /// In-flight read-your-writes directory ENUMERATION: the immediate-child names this transaction has + /// STAGED directly under `path` (one level, the directory prefix stripped). Forwarded to the metadata + /// transaction; default (e.g. local disk) is empty. + virtual std::vector listInFlightDirectory(const std::string & /*path*/) const { return {}; } }; using DiskTransactionPtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp index def7f6395860..5ab4565675a4 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +40,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int FILE_DOESNT_EXIST; extern const int CORRUPTED_DATA; + extern const int SUPPORT_IS_DISABLED; } std::unique_ptr IDataPartStorage::readFile( @@ -270,6 +273,16 @@ bool DataPartStorageOnDiskBase::isStoredOnRemoteDisk() const return volume->getDisk()->isRemote(); } +bool DataPartStorageOnDiskBase::isContentAddressed() const +{ + return volume->getDisk()->isContentAddressed(); +} + +bool DataPartStorageOnDiskBase::supportsAtomicFileWrites() const +{ + return volume->getDisk()->supportsAtomicFileWrites(); +} + std::optional DataPartStorageOnDiskBase::getCacheName() const { if (volume->getDisk()->supportsCache()) @@ -401,6 +414,18 @@ void DataPartStorageOnDiskBase::backup( auto disk = volume->getDisk(); + /// B34: the temporary-hard-link BACKUP path (used for Ordinary, non-UUID databases) calls + /// disk->createHardLink with a non-part-shaped temp path, which on a CAS disk + /// would otherwise surface as a raw LOGICAL_ERROR. Fail closed with a clear message instead. + /// The pointer-holding path (make_temporary_hard_links=false, used by Atomic/UUID databases) + /// uses getStorageObjects and round-trips on a CAS disk, so it is left untouched. + if (make_temporary_hard_links && disk->isContentAddressed()) + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "BACKUP via temporary hard links is not supported on a CAS disk yet (B16/B34); " + "use an Atomic database (which backs up via pointer-holding) instead; disk '{}'", + disk->getName()); + fs::path temp_part_dir; std::shared_ptr temp_dir_owner; if (make_temporary_hard_links) @@ -506,8 +531,20 @@ MutableDataPartStoragePtr DataPartStorageOnDiskBase::freeze( const ClonePartParams & params) const { auto disk = volume->getDisk(); - if (params.external_transaction) - params.external_transaction->createDirectories(to); + + /// A CAS disk models a part as one atomic unit (N files -> one manifest -> one ref). + /// The per-file createHardLink autocommit Backup uses with no enclosing transaction would publish a + /// one-file ref per file and overwrite the destination, leaving the clone with only its last file + /// (the B21 corruption mode — seen as system.detached_parts listing metadata_version.txt instead of + /// the detached part dir, B36). When the caller did not supply a transaction, run the whole clone + /// through ONE self-created disk transaction so all files land in a single content-addressed part. + DiskTransactionPtr owned_transaction; + if (!params.external_transaction && disk->isContentAddressed()) + owned_transaction = disk->createTransaction(); + const DiskTransactionPtr & clone_transaction = params.external_transaction ? params.external_transaction : owned_transaction; + + if (clone_transaction) + clone_transaction->createDirectories(to); else disk->createDirectories(to); @@ -522,17 +559,38 @@ MutableDataPartStoragePtr DataPartStorageOnDiskBase::freeze( /* max_level= */ {}, params.copy_instead_of_hardlink, params.files_to_copy_instead_of_hardlinks, - params.external_transaction); + clone_transaction); if (save_metadata_callback) save_metadata_callback(disk); - if (params.external_transaction) + if (clone_transaction) { - params.external_transaction->removeFileIfExists(fs::path(to) / dir_path / "delete-on-destroy.txt"); - params.external_transaction->removeFileIfExists(fs::path(to) / dir_path / VersionMetadata::TXN_VERSION_METADATA_FILE_NAME); + clone_transaction->removeFileIfExists(fs::path(to) / dir_path / "delete-on-destroy.txt"); + clone_transaction->removeFileIfExists(fs::path(to) / dir_path / VersionMetadata::TXN_VERSION_METADATA_FILE_NAME); if (!params.keep_metadata_version) - params.external_transaction->removeFileIfExists(fs::path(to) / dir_path / IMergeTreeDataPart::METADATA_VERSION_FILE_NAME); + clone_transaction->removeFileIfExists(fs::path(to) / dir_path / IMergeTreeDataPart::METADATA_VERSION_FILE_NAME); + + /// When the caller wants a fresh metadata version written into the clone (the Replicated queue + /// clone path — `executeReplaceRange`/`replacePartitionFrom`/`movePartitionToTable` set + /// `metadata_version_to_write`), write `metadata_version.txt` INSIDE the clone transaction so it + /// is part of the single whole-part commit. On a content-addressed disk the part is published + /// atomically at `commit`; a separate post-clone autocommit `writeFile` of this part file (what + /// `cloneAndLoadDataPart` does for non-CA disks) would hit the per-file-autocommit guard (B21). + /// `cloneAndLoadDataPart`'s own post-clone write now runs unconditionally (the freeze special + /// case was dropped with all-tree Task 10): identical bytes land as a byte-equal repoint + /// no-op, differing bytes as a legal repoint. + if (params.metadata_version_to_write.has_value()) + { + chassert(!params.keep_metadata_version); + auto out_metadata = clone_transaction->writeFile( + fs::path(to) / dir_path / IMergeTreeDataPart::METADATA_VERSION_FILE_NAME, + 4096, + WriteMode::Rewrite, + write_settings); + writeText(*params.metadata_version_to_write, *out_metadata); + out_metadata->finalize(); + } } else { @@ -542,6 +600,11 @@ MutableDataPartStoragePtr DataPartStorageOnDiskBase::freeze( disk->removeFileIfExists(fs::path(to) / dir_path / IMergeTreeDataPart::METADATA_VERSION_FILE_NAME); } + /// Commit the self-created transaction (the whole-part clone commit point for CA). An external + /// transaction is committed by its owner, as before. + if (owned_transaction) + owned_transaction->commit(); + auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); /// Do not initialize storage in case of DETACH because part may be broken. @@ -605,6 +668,51 @@ MutableDataPartStoragePtr DataPartStorageOnDiskBase::freezeRemote( return create(single_disk_volume, to, dir_path, /*initialize=*/ !to_detached && !params.external_transaction); } +namespace +{ + +/// Recursively copy every file under `source_path` on `src_disk` into `destination_path` through +/// `dst_transaction`'s NON-autocommit `writeFile` (IDiskTransaction::writeFile, NOT +/// writeFileWithAutoCommit) -- the same primitive `freeze` already uses for a single file (the +/// metadata_version.txt write, DataPartStorageOnDiskBase::freeze). Cross-disk, so it cannot reuse +/// Backup()/BackupImpl: that helper's transactional branch calls transaction->copyFile, which is +/// SAME-disk only (DiskObjectStorageTransaction::copyFile throws NOT_IMPLEMENTED across disks on +/// CA), and its non-transactional branch always autocommits per file via IDisk::copyFile / +/// copyDirectoryContent. Sequential, not the parallel copyThroughBuffers thread pool: a +/// content-addressed transaction batches every file into ONE eventual manifest, and its staging +/// map is not mutex-guarded; MOVE is a background, latency-insensitive operation, so +/// parallelizing this is a deferred optimization, not a correctness requirement. +void copyDirectoryContentIntoTransaction( + IDisk & src_disk, + const String & source_path, + IDiskTransaction & dst_transaction, + const String & destination_path, + const ReadSettings & read_settings, + const WriteSettings & write_settings, + const std::function & cancellation_hook) +{ + dst_transaction.createDirectories(destination_path); + for (auto it = src_disk.iterateDirectory(source_path); it->isValid(); it->next()) + { + auto source = it->path(); + auto destination = fs::path(destination_path) / it->name(); + + if (src_disk.existsDirectory(source)) + { + copyDirectoryContentIntoTransaction( + src_disk, source, dst_transaction, destination, read_settings, write_settings, cancellation_hook); + continue; + } + + auto in = src_disk.readFile(source, read_settings); + auto out = dst_transaction.writeFile(destination, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite, write_settings); + copyData(*in, *out, cancellation_hook); + out->finalize(); + } +} + +} + MutableDataPartStoragePtr DataPartStorageOnDiskBase::clonePart( const std::string & to, const std::string & dir_path, @@ -624,18 +732,46 @@ MutableDataPartStoragePtr DataPartStorageOnDiskBase::clonePart( dir_path, getRelativePath(), path_to_clone, fullPath(dst_disk, path_to_clone)); } - try + if (dst_disk->isContentAddressed()) { - dst_disk->createDirectories(to); - src_disk->copyDirectoryContent(getRelativePath(), dst_disk, path_to_clone, read_settings, write_settings, cancellation_hook); + /// L2 (MOVE-to-CA fix): a content-addressed disk models a part as ONE atomic unit (N + /// files -> one manifest -> one ref). The generic per-file autocommit path below would + /// publish a separate one-file ref per file -- colliding on the shared "moving" ref + /// before L1, and throwing NOT_IMPLEMENTED on a non-first content file even after L1 + /// ("Autocommit writes are not supported for content part files"). Run the whole clone + /// through ONE self-created disk transaction instead, mirroring freeze's + /// owned_transaction shape -- but streaming cross-disk bytes, since freeze's Backup() is + /// same-disk hardlink/copyFile (throws NOT_IMPLEMENTED for CA cross-disk). + auto clone_transaction = dst_disk->createTransaction(); + try + { + copyDirectoryContentIntoTransaction( + *src_disk, getRelativePath(), *clone_transaction, path_to_clone, + read_settings, write_settings, cancellation_hook); + clone_transaction->commit(); + } + catch (...) + { + LOG_WARNING(log, "Rolling back transaction after failed attempt to move a data part to {}", path_to_clone); + clone_transaction->undo(); + throw; + } } - catch (...) + else { - /// It's safe to remove it recursively (even with zero-copy-replication) - /// because we've just did full copy through copyDirectoryContent - LOG_WARNING(log, "Removing directory {} after failed attempt to move a data part", path_to_clone); - dst_disk->removeRecursive(path_to_clone); - throw; + try + { + dst_disk->createDirectories(to); + src_disk->copyDirectoryContent(getRelativePath(), dst_disk, path_to_clone, read_settings, write_settings, cancellation_hook); + } + catch (...) + { + /// It's safe to remove it recursively (even with zero-copy-replication) + /// because we've just did full copy through copyDirectoryContent + LOG_WARNING(log, "Removing directory {} after failed attempt to move a data part", path_to_clone); + dst_disk->removeRecursive(path_to_clone); + throw; + } } auto single_disk_volume = std::make_shared(dst_disk->getName(), dst_disk, 0); diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h index e1c55719954f..1bd66ff660a6 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h @@ -40,6 +40,8 @@ class DataPartStorageOnDiskBase : public IDataPartStorage std::string getDiskName() const override; std::string getDiskType() const override; bool isStoredOnRemoteDisk() const override; + bool isContentAddressed() const override; + bool supportsAtomicFileWrites() const override; std::optional getCacheName() const override; bool supportZeroCopyReplication() const override; bool supportParallelWrite() const override; diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp index 5503d11c58d9..fd25e35153c4 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -9,14 +10,24 @@ #include #include #include +#include #include +#include +#include + namespace DB { +namespace FailPoints +{ + extern const char part_storage_fail_commit_transaction[]; +} + namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int FAULT_INJECTED; } DataPartStorageOnDiskFull::DataPartStorageOnDiskFull(VolumePtr volume_, std::string root_path_, std::string part_dir_) @@ -48,22 +59,39 @@ DataPartStoragePtr DataPartStorageOnDiskFull::getProjection(const std::string & bool DataPartStorageOnDiskFull::exists() const { - return volume->getDisk()->existsDirectory(fs::path(root_path) / part_dir); + auto path = fs::path(root_path) / part_dir; + /// CA read-your-writes: a part dir being assembled by this transaction (e.g. a carried-forward + /// projection dir staged into the open whole-part txn) is not on committed metadata yet. Mirrors + /// existsDirectory at directory granularity for the part's OWN directory. + if (transaction && transaction->hasInFlightDirectory(path)) + return true; + return volume->getDisk()->existsDirectory(path); } bool DataPartStorageOnDiskFull::existsFile(const std::string & name) const { + auto path = fs::path(root_path) / part_dir / name; + /// B59: a part still being assembled by this transaction can have staged-but-uncommitted files + /// (e.g. projection temp blocks on a content-addressed disk). Consult the held transaction first. + if (transaction && transaction->tryGetInFlightFileSize(path).has_value()) + return true; if (looksLikePackedSkipIndexFile(name)) { if (auto reader = getSkipIndicesPackedReader(); reader && reader->exists(name)) return true; } - return volume->getDisk()->existsFile(fs::path(root_path) / part_dir / name); + return volume->getDisk()->existsFile(path); } bool DataPartStorageOnDiskFull::existsDirectory(const std::string & name) const { - return volume->getDisk()->existsDirectory(fs::path(root_path) / part_dir / name); + auto path = fs::path(root_path) / part_dir / name; + /// CA read-your-writes: a part still being assembled by this transaction can have a staged-but-uncommitted + /// directory (e.g. a carried-forward projection hardlinked into the open whole-part txn) that committed + /// metadata cannot see yet. Mirrors existsFile (B59) at directory granularity. + if (transaction && transaction->hasInFlightDirectory(path)) + return true; + return volume->getDisk()->existsDirectory(path); } class DataPartStorageIteratorOnDisk final : public IDataPartStorageIterator @@ -85,11 +113,52 @@ class DataPartStorageIteratorOnDisk final : public IDataPartStorageIterator DirectoryIteratorPtr it; }; +/// CA read-your-writes directory enumeration: a merged view of the committed disk entries PLUS the +/// immediate children this transaction has STAGED under the part dir (deduplicated). Used so +/// loadProjections' withPartFormatFromDisk can iterate a staged-but-uncommitted projection directory and +/// find its mark file. Mirrors existsFile/existsDirectory (B59) at the enumeration level; the committed +/// entries dominate (a name present both on disk and staged appears once). +class DataPartStorageMergedIterator final : public IDataPartStorageIterator +{ +public: + DataPartStorageMergedIterator(DiskPtr disk_, std::string dir_path_, std::vector names_) + : disk(std::move(disk_)), dir_path(std::move(dir_path_)), names(std::move(names_)) + { + } + + void next() override { ++pos; } + bool isValid() const override { return pos < names.size(); } + std::string name() const override { return names[pos]; } + std::string path() const override { return fs::path(dir_path) / names[pos]; } + bool isFile() const override { return isValid() && disk->existsFile(path()); } + +private: + DiskPtr disk; + std::string dir_path; + std::vector names; + size_t pos = 0; +}; + DataPartStorageIteratorPtr DataPartStorageOnDiskFull::iterate() const { + auto dir_path = fs::path(root_path) / part_dir; + if (transaction) + { + if (auto staged = transaction->listInFlightDirectory(dir_path); !staged.empty()) + { + /// Union the committed entries with the staged children (set semantics, committed dominates). + std::set names(staged.begin(), staged.end()); + if (volume->getDisk()->existsDirectory(dir_path)) + for (auto it = volume->getDisk()->iterateDirectory(dir_path); it->isValid(); it->next()) + names.insert(it->name()); + return std::make_unique( + volume->getDisk(), dir_path, std::vector(names.begin(), names.end())); + } + } + return std::make_unique( volume->getDisk(), - volume->getDisk()->iterateDirectory(fs::path(root_path) / part_dir)); + volume->getDisk()->iterateDirectory(dir_path)); } Poco::Timestamp DataPartStorageOnDiskFull::getFileLastModified(const String & file_name) const @@ -99,12 +168,17 @@ Poco::Timestamp DataPartStorageOnDiskFull::getFileLastModified(const String & fi size_t DataPartStorageOnDiskFull::getFileSize(const String & file_name) const { + auto path = fs::path(root_path) / part_dir / file_name; + /// B59: see existsFile — the merge stats the staged temp files before reading them back. + if (transaction) + if (auto size = transaction->tryGetInFlightFileSize(path)) + return *size; if (looksLikePackedSkipIndexFile(file_name)) { if (auto reader = getSkipIndicesPackedReader(); reader && reader->exists(file_name)) return reader->getFileSize(file_name); } - return volume->getDisk()->getFileSize(fs::path(root_path) / part_dir / file_name); + return volume->getDisk()->getFileSize(path); } UInt32 DataPartStorageOnDiskFull::getRefCount(const String & file_name) const @@ -115,7 +189,17 @@ UInt32 DataPartStorageOnDiskFull::getRefCount(const String & file_name) const std::vector DataPartStorageOnDiskFull::getRemotePaths(const std::string & file_name) const { const std::string path = fs::path(root_path) / part_dir / file_name; - auto objects = volume->getDisk()->getStorageObjects(path); + + /// B59: a file staged by this transaction resolves to its already-uploaded blob object(s) before commit. + /// A mutable per-part file intentionally does NOT resolve here (tryGetInFlightStorageObjects returns + /// nullopt → falls through): it has no blob object and must be read via tryReadFileInFlight. The merge + /// reads projection column blocks (blob-backed) through this path, not mutable files. + StoredObjects objects; + if (transaction) + if (auto inflight = transaction->tryGetInFlightStorageObjects(path)) + objects = std::move(*inflight); + if (objects.empty()) + objects = volume->getDisk()->getStorageObjects(path); std::vector remote_paths; remote_paths.reserve(objects.size()); @@ -141,6 +225,42 @@ void DataPartStorageOnDiskFull::prepareRead( std::optional read_hint, ReadPipeline & pipeline) const { + auto path = fs::path(root_path) / part_dir / name; + + /// B59: read-your-writes for a part still being assembled by this transaction. A projection + /// spill-and-merge reads its own temp blocks back before the parent part's single commit; on a + /// content-addressed disk those files are staged in the transaction (blob uploaded, no ref yet), + /// so the committed metadata path can't see them. If the held transaction resolves the file + /// in-flight, serve it via a custom pipeline source that reads through the transaction. Gated on + /// `transaction != nullptr` so committed-part reads (no open transaction) are unchanged. + if (transaction) + { + StoredObjects inflight_objects; + if (auto objs = transaction->tryGetInFlightStorageObjects(path)) + inflight_objects = std::move(*objs); + else if (auto size = transaction->tryGetInFlightFileSize(path)) + /// Mutable per-part file staged inline (no blob object); synthesize a placeholder so the + /// single-object pipeline is satisfied — the custom creator below ignores it and reads the + /// inline bytes through the transaction. + inflight_objects = StoredObjects{StoredObject(path, path, *size)}; + + if (!inflight_objects.empty()) + { + /// Safe to capture the raw transaction pointer: no cache/gather/async stage is added on this + /// branch, so the custom source is consumed synchronously inside build() during this read and + /// the pointer is never retained past it. + auto * tx = transaction.get(); + pipeline.setSource( + [tx, path](const StoredObject &, const ReadSettings & read_settings, bool /*use_external_buffer*/, bool /*restrict_seek*/) + { + return tx->tryReadFileInFlight(path, read_settings, std::nullopt); + }, + std::move(inflight_objects), + settings); + return; + } + } + if (looksLikePackedSkipIndexFile(name)) { if (auto reader = getSkipIndicesPackedReader(); reader && reader->exists(name)) @@ -161,7 +281,8 @@ void DataPartStorageOnDiskFull::prepareRead( return; } } - volume->getDisk()->prepareRead(fs::path(root_path) / part_dir / name, settings, read_hint, pipeline); + + volume->getDisk()->prepareRead(path, settings, read_hint, pipeline); } std::unique_ptr DataPartStorageOnDiskFull::readFileIfExists( @@ -169,6 +290,13 @@ std::unique_ptr DataPartStorageOnDiskFull::readFileIfExi const ReadSettings & settings, std::optional read_hint) const { + auto path = fs::path(root_path) / part_dir / name; + /// B59: serve a file staged by this transaction (uploaded blob or inline mutable bytes) before commit. + /// This direct delegate bypasses prepareRead, so the in-flight guard must be repeated here; it is the + /// only path that reaches the inline-mutable case via a returned buffer. + if (transaction) + if (auto rb = transaction->tryReadFileInFlight(path, settings, read_hint)) + return rb; if (looksLikePackedSkipIndexFile(name)) { if (auto reader = getSkipIndicesPackedReader(); reader && reader->exists(name)) @@ -177,7 +305,7 @@ std::unique_ptr DataPartStorageOnDiskFull::readFileIfExi fs::path(root_path) / part_dir / String(SKIP_INDICES_PACKED_FILENAME), name, settings, read_hint); } - return volume->getDisk()->readFileIfExists(fs::path(root_path) / part_dir / name, settings, read_hint); + return volume->getDisk()->readFileIfExists(path, settings, read_hint); } std::unique_ptr DataPartStorageOnDiskFull::writeFile( @@ -266,20 +394,38 @@ void DataPartStorageOnDiskFull::createProjection(const std::string & name) void DataPartStorageOnDiskFull::beginTransaction() { + /// A borrowed projection sub-part shares the PARENT part's whole-part transaction (on a + /// content-addressed disk a part is one atomic unit: one manifest + one ref). It must not open its + /// own — riding the parent transaction is the point (B58) — so begin is a no-op here. This + /// centralizes the rule the 6 merge/mutate call sites used to duplicate as + /// `if (!isContentAddressed()) beginTransaction()`. + if (has_shared_transaction) + return; + if (transaction) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Uncommitted{}transaction already exists", has_shared_transaction ? " shared " : " "); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Uncommitted transaction already exists"); transaction = volume->getDisk()->createTransaction(); } void DataPartStorageOnDiskFull::commitTransaction() { + /// The mirror of beginTransaction: a borrowed projection sub-part rides the parent's transaction and + /// is published by the parent's single commit. Committing here would be committing someone else's + /// transaction, so it is a no-op. + if (has_shared_transaction) + return; + if (!transaction) throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no uncommitted transaction"); - if (has_shared_transaction) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot commit shared transaction"); + /// Regression gate for the part-durability-before-Keeper-commit invariant: lets a test fail the + /// close of the PART's deferred disk transaction specifically (autocommit one-shot disk ops are + /// not affected, unlike disk_object_storage_fail_commit_metadata_transaction). + fiu_do_on(FailPoints::part_storage_fail_commit_transaction, + { + throw Exception(ErrorCodes::FAULT_INJECTED, "part_storage_fail_commit_transaction"); + }); transaction->commit(); transaction.reset(); diff --git a/src/Storages/MergeTree/IDataPartStorage.h b/src/Storages/MergeTree/IDataPartStorage.h index 37f31ade32b9..37cc719215a8 100644 --- a/src/Storages/MergeTree/IDataPartStorage.h +++ b/src/Storages/MergeTree/IDataPartStorage.h @@ -189,6 +189,15 @@ class IDataPartStorage : public boost::noncopyable virtual std::string getDiskName() const = 0; virtual std::string getDiskType() const = 0; virtual bool isStoredOnRemoteDisk() const { return false; } + /// True when the underlying disk stores a part as one atomic content-addressed unit (one manifest + /// + one ref). On such disks a projection sub-part must be written through the PARENT part's + /// whole-part transaction rather than its own sub-transaction (otherwise the projection is lost + /// from the committed manifest — B58). + virtual bool isContentAddressed() const { return false; } + /// True when the underlying disk publishes a file write atomically in one shot (no partial + /// content ever becomes visible under the file's final name). Such disks do not need the + /// tmp-file + `replaceFile` crash-safety dance that plain local writes require. + virtual bool supportsAtomicFileWrites() const { return false; } virtual std::optional getCacheName() const { return std::nullopt; } virtual bool supportZeroCopyReplication() const { return false; } virtual bool supportParallelWrite() const = 0; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 5a29ab7a5f91..465f64c91a80 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1356,7 +1356,13 @@ MergeTreeDataPartBuilder IMergeTreeDataPart::getProjectionPartBuilder( const String & projection_name, ProjectionDescriptionRawPtr projection, bool is_temp_projection) { const char * projection_extension = is_temp_projection ? ".tmp_proj" : ".proj"; - auto projection_storage = getDataPartStorage().getProjection(projection_name + projection_extension, !is_temp_projection); + /// On a content-addressed disk a part is one atomic unit, so a temp projection sub-part (written + /// during a merge/mutate rebuild under `.tmp_proj`) must share the PARENT part's whole-part + /// transaction — its files are re-keyed into the parent manifest when `.tmp_proj` is renamed to + /// `.proj` (B58). On a non-CA disk a temp projection keeps its own sub-transaction (the + /// historical behavior): `use_parent_transaction = !is_temp_projection`. + const bool use_parent_transaction = !is_temp_projection || getDataPartStorage().isContentAddressed(); + auto projection_storage = getDataPartStorage().getProjection(projection_name + projection_extension, use_parent_transaction); MergeTreeDataPartBuilder builder(storage, projection_name, projection_storage, getReadSettings()); return builder.withPartInfo(MergeListElement::FAKE_RESULT_PART_FOR_PROJECTION).withParentPart(this).withProjection(projection); } diff --git a/src/Storages/MergeTree/MergeProjectionPartsTask.cpp b/src/Storages/MergeTree/MergeProjectionPartsTask.cpp index 8956d382ef6b..cb8b7171ab7f 100644 --- a/src/Storages/MergeTree/MergeProjectionPartsTask.cpp +++ b/src/Storages/MergeTree/MergeProjectionPartsTask.cpp @@ -129,6 +129,10 @@ bool MergeProjectionPartsTask::executeStep() /// FIXME (alesapin) we should use some temporary storage for this, /// not commit each subprojection part + /// + /// A borrowed (CA) recursively-merged projection sub-part shares the parent part's whole-part + /// transaction (the nested MergeTask skipped its own begin), so it is committed by the parent's + /// single commit; the storage makes commitTransaction a no-op there, so this is unconditional (B58). next_level_parts.back()->getDataPartStorage().commitTransaction(); next_level_parts.back()->is_temp = true; next_level_parts.back()->temp_projection_block_number = block_num; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 5e7d3c71435b..7a631e06baaf 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -559,7 +559,14 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const std::optional builder; if (global_ctx->parent_part) { - auto data_part_storage = global_ctx->parent_part->getDataPartStorage().getProjection(local_tmp_part_basename, /* use parent transaction */ false); + /// On a content-addressed disk a part is one atomic unit (one manifest + one ref). The projection + /// sub-part must therefore be written through the PARENT part's whole-part transaction (mirroring + /// the INSERT path, `MergeTreeDataWriter::writeProjectionPartImpl` with `use_parent_transaction = + /// true`) so its files land in the parent manifest and survive a reload (B58). On a non-CA disk we + /// keep the historical behavior: the projection sub-part opens and commits its own sub-transaction. + global_ctx->projection_uses_parent_transaction = global_ctx->parent_part->getDataPartStorage().isContentAddressed(); + auto data_part_storage = global_ctx->parent_part->getDataPartStorage().getProjection( + local_tmp_part_basename, /* use_parent_transaction */ global_ctx->projection_uses_parent_transaction); builder.emplace(*global_ctx->data, global_ctx->future_part->name, data_part_storage, getReadSettings()); builder->withParentPart(global_ctx->parent_part); } @@ -579,6 +586,8 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const if (data_part_storage->exists()) throw Exception(ErrorCodes::DIRECTORY_ALREADY_EXISTS, "Directory {} already exists", data_part_storage->getFullPath()); + /// A borrowed projection sub-part shares the parent's already-open transaction; the storage makes + /// beginTransaction a no-op in that case, so this can be called unconditionally. data_part_storage->beginTransaction(); /// Background temp dirs cleaner will not touch tmp projection directory because @@ -1314,6 +1323,9 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::calculateProjectionForBlock( *global_ctx->data, ctx->log, result, projection, global_ctx->new_data_part.get(), ++ctx->projection_block_num, global_ctx->context); tmp_part->finalize(); + /// A borrowed (CA) temp projection sub-part rides the parent's whole-part transaction and is + /// committed by the parent's single commit; the storage makes commitTransaction a no-op there, + /// so this can be called unconditionally (B58). tmp_part->part->getDataPartStorage().commitTransaction(); ctx->projection_parts[projection.name].emplace_back(std::move(tmp_part->part)); } @@ -1356,6 +1368,8 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::finalizeProjections() const *global_ctx->data, ctx->log, result, projection, global_ctx->new_data_part.get(), ++ctx->projection_block_num, global_ctx->context); temp_part->finalize(); + /// See the matching note above: a borrowed (CA) temp projection sub-part rides the parent + /// transaction, so commitTransaction is a no-op and can be called unconditionally. temp_part->part->getDataPartStorage().commitTransaction(); ctx->projection_parts[projection.name].emplace_back(std::move(temp_part->part)); } diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index 21d027b5db20..a982d9ba58d6 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -219,6 +219,10 @@ class MergeTask ProjectionDescriptionRawPtr projection{nullptr}; /// This will be either nullptr or new_data_part, so raw pointer is ok. IMergeTreeDataPart * parent_part{nullptr}; + /// True only when this MergeTask builds a projection sub-part (`parent_part != nullptr`) whose + /// parent lives on a content-addressed disk: the sub-part then shares the parent's whole-part + /// transaction and must NOT begin/commit its own (B58). False for non-CA disks and top-level parts. + bool projection_uses_parent_transaction{false}; MergedPartOffsetsPtr merged_part_offsets; ContextPtr context{nullptr}; time_t time_of_merge{0}; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0d46d17862db..0ffa9443770d 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -33,9 +33,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -5915,6 +5917,27 @@ MergeTreeData::PartsToRemoveFromZooKeeper MergeTreeData::removePartsInRangeFromW MergeTreeData::Transaction transaction(*this, NO_TRANSACTION_RAW); renameTempPartAndAdd(new_data_part, transaction, lock, /*rename_in_transaction=*/ false); /// All covered parts must be already removed + /// On a content-addressed disk a part directory becomes durable only when its disk-storage + /// transaction is committed (the ref to its manifest is published at commit, not at rename). + /// The flow below rolls back the in-memory MergeTreeData transaction (to keep the empty part + /// Outdated, not Active), which never calls commitTransaction on the disk storage — so on a CA + /// disk the empty covering part would leave NO on-disk ref and vanish on restart/reattach, + /// defeating its sole purpose (it exists only to cover the dropped parts on disk so a restart + /// does not treat them as uncovered unexpected parts and trip TOO_MANY_UNEXPECTED_DATA_PARTS). + /// On a plain disk the rename in renameTempPartAndAdd is already durable, so this is a no-op + /// there. Commit the disk storage transaction here (CA only) so the ref is published before the + /// in-memory rollback; the part still ends up Outdated, exactly as on a plain disk. + /// + /// [TXN-ONE-PIPELINE] (`2026-07-16-cas-txn-one-pipeline-design.md`, Audit 7 / Tension 2): this + /// hand-placed `commitTransaction()` is NOT made redundant by moving publication into `commit` + /// — it is the direct consequence of that design. There is no `precommit` phase under the + /// one-pipeline model, and this rollback path (by construction, to keep the part Outdated) never + /// reaches `MergeTreeData::Transaction::commit`, the only other place a disk transaction is + /// committed. So this call remains the ONLY thing that publishes the empty cover's ref. Keep it. + if (new_data_part->getDataPartStorage().isContentAddressed() + && new_data_part->getDataPartStorage().hasActiveTransaction()) + new_data_part->getDataPartStorage().commitTransaction(); + /// It will add the empty part to the set of Outdated parts without making it Active (exactly what we need) transaction.rollback(&lock); new_data_part->remove_time.store(0, std::memory_order_relaxed); @@ -6725,6 +6748,54 @@ void MergeTreeData::checkAlterPartitionIsPossible( can_execute_alter_on_disk = std::ranges::contains(supported_commands, command.type); break; } + case MetadataStorageType::CAS: + { + /// On a CAS disk a part clone is cheap: identical content has the same + /// `part_id`, so cloning is publishing a ref (no byte copy). The clone path is now + /// transactional — `DataPartStorageOnDiskBase::freeze` runs the whole clone through ONE + /// CA transaction, and `moveDirectory` re-keys the detached-staging → active rename into + /// a complete active ref — so these are SUPPORTED and verified (read back identical data, + /// survive restart): `ATTACH PARTITION`/`ATTACH PART` (re-clone of the table's own + /// detached parts), `REPLACE PARTITION`/`ATTACH PARTITION ... FROM` (parses to + /// `REPLACE_PARTITION`), and `MOVE PARTITION ... TO TABLE`. The pointer-unlink commands + /// `DROP PARTITION` / `DETACH PARTITION` / `DROP DETACHED PARTITION` are also fine. + /// `FETCH PARTITION`/`FETCH PART` is also SUPPORTED — it is a `ReplicatedMergeTree` op + /// (now supported on CA), and a `to_detached` fetch takes the byte-fetch path: the + /// downloaded files content-address into the `detached/` namespace (relink-into-detached + /// is deferred, see backlog). `ALTER ... FETCH PART` parses to the same `FETCH_PARTITION` + /// command type (with `part=true`), so this entry covers both. + /// `FREEZE PARTITION`/`FREEZE ALL` and `UNFREEZE PARTITION`/`UNFREEZE ALL` are now SUPPORTED: + /// a freeze publishes each part as its own ref in the `shadow/` namespace (a GC root sharing + /// the live blobs zero-copy — no byte copy); UNFREEZE removes the backup's refs. + /// `FORGET PARTITION` is SUPPORTED on CA — it only manipulates ZooKeeper partition metadata + /// (removes block-number nodes from ZooKeeper) and does not write, clone, or touch any part + /// files on disk, so it is safe on a content-addressed disk. + /// NOTE: `MOVE_PARTITION` also admits cross-disk + /// `MOVE ... TO DISK/VOLUME` (this check cannot distinguish the destination); that uses + /// the byte-copy `clonePart` path (NOT the corrupting per-file hardlink), but only + /// same-disk `MOVE ... TO TABLE` is verified here — cross-disk is a follow-up to verify. + const static auto supported_commands = { + PartitionCommand::DROP_PARTITION, + PartitionCommand::DROP_DETACHED_PARTITION, + PartitionCommand::FORGET_PARTITION, + PartitionCommand::ATTACH_PARTITION, + PartitionCommand::REPLACE_PARTITION, + PartitionCommand::MOVE_PARTITION, + PartitionCommand::FETCH_PARTITION, + PartitionCommand::FREEZE_PARTITION, + PartitionCommand::FREEZE_ALL_PARTITIONS, + PartitionCommand::UNFREEZE_PARTITION, + PartitionCommand::UNFREEZE_ALL_PARTITIONS, + }; + + if (!std::ranges::contains(supported_commands, command.type)) + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "Partition operation ALTER TABLE {} is not supported on a CAS disk yet " + "(it clones parts file-by-file with no transaction, which would corrupt the clone); disk '{}'", + command.typeToString(), disk->getName()); + break; + } case MetadataStorageType::StaticWeb: { can_execute_alter_on_disk = false; @@ -7465,6 +7536,14 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr r /// Copy files from the backup to the directory `tmp_part_dir`. disk->createDirectories(temp_part_dir); + /// A content-addressed disk publishes a part as ONE manifest (N files -> one ref) atomically, so the + /// per-file copyFileToDisk autocommit below is rejected for content part files. Route the restore + /// through one whole-part transaction (mirrors DataPartStorageOnDiskBase::freeze's owned_transaction): + /// all files land in a single content-addressed part at tmp_restore_, published by tx->commit(). + DiskTransactionPtr restore_tx; + if (disk->isContentAddressed()) + restore_tx = disk->createTransaction(); + for (const String & filename : filenames) { /// Needs to create subdirectories before copying the files. Subdirectories are used to represent projections. @@ -7485,10 +7564,24 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr r continue; } - size_t file_size = backup->copyFileToDisk(part_path_in_backup_fs / filename, disk, temp_part_dir / filename, WriteMode::Rewrite); - reservation->update(reservation->getSize() - file_size); + if (restore_tx) + { + auto in = backup->readFile(part_path_in_backup_fs / filename); + auto out = restore_tx->writeFile(temp_part_dir / filename, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite, getContext()->getWriteSettings()); + copyData(*in, *out); + out->finalize(); + reservation->update(reservation->getSize() - backup->getFileSize(part_path_in_backup_fs / filename)); + } + else + { + size_t file_size = backup->copyFileToDisk(part_path_in_backup_fs / filename, disk, temp_part_dir / filename, WriteMode::Rewrite); + reservation->update(reservation->getSize() - file_size); + } } + if (restore_tx) + restore_tx->commit(); + if (auto part = loadPartRestoredFromBackup(part_name, disk, temp_part_dir, detach_if_broken)) restored_parts_holder->addPart(part); else @@ -8901,12 +8994,31 @@ void MergeTreeData::Transaction::clear() void MergeTreeData::Transaction::renameParts() { + /// Materialize every part of this transaction: perform the deferred tmp->final renames, then + /// close each part's disk-storage transaction, making the parts DURABLE on their disks. + /// + /// Contract: after renameParts returns, every part of this transaction is durable at its + /// final name. commit only flips in-memory visibility (its commitTransaction loop remains as + /// a safety net for paths that do not come through here); rollback compensates with new + /// operations over committed disk state (removing a rolled-back part reclaims its disk data; + /// on a content-addressed disk that drops the published ref). + /// + /// Ordering is load-bearing: every call site invokes renameParts BEFORE its external Keeper + /// commit decision. A part must be durable before its block_id/part-znode is registered, + /// otherwise a fault between the Keeper commit and the disk commit leaves a phantom part whose + /// surviving block_id silently dedups a byte-identical client retry (acked data loss). This + /// also keeps the disk commit (network I/O on object storages) off the data_parts lock, which + /// Transaction::commit holds. for (const auto & part_need_rename : precommitted_parts_need_rename) { LOG_TEST(data.log, "Renaming part to {}", part_need_rename->name); part_need_rename->renameTo(part_need_rename->name, true); } precommitted_parts_need_rename.clear(); + + for (const auto & part : precommitted_parts) + if (part->getDataPartStorage().hasActiveTransaction()) + part->getDataPartStorage().commitTransaction(); } MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit() diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 826f4eead8f5..349341d8a81f 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -366,9 +366,17 @@ class MergeTreeData : public WithMutableContext, public IStorage, public IBackgr DataPartsVector commit(); DataPartsVector commit(DataPartsLock & lock); - /// Rename should be done explicitly, before calling commit(), to - /// guarantee that no lock held during rename (since rename is IO - /// bound, while data parts lock is the bottleneck) + /// Renames should be done explicitly, before calling commit, to + /// guarantee that no lock is held during the rename and the disk + /// commit (both are IO bound, while the data parts lock is the + /// bottleneck). Contract: after renameParts every part of this + /// transaction is durable on its disk at its final name; commit only + /// flips in-memory visibility, and rollback compensates via new disk + /// operations (part removal). Every caller runs this BEFORE its + /// external Keeper commit decision: a part must be durable before its + /// block_id/part-znode is registered in Keeper, otherwise a fault between the two commits + /// leaves a phantom part whose surviving block_id silently dedups a byte-identical client + /// retry (acked data loss). void renameParts(); void addPart(MutableDataPartPtr & part, bool need_rename); diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 8e64543510c3..aaf7ddc527a9 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -1053,6 +1053,8 @@ MergeTreeTemporaryPartPtr MergeTreeDataWriter::writeProjectionPartImpl( auto projection_part_storage = new_data_part->getDataPartStoragePtr(); auto data_settings = data.getSettings(&projection.settings_changes); + /// A temp projection sub-part opens a transaction only if it owns one; a borrowed (CA) projection + /// storage makes beginTransaction a no-op, so the `isContentAddressed()` branch is no longer needed. if (is_temp) projection_part_storage->beginTransaction(); diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 918ead21658d..0eb164c8e7a7 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -1871,6 +1871,9 @@ void PartMergerWriter::writeTempProjectionPart(size_t projection_idx, Chunk chun ctx->context); tmp_part->finalize(); + /// A borrowed (CA) temp projection sub-part shares the new (parent) part's whole-part transaction + /// (see `IMergeTreeDataPart::getProjectionPartBuilder`) and is committed by the parent's single + /// commit; the storage makes commitTransaction a no-op there, so this is called unconditionally (B58). tmp_part->part->getDataPartStorage().commitTransaction(); projection_parts[projection.name].emplace_back(std::move(tmp_part->part)); } diff --git a/src/Storages/MergeTree/tests/gtest_projection_borrowed_transaction.cpp b/src/Storages/MergeTree/tests/gtest_projection_borrowed_transaction.cpp new file mode 100644 index 000000000000..ae079a61450f --- /dev/null +++ b/src/Storages/MergeTree/tests/gtest_projection_borrowed_transaction.cpp @@ -0,0 +1,86 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + /// A DiskLocal-backed parent part storage. `DiskLocal::createTransaction` yields a real + /// transaction object, which is all `beginTransaction` needs to hand a NON-NULL transaction to a + /// borrowed projection sub-part (the `has_shared_transaction == true` case). + struct ParentStorageFixture + { + std::filesystem::path base_path; + DiskPtr disk; + VolumePtr volume; + MutableDataPartStoragePtr parent; + + ParentStorageFixture() + { + const auto unique = std::to_string(::getpid()) + "_" + + std::to_string(reinterpret_cast(this)); + base_path = std::filesystem::temp_directory_path() / ("proj_txn_gtest_" + unique); + std::filesystem::create_directories(base_path / "all_1_1_0"); + disk = std::make_shared("test_disk_" + unique, base_path.string()); + volume = std::make_shared("test_volume", disk); + parent = std::make_shared(volume, /*root_path=*/"", "all_1_1_0"); + } + + ~ParentStorageFixture() + { + std::error_code ec; + std::filesystem::remove_all(base_path, ec); + } + }; +} + +/// A projection sub-part that BORROWS the parent's whole-part transaction (the CA-disk shape: +/// getProjection(..., use_parent_transaction = true)) must let begin/commit be NO-OPS — it rides the +/// parent's single commit. Before the encapsulation this threw "Uncommitted shared transaction already +/// exists" / "Cannot commit shared transaction", forcing every caller to branch on isContentAddressed(). +TEST(ProjectionBorrowedTransaction, BorrowedStorageBeginCommitAreNoOps) +{ + ParentStorageFixture fx; + + /// Parent opens the whole-part transaction (as MergeTask/writer do for a CA part). + fx.parent->beginTransaction(); + ASSERT_TRUE(fx.parent->hasActiveTransaction()); + + /// Borrowed projection sub-part: shares the parent transaction (has_shared_transaction == true). + auto proj = fx.parent->getProjection("p.proj", /*use_parent_transaction=*/true); + EXPECT_TRUE(proj->hasActiveTransaction()); + + /// The encapsulated rule: begin/commit on the borrowed storage are silent no-ops (they must NOT + /// open a second transaction, nor commit the parent's). + EXPECT_NO_THROW(proj->beginTransaction()); + EXPECT_NO_THROW(proj->commitTransaction()); + + /// The parent's transaction is untouched by the projection's no-ops and still commits cleanly. + EXPECT_TRUE(fx.parent->hasActiveTransaction()); + EXPECT_NO_THROW(fx.parent->commitTransaction()); + EXPECT_FALSE(fx.parent->hasActiveTransaction()); +} + +/// The non-CA temp-projection shape (use_parent_transaction = false) is unchanged: the sub-part OWNS +/// its transaction, so begin creates it and commit commits it (has_shared_transaction == false, so the +/// no-op path never triggers). +TEST(ProjectionBorrowedTransaction, OwnedProjectionStorageStillBeginsAndCommits) +{ + ParentStorageFixture fx; + + auto proj = fx.parent->getProjection("q.proj", /*use_parent_transaction=*/false); + EXPECT_FALSE(proj->hasActiveTransaction()); + + EXPECT_NO_THROW(proj->beginTransaction()); + EXPECT_TRUE(proj->hasActiveTransaction()); + EXPECT_NO_THROW(proj->commitTransaction()); + EXPECT_FALSE(proj->hasActiveTransaction()); +} From 3d35dfce282feea62675b53a4f18dfa9507233f6 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:33 +0200 Subject: [PATCH 10/30] Unwrap table proxies in single-table SYSTEM commands Single-table SYSTEM commands (SYNC/RESTORE/RESTART/DROP REPLICA, WAIT LOADING PARTS, PREWARM, ...) cast the storage directly and missed tables behind a proxy; unwrap the proxy before the cast. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- src/Storages/StorageProxy.h | 9 +++++++++ src/Storages/StorageTableProxy.h | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index dc1c0570e9b6..fe5b91e3bfba 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -143,6 +143,15 @@ class StorageProxy : public IStorage void mutate(const MutationCommands & commands, ContextPtr context) override { getNested()->mutate(commands, context); } + /// Must forward alongside `mutate`: `IStorage`'s default throws NOT_IMPLEMENTED ("doesn't + /// support mutations"), so a non-forwarding proxy rejects every mutation on a wrapped table + /// even though the nested engine supports them (found via `ALTER TABLE ... MATERIALIZE TTL` + /// on a `lazy_load_tables = 1` table wrapped in `StorageTableProxy`). + void checkMutationIsPossible(const MutationCommands & commands, const Settings & settings) const override + { + getNested()->checkMutationIsPossible(commands, settings); + } + CancellationCode killMutation(const String & mutation_id) override { return getNested()->killMutation(mutation_id); } void startup() override { getNested()->startup(); } diff --git a/src/Storages/StorageTableProxy.h b/src/Storages/StorageTableProxy.h index fa998bab42ac..e935de70e221 100644 --- a/src/Storages/StorageTableProxy.h +++ b/src/Storages/StorageTableProxy.h @@ -54,6 +54,14 @@ class StorageTableProxy final : public StorageProxy StoragePolicyPtr getStoragePolicy() const override { return nullptr; } bool isView() const override { return false; } + /// NOTE: this proxy deliberately does NOT forward `checkTableCanBeRenamed` to the nested engine. + /// Doing so would materialize the lazy table (`getNested`) while `DatabaseAtomic` holds its + /// non-recursive database mutex, and a schema-inferred lazy `Buffer` resolves its destination via + /// `DatabaseCatalog::getTable` in its constructor -- re-entering the same database and self- + /// deadlocking. Bypassing the nested engine's rename restriction for a lazy (never-accessed) table + /// is a pre-existing gap tracked in docs/superpowers/cas/BACKLOG.md; the correct fix is to + /// materialize before the database mutex is taken, at the interpreter level. + /// /// Startup is deferred until first access via `getNested`. void startup() override { } From f85cb4330c88d7619228d2784f1658ea28240254 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:33 +0200 Subject: [PATCH 11/30] clickhouse-disks: non-interactive runs exit nonzero on a failed command clickhouse-disks --query always exited 0 and reported failures only on stderr, so scripts, cron jobs and CI could not gate on it at all. Record each command's error code in processQueryText and return it as the process exit code for non-interactive runs; interactive REPL sessions keep exiting 0. Within one semicolon-separated batch a later success does not clear an earlier failure. Carries the two test fixes the contract exposed: a 2024 typo in test_disks_app_func ("d/a" instead of "a/d/a" always failed inside the tool and was swallowed), and test_replicated_table_structure_alter reading metadata_path from system.tables after DETACH DATABASE (empty path, the remove never ran, the recovery scenario was never exercised). Mixed-file note: DisksApp.cpp also carries the cas-* command registration and CA pool initialization, wired by the later CAS integration commits. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- programs/disks/DisksApp.cpp | 35 +++++++++++++++++++ programs/disks/DisksApp.h | 4 +++ tests/integration/test_disks_app_func/test.py | 10 +++--- .../test_replicated_database/test.py | 11 ++++-- 4 files changed, 52 insertions(+), 8 deletions(-) diff --git a/programs/disks/DisksApp.cpp b/programs/disks/DisksApp.cpp index dbde5cb4672c..871316672a43 100644 --- a/programs/disks/DisksApp.cpp +++ b/programs/disks/DisksApp.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include #include "config.h" @@ -34,6 +35,7 @@ #include #include #include +#include #include @@ -44,6 +46,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int STD_EXCEPTION; }; LineReader::Patterns DisksApp::query_extenders = {"\\"}; @@ -212,6 +215,8 @@ bool DisksApp::processQueryText(const String & text) return false; CommandPtr command; + last_command_exit_code = 0; + auto subqueries = splitOnUnquotedSemicolons(text); for (const auto & subquery : subqueries) { @@ -230,6 +235,7 @@ bool DisksApp::processQueryText(const String & text) { int code = err.code(); error_string = getExceptionMessageForLogging(err, true, false); + last_command_exit_code = code; if (code == ErrorCodes::BAD_ARGUMENTS) { if (command.get()) @@ -246,10 +252,12 @@ bool DisksApp::processQueryText(const String & text) catch (std::exception & err) { error_string = err.what(); + last_command_exit_code = ErrorCodes::STD_EXCEPTION; } catch (...) // Ok: report unknown exception { error_string = "Unknown exception"; + last_command_exit_code = ErrorCodes::STD_EXCEPTION; } if (error_string.has_value()) { @@ -334,6 +342,11 @@ void DisksApp::registerCommands() command_descriptions.emplace("switch-disk", makeCommandSwitchDisk()); command_descriptions.emplace("current_disk_with_path", makeCommandGetCurrentDiskAndPath()); command_descriptions.emplace("touch", makeCommandTouch()); + command_descriptions.emplace("cas-fsck", makeCommandFsck()); + command_descriptions.emplace("cas-gc-dryrun", makeCommandCaGcDryRun()); + command_descriptions.emplace("cas-gc-rebuild", makeCommandCaGcRebuild()); + command_descriptions.emplace("cas-inspect", makeCommandCaInspect()); + command_descriptions.emplace("cas-drop-member", makeCommandCaDropMember()); command_descriptions.emplace("read-checksums", makeCommandReadChecksums()); command_descriptions.emplace("help", makeCommandHelp(*this)); #if CLICKHOUSE_CLOUD @@ -539,6 +552,13 @@ int DisksApp::main(const std::vector & /*args*/) /*max_io_thread_pool_free_size*/ 0, /*io_thread_pool_queue_size*/ 10000); + /// `clickhouse-disks` loads no `ServerSettings`, so this can't read + /// `cas_blob_upload_pool_size`; 16 mirrors that setting's default + /// (`src/Core/ServerSettings.cpp`). A `write` command that commits through a + /// `cas` disk reaches `uploadPendingBlobs`, which calls this pool + /// unconditionally (see the analogous init in `Server.cpp`/`LocalServer.cpp`). + DB::Cas::initializeBlobUploadPool(16); + registerCommands(); registerDisks(/* global_skip_access_check= */ true); @@ -571,6 +591,16 @@ int DisksApp::main(const std::vector & /*args*/) global_context->setPath(path); + /// Load the server UUID so that live CA namespaces resolve correctly. + /// Only load when the uuid file already exists — clickhouse-disks inspects existing + /// pools and must NOT create or mutate the uuid file (the disk may be read-only). + /// If the file is absent, ServerUUID stays Nil and shadow/non-live navigation works. + { + fs::path uuid_file = fs::path(path) / "uuid"; + if (fs::exists(uuid_file)) + ServerUUID::load(uuid_file, &logger()); + } + client = std::make_unique(config(), global_context); suggest.setCompletionsCallback([&](const String & prefix, size_t /* prefix_length */) { return getCompletions(prefix); }); @@ -587,6 +617,10 @@ int DisksApp::main(const std::vector & /*args*/) if (log_file) log_file->close(); + /// Non-interactive runs surface a failing command as a nonzero process exit (CI/cron gating, + /// e.g. `cas-fsck` reporting dangling objects). Interactive sessions are unaffected. + if (query.has_value() && last_command_exit_code != 0) + return last_command_exit_code; return Application::EXIT_OK; } @@ -636,6 +670,7 @@ int mainEntryClickHouseDisks(int argc, char ** argv) /// That way, accesses happen-before destruction. SCOPE_EXIT_SAFE({ DB::StaticThreadPool::shutdownAll(); + DB::Cas::shutdownBlobUploadPool(); GlobalThreadPool::shutdown(); }); diff --git a/programs/disks/DisksApp.h b/programs/disks/DisksApp.h index fbe0639e00f3..27d7e03a6720 100644 --- a/programs/disks/DisksApp.h +++ b/programs/disks/DisksApp.h @@ -90,6 +90,10 @@ class DisksApp : public Poco::Util::Application std::optional query; + /// Set when a command threw during processQueryText; used to make non-interactive (--query) + /// runs exit nonzero (e.g. `fsck` reporting dangling objects). Reset per processQueryText call. + int last_command_exit_code = 0; + const std::unordered_map aliases = { {"cp", "copy"}, {"mv", "move"}, diff --git a/tests/integration/test_disks_app_func/test.py b/tests/integration/test_disks_app_func/test.py index 87ab2e24b9f3..b635f6aad682 100755 --- a/tests/integration/test_disks_app_func/test.py +++ b/tests/integration/test_disks_app_func/test.py @@ -178,7 +178,7 @@ def init_data_s3_rm_rec(source): write(source, "test3", "a/b/d") write(source, "test3", "a/b/e") - write(source, "test3", "d/a") + write(source, "test3", "a/d/a") def test_disks_app_func_ld(started_cluster): @@ -330,22 +330,22 @@ def test_disks_app_func_rm_shared_recursive(started_cluster): out = ls(source, "test3", ". --recursive") assert ( out - == ".:\na\n\n./a:\na\nb\nc\nd\n\n./a/a:\na\nb\nc\n\n./a/b:\na\nb\nc\nd\ne\n\n./a/c:\n\n./a/d:\n\n" + == ".:\na\n\n./a:\na\nb\nc\nd\n\n./a/a:\na\nb\nc\n\n./a/b:\na\nb\nc\nd\ne\n\n./a/c:\n\n./a/d:\na\n\n" ) remove(source, "test3", "a/a --recursive") out = ls(source, "test3", ". --recursive") assert ( - out == ".:\na\n\n./a:\nb\nc\nd\n\n./a/b:\na\nb\nc\nd\ne\n\n./a/c:\n\n./a/d:\n\n" + out == ".:\na\n\n./a:\nb\nc\nd\n\n./a/b:\na\nb\nc\nd\ne\n\n./a/c:\n\n./a/d:\na\n\n" ) remove(source, "test3", "a/b --recursive") out = ls(source, "test3", ". --recursive") - assert out == ".:\na\n\n./a:\nc\nd\n\n./a/c:\n\n./a/d:\n\n" + assert out == ".:\na\n\n./a:\nc\nd\n\n./a/c:\n\n./a/d:\na\n\n" remove(source, "test3", "a/c --recursive") out = ls(source, "test3", ". --recursive") - assert out == ".:\na\n\n./a:\nd\n\n./a/d:\n\n" + assert out == ".:\na\n\n./a:\nd\n\n./a/d:\na\n\n" remove(source, "test3", "a --recursive") out = ls(source, "test3", ". --recursive") diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 804398c8307d..8226c58aaa42 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -1379,6 +1379,14 @@ def test_replicated_table_structure_alter(started_cluster): ) competing_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") + + # `system.tables` only lists an attached database, so the metadata path of `mem` must be read + # before the DETACH below; afterwards the SELECT returns nothing. + metadata_path = competing_node.query( + "SELECT metadata_path FROM system.tables WHERE database='table_structure' AND name='mem'" + ).strip() + assert metadata_path, "metadata_path of table_structure.mem is empty" + competing_node.query("DETACH DATABASE table_structure") main_node.query( @@ -1389,9 +1397,6 @@ def test_replicated_table_structure_alter(started_cluster): ) main_node.query("INSERT INTO table_structure.rmt VALUES (1, 2, 3)") - metadata_path = competing_node.query( - "SELECT metadata_path FROM system.tables WHERE database='table_structure' AND name='mem'" - ).strip() db_disk_name = get_database_disk_name(competing_node) competing_node.exec_in_container( [ From 407935b893b29f048d2ac66e2c281e82c087b3c0 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:33 +0200 Subject: [PATCH 12/30] Fix MSan build: include DataTypesDecimal.h in IcebergWrites.cpp Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index 5bc1fe22fbce..7a267e3b2b6d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include From 83a04256953c16e08bab20d8c3c3568f6af92261 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:33 +0200 Subject: [PATCH 13/30] CAS subsystem: Primitives layer Core value types of the content-addressed storage subsystem: identifiers, hashes, tokens, namespaces. No dependencies on other CAS layers. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../Primitives/CasBlobDigest.cpp | 47 +++ .../Primitives/CasBlobDigest.h | 247 +++++++++++++ .../Primitives/CasBlobHashingWriteBuffer.cpp | 260 ++++++++++++++ .../Primitives/CasBlobHashingWriteBuffer.h | 50 +++ .../Primitives/CasCodecUtil.h | 118 +++++++ .../ContentAddressed/Primitives/CasEvent.cpp | 95 +++++ .../ContentAddressed/Primitives/CasEvent.h | 126 +++++++ .../Primitives/CasNamespaceLifeId.h | 116 ++++++ .../ContentAddressed/Primitives/CasTypes.h | 331 ++++++++++++++++++ .../Primitives/CasXxh3Streamer.h | 83 +++++ 10 files changed, 1473 insertions(+) create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobDigest.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobDigest.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobHashingWriteBuffer.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobHashingWriteBuffer.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasCodecUtil.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasNamespaceLifeId.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasTypes.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasXxh3Streamer.h diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobDigest.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobDigest.cpp new file mode 100644 index 000000000000..3c95ff171e4f --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobDigest.cpp @@ -0,0 +1,47 @@ +#include + +namespace DB::Cas +{ + +std::string_view blobHashAlgoName(BlobHashAlgo algo) +{ + switch (algo) + { + case BlobHashAlgo::CityHash128: + return "ch128"; + case BlobHashAlgo::XXH3_128: + return "xxh3"; + case BlobHashAlgo::Sha256: + return "sha256"; + } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "blobHashAlgoName: unknown BlobHashAlgo {}", static_cast(algo)); +} + +uint64_t blobHashLenFor(BlobHashAlgo algo) +{ + switch (algo) + { + case BlobHashAlgo::CityHash128: + case BlobHashAlgo::XXH3_128: + return 16; + case BlobHashAlgo::Sha256: + return 32; + } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "blobHashLenFor: unknown BlobHashAlgo {}", static_cast(algo)); +} + +BlobHashAlgo parseBlobHashAlgo(std::string_view config_value) +{ + if (config_value == "cityhash128") + return BlobHashAlgo::CityHash128; + if (config_value == "xxh3-128") + return BlobHashAlgo::XXH3_128; + if (config_value == "sha256") + return BlobHashAlgo::Sha256; + + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "parseBlobHashAlgo: unknown blob_hash config value '{}' (expected one of " + "cityhash128|xxh3-128|sha256)", config_value); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobDigest.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobDigest.h new file mode 100644 index 000000000000..cc557fd753e1 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobDigest.h @@ -0,0 +1,247 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} +} + +namespace DB::Cas +{ + +/// Blob identity for the content-addressed pool: the hash-algorithm vocabulary (`BlobHashAlgo`), +/// the digest value (`BlobDigest`), its width-aware representation converter (`DigestCodec`), and +/// the complete identity pair (`BlobRef`). The streaming machinery that PRODUCES digests lives in +/// `CasBlobHashingWriteBuffer.h`; this header is dependency-light on purpose — it is included by +/// virtually every CAS translation unit through `CasTypes.h`. + +/// The content-address hash function selected for a blob pool. The numeric values are persisted as a +/// byte in the binary source-edge run format and must remain stable; the textual name used in object +/// paths and pool metadata is returned by `blobHashAlgoName`. +/// +/// `CityHash128` and `XXH3_128` produce 16-byte digests, while `Sha256` produces a 32-byte digest. +/// The digest representation and codec derive their width from this algorithm for each blob; there +/// is no single pool-wide digest width when a pool admits more than one algorithm. +enum class BlobHashAlgo : uint8_t +{ + CityHash128 = 1, + XXH3_128 = 2, + Sha256 = 3, +}; + +/// The blob PATH SEGMENT for `algo`, e.g. `/blobs///`: `"ch128"` | `"xxh3"` | +/// `"sha256"`. Throws `BAD_ARGUMENTS` for an out-of-range enum value. +std::string_view blobHashAlgoName(BlobHashAlgo algo); + +/// Returns the digest byte width for `algo`: 16 for `CityHash128` and `XXH3_128`, or 32 for +/// `Sha256`. This is also the width used by `Cas::codecFor(algo)`'s `DigestCodec`; callers must +/// derive it from the algorithm rather than from pool state. Throws `BAD_ARGUMENTS` for an +/// out-of-range enum value, preserving the fail-closed contract of `blobHashAlgoName`. +uint64_t blobHashLenFor(BlobHashAlgo algo); + +/// Parses the per-disk `blob_hash` CONFIG value: `"cityhash128"` | `"xxh3-128"` | `"sha256"`. Throws +/// `BAD_ARGUMENTS` on any other value (fail-closed). +BlobHashAlgo parseBlobHashAlgo(std::string_view config_value); + +/// A content digest whose width is selected by its hash algorithm. The fixed 32-byte big-endian +/// storage accommodates the existing 128-bit algorithms (`cityHash128` and `xxh3-128`) and +/// `sha256`; only the first `blobHashLenFor(algo)` bytes are meaningful and the remaining bytes +/// must be zero. A fixed array avoids a per-manifest-entry allocation that a variable `String` +/// would require for 32-byte digests. +/// +/// This type is reserved for content hashes. Protocol identifiers such as `payload_digest`, +/// `RunRef::checksum`, source-edge identifiers, lease owners, and cleanup shards remain +/// `UInt128`, because widening the content digest does not change their separate wire or ordering +/// contracts. A blob's complete identity is `BlobRef`, which pairs this digest with its algorithm. +struct BlobDigest +{ + std::array bytes{}; + + auto operator<=>(const BlobDigest &) const = default; + bool operator==(const BlobDigest &) const = default; + + /// Converts a 128-bit content hash to the common representation: big-endian in `bytes[0:16]` + /// and zero in the tail. This is the bridge for the 128-bit hash algorithms. + static BlobDigest fromU128(const UInt128 & v) + { + BlobDigest d; + for (int i = 0; i < 16; ++i) + d.bytes[static_cast(i)] = static_cast(static_cast(v >> (8 * (15 - i)))); + return d; + } + + /// Reads `bytes[0:16]` as big-endian into a `UInt128`. The conversion is meaningful only for a + /// 128-bit digest; the caller is responsible for selecting that width and the tail is ignored. + UInt128 toU128() const + { + UInt128 v = 0; + for (int i = 0; i < 16; ++i) + v = (v << 8) | static_cast(bytes[static_cast(i)]); + return v; + } +}; + +/// Hasher for `BlobDigest` as an `unordered_map`/`unordered_set` key. This is an in-process hash +/// table key, not a content address, so a cheap FNV-1a mix over the raw bytes is sufficient -- no +/// cryptographic property is needed here. +struct BlobDigestHash +{ + size_t operator()(const BlobDigest & d) const noexcept + { + size_t h = 1469598103934665603ull; /// FNV-1a 64-bit offset basis + for (uint8_t b : d.bytes) + { + h ^= b; + h *= 1099511628211ull; /// FNV-1a 64-bit prime + } + return h; + } +}; + +/// Converts a `BlobDigest` using one algorithm's width. A codec must be obtained from the algorithm +/// through `codecFor`, never from a pool-wide width: a pool may contain multiple algorithms. Hex +/// and raw-byte conversions accept and produce exactly the selected width. `shardOf` reads the +/// first eight digest bytes in big-endian order, preserving the existing shard mapping for every +/// 128-bit digest. +class DigestCodec +{ +public: + /// Creates a codec for a supported digest width: 16 bytes for the 128-bit algorithms or + /// 32 bytes for `sha256`. Any other width violates the per-algorithm representation invariant. + explicit DigestCodec(uint64_t digest_len_) : len(digest_len_) + { + chassert(len == 16 || len == 32, "DigestCodec: digest length must be 16 or 32 bytes"); + } + + /// Renders exactly `2 * len` lowercase hex chars. + String toHex(const BlobDigest & d) const + { + checkZeroTail(d, "toHex"); + return hexString(d.bytes.data(), len); + } + + /// Requires exactly `2 * len` hex chars; throws `BAD_ARGUMENTS` otherwise (wrong + /// width or a non-hex character). Zero-fills the tail beyond `len`. + BlobDigest fromHex(std::string_view hex) const + { + if (hex.size() != 2 * len) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "DigestCodec::fromHex: expected {} hex chars for a {}-byte digest, got {}", + 2 * len, len, hex.size()); + + for (char c : hex) + { + if (unhex(c) == 0xff) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "DigestCodec::fromHex: invalid hex character '{}'", c); + } + + BlobDigest d; + for (uint64_t i = 0; i < len; ++i) + d.bytes[i] = unhex2(hex.data() + i * 2); + return d; + } + + /// Serializes exactly `len` raw bytes, big-endian (i.e. `bytes[0:len]`). + String toBytesBE(const BlobDigest & d) const + { + checkZeroTail(d, "toBytesBE"); + return String(reinterpret_cast(d.bytes.data()), len); + } + + /// Requires exactly `len` bytes; throws `BAD_ARGUMENTS` otherwise. Zero-fills the + /// tail beyond `len`. + BlobDigest fromBytesBE(std::string_view b) const + { + if (b.size() != len) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "DigestCodec::fromBytesBE: expected {} bytes for the pool's digest width, got {}", len, b.size()); + + BlobDigest d; + memcpy(d.bytes.data(), b.data(), len); + return d; + } + + /// Returns the first eight digest bytes as a big-endian `uint64_t`. Keep this explicit rather + /// than using a native-endian `memcpy`: changing the byte order would silently remap shards on + /// little-endian hosts and break compatibility with the 128-bit hash mapping. + uint64_t shardOf(const BlobDigest & d) const + { + uint64_t v = 0; + for (int i = 0; i < 8; ++i) + v = (v << 8) | d.bytes[static_cast(i)]; + return v; + } + +private: + uint64_t len; + + /// Checks the representation invariant that bytes beyond the selected width are zero. This is + /// a debug-only internal assertion; wrong-width external input is rejected by the decoding + /// methods above. + void checkZeroTail(const BlobDigest & d, [[maybe_unused]] const char * what) const + { + for (uint64_t i = len; i < d.bytes.size(); ++i) + chassert(d.bytes[i] == 0, fmt::format("DigestCodec::{}: non-zero byte at tail position {} (len={})", what, i, len)); + } +}; + +/// The complete blob identity is the pair of hash algorithm and +/// digest. A bare digest is NOT a blob identity anywhere -- `ch128` and `xxh3` digests are both +/// 16-byte, so the same digest value under two algos names two DIFFERENT objects. BlobRef is +/// constructed ONLY where algo and digest are born together (the write mint / the hasher) or read +/// together (a durable form: settlement key, blob path, manifest entry, envelope). Every other +/// site COPIES BlobRefs -- never assemble one from an algo and a digest obtained separately. +struct BlobRef +{ + BlobHashAlgo algo = BlobHashAlgo::CityHash128; + BlobDigest digest{}; + + auto operator<=>(const BlobRef &) const = default; + bool operator==(const BlobRef &) const = default; +}; + +/// Hasher for unordered_map/unordered_set keys (in-process only, not a content address). +struct BlobRefHash +{ + size_t operator()(const BlobRef & r) const noexcept + { + size_t h = BlobDigestHash{}(r.digest); + h ^= static_cast(r.algo) + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2); + return h; + } +}; + +/// Returns the codec whose width belongs to `algo`; callers must not substitute a pool-wide width. +inline DigestCodec codecFor(BlobHashAlgo algo) +{ + return DigestCodec(blobHashLenFor(algo)); +} + +/// Bare lowercase hex of the digest at the algo's width -- for OBJECT KEY construction only +/// (the algo lives in the key's path segment `blobs//...`). +inline String blobHexOf(const BlobRef & r) +{ + return codecFor(r.algo).toHex(r.digest); +} + +/// Human/log identity: ":", e.g. "sha256:ab12...". Rendered ids must never be a +/// bare hex (ambiguous across algos) -- events, inspect JSON and error messages use this. +inline String blobIdOf(const BlobRef & r) +{ + return String(blobHashAlgoName(r.algo)) + ":" + blobHexOf(r); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobHashingWriteBuffer.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobHashingWriteBuffer.cpp new file mode 100644 index 000000000000..b5c3d530e535 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobHashingWriteBuffer.cpp @@ -0,0 +1,260 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include "config.h" + +#if USE_SSL +# include +# include +#endif + +/// `XXH_INLINE_ALL` renames every public symbol under the `XXH_INLINE_` prefix (`XXH_NAMESPACE`) and +/// makes the whole library a header-only, static-inline implementation local to THIS translation +/// unit -- no link dependency on the separately-compiled `ch_contrib::xxHash` object. This file is +/// part of the `dbms` target (not `clickhouse_functions_obj`, which gets the flag via its own +/// `target_link_libraries(... ch_contrib::xxHash)`), so the macro is defined locally here, same +/// effect, same prefixed names as `Functions/FunctionsHashing.h` uses. +/// xxHash is included through this wrapper (which marks it a system header) to suppress the vendored-C +/// warnings from lz4's shadowing copy under `-Werror -Weverything`. See `CasXxh3Streamer.h`. +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int CANNOT_ALLOCATE_MEMORY; + extern const int OPENSSL_ERROR; + extern const int SUPPORT_IS_DISABLED; +} +} + +namespace DB::Cas +{ + +namespace +{ + +/// Thin adapter over the existing `HashingWriteBuffer` so `CityHash128` blob hashes stay +/// byte-identical to today. Bytes written to `*this` alias directly into `hashing`'s own buffer +/// (the same zero-copy trick `HashingWriteBuffer` itself uses against its nested sink), so this +/// adds no extra copy and no change to the chunked `CityHash128WithSeed` chaining. +class CityHash128BlobHashingWriteBuffer : public IBlobHashingWriteBuffer +{ +public: + explicit CityHash128BlobHashingWriteBuffer(WriteBuffer & sink) + : IBlobHashingWriteBuffer() + , hashing(sink) + { + working_buffer = hashing.buffer(); + pos = working_buffer.begin(); + } + + void sync() override + { + hashing.sync(); + } + + String getHashHex() override + { + next(); + return getHexUIntLowercase(hashing.getHash()); + } + +private: + void nextImpl() override + { + hashing.position() = pos; + hashing.next(); + working_buffer = hashing.buffer(); + } + + void finalizeImpl() override + { + next(); + hashing.finalize(); + } + + void cancelImpl() noexcept override + { + hashing.cancel(); + } + + HashingWriteBuffer hashing; +}; + +/// A hash-and-passthrough buffer over the xxhash library's streaming `XXH3_128bits` state. Unlike +/// `CityHash128`, xxh3's streaming digest is defined to agree with its one-shot digest, so there is +/// no chunked-convention to preserve -- this just needs to feed every byte to the streaming state +/// (`update`) and forward the same bytes to `sink` unchanged. +class Xxh3128BlobHashingWriteBuffer : public BufferWithOwnMemory +{ +public: + explicit Xxh3128BlobHashingWriteBuffer(WriteBuffer & sink_, size_t buf_size = DBMS_DEFAULT_HASHING_BLOCK_SIZE) + : BufferWithOwnMemory(buf_size) + , sink(sink_) + { + if (!state.valid()) + throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Xxh3128BlobHashingWriteBuffer: failed to allocate the xxh3 streaming state"); + } + + void sync() override + { + sink.sync(); + } + + String getHashHex() override + { + next(); + UInt64 low = 0; + UInt64 high = 0; + state.digest(low, high); + return getHexUIntLowercase(UInt128{low, high}); + } + +private: + void nextImpl() override + { + const size_t len = offset(); + if (!len) + return; + + state.update(working_buffer.begin(), len); + sink.write(working_buffer.begin(), len); + } + + WriteBuffer & sink; + Xxh3Streamer state; +}; + +#if USE_SSL +/// A hash-and-passthrough buffer over OpenSSL's streaming EVP SHA-256 digest. Unlike the 128-bit +/// hashes above, `Sha256` produces a 32-byte digest (64 lowercase hex chars, see `blobHashLenFor`). +/// Every byte written is folded into the running EVP digest (`EVP_DigestUpdate`) AND forwarded +/// unchanged to `sink`, exactly like `Xxh3128BlobHashingWriteBuffer` above -- streaming SHA-256 is +/// defined to agree with the one-shot digest, so there is no chunked-convention to preserve either. +class Sha256BlobHashingWriteBuffer : public BufferWithOwnMemory +{ +public: + explicit Sha256BlobHashingWriteBuffer(WriteBuffer & sink_, size_t buf_size = DBMS_DEFAULT_HASHING_BLOCK_SIZE) + : BufferWithOwnMemory(buf_size) + , sink(sink_) + , ctx(EVP_MD_CTX_new(), &EVP_MD_CTX_free) + { + if (!ctx) + throw Exception(ErrorCodes::OPENSSL_ERROR, + "Sha256BlobHashingWriteBuffer: EVP_MD_CTX_new failed: {}", getOpenSSLErrors()); + + if (EVP_DigestInit_ex(ctx.get(), EVP_sha256(), nullptr) != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, + "Sha256BlobHashingWriteBuffer: EVP_DigestInit_ex failed: {}", getOpenSSLErrors()); + } + + void sync() override + { + sink.sync(); + } + + String getHashHex() override + { + next(); + + unsigned char digest[EVP_MAX_MD_SIZE]; + unsigned int digest_len = 0; + if (EVP_DigestFinal_ex(ctx.get(), digest, &digest_len) != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, + "Sha256BlobHashingWriteBuffer: EVP_DigestFinal_ex failed: {}", getOpenSSLErrors()); + + chassert(digest_len == 32); + return hexString(digest, digest_len); + } + +private: + using EVP_MD_CTX_ptr = std::unique_ptr; + + void nextImpl() override + { + const size_t len = offset(); + if (!len) + return; + + if (EVP_DigestUpdate(ctx.get(), working_buffer.begin(), len) != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, + "Sha256BlobHashingWriteBuffer: EVP_DigestUpdate failed: {}", getOpenSSLErrors()); + + sink.write(working_buffer.begin(), len); + } + + WriteBuffer & sink; + EVP_MD_CTX_ptr ctx; +}; +#endif + +} + +std::unique_ptr makeBlobHashingWriteBuffer(BlobHashAlgo algo, WriteBuffer & sink) +{ + switch (algo) + { + case BlobHashAlgo::CityHash128: + return std::make_unique(sink); + case BlobHashAlgo::XXH3_128: + return std::make_unique(sink); + case BlobHashAlgo::Sha256: +#if USE_SSL + return std::make_unique(sink); +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "blob_hash = 'sha256' requires ClickHouse built with SSL support"); +#endif + } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "makeBlobHashingWriteBuffer: unknown BlobHashAlgo {}", static_cast(algo)); +} + +String blobHashHexOneShot(BlobHashAlgo algo, std::string_view bytes) +{ + switch (algo) + { + case BlobHashAlgo::CityHash128: + { + /// Preserve the blob content-hash convention used by `poolContentHash`: hash through + /// `HashingReadBuffer` in `DBMS_DEFAULT_HASHING_BLOCK_SIZE` chunks, chaining + /// `CityHash128WithSeed`. A one-shot `CityHash128WithSeed` call would produce a different + /// result for payloads larger than one hash block. + ReadBufferFromMemory in(bytes.data(), bytes.size()); + HashingReadBuffer hashing(in); + hashing.ignoreAll(); + return getHexUIntLowercase(hashing.getHash()); + } + case BlobHashAlgo::XXH3_128: + { + UInt64 low = 0; + UInt64 high = 0; + xxh3_128_oneshot(bytes.data(), bytes.size(), low, high); + return getHexUIntLowercase(UInt128{low, high}); + } + case BlobHashAlgo::Sha256: + { +#if USE_SSL + /// One-shot SHA-256 is defined to agree with the streaming EVP digest above (there is no + /// chunked convention to preserve, unlike `CityHash128`), so this can go straight through + /// `encodeSHA256`'s one-shot path instead of round-tripping through a streaming buffer. + unsigned char digest[32]; + encodeSHA256(bytes.data(), bytes.size(), digest); + return hexString(digest, sizeof(digest)); +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "blob_hash = 'sha256' requires ClickHouse built with SSL support"); +#endif + } + } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "blobHashHexOneShot: unknown BlobHashAlgo {}", static_cast(algo)); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobHashingWriteBuffer.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobHashingWriteBuffer.h new file mode 100644 index 000000000000..0779363a2f54 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasBlobHashingWriteBuffer.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// A streaming hash-and-passthrough `WriteBuffer`: every byte written is forwarded unchanged to the +/// nested sink AND folded into the running digest, exposed as lowercase hex once flushed (32 hex +/// chars for the 128-bit hashes, 64 hex chars for `Sha256` -- see `blobHashLenFor`) (`getHashHex` +/// calls `next()` first, mirroring `HashingWriteBuffer::getHash`). It does not finalize or cancel the +/// underlying sink -- that stays the caller's responsibility, the same contract `CaContentWriteBuffer` +/// already relies on for `HashingWriteBuffer` and its nested sink. +class IBlobHashingWriteBuffer : public WriteBuffer +{ +public: + explicit IBlobHashingWriteBuffer(Position ptr = nullptr, size_t size = 0) : WriteBuffer(ptr, size) {} + ~IBlobHashingWriteBuffer() override = default; + + /// Flushes any pending bytes (like `HashingWriteBuffer::getHash`) and returns the digest as + /// lowercase hex (length depends on `algo`, see `blobHashLenFor`). May be called only once useful + /// data has stopped flowing; does not itself finalize the buffer (mirrors + /// `HashingWriteBuffer::getHash`, which also does not finalize). + virtual String getHashHex() = 0; +}; + +/// Builds a streaming hash-and-passthrough buffer for `algo` over `sink`. Every byte written to the +/// returned buffer is forwarded unchanged to `sink` and included in its digest. `sink` must outlive +/// the returned buffer, as with `HashingWriteBuffer`. +/// +/// `CityHash128` is a thin adapter over the existing `HashingWriteBuffer`: it retains the +/// `DBMS_DEFAULT_HASHING_BLOCK_SIZE` chunks and chained `CityHash128WithSeed` convention, so the +/// default algorithm produces byte-identical blob hashes. `XXH3_128` uses the xxhash library's +/// streaming `XXH3_128bits` state, and `Sha256` uses OpenSSL's streaming EVP SHA-256 digest. Both +/// of those algorithms define their streaming digest to agree with their one-shot digest. +std::unique_ptr makeBlobHashingWriteBuffer(BlobHashAlgo algo, WriteBuffer & sink); + +/// Computes the lowercase-hex digest of `bytes` for `algo`. This is used by the re-hash and +/// copy-forward path and by tests. For `CityHash128`, the implementation deliberately uses +/// `HashingReadBuffer` so it follows the same chunked, chained convention as the streaming write +/// path; a one-shot `CityHash128WithSeed` call would diverge for payloads larger than one hash block. +/// The one-shot `XXH3_128bits` and OpenSSL `encodeSHA256` paths agree with their streaming +/// counterparts. +String blobHashHexOneShot(BlobHashAlgo algo, std::string_view bytes); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasCodecUtil.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasCodecUtil.h new file mode 100644 index 000000000000..53f14a120174 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasCodecUtil.h @@ -0,0 +1,118 @@ +#pragma once +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +/// Shared low-level byte-encoding helpers for CAS: the big-endian `UInt128` wire form, exact reads, +/// and validation of identifiers embedded in persisted data. They remain independent of any +/// particular object format. + +/// On-disk UInt128 wire form: the 16-byte big-endian representation used by raw CAS byte fields and +/// key components. It is FROZEN — changing the bytes breaks every object already written. (The +/// lowercase-hex form lives in `CasTypes.h` as `u128ToHex` / `hexToU128` and is out of scope here.) +/// +/// Converts `v` to the frozen 16-byte big-endian representation used in raw byte fields and keys. +inline std::string u128ToBytesBE(const UInt128 & v) +{ + std::string out(16, '\0'); + for (int i = 0; i < 16; ++i) + out[i] = static_cast(static_cast(v >> (8 * (15 - i)))); + return out; +} + +/// Parses a frozen 16-byte big-endian value. `what` identifies the containing field in corruption +/// diagnostics; any other length is malformed persisted data and raises `CORRUPTED_DATA`. +inline UInt128 u128FromBytesBE(const std::string & b, std::string_view what) +{ + if (b.size() != 16) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: big-endian UInt128 field must be 16 bytes, got {}", what, b.size()); + UInt128 v = 0; + for (int i = 0; i < 16; ++i) + v = (v << 8) | static_cast(b[i]); + return v; +} + +/// Read exactly `n` raw bytes. The bounds check MUST precede the allocation: `n` typically comes +/// from a length field just read off the wire, so on corrupted input it can be huge (a u32 field +/// admits 4 GiB) — allocating first would mean a multi-GiB transient allocation, which under a +/// memory tracker surfaces as MEMORY_LIMIT_EXCEEDED instead of the pinned CORRUPTED_DATA. +/// Comparing against `available` as the exact remainder is valid because all CAS codec decoding +/// reads from `ReadBufferFromMemory`: the whole object is in memory, so `available` is exactly +/// the number of bytes left. +/// Throws `CORRUPTED_DATA` before allocating when the encoded object cannot contain `n` bytes. +inline String readFixedBytes(ReadBuffer & in, size_t n) +{ + if (n > in.available()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS codec: truncated encoded data: need {} bytes, {} available", n, in.available()); + String s(n, '\0'); + in.readStrict(s.data(), n); + return s; +} + +/// Canonical clean relative path for ref/file names: non-empty, no NUL byte, no backslash, and no +/// segment that is empty (rejects a leading/trailing/doubled '/'), ".", or "..". Names in this +/// family originate from part names -- a NUL byte is never legitimate there, so it fails closed +/// rather than being silently truncated or passed through. +/// Returns true only for a non-empty relative path whose slash-separated components are all normal +/// names; it does not normalize or rewrite the input. +inline bool isCanonicalRefName(std::string_view name) +{ + if (name.empty() || name.find('\0') != std::string_view::npos || name.find('\\') != std::string_view::npos) + return false; + size_t start = 0; + while (true) + { + const size_t end = name.find('/', start); + const std::string_view segment + = name.substr(start, end == std::string_view::npos ? std::string_view::npos : end - start); + if (segment.empty() || segment == "." || segment == "..") + return false; + if (end == std::string_view::npos) + break; + start = end + 1; + } + return true; +} + +/// Throws CORRUPTED_DATA naming both `caller` (the codec, e.g. "RefLogTxn") and `what` (the field, +/// e.g. "set_published_at ref_name") when `name` fails `isCanonicalRefName`. +/// On success, the input is unchanged; on failure, no partial normalization is attempted. +inline void checkCanonicalRefName(std::string_view name, std::string_view caller, std::string_view what) +{ + if (!isCanonicalRefName(name)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "{}: {} is not a canonical clean relative path: '{}'", caller, what, name); +} + +/// `ManifestRef` field validity, shared by the ref codecs (`CasRefLogFormat`, +/// `CasRefSnapshotFormat`): `writer_epoch`/`build_sequence` nonzero, `manifest_ordinal` in +/// `[1, kMaxManifestOrdinal]` -- the same range `manifestOrdinalFileName` (`CasManifestId.h`) enforces +/// at key-construction time. Throws CORRUPTED_DATA naming both `caller` (the codec) and `what` (the +/// field, e.g. "set_published_at manifest_ref"). +/// This keeps the value-level invariant aligned with the range enforced by manifest-key construction +/// before either a codec encoder or decoder accepts the reference. +inline void checkManifestRef(const ManifestRef & ref, std::string_view caller, std::string_view what) +{ + if (ref.writer_epoch == 0 || ref.build_sequence == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "{}: {} manifest_ref writer_epoch/build_sequence must both be nonzero, got {}-{}", + caller, what, ref.writer_epoch, ref.build_sequence); + if (ref.manifest_ordinal == 0 || ref.manifest_ordinal > kMaxManifestOrdinal) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "{}: {} manifest_ref manifest_ordinal {} out of range", caller, what, ref.manifest_ordinal); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.cpp new file mode 100644 index 000000000000..a981d1cd0575 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.cpp @@ -0,0 +1,95 @@ +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +String toString(CasEventType type) +{ + switch (type) + { + case CasEventType::BlobPut: return "blob_put"; + case CasEventType::BlobReuseAdopt: return "blob_reuse_adopt"; + case CasEventType::BlobReuseResurrect: return "blob_reuse_resurrect"; + case CasEventType::BlobRetire: return "blob_retire"; + case CasEventType::BlobRetireReplaced: return "blob_retire_replaced"; + case CasEventType::BlobDelete: return "blob_delete"; + case CasEventType::BlobForget: return "blob_forget"; + case CasEventType::ManifestPut: return "manifest_put"; + case CasEventType::ManifestDelete: return "manifest_delete"; + case CasEventType::RefPublish: return "ref_publish"; + case CasEventType::RefDrop: return "ref_drop"; + case CasEventType::RefRepoint: return "ref_repoint"; + case CasEventType::RootAdd: return "root_add"; + case CasEventType::RootRemove: return "root_remove"; + case CasEventType::RootRepoint: return "root_repoint"; + case CasEventType::IndegZero: return "indegree_zero"; + case CasEventType::GcFoldBegin: return "gc_fold_begin"; + case CasEventType::GcFoldEnd: return "gc_fold_end"; + case CasEventType::GcRetireObserve: return "gc_retire_observe"; + case CasEventType::GcRetireDecision: return "gc_retire_decision"; + case CasEventType::GcRecheckVerdict: return "gc_recheck_verdict"; + case CasEventType::GcFence: return "gc_fence"; + case CasEventType::GcCursorAdvance: return "gc_cursor_advance"; + case CasEventType::GcShardReclaim: return "gc_shard_reclaim"; + case CasEventType::GcFenceOut: return "gc_fence_out"; + case CasEventType::GcRebuild: return "gc_rebuild"; + case CasEventType::GcFoldClamp: return "gc_fold_clamp"; + case CasEventType::GcAnomaly: return "gc_anomaly"; + case CasEventType::GcLeaseAcquire: return "gc_lease_acquire"; + case CasEventType::GcLeaseSteal: return "gc_lease_steal"; + case CasEventType::GcLeaseHeartbeat: return "gc_lease_heartbeat"; + case CasEventType::BuildStart: return "build_start"; + case CasEventType::BuildPublish: return "build_publish"; + case CasEventType::BuildAbort: return "build_abort"; + case CasEventType::Precommit: return "precommit"; + case CasEventType::PrecommitRemoved: return "precommit_removed"; + case CasEventType::PrecommitReclaim: return "precommit_reclaim"; + case CasEventType::GateRevalidate: return "gate_revalidate"; + case CasEventType::GateResurrect: return "gate_resurrect"; + case CasEventType::WatermarkRenew: return "watermark_renew"; + case CasEventType::MountRemount: return "mount_remount"; + case CasEventType::MountClaim: return "mount_claim"; + case CasEventType::MountRelease: return "mount_release"; + case CasEventType::MountConflict: return "mount_conflict"; + case CasEventType::MemberDecommission: return "member_decommission"; + case CasEventType::ForeignInterference: return "foreign_interference"; + case CasEventType::RefResolve: return "ref_resolve"; + case CasEventType::ReadMissing: return "read_missing"; + case CasEventType::DanglingAccess: return "dangling_access"; + case CasEventType::CorruptDangle: return "corrupt_dangle"; + case CasEventType::CorruptDecode: return "corrupt_decode"; + case CasEventType::SnapJournalIncoherent: return "snap_journal_incoherent"; + case CasEventType::Exception: return "exception"; + } + throw DB::Exception( + DB::ErrorCodes::LOGICAL_ERROR, + "CasEvent: unknown CasEventType value {}", + static_cast(type)); +} + +String toString(CasEventObjectKind kind) +{ + switch (kind) + { + case CasEventObjectKind::None: return "none"; + case CasEventObjectKind::Blob: return "blob"; + case CasEventObjectKind::Manifest: return "manifest"; + case CasEventObjectKind::Root: return "root"; + case CasEventObjectKind::Snap: return "snapshot"; + } + throw DB::Exception( + DB::ErrorCodes::LOGICAL_ERROR, + "CasEvent: unknown CasEventObjectKind value {}", + static_cast(kind)); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.h new file mode 100644 index 000000000000..752ac5cf3f84 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.h @@ -0,0 +1,126 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Names the append-only audit events emitted by content-addressed storage. The metadata storage +/// converts each `CasEvent` into a `ContentAddressedLogElement` and forwards it to the `SystemLog`; +/// this layer deliberately keeps only pure data so the core and its unit tests do not depend on +/// system-log machinery. The event log reconstructs each entity's lifetime, so this taxonomy +/// includes state-changing decisions, GC transitions, and errors or anomalies rather than only +/// successful user-visible operations. +enum class CasEventType +{ + BlobPut, BlobReuseAdopt, BlobReuseResurrect, BlobRetire, BlobRetireReplaced, BlobDelete, BlobForget, + ManifestPut, ManifestDelete, + RefPublish, RefDrop, RefRepoint, RootAdd, RootRemove, RootRepoint, IndegZero, + GcFoldBegin, GcFoldEnd, GcRetireObserve, GcRetireDecision, GcRecheckVerdict, + GcFence, GcCursorAdvance, GcShardReclaim, GcFenceOut, GcRebuild, GcFoldClamp, + /// One GC round anomaly, with the context of the decision it forced. `gc_fold_end` reports only a + /// COUNT of anomalies, which cannot even distinguish a ref-prefix enumeration disagreement from an + /// undecodable ref-log body -- so every investigation had to fall back on the rotated text log, which + /// is how a masked permission error once produced a confident "no occurrences". An anomaly aborts ref + /// folding; that decision deserves a queryable record, not a counter. + GcAnomaly, + GcLeaseAcquire, GcLeaseSteal, GcLeaseHeartbeat, + BuildStart, BuildPublish, BuildAbort, Precommit, PrecommitRemoved, PrecommitReclaim, + GateRevalidate, GateResurrect, WatermarkRenew, MountRemount, + MountClaim, MountRelease, MountConflict, + /// Operator-driven erasure of a dead pool member's namespace. `decommissionPoolMember` runs as + /// a writer, never as GC: it claims the member's mount, drains its namespaces and debris, and + /// deletes the mount slot only after the drain is confirmed. The slot is the interrupted-operation + /// resume anchor, so a failed drain leaves it terminated for a later retry. `outcome` is one of + /// "begin", "namespace_removed", or "end". + MemberDecommission, + /// Incidental-detection reaction to foreign bytes at a ref-log wedge key owned by this mount. + /// The mount lease makes the key exclusive, so this is impossible under legitimate + /// single-writer operation and indicates that the wedge hard contract was violated. The + /// reaction records the anomaly and fails the local write path closed; it does not treat the + /// foreign bytes as valid ref-log state. + ForeignInterference, + RefResolve, ReadMissing, DanglingAccess, + CorruptDangle, CorruptDecode, SnapJournalIncoherent, Exception, +}; + +/// Identifies the kind of object described by a `CasEvent`. `None` is used for events about a +/// protocol action, mount, or anomaly that is not tied to one stored object. +enum class CasEventObjectKind { None, Blob, Manifest, Root, Snap }; + +/// Pure-data event passed from the content-addressed core to the metadata-storage audit-log sink. +/// Fields that do not apply to an event remain empty or zero. `reason` is mandatory for decisions +/// and must explain why the operation took its outcome; `detail` carries structured facts needed +/// to reconstruct the event without parsing the free-form reason. Hashes are lowercase hexadecimal, +/// tokens identify object incarnations, and the numeric fields identify GC rounds, snapshot +/// generations, or the manifest journal version as applicable. +struct CasEvent +{ + CasEventType type = CasEventType::BlobPut; + String namespace_; /// roots/ (empty if N/A) + String ref_name; /// the ref name — a mutable directory handle, git-style (empty if N/A) + CasEventObjectKind object_kind = CasEventObjectKind::None; + String object_hash; /// lowercase hex (empty if N/A) + String token; /// incarnation token (empty if N/A) + UInt64 round = 0; + UInt64 gen = 0; + UInt64 at_version = 0; + String outcome; /// e.g. "ok","adopt","deleted","zeroed" (empty if N/A) + String reason; /// REQUIRED: the human-readable WHY of the decision + std::map detail; +}; + +/// Receives events by value so emission sites can move the complete record, including its `detail` +/// map, into the sink instead of deep-copying it on the emitter thread. Emission sites pass an rvalue +/// for a completed event; the sink consumes that event while converting it to the system-log row. +using CasEventSink = std::function; + +/// Builds and emits a `CasEvent` for a store that owns the event sink. The builder supplies the +/// per-event fields; the emitter supplies the sink owner shared by all events from that store. +/// +/// If the store has no sink, the builder is not invoked and no event is constructed; the disabled +/// path is therefore only the sink-presence check. Otherwise, `emit` moves the completed event into +/// `emitEvent`, preserving the fields supplied by the builder without adding a copy. This class is +/// templated to avoid a header-layering cycle (`CasPool.h` includes this header); any `S` exposing +/// `hasEventSink` and `emitEvent` with the expected contracts can be used. +template +class EventEmitter +{ +public: + /// Keeps a reference to the store; the store must outlive this short-lived emitter. + explicit EventEmitter(const S & store_) : store(store_) {} + + template + /// Invokes `build` only when the store has an enabled sink, then moves the resulting event into + /// the store. `Builder` must accept `CasEvent &` and may populate any applicable fields; any + /// exception from the builder or the store is propagated to the caller. + void emit(Builder && build) const + { + if (!store.hasEventSink()) + return; + CasEvent event; + build(event); + store.emitEvent(std::move(event)); + } + +private: + const S & store; +}; + +template +EventEmitter(const S &) -> EventEmitter; + +/// Converts an event taxonomy value to the stable snake_case name stored in the `SystemLog` +/// `event_type` column. Every enumerator must have a mapping because these names are queried by +/// users and are part of the audit-log schema; an unknown value raises a logical-error exception. +String toString(CasEventType type); + +/// Converts an object-kind value to the stable snake_case name stored in the `SystemLog` +/// `object_kind` column. An unknown value raises a logical-error exception rather than silently +/// producing an unrecognized schema value. +String toString(CasEventObjectKind kind); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasNamespaceLifeId.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasNamespaceLifeId.h new file mode 100644 index 000000000000..c2bab7f2bf8a --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasNamespaceLifeId.h @@ -0,0 +1,116 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +/// Opaque pool-wide physical identity of one namespace life. The catalog's existing `incarnation` +/// field is this value; the alias makes listed-key APIs explicit without renaming catalog wire data. +using NamespaceLifePhysicalId = UInt128; + +/// Builds the key segment for `incarnation`: 32 fixed-width lower-case hex digits, so the segment has +/// one canonical spelling and `/` sorts stably. Throws `LOGICAL_ERROR` on a zero incarnation +/// for the same reason `renderRefTxnId` does: this render becomes an object key, and an invalid +/// identity must never silently produce a well-formed-looking one. +inline String renderIncarnation(const UInt128 & incarnation) +{ + if (incarnation == 0) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, + "NamespaceLifeId: incarnation must be nonzero -- 0 never names a life"); + return u128ToHex(incarnation); +} + +/// Inverse of `renderIncarnation` for ONE listed path segment. Accepts the canonical form only: +/// exactly 32 lower-case hex digits encoding a nonzero value. Upper case, a short or long segment, +/// non-hex characters and an all-zero segment all return `std::nullopt`; the CALLER decides whether +/// that is "not one of our keys" or corruption, because only the caller knows whether the rest of the +/// key already identified the object as ours. +inline std::optional parseIncarnation(std::string_view s) +{ + constexpr size_t kHexLen = 32; + if (s.size() != kHexLen) + return std::nullopt; + + /// `unhexUInt` also accepts upper case, which the canonical form must reject, so the digits are + /// validated by hand first and only then handed to it. + for (char c : s) + { + const bool canonical_digit = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); + if (!canonical_digit) + return std::nullopt; + } + + /// `unhexUInt` reads exactly sizeof(UInt128)*2 == kHexLen bytes from the pointer, already + /// validated above; it needs no NUL terminator. + // NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage) + const UInt128 value = unhexUInt(s.data()); + if (value == 0) + return std::nullopt; + return value; +} + +class Layout; + +/// The logical and physical identity of ONE LIFE of a namespace: the catalog name plus its opaque, +/// pool-wide incarnation. Physical life-owned keys use only `incarnation`; the name remains the +/// catalog authority used after a listed physical id is joined through one immutable catalog cut. +/// +/// Part manifests and loose mountpoint objects are deliberately NOT qualified (Constraint 12, +/// directive §2): manifests already carry globally unique build identities, and a loose mountpoint +/// object is outside namespace ownership altogether. +/// +/// There is deliberately NO conversion from a bare `RootNamespace`, none to one, and no default +/// construction: code holding only the name cannot name a ref object or a namespace file at all, so +/// losing the incarnation is a compile error rather than a runtime aliasing bug (spec §2 r9-3). The +/// call sites that legitimately need the bare name -- manifest identities, loose mountpoint objects -- +/// say `.ns`, and are visible in review because they say it. +/// +/// Permanent logical/physical pairs come only from `fromCatalogEntry` (recovery, fold, fsck and the +/// sweeps). A held reader (e.g. `RefTableRuntime`) keeps the `NamespaceLifeId` it resolved this way +/// at table-open time and threads it through, rather than re-deriving it. `Layout` parsers +/// deliberately return only an untrusted `NamespaceLifePhysicalId`; a listed key can name any +/// physical id, including one no longer in the catalog, and only one immutable catalog cut may +/// attach a logical name to it. +struct NamespaceLifeId +{ + RootNamespace ns; + NamespaceLifePhysicalId incarnation; /// 0 is INVALID -- never a wildcard, never "any life" + + bool operator==(const NamespaceLifeId &) const = default; + + /// The catalog is the universe authority (INV-3): a discovery path learns of a namespace ONLY + /// from a catalog entry, and takes BOTH fields from that same entry. Pairing a namespace with an + /// incarnation obtained from anywhere else re-opens the rebirth alias this type exists to close. + static NamespaceLifeId fromCatalogEntry(RootNamespace ns, const UInt128 & incarnation) + { + return NamespaceLifeId{std::move(ns), incarnation}; + } + +private: + /// Kept private so only the explicit factories above can construct a logical/physical pair. + friend class Layout; + + NamespaceLifeId(RootNamespace ns_, const UInt128 & incarnation_) + : ns(std::move(ns_)), incarnation(incarnation_) + { + if (incarnation_ == 0) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, + "NamespaceLifeId: incarnation must be nonzero for namespace '{}' -- 0 never names a life", + ns.string()); + } +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasTypes.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasTypes.h new file mode 100644 index 000000000000..84f24dba3f73 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasTypes.h @@ -0,0 +1,331 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +/// Strongly typed value types and codecs shared by the content-addressed metadata and object paths. +/// +/// These types deliberately keep protocol identity, content digests, backend tokens, and their +/// textual forms distinct. Their ordering and equality operators make them usable as ordered keys; +/// hash specializations below support unordered containers. Conversion to a storage key or wire +/// representation remains explicit at the boundary that owns that representation. +/// +/// Opaque namespace under which root manifests live. The core never interprets its contents: +/// the wiring composes strings like "srv1/" or "shadow//". +/// Layout only validates its shape (non-empty, no leading/trailing or empty segments, +/// no segment equal to the reserved "_files"). +/// Construction from `String` is explicit, preventing unrelated identifier types from being mixed; +/// the underlying string is exposed only through `string` at object-storage boundaries. +class RootNamespace +{ +public: + RootNamespace() = default; + explicit RootNamespace(String value_) : value(std::move(value_)) {} + const String & string() const { return value; } + auto operator<=>(const RootNamespace &) const = default; + bool operator==(const RootNamespace &) const = default; + +private: + String value; +}; + +/// Returns 32 lowercase hex chars encoding `v`. +inline String u128ToHex(const UInt128 & v) +{ + return getHexUIntLowercase(v); +} + +/// Parses 32-char lowercase (or uppercase) hex string to UInt128. +/// Throws BAD_ARGUMENTS on wrong length or non-hex characters. +inline UInt128 hexToU128(const String & hex) +{ + if (hex.size() != 32) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "hexToU128: expected 32 hex chars, got {}", hex.size()); + + // Validate each character is a valid hex digit (unhex(c) returns 0xff for invalid chars). + for (char c : hex) + { + if (::unhex(c) == 0xff) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "hexToU128: invalid hex character '{}'", c); + } + + return unhexUInt(hex.data()); +} + +} + +namespace std +{ + +template <> +struct hash +{ + size_t operator()(const DB::Cas::RootNamespace & value) const noexcept + { + return std::hash{}(value.string()); + } +}; + +} + +namespace DB::Cas +{ + +/// The compact reference a root journal stores for a part manifest. It is not a string key: +/// `CasLayout::manifestKey` derives the object key from this reference and the owning namespace. +/// The namespace is deliberately absent because it comes from the owning root context and must not +/// be serialized into the journal reference. +/// +/// writer_epoch - durable monotone writer epoch allocated under the mounted `server_root_id`; +/// never reused for that server root. +/// build_sequence - monotone build sequence inside one writer incarnation; part of identity +/// and of the build-scoped debris prefix. +/// manifest_ordinal - monotone ordinal inside one build, rendered as `000001.zst` in the key. +struct ManifestRef +{ + uint64_t writer_epoch = 0; + uint64_t build_sequence = 0; + uint32_t manifest_ordinal = 0; + + bool operator==(const ManifestRef & o) const = default; + + /// Total order for std::map / std::set keys. Field order is arbitrary but stable. + bool operator<(const ManifestRef & o) const + { + return std::tie(writer_epoch, build_sequence, manifest_ordinal) + < std::tie(o.writer_epoch, o.build_sequence, o.manifest_ordinal); + } +}; + +/// The namespace-qualified protocol identity used by GC for source edges, blob deltas, cleanup work, +/// and addressing. It is the pair `(root_namespace, ManifestRef)`. +/// Two namespaces may legally carry the same `ManifestRef` tuple without addressing the same object; +/// therefore those structures must use `ManifestId`, never `ManifestRef` alone. +struct ManifestId +{ + RootNamespace root_namespace; /// owning namespace; NOT part of the journal ref + ManifestRef ref; + + bool operator==(const ManifestId & o) const = default; + + bool operator<(const ManifestId & o) const + { + if (root_namespace.string() != o.root_namespace.string()) + return root_namespace.string() < o.root_namespace.string(); + return ref < o.ref; + } +}; + +inline constexpr uint32_t kMaxManifestOrdinal = 999999; + +/// Six-digit filename for a per-build part-manifest ordinal: `000001.zst` through `999999.zst` +/// (the registered v3 stored suffix for `FormatId::PartManifest`). `0` is reserved as an invalid +/// sentinel and is never emitted. +inline String manifestOrdinalFileName(uint32_t manifest_ordinal) +{ + if (manifest_ordinal == 0 || manifest_ordinal > kMaxManifestOrdinal) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "Manifest ordinal must be in [1, {}], got {}", kMaxManifestOrdinal, manifest_ordinal); + return fmt::format("{:06}{}", manifest_ordinal, storedSuffix(FormatId::PartManifest)); +} + +/// The canonical `writer_epoch:build_sequence:manifest_ordinal` text form of a manifest reference. +/// It is not an object-key encoding: keys are derived by `CasLayout::manifestKey`. Besides logs and +/// diagnostics it is the wire form of the relink confirm token (spec §confirm-primitive), so +/// `tryParseManifestRef` below is its exact inverse and the two must be changed together. +inline String manifestRefDebugString(const ManifestRef & ref) +{ + return fmt::format("{}:{}:{}", ref.writer_epoch, ref.build_sequence, ref.manifest_ordinal); +} + +/// Parses the canonical text form produced by `manifestRefDebugString` back into a reference. +/// +/// Everything that is not EXACTLY three colon-separated decimal fields in range is `nullopt`. The text +/// arrives from a remote peer in the relink confirm token, so it is untrusted: `from_chars` is used +/// precisely because it accepts no sign, no whitespace and no partial consumption, and the ordinal is +/// range-checked the same way `manifestOrdinalFileName` checks it (`0` is the reserved invalid +/// sentinel, never emitted, so a token carrying it can only be malformed or forged). A refusal here is +/// never a `No`: the caller cannot tell what was asked, which is an ambiguity, not knowledge. +inline std::optional tryParseManifestRef(std::string_view text) +{ + const auto parse_field = [](std::string_view field, auto & out) -> bool + { + if (field.empty()) + return false; + const auto * const begin = field.data(); + const auto * const end = field.data() + field.size(); + const auto result = std::from_chars(begin, end, out); + return result.ec == std::errc{} && result.ptr == end; + }; + + const size_t first = text.find(':'); + if (first == std::string_view::npos) + return std::nullopt; + const size_t second = text.find(':', first + 1); + if (second == std::string_view::npos || text.find(':', second + 1) != std::string_view::npos) + return std::nullopt; + + ManifestRef ref; + if (!parse_field(text.substr(0, first), ref.writer_epoch) + || !parse_field(text.substr(first + 1, second - first - 1), ref.build_sequence) + || !parse_field(text.substr(second + 1), ref.manifest_ordinal)) + return std::nullopt; + if (ref.manifest_ordinal == 0 || ref.manifest_ordinal > kMaxManifestOrdinal) + return std::nullopt; + return ref; +} + +} + +/// Hash specializations for the identity types. Each combines exactly the fields used by its +/// corresponding `operator==`, so equal keys always have equal hashes. +namespace std +{ + +template <> +struct hash +{ + size_t operator()(const DB::Cas::ManifestRef & r) const + { + const size_t h1 = std::hash{}(r.writer_epoch); + const size_t h2 = std::hash{}(r.build_sequence); + const size_t h3 = std::hash{}(r.manifest_ordinal); + size_t h = h1; + h ^= h2 + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2); + h ^= h3 + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2); + return h; + } +}; + +template <> +struct hash +{ + size_t operator()(const DB::Cas::ManifestId & id) const + { + const size_t h1 = std::hash<::String>{}(id.root_namespace.string()); + const size_t h2 = std::hash{}(id.ref); + return h1 ^ (h2 + 0x9e3779b97f4a7c15ULL + (h1 << 6) + (h1 >> 2)); + } +}; + +} + +namespace DB::Cas +{ + +/// How a backend identifies one physical incarnation of an object key. +enum class TokenType : uint8_t +{ + ETag = 1, /// S3-family / Azure + Generation = 2, /// GCS (binding deferred; fail-closed until probed) + Emulated = 3, /// test backends (in-memory fake, Local emulation) +}; + +/// A backend-native incarnation token. Opaque; sent back to the backend EXACTLY as observed. +struct Token +{ + String value; + TokenType type = TokenType::ETag; + + bool empty() const { return value.empty(); } + bool operator==(const Token &) const = default; +}; + +/// The ordered ref-transaction identifier. A successful writer mount establishes a strictly newer +/// `writer_epoch`; within an epoch, one namespace's `ref_sequence` values are CONTIGUOUS from 1 -- +/// derived per append from the table's own greatest applied id (`nextRefTxnId`), not drawn from any +/// counter, so two namespaces of the same mount both count 1, 2, 3... independently. Both fields +/// are nonzero for a valid id -- {0, 0} is never a real transaction. `writer_epoch` is the primary +/// ordering component, so tuple order matches the intended timeline even across an epoch restart that +/// resets `ref_sequence` back to one. +struct RefTxnId +{ + uint64_t writer_epoch = 0; + uint64_t ref_sequence = 0; + + auto operator<=>(const RefTxnId &) const = default; +}; + +/// Renders the canonical form: two fixed-width, lower-case, 16-digit hexadecimal numbers joined by +/// '-' (e.g. "0000000000000007-000000000000008e"). Lexical order of the render equals tuple order of +/// `id`, because '-' (0x2d) sorts below every hex digit character and both fields are fixed-width. +/// Throws LOGICAL_ERROR if either field is zero: this render becomes an object key, and an invalid id +/// must never silently produce a well-formed-looking one. +inline String renderRefTxnId(const RefTxnId & id) +{ + if (id.writer_epoch == 0 || id.ref_sequence == 0) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, + "RefTxnId: writer_epoch and ref_sequence must both be nonzero, got {}-{}", + id.writer_epoch, id.ref_sequence); + return getHexUIntLowercase(id.writer_epoch) + "-" + getHexUIntLowercase(id.ref_sequence); +} + +/// Parses the canonical form only: exactly 33 characters, '-' at index 16, exactly 16 lower-case hex +/// digits ('0'-'9', 'a'-'f') either side, and both parsed fields nonzero. Any other shape -- short, +/// long, upper-case, non-hex, misplaced separator, or a zero component -- returns nullopt rather than +/// throwing, since parsing an untrusted listed key is an ordinary "is this ours" question. +inline std::optional parseRefTxnId(std::string_view s) +{ + constexpr size_t kFieldLen = 16; + constexpr size_t kTotalLen = kFieldLen * 2 + 1; + if (s.size() != kTotalLen || s[kFieldLen] != '-') + return std::nullopt; + + /// Strict lower-case-hex-only parse: `unhexUInt` (base/hex.h) also accepts upper-case, which the + /// canonical form must reject, so digits are validated and accumulated by hand here. + const auto parseField = [](std::string_view field) -> std::optional + { + uint64_t value = 0; + for (char c : field) + { + uint64_t digit = 0; + if (c >= '0' && c <= '9') + digit = static_cast(c - '0'); + else if (c >= 'a' && c <= 'f') + digit = static_cast(c - 'a') + 10; + else + return std::nullopt; + value = (value << 4) | digit; + } + return value; + }; + + const auto epoch = parseField(s.substr(0, kFieldLen)); + const auto seq = parseField(s.substr(kFieldLen + 1, kFieldLen)); + if (!epoch || !seq || *epoch == 0 || *seq == 0) + return std::nullopt; + return RefTxnId{*epoch, *seq}; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasXxh3Streamer.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasXxh3Streamer.h new file mode 100644 index 000000000000..ff3b1e3c1a8d --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasXxh3Streamer.h @@ -0,0 +1,83 @@ +#pragma once + +/// Isolated include wrapper + tiny helper API for the xxHash XXH3-128 hash used by `CasBlobHashingWriteBuffer`. +/// +/// Two problems this header contains in one place: +/// 1. In the `dbms` target a plain `#include ` resolves to lz4's bundled copy +/// (`contrib/lz4/lib` is a higher-priority `-I` than the `-isystem contrib/xxHash`), and that copy +/// provides only XXH32/64 — NOT the XXH3 API. So xxHash is referenced by an explicit repo-relative +/// path (from this file's directory up to the repo root) which unambiguously picks the full +/// standalone `contrib/xxHash` that has XXH3. +/// 2. `XXH_INLINE_ALL` makes xxHash a header-only static-inline implementation whose vendored C is not +/// clean under the CAS `-Werror -Weverything` flags, and whose inline functions carry an "unused" +/// attribute that trips `-Wused-but-marked-unused` at every call site. `#pragma clang system_header` +/// marks the rest of THIS header (and everything it includes, plus the helper calls below) as a +/// system header, so ALL of those warnings are suppressed here — without disabling warnings for any +/// real `CasBlobHashingWriteBuffer` code, which only ever touches the clean `DB::Cas` helpers defined below. +/// +/// The explicit include is a build-selection detail: it does not change the XXH3-128 algorithm or +/// the digest representation exposed to CAS. Keeping the dependency and warning suppression here +/// prevents callers from depending on either vendored implementation directly. +#pragma clang system_header + +#include +#include + +#define XXH_INLINE_ALL +#include "../../../../../../contrib/xxHash/xxhash.h" + +namespace DB::Cas +{ + +/// Owns one streaming XXH3-128 state and exposes only the operations needed by `CasBlobHashingWriteBuffer`. +/// The state is allocated by the constructor and released by the destructor; the wrapper is +/// deliberately non-copyable because copying an xxHash state would make ownership and continuation +/// semantics ambiguous. Callers normally check `valid` immediately after construction, then feed +/// the complete byte sequence with `update` before retrieving the digest with `digest`. +/// +/// All raw xxHash symbols are confined to this system header, so callers see no xxHash warnings. +class Xxh3Streamer +{ +public: + /// Allocates and resets a fresh state. `valid` reports allocation failure; callers must not call + /// `update` or `digest` on an invalid wrapper. + Xxh3Streamer() : state(XXH3_createState()) { XXH3_128bits_reset(state); } + + /// Releases the state owned by this wrapper. It does not perform or publish a digest. + ~Xxh3Streamer() { XXH3_freeState(state); } + + Xxh3Streamer(const Xxh3Streamer &) = delete; + Xxh3Streamer & operator=(const Xxh3Streamer &) = delete; + + /// Returns whether the constructor obtained an xxHash state successfully. + bool valid() const { return state != nullptr; } + + /// Adds the next byte range to the running digest. The range must remain readable for the + /// duration of the call; it is consumed immediately and is not retained by the wrapper. + void update(const void * data, size_t len) { XXH3_128bits_update(state, data, len); } + + /// Writes the current 128-bit digest into its low and high 64-bit halves. This does not reset + /// the state, so it can be used for inspection before the stream is destroyed; callers should + /// finish all `update` calls before relying on the result. + void digest(UInt64 & low, UInt64 & high) const + { + const XXH128_hash_t d = XXH3_128bits_digest(state); + low = d.low64; + high = d.high64; + } + +private: + XXH3_state_t * state; +}; + +/// Hashes one byte range with XXH3-128 and writes the digest into its low and high 64-bit halves. +/// The input is consumed during the call and no state is retained. This is the one-shot counterpart +/// to `Xxh3Streamer` and is used as the reference path for the streaming CAS hash. +inline void xxh3_128_oneshot(const void * data, size_t len, UInt64 & low, UInt64 & high) +{ + const XXH128_hash_t d = XXH3_128bits(data, len); + low = d.low64; + high = d.high64; +} + +} From 7abc22a333ee5b129d8682fc77eed1af23f8335d Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:33 +0200 Subject: [PATCH 14/30] CAS subsystem: Formats layer On-wire/on-disk encodings: manifests, ref-log records, GC state, seals, codecs. Depends only on Primitives. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../Formats/CasBlobEnvelopeFormat.cpp | 253 ++++++++ .../Formats/CasBlobEnvelopeFormat.h | 110 ++++ .../Formats/CasBlobMetaFormat.cpp | 94 +++ .../Formats/CasBlobMetaFormat.h | 46 ++ .../ContentAddressed/Formats/CasByteBudget.h | 57 ++ .../Formats/CasFoldSealFormat.cpp | 546 ++++++++++++++++++ .../Formats/CasFoldSealFormat.h | 225 ++++++++ .../ContentAddressed/Formats/CasFormat.cpp | 198 +++++++ .../ContentAddressed/Formats/CasFormat.h | 203 +++++++ .../Formats/CasGcMaintenanceStateFormat.cpp | 67 +++ .../Formats/CasGcMaintenanceStateFormat.h | 21 + .../Formats/CasGcOutcomesFormat.cpp | 129 +++++ .../Formats/CasGcOutcomesFormat.h | 58 ++ .../Formats/CasGcStateFormat.cpp | 118 ++++ .../Formats/CasGcStateFormat.h | 71 +++ .../ContentAddressed/Formats/CasLayout.cpp | 345 +++++++++++ .../ContentAddressed/Formats/CasLayout.h | 480 +++++++++++++++ .../Formats/CasPartManifestFormat.cpp | 353 +++++++++++ .../Formats/CasPartManifestFormat.h | 120 ++++ .../Formats/CasPoolMetaFormat.cpp | 182 ++++++ .../Formats/CasPoolMetaFormat.h | 91 +++ .../Formats/CasRecordStreamFormat.cpp | 325 +++++++++++ .../Formats/CasRecordStreamFormat.h | 164 ++++++ .../Formats/CasRefCatalogFormat.cpp | 397 +++++++++++++ .../Formats/CasRefCatalogFormat.h | 168 ++++++ .../Formats/CasRefCkptFormat.cpp | 179 ++++++ .../Formats/CasRefCkptFormat.h | 124 ++++ .../Formats/CasRefLogFormat.cpp | 443 ++++++++++++++ .../Formats/CasRefLogFormat.h | 177 ++++++ .../Formats/CasRefSnapshotFormat.cpp | 305 ++++++++++ .../Formats/CasRefSnapshotFormat.h | 95 +++ .../Formats/CasRefWireVocab.cpp | 47 ++ .../Formats/CasRefWireVocab.h | 65 +++ .../Formats/CasServerRootFormats.cpp | 178 ++++++ .../Formats/CasServerRootFormats.h | 93 +++ .../Formats/CasTextFormat.cpp | 414 +++++++++++++ .../ContentAddressed/Formats/CasTextFormat.h | 241 ++++++++ .../ContentAddressed/Formats/CasWireVocab.cpp | 103 ++++ .../ContentAddressed/Formats/CasWireVocab.h | 62 ++ .../ContentAddressed/Formats/README.md | 63 ++ 40 files changed, 7410 insertions(+) create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasByteBudget.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefWireVocab.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefWireVocab.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasWireVocab.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasWireVocab.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.cpp new file mode 100644 index 000000000000..65176572e896 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.cpp @@ -0,0 +1,253 @@ +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +namespace +{ + +constexpr std::string_view kBlobType = "cas_blob"; + +std::string_view opToWord(ProvenanceOp op) +{ + switch (op) + { + case ProvenanceOp::Other: return "other"; + case ProvenanceOp::Insert: return "insert"; + case ProvenanceOp::Merge: return "merge"; + case ProvenanceOp::Mutation: return "mutation"; + case ProvenanceOp::Attach: return "attach"; + case ProvenanceOp::Repack: return "repack"; + } + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS blob envelope: unknown ProvenanceOp {}", static_cast(op)); +} + +ProvenanceOp opFromWord(std::string_view w) +{ + if (w == "other") return ProvenanceOp::Other; + if (w == "insert") return ProvenanceOp::Insert; + if (w == "merge") return ProvenanceOp::Merge; + if (w == "mutation") return ProvenanceOp::Mutation; + if (w == "attach") return ProvenanceOp::Attach; + if (w == "repack") return ProvenanceOp::Repack; + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS blob envelope: unknown op '{}'", w); +} + +/// The escaped byte-length of one raw ref char under the frozen envelope alphabet (see writeEnvelopeRefField). +size_t escapedLen(char c) +{ + const unsigned char u = static_cast(c); + if (c == '"' || c == '\\') + return 2; + if (u < 0x20) + return 6; /// \uXXXX + return 1; /// everything else, INCLUDING '/', verbatim +} + +void appendEscaped(String & out, char c) +{ + const unsigned char u = static_cast(c); + if (c == '"') { out += "\\\""; return; } + if (c == '\\') { out += "\\\\"; return; } + if (u < 0x20) + { + static constexpr char hexd[] = "0123456789abcdef"; + out += "\\u00"; + out += hexd[(u >> 4) & 0xF]; + out += hexd[u & 0xF]; + return; + } + out += c; +} + +/// The blob-envelope's OWN ref-string writer. DELIBERATELY NOT `writeStringValue` and MUST NOT be +/// "unified" with it: the 256-byte header budget arithmetic and the stored blob bytes depend on this +/// alphabet being codec-owned and FROZEN — only `"`, `\`, and control chars (< 0x20, as `\uXXXX`) +/// escape; `/` and every other byte pass verbatim. (`writeStringValue`/`FormatSettings::JSON` may +/// legitimately evolve for the control-plane formats; this codec must not inherit that.) Writes the +/// opening quote, the ref content escaped and truncated to at most `budget` escaped bytes (stopping at +/// the first char that would overflow — never splitting an escape), then the closing quote. +void writeEnvelopeRefField(String & json, size_t budget, std::string_view raw_ref) +{ + json += '"'; + size_t used = 0; + for (char c : raw_ref) + { + const size_t need = escapedLen(c); + if (used + need > budget) + break; + appendEscaped(json, c); + used += need; + } + json += '"'; +} + +} + +String encodeEnvelopeHeader(EnvelopeHeader & header, uint32_t blob_header_len) +{ + if (header.kind != ObjectKind::Blob) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS blob envelope: unexpected ObjectKind {}", static_cast(header.kind)); + + /// Build every field EXCEPT `ref` into a buffer (small, bounded by blob_header_len). `ref` is the + /// only truncated field, appended last so the truncation never disturbs another field. + String json; + { + CasJsonWriter buf(256); + bool first = true; + writeKey(buf, "type", first); writeStringValue(buf, kBlobType); + writeKey(buf, "v", first); writeIntText(currentCompatibilityVersion(), buf); + writeKey(buf, "tag", first); writeHex128Value(buf, header.incarnation_tag); + writeKey(buf, "bld", first); writeHex128Value(buf, header.build_id); + if (header.provenance) + { + writeKey(buf, "ts", first); writeIntText(header.provenance->created_at_ms, buf); + writeKey(buf, "by", first); writeHex128Value(buf, header.provenance->creator_server_id); + writeKey(buf, "op", first); writeStringValue(buf, opToWord(header.provenance->op)); + writeKey(buf, "ch", first); writeIntText(header.provenance->ch_version, buf); + } + /// Test-only critical extension: an unknown `!`-key BEFORE `ref`. + if (header.emit_unknown_critical_key) + { + writeKey(buf, "!x", first); writeStringValue(buf, "1"); + } + json = std::move(buf).take(); /// e.g. {"type":"cas_blob","v":3,...,"ch":26006001 (no ref, no closing brace) + } + + /// Optional `ref`, truncated to the exact remaining budget. Layout after this block: + /// json + `,"ref":` + `"` + + `"` + `}` must be <= blob_header_len-1 + /// (byte blob_header_len-1 is reserved for '\n'; the pad zone fills the gap with spaces). + if (header.intended_ref) + { + static constexpr std::string_view ref_key = ",\"ref\":"; + /// +3 = opening quote + closing quote + closing brace. + const size_t fixed = json.size() + ref_key.size() + 3; + if (blob_header_len < 1 || fixed > static_cast(blob_header_len) - 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS blob envelope: non-ref fields ({} bytes) do not fit blob_header_len {} before the ref", + fixed, blob_header_len); + const size_t budget = (static_cast(blob_header_len) - 1) - fixed; + json += ref_key; + writeEnvelopeRefField(json, budget, *header.intended_ref); + } + json += '}'; + + if (json.size() > static_cast(blob_header_len) - 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS blob envelope: header object {} bytes exceeds blob_header_len {} - 1", + json.size(), blob_header_len); + + /// Space pad to byte blob_header_len-2, then '\n' at byte blob_header_len-1. + String out = std::move(json); + out.append((static_cast(blob_header_len) - 1) - out.size(), ' '); + out += '\n'; + + header.header_len = blob_header_len; + return out; +} + +EnvelopeHeader decodeEnvelopeHeader(std::string_view head_bytes, uint64_t /*object_size*/, ObjectKind expected_kind) +{ + ReadBufferFromMemory in(head_bytes.data(), head_bytes.size()); + JsonObjectReader r(in, KeyStrictness::Tolerant, "blob envelope"); + + EnvelopeHeader h; + h.kind = ObjectKind::Blob; + bool saw_type = false; + bool saw_v = false; + bool have_prov = false; + Provenance prov; + String key; + while (r.nextKey(key)) + { + if (key == "type") + { + const String t = r.readString(); + if (t != kBlobType) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS blob envelope: object is a '{}', not a '{}'", t, kBlobType); + saw_type = true; + } + else if (key == "v") + { + h.compatibility_version = r.readU32Number(); + checkCompatibility(h.compatibility_version, "blob envelope"); + saw_v = true; + } + else if (key == "tag") + h.incarnation_tag = r.readHex128(); + else if (key == "bld") + h.build_id = r.readHex128(); + else if (key == "ts") + { + prov.created_at_ms = r.readU64Number(); + have_prov = true; + } + else if (key == "by") + { + prov.creator_server_id = r.readHex128(); + have_prov = true; + } + else if (key == "op") + { + prov.op = opFromWord(r.readString()); + have_prov = true; + } + else if (key == "ch") + { + prov.ch_version = static_cast(r.readU64Number()); + have_prov = true; + } + else if (key == "ref") + h.intended_ref = r.readString(); + else + r.skipUnknown(key); /// `!`-key -> UNKNOWN_FORMAT_VERSION; unknown plain key -> skipped (tolerant) + } + if (!saw_type) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS blob envelope: missing type"); + if (!saw_v) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS blob envelope: missing v"); + if (h.kind != expected_kind) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS blob envelope: kind {} does not match expected {}", + static_cast(h.kind), static_cast(expected_kind)); + if (have_prov) + h.provenance = prov; + + /// Pad-verify: JsonObjectReader consumed through the closing '}', so in.count() == json_len. Every + /// byte up to the terminating '\n' must be an ASCII space (no smuggling); header_len is DERIVED from + /// the '\n' position (blob_header_len is never passed to decode). + while (true) + { + if (in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS blob envelope: pad zone has no '\\n' terminator"); + const char c = *in.position(); + ++in.position(); + if (c == '\n') + { + h.header_len = static_cast(in.count()); + break; + } + if (c != ' ') + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS blob envelope: non-space byte 0x{:02x} in the header pad zone", static_cast(static_cast(c))); + } + + return h; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.h new file mode 100644 index 000000000000..19250fe69ddd --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobEnvelopeFormat.h @@ -0,0 +1,110 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Identifies the kind of content represented by an envelope. `Blob` is currently the only envelope +/// kind, but the enum remains a distinct switch-friendly type because callers use the kind as part of +/// their object and audit-event vocabulary. +enum class ObjectKind : uint8_t +{ + Blob = 1, +}; + +/// Describes the operation that produced an envelope's optional diagnostic provenance. It is metadata +/// for inspection and attribution only; readers do not use it to make storage or compatibility +/// decisions. +enum class ProvenanceOp : uint8_t +{ + Other = 0, + Insert = 1, + Merge = 2, + Mutation = 3, + Attach = 4, + Repack = 5, +}; + +/// Optional diagnostic metadata recorded with an envelope. The fields identify when and where the +/// incarnation was created, the ClickHouse build that wrote it, and the operation that produced it; +/// none of them participates in object identity or a protocol decision. +struct Provenance +{ + uint64_t created_at_ms = 0; + UInt128 creator_server_id{}; + uint32_t ch_version = 0; + ProvenanceOp op = ProvenanceOp::Other; +}; + +/// The blob envelope is a fixed-size JSON descriptor followed by the raw payload. The JSON object +/// occupies bytes [0, json_len), ASCII spaces occupy [json_len, blob_header_len-1), and '\n' is at byte +/// blob_header_len-1. The payload therefore begins at the pool-wide constant offset +/// `blob_header_len` (256 for blob pools, a `PoolMeta` parameter), allowing the locate path to use a +/// constant shift without reading an object-specific header first. The header is also the incarnation +/// zone: it may differ between incarnations of one logical object, and each upload attempt gets a fresh +/// random u128 `tag`, which is used as the exact-token delete identity. +/// +/// The envelope intentionally does not duplicate identity or unused integrity metadata. The identity +/// algorithm and digest are already present in the object key and manifest reference, `domain_id` had +/// no validating consumer, and `header_hash` had no consumer once the CityHash64 check left the +/// envelope. Writer forensics are represented +/// by `ch` and `bld`, so a separate `writer_version` is unnecessary. The `v` field is the sole format +/// compatibility gate; a reader rejects a version it does not understand before interpreting the body. +struct EnvelopeHeader +{ + ObjectKind kind = ObjectKind::Blob; + /// Set by decode from the header `v`; encode stamps `currentCompatibilityVersion`. A reader + /// fails closed (UNKNOWN_FORMAT_VERSION) when `v` exceeds what this build understands. + uint32_t compatibility_version = 0; + UInt128 incarnation_tag{}; /// `tag` + UInt128 build_id{}; /// `bld` + std::optional provenance; /// `ts` / `by` / `op` / `ch` + std::optional intended_ref; /// `ref` (diagnostic; truncated on encode to fit the header) + uint32_t header_len = 0; /// filled by encode/decode = blob_header_len (payload offset) + /// Test-only knob: emit an unknown `!`-critical key. Decoding the resulting header must fail + /// closed with `UNKNOWN_FORMAT_VERSION`, exercising the compatibility rule for critical extensions. + bool emit_unknown_critical_key = false; +}; + +/// Builds the fixed-length header for a pool whose `blob_header_len` is `blob_header_len` (256 for blob +/// pools). Sets `header.header_len = blob_header_len` and returns exactly that many bytes. The +/// diagnostic `ref` is the only truncatable field and is shortened, never dropped, when necessary to +/// preserve the fixed layout. The header is built without payload bytes, so an upload can stage the +/// header before the payload is streamed. +String encodeEnvelopeHeader(EnvelopeHeader & header, uint32_t blob_header_len); + +/// Parses and validates the JSON descriptor, its expected `type`, and its compatibility version. +/// Derives `header_len` from the terminating '\n' and requires every preceding byte in the pad zone to +/// be an ASCII space, preventing bytes from being smuggled between the descriptor and payload. Malformed +/// type, padding, or truncation produces `CORRUPTED_DATA`; a future `v` or unknown `!`-prefixed critical +/// key produces `UNKNOWN_FORMAT_VERSION`. `object_size` is accepted for symmetry with read and GC call +/// sites; the payload length is derived downstream as `object_size - header_len`, so this function does +/// not otherwise inspect it. +EnvelopeHeader decodeEnvelopeHeader(std::string_view head_bytes, uint64_t object_size, ObjectKind expected_kind); + +/// Payload starts right after the header. +inline uint64_t payloadOffset(const EnvelopeHeader & header) +{ + return static_cast(header.header_len); +} + +/// Map an internal `ObjectKind` to the audit-log `CasEventObjectKind`. Single source for the mapping +/// previously open-coded as a ternary at each emission site. Lives here (Formats) rather than in +/// CasEvent.h (Primitives) so the include direction Formats -> Primitives is respected and +/// `CasEvent` stays dependency-free. +inline CasEventObjectKind toEventKind(ObjectKind kind) +{ + switch (kind) + { + case ObjectKind::Blob: return CasEventObjectKind::Blob; + } +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.cpp new file mode 100644 index 000000000000..b62fd3b82424 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.cpp @@ -0,0 +1,94 @@ +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +namespace +{ + +std::string_view metaStateToWord(MetaState s) +{ + switch (s) + { + case MetaState::Clean: return "clean"; + case MetaState::Condemned: return "condemned"; + } + // The enum is persisted as a closed vocabulary. Do not silently invent a spelling for a value + // added without a corresponding format decision: that would make the writer emit data older + // readers cannot classify. + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS blob meta: unknown MetaState {}", static_cast(s)); +} + +MetaState metaStateFromWord(std::string_view w) +{ + if (w == "clean") return MetaState::Clean; + if (w == "condemned") return MetaState::Condemned; + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS blob meta: unknown state '{}'", w); +} + +} + +String encodeBlobMeta(const BlobMeta & meta) +{ + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::BlobMeta); + // `version` is represented by the header line. The JSON body contains only fields that describe + // the current marker and its accounting data. + bool first = true; + writeKey(out, "st", first); + writeStringValue(out, metaStateToWord(meta.state)); + writeKey(out, "cr", first); + writeU64StringValue(out, meta.condemn_round); + writeKey(out, "sz", first); + writeU64StringValue(out, meta.size); + closeObject(out, first); + writeChar('\n', out); + return std::move(out).take(); +} + +BlobMeta decodeBlobMeta(std::string_view bytes) +{ + ReadBufferFromMemory in(bytes.data(), bytes.size()); + expectHeaderLine(in, FormatId::BlobMeta); + const String body = readLine(in, traitsFor(FormatId::BlobMeta).line_cap, "blob meta"); + ReadBufferFromMemory body_in(body.data(), body.size()); + JsonObjectReader r(body_in, KeyStrictness::Tolerant, "blob meta"); + + // Start with the documented defaults. In particular, `version` stays at 1 because the header's + // version is authoritative and is not copied into the body struct. + BlobMeta m; + bool saw_state = false; + String key; + while (r.nextKey(key)) + { + if (key == "st") + { + m.state = metaStateFromWord(r.readString()); + saw_state = true; + } + else if (key == "cr") + m.condemn_round = r.readU64String(); + else if (key == "sz") + m.size = r.readU64String(); + else + r.skipUnknown(key); + } + if (!saw_state) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS blob meta: missing st"); + if (!body_in.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS blob meta: trailing bytes"); + return m; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.h new file mode 100644 index 000000000000..bf1fa832a3b5 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasBlobMetaFormat.h @@ -0,0 +1,46 @@ +#pragma once +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// The two states of the per-hash freshness marker used by the writer's deduplication gate and by GC. +/// This marker is only a point-read hint, not the linearization point for blob lifetime: the body's +/// in-body `incarnation_tag` and the body's exact-token delete provide the safety guarantee. A stale +/// marker can therefore make a writer re-upload conservatively, but it is never authority for deleting +/// the body. +enum class MetaState : uint8_t +{ + Clean = 0, /// The body is present and may be referenced. + Condemned = 1, /// GC observed zero in-degree; the body remains present until exact-token deletion, + /// so a writer may resurrect it by replacing the body and updating this marker. +}; + +/// The durable per-hash meta record. Its text representation consists of a format header followed by +/// one JSON object with the state word, the GC condemnation round, and the raw body size. `size` is +/// retained for introspection, fsck, and GC accounting; reads of the blob never consult the meta. +/// Lifecycle transitions are conditional on the backend etag, while the encoded bytes themselves are +/// not compared. The body header's `v` is the authoritative format version, so `version` remains only +/// for the inspection interface and is deliberately not serialized in the JSON body. +struct BlobMeta +{ + uint8_t version = 1; + MetaState state = MetaState::Clean; + uint64_t condemn_round = 0; /// The GC round that condemned this blob; distinguishes a stale + /// condemnation from a later spare-and-recondemn transition. + uint64_t size = 0; +}; + +/// Serialize `meta` as the header line and one JSON body line. Invalid `MetaState` values are rejected +/// with `CORRUPTED_DATA`; the body does not contain `version` because the header owns format versioning. +String encodeBlobMeta(const BlobMeta & meta); + +/// Decode a stored meta record. The header, required state, field types, and complete input are checked; +/// malformed input, an unknown state, or trailing bytes throws `CORRUPTED_DATA`. Unknown JSON keys are +/// tolerated so the format can add nonessential fields without breaking older readers. +BlobMeta decodeBlobMeta(std::string_view bytes); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasByteBudget.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasByteBudget.h new file mode 100644 index 000000000000..28e5900f7674 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasByteBudget.h @@ -0,0 +1,57 @@ +#pragma once +#include + +namespace DB::Cas +{ + +/// The byte arithmetic every write-once text control object owes BEFORE its PUT. +/// +/// Two caps, deliberately kept apart (they answer different questions and fail differently): +/// +/// * the LINE cap bounds ONE encoded record. A record longer than it is not merely large, it is +/// UNREADABLE: the streaming reader refuses the line, so an object containing one can never be +/// decoded again. This predicate belongs at the point a record is emitted. +/// * the OBJECT cap bounds the whole object. It is the ADDITIVE question — a fixed frame (header, +/// meta, trailer) plus the worst-case reservation of every entry — because a producer that must +/// decide whether one MORE entry still fits cannot encode first and measure afterwards. +/// +/// Both predicates accept at EQUALITY: a cap is the largest permitted value, not the first forbidden +/// one, and the readers enforce it the same way. +/// +/// Additions AND multiplications saturate. A modular sum or product that wrapped would answer "fits" +/// for an object that does not, turning an overflow into a durable unreadable object — the one +/// outcome the caps exist to prevent. + +/// `a + b`, clamped to `UINT64_MAX` instead of wrapping. +constexpr uint64_t addByteBudget(uint64_t a, uint64_t b) +{ + return a > UINT64_MAX - b ? UINT64_MAX : a + b; +} + +/// `a * b`, clamped to `UINT64_MAX` instead of wrapping -- the same saturation discipline as +/// `addByteBudget`, one step earlier: a per-entry byte cost multiplied by an entry count before the +/// caller adds it to a fixed frame (`fitsObjectCap`'s second argument) can overflow first, and a +/// wrapped product can land BELOW the true fixed-plus-reservation sum even though the real total is +/// far over it -- the exact "answers fits for an object that does not" failure this file exists to +/// prevent, one arithmetic step upstream of the sum itself. +constexpr uint64_t mulByteBudget(uint64_t a, uint64_t b) +{ + return a != 0 && b > UINT64_MAX / a ? UINT64_MAX : a * b; +} + +/// LINE predicate: `encoded_row_bytes <= line_cap`, measured EXCLUDING the '\n' terminator (the same +/// bytes `readLine` measures). `line_cap == 0` means the format declares no line cap. +constexpr bool fitsLineCap(uint64_t encoded_row_bytes, uint64_t line_cap) +{ + return line_cap == 0 || encoded_row_bytes <= line_cap; +} + +/// OBJECT predicate: `fixed_bytes + entries_reservation <= object_cap`, evaluated with saturating +/// addition. `object_cap == 0` means the format declares no whole-object cap (a streamed format that +/// is never materialized whole). +constexpr bool fitsObjectCap(uint64_t fixed_bytes, uint64_t entries_reservation, uint64_t object_cap) +{ + return object_cap == 0 || addByteBudget(fixed_bytes, entries_reservation) <= object_cap; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.cpp new file mode 100644 index 000000000000..b4bba2bff08f --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.cpp @@ -0,0 +1,546 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int LIMIT_EXCEEDED; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +std::string_view holdReasonToWord(HoldReason r) +{ + switch (r) + { + case HoldReason::GapBelowWitness: return "gap_below_witness"; + case HoldReason::UnconsumedSealCrossing: return "unconsumed_seal_crossing"; + case HoldReason::WitnessDisappeared: return "witness_disappeared"; + case HoldReason::BodyUndecodable: return "body_undecodable"; + case HoldReason::ManifestBodyMissing: return "manifest_body_missing"; + case HoldReason::CheckpointUndecodable: return "checkpoint_undecodable"; + } + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: unknown hold reason {}", static_cast(r)); +} + +namespace +{ + +HoldReason holdReasonFromWord(std::string_view w) +{ + if (w == "gap_below_witness") return HoldReason::GapBelowWitness; + if (w == "unconsumed_seal_crossing") return HoldReason::UnconsumedSealCrossing; + if (w == "witness_disappeared") return HoldReason::WitnessDisappeared; + if (w == "body_undecodable") return HoldReason::BodyUndecodable; + if (w == "manifest_body_missing") return HoldReason::ManifestBodyMissing; + if (w == "checkpoint_undecodable") return HoldReason::CheckpointUndecodable; + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: unknown hold reason '{}'", w); +} + +/// The classification set is CLOSED. Every consumer of a coverage row branches on exact values — the +/// sweep's §6 deletion premise refuses a row by testing `== 4` and then `== 0` — so a value outside the +/// set is not an unknown variant to be tolerated forward: it is a row that passes every refusal written +/// in terms of the set and reaches the irreversible delete. One predicate, used by both directions, so +/// the writer's self-check and the reader's fail-close can never name different sets. +bool isKnownClassification(uint64_t classification) +{ + return classification == 0 || classification == 1 || classification == 2 || classification == 4; +} + +/// A hold names a position the fold must resolve, and both components of that id are nonzero (the +/// canonical `RefTxnId` rule `renderRefTxnId` enforces for every id that becomes a key). A zero +/// component is not a weaker hold, it is a self-erasing one: no position sorts below `{0, 0}`, so the +/// carry rule clears it on the first round that folds anything, and the durable evidence that the +/// namespace ever stopped disappears without a single record having been resolved. +bool isCanonicalHoldPosition(const RefTxnId & at) +{ + return at.writer_epoch != 0 && at.ref_sequence != 0; +} + +/// Insert a decoded record under a key that must appear at most ONCE in the object. Plain +/// `map[key] = value` is last-wins, and last-wins is not a lossy nicety here: a repeated coverage key +/// lets a later, clean row overwrite the held one that a whole namespace's retention rests on, which is +/// exactly how a forged or mis-merged seal would erase the only durable record of an unresolved +/// position. One record per key, or the object is corrupt. +template +void insertRecordOnce(Map & map, const Key & key, Value && value, std::string_view what) +{ + if (!map.emplace(key, std::forward(value)).second) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: a second {} record for '{}' — a key appears at most once, and accepting the " + "duplicate would silently overwrite the record already read", + what, key); +} + +/// Emit one run record (`k` = "btr") WITHOUT its line terminator; the caller closes (and measures) the +/// line, and sorts the vector by key first. +void writeRun(CasJsonWriter & out, std::string_view kind, const RunRef & r) +{ + bool first = true; + writeKey(out, "k", first); writeStringValue(out, kind); + writeKey(out, "key", first); writeStringValue(out, r.key); + writeKey(out, "ck", first); writeHex128Value(out, r.checksum); + writeKey(out, "shard", first); writeIntText(r.shard, out); + writeKey(out, "gen", first); writeU64StringValue(out, r.generation); + closeObject(out, first); +} + +void validateFoldSealStructure( + const CasFoldSeal & seal, const Layout & layout, uint64_t gc_shards, + int error_code, std::string_view source) +{ + if (gc_shards == 0) + throw Exception(error_code, "CAS fold seal {}: gc_shards must be nonzero", source); + + std::vector run_seen(gc_shards, false); + for (const RunRef & run : seal.blob_target_runs) + { + if (run.key.empty() || run.generation == 0) + throw Exception(error_code, + "CAS fold seal {}: blob-target run requires a nonempty key and nonzero physical generation", + source); + if (run.shard >= gc_shards) + throw Exception(error_code, + "CAS fold seal {}: blob-target shard {} is outside [0, {})", + source, run.shard, gc_shards); + if (run_seen[run.shard]) + throw Exception(error_code, + "CAS fold seal {}: duplicate blob-target shard {} -- at most one run per shard is allowed", + source, run.shard); + run_seen[run.shard] = true; + + const auto parsed = layout.parseBlobTargetRunKey(run.key); + if (!parsed || parsed->generation != run.generation || parsed->shard != run.shard || parsed->seq != 0) + throw Exception(error_code, + "CAS fold seal {}: blob-target run key '{}' is not canonical for generation {}, shard {}, sequence 0", + source, run.key, run.generation, run.shard); + } + + if (seal.condemned_summary.size() != gc_shards) + throw Exception(error_code, + "CAS fold seal {}: condemned summary has {} rows, but exactly {} shards are required", + source, seal.condemned_summary.size(), gc_shards); + for (uint64_t shard = 0; shard < gc_shards; ++shard) + { + const auto it = seal.condemned_summary.find(shard); + if (it == seal.condemned_summary.end()) + throw Exception(error_code, + "CAS fold seal {}: condemned summary is missing shard {} from [0, {})", + source, shard, gc_shards); + const CondemnedSummary & summary = it->second; + if (summary.pending_total > summary.condemned_total) + throw Exception(error_code, + "CAS fold seal {}: shard {} has pending_total {} greater than condemned_total {}", + source, shard, summary.pending_total, summary.condemned_total); + const bool has_nonpending = summary.pending_total < summary.condemned_total; + const bool has_real_oldest = summary.oldest_nonpending_condemn_round != UINT64_MAX; + if (has_nonpending != has_real_oldest) + throw Exception(error_code, + "CAS fold seal {}: shard {} must carry a real oldest non-pending condemn round exactly when non-pending rows exist", + source, shard); + } +} + +} + +FoldSealCaps foldSealCaps() +{ + const FormatTraits & t = traitsFor(FormatId::FoldSeal); + return FoldSealCaps{.line_cap = t.line_cap, .object_cap = t.object_cap}; +} + +void checkFoldSealObjectBytes(uint64_t encoded_bytes) +{ + const uint64_t object_cap = foldSealCaps().object_cap; + if (!fitsObjectCap(encoded_bytes, /*entries_reservation*/0, object_cap)) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "CAS fold seal: the seal encodes to {} bytes, over the {}-byte object cap. Writing it " + "would leave a durable seal no later round can read; the round is refused before the PUT " + "and retries.", + encoded_bytes, object_cap); +} + +String encodeFoldSeal(const CasFoldSeal & seal) +{ + const FoldSealCaps caps = foldSealCaps(); + CasJsonWriter out(256); + + /// EVERY line this encoder emits is measured against the LINE cap, on the bytes actually emitted -- + /// escaping, framing and all -- rather than on an estimate of them. A line that does not fit is not + /// a large line, it is an UNREADABLE one: `readLine` refuses it, so the whole object is lost. + /// Refuse here, where nothing is durable yet. The header and trailer are measured too, even though + /// their lengths are bounded by construction — a gate with an unstated exception is one that a + /// later edit widens without noticing. + const auto checkLineBytes = [&](uint64_t bytes, std::string_view what) + { + if (!fitsLineCap(bytes, caps.line_cap)) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "CAS fold seal: the {} line encodes to {} bytes, over the {}-byte line cap; a longer " + "line cannot be read back, so the seal is refused before it is written", + what, bytes, caps.line_cap); + }; + + writeHeaderLine(out, FormatId::FoldSeal); /// emits its own terminator + checkLineBytes(out.size() - 1, "header"); + + size_t line_start = out.size(); + const auto closeLine = [&](std::string_view what) + { + checkLineBytes(out.size() - line_start, what); + writeChar('\n', out); + line_start = out.size(); + }; + + /// meta line + { + bool first = true; + writeKey(out, "g", first); writeU64StringValue(out, seal.generation); + writeKey(out, "pg", first); writeU64StringValue(out, seal.parent_generation); + closeObject(out, first); + closeLine("meta"); + } + + uint64_t n = 0; + + /// Ref-life rows (`std::map` => opaque-id-sorted). This is the sole serialized + /// producer of ref coverage and removal evidence. + for (const auto & [life_id, life_state] : seal.ref_lives) + { + const RefCoverage & cov = life_state.coverage; + const String life_hex = u128ToHex(life_id); + if (life_id == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS fold seal: a ref-life row has a zero life id -- 0 never names a life"); + /// THE STRICT GRAMMAR, enforced where the bytes are produced. Every refusal below is + /// `LOGICAL_ERROR`: this row came from our own fold, so an ill-formed one is a bug in this + /// process, not corruption arriving from a store — and none of these shapes is repairable once + /// durable, so none is ever written. + /// + /// A classification outside the closed set first, because the two checks after it are stated in + /// terms of the set and a row they cannot classify makes their answers meaningless. + if (!isKnownClassification(cov.classification)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS fold seal: coverage '{}' has classification {}, which is not one of the four the " + "fold grammar defines (0 absent, 1 unchanged, 2 folded, 4 clamped) — every consumer " + "branches on those exact values, so this row would pass refusals meant to stop it", + life_hex, cov.classification); + /// A classification-4 row whose hold was dropped is indistinguishable, once durable, from a + /// namespace that stopped for no reason — and a hold on any other classification claims a stop + /// that did not happen. + if ((cov.classification == 4) != cov.hold.has_value()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS fold seal: coverage '{}' has classification {} and {} hold — the hold fields are " + "required for classification 4 and forbidden otherwise", + life_hex, cov.classification, cov.hold ? "a" : "no"); + /// A hold that names no position resolves itself on the next round (nothing sorts below + /// `{0, 0}`) and cannot be rendered where the sweep reports why it retained a manifest. + if (cov.hold && !isCanonicalHoldPosition(cov.hold->offending_position)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS fold seal: coverage '{}' is held at {}-{} — a hold's offending position has both " + "components nonzero; a zero one would be cleared by the first record the next round " + "folds, erasing the only durable evidence that the namespace stopped", + life_hex, cov.hold->offending_position.writer_epoch, cov.hold->offending_position.ref_sequence); + + if (life_state.cleanup_evidence + && (life_state.cleanup_evidence->remove_txn_id.writer_epoch == 0 + || life_state.cleanup_evidence->remove_txn_id.ref_sequence == 0)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS fold seal: ref life '{}' carries cleanup evidence with non-canonical removal " + "transaction {}-{} -- both components are required and nonzero", + life_hex, + life_state.cleanup_evidence->remove_txn_id.writer_epoch, + life_state.cleanup_evidence->remove_txn_id.ref_sequence); + + bool first = true; + writeKey(out, "k", first); writeStringValue(out, "rfl"); + writeKey(out, "life", first); writeHex128Value(out, life_id); + writeKey(out, "cls", first); writeIntText(static_cast(cov.classification), out); + writeKey(out, "lfe", first); writeU64StringValue(out, cov.last_folded_ref_id.writer_epoch); + writeKey(out, "lfs", first); writeU64StringValue(out, cov.last_folded_ref_id.ref_sequence); + if (cov.hold) + { + writeKey(out, "hr", first); writeStringValue(out, holdReasonToWord(cov.hold->reason)); + writeKey(out, "hpe", first); writeU64StringValue(out, cov.hold->offending_position.writer_epoch); + writeKey(out, "hps", first); writeU64StringValue(out, cov.hold->offending_position.ref_sequence); + writeKey(out, "hrc", first); writeIntText(cov.hold->retry_count, out); + writeKey(out, "hnr", first); writeU64StringValue(out, cov.hold->next_retry_round); + } + if (life_state.cleanup_evidence) + { + writeKey(out, "rte", first); + writeU64StringValue(out, life_state.cleanup_evidence->remove_txn_id.writer_epoch); + writeKey(out, "rts", first); + writeU64StringValue(out, life_state.cleanup_evidence->remove_txn_id.ref_sequence); + } + closeObject(out, first); + closeLine("rfl"); + ++n; + } + + { + std::vector runs = seal.blob_target_runs; + std::sort(runs.begin(), runs.end(), [](const RunRef & a, const RunRef & b) { return a.key < b.key; }); + for (const RunRef & r : runs) + { + writeRun(out, "btr", r); + closeLine("btr"); + } + } + n += seal.blob_target_runs.size(); + + /// condemned summary (std::map => shard-sorted) + for (const auto & [shard, s] : seal.condemned_summary) + { + bool first = true; + writeKey(out, "k", first); writeStringValue(out, "cnd"); + writeKey(out, "shard", first); writeIntText(shard, out); + writeKey(out, "ct", first); writeIntText(s.condemned_total, out); + writeKey(out, "pt", first); writeIntText(s.pending_total, out); + writeKey(out, "ocr", first); writeU64StringValue(out, s.oldest_nonpending_condemn_round); + closeObject(out, first); + closeLine("cnd"); + ++n; + } + + const size_t trailer_start = out.size(); + writeTrailerLine(out, n); /// emits its own terminator + checkLineBytes(out.size() - trailer_start - 1, "trailer"); + + String text = std::move(out).take(); + checkFoldSealObjectBytes(text.size()); + return text; +} + +CasFoldSeal decodeFoldSeal(std::string_view data, std::optional expected_generation) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::FoldSeal); + const uint64_t line_cap = traitsFor(FormatId::FoldSeal).line_cap; + + CasFoldSeal seal; + + /// meta line + { + const String meta = readLine(in, line_cap, "fold seal"); + ReadBufferFromMemory m(meta.data(), meta.size()); + JsonObjectReader r(m, KeyStrictness::Strict, "fold seal"); + String key; + while (r.nextKey(key)) + { + if (key == "g") seal.generation = r.readU64String(); + else if (key == "pg") seal.parent_generation = r.readU64String(); + else r.skipUnknown(key); /// Strict => any unknown key is CORRUPTED_DATA + } + } + + uint64_t seen = 0; + while (true) + { + const String line = readLine(in, line_cap, "fold seal"); + ReadBufferFromMemory l(line.data(), line.size()); + JsonObjectReader r(l, KeyStrictness::Strict, "fold seal"); + String key; + if (!r.nextKey(key)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: empty line"); + + if (key == "n") + { + const uint64_t n = r.readU64Number(); + if (r.nextKey(key)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: trailer has extra keys"); + if (!l.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: bytes after trailer"); + if (n != seen) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: trailer count {} != {} records", n, seen); + if (expected_generation && seal.generation != *expected_generation) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: body generation {} does not match the requested generation {}", + seal.generation, *expected_generation); + return seal; + } + if (key != "k") + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: record must start with \"k\""); + const String kind = r.readString(); + + if (kind == "rfl") + { + std::optional life_id; + RefCoverage cov; + /// Read WIDE and validated before it is narrowed to the persisted byte. `cls` is the field + /// every consumer branches on, and a plain `static_cast` maps 258 onto 2 ("all + /// records through the cursor were folded") and 256 onto 0 — a forged or damaged seal would + /// buy full coverage with an integer no reader ever sees. + std::optional classification; + /// The hold fields are read individually so the grammar can be checked on WHICH of them + /// arrived, not merely on how many. `JsonObjectReader` already rejects a duplicate key, so + /// a second `hr` can never quietly rewrite the reason. + std::optional hold_reason; + std::optional hold_epoch; + std::optional hold_sequence; + std::optional hold_retry_count; + std::optional hold_next_retry_round; + std::optional remove_txn_epoch; + std::optional remove_txn_sequence; + while (r.nextKey(key)) + { + if (key == "life") life_id = r.readHex128(); + else if (key == "cls") classification = r.readU64Number(); + else if (key == "lfe") cov.last_folded_ref_id.writer_epoch = r.readU64String(); + else if (key == "lfs") cov.last_folded_ref_id.ref_sequence = r.readU64String(); + else if (key == "hr") hold_reason = holdReasonFromWord(r.readString()); + else if (key == "hpe") hold_epoch = r.readU64String(); + else if (key == "hps") hold_sequence = r.readU64String(); + else if (key == "hrc") hold_retry_count = r.readU32Number(); + else if (key == "hnr") hold_next_retry_round = r.readU64String(); + else if (key == "rte") remove_txn_epoch = r.readU64String(); + else if (key == "rts") remove_txn_sequence = r.readU64String(); + else throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: unknown rfl key '{}'", key); + } + + if (!life_id || *life_id == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: a ref-life row is missing a nonzero opaque life id"); + const String life_hex = u128ToHex(*life_id); + + /// `cls` is required, not defaulted: an absent one would read as 0 ("no round folded this + /// namespace"), which is a claim about a fold, not the absence of one. + if (!classification) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: rfl '{}' missing cls", life_hex); + if (!isKnownClassification(*classification)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: coverage '{}' has classification {}, which is not one of the four " + "the fold grammar defines (0 absent, 1 unchanged, 2 folded, 4 clamped)", + life_hex, *classification); + cov.classification = static_cast(*classification); /// in range, so narrowing is exact + + /// The same strict grammar the encoder enforces, applied to bytes we did not write. A + /// PARTIAL hold is corruption, never a hold with defaults: a hold whose offending position + /// defaulted to `{0,0}` would be cleared by the very first record the next round folds. + const bool any_hold_field = hold_reason || hold_epoch || hold_sequence + || hold_retry_count || hold_next_retry_round; + const bool every_hold_field = hold_reason && hold_epoch && hold_sequence + && hold_retry_count && hold_next_retry_round; + if (cov.classification == 4) + { + if (!every_hold_field) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: coverage '{}' is held (classification 4) but its hold is " + "incomplete — reason, offending position, retry count and next retry round are " + "all required", life_hex); + /// PRESENT is not enough: the position must be one a fold can actually retry. `{0, 0}` + /// (or either component zero) is the shape that quietly deletes the hold — the carry + /// rule keeps a hold only while the walk stops BELOW it, and nothing is below zero — and + /// it cannot be rendered where the sweep names the position it retained a manifest for. + if (!isCanonicalHoldPosition(RefTxnId{*hold_epoch, *hold_sequence})) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: coverage '{}' is held at {}-{} — a hold's offending position has " + "both components nonzero; a zero one clears itself on the next round", + life_hex, *hold_epoch, *hold_sequence); + cov.hold = RefHold{.reason = *hold_reason, + .offending_position = RefTxnId{*hold_epoch, *hold_sequence}, + .retry_count = *hold_retry_count, + .next_retry_round = *hold_next_retry_round}; + } + else if (any_hold_field) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: coverage '{}' carries hold fields at classification {} — they are " + "forbidden on anything but a held (classification 4) row", + life_hex, cov.classification); + + const bool any_cleanup_field = remove_txn_epoch || remove_txn_sequence; + const bool every_cleanup_field = remove_txn_epoch && remove_txn_sequence; + std::optional cleanup_evidence; + if (any_cleanup_field) + { + if (!every_cleanup_field || *remove_txn_epoch == 0 || *remove_txn_sequence == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: ref life '{}' carries incomplete or zero cleanup evidence -- " + "both removal transaction components are required and nonzero", + life_hex); + cleanup_evidence = RefCleanupEvidence{ + .remove_txn_id = RefTxnId{*remove_txn_epoch, *remove_txn_sequence}}; + } + if (!seal.ref_lives.emplace( + *life_id, RefLifeFoldState{.coverage = cov, .cleanup_evidence = cleanup_evidence}).second) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: a second ref-life record for '{}' -- a life id appears at most once", + life_hex); + } + else if (kind == "btr") + { + std::optional run_key; + std::optional checksum; + std::optional shard; + std::optional generation; + while (r.nextKey(key)) + { + if (key == "key") run_key = r.readString(); + else if (key == "ck") checksum = r.readHex128(); + else if (key == "shard") shard = r.readU64Number(); + else if (key == "gen") generation = r.readU64String(); + else throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: unknown run key '{}'", key); + } + if (!run_key || !checksum || !shard || !generation) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: btr requires key, ck, shard, and gen"); + seal.blob_target_runs.push_back(RunRef{ + .key = std::move(*run_key), .checksum = *checksum, .shard = *shard, .generation = *generation}); + } + else if (kind == "cnd") + { + std::optional shard; + std::optional condemned_total; + std::optional pending_total; + std::optional oldest_nonpending_condemn_round; + while (r.nextKey(key)) + { + if (key == "shard") shard = r.readU64Number(); + else if (key == "ct") condemned_total = r.readU64Number(); + else if (key == "pt") pending_total = r.readU64Number(); + else if (key == "ocr") oldest_nonpending_condemn_round = r.readU64String(); + else throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: unknown cnd key '{}'", key); + } + if (!shard || !condemned_total || !pending_total || !oldest_nonpending_condemn_round) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS fold seal: cnd requires shard, ct, pt, and ocr"); + insertRecordOnce(seal.condemned_summary, *shard, CondemnedSummary{ + .condemned_total = *condemned_total, + .pending_total = *pending_total, + .oldest_nonpending_condemn_round = *oldest_nonpending_condemn_round}, "condemned-summary"); + } + else + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: unknown record kind '{}'", kind); + + if (!l.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS fold seal: junk after record"); + ++seen; + } +} + +CasFoldSeal decodeFoldSeal( + std::string_view data, const Layout & layout, uint64_t gc_shards, + std::optional expected_generation) +{ + CasFoldSeal seal = decodeFoldSeal(data, expected_generation); + validateFoldSealStructure(seal, layout, gc_shards, ErrorCodes::CORRUPTED_DATA, "adoption"); + return seal; +} + +void validateFoldSealForWrite(const CasFoldSeal & seal, const Layout & layout, uint64_t gc_shards) +{ + validateFoldSealStructure(seal, layout, gc_shards, ErrorCodes::LOGICAL_ERROR, "producer"); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.h new file mode 100644 index 000000000000..f6d00a6e3b50 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFoldSealFormat.h @@ -0,0 +1,225 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +class Layout; + +/// A reference to one write-once run object and its whole-object checksum. A retry can compare the +/// checksum with the bytes already sealed before it adopts or consumes the run. +/// +/// `shard` and `generation` are required on `blob_target_runs`. An idle shard can carry a parent's run +/// into a newer seal, even though the object remains under the older generation's key namespace. The +/// explicit fields therefore preserve both the shard association and the physical generation without +/// making consumers parse a storage key. Run references for objects that are always local to the current +/// generation may leave these fields at their defaults because their consumers resolve them by key. +struct RunRef +{ + String key; + UInt128 checksum{}; + uint64_t shard = 0; /// gc-shard this run belongs to (REQUIRED for blob_target_runs) + uint64_t generation = 0; /// generation whose key namespace physically holds the object (for retention) + bool operator==(const RunRef &) const = default; +}; + +/// Why one namespace is held below its ref-log frontier. A BOUNDED enum: every value is a shape the +/// fold can name exactly, so an operator reading a seal learns what stopped the namespace without +/// correlating logs. Persisted as a word, so an unknown word is `CORRUPTED_DATA` rather than a silently +/// reinterpreted integer. +/// +/// THESE ARE WIRE VALUES, AND THEY ARE APPEND-ONLY. A durable seal written by one build is read by +/// another, so a renumbered value or a reused word makes an older seal describe a hold that is not the +/// one it recorded — and a hold's whole job is to say truthfully what stopped a namespace and where. +/// Add new reasons at the end; never renumber, never repurpose a retired word. +enum class HoldReason : uint8_t +{ + GapBelowWitness = 1, /// 404 at the expected id with a durable witness above it, same epoch + UnconsumedSealCrossing = 2, /// a later epoch is reachable but this epoch's closing seal was never consumed + WitnessDisappeared = 3, /// an above-cursor record stopped answering GETs — corruption, never clearance + BodyUndecodable = 4, /// the ref-log body at the position is present but cannot be decoded/extracted + ManifestBodyMissing = 5, /// a part-manifest body the position's edges name is absent (the fold barrier) + CheckpointUndecodable = 6, /// the namespace's `_ckpt` is present but its body cannot be decoded +}; + +/// The wire word one `HoldReason` is persisted as. Exported because the reason is operator-facing well +/// beyond the codec — the sweep names it when the §6 deletion premise retains a manifest under a held +/// namespace — and a second rendering of these words elsewhere would be a second place for them to drift. +std::string_view holdReasonToWord(HoldReason r); + +/// The durable hold on one namespace. It rides `RefCoverage` across rounds and across `REBUILD`, and +/// clears ONLY by folding through `offending_position` and adopting the result in `gc/state` — never by +/// observing another absent, because an absent is exactly the observation a lying store produces. +struct RefHold +{ + HoldReason reason = HoldReason::GapBelowWitness; + + /// The exact position the fold must resolve before this namespace may advance. A carried hold makes + /// the next round read this key even when the round's hint omits the namespace entirely. + /// + /// A CANONICAL id: both components are nonzero, and both codecs enforce it. `{0, 0}` is not a + /// degenerate hold, it is a hold that ERASES ITSELF — every position compares at or above it, so the + /// first fold that reaches any record clears the hold as resolved, and the namespace advances with + /// nothing recording that it ever stopped. It is also unnameable: `renderRefTxnId` refuses a zero + /// component, so the sweep that retains a manifest "because the namespace is held at " + /// cannot even state its reason. The decoder rejects it as `CORRUPTED_DATA` and the encoder as + /// `LOGICAL_ERROR`. + RefTxnId offending_position{}; + + /// How many rounds have retried `offending_position` without resolving it. Purely observational: + /// it is what distinguishes a transient barrier (a writer still uploading a manifest body) from a + /// namespace that has been stuck for hours. + uint32_t retry_count = 0; + + /// The first round that retries `offending_position`. The fold retries every round today (a hold + /// costs one exact `GET`, and a transient barrier must clear the moment its body lands), so this is + /// always `current_round + 1`; it exists so a future backoff policy needs no format change. + uint64_t next_retry_round = 0; + + bool operator==(const RefHold &) const = default; +}; + +/// Records what the current round did for one life-keyed `CasFoldSeal::ref_lives` row. +/// `classification` is a persisted byte: +/// 0 means absent, 1 means unchanged, 2 means all records through the observed cursor were folded, and 4 +/// means folding was clamped below the ref-log cursor. A clamped entry must be read again in the next +/// round, because an unfolded event may become foldable by then. +/// +/// THE SET {0, 1, 2, 4} IS CLOSED, and both codecs enforce it (decode `CORRUPTED_DATA`, encode +/// `LOGICAL_ERROR`). Every consumer branches on exact values — the sweep's §6 deletion premise refuses a +/// row by testing `== 4` and `== 0` — so an unrecognized byte is not a variant to tolerate: it passes +/// every refusal stated in terms of the set and reaches the delete. The decoder also validates BEFORE +/// narrowing to the byte, because a wide integer on the wire (258, say) truncates into the set and would +/// otherwise claim a coverage the fold never proved. +struct RefCoverage +{ + uint8_t classification = 0; + + /// The greatest `RefTxnId` whose owner changes have contributed their manifest-edge deltas. There is + /// one ref-log stream per namespace life, so this cursor is stored in that life-keyed row. + /// `{0, 0}` means that no transaction has been folded yet. A clamp leaves the cursor below the + /// offending transaction so the complete transaction is retried rather than partially applied. + RefTxnId last_folded_ref_id{}; + + /// STRICT GRAMMAR: present if and only if `classification == 4`. Both directions enforce it — the + /// encoder refuses to write a classification-4 row without a hold (a clamp whose reason was lost is + /// indistinguishable from a clean cursor once it is durable) and refuses to write a hold on any + /// other classification (`LOGICAL_ERROR`); the decoder rejects both shapes as `CORRUPTED_DATA`. The + /// pairing lives in the type, not only in the codec, so no producer can construct the forbidden + /// combination by forgetting a field. + std::optional hold = std::nullopt; + + bool operator==(const RefCoverage &) const = default; +}; + +/// Positive evidence that the terminal `remove_namespace` transaction for one life was folded into +/// the adopted seal. The owning opaque life id is the `CasFoldSeal::ref_lives` map key; the evidence +/// therefore carries no duplicate namespace or incarnation and has no pending/completed state. +struct RefCleanupEvidence +{ + RefTxnId remove_txn_id{}; + + bool operator==(const RefCleanupEvidence &) const = default; +}; + +/// The complete durable fold state for one cataloged ref life. Coverage and optional terminal +/// evidence live in the same row so neither can be admitted by an independent producer. +struct RefLifeFoldState +{ + RefCoverage coverage; + std::optional cleanup_evidence = std::nullopt; + + bool operator==(const RefLifeFoldState &) const = default; +}; + +/// Per-shard summary of condemned rows carried in the sealed source-edge run. It lets graduation and +/// pure reference-carry decisions inspect the seal without reading a run. Every newly written seal has +/// an entry for every shard in `0..gc_shards-1`: a folding shard computes its entry from its remaining +/// condemned rows, while a pure-carry shard copies the parent's entry. Missing entries are invalid and +/// must not be interpreted as zero. +struct CondemnedSummary +{ + uint64_t condemned_total = 0; /// count of `kCondemned` rows in this shard's sealed run + uint64_t pending_total = 0; /// how many of those are `delete_pending` (a graduation is due) + uint64_t oldest_nonpending_condemn_round = UINT64_MAX; /// min condemn_round over non-pending; UINT64_MAX = none + bool operator==(const CondemnedSummary &) const = default; +}; + +/// The write-once fold seal for one GC generation at +/// `/gc/gen//attempt//fold_seal`. It is the generation's durable coverage +/// record: it stores cursors and run references, not one record per edge, manifest, or candidate. A retry +/// and the next round use the adopted seal to determine what was folded and which parent runs can be +/// carried forward. Its run references and ref-life rows are also the durable inputs to retention. +/// Manifest cleanup is intentionally not represented here: those cleanups execute +/// inline from the in-memory cleanup map, and no durable cleanup-run reader exists. +struct CasFoldSeal +{ + uint64_t generation = 0; + uint64_t parent_generation = 0; + /// Exactly one row per `Live` or `Removing` catalog life admitted by `buildRefWalkPlan`. + std::map ref_lives; + std::vector blob_target_runs; /// the blob in-degree run segments this gen sealed + std::map condemned_summary; /// gc-shard -> summary; TOTAL over gc_shards + bool operator==(const CasFoldSeal &) const = default; +}; + +/// The two byte caps a fold seal is measured against, read from the format registry so the writer's +/// gate and its boundary tests can never drift from the reader's limits. +struct FoldSealCaps +{ + uint64_t line_cap; /// longest decodable record, excluding the '\n' terminator + uint64_t object_cap; /// largest whole seal object +}; +FoldSealCaps foldSealCaps(); + +/// PRE-PUT GATE. A seal larger than the object cap is writable but not readable — nothing enforces the +/// cap on the fold-seal read path, so an oversized PUT would leave the pool with a durable seal that no +/// later round can decode, which is unrecoverable. `encodeFoldSeal` therefore refuses BEFORE returning +/// any bytes, which is before either PUT site can issue its write. `LIMIT_EXCEEDED`, not +/// `CORRUPTED_DATA`: the bytes are well formed, the round is over budget, and the round fails closed and +/// retries. Equality fits; one byte more does not. +void checkFoldSealObjectBytes(uint64_t encoded_bytes); + +/// Encodes a fold seal as a strict, raw text control object. The header and meta lines are followed by +/// tagged records in the fixed `rfl`/`btr`/`cnd` order and a record-count trailer. Map iteration and +/// run references are sorted so retries produce byte-identical output for write-once adoption. +/// +/// Enforces the whole coverage grammar — the closed classification set, the classification-4 hold +/// pairing, and the hold's canonical offending position — and BOTH byte caps: every emitted line against +/// `line_cap` — header, meta, records and trailer alike, with no exception — and the whole object +/// against `object_cap`. Both PUT sites go through this function, so the gate cannot be bypassed by +/// adding a third one. +/// +/// A grammar violation here is `LOGICAL_ERROR`, not `CORRUPTED_DATA`: these bytes do not come from a +/// store, they come from THIS process, so the seal it is about to make durable is a bug in our own +/// writer — the same code `encodeGcState` raises for its own impossible input. The caps stay +/// `LIMIT_EXCEEDED` (a well-formed seal, over budget). Nothing is returned on any refusal, so no PUT +/// site can issue the write. +String encodeFoldSeal(const CasFoldSeal & seal); + +/// Decodes and validates a fold seal, rejecting unknown fields, malformed records, trailing bytes, a +/// second record for a key already read, and a trailer count that differs from the records read. +/// Invalid persisted data raises `CORRUPTED_DATA` — including every shape the encoder refuses, since +/// these bytes may have been written by anything at all. +CasFoldSeal decodeFoldSeal(std::string_view data, std::optional expected_generation = std::nullopt); + +/// Decodes a seal at an adoption boundary and validates the shard-indexed structure against the +/// pool-authoritative nonzero `gc_shards`: canonical blob-target keys, at most one run per shard, +/// and an exactly total, semantically consistent condemned summary. +CasFoldSeal decodeFoldSeal( + std::string_view data, const Layout & layout, uint64_t gc_shards, + std::optional expected_generation = std::nullopt); + +/// Applies the same shard-indexed structural checks to an in-memory producer immediately before its +/// seal PUT. Invalid producer state is a `LOGICAL_ERROR`; no bytes are written. +void validateFoldSealForWrite(const CasFoldSeal & seal, const Layout & layout, uint64_t gc_shards); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.cpp new file mode 100644 index 000000000000..cf46c2e51b62 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.cpp @@ -0,0 +1,198 @@ +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int UNKNOWN_FORMAT_VERSION; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +namespace +{ + +/// Generation-1 baseline for every class. A future format change appends to that class's array and +/// bumps `G_BUILD`: additive changes use the previous reader floor, while breaking changes use the +/// new generation as the floor. Existing entries are immutable history. +constexpr FormatChangePoint BASELINE[] = {{1, 1}}; + +/// The two ref classes changed at generation 4 (INV-1, per-namespace contiguous ids) AND AGAIN at +/// generation 5 (Stage B's recreate-only "format bump B": the ref layer re-keyed under +/// `//`). Both changes are BREAKING even though not one byte of the encoding moved +/// either time -- a generation-3 stream's ids came from a pool-wide counter and legitimately skip, +/// which a generation-4 reader reports as corruption, and a generation-4 key names no incarnation at +/// all, which a generation-5 reader also reports as corruption (`Layout::parseRefObjectKey`). Each +/// floor is the change generation itself. +constexpr FormatChangePoint REF_STREAM[] = { + {1, 1}, + {kContiguousRefStreamsGeneration, kContiguousRefStreamsGeneration}, + {kNamespaceLifeKeyedGeneration, kNamespaceLifeKeyedGeneration}, + {kOpaqueNamespaceLifeLayoutGeneration, kOpaqueNamespaceLifeLayoutGeneration}, +}; + +/// `cas_ref_ckpt` is BORN at generation 4, so it has no generation-1 baseline to inherit: there is no +/// such thing as a generation-1 `_ckpt` object, and claiming one would say a generation-1 reader could +/// read it. Generation 5 re-keys it under `//` exactly like `REF_STREAM` above, for +/// the same reason and with the same floor. +constexpr FormatChangePoint REF_CKPT[] = { + {kContiguousRefStreamsGeneration, kContiguousRefStreamsGeneration}, + {kNamespaceLifeKeyedGeneration, kNamespaceLifeKeyedGeneration}, + {kOpaqueNamespaceLifeLayoutGeneration, kOpaqueNamespaceLifeLayoutGeneration}, + {kCommittedRefFrontierGeneration, kCommittedRefFrontierGeneration}, +}; + +/// `cas_ref_catalog` is BORN at generation 4, one generation BEFORE the bump that makes namespace +/// existence catalog-authoritative (Stage B's Task 4, "format bump B" -- `kNamespaceLifeKeyedGeneration`): +/// Task 2 introduced the catalog OBJECT while `G_BUILD` was still the value +/// `kContiguousRefStreamsGeneration` names, and Task 4 is the later change that actually wires +/// discovery to read it and bumps the floor. The catalog's own encoding is unaffected by that bump (it +/// reuses `kContiguousRefStreamsGeneration` as its birth generation, not a second constant named after +/// itself, for the same reason `REF_CKPT` originally did), so it carries no second change point here. +constexpr FormatChangePoint REF_CATALOG[] = {{kContiguousRefStreamsGeneration, kContiguousRefStreamsGeneration}}; +constexpr FormatChangePoint GC_MAINTENANCE_STATE[] = {{kUnifiedRefLifeFoldGeneration, kUnifiedRefLifeFoldGeneration}}; +constexpr FormatChangePoint POOL_META[] = { + {1, 1}, + {kPoolGcShardsGeneration, kPoolGcShardsGeneration}, + {kCommittedRefFrontierGeneration, kCommittedRefFrontierGeneration}, +}; + +} + +std::span changePoints(FormatId id) +{ + switch (id) + { + case FormatId::RefLog: + case FormatId::RefSnapshot: + return REF_STREAM; + case FormatId::RefCkpt: + return REF_CKPT; + case FormatId::RefCatalog: + return REF_CATALOG; + case FormatId::GcMaintenanceState: + return GC_MAINTENANCE_STATE; + case FormatId::PoolMeta: + return POOL_META; + case FormatId::Blob: + case FormatId::GcState: + case FormatId::Roster: + case FormatId::GcOutcomes: + case FormatId::PartManifest: + case FormatId::RunFile: + case FormatId::FoldSeal: + case FormatId::Owner: + case FormatId::ServerEpoch: + case FormatId::MountLease: + case FormatId::BlobMeta: + case FormatId::GcHeartbeat: + return BASELINE; + } + throw Exception(ErrorCodes::LOGICAL_ERROR, "CasFormat: unknown FormatId {}", static_cast(id)); +} + +uint32_t currentWriterVersion() +{ + return G_BUILD; +} + +uint32_t currentCompatibilityVersion() +{ + /// Until roster-based write-down is implemented, every object carries the current build as its + /// compatibility floor. + return G_BUILD; +} + +void checkCompatibility(uint32_t compatibility_version, std::string_view what) +{ + if (compatibility_version > G_BUILD) + throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, + "CAS {}: object requires reader generation {} but this build supports at most {}", + what, compatibility_version, G_BUILD); +} + +namespace +{ +constexpr uint64_t kKiB = 1024; +constexpr uint64_t kMiB = 1024 * 1024; + +/// Caps are 100-1000x above realistic sizes; hitting one indicates a corrupt object or protocol bug. +/// `RefLog` and `RefSnapshot` objects are read whole, so their 64 MiB decompressed object cap +/// accommodates the JSON-inflated removal-class transaction and full snapshot. Their codecs +/// independently enforce the existing `ref_txn_max_bytes` (20 MiB) and 64 MiB removal/snapshot budgets +/// before sealing. +/// +/// Their `line_cap` intentionally equals `object_cap`. A smaller per-line limit would add no memory +/// protection to a whole-read format, while creating a write/read split: admission measures the whole +/// transaction against the object budget, so a large individual ref payload could be accepted on +/// write and rejected on decode, leaving a persisted ref object that cannot be read. The line cap is +/// instead meaningful for streaming formats, where it bounds the resident O(line) record. Matching +/// `line_cap` to `object_cap` lets any individually valid record consume the available object budget. +/// +/// Compression is per type and deterministic, with no size threshold. `Always` types can grow large +/// and use a `.zst` key suffix; `PinnedRaw` types need stable bytes for adoption; `Never` types are +/// small raw singletons. +constexpr FormatTraits TRAITS[] = +{ + {FormatId::Blob, "cas_blob", TextFamily::PayloadHybrid, KeyStrictness::Tolerant, CompressionPolicy::Never, 256, 256}, + {FormatId::BlobMeta, "cas_blob_meta", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::PoolMeta, "cas_pool_meta", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::RefLog, "cas_ref_log", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Always, 64 * kMiB, 64 * kMiB}, + {FormatId::RefSnapshot, "cas_ref_snap", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Always, 64 * kMiB, 64 * kMiB}, + /// `cas_ref_ckpt` is a three-field mutable singleton read by a point GET on every recovery and on + /// every cleanup decision, so its caps are deliberately TIGHT (64 KiB / 4 KiB rather than the + /// megabyte scale its Control-family siblings use): nothing legitimate approaches them, and the cap + /// is the first thing that fires if a foreign object ever lands at the key. STRICT for the same + /// reason its decoder is -- every field changes what cleanup may delete, so nothing in it may be + /// skipped. Raw (`Never`): a small singleton, and `publishCkpt` re-encodes it on every attempt. + {FormatId::RefCkpt, "cas_ref_ckpt", TextFamily::Control, KeyStrictness::Strict, CompressionPolicy::Never, 64 * kKiB, 4 * kKiB}, + /// `cas_ref_catalog` (INV-3): one object for the whole pool, token-CAS like `gc/state`, read on + /// every fold round and every recovery. STRICT for the same reason `cas_ref_ckpt` is -- every + /// field decides a namespace's lifecycle, so nothing in it may be skipped. Raw (`Never`): the + /// admission gate measures `encodeRefCatalog`'s own output directly, so a compressed size would + /// answer the wrong question. The object cap is the fold-seal's own 256 MiB (predicate (2) of the + /// additive admission check bounds it further via the entry count); the line cap is tight (4 KiB) + /// because one entry's record is ordinarily small -- but not always small enough: a namespace or + /// `server_root_id` near their own byte bounds, worst-case escaped, can push a single line past + /// 4 KiB, and `encodeRefCatalog` REFUSES that entry (`LIMIT_EXCEEDED`, `CasRefCatalogFormat.cpp`'s + /// `checkLineBytes`) rather than writing an object no reader could later decode. + {FormatId::RefCatalog, "cas_ref_catalog", TextFamily::Control, KeyStrictness::Strict, CompressionPolicy::Never, 256 * kMiB, 4 * kKiB}, + {FormatId::GcMaintenanceState, "cas_gc_maintenance_state", TextFamily::Control, KeyStrictness::Strict, CompressionPolicy::Never, 512 * kKiB, 512 * kKiB}, + {FormatId::PartManifest, "cas_part_manifest", TextFamily::PayloadHybrid, KeyStrictness::Tolerant, CompressionPolicy::Always, 256 * kMiB, 64 * kKiB}, + {FormatId::RunFile, "cas_run", TextFamily::RecordStream, KeyStrictness::Strict, CompressionPolicy::PinnedRaw, 0, 4 * kKiB}, + {FormatId::FoldSeal, "cas_fold_seal", TextFamily::Control, KeyStrictness::Strict, CompressionPolicy::PinnedRaw, 256 * kMiB, 64 * kKiB}, + {FormatId::GcState, "cas_gc_state", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::GcHeartbeat, "cas_gc_hb", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::GcOutcomes, "cas_gc_outcomes", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Always, 256 * kMiB, 64 * kKiB}, + {FormatId::Owner, "cas_owner", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::ServerEpoch, "cas_epoch", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, + {FormatId::MountLease, "cas_mount_lease", TextFamily::Control, KeyStrictness::Tolerant, CompressionPolicy::Never, 1 * kMiB, 64 * kKiB}, +}; +} + +const FormatTraits & traitsFor(FormatId id) +{ + for (const FormatTraits & t : TRAITS) + if (t.id == id) + return t; + throw Exception(ErrorCodes::LOGICAL_ERROR, "CasFormat: no traits for FormatId {} (reserved?)", static_cast(id)); +} + +const FormatTraits * traitsForType(std::string_view type) +{ + for (const FormatTraits & t : TRAITS) + if (t.type == type) + return &t; + return nullptr; +} + +std::string_view storedSuffix(FormatId id) +{ + return traitsFor(id).compression == CompressionPolicy::Always ? ".zst" : ""; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h new file mode 100644 index 000000000000..633f8f8fd7be --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h @@ -0,0 +1,203 @@ +#pragma once +#include +#include +#include + +namespace DB::Cas +{ + +/// Shared vocabulary and policy registry for persisted content-addressed objects. The registry +/// supplies the version gate and the per-object text-file traits used by the codecs in this +/// directory; it does not own object bytes or storage lifecycle state. + +/// The highest pool-format generation this build understands. A build keeps every decoder for +/// generations 1..G_BUILD (new code always reads old); an object is readable iff its +/// compatibility_version <= G_BUILD. Bump this (and append a change-point in CasFormat.cpp) when a new +/// format generation is introduced. +/// +/// Generation 2 is the first generation that understands mixed-algorithm pools: the schema-3 +/// source-edge settlement key includes the algorithm prefix, so a generation-1 reader can open the +/// pool but cannot decode its GC state. Pool admission CAS-raises `min_reader_generation` to this +/// build's own floor (`G_BUILD`), and a persisted floor above `G_BUILD` fails closed. +/// +/// Generation 3 replaced mutable ref-shard objects with immutable `_log` and `_snap` objects. +/// +/// Generation 4 makes each namespace's ref-log ids per-namespace and CONTIGUOUS within a writer epoch +/// (INV-1). The bytes of a `_log`/`_snap` object did not change, but their MEANING did: a generation-3 +/// pool's ids were drawn from a pool-wide counter and are full of legitimate holes, which this build +/// reads as a truncated -- i.e. corrupt -- stream. The per-object forward gate cannot reject such a +/// pool (its version is not in the future), so pool-meta decoding applies +/// `kContiguousRefStreamsGeneration` as a backward floor. Pools below the floor must be recreated; +/// there is no migration path in the pre-release format. +/// +/// Generation 5 (Stage B's own recreate-only bump, the plan's "format bump B") re-keys the ref layer +/// under `//` (spec INV-3: the whole-pool namespace catalog mints the incarnation). +/// Again the bytes of `_log`/`_snap`/`_ckpt` objects did not change, but the KEY SHAPE they live under +/// did: a generation-4 key named a namespace directly (`cas/refs//_log/`), while this +/// generation's reader recognizes only the incarnation-qualified shape +/// (`cas/refs///_log/`) -- `Layout::parseRefObjectKey`/`parseRefCkptKey` already +/// refuse the un-incarnated shape with `CORRUPTED_DATA` (Stage B Tasks 1/1c landed that refusal ahead +/// of this bump, deliberately: the pre-release format carries zero persisted data and zero compat +/// obligation, so the key shapes and the bump that makes them the ONLY readable shape need not land in +/// the same commit). `kNamespaceLifeKeyedGeneration` is the backward floor for this change, applied the +/// same way `kContiguousRefStreamsGeneration` is. +/// +/// Generation 6 replaces that namespace-bearing grammar with opaque pool-wide life identifiers and +/// splits hot ref streams from point-read state: `cas/ns/stream//...` contains `_log`, `_snap` +/// while `cas/ns/state//...` contains `_ckpt` and `_files`. A generation-5 +/// pool must be recreated; no dual parser or copy-forward path exists. +/// +/// Generation 7 replaces the fold seal's independent namespace-keyed coverage and cleanup +/// collections with one opaque-life-keyed row and removes the retired terminal-marker object class. A generation-6 +/// pool must be recreated; there is no dual reader for the split grammar. +/// +/// Generation 8 persists the creation-time `gc_shards` authority in `_pool_meta`. Generation-7 pools +/// must be recreated because namespace admission can precede creation of `gc/state`; accepting a +/// metadata object without this field would leave different openers charging different seal bounds. +/// Generation 9 adds `_ckpt.committed_through`, the exact recovery frontier. Generation-8 pools +/// must be recreated: the absence of this field has the incompatible meaning that no transaction has +/// entered durable logical history. +constexpr uint32_t G_BUILD = 9; + +/// The pool-format generation at which ref-log ids became per-namespace and contiguous. Pool metadata +/// below this value cannot be opened, because its ref streams carry holes this build reports as +/// corruption; the backward-floor check is applied by `decodePoolMeta`. Named separately from `G_BUILD` +/// so a later generation that CAN still read a generation-4 pool does not silently move the floor with +/// it. +constexpr uint32_t kContiguousRefStreamsGeneration = 4; + +/// The pool-format generation at which the ref layer (and, per Stage B's Task 4b, namespace files) +/// became incarnation-scoped under `//`. Pool metadata below this value cannot be +/// opened: its ref-object keys carry no incarnation segment, which this build's parsers refuse as +/// corruption rather than read as a compatibility case (see the `G_BUILD` doc above). The backward- +/// floor check is applied by `decodePoolMeta`, exactly mirroring `kContiguousRefStreamsGeneration`; +/// named separately for the same reason that one is -- so a later generation that can still read a +/// generation-5 pool does not silently move this floor with it. Pools below the floor must be +/// recreated; there is no migration path in the pre-release format. +constexpr uint32_t kNamespaceLifeKeyedGeneration = 5; + +/// The recreate-only generation at which namespace text disappeared from physical life keys and hot +/// ref streams were separated from point-read namespace state. +constexpr uint32_t kOpaqueNamespaceLifeLayoutGeneration = 6; + +/// The recreate-only generation at which one unified ref-life row replaced the split coverage and +/// namespace-cleanup grammar. +constexpr uint32_t kUnifiedRefLifeFoldGeneration = 7; + +/// The recreate-only generation at which `_pool_meta` became the authority for `gc_shards`. +constexpr uint32_t kPoolGcShardsGeneration = 8; + +/// The recreate-only generation at which `_ckpt` gained its exact committed-transaction frontier. +constexpr uint32_t kCommittedRefFrontierGeneration = 9; + +/// Stable identifiers for every self-describing persisted object class. The text registry uses the +/// corresponding `type` string as the on-disk identity. Numeric values are part of the format history: +/// retired values remain unused so an old object can never be mistaken for a later class. +enum class FormatId : uint16_t +{ + Blob = 1, + /// Values 2, 3, and 4 are retired. The former tree and GC-snapshot classes were replaced by the + /// root-local part manifest, while the former mutable `cas_ref_shard` class was replaced by the + /// immutable `RefSnapshot` and `RefLog` objects. Keep all three values unused. + GcState = 5, + /// Value 6 is retired: condemned state now rides source-edge runs and the fold-seal + /// `condemned_summary`. Keep it unused. Value 7 is also retired: the build-watermark floor is + /// carried by the `MountLease` beat rather than a standalone object. + PoolMeta = 8, + Roster = 9, + /// Value 10 is retired: discovery authority is the pool-wide `cas/ref_catalog` object rather + /// than a roots registry object or a physical stream listing. + GcOutcomes = 11, + PartManifest = 12, /// Immutable root-local `cas_part_manifest` payload-hybrid text object. + RunFile = 13, /// Deterministic, uncompressed `cas_run` GC source-edge NDJSON stream. + FoldSeal = 14, /// Write-once `cas_fold_seal` coverage and blob-target/cleanup-run object. + /// Value 15 is retired: the fold seal is the sole per-generation coverage record after the + /// one-pass GC round. The following three classes are per-server-root mount-safety objects. + Owner = 16, /// `cas_owner` anchor from server-root ID to server UUID. + ServerEpoch = 17, /// `cas_epoch` writer-epoch fence carrying `next_writer_epoch`. + MountLease = 18, /// Live `cas_mount_lease` object. + /// These identifiers cover objects that were added to the registry after their initial codecs: + /// the ref transaction log, complete ref snapshot, blob freshness sidecar, and GC heartbeat. + /// Their values are frozen and must never be reused. + RefLog = 19, /// cas_ref_log — ref transaction log object + RefSnapshot = 20, /// cas_ref_snap — complete per-namespace ref table + BlobMeta = 21, /// cas_blob_meta — per-blob freshness sidecar + GcHeartbeat = 22, /// cas_gc_hb — GC leader heartbeat + RefCkpt = 23, /// cas_ref_ckpt — per-namespace checkpoint (INV-4) + RefCatalog = 24, /// cas_ref_catalog — the whole-pool namespace catalog (INV-3) + GcMaintenanceState = 25, /// cas_gc_maintenance_state — leak-only namespace-janitor cursor +}; + +/// Returns the writer generation stamped on newly written objects. The current pre-roster writer +/// always stamps `G_BUILD`. +uint32_t currentWriterVersion(); + +/// Returns the compatibility generation stamped on newly written objects. Until the roster and +/// write-down-to-floor policy exist, this is always `G_BUILD`; readers reject values above `G_BUILD`. +uint32_t currentCompatibilityVersion(); + +/// Applies the common fail-closed reader gate. If an object's `compatibility_version` exceeds +/// `G_BUILD`, throws `UNKNOWN_FORMAT_VERSION` before the caller interprets the body; `what` identifies +/// the object in the exception message. +void checkCompatibility(uint32_t compatibility_version, std::string_view what); + +/// One append-only entry in a class's format history. At `generation`, the class's ENCODING or the +/// MEANING of what it encodes changed, and a reader must understand at least `min_reader` to read an +/// object written at that generation. Additive changes retain the previous reader floor; breaking +/// changes set the floor to the change generation itself. Generation 4's ref-stream entry is the +/// worked example of the second kind: not one byte of `cas_ref_log` moved, but its ids became dense, +/// so an older stream is unreadable to this build and the floor is the change generation. +struct FormatChangePoint +{ + uint16_t generation; + uint16_t min_reader; +}; + +/// Returns the append-only change-point history for `id`, oldest first. A class's history begins at +/// the generation it was BORN in, not at 1: the classes that existed from the start carry the frozen +/// `{1, 1}` baseline, while `RefCkpt` — introduced at generation 4 — begins at `{4, 4}`, because there +/// is no such thing as a generation-1 `_ckpt` and claiming one would say a generation-1 reader could +/// read it. Future changes append entries without editing old ones. +std::span changePoints(FormatId id); + +/// The text-format registry has one row per decodable persisted object. Each row is the single source +/// for the header-line `type`, body family, unknown-key policy, compression policy, and fail-closed +/// size caps. A format missing from this registry cannot be decoded. + +/// The shape of a text object body: a materialized control object, a streamed sorted record sequence, +/// or a descriptor followed by raw payload bytes. +enum class TextFamily : uint8_t { Control = 1, RecordStream = 2, PayloadHybrid = 3 }; + +/// Whether a decoder skips unknown ordinary keys or rejects them. Critical keys prefixed with `!` +/// are rejected by all families because they signal a required extension. +enum class KeyStrictness : uint8_t { Tolerant = 1, Strict = 2 }; + +/// Deterministic storage policy. `Always` uses whole-object zstd and a `.zst` key suffix; `Never` +/// remains raw; `PinnedRaw` is raw because byte adoption compares the serialized bytes. +enum class CompressionPolicy : uint8_t { Never = 1, Always = 2, PinnedRaw = 3 }; + +/// Codec metadata for one registered text object. The byte caps apply to decompressed object content +/// and individual text lines; `object_cap == 0` means a streamed format has no whole-object cap. +struct FormatTraits +{ + FormatId id; + std::string_view type; /// header-line "type" value + TextFamily family; + KeyStrictness strictness; + CompressionPolicy compression; + uint64_t object_cap; /// max DECOMPRESSED object bytes; 0 = uncapped (streamed) + uint64_t line_cap; /// max bytes of one text line +}; + +/// Returns the traits for `id`. Throws `LOGICAL_ERROR` for `FormatId::Roster`, which is reserved and +/// has no codec or traits row yet. +const FormatTraits & traitsFor(FormatId id); +/// Looks up a header-line `type` string. Returns nullptr for an unregistered type; it does not throw +/// because callers use this result to classify the input before decoding it. +const FormatTraits * traitsForType(std::string_view type); +/// Returns the storage-key suffix for `id`: `.zst` for `Always`, and an empty suffix otherwise. +/// Key builders use this policy directly so a point lookup never has to inspect the object body or +/// try multiple keys. +std::string_view storedSuffix(FormatId id); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.cpp new file mode 100644 index 000000000000..c5dda3286ad6 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.cpp @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include + +namespace DB::ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int LIMIT_EXCEEDED; +} + +namespace DB::Cas +{ + +String encodeGcMaintenanceState(const GcMaintenanceState & state) +{ + if (state.janitor_cursor.size() > kMaxGcMaintenanceCursorBytes) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, "CAS gc maintenance state: cursor has {} bytes, limit {}", + state.janitor_cursor.size(), kMaxGcMaintenanceCursorBytes); + + CasJsonWriter out; + writeHeaderLine(out, FormatId::GcMaintenanceState); + bool first = true; + writeKey(out, "cur", first); + writeStringValue(out, state.janitor_cursor); + closeObject(out, first); + writeChar('\n', out); + return std::move(out).take(); +} + +GcMaintenanceState decodeGcMaintenanceState(std::string_view data) +{ + const uint64_t object_cap = traitsFor(FormatId::GcMaintenanceState).object_cap; + if (data.size() > object_cap) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS gc maintenance state: object has {} bytes, limit {}", data.size(), object_cap); + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::GcMaintenanceState); + const String body = readLine(in, traitsFor(FormatId::GcMaintenanceState).line_cap, "cas_gc_maintenance_state"); + ReadBufferFromMemory body_in(body.data(), body.size()); + JsonObjectReader reader(body_in, KeyStrictness::Strict, "cas_gc_maintenance_state"); + + GcMaintenanceState result; + bool has_cursor = false; + String key; + while (reader.nextKey(key)) + { + if (key == "cur") + { + result.janitor_cursor = reader.readString(); + has_cursor = true; + } + else + reader.skipUnknown(key); + } + if (!has_cursor) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS gc maintenance state: missing cur"); + if (!body_in.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS gc maintenance state: trailing bytes"); + if (result.janitor_cursor.size() > kMaxGcMaintenanceCursorBytes) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS gc maintenance state: cursor has {} bytes, limit {}", + result.janitor_cursor.size(), kMaxGcMaintenanceCursorBytes); + return result; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.h new file mode 100644 index 000000000000..49d495995698 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcMaintenanceStateFormat.h @@ -0,0 +1,21 @@ +#pragma once +#include +#include +#include + +namespace DB::Cas +{ + +constexpr size_t kMaxGcMaintenanceCursorBytes = 64 * 1024; + +/// Durable, leak-only progress for namespace maintenance. It has no GC authority fields. +struct GcMaintenanceState +{ + String janitor_cursor; + bool operator==(const GcMaintenanceState &) const = default; +}; + +String encodeGcMaintenanceState(const GcMaintenanceState & state); +GcMaintenanceState decodeGcMaintenanceState(std::string_view data); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.cpp new file mode 100644 index 000000000000..e69ea98a799e --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.cpp @@ -0,0 +1,129 @@ +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +namespace +{ + +std::string_view outcomeKindToWord(OutcomeKind o) +{ + switch (o) + { + case OutcomeKind::Deleted: return "deleted"; + case OutcomeKind::Absent: return "absent"; + case OutcomeKind::Replaced: return "replaced"; + case OutcomeKind::Spared: return "spared"; + } + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS outcome log: unknown OutcomeKind {}", static_cast(o)); +} + +OutcomeKind outcomeKindFromWord(std::string_view w) +{ + if (w == "deleted") return OutcomeKind::Deleted; + if (w == "absent") return OutcomeKind::Absent; + if (w == "replaced") return OutcomeKind::Replaced; + if (w == "spared") return OutcomeKind::Spared; + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS outcome log: unknown outcome '{}'", w); +} + +} + +String encodeOutcomeLog(const OutcomeLog & log) +{ + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::GcOutcomes); + for (const OutcomeEntry & e : log.entries) + { + bool first = true; + writeKey(out, "k", first); + writeStringValue(out, objectKindToWord(e.kind)); + writeBlobRefFields(out, first, e.ref); /// ha + h + writeTokenFields(out, first, e.token); /// tt + tv + writeKey(out, "oc", first); + writeStringValue(out, outcomeKindToWord(e.outcome)); + closeObject(out, first); + writeChar('\n', out); + } + writeTrailerLine(out, log.entries.size()); + return std::move(out).take(); +} + +OutcomeLog decodeOutcomeLog(std::string_view data) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::GcOutcomes); + const uint64_t line_cap = traitsFor(FormatId::GcOutcomes).line_cap; + + OutcomeLog log; + while (true) + { + const String line = readLine(in, line_cap, "outcome log"); + ReadBufferFromMemory line_in(line.data(), line.size()); + JsonObjectReader r(line_in, KeyStrictness::Tolerant, "outcome log"); + + String key; + /// The first key distinguishes a trailer ("n") from a record ("k"). + if (!r.nextKey(key)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS outcome log: empty line"); + if (key == "n") + { + const uint64_t n = r.readU64Number(); + while (r.nextKey(key)) + r.skipUnknown(key); + if (!line_in.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS outcome log: bytes after trailer"); + if (n != log.entries.size()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS outcome log: trailer count {} != {} records", n, log.entries.size()); + return log; + } + + OutcomeEntry e; + String ha; + String hhex; + String tv; + bool have_ha = false; + bool have_h = false; + bool have_tt = false; + TokenType tt{}; + do + { + if (key == "k") e.kind = objectKindFromWord(r.readString(), "outcome log"); + else if (key == "ha") { ha = r.readString(); have_ha = true; } + else if (key == "h") { hhex = r.readString(); have_h = true; } + else if (key == "tt") { tt = tokenTypeFromWord(r.readString(), "outcome log"); have_tt = true; } + else if (key == "tv") tv = r.readString(); + else if (key == "oc") e.outcome = outcomeKindFromWord(r.readString()); + else r.skipUnknown(key); + } while (r.nextKey(key)); + + if (!have_ha || !have_h || !have_tt) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS outcome log: record missing ha/h/tt"); + const BlobHashAlgo algo = blobHashAlgoFromWord(ha, "outcome log"); + /// Validate the digest width before `fromHex`: a width mismatch must surface as the + /// CORRUPTED_DATA required for malformed serialized input, not fromHex's BAD_ARGUMENTS. + if (hhex.size() != blobHashLenFor(algo) * 2) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS outcome log: digest width {} does not match algo '{}'", hhex.size(), ha); + e.ref = BlobRef{algo, codecFor(algo).fromHex(hhex)}; + e.token = Token{tv, tt}; + if (!line_in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS outcome log: junk after record"); + log.entries.push_back(std::move(e)); + } +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.h new file mode 100644 index 000000000000..09a850ee66ff --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcOutcomesFormat.h @@ -0,0 +1,58 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// The outcome log records observations made while settling GC candidates. Each log belongs to one +/// attempt-scoped generation, round, and shard at +/// `gc/gen/{g}/attempt/{a}/outcomes/{round}/{shard}`. It contains the results of exact-token deletes +/// for entries that were already published as `delete_pending`, as well as candidates spared when +/// the one-pass merge found a live in-degree. The log is written before the round's single state CAS; +/// `putIfAbsent` adopts an existing durable log on replay rather than treating a byte difference as +/// an error. The uncompressed payload is a header line, one flat JSON record per entry in insertion +/// order, and an `{"n":count}` trailer. `FormatId::GcOutcomes` stores the sealed payload in one zstd +/// frame, so its object-storage key has the `.zst` suffix. +enum class OutcomeKind : uint8_t +{ + Deleted = 1, /// The exact-token delete succeeded. + Absent = 2, /// The object was already absent, for example after a prior round's delete. + Replaced = 3, /// A 412 showed that a writer recreated the object with a new token. + Spared = 4, /// The merge found a positive in-degree, so the candidate was kept alive. +}; + +/// One observation about a blob incarnation considered by GC. `token` identifies the exact +/// incarnation that GC examined, while `ref` identifies the content address; retaining both lets +/// replay and inspection distinguish an absent object from a replacement that won a race with GC. +struct OutcomeEntry +{ + ObjectKind kind = ObjectKind::Blob; + BlobRef ref{}; + Token token; + OutcomeKind outcome = OutcomeKind::Spared; +}; + +/// The ordered records for one GC outcome object. Encoding preserves this insertion order because +/// the log is observation-bearing rather than a canonical deterministic artifact; decoding returns +/// only after the trailer count matches the records that were read. +struct OutcomeLog +{ + std::vector entries; +}; + +/// Encodes a log as the uncompressed `GcOutcomes` text payload. The caller is responsible for +/// applying the format registry's compression policy before storing the returned bytes. +String encodeOutcomeLog(const OutcomeLog & log); + +/// Decodes and validates a `GcOutcomes` text payload. The header, required record fields, supported +/// enum words, line boundaries, trailer count, and end-of-object condition are checked; malformed +/// input raises `CORRUPTED_DATA`. +OutcomeLog decodeOutcomeLog(std::string_view data); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.cpp new file mode 100644 index 000000000000..7012c6787f70 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.cpp @@ -0,0 +1,118 @@ +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +String encodeGcState(const GcState & state) +{ + if (state.gc_shards < 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "encodeGcState: gc_shards must be >= 1 -- refusing to persist an unreadable gc/state"); + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::GcState); + bool first = true; + writeKey(out, "rnd", first); writeU64StringValue(out, state.round); + writeKey(out, "gcs", first); writeIntText(state.gc_shards, out); + writeKey(out, "sg", first); writeU64StringValue(out, state.snap_generation); + writeKey(out, "spt", first); writeU64StringValue(out, state.snap_pruned_through); + writeKey(out, "sa", first); writeU64StringValue(out, state.snap_attempt); + writeKey(out, "msc", first); writeStringValue(out, state.manifest_sweep_cursor); + writeKey(out, "lo", first); writeHex128Value(out, state.lease.owner); + writeKey(out, "ls", first); writeU64StringValue(out, state.lease.seq); + closeObject(out, first); + writeChar('\n', out); + return std::move(out).take(); +} + +GcState decodeGcState(std::string_view data) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::GcState); + const String body = readLine(in, traitsFor(FormatId::GcState).line_cap, "gc/state"); + ReadBufferFromMemory body_in(body.data(), body.size()); + JsonObjectReader r(body_in, KeyStrictness::Tolerant, "gc/state"); + + GcState state; + bool saw_gcs = false; + String key; + while (r.nextKey(key)) + { + if (key == "rnd") state.round = r.readU64String(); + else if (key == "gcs") { state.gc_shards = r.readU64Number(); saw_gcs = true; } + else if (key == "sg") state.snap_generation = r.readU64String(); + else if (key == "spt") state.snap_pruned_through = r.readU64String(); + else if (key == "sa") state.snap_attempt = r.readU64String(); + else if (key == "msc") state.manifest_sweep_cursor = r.readString(); + else if (key == "lo") state.lease.owner = r.readHex128(); + else if (key == "ls") state.lease.seq = r.readU64String(); + else r.skipUnknown(key); + } + /// Fail closed on an absent gcs: the writer always emits it, so a missing key means a corrupt object. + /// Do NOT silently keep the struct default (1) — that would hide corruption (no-fallback principle). + if (!saw_gcs) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS gc/state: missing gcs"); + if (state.gc_shards == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS gc/state: gc_shards must be >= 1"); + if (!body_in.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS gc/state: trailing bytes"); + return state; +} + +String encodeGcHeartbeat(const GcHeartbeat & hb) +{ + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::GcHeartbeat); + bool first = true; + writeKey(out, "by", first); writeHex128Value(out, hb.owner); + writeKey(out, "seq", first); writeU64StringValue(out, hb.hb_seq); + closeObject(out, first); + writeChar('\n', out); + return std::move(out).take(); +} + +GcHeartbeat decodeGcHeartbeat(std::string_view data) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::GcHeartbeat); + const String body = readLine(in, traitsFor(FormatId::GcHeartbeat).line_cap, "gc heartbeat"); + ReadBufferFromMemory body_in(body.data(), body.size()); + JsonObjectReader r(body_in, KeyStrictness::Tolerant, "gc heartbeat"); + + GcHeartbeat hb; + bool saw_by = false; + bool saw_seq = false; + String key; + while (r.nextKey(key)) + { + if (key == "by") + { + hb.owner = r.readHex128(); + saw_by = true; + } + else if (key == "seq") + { + hb.hb_seq = r.readU64String(); + saw_seq = true; + } + else r.skipUnknown(key); + } + if (!saw_by || !saw_seq) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS gc heartbeat: missing identity field"); + if (!body_in.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS gc heartbeat: trailing bytes"); + return hb; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.h new file mode 100644 index 000000000000..88bce088ed13 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasGcStateFormat.h @@ -0,0 +1,71 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// The durable lease portion of the `gc/state` control object. `owner` identifies the current GC +/// leader and `seq` changes when that leader renews the lease. A contender compares the pair across +/// observations before stealing a stalled lease; a zero owner means that the lease has never been +/// held. +struct GcLease +{ + UInt128 owner{}; /// Random GC leader identifier; zero means the lease has never been held. + uint64_t seq = 0; /// Renewal counter observed by contenders when deciding whether a lease stalled. +}; + +/// The durable control state for GC rounds and snapshot generations. It is materialized as one +/// versioned JSON control object by `encodeGcState` and read back by `decodeGcState`. +/// +/// `round` and the snapshot fields are publication cursors: a committed round makes its retire sets +/// durable, while `snap_generation` and `snap_attempt` identify the fold seal whose snapshot was +/// adopted. The fold's per-namespace and per-shard cursor lives in that write-once fold seal, not in +/// this object. `gc_shards` is chosen once and must remain at least one; decoders reject an absent or +/// zero value instead of silently accepting the C++ default. The manifest sweep cursor is only a +/// best-effort orphan-manifest cleanup position and is never used for reachability decisions. +struct GcState +{ + uint64_t round = 0; /// the highest GC round whose retire sets are durable + uint64_t gc_shards = 1; /// GC blob-target-shard count; set once, immutable; must be >= 1 + uint64_t snap_generation = 0; /// monotone; the authoritative snap objects' generation + uint64_t snap_pruned_through = 0; /// highest snap generation fully pruned (retention cursor) + uint64_t snap_attempt = 0; /// adopted attempt id (folding leader's lease.seq) for snap_generation + String manifest_sweep_cursor; /// best-effort orphan part-manifest cleanup cursor; reachability ignores it + GcLease lease; +}; + +/// Encode `state` as the canonical `cas_gc_state` text object: a versioned header line followed by +/// one JSON body object. The writer always emits the complete field set and asserts the invariant +/// that `gc_shards` is nonzero. +String encodeGcState(const GcState & state); + +/// Decode a complete `cas_gc_state` text object. The header and size limits are checked before the +/// body is parsed; unknown non-reserved fields are tolerated for forward evolution, but malformed +/// input, trailing bytes, a missing `gcs`, or a zero shard count raises `CORRUPTED_DATA` rather than +/// falling back to a default state. +GcState decodeGcState(std::string_view data); + +/// Advisory liveness state for the GC leader lease. The leader increments `hb_seq` on a fast cadence +/// independently of round progress, because its lease renewal counter can remain unchanged during a +/// long fold. A follower that observes the heartbeat advance backs off from stealing the lease; this +/// prevents mistaking an alive, mid-round leader for a stalled one. The value is persisted as the +/// versioned `cas_gc_hb` text object, whose body contains `by` and `seq` string values, replacing the +/// former unversioned 24-byte record. +struct GcHeartbeat +{ + UInt128 owner{}; + uint64_t hb_seq = 0; +}; + +/// Encode a complete heartbeat as a canonical versioned header line and one JSON body object. +String encodeGcHeartbeat(const GcHeartbeat & hb); + +/// Decode a complete `cas_gc_hb` text object, rejecting malformed input and trailing bytes with +/// `CORRUPTED_DATA`. Unknown non-reserved fields remain skippable for forward evolution. +GcHeartbeat decodeGcHeartbeat(std::string_view data); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.cpp new file mode 100644 index 000000000000..5bd928ec01a3 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.cpp @@ -0,0 +1,345 @@ +#include +#include +#include +#include + +namespace DB::Cas +{ + +namespace +{ + +/// Parses a canonical unsigned-decimal path segment (the shape `std::to_string` produces): non-empty, +/// digits only, no leading zero unless the segment is exactly "0", and fits in a `uint64_t`. Returns +/// `std::nullopt` for anything else -- never throws, since key parsing classifies a foreign/malformed +/// segment as debris, not an error (mirrors `parseManifestKey`'s ordinal parse). +std::optional parseCanonicalU64(std::string_view s) +{ + if (s.empty() || (s.size() > 1 && s[0] == '0')) + return std::nullopt; + for (char c : s) + if (c < '0' || c > '9') + return std::nullopt; + uint64_t v = 0; + const auto [ptr, ec] = std::from_chars(s.data(), s.data() + s.size(), v); + if (ec != std::errc{} || ptr != s.data() + s.size()) + return std::nullopt; /// overflowed uint64_t or otherwise not fully consumed + return v; +} + +} + +/// Blob-key operations are kept together with their parsing inverse. `CasTypes.h` supplies the +/// complete `BlobRef` type and the hash-algorithm helpers used to render and validate the path. +String Layout::blobKey(const BlobRef & ref) const +{ + return shardedKey("blobs/" + String(blobHashAlgoName(ref.algo)), blobHexOf(ref)); +} + +String Layout::blobMetaKey(const BlobRef & ref) const +{ + return blobKey(ref) + ".meta"; +} + +std::optional Layout::parseBlobKey(std::string_view key) const +{ + std::string_view rest = key; + static constexpr std::string_view kMetaSuffix = ".meta"; + if (rest.size() >= kMetaSuffix.size() && rest.substr(rest.size() - kMetaSuffix.size()) == kMetaSuffix) + rest.remove_suffix(kMetaSuffix.size()); + + const String blobs_root = blobsPrefix(); /// "/blobs/" + if (rest.size() <= blobs_root.size() || !rest.starts_with(blobs_root)) + return std::nullopt; + rest.remove_prefix(blobs_root.size()); + + const size_t algo_sep = rest.find('/'); + if (algo_sep == std::string_view::npos) + return std::nullopt; + const std::string_view algo_name = rest.substr(0, algo_sep); + rest.remove_prefix(algo_sep + 1); + + const size_t shard_sep = rest.find('/'); + if (shard_sep == std::string_view::npos) + return std::nullopt; + const std::string_view shard = rest.substr(0, shard_sep); + const std::string_view hex = rest.substr(shard_sep + 1); + if (shard.size() != 2 || hex.size() < 2 || shard != hex.substr(0, 2)) + return std::nullopt; /// malformed shard/hex shape -- not ours + + /// `` -> `BlobHashAlgo`: the small enum-value set makes a linear scan against + /// `blobHashAlgoName` (the ONE name authority) cheaper and safer than a second name table that + /// could drift from it. + std::optional algo; + for (BlobHashAlgo candidate : {BlobHashAlgo::CityHash128, BlobHashAlgo::XXH3_128, BlobHashAlgo::Sha256}) + if (algo_name == blobHashAlgoName(candidate)) + { + algo = candidate; + break; + } + if (!algo) + return std::nullopt; /// unknown/foreign algo segment -- debris, not ours + + if (hex.size() != 2 * blobHashLenFor(*algo)) + return std::nullopt; /// a KNOWN algo but the wrong-width hex -- not ours either + + try + { + return BlobRef{*algo, codecFor(*algo).fromHex(String(hex))}; + } + catch (const Exception &) + { + return std::nullopt; /// non-hex characters -- malformed, not ours + } +} + +std::optional Layout::parseRefObjectKey(std::string_view key) const +{ + const String base = casRefsPrefix(); + if (!key.starts_with(base)) + return std::nullopt; + std::string_view rest = key; + rest.remove_prefix(base.size()); + + const size_t id_sep = rest.rfind('/'); + if (id_sep == std::string_view::npos) + return std::nullopt; + std::string_view id_part = rest.substr(id_sep + 1); + std::string_view before_id = rest.substr(0, id_sep); + + const size_t kind_sep = before_id.rfind('/'); + if (kind_sep == std::string_view::npos) + return std::nullopt; + const std::string_view kind_seg = before_id.substr(kind_sep + 1); + const std::string_view before_kind = before_id.substr(0, kind_sep); + if (before_kind.empty()) + return std::nullopt; + + RefObjectKind kind{}; + if (kind_seg == "_log") + kind = RefObjectKind::Log; + else if (kind_seg == "_snap") + kind = RefObjectKind::Snap; + else + return std::nullopt; + + std::string_view render = id_part; + /// `_log` and `_snap` are always-compressed text stored under a `.zst` suffix. + constexpr std::string_view kZstSuffix = ".zst"; + if (!render.ends_with(kZstSuffix)) + return std::nullopt; + render.remove_suffix(kZstSuffix.size()); + + const auto txn_id = parseRefTxnId(render); + if (!txn_id) + return std::nullopt; + + if (before_kind.find('/') != std::string_view::npos) + return std::nullopt; + return ParsedRefObjectKey{namespaceLifePhysicalIdOf(key, before_kind), kind, *txn_id}; +} + +std::optional Layout::parseRefCkptKey(std::string_view key) const +{ + const String base = namespaceStateRootPrefix(); + if (!key.starts_with(base)) + return std::nullopt; + std::string_view rest = key; + rest.remove_prefix(base.size()); + + /// `/_ckpt` -- the leaf is the fixed name plus whatever the registry's + /// compression policy appends. Built from the same pieces `refCkptKey` uses, so the two cannot + /// drift apart. + const String leaf = "_ckpt" + String(storedSuffix(FormatId::RefCkpt)); + const size_t leaf_sep = rest.rfind('/'); + if (leaf_sep == std::string_view::npos || rest.substr(leaf_sep + 1) != leaf) + return std::nullopt; + const std::string_view life_id = rest.substr(0, leaf_sep); + if (life_id.empty() || life_id.find('/') != std::string_view::npos) + return std::nullopt; + + return namespaceLifePhysicalIdOf(key, life_id); +} + +std::optional Layout::parseNamespaceFileKey(std::string_view key) const +{ + const String base = namespaceStateRootPrefix(); + if (!key.starts_with(base)) + return std::nullopt; + std::string_view rest = key; + rest.remove_prefix(base.size()); + + /// The first `/_files/` separates the single `` segment from the relative name. A + /// relative name may itself contain `_files`, so the first occurrence is the delimiter. + static constexpr std::string_view kFilesSegment = "/_files/"; + const size_t files_pos = rest.find(kFilesSegment); + if (files_pos == std::string_view::npos) + return std::nullopt; /// no reserved segment: a loose mountpoint object, not one of our files + + const std::string_view life_id = rest.substr(0, files_pos); + const std::string_view relative_name = rest.substr(files_pos + kFilesSegment.size()); + if (relative_name.empty()) + return std::nullopt; /// the files prefix itself names no file + + if (life_id.empty() || life_id.find('/') != std::string_view::npos) + return std::nullopt; + /// Mirror `namespaceFileKey`'s writer-side grammar: a relative name no current writer could have + /// produced (a dirty path, `..` segment, doubled slash, ...) is not one of ours, even though the + /// `_files/` segment and life id both parse. `namespaceLifePhysicalIdOf` below throws + /// `CORRUPTED_DATA` for the sibling life-id case; this mirrors that severity for symmetry. + if (!isCleanRelativeNamespaceFileName(relative_name)) + throw DB::Exception(DB::ErrorCodes::CORRUPTED_DATA, + "CasLayout: object '{}' names no clean namespace file: '{}' is not a relative path any " + "current writer could have produced", key, relative_name); + return ParsedNamespaceFileKey{namespaceLifePhysicalIdOf(key, life_id), String(relative_name)}; +} + +NamespaceLifePhysicalId Layout::namespaceLifePhysicalIdOf(std::string_view key, std::string_view segment) const +{ + const auto incarnation = parseIncarnation(segment); + if (!incarnation) + throw DB::Exception(DB::ErrorCodes::CORRUPTED_DATA, + "CasLayout: object '{}' names no life: '{}' is not 32 lower-case hex digits of a nonzero " + "life id. Generation-5 namespace-bearing pools are rejected by the pool-metadata format " + "gate before this generation-6 physical-key parser is reached", + key, segment); + return *incarnation; +} + +std::optional Layout::parseManifestKey(std::string_view key) const +{ + const String base = casManifestsPrefix(); + if (!key.starts_with(base)) + return std::nullopt; + std::string_view rest = key; + rest.remove_prefix(base.size()); + + const size_t file_sep = rest.rfind('/'); + if (file_sep == std::string_view::npos) + return std::nullopt; + const std::string_view file = rest.substr(file_sep + 1); + const std::string_view before_file = rest.substr(0, file_sep); + + const size_t build_sep = before_file.rfind('/'); + if (build_sep == std::string_view::npos) + return std::nullopt; + const std::string_view build_seg = before_file.substr(build_sep + 1); + const std::string_view ns_part = before_file.substr(0, build_sep); + if (ns_part.empty()) + return std::nullopt; + + const auto build = parseRefTxnId(build_seg); + if (!build) + return std::nullopt; + + const std::string_view kManifestSuffix = storedSuffix(FormatId::PartManifest); + constexpr size_t kOrdinalDigits = 6; + if (file.size() != kOrdinalDigits + kManifestSuffix.size() || !file.ends_with(kManifestSuffix)) + return std::nullopt; + const std::string_view ordinal_str = file.substr(0, kOrdinalDigits); + uint32_t ordinal = 0; + for (char c : ordinal_str) + { + if (c < '0' || c > '9') + return std::nullopt; + ordinal = ordinal * 10 + static_cast(c - '0'); + } + if (ordinal == 0 || ordinal > kMaxManifestOrdinal) + return std::nullopt; + + ManifestId parsed; + parsed.root_namespace = RootNamespace{String(ns_part)}; + parsed.ref.writer_epoch = build->writer_epoch; + parsed.ref.build_sequence = build->ref_sequence; + parsed.ref.manifest_ordinal = ordinal; + return parsed; +} + +std::optional Layout::parseBlobTargetRunKey(std::string_view key) const +{ + const String base = prefix + "/gc/gen/"; + if (!key.starts_with(base)) + return std::nullopt; + std::string_view rest = key; + rest.remove_prefix(base.size()); + + /// Splits the next '/'-delimited segment off the front of `rest`, returning `std::nullopt` if + /// `rest` has no further '/' (an incomplete key shape). + auto takeSegment = [](std::string_view & s) -> std::optional + { + const size_t sep = s.find('/'); + if (sep == std::string_view::npos) + return std::nullopt; + const std::string_view seg = s.substr(0, sep); + s.remove_prefix(sep + 1); + return seg; + }; + + const auto generation_seg = takeSegment(rest); + if (!generation_seg) + return std::nullopt; + const auto generation = parseCanonicalU64(*generation_seg); + if (!generation) + return std::nullopt; + + const auto attempt_lit = takeSegment(rest); + if (!attempt_lit || *attempt_lit != "attempt") + return std::nullopt; + + const auto attempt_seg = takeSegment(rest); + if (!attempt_seg) + return std::nullopt; + const auto attempt = parseCanonicalU64(*attempt_seg); + if (!attempt) + return std::nullopt; + + const auto blob_target_lit = takeSegment(rest); + if (!blob_target_lit || *blob_target_lit != "blob_target") + return std::nullopt; + + const auto shard_seg = takeSegment(rest); + if (!shard_seg) + return std::nullopt; + const auto shard = parseCanonicalU64(*shard_seg); + if (!shard) + return std::nullopt; + + /// `rest` is now the final segment (`seq`): reject trailing garbage (a further '/'). + if (rest.find('/') != std::string_view::npos) + return std::nullopt; + const auto seq = parseCanonicalU64(rest); + if (!seq) + return std::nullopt; + + return ParsedBlobTargetRunKey{*generation, *attempt, *shard, *seq}; +} + +/// A namespace must be non-empty, with no leading/trailing '/', no empty segment ("//"), and no +/// segment equal to the reserved "_files". +void Layout::checkNamespace(const RootNamespace & ns) const +{ + const String & s = ns.string(); + if (s.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "CasLayout: namespace must be non-empty"); + + size_t start = 0; + while (true) + { + size_t end = s.find('/', start); + const String segment = s.substr(start, end == String::npos ? String::npos : end - start); + if (segment.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "CasLayout: namespace '{}' has an empty segment (leading/trailing or doubled '/')", s); + if (segment == "_files") + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "CasLayout: namespace '{}' uses the reserved segment '_files'", s); + if (segment == "_manifests") + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "CasLayout: namespace '{}' uses the reserved segment '_manifests'", s); + if (end == String::npos) + break; + start = end + 1; + } +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h new file mode 100644 index 000000000000..093f380bd0e2 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h @@ -0,0 +1,480 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int CORRUPTED_DATA; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +/// Shared writer/parser grammar for a namespace file's relative name (the part after `_files/`): +/// no empty name, no leading or trailing `/`, no `//`, and no `..` path segment anywhere in it. +/// `Layout::namespaceFileKey` (the writer) and `Layout::parseNamespaceFileKey` (the reader) both +/// call this so their definition of "clean" cannot drift apart. +inline bool isCleanRelativeNamespaceFileName(std::string_view name) +{ + return !(name.empty() || name.front() == '/' || name.back() == '/' + || name.find("//") != std::string_view::npos || name == ".." || name.starts_with("../") + || name.ends_with("/..") || name.find("/../") != std::string_view::npos); +} + +/// Which immutable ref-object kind a `_log` or `_snap` key names. +enum class RefObjectKind : uint8_t +{ + Log, + Snap, +}; + +/// The result of `Layout::parseRefObjectKey`: the opaque physical life id, ref-object kind, and +/// canonical transaction identifier recovered from a listed stream key. Logical resolution belongs +/// to the immutable catalog cut held by the consumer, never to the key parser. +struct ParsedRefObjectKey +{ + NamespaceLifePhysicalId life_id = 0; + RefObjectKind kind = RefObjectKind::Log; + RefTxnId txn_id; + + bool operator==(const ParsedRefObjectKey &) const = default; +}; + +/// The result of `Layout::parseNamespaceFileKey`: the opaque physical life id that owns a listed +/// verbatim file and the file name relative to that life's `_files/` prefix. +struct ParsedNamespaceFileKey +{ + NamespaceLifePhysicalId life_id; + String relative_name; + + bool operator==(const ParsedNamespaceFileKey &) const = default; +}; + +/// The result of `Layout::parseBlobTargetRunKey`: the (generation, attempt, shard, seq) coordinates +/// recovered from a listed blob-target in-degree/delta run segment key. +struct ParsedBlobTargetRunKey +{ + uint64_t generation = 0; + uint64_t attempt = 0; + uint64_t shard = 0; + uint64_t seq = 0; + + bool operator==(const ParsedBlobTargetRunKey &) const = default; +}; + +/// Builds and parses object-storage keys for one content-addressed pool. +/// +/// `Layout` owns only the pool prefix; it does not own storage, cache state, or a pool-wide blob hash +/// algorithm. Callers use its pure methods to derive keys for blobs, manifests, refs, verbatim files, +/// and GC control data, and to classify listed keys before acting on them. Logical-name key builders +/// validate namespaces, while parsers deliberately return `std::nullopt` for foreign or +/// malformed listed keys so sweeps can treat those keys as debris without exceptions. +/// +/// Every key is built from a pool prefix and a stable path subtree. The main families are: +/// - content objects: POOL/blobs/ALGO/S/HEX +/// - part manifests: POOL/cas/manifests/NAMESPACE/BUILD/ORDINAL.zst +/// - ref stream: POOL/cas/ns/stream/LIFE_ID/_log|_snap/ID +/// - namespace state: POOL/cas/ns/state/LIFE_ID/_ckpt +/// POOL/cas/ns/state/LIFE_ID/_files/FILE_NAME +/// - GC state: POOL/gc/... +/// - pool metadata: POOL/_pool_meta +/// +/// NAMESPACE is opaque to the core: the wiring composes strings like "srv1/" or +/// "shadow//". The reserved "_files" and "_manifests" segments cannot collide +/// with root shard keys because shard names are numeric, and `checkNamespace` rejects both as +/// namespace segments. +/// +/// The 2-char shard is always the first two characters of the id string. +/// This matches the protocol's fixed two-character shard layout. +/// +/// Blob bodies carry their own `BlobHashAlgo` as a path segment: +/// `POOL/blobs//S/`, where `` is `blobHashAlgoName(ref.algo)` (`"ch128"`, `"xxh3"`, +/// `"sha256"`) and ``/`S` are taken from `ref.digest` at the algo's own width. This applies to +/// ALL algos, including the pool's default, so blob keys are uniformly self-describing and a pool may +/// hold blobs under several algos at once. Trees/manifests/refs/gc keys are unaffected -- only +/// blob-body keys (`blobKey`/`blobMetaKey`) gain the segment. `Layout` itself carries NO +/// algo -- there is no pool-wide "the" algo anymore; every blob key is built from a `BlobRef` alone. +class Layout +{ +public: + explicit Layout(String prefix_) : prefix(std::move(prefix_)) {} + + /// Content objects: POOL/blobs//S/, with ``/`` taken from `ref` itself. + /// Defined out-of-line in `CasLayout.cpp`, where the key construction and parsing helpers remain + /// together. + String blobKey(const BlobRef & ref) const; + /// The per-hash meta descriptor sibling of the blob body. + String blobMetaKey(const BlobRef & ref) const; + + /// Inverse of `blobKey`/`blobMetaKey`: parses a listed object key of the shape + /// `/blobs///` (the `.meta` sibling is accepted identically -- its + /// trailing `.meta` is stripped first, so a body and its meta parse to the SAME `BlobRef`). + /// Returns `std::nullopt` for anything that is not one of OUR blob keys: a foreign top-level + /// prefix, a missing shard/hex segment, an `` this build does not recognize + /// (`blobHashAlgoName` never rendered it), a hex payload of the wrong width for a KNOWN algo, or + /// non-hex characters -- every case is "debris, not ours", never an exception (callers classify + /// it as foreign/unaccounted, mirroring the LIST sweep's existing `catch (...) continue` + /// contract). Defined out-of-line in `CasLayout.cpp` alongside `blobKey` and `blobMetaKey`. + std::optional parseBlobKey(std::string_view key) const; + + /// Immutable stream prefix for one physical life. + String namespaceStreamPrefix(const NamespaceLifeId & life) const + { + return namespaceStreamRootPrefix() + renderIncarnation(life.incarnation) + "/"; + } + + /// Point/path-addressed state prefix for one physical life. + String namespaceStatePrefix(const NamespaceLifeId & life) const + { + return namespaceStateRootPrefix() + renderIncarnation(life.incarnation) + "/"; + } + + /// Pool-wide immutable stream prefix. GC's one hot namespace enumeration is scoped here. + String casRefsPrefix() const + { + return namespaceStreamRootPrefix(); + } + + String namespaceStreamRootPrefix() const { return prefix + "/cas/ns/stream/"; } + String namespaceStateRootPrefix() const { return prefix + "/cas/ns/state/"; } + String namespaceRootPrefix() const { return prefix + "/cas/ns/"; } + + /// Immutable transaction log object at + /// `/cas/ns/stream//_log/.zst`. The log is the text `cas_ref_log` stored with + /// the format's always-compressed `.zst` suffix; readers construct the one canonical key and do + /// not try an uncompressed variant. + String refLogKey(const NamespaceLifeId & ns_id, const RefTxnId & id) const + { + return namespaceStreamPrefix(ns_id) + "_log/" + renderRefTxnId(id) + String(storedSuffix(FormatId::RefLog)); + } + + /// Writer-published table snapshot at `.../_snap/.zst`. The snapshot + /// is the text `cas_ref_snap` stored with the format's always-compressed `.zst` suffix. Snapshot + /// `X` reuses the `RefTxnId` of the last log it covers. + String refSnapshotKey(const NamespaceLifeId & ns_id, const RefTxnId & id) const + { + return namespaceStreamPrefix(ns_id) + "_snap/" + renderRefTxnId(id) + String(storedSuffix(FormatId::RefSnapshot)); + } + + /// The life's checkpoint object (spec INV-4) at `/cas/ns/state//_ckpt`. Unlike + /// immutable stream objects it is mutable (token-CAS), carries no transaction id, and therefore lives + /// in the point/path-addressed state tree rather than a `_log`/`_snap` directory -- + /// which is also why `parseRefObjectKey` does not recognize it and `parseRefCkptKey` exists. + String refCkptKey(const NamespaceLifeId & ns_id) const + { + return namespaceStatePrefix(ns_id) + "_ckpt" + String(storedSuffix(FormatId::RefCkpt)); + } + + /// Inverse of `refCkptKey`: returns the opaque physical life id when `key` is exactly one of our `_ckpt` + /// keys, and `std::nullopt` when the key is not one at all (a foreign pool prefix, a different + /// leaf name, an id-bearing ref object) -- classifying an untrusted listed key stays an ordinary + /// "is this ours" question. Logical resolution is a separate join against one catalog cut. + /// + /// It REFUSES, with `CORRUPTED_DATA` naming the key, a key whose leaf IS `_ckpt` but whose + /// incarnation segment is missing, non-canonical or zero: behind Stage B's format bump the + /// un-incarnated (Stage A) shape names no live object, and treating it as foreign debris would let + /// it sit unnoticed under a live namespace. + /// + /// Stream sweeps never consult this parser: checkpoints are deliberately outside the hot + /// `casRefsPrefix` enumeration. + std::optional parseRefCkptKey(std::string_view key) const; + + /// Inverse of `refLogKey`/`refSnapshotKey`: classifies a LISTED key under `casRefsPrefix()` by its + /// kind directory (`_log` or `_snap`) and parses the trailing + /// `RefTxnId` and the life it belongs to. Strict: returns `std::nullopt` for anything that is not + /// one of our ref-object keys: a foreign top-level prefix, a missing life/kind/id segment, an + /// unrecognized kind directory, a `_log`/`_snap` id missing its + /// `.zst` suffix (both are always-compressed text), trailing garbage after the id, or a + /// non-canonical `RefTxnId` render (delegates to + /// `parseRefTxnId`). + /// + /// It throws `CORRUPTED_DATA` when the key otherwise has a recognized stream-object shape but its + /// physical life-id segment is missing, non-canonical or zero. Logical names never come from keys. + /// Defined out-of-line in `CasLayout.cpp`, where the key construction and parsing helpers remain + /// together. + std::optional parseRefObjectKey(std::string_view key) const; + + /// Prefix that covers all part manifests of a namespace: `/cas/manifests//`. + String manifestNamespacePrefix(const RootNamespace & ns) const + { + checkNamespace(ns); + return prefix + "/cas/manifests/" + ns.string() + "/"; + } + + /// Pool-wide part-manifest prefix: `/cas/manifests/`. + String casManifestsPrefix() const + { + return prefix + "/cas/manifests/"; + } + + /// Verbatim (non-content-addressed) file stored under ONE LIFE of a namespace. Names may be nested + /// (relative sub-paths — the wiring stores table-level subdirectory files such as + /// deduplication_logs/deduplication_log_1.txt verbatim); empty segments, leading or + /// trailing '/', and '..' segments are rejected (no escaping the life's files prefix). + /// + /// The incarnation segment is what makes a file of a previous life structurally unreachable from + /// the next one (directive §2): once the catalog entry is gone, nothing can name that life's files + /// again, so a LIST that omitted an old file can only leak storage — it can no longer let the file + /// become visible under a reborn namespace of the same name. + String namespaceFileKey(const NamespaceLifeId & life, const String & file_name) const + { + if (!isCleanRelativeNamespaceFileName(file_name)) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "CasLayout: namespace file name must be a clean relative path, got '{}'", file_name); + return namespaceFilesPrefix(life) + file_name; + } + + /// Prefix that covers all verbatim files of ONE LIFE of a namespace (for list). A LIST of it + /// enumerates that life's files and nothing of any earlier life of the same namespace name. + String namespaceFilesPrefix(const NamespaceLifeId & life) const + { + return namespaceStatePrefix(life) + "_files/"; + } + + /// Inverse of `namespaceFileKey`: classifies a listed key under `namespaceStateRootPrefix` by the reserved + /// `_files` segment and returns the life it belongs to plus the name relative to that life's files + /// prefix. Returns `std::nullopt` for anything that is not one of OUR namespace files: a foreign + /// top-level prefix, a key with no `_files` segment at all, or nothing after the reserved segment. + /// + /// It THROWS `CORRUPTED_DATA` naming the key in the same one situation the ref parsers do: the key + /// carries the reserved segment, but the segment where its incarnation belongs is missing, + /// non-canonical or zero. Behind Stage B's format bump that is the un-incarnated (Stage A) shape, + /// and classifying it as foreign debris would leave it sitting unnoticed under a live namespace. + /// + /// The first `_files` segment separates the single life-id segment from the relative name; a + /// relative name may itself contain `_files`. Defined out-of-line in `CasLayout.cpp`, where the key construction and + /// parsing helpers remain together. + std::optional parseNamespaceFileKey(std::string_view key) const; + + /// Part manifest body key, in canonical hex form: + /// /cas/manifests//-/<000001>.zst + /// The build-scoped directory reuses `RefTxnId`'s hex rendering for `{writer_epoch, + /// build_sequence}` (same durable-epoch fence and hex width as a ref transaction id -- a different + /// counter with different semantics, not the same identifier). `manifest_ordinal` is a + /// per-build ordinal rendered as a six-digit filename. `root_namespace_id` comes from the owning + /// context (the `ManifestId`), never from the journal ref. + String manifestKey(const ManifestId & id) const + { + checkNamespace(id.root_namespace); + return prefix + "/cas/manifests/" + id.root_namespace.string() + "/" + + renderRefTxnId(RefTxnId{id.ref.writer_epoch, id.ref.build_sequence}) + "/" + + manifestOrdinalFileName(id.ref.manifest_ordinal); + } + + /// Inverse of `manifestKey`: parses `/cas/manifests//-/.zst`. + /// Strict: rejects the old decimal directory shape (it is not two fixed-width hex fields joined by + /// '-'), a missing namespace/build/ordinal segment, trailing garbage, a file not ending in the + /// registered suffix or of the wrong width, and an out-of-range or non-canonical ordinal. + /// Foreign/malformed keys return `std::nullopt`, never throw -- LIST sweep / fsck classify by key + /// shape, not by validity. All manifest-path parsing (sweep, fsck) routes through this one function. + /// Defined out-of-line in `CasLayout.cpp`, where the key construction and parsing helpers remain + /// together. + std::optional parseManifestKey(std::string_view key) const; + + /// A plain mountpoint object is a loose, non-content-addressed file mirrored at its + /// ClickHouse path under `roots/`, with NO namespace and NO `_files` wrapper. `key` is the + /// server-prefixed mirrored path (e.g. `srv1/clickhouse_access_check_abc`). It must NOT end in a + /// reserved area. Namespace discovery is catalog-authoritative, not derived from loose roots. + /// The `_files`/`_pool_meta` reservations still apply to its segments via the path itself + /// (these never appear in a real ClickHouse loose-file path). + String mountpointObjectKey(const String & key) const + { + if (key.empty() || key.front() == '/' || key.back() == '/' || key.find("//") != String::npos) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "CasLayout: mountpoint object key must be a clean relative path, got '{}'", key); + return prefix + "/roots/" + key; + } + + /// GC keys. + String gcStateKey() const + { + return prefix + "/gc/state"; + } + + /// Leak-only namespace-janitor cursor. It is deliberately separate from GC authority state. + String gcMaintenanceStateKey() const + { + return prefix + "/gc/maintenance_state"; + } + + /// GC heartbeat (advisory liveness pulse): `/gc/hb`. + String gcHbKey() const + { + return prefix + "/gc/hb"; + } + + /// Prefix that covers EVERY per-round artifact of one generation (all attempts): /gc/gen// + /// The wholesale retention prune reclaims a whole generation (every attempt's debris) by this prefix. + String gcGenPrefix(uint64_t generation) const + { + return prefix + "/gc/gen/" + std::to_string(generation) + "/"; + } + + /// Prefix that covers one (generation, attempt)'s artifacts: /gc/gen//attempt// + /// `attempt` is the folding leader's monotonic per-round id; only the adopted attempt is reader-visible. + String gcGenAttemptPrefix(uint64_t generation, uint64_t attempt) const + { + return gcGenPrefix(generation) + "attempt/" + std::to_string(attempt) + "/"; + } + + /// Per-(generation, attempt) FOLD seal (write-once): /gc/gen//attempt//fold_seal. + String foldSealKey(uint64_t generation, uint64_t attempt) const + { + return gcGenAttemptPrefix(generation, attempt) + "fold_seal"; + } + + /// One blob-target in-degree/delta run segment: + /// /gc/gen//attempt//blob_target// + String blobTargetRunKey(uint64_t generation, uint64_t attempt, uint64_t shard, uint64_t seq) const + { + return gcGenAttemptPrefix(generation, attempt) + "blob_target/" + + std::to_string(shard) + "/" + std::to_string(seq); + } + + /// Inverse of `blobTargetRunKey`: parses + /// `/gc/gen//attempt//blob_target//`. Strict: rejects a + /// missing/foreign top-level prefix, a missing `attempt`/`blob_target` literal segment, a missing + /// generation/attempt/shard/seq segment, a non-canonical decimal component (empty, non-digit, a + /// leading zero other than a bare "0", or a value that overflows `uint64_t`), or trailing garbage. + /// Foreign/malformed keys return `std::nullopt`, never throw -- callers (`cas-inspect`) classify by + /// key shape, not by validity. Defined out-of-line in `CasLayout.cpp`, where the key construction + /// and parsing helpers remain together. + std::optional parseBlobTargetRunKey(std::string_view key) const; + + /// Outcomes key: /gc/gen//attempt//outcomes//.zst + /// The `.zst` suffix comes from the traits table (cas_gc_outcomes is the one Always-compressed + /// control object): a constructed key names the compressed object deterministically, no body sniff. + String outcomesKey(uint64_t generation, uint64_t attempt, uint64_t round, uint64_t shard) const + { + return gcGenAttemptPrefix(generation, attempt) + "outcomes/" + std::to_string(round) + "/" + std::to_string(shard) + + String(storedSuffix(FormatId::GcOutcomes)); + } + + /// Prefix that covers every root-shard manifest and namespace file (GC round discovery). + String rootsPrefix() const + { + return prefix + "/roots/"; + } + + /// Prefix that covers every content blob (raw object listing for fsck). Deliberately stays + /// `/blobs/` (no algo segment) even though `blobKey` nests one level deeper under + /// `blobs//S/ID`: a recursive LIST of this prefix still returns every blob object across all + /// algos in one sweep. Any code that PARSES a listed key back to a hash must take the LAST path + /// component (the hex digest), which stays correct regardless of the algo segment (`CasGc.cpp` / + /// `CasFsck.cpp` already do this via `rfind('/')`). + /// + /// The S3-staging area lives under `/staging//` — a distinct top-level sibling of + /// `blobs/`, `cas/ns/`, and + /// `cas/manifests/`, never a sub-path of any of them. Every GC blob-discovery LIST (`CasGc.cpp`, + /// `CasFsck.cpp`) enumerates ONLY this `blobsPrefix()`, so a `staging/` object can never be listed, + /// HEAD'd, or condemned as an orphan blob — `Cas::sweepOwnMountStaging` (`CasStagingSweeper.h`) is + /// the sole reclaimer of `staging/` debris. + String blobsPrefix() const { return prefix + "/blobs/"; } + + /// Per-server-root control subtree, keyed by the configured `server_root_id` + /// (validated by `DB::Cas::validateServerRootId`). All four control objects live together under + /// `/gc/server-roots//` so a server's mount-safety state is one subtree. + String serverRootPrefix(const String & server_root_id) const + { + return prefix + "/gc/server-roots/" + server_root_id + "/"; + } + + /// Pool-wide server-roots prefix: `/gc/server-roots/`. The base of every + /// `serverRootPrefix`; the GC heartbeat gate LISTs it to enumerate all mount objects (it must + /// filter to keys ending in `/mount`, since `/owner` and `/epoch` objects share the subtree). + String serverRootsPrefix() const + { + return prefix + "/gc/server-roots/"; + } + + /// Owner anchor: `/gc/server-roots//owner`. + String ownerKey(const String & server_root_id) const + { + return serverRootPrefix(server_root_id) + "owner"; + } + + /// Writer-epoch fence: `/gc/server-roots//epoch`. + String epochKey(const String & server_root_id) const + { + return serverRootPrefix(server_root_id) + "epoch"; + } + + /// Mount lease: `/gc/server-roots//mount`. + String mountKey(const String & server_root_id) const + { + return serverRootPrefix(server_root_id) + "mount"; + } + + /// The data subtree owned by a server root: `/roots//`. The mount-safety empty-root + /// precondition lists this prefix before data, ref, or manifest writes are admitted. + String serverRootDataPrefix(const String & server_root_id) const + { + return prefix + "/roots/" + server_root_id + "/"; + } + + /// Per-server-root content-addressed manifest subtree: `/cas/manifests//`. It is + /// included in the mount-safety empty-root check even before the first manifest is written. + String casManifestsServerPrefix(const String & server_root_id) const + { + return prefix + "/cas/manifests/" + server_root_id + "/"; + } + + /// Pool-level metadata object. + String poolMetaKey() const + { + return prefix + "/_pool_meta"; + } + + /// The whole-pool namespace catalog (spec INV-3): one object, token-CAS like `gc/state`, read on + /// every fold round and every recovery. + String refCatalogKey() const + { + return prefix + "/cas/ref_catalog"; + } + + /// Public validator for a namespace reconstructed from an untrusted listed key (GC ref intake): + /// `parseRefObjectKey` returns the namespace without checking its shape, so a + /// consumer that will act on it must re-validate. Throws BAD_ARGUMENTS on a malformed namespace, + /// exactly as every key-building method does. + void validateNamespace(const RootNamespace & ns) const { checkNamespace(ns); } + + /// The pool's root key prefix, i.e. the constructor's `prefix_` verbatim. Exposed for diagnostics + /// only (e.g. scoping a free-function's log line to the pool it operates on when no LoggerPtr is + /// threaded that deep) -- no key-building method needs this, they already have `prefix` in scope. + const String & poolPrefix() const { return prefix; } + +private: + String prefix; + + /// A namespace must be non-empty, with no leading/trailing '/', no empty segment ("//"), + /// and no segment equal to the reserved "_files". Defined out-of-line in `CasLayout.cpp`, where + /// the key construction and parsing helpers remain together. + void checkNamespace(const RootNamespace & ns) const; + + /// Parses the one physical-id segment after the rest of a life-owned key identified its family. + NamespaceLifePhysicalId namespaceLifePhysicalIdOf(std::string_view key, std::string_view segment) const; + + /// Build ///. + /// Throws BAD_ARGUMENTS if id is shorter than 2 characters. + String shardedKey(const String & ns, const String & id) const + { + if (id.size() < 2) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "CasLayout: id must be at least 2 characters, got '{}'", id); + return prefix + "/" + ns + "/" + id.substr(0, 2) + "/" + id; + } +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.cpp new file mode 100644 index 000000000000..c944bce3d284 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.cpp @@ -0,0 +1,353 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +namespace +{ + +std::string_view placementToWord(EntryPlacement p) +{ + switch (p) + { + case EntryPlacement::Inline: return "inline"; + case EntryPlacement::Blob: return "blob"; + } + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: unknown placement {}", static_cast(p)); +} + +EntryPlacement placementFromWord(std::string_view w) +{ + if (w == "inline") return EntryPlacement::Inline; + if (w == "blob") return EntryPlacement::Blob; + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: unknown placement '{}'", w); +} + +/// One entry-record line: {"p","pm", then either the Blob's "ha"/"h"/"sz" or the Inline's "il"}. +void writeEntryRecord(CasJsonWriter & out, const ManifestEntry & e) +{ + bool first = true; + writeKey(out, "p", first); + writeStringValue(out, e.path); + writeKey(out, "pm", first); + writeStringValue(out, placementToWord(e.placement)); + if (e.placement == EntryPlacement::Blob) + { + writeBlobRefFields(out, first, e.ref); /// ha + h + writeKey(out, "sz", first); + writeIntText(e.blob_size, out); + } + else + { + writeKey(out, "il", first); + writeIntText(e.inline_bytes.size(), out); + } + closeObject(out, first); + writeChar('\n', out); +} + +/// The exact banner text for one Inline entry's payload-zone chunk: `==> il= <==`. Takes +/// `path`/`n` explicitly (not a `ManifestEntry`): on decode, `inline_bytes` is not yet populated at +/// the point the expected banner is computed (that's the whole point of reading it from here first). +String bannerFor(std::string_view path, uint64_t n) +{ + return "==> " + String(path) + " il=" + std::to_string(n) + " <=="; +} + +} + +String encodePartManifest(const PartManifest & m) +{ + /// Canonical path order plus duplicate-path rejection makes the encoded record sequence + /// deterministic and establishes the ordering required by the lookup helpers. + std::vector sorted; + sorted.reserve(m.entries.size()); + for (const auto & e : m.entries) + sorted.push_back(&e); + std::sort(sorted.begin(), sorted.end(), + [](const ManifestEntry * a, const ManifestEntry * b) { return a->path < b->path; }); + for (size_t i = 1; i < sorted.size(); ++i) + if (sorted[i]->path == sorted[i - 1]->path) + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: duplicate path '{}'", sorted[i]->path); + + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::PartManifest); + + /// descriptor meta line: ManifestRef (me/mb/mo, shared rendering with refsnaplog) + root + /// namespace + payload digest. + { + bool first = true; + writeManifestRefFields(out, first, "", m.ref); + writeKey(out, "ns", first); + writeStringValue(out, m.root_namespace_id.string()); + writeKey(out, "pd", first); + writeHex128Value(out, m.payload_digest); + closeObject(out, first); + writeChar('\n', out); + } + + for (const ManifestEntry * e : sorted) + writeEntryRecord(out, *e); + + writeTrailerLine(out, sorted.size()); + + /// payload zone: one banner + raw bytes + '\n' per Inline entry, in path order. Blob entries + /// carry no payload-zone bytes (their bytes live in a separately addressed CAS blob). + for (const ManifestEntry * e : sorted) + { + if (e->placement != EntryPlacement::Inline) + continue; + const String banner = bannerFor(e->path, e->inline_bytes.size()); + out.append(banner); + writeChar('\n', out); + out.append(e->inline_bytes); + writeChar('\n', out); + } + + return std::move(out).take(); +} + +PartManifest decodePartManifest(std::string_view data) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::PartManifest); + const uint64_t line_cap = traitsFor(FormatId::PartManifest).line_cap; + + PartManifest m; + + /// descriptor meta line + { + const String meta = readLine(in, line_cap, "cas_part_manifest"); + ReadBufferFromMemory mm(meta.data(), meta.size()); + JsonObjectReader r(mm, KeyStrictness::Tolerant, "cas_part_manifest"); + std::optional me; + std::optional mb; + std::optional mo; + std::optional ns; + std::optional pd; + String key; + while (r.nextKey(key)) + { + if (key == "me") me = r.readU64String(); + else if (key == "mb") mb = r.readU64String(); + else if (key == "mo") mo = r.readU64Number(); + else if (key == "ns") ns = r.readString(); + else if (key == "pd") pd = r.readHex128(); + else r.skipUnknown(key); + } + if (!me || !mb || !mo) + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: descriptor missing me/mb/mo"); + if (!ns) + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: descriptor missing ns"); + if (!pd) + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: descriptor missing pd"); + m.ref = manifestRefFromFields(*me, *mb, *mo, "PartManifest", "descriptor"); + m.root_namespace_id = RootNamespace(*ns); + m.payload_digest = *pd; + if (!mm.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: junk after descriptor line"); + } + + /// entry record lines, until the trailer. Inline entries remember their declared `il` length so + /// the payload zone below can read exactly that many raw bytes back into `inline_bytes`. + /// Index-aligned with `m.entries` (Blob entries push an unused 0 placeholder). + std::vector inline_lens; + while (true) + { + const String line = readLine(in, line_cap, "cas_part_manifest"); + ReadBufferFromMemory l(line.data(), line.size()); + JsonObjectReader r(l, KeyStrictness::Tolerant, "cas_part_manifest"); + String key; + if (!r.nextKey(key)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: empty line"); + + if (key == "n") + { + const uint64_t declared_n = r.readU64Number(); + while (r.nextKey(key)) + r.skipUnknown(key); + if (!l.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: junk after trailer"); + if (declared_n != m.entries.size()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "PartManifest: trailer count {} != {} records", declared_n, m.entries.size()); + break; + } + + if (key != "p") + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: record must start with \"p\""); + ManifestEntry e; + e.path = r.readString(); + + /// Manifest bytes arrive over the interserver relink channel: enforce the same path hygiene + /// as CasLayout::checkNamespace so no future consumer can inherit a traversal. Relative, + /// no empty/'.'/'..' segments. (Syntactic only — legal projection subdirs pass.) + if (e.path.empty() || e.path.front() == '/') + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS part manifest: invalid entry path '{}'", e.path); + for (std::string_view rest = e.path; !rest.empty();) + { + const size_t slash = rest.find('/'); + const std::string_view seg = rest.substr(0, slash); + if (seg.empty() || seg == "." || seg == "..") + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS part manifest: invalid entry path '{}'", e.path); + rest = (slash == std::string_view::npos) ? std::string_view{} : rest.substr(slash + 1); + } + + std::optional pm; + std::optional ha; + std::optional h; + std::optional sz; + std::optional il; + while (r.nextKey(key)) + { + if (key == "pm") pm = r.readString(); + else if (key == "ha") ha = r.readString(); + else if (key == "h") h = r.readString(); + else if (key == "sz") sz = r.readU64Number(); + else if (key == "il") il = r.readU64Number(); + else r.skipUnknown(key); + } + if (!l.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: junk after record"); + if (!pm) + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: entry '{}' missing pm", e.path); + e.placement = placementFromWord(*pm); + + if (e.placement == EntryPlacement::Blob) + { + if (!ha || !h || !sz) + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: blob entry '{}' missing ha/h/sz", e.path); + const BlobHashAlgo algo = blobHashAlgoFromWord(*ha, "PartManifest entry"); + /// Validate the digest width before calling `fromHex`. A width mismatch otherwise + /// produces `BAD_ARGUMENTS` instead of the `CORRUPTED_DATA` required for malformed + /// serialized input, allowing an invalid manifest to escape the decoder's fail-closed + /// error contract. + const uint64_t expected_hex_len = blobHashLenFor(algo) * 2; + if (h->size() != expected_hex_len) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "PartManifest: entry '{}' digest hex width {} does not match algo width {}", + e.path, h->size(), expected_hex_len); + e.ref = BlobRef{algo, codecFor(algo).fromHex(*h)}; + e.blob_size = *sz; + inline_lens.push_back(0); /// unused for Blob; keeps inline_lens index-aligned with entries + } + else + { + if (!il) + throw Exception(ErrorCodes::CORRUPTED_DATA, "PartManifest: inline entry '{}' missing il", e.path); + inline_lens.push_back(*il); /// bytes filled from the payload zone below + } + + /// Canonical ascending-order and no-duplicate-path enforcement: compare only against the + /// immediately preceding entry, requiring + /// strict '<'. This is sufficient to catch a NON-adjacent duplicate too (e.g. forging entry + /// c's path to equal entry a's path in an a(h.high64) << 64) | static_cast(h.low64); +} + +bool refMatchesBody(const ManifestRef & journal_ref, const PartManifest & body) +{ + return journal_ref == body.ref; +} + +bool manifestNamespaceMatches(const RootNamespace & owning, const PartManifest & body) +{ + return owning == body.root_namespace_id; +} + +const ManifestEntry * findEntry(const std::vector & entries, std::string_view path) +{ + const auto it = std::lower_bound(entries.begin(), entries.end(), path, + [](const ManifestEntry & e, std::string_view p) { return std::string_view(e.path) < p; }); + if (it == entries.end() || std::string_view(it->path) != path) + return nullptr; + return &*it; +} + +std::pair +entryRange(const std::vector & entries, std::string_view dir_prefix) +{ + if (dir_prefix.empty()) + return {entries.data(), entries.data() + entries.size()}; + /// Every path starting with `dir_prefix` compares >= `dir_prefix`, and prefixed paths form a + /// contiguous run from the first such position. + const auto first = std::lower_bound(entries.begin(), entries.end(), dir_prefix, + [](const ManifestEntry & e, std::string_view p) { return std::string_view(e.path) < p; }); + auto last = first; + while (last != entries.end() && std::string_view(last->path).starts_with(dir_prefix)) + ++last; + return {entries.data() + (first - entries.begin()), entries.data() + (last - entries.begin())}; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.h new file mode 100644 index 000000000000..ab91c709e468 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPartManifestFormat.h @@ -0,0 +1,120 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Text codec for the immutable, root-local `cas_part_manifest` body. It is a `PayloadHybrid` object +/// made of JSON lines followed by a raw payload zone. The public types and helper signatures remain +/// stable for the surrounding CAS protocol. +/// +/// header line {"type":"cas_part_manifest","v":N} +/// descriptor meta line {"me","mb","mo"} (the ManifestRef, shared rendering with +/// refsnaplog, `CasWireVocab.h`) + "ns" (root namespace) + "pd" +/// (payload digest, 32 lowercase hex) +/// one entry-record line each {"p":path,"pm":placement-word, then either the Blob's +/// {"ha","h","sz"} or the Inline's {"il"}}, in canonical path order +/// trailer line {"n":entry-count} +/// PAYLOAD ZONE (raw, follows the trailer): for each Inline entry, in path order, a +/// `head -v`-style banner line `==> il= <==\n`, then +/// exactly `n` raw bytes, then `\n`. Blob entries carry no +/// payload-zone bytes — their bytes live in a separately addressed +/// CAS blob; the manifest carries only the `BlobRef` + size. +/// +/// The payload zone is why this format is `PayloadHybrid` rather than a plain `RecordStream`/ +/// `Control` object: `inline_bytes` is arbitrary binary, not necessarily valid UTF-8, so it cannot be +/// JSON-string-encoded safely and instead rides outside the JSON-line region entirely. + +/// Where a manifest entry's file bytes live. There are no nested tree objects: a directory is a path +/// prefix, not a placement. `Inline` bytes belong to the manifest's raw payload zone; `Blob` bytes +/// are stored separately under `blobKey`. +enum class EntryPlacement : uint8_t +{ + Inline = 1, /// bytes embedded in `inline_bytes` + Blob = 2, /// bytes stored as a content-addressed blob at `blobKey` +}; + +/// One file entry inside a part manifest. `ref` is meaningful only for `Blob`; `inline_bytes` only +/// for `Inline`. `blob_size` is the raw `Blob` byte count (0 for `Inline` — decode never fills it for +/// an inline entry, since the wire format carries no redundant size for inline bytes). Use `size()` +/// for the entry's logical file size regardless of placement; no consumer should branch on +/// `placement` just to answer "how big is this file". `ref` is the full blob identity: the algorithm +/// travels with the digest, per entry, so a manifest may mix hash algorithms. A bare digest is never +/// the identity. +struct ManifestEntry +{ + String path; + EntryPlacement placement = EntryPlacement::Inline; + BlobRef ref{}; + uint64_t blob_size = 0; + String inline_bytes; + bool operator==(const ManifestEntry &) const = default; + + /// The single source of truth for this entry's logical file size, independent of where its bytes + /// live. Decoding an `Inline` entry leaves `blob_size == 0`, because the wire record has no + /// redundant blob size; a carried-forward inline entry can therefore be non-empty even though + /// `blob_size` is zero. Consumers that need the logical file size must use this method rather + /// than inspecting the placement-specific fields themselves. + uint64_t size() const { return placement == EntryPlacement::Inline ? inline_bytes.size() : blob_size; } +}; + +/// The immutable body of one root-local part manifest. It repeats `ref` and `root_namespace_id` so +/// readers can validate the journal reference and owning root namespace against the body; neither +/// repetition is a second identity. `payload_digest` is integrity/debug metadata only: it is never a +/// key, deduplication input, or in-degree. Mutable per-reference payload remains in the root +/// `RefRecord`. Entries carry their own digest algorithm, so one manifest may mix digest widths. +/// Entries are strictly ascending by `path` after decoding, which permits index-free binary-search +/// and prefix-range lookup without adding a directory index to the immutable body. +struct PartManifest +{ + ManifestRef ref; + RootNamespace root_namespace_id; + UInt128 payload_digest{}; + std::vector entries; + bool operator==(const PartManifest &) const = default; +}; + +/// Deterministic, streaming-capable encode. Entries are written in canonical path order (the encoder +/// sorts them); a duplicate path throws `CORRUPTED_DATA`. Byte output is reproducible for identical +/// input (no timestamps, no nondeterministic compression). Returns the canonical TEXT (NOT sealed); +/// the caller compresses via `sealObject(FormatId::PartManifest, …)` on the persist path +/// (`CompressionPolicy::Always`). +String encodePartManifest(const PartManifest & m); + +/// Decode the canonical TEXT (the caller `openObject`s a stored `.zst` first). Throws `CORRUPTED_DATA` +/// on a malformed header/descriptor/record/trailer/payload-zone shape or an unknown placement word; +/// `UNKNOWN_FORMAT_VERSION` for a header `v` above this build. +PartManifest decodePartManifest(std::string_view data); + +/// Content digest of the canonical encoded body, using the CAS content-hash primitive +/// (`CityHash_v1_0_2::CityHash128`, the same one used for blob/tree hashing, not a second hash +/// primitive). Callers set `PartManifest.payload_digest` from this. It is +/// integrity/debug ONLY - never a key, never dedup, never in-degree. Stable for identical bodies; +/// changes when any byte of the canonical encoding does, and is independent of the `payload_digest` +/// field itself (computed with it zeroed). +UInt128 computePayloadDigest(const PartManifest & m); + +/// Fail-closed identity checks used when reading or folding a manifest. The journal `ManifestRef` +/// must equal the `ref` inside the decoded body. +bool refMatchesBody(const ManifestRef & journal_ref, const PartManifest & body); +/// The body `root_namespace_id` must equal the owning root namespace. +bool manifestNamespaceMatches(const RootNamespace & owning, const PartManifest & body); + +/// Pure ordered-entry lookup primitives over a decoded manifest. `decodePartManifest` guarantees +/// strict ascending order by `path`; `PartFolderView` composes these lookups with wiring policy. + +/// Binary search. Returns nullptr when absent. The pointer aliases `entries` — do not outlive it. +const ManifestEntry * findEntry(const std::vector & entries, std::string_view path); + +/// The contiguous [first, last) sub-span of entries whose path starts with `dir_prefix` (canonical +/// order makes matches contiguous). Empty prefix = the whole span. +std::pair +entryRange(const std::vector & entries, std::string_view dir_prefix); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.cpp new file mode 100644 index 000000000000..fab51431c308 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.cpp @@ -0,0 +1,182 @@ +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int UNKNOWN_FORMAT_VERSION; +} +} + +namespace DB::Cas +{ + +/// Minimum `blob_header_len` that provably fits the v3 `cas_blob` JSON envelope's mandatory (always- +/// written) non-ref fields, computed at type maxima from `encodeEnvelopeHeader` (CasBlobEnvelopeFormat.cpp): +/// {"type":"cas_blob" 18 +/// ,"v": 5 + 10 (currentCompatibilityVersion) 15 +/// ,"tag":"<32 hex>" 7 + 34 41 +/// ,"bld":"<32 hex>" 7 + 34 41 +/// ,"ts": 6 + 20 (created_at_ms) 26 +/// ,"by":"<32 hex>" 7 + 34 41 +/// ,"op":"" 6 + 10 (longest op word "mutation") 16 +/// ,"ch": 6 + 10 (VERSION_INTEGER) 16 +/// non-ref JSON = 214 bytes +/// The encoder then always frames the ref: `,"ref":` (7) + `""` (2) + `}` (1), and reserves byte +/// blob_header_len-1 for '\n' (1) = 11 bytes. So the mandatory content needs 214 + 11 = 225 bytes; +/// below that, encodeEnvelopeHeader throws LOGICAL_ERROR on the FIRST blob write (the old drop-and-retry +/// that used to mask this is gone). We floor at 240 (a multiple of 8 comfortably above 225, leaving +/// >= 15 bytes for the diagnostic ref even at type maxima, and well under the 256 default) so a +/// misconfigured pool fails at CREATION with BAD_ARGUMENTS, not at first write with LOGICAL_ERROR. +static constexpr uint64_t kMinBlobHeaderLen = 240; + +void validatePoolBlobHeaderLen(uint64_t blob_header_len, int error_code, std::string_view what) +{ + if (blob_header_len < kMinBlobHeaderLen) + throw Exception(error_code, "CAS {}: blob_header_len must be >= {} (v3 envelope minimum), got {}", + what, kMinBlobHeaderLen, blob_header_len); + if (blob_header_len % 8 != 0) + throw Exception(error_code, "CAS {}: blob_header_len must be a multiple of 8, got {}", what, blob_header_len); + if (blob_header_len > 16 * 1024) + throw Exception(error_code, "CAS {}: blob_header_len must be <= 16384, got {}", what, blob_header_len); +} + +void validatePoolAlgosUsed(const std::vector & algos_used, int error_code, std::string_view what) +{ + if (algos_used.empty()) + throw Exception(error_code, "CAS {}: algos_used must be non-empty", what); + for (size_t i = 0; i < algos_used.size(); ++i) + { + try + { + blobHashAlgoName(static_cast(algos_used[i])); + } + catch (const Exception &) + { + throw Exception(error_code, "CAS {}: algos_used contains an unknown algo {}", what, algos_used[i]); + } + if (i > 0 && algos_used[i] <= algos_used[i - 1]) + throw Exception(error_code, + "CAS {}: algos_used must be strictly sorted with no duplicates, got {} at index {} not after {}", + what, algos_used[i], i, algos_used[i - 1]); + } +} + +String encodePoolMeta(const PoolMeta & pm) +{ + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::PoolMeta); + + bool first = true; + writeKey(out, "pid", first); + writeHex128Value(out, pm.pool_id); + writeKey(out, "hln", first); + writeIntText(pm.blob_header_len, out); + writeKey(out, "gcs", first); + writeIntText(pm.gc_shards, out); + writeKey(out, "mrg", first); + writeIntText(pm.min_reader_generation, out); + writeKey(out, "alg", first); + { + /// Comma-joined algo words (tiny list, <=3): "ch128" or "ch128,sha256". + String joined; + for (size_t i = 0; i < pm.algos_used.size(); ++i) + { + if (i != 0) + joined += ','; + joined += blobHashAlgoName(static_cast(pm.algos_used[i])); + } + writeStringValue(out, joined); + } + closeObject(out, first); + writeChar('\n', out); + + return std::move(out).take(); +} + +PoolMeta decodePoolMeta(std::string_view data) +{ + ReadBufferFromMemory in(data.data(), data.size()); + const TextHeader header = expectHeaderLine(in, FormatId::PoolMeta); + + /// An older pool predates a breaking ref-layer change this build cannot reconcile, so + /// reject it before reading the metadata body. Writers always emit the current generation, while + /// `expectHeaderLine` separately rejects a future generation that this build cannot understand. + /// Generation 9 is the latest recreate-only authority floor and subsumes the earlier stream floors. + if (header.v < kCommittedRefFrontierGeneration) + throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, + "CAS pool format {} predates generation-9 exact _ckpt committed_through recovery frontier; recreate the pool. " + "This build requires the namespace-admission shard bound and exact recovery frontier " + "in the generation-9 format " + "(generation {}+), and CAS is pre-release: there is no in-place migration.", + header.v, kCommittedRefFrontierGeneration); + + const String body = readLine(in, traitsFor(FormatId::PoolMeta).line_cap, "pool meta"); + ReadBufferFromMemory body_in(body.data(), body.size()); + JsonObjectReader r(body_in, KeyStrictness::Tolerant, "pool meta"); + + PoolMeta pm; + bool saw_pid = false; + bool saw_gc_shards = false; + String key; + while (r.nextKey(key)) + { + if (key == "pid") + { + pm.pool_id = r.readHex128(); + saw_pid = true; + } + else if (key == "hln") + pm.blob_header_len = r.readU64Number(); + else if (key == "gcs") + { + pm.gc_shards = r.readU64Number(); + saw_gc_shards = true; + } + else if (key == "mrg") + pm.min_reader_generation = r.readU64Number(); + else if (key == "alg") + { + const String joined = r.readString(); + size_t start = 0; + while (start <= joined.size()) + { + const size_t comma = joined.find(',', start); + const String word = joined.substr(start, comma == String::npos ? String::npos : comma - start); + if (word.empty()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS pool meta: empty algo word in '{}'", joined); + pm.algos_used.push_back(static_cast(blobHashAlgoFromWord(word, "pool meta algo"))); + if (comma == String::npos) + break; + start = comma + 1; + } + } + else + r.skipUnknown(key); + } + if (!saw_pid) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS pool meta: missing pid"); + if (!saw_gc_shards || pm.gc_shards == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS pool meta: missing or zero gcs"); + if (!body_in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS pool meta: junk after body object"); + if (!in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS pool meta: trailing bytes after body line"); + + validatePoolBlobHeaderLen(pm.blob_header_len, ErrorCodes::CORRUPTED_DATA, "pool meta"); + validatePoolAlgosUsed(pm.algos_used, ErrorCodes::CORRUPTED_DATA, "pool meta"); + + if (G_BUILD < pm.min_reader_generation) + throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, + "CAS pool meta: pool requires reader generation {} but this build supports at most {}", + pm.min_reader_generation, G_BUILD); + + return pm; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.h new file mode 100644 index 000000000000..2ca0894d2f01 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasPoolMetaFormat.h @@ -0,0 +1,91 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +class Backend; +class Layout; + +/// `_pool_meta` — the pool identity and the pool-wide constants that every reader and writer must +/// agree on. The v3 text representation is a header line followed by one JSON body object: +/// {"pid":"<32hex>","hln":,"mrg":,"alg":""}. +/// +/// The persisted object is authoritative after creation. On reopen, `createOrValidate` uses its +/// `blob_header_len` and reader-generation floor rather than replacing them with local configuration; +/// the configuration's hash algorithm may only be admitted through the explicit opt-in path. The +/// `pool_id` is also the envelope `domain_id`, so it remains stable for the entire pool lifetime. +struct PoolMeta +{ + UInt128 pool_id{}; + uint64_t blob_header_len = 0; + uint64_t gc_shards = 1; + uint64_t min_reader_generation = 0; + /// Every hash algorithm ever admitted, encoded as `static_cast(BlobHashAlgo)`, in strictly + /// increasing order. Admission only appends a new algorithm to this durable set. + std::vector algos_used; + + /// Creates the pool metadata if `_pool_meta` is absent, or validates and possibly admits the + /// configured hash algorithm if it already exists. Initial creation validates the supplied header + /// size and records this build's reader-generation floor. Reopen ignores the supplied header size, + /// because changing it would move the blob payload offset for existing objects; a new hash algorithm + /// is rejected unless `allow_new` is set, and concurrent admission is retried from fresh metadata. + /// + /// `allow_mint` (spec §2 [C4][D2]) gates the create-if-absent path: minting a fresh `_pool_meta` is a + /// consequential write that establishes a brand-new pool identity, so it is permitted ONLY on the + /// writable startup path that has just passed the zero-write residual proof (`Pool::open`). Every + /// non-bootstrap caller — a read-only/observe open, `openForDecommission` — passes `false`; an absent + /// `_pool_meta` then fails closed with `INVALID_STATE` instead of silently minting (which, on an + /// observe scan over a partially-erased pool, would poison the next writable mount). The validate path + /// (meta already present) never consults it. + /// + /// Defaults to `false` — a safety gate must fail CLOSED when a caller leaves it unstated, so a future + /// pool-lifecycle entry point cannot silently re-arm the observe-mint footgun by omission. The two + /// production callers pass it explicitly; only test minting sites opt in with `allow_mint=true`. + static PoolMeta createOrValidate( + Backend &, const Layout &, uint64_t blob_header_len, uint64_t gc_shards, + BlobHashAlgo blob_hash_algo = BlobHashAlgo::CityHash128, bool allow_new = false, + bool allow_mint = false); + + /// Convenience for single-shard callers. Production pool opening passes the configured value to + /// the explicit overload above; this preserves compact single-shard codec/unit fixtures. + static PoolMeta createOrValidate( + Backend & backend, const Layout & layout, uint64_t blob_header_len, + BlobHashAlgo blob_hash_algo = BlobHashAlgo::CityHash128, bool allow_new = false, + bool allow_mint = false) + { + return createOrValidate( + backend, layout, blob_header_len, /*gc_shards=*/1, blob_hash_algo, allow_new, allow_mint); + } +}; + +/// Serializes valid pool metadata as the versioned `_pool_meta` text object. The output includes the +/// format header, one JSON body line, and its terminating newline; it is suitable for a conditional +/// backend write and preserves the sorted algorithm set as comma-separated vocabulary words. +String encodePoolMeta(const PoolMeta &); + +/// Parses and validates a persisted `_pool_meta` object. Unknown JSON keys are tolerated for additive +/// evolution, while missing required data, malformed values, invariant violations, an unsupported +/// ref-state generation, or a pool requiring a newer reader produce an exception with the appropriate +/// corruption or compatibility error code. +PoolMeta decodePoolMeta(std::string_view); + +/// Checks the fixed blob-envelope size invariant. The length must be 8-byte aligned, at most 16 KiB, +/// and at least 240 bytes: v3's mandatory envelope fields, framing, and newline consume 225 bytes at +/// type maxima, while 240 leaves room for a diagnostic `ref`. The caller supplies the error code so +/// persisted violations can be reported as `CORRUPTED_DATA` and bad creation arguments as +/// `BAD_ARGUMENTS`. +void validatePoolBlobHeaderLen(uint64_t blob_header_len, int error_code, std::string_view what); + +/// Checks that every admitted hash algorithm is known, that the set is non-empty, and that its numeric +/// representation is strictly increasing with no duplicates. The caller supplies the error code to +/// distinguish invalid persisted metadata from invalid creation or admission input. +void validatePoolAlgosUsed(const std::vector & algos_used, int error_code, std::string_view what); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.cpp new file mode 100644 index 000000000000..b21458aaf6e2 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.cpp @@ -0,0 +1,325 @@ +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +namespace +{ + +UInt128 toWideChecksum(CityHash_v1_0_2::uint128 h) +{ + /// Keep the high and low halves in the same order for the write-side helper and the streaming + /// reader. The value is compared internally rather than exposed as a separately interpreted wire + /// field, but both paths must use the same packing for a stored run to verify successfully. + return (static_cast(h.high64) << 64) | static_cast(h.low64); +} + +int hexNibble(char c) +{ + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + return -1; +} + +BlobHashAlgo algoFromByte(uint8_t b, std::string_view what) +{ + switch (b) + { + case static_cast(BlobHashAlgo::CityHash128): return BlobHashAlgo::CityHash128; + case static_cast(BlobHashAlgo::XXH3_128): return BlobHashAlgo::XXH3_128; + case static_cast(BlobHashAlgo::Sha256): return BlobHashAlgo::Sha256; + default: + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: unknown algo byte {} in record key", what, b); + } +} + +/// `b` = the algo byte as two lowercase hex chars, then the digest hex at the algo's width. The algo +/// byte leads so that string-sorting `b` reproduces the binary (algo, digest) byte order. +String renderB(const BlobRef & ref) +{ + static constexpr char H[] = "0123456789abcdef"; + const uint8_t a = static_cast(ref.algo); + String b; + b.push_back(H[(a >> 4) & 0xF]); + b.push_back(H[a & 0xF]); + b += codecFor(ref.algo).toHex(ref.digest); + return b; +} + +BlobRef parseB(std::string_view b) +{ + if (b.size() < 2) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: record key too short"); + const int hi = hexNibble(b[0]); + const int lo = hexNibble(b[1]); + if (hi < 0 || lo < 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: non-hex algo byte in record key"); + const BlobHashAlgo algo = algoFromByte(static_cast((hi << 4) | lo), "cas_run"); + const std::string_view digest_hex = b.substr(2); + if (digest_hex.size() != static_cast(blobHashLenFor(algo)) * 2) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS cas_run: digest hex width {} does not match algo width {}", digest_hex.size(), blobHashLenFor(algo) * 2); + BlobRef ref; + ref.algo = algo; + ref.digest = codecFor(algo).fromHex(String(digest_hex)); + return ref; +} + +std::string_view markerToWord(char m) +{ + switch (m) + { + case kEdgeActive: return "edge"; + case kZeroMarker: return "zero"; + case kCondemned: return "condemned"; + default: + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: unknown row marker 0x{:02x}", static_cast(m)); + } +} + +char markerFromWord(std::string_view w) +{ + if (w == "edge") return kEdgeActive; + if (w == "zero") return kZeroMarker; + if (w == "condemned") return kCondemned; + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: unknown row marker '{}'", w); +} + +} + +void writeRunHeaderLine(WriteBuffer & out, std::string_view kind) +{ + const FormatTraits & t = traitsFor(FormatId::RunFile); + CasJsonWriter line(64); + bool first = true; + writeKey(line, "type", first); + writeStringValue(line, t.type); + writeKey(line, "v", first); + writeIntText(currentCompatibilityVersion(), line); + writeKey(line, "kind", first); + writeStringValue(line, kind); + closeObject(line, first); + writeChar('\n', line); + const std::string_view line_view = line.view(); + out.write(line_view.data(), line_view.size()); +} + +void expectRunHeaderLine(ReadBuffer & in, std::string_view expected_kind) +{ + const FormatTraits & t = traitsFor(FormatId::RunFile); + const String line = readLine(in, t.line_cap, t.type); + ReadBufferFromMemory buf(line.data(), line.size()); + JsonObjectReader r(buf, KeyStrictness::Tolerant, t.type); + + String key; + if (!r.nextKey(key) || key != "type") + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: header line must start with \"type\""); + const String type = r.readString(); + if (type != t.type) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: object is a '{}', not a '{}'", type, t.type); + + if (!r.nextKey(key) || key != "v") + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: header line must carry \"v\" second"); + const uint32_t v = r.readU32Number(); + checkCompatibility(v, t.type); + + if (!r.nextKey(key) || key != "kind") + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: header line must carry \"kind\" third"); + const String kind = r.readString(); + if (kind != expected_kind) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: unknown run kind '{}'", kind); + + while (r.nextKey(key)) + r.skipUnknown(key); + if (!buf.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: junk after the header object"); +} + +SourceEdgeRunWriter::SourceEdgeRunWriter(WriteBuffer & out_) + : out(out_) +{ + writeRunHeaderLine(out, kSourceEdgeKindWord); +} + +void SourceEdgeRunWriter::append(const SourceEdgeRecord & rec) +{ + if (finished) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS cas_run: append after finish"); + /// Non-decreasing (ref, source_id) is a HARD writer contract (deterministic run + streaming merge). + /// A regression is a programming bug at the producer, not corrupt on-disk data => LOGICAL_ERROR. + if (have_prev) + { + const bool regressed = (rec.ref < prev_ref) + || (rec.ref == prev_ref && rec.source_id < prev_source_id); + if (regressed) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS cas_run: records appended out of (ref, source_id) order"); + } + have_prev = true; + prev_ref = rec.ref; + prev_source_id = rec.source_id; + + scratch.clear(); + bool first = true; + writeKey(scratch, "b", first); + writeStringValue(scratch, renderB(rec.ref)); + writeKey(scratch, "s", first); + writeHex128Value(scratch, rec.source_id); + writeKey(scratch, "m", first); + writeStringValue(scratch, markerToWord(rec.marker)); + if (rec.marker == kCondemned) + { + writeKey(scratch, "pend", first); + writeBoolValue(scratch, rec.delete_pending); + writeTokenFields(scratch, first, rec.token); /// tt + tv + writeKey(scratch, "sz", first); + writeIntText(rec.size, scratch); + writeKey(scratch, "cr", first); + writeU64StringValue(scratch, rec.condemn_round); + writeKey(scratch, "mc", first); + writeBoolValue(scratch, rec.marker_confirmed); + } + closeObject(scratch, first); + writeChar('\n', scratch); + { + const std::string_view scratch_view = scratch.view(); + out.write(scratch_view.data(), scratch_view.size()); + } + ++count; +} + +void SourceEdgeRunWriter::finish() +{ + if (finished) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS cas_run: finish called twice"); + scratch.clear(); + writeTrailerLine(scratch, count); + const std::string_view scratch_view = scratch.view(); + out.write(scratch_view.data(), scratch_view.size()); + finished = true; +} + +UInt128 sourceEdgeRunChecksum(std::string_view stored_bytes) +{ + /// Use the same chained `CityHash128` and default block size as the reader. A one-shot hash would + /// diverge from the streaming hash for sufficiently large input, so the producer and later fold + /// must both process exactly the stored bytes through `HashingReadBuffer`. + ReadBufferFromMemory mem(stored_bytes.data(), stored_bytes.size()); + HashingReadBuffer hashing(mem); + hashing.ignoreAll(); /// drain the whole object through the hash + return toWideChecksum(hashing.getHash()); +} + +SourceEdgeRunReader::SourceEdgeRunReader(ReadBuffer & in_) + : hashing(in_) +{ + /// Typed open: gate type/v/kind (and hash the header bytes) before any record is interpreted. + expectRunHeaderLine(hashing, kSourceEdgeKindWord); +} + +bool SourceEdgeRunReader::next(SourceEdgeRecord & rec) +{ + if (done) + return false; + + const String line = readLine(hashing, traitsFor(FormatId::RunFile).line_cap, "cas_run"); + ReadBufferFromMemory line_in(line.data(), line.size()); + JsonObjectReader r(line_in, KeyStrictness::Strict, "cas_run"); + + String key; + if (!r.nextKey(key)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: empty line"); + + if (key == "n") + { + const uint64_t n = r.readU64Number(); + if (r.nextKey(key)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: trailer has extra keys"); + if (!line_in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: junk after trailer object"); + /// The trailer must be the last line of the object; hashing must be at EOF (this also drains and + /// hashes the final bytes so accumulatedChecksum covers the whole object). + if (!hashing.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: bytes after trailer"); + if (n != seen) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS cas_run: trailer count {} != {} records (line truncation?)", n, seen); + done = true; + return false; + } + + SourceEdgeRecord out; + String b; + String tv; + bool have_b = false; + bool have_s = false; + bool have_m = false; + bool have_pend = false; + bool have_tt = false; + bool have_tv = false; + bool have_sz = false; + bool have_cr = false; + bool have_mc = false; + TokenType tt{}; + do + { + if (key == "b") { b = r.readString(); have_b = true; } + else if (key == "s") { out.source_id = r.readHex128(); have_s = true; } + else if (key == "m") { out.marker = markerFromWord(r.readString()); have_m = true; } + else if (key == "pend") { out.delete_pending = r.readBool(); have_pend = true; } + else if (key == "tt") { tt = tokenTypeFromWord(r.readString(), "cas_run"); have_tt = true; } + else if (key == "tv") { tv = r.readString(); have_tv = true; } + else if (key == "sz") { out.size = r.readU64Number(); have_sz = true; } + else if (key == "cr") { out.condemn_round = r.readU64String(); have_cr = true; } + else if (key == "mc") { out.marker_confirmed = r.readBool(); have_mc = true; } + else r.skipUnknown(key); /// Strict => any unknown key is CORRUPTED_DATA + } while (r.nextKey(key)); + + if (!have_b || !have_s || !have_m) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: record missing b/s/m"); + out.ref = parseB(b); + if (out.marker == kCondemned) + { + if (!have_pend || !have_tt || !have_tv || !have_sz || !have_cr || !have_mc) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: condemned record missing pend/tt/tv/sz/cr/mc"); + out.token = Token{tv, tt}; + } + else if (have_pend || have_tt || have_tv || have_sz || have_cr || have_mc) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: non-condemned record carries condemned fields"); + + if (!line_in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_run: junk after record object"); + + rec = std::move(out); + ++seen; + return true; +} + +UInt128 SourceEdgeRunReader::accumulatedChecksum() +{ + return toWideChecksum(hashing.getHash()); +} + +void SourceEdgeRunReader::verifyAgainst(const UInt128 & expected) +{ + const UInt128 got = accumulatedChecksum(); + if (got != expected) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS cas_run: whole-file seal-checksum mismatch (the run bytes do not match the fold seal's " + "RunRef.checksum); refusing to act on this run"); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.h new file mode 100644 index 000000000000..d5f9a4801caf --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRecordStreamFormat.h @@ -0,0 +1,164 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Row tags for the sealed source-edge run. These byte values are part of the source-edge payload +/// format, shared by this codec and the GC fold that interprets the rows. +/// +/// Source-edge rows use `source_id == 0` as a sentinel key. A real active edge must never use that key; +/// both sentinel tags are restricted to it. `kZeroMarker` describes a zero transition for the current +/// generation and is dropped when the row is carried forward. `kCondemned` carries the condemned +/// incarnation at the sentinel key across generations until settlement; its payload contains the full +/// deletion token and other condemned-row state. A condemned row subsumes the zero marker for that +/// generation. +constexpr char kEdgeActive = 0x01; +constexpr char kZeroMarker = 0x00; +constexpr char kCondemned = 0x02; + +/// The `cas_run` codec represents the GC source-edge in-degree data plane as sorted NDJSON. This is +/// the `RecordStream` family +/// (`FormatId::RunFile`): unbounded-cardinality sorted records, `object_cap = 0` (NEVER materialized +/// whole — streamed one line at a time over a `ReadBuffer`), `line_cap = 4 KiB`, `PinnedRaw` (no +/// compression) + `Strict` (byte-deterministic for `putDeterministicArtifact` adoption). +/// +/// This file is backend-free: it accepts caller-owned `ReadBuffer`/`WriteBuffer` objects and never +/// includes backend or GC subsystem headers. The GC layer owns the stream lifetime and the bridge to +/// packed keys and condemned rows; this codec owns only the durable text representation and its +/// identifier-layer types. Keeping that boundary physical prevents storage or GC dependencies from +/// leaking into the format implementation. +/// +/// File shape: +/// {"type":"cas_run","v":3,"kind":"source_edge"} header line (type + v + kind gate) +/// {"b":"01","s":"<32hex>","m":"edge"} an active-edge / zero-marker row +/// {"b":"01","s":"00000000000000000000000000000000","m":"condemned","pend":false,"tt":"etag","tv":"...","sz":123,"cr":"7","mc":false} +/// {"n":184267} trailer: record count +/// +/// The record key `b` is the algo BYTE as two lowercase hex chars followed by the digest hex at the +/// algo's width; `s` is the 32-hex source id. String-sorting records by (b, s) reproduces the current +/// `(algorithm, digest, source_id)` byte order (lowercase hex preserves unsigned byte order and the +/// algorithm byte is emitted first) — the invariant the fold's two-cursor merge depends on. The row-tag word +/// `m` maps to the `kEdgeActive`/`kZeroMarker`/`kCondemned` bytes; a `condemned` row additionally +/// carries the retired incarnation (`pend`/`tt`/`tv`/`sz`/`cr`) and the durable condemn-marker +/// confirmation bit (`mc`). + +/// One decoded source-edge row. All fields are identifier-layer types so the codec stays backend-free. +/// The condemned-only fields (`delete_pending`/`token`/`size`/`condemn_round`/`marker_confirmed`) are +/// meaningful only when `marker == kCondemned`. +struct SourceEdgeRecord +{ + BlobRef ref{}; + UInt128 source_id{}; + char marker = kEdgeActive; + bool delete_pending = false; + Token token{}; + uint64_t size = 0; + uint64_t condemn_round = 0; + bool marker_confirmed = false; /// durable Condemned meta confirmed for this entry (graduation gate) +}; + +/// The header-line `kind` word for the only live `cas_run` kind. +inline constexpr std::string_view kSourceEdgeKindWord = "source_edge"; + +/// Write the typed header line `{"type":"cas_run","v":G_BUILD,"kind":""}\n` with a fixed key +/// order for byte-determinism. The `kind` field distinguishes the record schema within the run +/// family, so a reader can reject a valid run of the wrong kind before interpreting any records. +void writeRunHeaderLine(WriteBuffer & out, std::string_view kind); + +/// Read + gate the typed header line: `type` must be `cas_run`, `v` is gated by `checkCompatibility` +/// (future `v` -> `UNKNOWN_FORMAT_VERSION`), and `kind` must equal `expected_kind` (else +/// `CORRUPTED_DATA`, "unknown run kind"). This is the typed-open — all three are validated before any +/// record is interpreted. +void expectRunHeaderLine(ReadBuffer & in, std::string_view expected_kind); + +/// Sorted NDJSON writer over a caller-owned `WriteBuffer` (backend-free; writes plainly — the whole- +/// object checksum is `sourceEdgeRunChecksum` over the finished bytes, which keeps this writer free of a +/// HashingWriteBuffer finalize-ordering hazard). `append` asserts records arrive in non-decreasing +/// (ref, source_id) order and throws on a regression (this replaces the old `prev_key` monotonicity +/// check). `finish` writes the `{"n":count}` trailer. +class SourceEdgeRunWriter +{ +public: + /// Write the typed source-edge header immediately. The writer borrows `out` for its entire + /// lifetime; the caller must keep it alive and must call `finish` exactly once after the final + /// record so the count trailer is present. + explicit SourceEdgeRunWriter(WriteBuffer & out_); + + /// Append one record in non-decreasing `(ref, source_id)` order. Equal keys are allowed because + /// the merge layer may produce multiple rows for the same key. A regression is a producer + /// programming error and raises `LOGICAL_ERROR`; no partial record is written for that call. + void append(const SourceEdgeRecord & rec); + + /// Write the record-count trailer and mark the stream finished. Calling `finish` twice raises + /// `LOGICAL_ERROR`; appending after it is likewise rejected so a completed run cannot be extended. + void finish(); + +private: + WriteBuffer & out; + uint64_t count = 0; + bool have_prev = false; + BlobRef prev_ref{}; + UInt128 prev_source_id{}; + bool finished = false; + /// Reused line-scratch: each record is assembled here, then bulk-written to `out` in one call. + /// `clear` keeps the buffer's capacity, so memory stays bounded by the largest line ever + /// assembled, never by record count. + CasJsonWriter scratch; +}; + +/// The whole-object seal-checksum (`RunRef.checksum`) of a stored `cas_run`: the chained CityHash128 a +/// `HashingReadBuffer` computes over ALL the object bytes. The reader accumulates the IDENTICAL hash as +/// it streams (`SourceEdgeRunReader::verifyAgainst`), so a run PUT by the producer and later read by the +/// fold agree byte-for-byte. Computed over the finished bytes on the write side (the producer already +/// holds them to PUT); streamed on the read side (the run is never materialized whole to verify). +UInt128 sourceEdgeRunChecksum(std::string_view stored_bytes); + +/// Sequential streaming reader over a caller-owned `ReadBuffer` (backend-free, O(one 4 KiB line) +/// resident). The ctor reads + gates the typed header line. `next` yields records in stored order and +/// returns false once the `{"n"}` trailer is consumed (the count is verified there — the line-truncation +/// guard). Every byte read is fed through a chained CityHash128; after the trailer, `verifyAgainst` +/// compares the accumulated whole-object hash to the seal's `RunRef.checksum` and throws `CORRUPTED_DATA` +/// on a mismatch — the caller calls it after draining and BEFORE acting on the records (the deletion +/// decision). Non-movable/non-copyable (owns a `HashingReadBuffer`) — construct in place. +class SourceEdgeRunReader +{ +public: + /// Construct a reader that borrows `in`, hashes every byte read, and validates the typed header + /// before exposing any record. The caller must drain the reader through the trailer before using + /// `verifyAgainst`, because the seal covers the complete object rather than only decoded rows. + explicit SourceEdgeRunReader(ReadBuffer & in_); + SourceEdgeRunReader(const SourceEdgeRunReader &) = delete; + SourceEdgeRunReader & operator=(const SourceEdgeRunReader &) = delete; + + /// Decode the next record in stored order. Returns `false` only after consuming and validating + /// the count trailer and confirming that it is the final line; malformed or truncated input + /// raises `CORRUPTED_DATA`. + bool next(SourceEdgeRecord & rec); + + /// Compare the accumulated whole-object checksum with the seal recorded for the run. Call only + /// after `next` has returned `false`; a mismatch raises `CORRUPTED_DATA` so callers can verify + /// the run before acting on decoded condemned rows. + void verifyAgainst(const UInt128 & expected); + + /// Return the whole-object hash accumulated so far. It is meaningful for seal verification only + /// after the trailer has been consumed, when the reader has hashed every byte of the object. + UInt128 accumulatedChecksum(); + +private: + HashingReadBuffer hashing; + uint64_t seen = 0; + bool done = false; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.cpp new file mode 100644 index 000000000000..c5b119ba44ca --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.cpp @@ -0,0 +1,397 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int LIMIT_EXCEEDED; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +std::string_view nsStateToWord(NsState s) +{ + switch (s) + { + case NsState::Creating: return "creating"; + case NsState::Live: return "live"; + case NsState::Removing: return "removing"; + } + /// Every value reaching here came from a live `NsState` or from `nsStateFromWord`, which already + /// validated it on decode -- so this is a bug in THIS process, not corruption arriving from a + /// store. + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS ref catalog: unknown ns state {}", static_cast(s)); +} + +NsState nsStateFromWord(std::string_view w) +{ + if (w == "creating") return NsState::Creating; + if (w == "live") return NsState::Live; + if (w == "removing") return NsState::Removing; + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: unknown ns state '{}'", w); +} + +namespace +{ + +/// `creator` is required iff `state == Creating`, forbidden otherwise -- one predicate, used by both +/// directions of the codec, so the writer's self-check and the reader's fail-close can never disagree. +bool creatorPairingOk(const CatalogEntry & e) +{ + return (e.state == NsState::Creating) == e.creator.has_value(); +} + +bool removalRoundPairingOk(const CatalogEntry & e) +{ + return (e.state == NsState::Removing) == e.removal_started_round.has_value(); +} + +/// Whether `entries` is already in the catalog's canonical shape: strictly ascending by namespace +/// bytes, no duplicate namespace. A duplicate namespace fails the SAME check as an out-of-order pair +/// (equal keys never compare strictly less), which is exactly right -- both are "not canonical". +bool isCanonicalCatalogOrder(const std::vector & entries) +{ + for (size_t i = 1; i < entries.size(); ++i) + if (!(entries[i - 1].ns.string() < entries[i].ns.string())) + return false; + return true; +} + +} + +String encodeRefCatalog(const RefCatalog & catalog) +{ + const uint64_t line_cap = traitsFor(FormatId::RefCatalog).line_cap; + CasJsonWriter out(256); + + /// EVERY line this encoder emits is measured against the LINE cap, on the bytes actually emitted + /// -- escaping, framing and all -- mirroring `encodeFoldSeal`'s `checkLineBytes` EXACTLY, + /// including its error code: `LIMIT_EXCEEDED`, not `LOGICAL_ERROR`. A line that does not fit is + /// not a large line, it is an UNREADABLE one: `readLine` refuses it, so the whole object is lost. + /// This is a capacity refusal on otherwise well-formed input (an admitted namespace name or + /// server_root_id near their own byte bounds, worst-case escaped, can reach ~4.7 KiB on its own -- + /// reachable, not theoretical), not a bug in this process, so a caller catching `LIMIT_EXCEEDED` + /// for a capacity refusal must not see it misreported as a `LOGICAL_ERROR`. Refuse here, where + /// nothing is durable yet. + const auto checkLineBytes = [&](uint64_t bytes, std::string_view what) + { + if (!fitsLineCap(bytes, line_cap)) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "CAS ref catalog: the {} line encodes to {} bytes, over the {}-byte line cap; a longer " + "line cannot be read back, so the catalog is refused before it is written", + what, bytes, line_cap); + }; + + writeHeaderLine(out, FormatId::RefCatalog); /// emits its own terminator + checkLineBytes(out.size() - 1, "header"); + + /// This is our own state, about to become durable: an out-of-order or duplicate-keyed vector is a + /// bug in the writer, not corruption arriving from a store. + if (!isCanonicalCatalogOrder(catalog.entries)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS ref catalog: entries are not canonically ordered -- strictly ascending by namespace " + "bytes with no duplicate namespace is required before a catalog may be encoded"); + + size_t line_start = out.size(); + const auto closeLine = [&](std::string_view what) + { + checkLineBytes(out.size() - line_start, what); + writeChar('\n', out); + line_start = out.size(); + }; + + for (const CatalogEntry & e : catalog.entries) + { + if (e.ns.string().empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS ref catalog: an entry's namespace must not be empty"); + if (e.ns.string().size() > kMaxNamespaceBytes) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS ref catalog: namespace '{}' is {} bytes, over the {}-byte admission bound", + e.ns.string(), e.ns.string().size(), kMaxNamespaceBytes); + if (e.incarnation == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS ref catalog: namespace '{}' has a zero incarnation -- 0 never names a life", + e.ns.string()); + if (!creatorPairingOk(e)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS ref catalog: namespace '{}' is {} and {} a creator fence -- creator is required " + "iff state == Creating and forbidden otherwise", + e.ns.string(), nsStateToWord(e.state), e.creator ? "carries" : "lacks"); + if (!removalRoundPairingOk(e)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS ref catalog: namespace '{}' is {} and {} removal_started_round -- the field is " + "required iff state == Removing", + e.ns.string(), nsStateToWord(e.state), e.removal_started_round ? "carries" : "lacks"); + + bool first = true; + writeKey(out, "k", first); writeStringValue(out, "ent"); + writeKey(out, "ns", first); writeStringValue(out, e.ns.string()); + writeKey(out, "st", first); writeStringValue(out, nsStateToWord(e.state)); + writeKey(out, "inc", first); writeHex128Value(out, e.incarnation); + if (e.removal_started_round) + { + writeKey(out, "rsr", first); writeU64StringValue(out, *e.removal_started_round); + } + if (e.creator) + { + writeKey(out, "csr", first); writeStringValue(out, e.creator->server_root_id); + writeKey(out, "cwe", first); writeU64StringValue(out, e.creator->writer_epoch); + writeKey(out, "cfg", first); writeU64StringValue(out, e.creator->fence_generation); + } + closeObject(out, first); + closeLine("ent"); + } + + const size_t trailer_start = out.size(); + writeTrailerLine(out, catalog.entries.size()); /// emits its own terminator + checkLineBytes(out.size() - trailer_start - 1, "trailer"); + + return std::move(out).take(); +} + +RefCatalog decodeRefCatalog(std::string_view data) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::RefCatalog); + const uint64_t line_cap = traitsFor(FormatId::RefCatalog).line_cap; + + RefCatalog catalog; + uint64_t seen = 0; + for (;;) + { + const String line = readLine(in, line_cap, "ref catalog"); + ReadBufferFromMemory l(line.data(), line.size()); + JsonObjectReader r(l, KeyStrictness::Strict, "ref catalog"); + String key; + if (!r.nextKey(key)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: empty line"); + + if (key == "n") + { + const uint64_t n = r.readU64Number(); + if (r.nextKey(key)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: trailer has extra keys"); + if (!l.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: bytes after trailer"); + if (n != seen) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref catalog: trailer count {} != {} records", n, seen); + return catalog; + } + if (key != "k") + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: record must start with \"k\""); + const String kind = r.readString(); + if (kind != "ent") + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: unknown record kind '{}'", kind); + + String ns_str; + std::optional st_word; + std::optional inc; + std::optional csr; + std::optional cwe; + std::optional cfg; + std::optional removal_started_round; + while (r.nextKey(key)) + { + if (key == "ns") ns_str = r.readString(); + else if (key == "st") st_word = r.readString(); + else if (key == "inc") inc = r.readHex128(); + else if (key == "csr") csr = r.readString(); + else if (key == "cwe") cwe = r.readU64String(); + else if (key == "cfg") cfg = r.readU64String(); + else if (key == "rsr") removal_started_round = r.readU64String(); + else throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: unknown ent key '{}'", key); + } + if (!l.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: junk after record"); + + if (!st_word) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: entry '{}' missing st", ns_str); + const NsState state = nsStateFromWord(*st_word); /// throws CORRUPTED_DATA on an unknown word + + /// A missing "ns" key reads as the same empty string a present-but-empty one would, and both + /// are refused identically here -- an empty namespace would sort first (every non-empty + /// namespace compares strictly greater than ""), passing the canonical-order check below, and + /// then wedge every later catalog-driven pass that tries to build a ref/namespace-file key + /// from it (`Layout::checkNamespace` refuses an empty namespace). + if (ns_str.empty()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: an entry's namespace must not be empty"); + if (ns_str.size() > kMaxNamespaceBytes) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref catalog: namespace '{}' is {} bytes, over the {}-byte admission bound", + ns_str, ns_str.size(), kMaxNamespaceBytes); + + if (!inc) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref catalog: entry '{}' missing inc", ns_str); + if (*inc == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref catalog: namespace '{}' has a zero incarnation -- 0 never names a life", ns_str); + + const bool any_creator_field = csr || cwe || cfg; + const bool every_creator_field = csr && cwe && cfg; + std::optional creator; + if (state == NsState::Creating) + { + if (!every_creator_field) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref catalog: namespace '{}' is Creating but its creator fence is incomplete -- " + "server_root_id, writer_epoch and fence_generation are all required", ns_str); + creator = CreatorFence{.server_root_id = *csr, .writer_epoch = *cwe, .fence_generation = *cfg}; + } + else if (any_creator_field) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref catalog: namespace '{}' carries a creator fence at state '{}' -- creator is " + "forbidden on anything but Creating", ns_str, *st_word); + + if ((state == NsState::Removing) != removal_started_round.has_value()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref catalog: namespace '{}' is {} and {} removal_started_round -- the field is " + "required iff state == Removing", + ns_str, *st_word, removal_started_round ? "carries" : "lacks"); + + /// Canonical order, checked incrementally as records stream in: a namespace that does not + /// compare strictly greater than the previous one is either a duplicate or out of order -- + /// both are "not canonical", and this one check rejects either shape. + if (!catalog.entries.empty() && !(catalog.entries.back().ns.string() < ns_str)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref catalog: namespace '{}' does not sort strictly after the previous entry " + "'{}' -- entries must be canonically ordered with no duplicate namespace", + ns_str, catalog.entries.back().ns.string()); + + catalog.entries.push_back(CatalogEntry{.ns = RootNamespace{ns_str}, .state = state, + .incarnation = *inc, .creator = creator, + .removal_started_round = removal_started_round}); + ++seen; + } +} + +void checkCatalogObjectBytes(uint64_t encoded_bytes, const RootNamespace & ns) +{ + const uint64_t cap = traitsFor(FormatId::RefCatalog).object_cap; + if (!fitsObjectCap(encoded_bytes, /*entries_reservation*/0, cap)) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "CAS ref catalog: admitting '{}' would grow the catalog to {} bytes, over the {}-byte " + "object cap (predicate 1: encoded_catalog_bytes <= catalog_object_cap) -- refused before " + "the write", ns.string(), encoded_bytes, cap); +} + +uint64_t foldSealFixedBytes() +{ + static const uint64_t bytes = [] + { + CasFoldSeal seal; + seal.generation = std::numeric_limits::max(); + seal.parent_generation = std::numeric_limits::max(); + const uint64_t empty_bytes = encodeFoldSeal(seal).size(); + /// The empty trailer is `{"n":0}`. A real seal may carry a 20-digit record count. + return addByteBudget(empty_bytes, std::numeric_limits::digits10); + }(); + return bytes; +} + +uint64_t worstCaseEntryFoldReservationBytes() +{ + /// Measured through the REAL fold-seal encoder, never a hand-kept formula, so a later change to + /// the fold seal's wire shape is felt here automatically instead of silently drifting. + static const uint64_t bytes = []() -> uint64_t + { + constexpr uint64_t kU64Max = std::numeric_limits::max(); + constexpr uint32_t kU32Max = std::numeric_limits::max(); + + CasFoldSeal seal; + /// One catalog entry admits exactly one ref-life row. Charge its widest legal form: a held + /// coverage record plus terminal cleanup evidence, all numeric fields at maximum width. + seal.ref_lives[std::numeric_limits::max()] = RefLifeFoldState{ + .coverage = RefCoverage{ + .classification = 4, + .last_folded_ref_id = RefTxnId{kU64Max, kU64Max}, + .hold = RefHold{.reason = HoldReason::UnconsumedSealCrossing, + .offending_position = RefTxnId{kU64Max, kU64Max}, + .retry_count = kU32Max, + .next_retry_round = kU64Max}}, + .cleanup_evidence = RefCleanupEvidence{.remove_txn_id = RefTxnId{kU64Max, kU64Max}}}; + + return encodeFoldSeal(seal).size() - encodeFoldSeal(CasFoldSeal{}).size(); + }(); + return bytes; +} + +uint64_t widestBlobTargetRunReservationBytes(const Layout & layout, uint64_t gc_shards) +{ + if (gc_shards == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS ref catalog: gc_shards must be nonzero when reserving blob-target rows"); + + constexpr uint64_t max = std::numeric_limits::max(); + CasFoldSeal seal; + seal.blob_target_runs.push_back(RunRef{ + .key = layout.blobTargetRunKey(max, max, gc_shards - 1, 0), + .checksum = std::numeric_limits::max(), + .shard = gc_shards - 1, + .generation = max}); + return encodeFoldSeal(seal).size() - encodeFoldSeal(CasFoldSeal{}).size(); +} + +uint64_t widestCondemnedSummaryReservationBytes(uint64_t gc_shards) +{ + if (gc_shards == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS ref catalog: gc_shards must be nonzero when reserving condemned-summary rows"); + + constexpr uint64_t max = std::numeric_limits::max(); + CasFoldSeal seal; + seal.condemned_summary.emplace(gc_shards - 1, CondemnedSummary{ + .condemned_total = max, + .pending_total = max, + .oldest_nonpending_condemn_round = max}); + return encodeFoldSeal(seal).size() - encodeFoldSeal(CasFoldSeal{}).size(); +} + +void checkFoldSealReservation( + uint64_t entry_count, uint64_t gc_shards, const Layout & layout, const RootNamespace & ns) +{ + const uint64_t cap = foldSealCaps().object_cap; + const uint64_t fixed = foldSealFixedBytes(); + /// Saturating, like `fitsObjectCap`'s own addition one step later: an unsaturated product can + /// wrap to a remainder far smaller than the true reservation, which would answer "fits" for an + /// `entry_count` that plainly does not. + const uint64_t ref_lives = mulByteBudget(entry_count, worstCaseEntryFoldReservationBytes()); + /// `validateFoldSealStructure` permits at most one canonical seq-0 `btr` per shard, so charging + /// one widest row for every shard covers the full legal run domain without per-entry arithmetic. + const uint64_t blob_target_runs = mulByteBudget( + gc_shards, widestBlobTargetRunReservationBytes(layout, gc_shards)); + const uint64_t condemned_summaries = mulByteBudget( + gc_shards, widestCondemnedSummaryReservationBytes(gc_shards)); + const uint64_t reservation = addByteBudget( + ref_lives, addByteBudget(blob_target_runs, condemned_summaries)); + if (!fitsObjectCap(fixed, reservation, cap)) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "CAS ref catalog: admitting '{}' would need a fold seal reserving fixed={} + ref_lives={} " + "+ blob_target_runs={} + condemned_summaries={} bytes (entries={}, gc_shards={}), over " + "the {}-byte fold-seal object cap (predicate 2) -- refused before the write", + ns.string(), fixed, ref_lives, blob_target_runs, condemned_summaries, entry_count, gc_shards, cap); +} + +String checkCatalogAdmission( + const RefCatalog & candidate, uint64_t gc_shards, const Layout & layout, + const RootNamespace & admitting_ns) +{ + const String encoded = encodeRefCatalog(candidate); /// grammar-checked; LOGICAL_ERROR on our own bug + checkCatalogObjectBytes(encoded.size(), admitting_ns); /// predicate (1) + checkFoldSealReservation(candidate.entries.size(), gc_shards, layout, admitting_ns); /// predicate (2) + return encoded; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.h new file mode 100644 index 000000000000..ca1fbe5a6ddc --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCatalogFormat.h @@ -0,0 +1,168 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +class Layout; + +/// The byte bound every namespace name admitted into `ref_catalog` must satisfy (spec INV-3: +/// "namespace names get a byte bound"). It keeps the catalog's operator-visible row and line grammar +/// bounded, and both directions of the codec enforce it. Logical namespace bytes do NOT enter +/// predicate (2): fold-seal `rfl` rows are keyed only by the fixed-width opaque life id. +constexpr size_t kMaxNamespaceBytes = 512; + +/// One namespace's catalog lifecycle state (spec INV-3, §3). `Creating` blocks publication and +/// requires a `creator` fence identity; `Live` is the steady state and forbids `creator`; +/// `Removing` forbids new positive ownership and, like `Live`, forbids `creator` (a namespace at +/// this state was already `Live`, so no creation fence identity applies to it any longer). +/// +/// THESE ARE WIRE VALUES, AND THEY ARE APPEND-ONLY, exactly like `HoldReason`: a catalog object +/// written by one build is read by another, so a renumbered or repurposed value would make an older +/// catalog name a different lifecycle than the one it recorded. Add new states at the end; never +/// renumber, never repurpose a retired word. +enum class NsState : uint8_t +{ + Creating = 1, + Live = 2, + Removing = 3, +}; + +/// The wire word one `NsState` is persisted as. Exported so any other reader of a lifecycle state +/// renders the SAME three words this codec does, rather than a second, independently drifting copy. +/// Every value it is ever called with comes from either a live `NsState` (a compile-time-closed +/// enumeration) or a value `nsStateFromWord` already validated on decode, so an unrecognized value +/// reaching it is a bug in THIS process, not corruption arriving from a store -- `LOGICAL_ERROR`. +std::string_view nsStateToWord(NsState s); +/// Inverse of `nsStateToWord`; throws `CORRUPTED_DATA` for anything but the three registered words. +NsState nsStateFromWord(std::string_view w); + +/// The fence identity of the mounted writer CREATING one namespace (spec §3): the server root plus +/// the writer epoch and admission fence generation captured at the moment `Creating` was minted. It +/// is what a reconciler compares against `CasServerRoot`'s liveness/fence machinery before a stalled +/// `Creating` entry may be CAS-reconciled away (INV-3: "stalled creators occupy entries until +/// fence-terminal reconciliation"). +struct CreatorFence +{ + String server_root_id; + uint64_t writer_epoch = 0; + uint64_t fence_generation = 0; + + bool operator==(const CreatorFence &) const = default; +}; + +/// One namespace's catalog row. `incarnation` is the ref-layer-scoped life identity minted once, at +/// `Creating` (spec INV-3; consumed as a `NamespaceLifeId` by every ref/namespace-file key helper -- +/// see `NamespaceLifeId::fromCatalogEntry`), and never changes for the rest of this row's life: a +/// namespace dropped and recreated gets a FRESH row with a FRESH incarnation, never a reused one -- +/// that is what makes rebirth structurally inert instead of an alias. `incarnation == 0` is always +/// invalid, at every state -- "0 never names a life", the same rule `NamespaceLifeId` enforces. +/// +/// `creator` is a STRICT GRAMMAR pairing: REQUIRED iff `state == Creating`, FORBIDDEN otherwise. +/// `removal_started_round` is similarly REQUIRED iff `state == Removing`: it is sampled once by the +/// `Live -> Removing` catalog CAS and never changes, so diagnostics can measure removal age without +/// inventing a caller-local epoch. Both pairings are enforced in both codec directions. +struct CatalogEntry +{ + RootNamespace ns; + NsState state = NsState::Creating; + UInt128 incarnation = 0; + std::optional creator = std::nullopt; + std::optional removal_started_round = std::nullopt; + + bool operator==(const CatalogEntry &) const = default; +}; + +/// The whole-pool namespace catalog (spec INV-3): one object, key `cas/ref_catalog` +/// (`Layout::refCatalogKey`), read on every fold round and every recovery, mutated by one token-CAS +/// write per lifecycle transition. `entries` is CANONICALLY ORDERED by namespace bytes, strictly +/// ascending -- no duplicate namespace -- and both directions of the codec enforce it, so an +/// out-of-order or duplicate-keyed catalog can never become durable. +struct RefCatalog +{ + std::vector entries; + + bool operator==(const RefCatalog &) const = default; +}; + +/// Encodes `catalog` as the canonical `cas_ref_catalog` text object: a header line, one "ent" record +/// per entry in canonical (ns-sorted) order, and a record-count trailer -- the same tagged-record +/// container `encodeFoldSeal` uses. Enforces the FULL strict grammar on the way out: canonical order +/// and no duplicate namespace, a non-empty namespace within the `kMaxNamespaceBytes` bound, nonzero +/// incarnation, and the `creator`/state pairing. This is our own state about to become durable, so a +/// violation is `LOGICAL_ERROR`, not `CORRUPTED_DATA`. Also enforces the per-line `LIMIT_EXCEEDED` +/// line-cap gate (mirroring `encodeFoldSeal`'s `checkLineBytes` exactly, including its error code -- +/// a caller catching `LIMIT_EXCEEDED` as a capacity refusal must not see a `LOGICAL_ERROR` bug report +/// instead) -- but deliberately does NOT enforce the whole-object cap itself: that predicate must +/// name the namespace under admission, which only a caller of `checkCatalogAdmission` knows. +/// +/// These bytes go to and come from the backend DIRECTLY, exactly like `cas_ref_ckpt`: the Pool-side +/// `CasRefCatalog::read`/`casUpdateImpl` (`Pool/CasRefCatalog.cpp`) bypass `sealObject`/`openObject`, +/// which are the identity under this class's `CompressionPolicy::Never` and would add nothing. A +/// policy flip to `Always` therefore breaks this silently -- and is caught, because `storedSuffix` +/// would stop being empty and the registry test asserting `storedSuffix(FormatId::RefCatalog) == ""` +/// fails. That assertion is the tripwire for this shortcut, not an incidental check of the key shape. +/// One consequence of the bypass, stated rather than fixed here (pre-existing for `RefCkpt` too): +/// `openObject`'s own object-cap enforcement is skipped on the read path, so nothing on either the +/// plain write path or the read path enforces the 256 MiB object cap outside `checkCatalogAdmission` +/// -- the cap is load-bearing only through THAT gate, never through the codec or the backend read. +String encodeRefCatalog(const RefCatalog & catalog); + +/// Decodes and validates a `cas_ref_catalog` object, re-checking every grammar rule `encodeRefCatalog` +/// enforces against bytes that may have come from anywhere: `CORRUPTED_DATA` on a duplicate namespace, +/// non-canonical order, a missing, empty, or over-bound namespace, a zero incarnation, an incomplete or +/// forbidden creator fence, an unknown state word, or trailing bytes. +RefCatalog decodeRefCatalog(std::string_view data); + +/// PRE-PUT GATE, predicate (1) of INV-3's additive admission: `encoded_bytes <= catalog_object_cap` +/// (the registry's own cap for `FormatId::RefCatalog`). Equality is accepted; refuses +/// (`LIMIT_EXCEEDED`, naming `ns`) one byte over. +void checkCatalogObjectBytes(uint64_t encoded_bytes, const RootNamespace & ns); + +/// Worst-case bytes for the fold-seal frame with no records: maximal generation fields plus the +/// widest possible `uint64_t` trailer count. Measured through the real encoder. +uint64_t foldSealFixedBytes(); + +/// The worst-case bytes ONE admitted catalog entry could ever add to a fold seal: one ref-life row +/// containing held coverage and terminal cleanup evidence at their widest legal shapes. Measured +/// through `encodeFoldSeal` itself, like `foldSealFixedBytes`. +uint64_t worstCaseEntryFoldReservationBytes(); + +/// Worst-case incremental bytes for one canonical blob-target run row. The serialized physical key +/// includes `layout`'s pool prefix, so layout is part of the bound. +uint64_t widestBlobTargetRunReservationBytes(const Layout & layout, uint64_t gc_shards); + +/// Worst-case incremental bytes for one condemned-summary row at the greatest configured shard. +uint64_t widestCondemnedSummaryReservationBytes(uint64_t gc_shards); + +/// PRE-PUT GATE, predicate (2) of INV-3's additive admission. Reserves the widest fixed frame, one +/// widest ref-life row per candidate catalog entry, and one widest blob-target plus condemned-summary +/// row per authoritative GC shard. The `btr` multiplier follows the authoritative fold-seal grammar: +/// at most one canonical sequence-0 run is legal for each shard. Equality is accepted; refuses +/// (`LIMIT_EXCEEDED`, naming `ns`) one entry over. Every multiplication and addition saturates, so an +/// unreachable-in-practice count can never wrap into something that reads as "fits". +void checkFoldSealReservation( + uint64_t entry_count, uint64_t gc_shards, const Layout & layout, const RootNamespace & ns); + +/// Runs BOTH admission predicates against `candidate` -- the catalog state as it would read +/// immediately AFTER the admission under consideration -- naming `admitting_ns` in whichever +/// predicate refuses (INV-3: "admission refuses loudly"; "TWO INDEPENDENT predicates"). `candidate` +/// is grammar-checked first (via `encodeRefCatalog`; `LOGICAL_ERROR` on our own bug), then predicate +/// (1) and predicate (2), in that order. Returns the encoded bytes on success, so a caller's `casPut` +/// writes EXACTLY what admission checked -- never a second, independently re-encoded copy. +/// +/// Constraint 13 (removal is never refused): this function is for entry-ADMITTING mutations only. +/// A removal transition (`Live` -> `Removing`) must go through the catalog's plain update path +/// instead, never through here. +String checkCatalogAdmission( + const RefCatalog & candidate, uint64_t gc_shards, const Layout & layout, + const RootNamespace & admitting_ns); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.cpp new file mode 100644 index 000000000000..6ff7fa5dda43 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.cpp @@ -0,0 +1,179 @@ +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +void checkRefCkptInvariants(const RefCkpt & ckpt, std::string_view what) +{ + /// PRESENT means REAL. `life_epoch` may be absent (no writer of this object knew the namespace's + /// genesis epoch), but a present one is a `writer_epoch`, and `RefTxnId` forbids a zero epoch. + /// Accepting zero would give the field two meanings -- "unknown" and "epoch zero" -- on an object + /// that gates destructive cleanup. + if (ckpt.life_epoch && *ckpt.life_epoch == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: a present life_epoch must be nonzero", what); + + /// A PRESENT id must be a real one. Both components nonzero is `RefTxnId`'s own validity rule + /// (`renderRefTxnId` refuses to build a key from anything else), so a half-zero id here would name + /// an object that cannot exist. + const auto check_id = [&](const std::optional & id, std::string_view field) + { + if (id && (id->writer_epoch == 0 || id->ref_sequence == 0)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: {} fields must both be nonzero, got {}-{}", + what, field, id->writer_epoch, id->ref_sequence); + }; + check_id(ckpt.checkpoint_snapshot_id, "checkpoint_snapshot_id"); + check_id(ckpt.last_epoch_seal, "last_epoch_seal"); + check_id(ckpt.committed_through, "committed_through"); + if (!ckpt.committed_through && (ckpt.checkpoint_snapshot_id || ckpt.last_epoch_seal)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: checkpoint_snapshot_id and last_epoch_seal require committed_through", what); + if (ckpt.committed_through) + { + if (ckpt.life_epoch && ckpt.committed_through->writer_epoch < *ckpt.life_epoch) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: committed_through must not precede life_epoch", what); + if (ckpt.checkpoint_snapshot_id && *ckpt.committed_through < *ckpt.checkpoint_snapshot_id) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: checkpoint_snapshot_id must not exceed committed_through", what); + if (ckpt.last_epoch_seal && *ckpt.committed_through < *ckpt.last_epoch_seal) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: last_epoch_seal must not exceed committed_through", what); + + /// The checkpoint carries the same finite proof as the ref-log chain. A current epoch is either + /// closed at the frontier itself, or its frontier follows the seal of exactly the preceding + /// numeric epoch. A seal from the same epoch below the frontier would claim that an epoch kept + /// accepting transactions after it was closed; a larger gap would let a missing epoch masquerade + /// as a proved boundary. + if (ckpt.last_epoch_seal) + { + if (*ckpt.last_epoch_seal != *ckpt.committed_through + && ckpt.last_epoch_seal->writer_epoch + 1 != ckpt.committed_through->writer_epoch) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: last_epoch_seal must equal committed_through or close its immediately preceding writer epoch", + what); + } + else if (ckpt.life_epoch && ckpt.committed_through->writer_epoch > *ckpt.life_epoch) + { + /// With a known genesis epoch, an unsealed later epoch has no chain evidence. Leave the + /// unknown-genesis contribution representable: another checkpoint writer may still merge + /// the genesis fact before this partial contribution is encoded as durable authority. + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: committed_through after life_epoch requires last_epoch_seal", what); + } + } +} + +String encodeRefCkpt(const RefCkpt & ckpt) +{ + checkRefCkptInvariants(ckpt, "cas_ref_ckpt encode"); + + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::RefCkpt); + bool first = true; + /// Every field is encoded WHEN SET and omitted otherwise, so "nobody knows the genesis epoch", + /// "no checkpoint yet", and "no epoch has been closed yet" are absences on the wire rather than + /// sentinel values a reader would have to interpret. The two ids are flat epoch/sequence PAIRS, + /// written by the one shared `RefTxnId` writer the `_log` and `_snap` formats also use, so the + /// three ref formats cannot disagree on the encoding. + if (ckpt.life_epoch) + { + writeKey(out, "le", first); + writeU64StringValue(out, *ckpt.life_epoch); + } + if (ckpt.committed_through) + writeRefTxnIdFields(out, first, "cte", "cts", *ckpt.committed_through); + if (ckpt.checkpoint_snapshot_id) + writeRefTxnIdFields(out, first, "cse", "css", *ckpt.checkpoint_snapshot_id); + if (ckpt.last_epoch_seal) + writeRefTxnIdFields(out, first, "lse", "lss", *ckpt.last_epoch_seal); + closeObject(out, first); + writeChar('\n', out); + + String text = std::move(out).take(); + /// The registry cap is a corruption brake, not a budget: this object has three fields and cannot + /// approach it. Checking on the WRITE side too means an encoder bug surfaces here rather than as an + /// object that was accepted on write and is unreadable on decode. + const uint64_t object_cap = traitsFor(FormatId::RefCkpt).object_cap; + if (text.size() > object_cap) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS cas_ref_ckpt encode: encoded size {} exceeds the object cap {}", text.size(), object_cap); + return text; +} + +RefCkpt decodeRefCkpt(std::string_view data) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::RefCkpt); + const String body = readLine(in, traitsFor(FormatId::RefCkpt).line_cap, "cas_ref_ckpt"); + ReadBufferFromMemory body_in(body.data(), body.size()); + /// STRICT: an unknown ordinary key is `CORRUPTED_DATA` and a `!`-prefixed one is + /// `UNKNOWN_FORMAT_VERSION`. `_ckpt` is a control object whose every field changes what cleanup is + /// allowed to delete, so a reader that silently ignored a key it did not understand would be + /// deciding deletions from a body it only partially read. Duplicate keys are rejected by + /// `JsonObjectReader` itself. + JsonObjectReader r(body_in, KeyStrictness::Strict, "cas_ref_ckpt"); + + RefCkpt ckpt; + std::optional cse; + std::optional css; + std::optional lse; + std::optional lss; + std::optional cte; + std::optional cts; + String key; + while (r.nextKey(key)) + { + if (key == "le") ckpt.life_epoch = r.readU64String(); + else if (key == "cte") cte = r.readU64String(); + else if (key == "cts") cts = r.readU64String(); + else if (key == "cse") cse = r.readU64String(); + else if (key == "css") css = r.readU64String(); + else if (key == "lse") lse = r.readU64String(); + else if (key == "lss") lss = r.readU64String(); + else r.skipUnknown(key); + } + + /// TRUNCATION IS REJECTED. Half an id pair is the dangerous shape: silently dropping it would turn + /// a truncated body into a well-formed `_ckpt` with NO checkpoint, which reads as "nothing is + /// deletable" today and as "recovery has no base" tomorrow -- both of which a reader would trust. + /// Fail closed instead. (A missing whole field is a legitimate absence, not truncation: every field + /// of this object is optional, so there is nothing to miss.) + if (cse || css) + { + if (!cse || !css) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_ref_ckpt: checkpoint_snapshot_id needs both cse and css"); + ckpt.checkpoint_snapshot_id = RefTxnId{*cse, *css}; + } + if (cte || cts) + { + if (!cte || !cts) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_ref_ckpt: committed_through needs both cte and cts"); + ckpt.committed_through = RefTxnId{*cte, *cts}; + } + if (lse || lss) + { + if (!lse || !lss) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_ref_ckpt: last_epoch_seal needs both lse and lss"); + ckpt.last_epoch_seal = RefTxnId{*lse, *lss}; + } + if (!body_in.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS cas_ref_ckpt: trailing bytes"); + + checkRefCkptInvariants(ckpt, "cas_ref_ckpt"); + return ckpt; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.h new file mode 100644 index 000000000000..d0331248db6f --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefCkptFormat.h @@ -0,0 +1,124 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// One namespace LIFE's checkpoint object (spec INV-4), persisted as the mutable, token-CAS +/// `cas_ref_ckpt` control object at `Layout::refCkptKey`, whose argument is a `NamespaceLifeId`. +/// +/// It exists because prefix cleaning made the ref stream unreadable from a LIST alone: a cleaned +/// prefix plus a hidden snapshot is indistinguishable from an empty one, so recovery cannot decide +/// which snapshot is its base by enumerating keys. `_ckpt` is the point-read answer -- it NAMES the +/// base -- and, being the only authority on it, it also becomes the gate on destructive cleanup: +/// +/// - snapshots are deletable only STRICTLY BELOW `checkpoint_snapshot_id` (so a STALE pointer can +/// only ever under-clean, never delete the base a live recovery is about to fetch); +/// - a sampled base that 404s is adjudicated against this object's TOKEN, not its content: an +/// advanced token means cleanup moved the base while we read (restart), an unchanged token means +/// the base was deleted under a live checkpoint, which is corruption. +/// +/// TWO writers update it -- the snapshot publisher and the sealer -- and both run the SAME algorithm +/// (`mergeCkpt` + `publishCkpt` in `Pool/CasRefCkpt.h`): read the whole body, merge by SEMANTIC +/// MAXIMUM per field, token-CAS. (`publishCkpt` adds one refusal the merge itself cannot express -- a +/// `life_epoch` BELOW the durable one -- because only the publish site knows which side is durable.) +/// Writing the whole body is what makes a stale field dangerous, and +/// the merge is what contains it: a writer that skipped it and wrote back the value it sampled +/// earlier would silently regress the OTHER writer's progress (TLC counterexample +/// `_sab_sealclobbersbase`, which loses an acked transaction). +/// +/// INVARIANT (Constraint 15), and it constrains what may ever be ADDED here, not merely what is here +/// today: `_ckpt` is a fixed-size product of SCALAR MONOTONE FACTS. Its encoded size is `O(1)` in refs, +/// files, transactions and writer epochs. Maps, collections and cardinality-growing fields belong in a +/// separate immutable object or ledger -- never in this one, because this one is MUTABLE, is rewritten +/// whole on every publish by two concurrent writers, and has NO REPAIR PATH: a `_ckpt` that grows with +/// the table is a body that eventually cannot be rewritten atomically, on the single object that names +/// recovery's base and gates destructive cleanup. +/// +/// The four dimensions do not all hold the same way, and the difference is worth stating so the +/// invariant is checkable rather than merely believed: +/// - refs and files enter in NO form. Two namespaces differing only in how many refs they hold encode +/// BYTE-IDENTICAL `_ckpt` bodies. +/// - transactions and writer epochs enter as the DECIMAL WIDTH of the two id pairs -- four orders of +/// magnitude of `ref_sequence` cost four bytes. That is `O(1)` because the components are +/// `uint64_t`, so the width is ceilinged at twenty digits and the whole object at a constant far +/// below `traitsFor(FormatId::RefCkpt).object_cap` (which stays what it is documented to be: a +/// corruption brake this object cannot approach, never the thing that bounds its size). +/// Both halves are fenced by `gtest_cas_ref_ckpt_join.cpp`, which also carries the compile-time +/// `std::is_trivially_copyable_v` assertion that a heap-owning field would break. +struct RefCkpt +{ + /// The namespace's birth epoch -- the `writer_epoch` of its `NamespaceBirth` record. It is what + /// makes the epoch-seal grammar checkable without walking to the beginning of the stream + /// (`validateEpochSealGrammarContextual` takes exactly this value). + /// + /// It is NOT a namespace-lifetime constant, and the previous version of this comment said it was -- + /// which is how the merge rule below came to be described as "its semantic maximum is itself". TWO + /// writers know a `life_epoch` and they derive it from different epochs: `completeCreation` from the + /// catalog creator's `writer_epoch`, and `commitRefChunk`'s birth chunk from the `NamespaceBirth` + /// record's. Those differ whenever a stalled `Creating` entry is resumed by a later actor over the + /// same incarnation, and whenever the mount's writer epoch advances between the creation and the + /// first write -- CREATE TABLE, restart, INSERT. The value that must survive is the LATER one (the + /// grammar needs the epoch the birth record actually landed at), and it is always the later + /// contribution, because writer epochs are durable-monotone per server root. So the semantic maximum + /// is right, and what is refused is a DECREASE rather than a disagreement -- by `publishCkpt`, not by + /// `mergeCkpt`, since only the publish site can tell which of the two values is the durable one. + /// + /// OPTIONAL, and the option is load-bearing rather than a convenience. Exactly ONE writer knows + /// this value -- the transaction that births the namespace -- and a table recovered from durable + /// objects written before it existed has no way to learn it. Every OTHER writer therefore + /// contributes `nullopt` and the merge leaves whatever is there alone. Making it mandatory would + /// force those writers to supply a number they do not have, and the semantic-max merge can never + /// lower a wrong one: a guess here is permanent. A consumer that NEEDS the genesis epoch (Stage B's + /// cross-epoch GC fold) must fail closed on `nullopt`, never substitute a floor. + std::optional life_epoch; + /// The greatest transaction admitted to durable logical history. Absent means this life has no + /// committed transaction; a snapshot or epoch seal is then invalid because neither can describe + /// history that was never committed. + std::optional committed_through = std::nullopt; + /// The snapshot recovery point-reads as its base, and the floor cleanup deletes strictly below. + /// `nullopt` until this namespace's first snapshot publication commits. + std::optional checkpoint_snapshot_id; + /// The `EpochSeal` transaction that closed the newest epoch known to have been closed. Consumed by + /// a later mount locating the previous epoch's terminating record and by the GC fold crossing + /// epochs. `nullopt` before this namespace has ever had an epoch closed. + std::optional last_epoch_seal; + + bool operator==(const RefCkpt &) const = default; +}; + +/// Encode `ckpt` as the canonical `cas_ref_ckpt` text object: a versioned header line followed by one +/// JSON body object. STRICT IN BOTH DIRECTIONS -- the same `checkRefCkptInvariants` that guards decode +/// runs here first, so a struct this build would refuse to read can never be written by it either +/// (`CORRUPTED_DATA`). Encoding is canonical and deterministic, which is what lets `publishCkpt` +/// compare a merged result against what it read. +/// +/// These bytes go to and come from the backend DIRECTLY: this pair bypasses `sealObject`/`openObject`, +/// which are the identity under this class's `CompressionPolicy::Never` and would add nothing. A +/// policy flip to `Always` therefore breaks this silently -- and is caught, because `storedSuffix` +/// would stop being empty and the registry test asserting `storedSuffix(FormatId::RefCkpt) == ""` +/// fails. That assertion is the tripwire for this shortcut, not an incidental check of the key shape. +String encodeRefCkpt(const RefCkpt & ckpt); + +/// Decode a complete `cas_ref_ckpt` text object. STRICT (`KeyStrictness::Strict`): an unknown ordinary +/// key, a duplicate key, a truncated object (a missing body line, or half of an optional +/// id pair), or trailing bytes all raise `CORRUPTED_DATA` -- never a partially-populated struct. This +/// object gates destructive cleanup and names recovery's base, so "decoded something" must mean +/// "decoded exactly what a writer of this format wrote". +RefCkpt decodeRefCkpt(std::string_view data); + +/// The shared field-level validity rule, applied on both encode and decode: every PRESENT field is a +/// real value -- a set `life_epoch` is nonzero, and a present id has both components nonzero. When a +/// frontier is present, its writer epoch may not precede `life_epoch`, and a snapshot may not exceed +/// it. Its `last_epoch_seal` is either that exact frontier or closes the immediately preceding numeric +/// writer epoch; without a seal, a known `life_epoch` permits only that genesis epoch. `what` +/// identifies the direction in the exception message. Exposed so a caller that assembles a `RefCkpt` +/// from several sources can fail closed before it reaches the wire. +void checkRefCkptInvariants(const RefCkpt & ckpt, std::string_view what); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.cpp new file mode 100644 index 000000000000..be7ee5567575 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.cpp @@ -0,0 +1,443 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +namespace +{ + +std::string_view opKindToWord(RefOpKind k) +{ + switch (k) + { + case RefOpKind::NamespaceBirth: return "namespace_birth"; + case RefOpKind::OwnerTransition: return "owner_transition"; + case RefOpKind::SetPublishedAt: return "set_published_at"; + case RefOpKind::RemoveNamespace: return "remove_namespace"; + case RefOpKind::EpochSeal: return "epoch_seal"; + } + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: unknown op kind {}", static_cast(k)); +} + +RefOpKind opKindFromWord(std::string_view w) +{ + if (w == "namespace_birth") return RefOpKind::NamespaceBirth; + if (w == "owner_transition") return RefOpKind::OwnerTransition; + if (w == "set_published_at") return RefOpKind::SetPublishedAt; + if (w == "remove_namespace") return RefOpKind::RemoveNamespace; + if (w == "epoch_seal") return RefOpKind::EpochSeal; + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: unknown op kind '{}'", w); +} + +/// Byte budget over the encoded text. A removal-class transaction uses the larger complete-table +/// budget and has neither an op-count nor a per-op cap; normal transactions are bounded by +/// `ref_txn_max_ops` and, per op, by `ref_op_max_bytes` (checked via `encodedOpSize`, one op at a +/// time -- no accumulation). +void checkBudget(const std::vector & ops, size_t encoded_bytes) +{ + const bool removal = refLogTxnIsRemovalClass(ops); + const size_t byte_limit = removal ? ref_removal_max_bytes : ref_txn_max_bytes; + if (encoded_bytes > byte_limit) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: encoded size {} exceeds the {}-class byte limit {}", + encoded_bytes, removal ? "removal" : "normal", byte_limit); + if (removal) + return; + if (ops.size() > ref_txn_max_ops) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: {} operations exceeds the normal-class op-count limit {}", ops.size(), ref_txn_max_ops); + for (const RefOp & op : ops) + { + const size_t op_bytes = encodedOpSize(op); + if (op_bytes > ref_op_max_bytes) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: op encoded size {} exceeds the normal-class per-op limit {}", op_bytes, ref_op_max_bytes); + } +} + +void writeBindingFields(CasJsonWriter & out, bool & first, std::string_view prefix, const RefOwnerBinding & b) +{ + checkCanonicalRefName(b.ref_name, "RefLogTxn", "owner binding ref_name"); + checkManifestRef(b.manifest_ref, "RefLogTxn", "owner binding manifest_ref"); + out.key(prefix, "bk", first); + writeStringValue(out, refOwnerKindToWord(b.kind)); + out.key(prefix, "rn", first); + writeStringValue(out, b.ref_name); + writeManifestRefFields(out, first, prefix, b.manifest_ref); +} + +void writeOp(CasJsonWriter & out, const RefOp & op) +{ + bool first = true; + writeKey(out, "op", first); + writeStringValue(out, opKindToWord(op.kind)); + switch (op.kind) + { + case RefOpKind::NamespaceBirth: + case RefOpKind::RemoveNamespace: + case RefOpKind::EpochSeal: + break; + case RefOpKind::OwnerTransition: + if (op.old_binding) + writeBindingFields(out, first, "o", *op.old_binding); + if (op.new_binding) + writeBindingFields(out, first, "n", *op.new_binding); + break; + case RefOpKind::SetPublishedAt: + checkCanonicalRefName(op.ref_name, "RefLogTxn", "set_published_at ref_name"); + checkManifestRef(op.expected_manifest_ref, "RefLogTxn", "set_published_at manifest_ref"); + writeKey(out, "rn", first); + writeStringValue(out, op.ref_name); + writeManifestRefFields(out, first, "", op.expected_manifest_ref); + writeKey(out, "ts", first); + writeIntText(op.published_at_ms, out); + break; + } + closeObject(out, first); + writeChar('\n', out); +} + +/// Collector for a ManifestRef's three flat fields under an optional prefix. +struct ManifestFields +{ + std::optional me; + std::optional mb; + std::optional mo; + + bool any() const { return me || mb || mo; } + ManifestRef build(std::string_view what) const + { + if (!me || !mb || !mo) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: {} manifest_ref missing me/mb/mo", what); + return manifestRefFromFields(*me, *mb, *mo, "RefLogTxn", what); + } +}; + +/// Collector for one binding (old/new) under a prefix. +struct BindingFields +{ + std::optional bk; + std::optional rn; + ManifestFields mf; + + bool any() const { return bk || rn || mf.any(); } + RefOwnerBinding build(std::string_view what) const + { + if (!bk || !rn) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: {} binding missing bk/rn", what); + RefOwnerBinding b; + b.kind = refOwnerKindFromWord(*bk, "RefLogTxn owner binding"); + b.ref_name = *rn; + checkCanonicalRefName(b.ref_name, "RefLogTxn", "owner binding ref_name"); + b.manifest_ref = mf.build(what); + return b; + } +}; + +/// The log transaction's header-object meta line (ns + txn_id + the optional `prev_epoch_seal` +/// chain). Shared by `encodeRefLogTxn` and `removalFramingSize` so the two never disagree by a byte; +/// `removalFramingSize` always passes `std::nullopt` -- a removal transaction is never a sequence-1 +/// epoch-transition record. Additive: the `"!pse"`/`"!pss"` pair is emitted only when +/// `prev_epoch_seal` is set, so a body without it is byte-identical to the pre-EpochSeal wire shape. +/// `!`-prefixed: `prev_epoch_seal` is INV-2 chain evidence, not cosmetic metadata -- a decoder that +/// doesn't understand it must refuse the object rather than silently drop the chain link while +/// otherwise passing the structural grammar (`JsonObjectReader::skipUnknown` rejects any unrecognized +/// `!`-key with `UNKNOWN_FORMAT_VERSION`, tolerant or not; see task-1 review finding M4). +void writeLogMeta(CasJsonWriter & out, const String & ns, const RefTxnId & txn_id, const std::optional & prev_epoch_seal) +{ + bool first = true; + writeKey(out, "ns", first); + writeStringValue(out, ns); + writeRefTxnIdFields(out, first, "we", "rs", txn_id); + if (prev_epoch_seal) + writeRefTxnIdFields(out, first, "!pse", "!pss", *prev_epoch_seal); + closeObject(out, first); + writeChar('\n', out); +} + +RefOp readOpRecord(JsonObjectReader & r, RefOpKind kind) +{ + RefOp op; + op.kind = kind; + + /// set_published_at fields + std::optional sp_rn; + ManifestFields sp_mf; + std::optional sp_ts; + /// owner_transition bindings + BindingFields ob; + BindingFields nb; + + String key; + while (r.nextKey(key)) + { + if (key == "rn") sp_rn = r.readString(); + else if (key == "me") sp_mf.me = r.readU64String(); + else if (key == "mb") sp_mf.mb = r.readU64String(); + else if (key == "mo") sp_mf.mo = r.readU64Number(); + else if (key == "ts") sp_ts = r.readU64Number(); + else if (key == "obk") ob.bk = r.readString(); + else if (key == "orn") ob.rn = r.readString(); + else if (key == "ome") ob.mf.me = r.readU64String(); + else if (key == "omb") ob.mf.mb = r.readU64String(); + else if (key == "omo") ob.mf.mo = r.readU64Number(); + else if (key == "nbk") nb.bk = r.readString(); + else if (key == "nrn") nb.rn = r.readString(); + else if (key == "nme") nb.mf.me = r.readU64String(); + else if (key == "nmb") nb.mf.mb = r.readU64String(); + else if (key == "nmo") nb.mf.mo = r.readU64Number(); + else if (key == "pl") + /// `"pl"` (payload) was removed from the op wire in stage-1 T12 (the `set_payload` op became + /// `set_published_at`). The retired op WORD is already rejected by `opKindFromWord`, but this + /// generic reader reads field keys before switching on kind, so a `"pl"` field paired with a + /// still-recognized op word would otherwise be `skipUnknown`'d. It is a KNOWN-removed field, + /// not a genuinely-unknown one -- reject it explicitly rather than silently discard it. + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: op record carries the removed \"pl\" (payload) field"); + else r.skipUnknown(key); + } + + switch (kind) + { + case RefOpKind::NamespaceBirth: + case RefOpKind::RemoveNamespace: + case RefOpKind::EpochSeal: + break; + case RefOpKind::OwnerTransition: + if (ob.any()) + op.old_binding = ob.build("old"); + if (nb.any()) + op.new_binding = nb.build("new"); + break; + case RefOpKind::SetPublishedAt: + if (!sp_rn || !sp_ts) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: set_published_at missing rn/ts"); + op.ref_name = *sp_rn; + checkCanonicalRefName(op.ref_name, "RefLogTxn", "set_published_at ref_name"); + op.expected_manifest_ref = sp_mf.build("set_published_at manifest_ref"); + op.published_at_ms = *sp_ts; + break; + } + return op; +} + +} + +bool refLogTxnIsEpochSeal(const RefLogTxn & txn) +{ + return txn.ops.size() == 1 && txn.ops.front().kind == RefOpKind::EpochSeal; +} + +void validateEpochSealGrammarStructural(const RefLogTxn & txn) +{ + const bool has_seal_op = std::any_of(txn.ops.begin(), txn.ops.end(), + [](const RefOp & op) { return op.kind == RefOpKind::EpochSeal; }); + if (has_seal_op && txn.ops.size() != 1) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: a transaction carrying an EpochSeal op must contain exactly that one op, got {} ops", + txn.ops.size()); + + if (txn.prev_epoch_seal) + { + checkRefTxnIdNonzero(*txn.prev_epoch_seal, "RefLogTxn", "prev_epoch_seal"); + if (txn.txn_id.ref_sequence != 1) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: prev_epoch_seal is only allowed at sequence 1, got txn_id {}-{}", + txn.txn_id.writer_epoch, txn.txn_id.ref_sequence); + /// INV-2 materializes every global writer epoch for an existing life. A sequence-1 transaction + /// in E therefore chains to the closing seal of exactly E-1, not merely an arbitrary earlier + /// epoch. A skip would make a missing intermediate seal look like a proved boundary and let a + /// fold or destructive tail walk bypass it. This is a context-free property of one body, so + /// reject it in the codec before any walker can interpret the link as evidence. + if (txn.prev_epoch_seal->writer_epoch >= txn.txn_id.writer_epoch) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: prev_epoch_seal writer_epoch {} must be strictly less than this " + "transaction's writer_epoch {}", txn.prev_epoch_seal->writer_epoch, txn.txn_id.writer_epoch); + if (txn.prev_epoch_seal->writer_epoch + 1 != txn.txn_id.writer_epoch) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: prev_epoch_seal writer_epoch {} must immediately precede this " + "transaction's writer_epoch {}", txn.prev_epoch_seal->writer_epoch, txn.txn_id.writer_epoch); + } +} + +void validateEpochSealGrammarContextual(const RefLogTxn & txn, uint64_t life_epoch) +{ + /// The unconditional "forbidden outside sequence 1" half of the rule is + /// `validateEpochSealGrammarStructural`'s job; this function only owns the required-iff rule, + /// which is meaningless off sequence 1. + if (txn.txn_id.ref_sequence != 1) + return; + const bool required = txn.txn_id.writer_epoch > life_epoch; + if (required && !txn.prev_epoch_seal) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: sequence-1 txn at writer_epoch {} (life_epoch {}) must carry prev_epoch_seal", + txn.txn_id.writer_epoch, life_epoch); + if (!required && txn.prev_epoch_seal) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: sequence-1 txn at writer_epoch {} (life_epoch {}) must not carry prev_epoch_seal", + txn.txn_id.writer_epoch, life_epoch); +} + +String encodeRefLogTxn(const RefLogTxn & txn) +{ + checkRefTxnIdNonzero(txn.txn_id, "RefLogTxn", "txn_id"); + validateEpochSealGrammarStructural(txn); + + CasJsonWriter out(512); + writeHeaderLine(out, FormatId::RefLog); + + writeLogMeta(out, txn.ns, txn.txn_id, txn.prev_epoch_seal); + + for (const RefOp & op : txn.ops) + writeOp(out, op); + + writeTrailerLine(out, txn.ops.size()); + String text = std::move(out).take(); + checkBudget(txn.ops, text.size()); + return text; +} + +RefLogTxn decodeRefLogTxn(std::string_view data, const String & expected_ns, const RefTxnId & expected_txn_id) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::RefLog); + const uint64_t line_cap = traitsFor(FormatId::RefLog).line_cap; + + RefLogTxn txn; + + /// meta line + { + const String line = readLine(in, line_cap, "cas_ref_log"); + ReadBufferFromMemory m(line.data(), line.size()); + JsonObjectReader r(m, KeyStrictness::Tolerant, "cas_ref_log"); + bool saw_ns = false; + bool saw_we = false; + bool saw_rs = false; + std::optional pse; + std::optional pss; + String key; + while (r.nextKey(key)) + { + if (key == "ns") { txn.ns = r.readString(); saw_ns = true; } + else if (key == "we") { txn.txn_id.writer_epoch = r.readU64String(); saw_we = true; } + else if (key == "rs") { txn.txn_id.ref_sequence = r.readU64String(); saw_rs = true; } + else if (key == "!pse") pse = r.readU64String(); + else if (key == "!pss") pss = r.readU64String(); + else r.skipUnknown(key); + } + if (!saw_ns || !saw_we || !saw_rs) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: meta line missing ns/we/rs"); + /// Both-or-neither: `nextKey` already rejects a repeated "!pse"/"!pss" (duplicate-key check), so + /// this only guards against a body carrying exactly one of the pair. + if (pse || pss) + { + if (!pse || !pss) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: prev_epoch_seal needs both !pse and !pss"); + txn.prev_epoch_seal = RefTxnId{*pse, *pss}; + } + if (!m.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: junk after meta line"); + } + + checkRefTxnIdNonzero(txn.txn_id, "RefLogTxn", "txn_id"); + /// The namespace and transaction id are duplicated in the body because the object key is the + /// source of truth for which transaction is being read. Reject a valid body copied under a + /// different key before accepting any of its operations. + if (txn.ns != expected_ns || txn.txn_id != expected_txn_id) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: body (ns='{}', txn_id={}-{}) does not match the key it was read from " + "(ns='{}', txn_id={}-{})", + txn.ns, txn.txn_id.writer_epoch, txn.txn_id.ref_sequence, + expected_ns, expected_txn_id.writer_epoch, expected_txn_id.ref_sequence); + + /// op record lines, until the trailer + while (true) + { + const String line = readLine(in, line_cap, "cas_ref_log"); + ReadBufferFromMemory l(line.data(), line.size()); + JsonObjectReader r(l, KeyStrictness::Tolerant, "cas_ref_log"); + String key; + if (!r.nextKey(key)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: empty line"); + if (key == "n") + { + const uint64_t n = r.readU64Number(); + while (r.nextKey(key)) + r.skipUnknown(key); + if (!l.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: bytes after trailer"); + if (n != txn.ops.size()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefLogTxn: trailer count {} != {} ops", n, txn.ops.size()); + break; + } + if (key != "op") + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: record must start with \"op\""); + const RefOpKind kind = opKindFromWord(r.readString()); + txn.ops.push_back(readOpRecord(r, kind)); + if (!l.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefLogTxn: junk after op record"); + } + + /// Finalization: `txn.ops` is now complete, so the context-free seal grammar (which needs the + /// full op list) can run. Both directions of the codec enforce the identical rule -- see + /// `encodeRefLogTxn`. + validateEpochSealGrammarStructural(txn); + checkBudget(txn.ops, data.size()); + return txn; +} + +size_t encodedOpSize(const RefOp & op) +{ + CasJsonWriter out(256); + writeOp(out, op); + return out.size(); +} + +bool refLogTxnIsRemovalClass(const std::vector & ops) +{ + return std::any_of(ops.begin(), ops.end(), [](const RefOp & op) { return op.kind == RefOpKind::RemoveNamespace; }); +} + +size_t removalOpEncodedSize(RefOwnerKind owner_kind, const String & ref_name, const ManifestRef & manifest_ref) +{ + /// One exact owner-removal op, exactly as it appears in a hypothetical whole-namespace removal + /// transaction: an owner_transition with only an old binding, no new binding. + RefOp op; + op.kind = RefOpKind::OwnerTransition; + op.old_binding = RefOwnerBinding{owner_kind, ref_name, manifest_ref}; + + CasJsonWriter out(256); + writeOp(out, op); + return out.size(); +} + +size_t removalFramingSize(const String & ns, const RefTxnId & txn_id, uint64_t op_count) +{ + /// Header + meta + the terminal remove_namespace op + trailer(op_count). `op_count` counts every op + /// including the remove_namespace op (i.e. committed + precommits + 1). + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::RefLog); + writeLogMeta(out, ns, txn_id, std::nullopt); + RefOp remove_op; + remove_op.kind = RefOpKind::RemoveNamespace; + writeOp(out, remove_op); + writeTrailerLine(out, op_count); + return out.size(); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.h new file mode 100644 index 000000000000..34347fb97aaf --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefLogFormat.h @@ -0,0 +1,177 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Text codec for `cas_ref_log`, the immutable object stored at `_log/`. Each object contains +/// exactly one committed transaction: its namespace, transaction id, and the batch of `RefOp`s applied +/// by that commit. The body has a header, a meta line `{"ns","we","rs",["!pse","!pss"]}`, one JSON +/// record per op, and a `{"n":count}` trailer. Records are emitted in the transaction's stored order +/// and contain no codec-generated timestamps, so encoding the same value is byte-identical. This +/// determinism is a property of the representation, not an adoption gate: ref commits use +/// `putIfAbsentControlled`, and the caller applies the `Always`/`.zst` storage policy by sealing the +/// returned text. +/// +/// `RefOpKind::EpochSeal` closes an epoch transition in-band (spec INV-2): a seal transaction contains +/// exactly that one op, and the meta line's optional `prev_epoch_seal` (wire fields `!pse`/`!pss`, +/// CRITICAL -- an unrecognized `!`-key fails closed with `UNKNOWN_FORMAT_VERSION` rather than being +/// silently skipped, since dropping it would lose INV-2's chain evidence while still passing the +/// structural grammar) chains to the transaction id of the seal that closed the PRECEDING epoch, and +/// its own `writer_epoch` is exactly one below the referencing transaction's. `prev_epoch_seal` is +/// required on exactly sequence 1 of every epoch above the namespace's genesis (`life_epoch`, the +/// writer epoch of its `NamespaceBirth`) and forbidden elsewhere -- see +/// `validateEpochSealGrammarStructural` and `validateEpochSealGrammarContextual`. + +/// One operation kind in a ref transaction log. The numeric values are part of the in-memory and +/// serialized representation; unknown values are rejected by the decoder. +enum class RefOpKind : uint8_t +{ + NamespaceBirth = 1, + OwnerTransition = 2, + SetPublishedAt = 3, + RemoveNamespace = 4, + EpochSeal = 5, +}; + +/// One operation inside a `RefLogTxn`. Only the fields documented next to `kind` are meaningful for +/// that kind, and the codec never reads or writes the others. `OwnerTransition` optionally removes +/// `old_binding` and/or installs `new_binding`; `SetPublishedAt` carries the expected manifest and the +/// publication timestamp. `RefOwnerBinding` is shared with the snapshot format through +/// `CasRefWireVocab.h`. `NamespaceBirth`, `RemoveNamespace`, and `EpochSeal` carry none of `RefOp`'s +/// fields -- `EpochSeal`'s only payload, `prev_epoch_seal`, lives on the containing `RefLogTxn`, not +/// here (it is a property of the transaction's position in the log, not of the op). +struct RefOp +{ + RefOpKind kind = RefOpKind::NamespaceBirth; + + std::optional old_binding; /// OwnerTransition: absent = pure add + std::optional new_binding; /// OwnerTransition: absent = pure removal + + String ref_name; /// SetPublishedAt + ManifestRef expected_manifest_ref; /// SetPublishedAt + uint64_t published_at_ms = 0; /// SetPublishedAt + + bool operator==(const RefOp &) const = default; +}; + +/// The complete immutable body of one ref transaction log object. `ns` and `txn_id` are repeated in +/// the body even though both are key-derived. `decodeRefLogTxn` compares them with the values supplied +/// from the object key, rejecting a valid body copied under a different key as corruption. +struct RefLogTxn +{ + String ns; + RefTxnId txn_id; + std::vector ops; + /// The id of the `EpochSeal` transaction that closed the PRECEDING epoch (spec INV-2's genesis + /// contract): required iff `txn_id.ref_sequence == 1` and `txn_id.writer_epoch` is above the + /// namespace's `life_epoch` (its `NamespaceBirth` writer epoch), forbidden at every other + /// sequence. Populated by both the recovery-minted seal that closes an empty epoch and the first + /// ordinary transaction a writer appends after a transition -- see + /// `validateEpochSealGrammarStructural` / `validateEpochSealGrammarContextual`. Declared LAST (not + /// grouped with `txn_id`) to keep it a strict append to `RefLogTxn`'s field order. This project's + /// `-Wmissing-field-initializers`/`-Wmissing-designated-field-initializers` still require every + /// pre-existing positional `RefLogTxn{ns, txn_id, ops}` aggregate-init call site across the tree to + /// spell out an explicit trailing `std::nullopt` -- appending is still the smaller, purely + /// mechanical diff (one token per call site) compared to inserting a field in the middle. + std::optional prev_epoch_seal; + + bool operator==(const RefLogTxn &) const = default; +}; + +/// Hard limits enforced by the codec at both encode and decode, measured over the JSON text bytes. +/// Normal transactions have an operation-count limit, a byte limit, and a per-op size limit. A +/// transaction containing `RemoveNamespace` is "removal-class": it shares the larger complete-table +/// byte budget and has neither a separate operation-count cap nor a per-op cap, because that byte +/// budget alone bounds it. +inline constexpr size_t ref_txn_max_ops = 5000; +/// Admission is ops-only (no accumulated-size estimation); this stays as a decode-side acceptance +/// bound plus a post-encode writer assert. The canonical writer cannot reach it: at most +/// `ref_txn_max_ops` ops at `ref_op_max_bytes` each is well under this, with framing headroom to +/// spare. +inline constexpr size_t ref_txn_max_bytes = 20 * 1024 * 1024; +inline constexpr size_t ref_removal_max_bytes = 64 * 1024 * 1024; +/// Per-op size cap on normal-class ops, enforced exactly per op (one op encoded alone, no +/// accumulation) both at admission and at decode. Not unreachable: `checkCanonicalRefName` imposes +/// no length limit and ref/part names grow with partition-key values. Removal-class ops are exempt +/// (they share the byte budget above and have no per-op cap). +inline constexpr size_t ref_op_max_bytes = 4096; + +/// Encode the transaction to canonical, uncompressed text. The persist path seals this text according +/// to the `Always`/`.zst` policy; keeping the codec unsealed lets the byte-budget check and preview +/// callers measure the actual text, and lets the state machine validate an uncompressed round-trip. +/// Operations retain their stored order. Throws CORRUPTED_DATA on a zero `txn_id` field, a +/// non-canonical `ref_name`, an out-of-range `manifest_ref`, an unknown op kind, or an op-count/byte +/// limit violation over the encoded text. +String encodeRefLogTxn(const RefLogTxn & txn); + +/// Decode canonical text after the caller has opened the stored `.zst` object. `expected_ns` and +/// `expected_txn_id` come from the object key; the decoded body must equal them, otherwise the body/key +/// binding fails with CORRUPTED_DATA. Unknown non-critical fields are skipped so additive fields can be +/// introduced without changing this reader, while a future header version is rejected with +/// UNKNOWN_FORMAT_VERSION. Truncation, an unknown op or owner kind, a non-canonical `ref_name`, a zero +/// transaction-id field, a body/key mismatch, and a limit violation are reported as CORRUPTED_DATA. +RefLogTxn decodeRefLogTxn(std::string_view data, const String & expected_ns, const RefTxnId & expected_txn_id); + +/// Encoded byte size of exactly one exact-owner-removal op line: an `owner_transition` with only an old +/// binding, no new binding, as it appears in a hypothetical whole-namespace removal transaction (one +/// such op per committed ref and precommit, followed by a terminal `remove_namespace`), encoded via +/// `encodeRefLogTxn`. +size_t removalOpEncodedSize(RefOwnerKind owner_kind, const String & ref_name, const ManifestRef & manifest_ref); + +/// Encoded byte size of a removal transaction's framing (header + meta + terminal remove_namespace op + +/// trailer) for `op_count` total ops, excluding the per-owner removal op lines. `removalFramingSize(...) +/// + Σ removalOpEncodedSize` equals `encodeRefLogTxn(...)`'s size, for the hypothetical whole-namespace +/// removal transaction described above, exactly. +size_t removalFramingSize(const String & ns, const RefTxnId & txn_id, uint64_t op_count); + +/// Encoded byte size of exactly one op, on its own, as it appears inside a `RefLogTxn` body (the +/// record line only -- no header, meta, or trailer framing). Used to enforce the per-op size cap at +/// admission and at decode without ever accumulating a whole transaction's bytes. +size_t encodedOpSize(const RefOp & op); + +/// The one canonical removal-class discriminator: a transaction (or a not-yet-encoded item's built +/// ops) is removal-class iff `ops` contains a `RemoveNamespace` op. `MutationScope::Kind::WholeShard` +/// is NOT a substitute -- the stale-precommit reclaim sweep is also `WholeShard`-scoped but is not +/// removal-class. Every site that selects the removal byte budget (`ref_removal_max_bytes` vs +/// `ref_txn_max_bytes`) or exempts an item from the normal-class op/per-op caps must call this, not +/// re-derive its own predicate. +bool refLogTxnIsRemovalClass(const std::vector & ops); + +/// True iff `txn` is exactly a well-formed epoch-seal transaction: one operation, and that operation +/// is `EpochSeal`. Unlike `refLogTxnIsRemovalClass` (a class predicate that tolerates other ops +/// alongside `RemoveNamespace`), a seal transaction's grammar forbids any companion op (spec INV-2), +/// so this checks the exact shape rather than mere presence -- callers that decode untrusted bytes +/// and branch on "is this a seal" (Task 2's slot-occupy walk) get a precise answer. +bool refLogTxnIsEpochSeal(const RefLogTxn & txn); + +/// The context-free half of the INV-2 seal grammar: a transaction carrying an `EpochSeal` op contains +/// EXACTLY that one op, and `prev_epoch_seal`, when present, is well-formed (both `RefTxnId` +/// components nonzero), appears ONLY at `txn_id.ref_sequence == 1`, and names the IMMEDIATELY +/// preceding epoch (`prev_epoch_seal->writer_epoch + 1 == txn_id.writer_epoch`) -- a self, forward, +/// or skipping pointer is rejected even though it cannot arise from `writeLogMeta`'s own writer, +/// because Tasks 2/6 walk this pointer backwards over untrusted decoded bodies. Called by both +/// `encodeRefLogTxn` and +/// `decodeRefLogTxn`, so a caller-built transaction and a decoded one are held to the identical rule. +/// Throws CORRUPTED_DATA on violation. Does NOT check the required-iff rule -- that needs +/// `life_epoch`, which the codec never sees; see `validateEpochSealGrammarContextual`. +void validateEpochSealGrammarStructural(const RefLogTxn & txn); + +/// The contextual half of the INV-2 seal grammar: `prev_epoch_seal` is required on exactly +/// `txn_id.ref_sequence == 1` of every epoch with `txn_id.writer_epoch > life_epoch` (a namespace's +/// `life_epoch` is the writer epoch of its `NamespaceBirth` record -- its genesis, which may be above +/// 1 for a namespace born after earlier epoch transitions) and forbidden at +/// `txn_id.writer_epoch <= life_epoch`; a no-op for `txn_id.ref_sequence != 1` (the unconditional +/// "forbidden elsewhere" half of the rule is `validateEpochSealGrammarStructural`'s job, not this +/// function's). Callers own `life_epoch`: the apply layer and the writer-side encode call sites that +/// mint a sequence-1 transaction. Throws CORRUPTED_DATA on violation. +void validateEpochSealGrammarContextual(const RefLogTxn & txn, uint64_t life_epoch); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.cpp new file mode 100644 index 000000000000..f31e9fed4ca2 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.cpp @@ -0,0 +1,305 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +namespace +{ + +void checkCommittedSorted(const std::vector & rows) +{ + for (size_t i = 1; i < rows.size(); ++i) + if (!(rows[i - 1].ref_name < rows[i].ref_name)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableSnapshot: committed rows are not strictly ascending by ref_name at '{}' -> '{}'", + rows[i - 1].ref_name, rows[i].ref_name); +} + +void checkPrecommitsSorted(const std::vector & rows) +{ + for (size_t i = 1; i < rows.size(); ++i) + { + const auto prev_key = std::tie(rows[i - 1].ref_name, rows[i - 1].manifest_ref); + const auto cur_key = std::tie(rows[i].ref_name, rows[i].manifest_ref); + if (!(prev_key < cur_key)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableSnapshot: precommit rows are not strictly ascending by (ref_name, manifest_ref) at '{}' -> '{}'", + rows[i - 1].ref_name, rows[i].ref_name); + } +} + +/// Whole-object validation: transaction IDs must be nonzero and both row vectors must be strictly +/// sorted. Applying the same +/// checks before encoding and after decoding keeps malformed caller state and malformed stored data +/// subject to the same contract. +void checkSnapshotInvariants(const RefTableSnapshot & snapshot) +{ + checkRefTxnIdNonzero(snapshot.snapshot_id, "RefTableSnapshot", "snapshot_id"); + + checkCommittedSorted(snapshot.committed); + checkPrecommitsSorted(snapshot.precommits); +} + +void writeCommittedRow(CasJsonWriter & out, const RefCommittedRow & row) +{ + checkCanonicalRefName(row.ref_name, "RefTableSnapshot", "committed ref_name"); + checkManifestRef(row.manifest_ref, "RefTableSnapshot", "committed"); + bool first = true; + writeKey(out, "k", first); + writeStringValue(out, "c"); + writeKey(out, "rn", first); + writeStringValue(out, row.ref_name); + writeManifestRefFields(out, first, "", row.manifest_ref); + writeKey(out, "ts", first); + writeIntText(row.published_at_ms, out); + closeObject(out, first); + writeChar('\n', out); +} + +void writePrecommitRow(CasJsonWriter & out, const RefOwnerBinding & row) +{ + if (row.kind != RefOwnerKind::Precommit) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableSnapshot: precommits entry '{}' has kind {}, expected Precommit", + row.ref_name, static_cast(row.kind)); + checkCanonicalRefName(row.ref_name, "RefTableSnapshot", "precommit ref_name"); + checkManifestRef(row.manifest_ref, "RefTableSnapshot", "precommit"); + bool first = true; + writeKey(out, "k", first); + writeStringValue(out, "p"); + writeKey(out, "rn", first); + writeStringValue(out, row.ref_name); + writeManifestRefFields(out, first, "", row.manifest_ref); + closeObject(out, first); + writeChar('\n', out); +} + +/// The snapshot's header-object meta line (`ns`, `snapshot_id`, and the required generation-8 +/// `lc:"live"` constant). Shared by +/// `encodeRefTableSnapshot` and `snapshotFramingSize` so the two never disagree by a +/// byte. Assumes the caller has already validated the snapshot (or is measuring framing only). +void writeSnapshotMeta(CasJsonWriter & out, const RefTableSnapshot & snapshot) +{ + bool first = true; + writeKey(out, "ns", first); + writeStringValue(out, snapshot.ns); + writeRefTxnIdFields(out, first, "we", "rs", snapshot.snapshot_id); + writeKey(out, "lc", first); + writeStringValue(out, "live"); + closeObject(out, first); + writeChar('\n', out); +} + +/// Collector for a ManifestRef's three flat fields (bare "me"/"mb"/"mo"). +struct ManifestFields +{ + std::optional me; + std::optional mb; + std::optional mo; + + /// Reconstruct a manifest reference after the tolerant reader has collected all three flat + /// fields. Missing fields are malformed input; `manifestRefFromFields` performs the remaining + /// range checks and reports the same corruption context as the row decoder. + ManifestRef build(std::string_view what) const + { + if (!me || !mb || !mo) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableSnapshot: {} manifest_ref missing me/mb/mo", what); + return manifestRefFromFields(*me, *mb, *mo, "RefTableSnapshot", what); + } +}; + +} + +String encodeRefTableSnapshot(const RefTableSnapshot & snapshot) +{ + checkSnapshotInvariants(snapshot); + + CasJsonWriter out(256 + 128 * (snapshot.committed.size() + snapshot.precommits.size())); + writeHeaderLine(out, FormatId::RefSnapshot); + + writeSnapshotMeta(out, snapshot); + + for (const RefCommittedRow & row : snapshot.committed) + writeCommittedRow(out, row); + for (const RefOwnerBinding & row : snapshot.precommits) + writePrecommitRow(out, row); + + writeTrailerLine(out, snapshot.committed.size() + snapshot.precommits.size()); + String text = std::move(out).take(); + if (text.size() > ref_snapshot_max_bytes) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableSnapshot: encoded size {} exceeds the snapshot byte limit {}", text.size(), ref_snapshot_max_bytes); + return text; +} + +RefTableSnapshot decodeRefTableSnapshot( + std::string_view data, const String & expected_ns, const RefTxnId & expected_snapshot_id) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::RefSnapshot); + const uint64_t line_cap = traitsFor(FormatId::RefSnapshot).line_cap; + + RefTableSnapshot snapshot; + + { + const String line = readLine(in, line_cap, "cas_ref_snap"); + ReadBufferFromMemory meta_buf(line.data(), line.size()); + JsonObjectReader r(meta_buf, KeyStrictness::Tolerant, "cas_ref_snap"); + bool saw_ns = false; + bool saw_we = false; + bool saw_rs = false; + bool saw_lc = false; + String key; + while (r.nextKey(key)) + { + if (key == "ns") { snapshot.ns = r.readString(); saw_ns = true; } + else if (key == "we") { snapshot.snapshot_id.writer_epoch = r.readU64String(); saw_we = true; } + else if (key == "rs") { snapshot.snapshot_id.ref_sequence = r.readU64String(); saw_rs = true; } + else if (key == "lc") + { + const String lifecycle = r.readString(); + if (lifecycle != "live") + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableSnapshot: lifecycle must be exactly 'live', got '{}'", lifecycle); + saw_lc = true; + } + else if (key == "rte" || key == "rts") + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableSnapshot: meta carries retired terminal field '{}'", key); + else r.skipUnknown(key); + } + if (!saw_ns || !saw_we || !saw_rs || !saw_lc) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableSnapshot: meta line missing ns/we/rs/lc"); + if (!meta_buf.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableSnapshot: junk after meta line"); + } + + /// record lines (committed then precommit), until the trailer + while (true) + { + const String line = readLine(in, line_cap, "cas_ref_snap"); + ReadBufferFromMemory l(line.data(), line.size()); + JsonObjectReader r(l, KeyStrictness::Tolerant, "cas_ref_snap"); + String key; + if (!r.nextKey(key)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableSnapshot: empty line"); + + if (key == "n") + { + const uint64_t n = r.readU64Number(); + while (r.nextKey(key)) + r.skipUnknown(key); + if (!l.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableSnapshot: bytes after trailer"); + if (n != snapshot.committed.size() + snapshot.precommits.size()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableSnapshot: trailer count {} != {} rows", n, snapshot.committed.size() + snapshot.precommits.size()); + break; + } + if (key != "k") + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableSnapshot: record must start with \"k\""); + const String k = r.readString(); + + std::optional rn; + ManifestFields mf; + std::optional ts; + while (r.nextKey(key)) + { + if (key == "rn") rn = r.readString(); + else if (key == "me") mf.me = r.readU64String(); + else if (key == "mb") mf.mb = r.readU64String(); + else if (key == "mo") mf.mo = r.readU64Number(); + else if (key == "ts") ts = r.readU64Number(); + else if (key == "pl") + /// `"pl"` (payload) was removed from the row wire in stage-1 T12. It is a KNOWN-removed + /// field, not a genuinely-unknown future one the tolerant reader may skip -- silently + /// discarding a persisted payload would lose data -- so reject it explicitly. + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableSnapshot: record carries the removed \"pl\" (payload) field"); + else r.skipUnknown(key); + } + if (!l.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableSnapshot: junk after record"); + + if (k == "c") + { + if (!rn || !ts) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableSnapshot: committed row missing rn/ts"); + RefCommittedRow row; + row.ref_name = *rn; + checkCanonicalRefName(row.ref_name, "RefTableSnapshot", "committed ref_name"); + row.manifest_ref = mf.build("committed"); + row.published_at_ms = *ts; + snapshot.committed.push_back(std::move(row)); + } + else if (k == "p") + { + if (!rn) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableSnapshot: precommit row missing rn"); + RefOwnerBinding row; + row.kind = RefOwnerKind::Precommit; + row.ref_name = *rn; + checkCanonicalRefName(row.ref_name, "RefTableSnapshot", "precommit ref_name"); + row.manifest_ref = mf.build("precommit"); + snapshot.precommits.push_back(std::move(row)); + } + else + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableSnapshot: unknown row kind '{}'", k); + } + + /// The object key is supplied separately from the body. Check the binding before accepting any + /// decoded state so bytes stored under one namespace or snapshot ID cannot be interpreted as + /// another object. + if (snapshot.ns != expected_ns || snapshot.snapshot_id != expected_snapshot_id) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableSnapshot: body (ns='{}', snapshot_id={}-{}) does not match the key it was read from " + "(ns='{}', snapshot_id={}-{})", + snapshot.ns, snapshot.snapshot_id.writer_epoch, snapshot.snapshot_id.ref_sequence, + expected_ns, expected_snapshot_id.writer_epoch, expected_snapshot_id.ref_sequence); + + checkSnapshotInvariants(snapshot); + return snapshot; +} + +size_t committedRowEncodedSize(const RefCommittedRow & row) +{ + CasJsonWriter out(256); + writeCommittedRow(out, row); + return out.size(); +} + +size_t precommitRowEncodedSize(const RefOwnerBinding & binding) +{ + CasJsonWriter out(256); + writePrecommitRow(out, binding); + return out.size(); +} + +size_t snapshotFramingSize(const String & ns, const RefTxnId & snapshot_id, uint64_t row_count) +{ + RefTableSnapshot meta_only; + meta_only.ns = ns; + meta_only.snapshot_id = snapshot_id; + + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::RefSnapshot); + writeSnapshotMeta(out, meta_only); + writeTrailerLine(out, row_count); + return out.size(); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.h new file mode 100644 index 000000000000..07972a0d47ab --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefSnapshotFormat.h @@ -0,0 +1,95 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Text codec for `cas_ref_snap`, the complete per-namespace ref table snapshot at +/// `_snap/`. The object is read whole rather than streamed and belongs to the Control +/// family: callers store the encoded text as an Always/`.zst` object. Its canonical text consists of +/// a header, a metadata line, committed and precommit row lines, and a `{"n":count}` trailer. +/// +/// There is no such thing as a "seal snapshot". An epoch is closed IN-BAND, by an `EpochSeal` +/// transaction the recovery CAS-walk places at `{E, T+1}` in the `_log` stream (INV-2) -- the exact key +/// a dying predecessor's in-flight PUT would have taken, so the store's write-once create is the fence. +/// The retired alternative was a synthetic snapshot at `{E-1, UINT64_MAX}` carrying a `sealed_from` +/// bound: it occupied no log key, so it fenced nothing and needed a separate after-the-fact detector for +/// the writes it let through. All `RefTxnId` components are still encoded as decimal strings. Rows are +/// emitted in canonical order, making re-encoding deterministic by construction; these objects are +/// published through the ordinary single-owner `putIfAbsentControlled` path, not a +/// `putDeterministicArtifact` byte-adoption gate. + +/// In-memory ref-table lifecycle. Only `Live` is serializable as a generation-8 snapshot; terminal +/// state lives in the removal log and fold evidence and has no snapshot DTO representation. +enum class RefLifecycle : uint8_t +{ + Live = 1, + Removed = 2, +}; + +/// One committed ref-name-to-manifest row in a `RefTableSnapshot`. `published_at_ms` is the only +/// mutable metadata field a committed row carries. +struct RefCommittedRow +{ + String ref_name; + ManifestRef manifest_ref; + uint64_t published_at_ms = 0; + + bool operator==(const RefCommittedRow &) const = default; +}; + +/// The complete state of one namespace's ref table in one canonical snapshot object. `precommits` +/// reuses `RefOwnerBinding` from `CasRefWireVocab.h`; every entry's `kind` must be `Precommit`. +/// Generation 8 serializes only `Live` snapshots. Both row vectors must already be strictly sorted by +/// their documented keys, because the codec +/// validates and emits the caller-provided order rather than sorting it. +struct RefTableSnapshot +{ + String ns; + RefTxnId snapshot_id; + std::vector committed; /// sorted by canonical bytewise ref_name, no duplicates + std::vector precommits; /// sorted by (ref_name, manifest_ref), no duplicates + + bool operator==(const RefTableSnapshot &) const = default; +}; + +/// Hard encoded-size limit over the uncompressed text. The snapshot reuses the removal-class +/// complete-table budget from `CasRefLogFormat.h`. +inline constexpr size_t ref_snapshot_max_bytes = ref_removal_max_bytes; + +/// Encode to the canonical text (not sealed): the caller compresses via +/// `sealObject(FormatId::RefSnapshot, …)` on the persist path (Always/`.zst`), and the in-memory +/// validation and `admits` size-estimate callers use the uncompressed text. Throws +/// CORRUPTED_DATA on: a zero `snapshot_id`; a non-canonical `ref_name`; an +/// out-of-range `manifest_ref`; non-strictly-ascending +/// `committed` / `precommits`; a `precommits` entry not `Precommit`; or an over-budget object. +String encodeRefTableSnapshot(const RefTableSnapshot & snapshot); + +/// Decode the canonical text (the caller `openObject`s the stored `.zst` first). `expected_ns` / +/// `expected_snapshot_id` are recovered from the object key; the decoded body must equal them (the +/// key↔body binding). Throws UNKNOWN_FORMAT_VERSION for a header `v` above this build, CORRUPTED_DATA +/// for truncation, a missing/non-`live` lifecycle word, either retired `rte`/`rts` field, an unknown +/// owner kind, or any validation failure listed above. +RefTableSnapshot decodeRefTableSnapshot( + std::string_view data, const String & expected_ns, const RefTxnId & expected_snapshot_id); + +/// Encoded byte size of exactly one committed row line, as `encodeRefTableSnapshot` would emit it. +/// Reuses the same writer, so it is byte-identical to that row's contribution to a full encode. +size_t committedRowEncodedSize(const RefCommittedRow & row); + +/// Encoded byte size of exactly one precommit row line, as `encodeRefTableSnapshot` would emit it. +size_t precommitRowEncodedSize(const RefOwnerBinding & binding); + +/// Encoded byte size of a snapshot's framing (header + meta line + trailer) for the given metadata and +/// row count, excluding all row lines. `snapshotFramingSize(...) + Σ committedRowEncodedSize + +/// Σ precommitRowEncodedSize` equals `encodeRefTableSnapshot(...).size()` exactly. +size_t snapshotFramingSize(const String & ns, const RefTxnId & snapshot_id, uint64_t row_count); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefWireVocab.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefWireVocab.cpp new file mode 100644 index 000000000000..bf1a5445e4df --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefWireVocab.cpp @@ -0,0 +1,47 @@ +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +std::string_view refOwnerKindToWord(RefOwnerKind k) +{ + switch (k) + { + case RefOwnerKind::Committed: return "committed"; + case RefOwnerKind::Precommit: return "precommit"; + } + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS ref wire: unknown RefOwnerKind {}", static_cast(k)); +} + +RefOwnerKind refOwnerKindFromWord(std::string_view w, std::string_view what) +{ + if (w == "committed") return RefOwnerKind::Committed; + if (w == "precommit") return RefOwnerKind::Precommit; + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: unknown owner kind '{}'", what, w); +} + +void checkRefTxnIdNonzero(const RefTxnId & id, std::string_view format, std::string_view field) +{ + if (id.writer_epoch == 0 || id.ref_sequence == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "{}: {} fields must both be nonzero, got {}-{}", format, field, id.writer_epoch, id.ref_sequence); +} + +void writeRefTxnIdFields(CasJsonWriter & out, bool & first, std::string_view epoch_key, std::string_view seq_key, const RefTxnId & id) +{ + writeKey(out, epoch_key, first); + writeU64StringValue(out, id.writer_epoch); + writeKey(out, seq_key, first); + writeU64StringValue(out, id.ref_sequence); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefWireVocab.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefWireVocab.h new file mode 100644 index 000000000000..fdccef8f7fdb --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasRefWireVocab.h @@ -0,0 +1,65 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Shared identifier vocabulary for the ref-log and ref-snapshot text formats. `RefOwnerBinding` and +/// `RefOwnerKind` are used in log `OwnerTransition` records and snapshot `precommits`, so neither +/// format owns a duplicate definition. This header has no backend or storage dependencies; the +/// manifest-reference JSON helpers are shared by both ref formats and part-manifest descriptors +/// through `CasWireVocab.h`. + +/// Identifies which ownership slot a `RefOwnerBinding` occupies. The numeric values are the +/// in-memory discriminators; text codecs render them as the words "committed" and "precommit". +enum class RefOwnerKind : uint8_t +{ + Committed = 1, + Precommit = 2, +}; + +/// One ref-name-to-manifest ownership binding. Log transitions use it for the optional old and new +/// owners; snapshots use it for precommit rows, whose `kind` must be `Precommit`. A precommit's build +/// identity is the pair `{manifest_ref.writer_epoch, manifest_ref.build_sequence}`; the binding has +/// no additional build token. `manifest_ref` is validated by the format that decodes the binding. +struct RefOwnerBinding +{ + RefOwnerKind kind = RefOwnerKind::Committed; + String ref_name; + ManifestRef manifest_ref; + + bool operator==(const RefOwnerBinding &) const = default; +}; + +/// Convert an owner-kind discriminator to its canonical text word. Throws `CORRUPTED_DATA` for a +/// value not represented by this format; accepting an unknown value would produce an ambiguous wire +/// record. +std::string_view refOwnerKindToWord(RefOwnerKind k); + +/// Parse a canonical owner-kind word. `what` identifies the containing field in the +/// `CORRUPTED_DATA` exception. Unknown words are rejected rather than treated as a default kind. +RefOwnerKind refOwnerKindFromWord(std::string_view w, std::string_view what); + +/// Append two flat `RefTxnId` fields -- its `writer_epoch` and `ref_sequence` components -- to an +/// in-progress JSON object, both as decimal STRINGS -- the representation is width-independent, so no +/// consumer has to care how large a `ref_sequence` can get. `epoch_key`/`seq_key` name the two fields, +/// letting each format distinguish its primary id from any secondary id it embeds (for example, +/// `cas_ref_log`'s `we`/`rs` versus its `prev_epoch_seal` pair) while sharing one writer so the +/// formats can never disagree on the representation. +void writeRefTxnIdFields(CasJsonWriter & out, bool & first, std::string_view epoch_key, std::string_view seq_key, const RefTxnId & id); + +/// `RefTxnId`'s validity rule applied to ONE field of a decoded or about-to-be-encoded record: both +/// components nonzero. `renderRefTxnId` refuses to build a key from anything else, so a half-zero id +/// here would name an object that cannot exist -- `CORRUPTED_DATA`, in both directions. +/// +/// Shared for the same reason `writeRefTxnIdFields` is: ref formats can embed ids besides their +/// primary one (for example, `cas_ref_log`'s `prev_epoch_seal`), and a per-format copy of the rule is +/// a rule that can drift per format. `format` names the record kind and `field` the member, so each +/// format keeps its own exception text without owning its own check. +void checkRefTxnIdNonzero(const RefTxnId & id, std::string_view format, std::string_view field); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp new file mode 100644 index 000000000000..a12062fd21cb --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp @@ -0,0 +1,178 @@ +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +namespace +{ + +/// Read exactly the one JSON body line allowed by a server-root control object. `readLine` rejects a +/// missing newline and a line over the format-specific cap; each decoder separately checks that no +/// bytes follow this line, so a concatenated object cannot be accepted accidentally. +String readBodyLine(ReadBuffer & in, FormatId id, std::string_view what) +{ + return readLine(in, traitsFor(id).line_cap, what); +} + +} + +String encodeOwner(const OwnerObject & o) +{ + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::Owner); + bool first = true; + writeKey(out, "su", first); + writeHex128Value(out, o.server_uuid); + if (o.retired_at_ms) + { + writeKey(out, "rt", first); + writeIntText(*o.retired_at_ms, out); + } + closeObject(out, first); + writeChar('\n', out); + return std::move(out).take(); +} + +OwnerObject decodeOwner(std::string_view data) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::Owner); + const String body = readBodyLine(in, FormatId::Owner, "owner"); + ReadBufferFromMemory body_in(body.data(), body.size()); + JsonObjectReader r(body_in, KeyStrictness::Tolerant, "owner"); + + OwnerObject o; + bool saw = false; + std::optional rt; + String key; + while (r.nextKey(key)) + { + if (key == "su") + { + o.server_uuid = r.readHex128(); + saw = true; + } + else if (key == "rt") + rt = r.readU64Number(); + else + r.skipUnknown(key); + } + o.retired_at_ms = rt; + if (!saw) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS owner: missing su"); + if (!body_in.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS owner: trailing bytes"); + return o; +} + +String encodeServerEpoch(const ServerEpoch & e) +{ + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::ServerEpoch); + bool first = true; + writeKey(out, "nwe", first); + writeU64StringValue(out, e.next_writer_epoch); + closeObject(out, first); + writeChar('\n', out); + return std::move(out).take(); +} + +ServerEpoch decodeServerEpoch(std::string_view data) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::ServerEpoch); + const String body = readBodyLine(in, FormatId::ServerEpoch, "server-epoch"); + ReadBufferFromMemory body_in(body.data(), body.size()); + JsonObjectReader r(body_in, KeyStrictness::Tolerant, "server-epoch"); + + ServerEpoch e; + bool saw = false; + String key; + while (r.nextKey(key)) + { + if (key == "nwe") + { + e.next_writer_epoch = r.readU64String(); + saw = true; + } + else + r.skipUnknown(key); + } + if (!saw) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS server-epoch: missing nwe"); + if (!body_in.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS server-epoch: trailing bytes"); + return e; +} + +String encodeMountLease(const MountLease & m) +{ + CasJsonWriter out(256); + writeHeaderLine(out, FormatId::MountLease); + bool first = true; + writeKey(out, "su", first); writeHex128Value(out, m.server_uuid); + writeKey(out, "we", first); writeU64StringValue(out, m.writer_epoch); + writeKey(out, "hn", first); writeStringValue(out, m.hostname); + writeKey(out, "pid", first); writeIntText(m.pid, out); + writeKey(out, "sat", first); writeIntText(m.started_at_ms, out); + writeKey(out, "seq", first); writeU64StringValue(out, m.seq); + writeKey(out, "eat", first); writeIntText(m.expires_at_ms, out); + writeKey(out, "ma", first); writeU64StringValue(out, m.min_active); + writeKey(out, "fen", first); writeBoolValue(out, m.gc_fenced); + closeObject(out, first); + writeChar('\n', out); + return std::move(out).take(); +} + +MountLease decodeMountLease(std::string_view data) +{ + ReadBufferFromMemory in(data.data(), data.size()); + expectHeaderLine(in, FormatId::MountLease); + const String body = readBodyLine(in, FormatId::MountLease, "mount-lease"); + ReadBufferFromMemory body_in(body.data(), body.size()); + JsonObjectReader r(body_in, KeyStrictness::Tolerant, "mount-lease"); + + MountLease m; + bool saw_su = false; + bool saw_we = false; + String key; + while (r.nextKey(key)) + { + if (key == "su") + { + m.server_uuid = r.readHex128(); + saw_su = true; + } + else if (key == "we") + { + m.writer_epoch = r.readU64String(); + saw_we = true; + } + else if (key == "hn") m.hostname = r.readString(); + else if (key == "pid") m.pid = r.readU64Number(); + else if (key == "sat") m.started_at_ms = r.readU64Number(); + else if (key == "seq") m.seq = r.readU64String(); + else if (key == "eat") m.expires_at_ms = r.readU64Number(); + else if (key == "ma") m.min_active = r.readU64String(); + else if (key == "fen") m.gc_fenced = r.readBool(); + else r.skipUnknown(key); + } + if (!saw_su || !saw_we) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS mount-lease: missing identity field"); + if (!body_in.eof() || !in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS mount-lease: trailing bytes"); + return m; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.h new file mode 100644 index 000000000000..447827d70dde --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.h @@ -0,0 +1,93 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Text codecs for the three per-server-root control objects under +/// `gc/server-roots//`. Each object is a CAS text object consisting of a type/version +/// header line and one JSON body line; the codec layer only maps fields and checks the syntax that is +/// local to that object. Coordination and CAS operations remain in the pool/server-root layer. +/// +/// The owner object permanently binds a configured `server_root_id` to one server UUID. The epoch +/// object stores the next writer epoch and is CAS-bumped so an epoch is never reused after a +/// restart or supersession. The mount object is the expirable liveness lease for one writer +/// incarnation; its `min_active` value carries the GC acknowledgement floor, while `gc_fenced` is a +/// terminal fence-out marker for that incarnation. + +/// Permanent identity anchor for one configured server root. It is created with put-if-absent, never +/// reassigned, and tombstoned in place when explicitly decommissioned, so `decodeOwner` returns both +/// the UUID that the caller must compare with its local server identity and the retirement state. +struct OwnerObject +{ + UInt128 server_uuid{}; + /// Set when this identity was explicitly decommissioned by an operator (`CasDecommission`'s + /// final step tombstones the owner object in place rather than deleting it, since its content + /// alone cannot distinguish decommissioned debris from a legitimate successor's live anchor). + /// A normal claim (`claimOwnerOrThrow`) must refuse to silently resume a tombstoned identity. + std::optional retired_at_ms; +}; + +/// Durable counter state used to allocate unique writer epochs. The stored value is the next epoch +/// to allocate; the caller CAS-writes a larger value and adopts the previous value for its mount. +struct ServerEpoch +{ + uint64_t next_writer_epoch = 0; +}; + +/// The current liveness lease for one `(server_uuid, writer_epoch)` writer incarnation. The pool +/// layer renews and replaces this object with CAS/overwrite operations, and GC may fence an expired +/// lease by setting `gc_fenced`; a fenced incarnation must not resume writing. `min_active` is the +/// merged GC acknowledgement floor, with `UINT64_MAX` marking a clean farewell (retired lease). +struct MountLease +{ + UInt128 server_uuid{}; + uint64_t writer_epoch = 0; + String hostname; + uint64_t pid = 0; + uint64_t started_at_ms = 0; + uint64_t seq = 0; + uint64_t expires_at_ms = 0; + uint64_t min_active = 0; /// UINT64_MAX = retired (farewell) + bool gc_fenced = false; /// GC fence-out of an expired lease; terminal +}; + +/// Encode the owner anchor as canonical text with the `cas_owner` header and a final newline. The +/// optional retirement timestamp is omitted for a never-retired owner, preserving its historical +/// bytes. This function does not perform the put-if-absent operation or validate ownership; those +/// decisions belong to the caller that coordinates the server-root object. +String encodeOwner(const OwnerObject & o); + +/// Decode an owner anchor, requiring its `su` field, tolerating an absent optional `rt` retirement +/// timestamp, and rejecting bytes after the body line. Unknown JSON fields are skipped for +/// forward-compatible reads; malformed input, a missing `su`, and trailing data throw +/// `CORRUPTED_DATA`. +OwnerObject decodeOwner(std::string_view data); + +/// Encode the durable next-writer-epoch counter as canonical text with the `cas_epoch` header and a +/// final newline. Allocation and the CAS loop that makes the counter monotone are outside this +/// codec. +String encodeServerEpoch(const ServerEpoch & e); + +/// Decode the epoch counter, requiring its `nwe` field and rejecting bytes after the body line. +/// Unknown JSON fields are skipped for forward-compatible reads; malformed input, a missing `nwe`, +/// and trailing data throw `CORRUPTED_DATA`. +ServerEpoch decodeServerEpoch(std::string_view data); + +/// Encode the complete mount-lease body as canonical text with the `cas_mount_lease` header and a +/// final newline. This preserves full-range `uint64_t` values such as `min_active` as decimal JSON +/// strings and writes `gc_fenced` as a JSON boolean; lease renewal, fencing, and token checks remain +/// in the caller. +String encodeMountLease(const MountLease & m); + +/// Decode a mount lease and reject bytes after its body line. The reader accepts unknown JSON fields +/// so newer lease metadata can be read by older code, while malformed field values and trailing data +/// throw `CORRUPTED_DATA`; cross-field lease validity is enforced by the mount/GC protocol, not here. +MountLease decodeMountLease(std::string_view data); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.cpp new file mode 100644 index 000000000000..9814d5d14811 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.cpp @@ -0,0 +1,414 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int UNKNOWN_FORMAT_VERSION; + extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED; + extern const int CANNOT_PARSE_QUOTED_STRING; + extern const int CANNOT_PARSE_NUMBER; + extern const int CANNOT_READ_ALL_DATA; + extern const int ATTEMPT_TO_READ_AFTER_EOF; + extern const int INCORRECT_DATA; +} +} + +namespace DB::Cas +{ + +namespace +{ +const FormatSettings::JSON & jsonReadSettings() +{ + static const FormatSettings::JSON settings; + return settings; +} +} + +/// ---- CasJsonWriter ---- + +namespace +{ +constexpr bool isSpecialJsonByte(unsigned char c) +{ + return c < 0x20 || c == '"' || c == '\\' || c == 0xE2; +} +} + +const char * findNextSpecialJsonByte(const char * pos, const char * end) +{ + for (; pos != end; ++pos) + if (isSpecialJsonByte(static_cast(*pos))) + return pos; + return end; +} + +void CasJsonWriter::stringValue(std::string_view s) +{ + appendChar('"'); + const char * pos = s.data(); + const char * const end = s.data() + s.size(); + while (pos != end) + { + const char * next = findNextSpecialJsonByte(pos, end); + if (next != pos) + { + buf.append(pos, static_cast(next - pos)); + pos = next; + if (pos == end) + break; + } + const unsigned char c = static_cast(*pos); + switch (c) + { + case '\b': append("\\b"); ++pos; break; + case '\f': append("\\f"); ++pos; break; + case '\n': append("\\n"); ++pos; break; + case '\r': append("\\r"); ++pos; break; + case '\t': append("\\t"); ++pos; break; + case '\\': append("\\\\"); ++pos; break; + case '"': append("\\\""); ++pos; break; + case 0xE2: + if (end - pos >= 3 && pos[1] == '\x80' && (pos[2] == '\xA8' || pos[2] == '\xA9')) + { + append(pos[2] == '\xA8' ? std::string_view{"\\u2028"} : std::string_view{"\\u2029"}); + pos += 3; + } + else + { + appendChar('\xE2'); + ++pos; + } + break; + default: + { + /// A control byte without a named escape: \u00XY with writeJSONString's exact + /// nibble rendering (uppercase A-F for the low nibble). + const unsigned char lower_half = c & 0xF; + append("\\u00"); + appendChar(static_cast('0' + (c >> 4))); + appendChar(static_cast(lower_half <= 9 ? '0' + lower_half : 'A' + lower_half - 10)); + ++pos; + break; + } + } + } + appendChar('"'); +} + +/// ---- read-side pull cursor ---- + +/// A canonical-text parse failure is CORRUPTED_DATA regardless of which ReadHelpers primitive +/// noticed it first; the primitives themselves throw a handful of parse-specific codes (assertion +/// failure, quoted-string, number, EOF flavors). This is the single place that narrows all of them +/// down to the two codes this format speaks: CORRUPTED_DATA (translated here) and +/// UNKNOWN_FORMAT_VERSION (thrown deliberately by skipUnknown and passed through unchanged). +template +auto JsonObjectReader::guarded(F && f) +{ + try + { + return f(); + } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::CORRUPTED_DATA || e.code() == ErrorCodes::UNKNOWN_FORMAT_VERSION) + throw; + if (e.code() == ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED + || e.code() == ErrorCodes::CANNOT_PARSE_QUOTED_STRING + || e.code() == ErrorCodes::CANNOT_PARSE_NUMBER + || e.code() == ErrorCodes::CANNOT_READ_ALL_DATA + || e.code() == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF + || e.code() == ErrorCodes::INCORRECT_DATA) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: {}", what, e.message()); + throw; + } +} + +JsonObjectReader::JsonObjectReader(ReadBuffer & in_, KeyStrictness strictness_, std::string_view what_) + : in(in_), strictness(strictness_), what(what_) +{ + guarded([&] { assertChar('{', in); }); +} + +bool JsonObjectReader::nextKey(String & key) +{ + return guarded([&]() -> bool + { + if (done) + return false; + if (first) + { + first = false; + if (checkChar('}', in)) + { + done = true; + return false; + } + } + else + { + if (checkChar('}', in)) + { + done = true; + return false; + } + assertChar(',', in); + } + readJSONString(key, in, jsonReadSettings()); + assertChar(':', in); + if (std::find(seen_keys.begin(), seen_keys.end(), key) != seen_keys.end()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: duplicate key '{}'", what, key); + seen_keys.push_back(key); + return true; + }); +} + +String JsonObjectReader::readString() +{ + return guarded([&] + { + String s; + readJSONString(s, in, jsonReadSettings()); + return s; + }); +} + +UInt128 JsonObjectReader::readHex128() +{ + return guarded([&] + { + const String hex = readString(); + if (hex.size() != 32 + || std::any_of(hex.begin(), hex.end(), [](char c) { return unhex(c) == 0xff || (c >= 'A' && c <= 'F'); })) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: expected 32 lowercase hex chars, got '{}'", what, hex); + return unhexUInt(hex.data()); + }); +} + +uint64_t JsonObjectReader::readU64String() +{ + return guarded([&] + { + const String s = readString(); + ReadBufferFromMemory buf(s.data(), s.size()); + uint64_t v = 0; + readIntText(v, buf); + if (s.empty() || !buf.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: expected decimal u64 string, got '{}'", what, s); + return v; + }); +} + +uint64_t JsonObjectReader::readU64Number() +{ + return guarded([&] + { + uint64_t v = 0; + readIntText(v, in); + return v; + }); +} + +uint32_t JsonObjectReader::readU32Number() +{ + const uint64_t v = readU64Number(); + if (v > std::numeric_limits::max()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: value out of uint32 range", what); + return static_cast(v); +} + +bool JsonObjectReader::readBool() +{ + return guarded([&] + { + if (checkString("true", in)) + return true; + assertString("false", in); + return false; + }); +} + +void JsonObjectReader::skipUnknown(const String & key) +{ + guarded([&] + { + if (!key.empty() && key[0] == '!') + throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, + "CAS {}: critical key '{}' is not understood by this build", what, key); + if (strictness == KeyStrictness::Strict) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: unknown key '{}' in a strict format", what, key); + skipJSONField(in, key, jsonReadSettings()); + }); +} + +/// ---- header line / trailer line / raw line access ---- + +void writeHeaderLine(CasJsonWriter & out, FormatId id) +{ + const FormatTraits & t = traitsFor(id); + bool first = true; + writeKey(out, "type", first); + writeStringValue(out, t.type); + writeKey(out, "v", first); + writeIntText(currentCompatibilityVersion(), out); + closeObject(out, first); + writeChar('\n', out); +} + +void writeTrailerLine(CasJsonWriter & out, uint64_t n) +{ + bool first = true; + writeKey(out, "n", first); + writeIntText(n, out); + closeObject(out, first); + writeChar('\n', out); +} + +String readLine(ReadBuffer & in, uint64_t line_cap, std::string_view what) +{ + String line; + while (true) + { + if (in.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: truncated object (line without terminator)", what); + const char c = *in.position(); + ++in.position(); + if (c == '\n') + return line; + line.push_back(c); + if (line.size() > line_cap) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: line exceeds the {}-byte cap", what, line_cap); + } +} + +namespace +{ +TextHeader parseHeaderObject(std::string_view line, std::string_view what) +{ + ReadBufferFromMemory buf(line.data(), line.size()); + JsonObjectReader r(buf, KeyStrictness::Tolerant, what); + String key; + if (!r.nextKey(key) || key != "type") + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: header line must start with \"type\"", what); + TextHeader h; + h.type = r.readString(); + if (!r.nextKey(key) || key != "v") + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: header line must carry \"v\" second", what); + h.v = r.readU32Number(); + while (r.nextKey(key)) + r.skipUnknown(key); + if (!buf.eof()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: junk after the header object", what); + return h; +} +} + +TextHeader expectHeaderLine(ReadBuffer & in, FormatId id) +{ + const FormatTraits & t = traitsFor(id); + const String line = readLine(in, t.line_cap, t.type); + const TextHeader h = parseHeaderObject(line, t.type); + if (h.type != t.type) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: object is a '{}', not a '{}'", t.type, h.type, t.type); + checkCompatibility(h.v, t.type); + return h; +} + +std::optional sniffHeaderLine(std::string_view bytes) +{ + constexpr uint64_t kSniffLineCap = 64 * 1024; + try + { + ReadBufferFromMemory buf(bytes.data(), bytes.size()); + const String line = readLine(buf, kSniffLineCap, "sniff"); + TextHeader h = parseHeaderObject(line, "sniff"); + if (traitsForType(h.type) == nullptr) + return std::nullopt; + return h; + } + catch (const Exception &) + { + return std::nullopt; + } +} + +/// ---- the zstd arm ---- + +bool looksZstd(std::string_view bytes) +{ + static constexpr char kZstdFramePrefix[4] = {'\x28', '\xB5', '\x2F', '\xFD'}; + return bytes.size() >= 4 && memcmp(bytes.data(), kZstdFramePrefix, 4) == 0; +} + +namespace +{ +constexpr int kZstdLevel = 3; +} + +String sealObject(FormatId id, String text) +{ + const FormatTraits & t = traitsFor(id); + if (t.compression != CompressionPolicy::Always) + return text; + + ZSTD_CCtx * cctx = ZSTD_createCCtx(); + if (cctx == nullptr) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: cannot create zstd context", t.type); + SCOPE_EXIT({ ZSTD_freeCCtx(cctx); }); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, kZstdLevel); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); + + String out; + out.resize(ZSTD_compressBound(text.size())); + const size_t written = ZSTD_compress2(cctx, out.data(), out.size(), text.data(), text.size()); + if (ZSTD_isError(written)) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: zstd compression failed: {}", t.type, ZSTD_getErrorName(written)); + out.resize(written); + return out; +} + +String openObject(FormatId id, std::string_view stored) +{ + const FormatTraits & t = traitsFor(id); + if (!looksZstd(stored)) + { + if (t.object_cap != 0 && stored.size() > t.object_cap) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: raw object size {} exceeds the {}-byte cap", t.type, stored.size(), t.object_cap); + return String(stored); + } + if (t.compression != CompressionPolicy::Always) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: compressed object in a format whose policy is raw", t.type); + + const uint64_t content = ZSTD_getFrameContentSize(stored.data(), stored.size()); + if (content == ZSTD_CONTENTSIZE_UNKNOWN || content == ZSTD_CONTENTSIZE_ERROR) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: zstd frame without a declared content size", t.type); + if (t.object_cap != 0 && content > t.object_cap) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: declared decompressed size {} exceeds the {}-byte cap", t.type, content, t.object_cap); + + String out; + out.resize(content); + const size_t got = ZSTD_decompress(out.data(), out.size(), stored.data(), stored.size()); + if (ZSTD_isError(got) || got != content) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: zstd decompression failed: {}", + t.type, ZSTD_isError(got) ? ZSTD_getErrorName(got) : "short output"); + return out; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.h new file mode 100644 index 000000000000..50edd33bd4ba --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasTextFormat.h @@ -0,0 +1,241 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Shared container mechanics for versioned content-addressed text objects: a header line +/// {"type":"cas_","v":N}, a body, an optional trailer line, and, for formats whose registry +/// policy requires it, one zstd frame around the whole object. This is the only code that knows +/// that container shape; per-object codecs add only key mappings and object-specific invariants. +/// +/// Writers produce canonical text without whitespace outside JSON strings, and readers reject such +/// whitespace as `CORRUPTED_DATA`. Values that may span the full u64 range are decimal strings; +/// hashes are 32-character lowercase hexadecimal strings. The JSON writer settings are pinned in +/// the implementation so global `FormatSettings` changes cannot alter CAS bytes: slash-containing +/// ref paths and deterministic artifacts must retain the same representation and golden files. + +/// Bulk-append writer for canonical CAS JSON text. Replaces WriteBuffer in every CAS encode +/// path: appends are inline stores into an owned String (no per-call finalized/canceled +/// lifecycle, no per-byte writes, no heap allocations per record). Two usage modes: +/// whole-object assembly (bounded formats; `take` at the end) and line-scratch (RecordStream: +/// assemble one line, bulk-write it to the surrounding WriteBuffer, `clear` — memory stays +/// bounded by the largest line). The JSON escaping semantics of `stringValue` are statically +/// fixed to the CAS canon (forward slashes NOT escaped); process-wide FormatSettings cannot +/// influence CAS bytes. +class CasJsonWriter +{ +public: + explicit CasJsonWriter(size_t reserve_hint = 256) + { + buf.reserve(reserve_hint); + } + + void append(std::string_view s) + { + buf.append(s.data(), s.size()); + } + + void appendChar(char c) + { + buf.push_back(c); + } + + /// '{' on the first call, ',' after, then "name": . `name` must be plain ASCII (written raw). + void key(std::string_view name, bool & first) + { + appendChar(first ? '{' : ','); + first = false; + appendChar('"'); + append(name); + append("\":"); + } + + /// Same, for the prefixed key vocabulary ("o"/"n" + "me"/"mb"/"mo"/"bk"/"rn") — the + /// prefix and name are appended back to back, no composed temporary. + void key(std::string_view prefix, std::string_view name, bool & first) + { + appendChar(first ? '{' : ','); + first = false; + appendChar('"'); + append(prefix); + append(name); + append("\":"); + } + + /// Quoted JSON string with full escaping (bulk-run scan). Defined in CasTextFormat.cpp. + void stringValue(std::string_view s); + + void u64Number(uint64_t v) + { + char digits[24]; + char * end = itoa(v, digits); + buf.append(digits, static_cast(end - digits)); + } + + void u64StringValue(uint64_t v) + { + appendChar('"'); + u64Number(v); + appendChar('"'); + } + + void hex128Value(const UInt128 & v) + { + char hex[32]; + writeHexUIntLowercase(v, hex); + appendChar('"'); + buf.append(hex, sizeof(hex)); + appendChar('"'); + } + + void boolValue(bool v) + { + append(v ? std::string_view{"true"} : std::string_view{"false"}); + } + + void closeObject(bool & first) + { + if (first) + appendChar('{'); + first = false; + appendChar('}'); + } + + void newline() + { + appendChar('\n'); + } + + size_t size() const + { + return buf.size(); + } + + std::string_view view() const + { + return buf; + } + + void clear() + { + buf.clear(); + } + + String take() && + { + return std::move(buf); + } + +private: + String buf; +}; + +/// The write-side JSON primitives used by the format codecs. `CasJsonWriter` is the only CAS +/// text writer; every codec assembles its object in one before handing bytes to the underlying +/// `WriteBuffer`. +inline void writeKey(CasJsonWriter & out, std::string_view key, bool & first) { out.key(key, first); } +inline void writeStringValue(CasJsonWriter & out, std::string_view s) { out.stringValue(s); } +inline void writeHex128Value(CasJsonWriter & out, const UInt128 & v) { out.hex128Value(v); } +inline void writeU64StringValue(CasJsonWriter & out, uint64_t v) { out.u64StringValue(v); } +inline void writeBoolValue(CasJsonWriter & out, bool v) { out.boolValue(v); } +inline void closeObject(CasJsonWriter & out, bool & first) { out.closeObject(first); } +/// Argument order mirrors the IO helpers so migrated codecs keep their call shapes. +inline void writeChar(char c, CasJsonWriter & out) { out.appendChar(c); } +inline void writeIntText(uint64_t v, CasJsonWriter & out) { out.u64Number(v); } +void writeHeaderLine(CasJsonWriter & out, FormatId id); +void writeTrailerLine(CasJsonWriter & out, uint64_t n); + +/// Pull cursor over one canonical JSON object. +/// +/// The reader borrows the input buffer and records the object name for exception messages. It +/// enforces unique keys and translates the several low-level parser exceptions into the CAS +/// `CORRUPTED_DATA` contract. Unknown keys follow the supplied evolution policy: ordinary keys +/// may be skipped in tolerant objects, while `!`-prefixed keys always fail with +/// `UNKNOWN_FORMAT_VERSION`. + +class JsonObjectReader +{ +public: + /// Consumes the opening `{`; throws `CORRUPTED_DATA` when the object does not start there. + JsonObjectReader(ReadBuffer & in_, KeyStrictness strictness_, std::string_view what_); + /// Advances to the next key; false when the closing '}' was consumed. The caller must + /// consume the value (one read* / skipUnknown) before the next call. Duplicate keys are + /// rejected with `CORRUPTED_DATA`. + bool nextKey(String & key); + /// Reads the value for the key returned by `nextKey` as a JSON string. + String readString(); + /// Reads a quoted 32-character lowercase hexadecimal string as a `UInt128`. + UInt128 readHex128(); + /// Reads a quoted decimal u64 string and rejects empty, trailing, or non-decimal text. + uint64_t readU64String(); + /// Reads an unquoted JSON number into a u64; low-level parse failures become `CORRUPTED_DATA`. + uint64_t readU64Number(); + /// Reads an unquoted JSON number into a u32; rejects a value that would silently narrow. + uint32_t readU32Number(); + /// Reads the bare JSON literals `true` and `false`. + bool readBool(); + /// Applies the evolution rule for an unrecognized key: `!`-prefixed keys produce + /// `UNKNOWN_FORMAT_VERSION`; strict objects produce `CORRUPTED_DATA`; tolerant objects skip + /// the value. + void skipUnknown(const String & key); + +private: + /// Runs one parser operation under the CAS error taxonomy while preserving version exceptions. + template + auto guarded(F && f); + + ReadBuffer & in; + KeyStrictness strictness; + String what; + std::vector seen_keys; + bool first = true; + bool done = false; +}; + +/// Header, trailer, and raw-line access for the common text container. + +/// Header metadata returned after parsing a self-describing CAS object header line. +struct TextHeader +{ + String type; + uint32_t v = 0; +}; + +/// Reads and gates line 1 against `id`'s registered type; wrong type -> CORRUPTED_DATA; v above +/// what this build understands -> UNKNOWN_FORMAT_VERSION. +TextHeader expectHeaderLine(ReadBuffer & in, FormatId id); +/// Best-effort "is this a CAS object, and which one" for fsck/dispatch: swallows every failure and +/// returns nullopt. Never the load-bearing gate — that is expectHeaderLine. +std::optional sniffHeaderLine(std::string_view bytes); +/// Reads one line (excluding the '\n' terminator); CORRUPTED_DATA on missing terminator or a line +/// longer than `line_cap`. +String readLine(ReadBuffer & in, uint64_t line_cap, std::string_view what); + +/// Position of the next byte `stringValue` treats specially (control byte, '"', '\\', or the +/// 0xE2 lead byte of the U+2028/U+2029 lookahead), or `end`. Scalar bulk-run scan: the win is on +/// the short hot strings, so SIMD is deferred to a benchmark-gated contingency. +const char * findNextSpecialJsonByte(const char * pos, const char * end); + +/// Zstd detection and per-format compression policy. + +/// True iff `bytes` starts with the zstd frame prefix 28 B5 2F FD. +bool looksZstd(std::string_view bytes); +/// Compression per the per-type policy: `Always` -> one zstd frame (any size, checksum on); +/// everything else -> identity (returns `text` unchanged). +String sealObject(FormatId id, String text); +/// Inverse of sealObject. A compressed body is only legal when `id`'s policy is `Always` +/// (declared content size checked against the cap before allocation); a raw body is accepted +/// (repair path — e.g. an operator-restored uncompressed copy) subject to the SAME `object_cap` -- +/// skipping compression must never also skip the size cap. +String openObject(FormatId id, std::string_view stored); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasWireVocab.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasWireVocab.cpp new file mode 100644 index 000000000000..31d44f260121 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasWireVocab.cpp @@ -0,0 +1,103 @@ +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +std::string_view tokenTypeToWord(TokenType t) +{ + switch (t) + { + case TokenType::ETag: return "etag"; + case TokenType::Generation: return "generation"; + case TokenType::Emulated: return "emulated"; + } + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS wire: unknown TokenType {}", static_cast(t)); +} + +TokenType tokenTypeFromWord(std::string_view w, std::string_view what) +{ + if (w == "etag") return TokenType::ETag; + if (w == "generation") return TokenType::Generation; + if (w == "emulated") return TokenType::Emulated; + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: unknown token type '{}'", what, w); +} + +BlobHashAlgo blobHashAlgoFromWord(std::string_view w, std::string_view what) +{ + if (w == "ch128") return BlobHashAlgo::CityHash128; + if (w == "xxh3") return BlobHashAlgo::XXH3_128; + if (w == "sha256") return BlobHashAlgo::Sha256; + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: unknown blob hash algo '{}'", what, w); +} + +std::string_view objectKindToWord(ObjectKind k) +{ + switch (k) + { + case ObjectKind::Blob: return "blob"; + } + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS wire: unknown ObjectKind {}", static_cast(k)); +} + +ObjectKind objectKindFromWord(std::string_view w, std::string_view what) +{ + if (w == "blob") return ObjectKind::Blob; + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS {}: unknown object kind '{}'", what, w); +} + +void writeTokenFields(CasJsonWriter & out, bool & first, const Token & t) +{ + writeKey(out, "tt", first); + writeStringValue(out, tokenTypeToWord(t.type)); + writeKey(out, "tv", first); + writeStringValue(out, t.value); +} + +void writeBlobRefFields(CasJsonWriter & out, bool & first, const BlobRef & r) +{ + writeKey(out, "ha", first); + writeStringValue(out, blobHashAlgoName(r.algo)); + writeKey(out, "h", first); + writeStringValue(out, codecFor(r.algo).toHex(r.digest)); +} + +void writeManifestRefFields(CasJsonWriter & out, bool & first, std::string_view prefix, const ManifestRef & r) +{ + /// Unlike the WriteBuffer overload, the two-part key() form appends the prefix and name back + /// to back with no composed String(prefix) + "..." temporary. + out.key(prefix, "me", first); + out.u64StringValue(r.writer_epoch); + out.key(prefix, "mb", first); + out.u64StringValue(r.build_sequence); + out.key(prefix, "mo", first); + out.u64Number(r.manifest_ordinal); +} + +ManifestRef manifestRefFromFields(uint64_t writer_epoch, uint64_t build_sequence, uint64_t manifest_ordinal, + std::string_view caller, std::string_view what) +{ + /// Check the upper bound before narrowing the caller-supplied value to the in-memory ordinal + /// type. `checkManifestRef` then applies the shared nonzero and lower-bound checks. + if (manifest_ordinal > kMaxManifestOrdinal) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: {} manifest_ordinal {} out of range", caller, what, manifest_ordinal); + ManifestRef r; + r.writer_epoch = writer_epoch; + r.build_sequence = build_sequence; + r.manifest_ordinal = static_cast(manifest_ordinal); + checkManifestRef(r, caller, what); + return r; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasWireVocab.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasWireVocab.h new file mode 100644 index 000000000000..8727e6e219b6 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasWireVocab.h @@ -0,0 +1,62 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Shared JSON vocabulary for the value sub-types embedded by the CAS text codecs. These helpers +/// keep the same sub-object key names and full-word enum values across outcome logs, record streams, +/// ref logs, ref snapshots, part manifests, and blob envelopes. Every reverse map rejects an +/// unrecognized value with `CORRUPTED_DATA`; silently choosing a default would turn malformed +/// persisted data into a different valid-looking record. + +/// Convert a token discriminator to its canonical wire word. Throws `CORRUPTED_DATA` if `t` is not +/// one of the token types understood by this build. +std::string_view tokenTypeToWord(TokenType t); + +/// Parse a canonical token-type word. `what` identifies the containing codec or field in the +/// `CORRUPTED_DATA` exception; unknown words are rejected rather than treated as a default type. +TokenType tokenTypeFromWord(std::string_view w, std::string_view what); + +/// Parse a canonical blob-hash algorithm word. The write side uses `blobHashAlgoName` directly, so +/// this is its fail-closed inverse. `what` identifies the containing codec or field in the +/// `CORRUPTED_DATA` exception. +BlobHashAlgo blobHashAlgoFromWord(std::string_view w, std::string_view what); + +/// Convert an envelope object-kind discriminator to its canonical wire word. Throws +/// `CORRUPTED_DATA` if `k` is not represented by this format. +std::string_view objectKindToWord(ObjectKind k); + +/// Parse a canonical envelope object-kind word. `what` identifies the containing codec or field in +/// the `CORRUPTED_DATA` exception; unknown words are rejected rather than treated as a default kind. +ObjectKind objectKindFromWord(std::string_view w, std::string_view what); + +/// Append the sibling fields `tt` and `tv` to an in-progress JSON object. The caller owns `first`, +/// which must describe the fields already written to that object; the token value is JSON-escaped. +void writeTokenFields(CasJsonWriter & out, bool & first, const Token & t); + +/// Append the sibling fields `ha` and `h` to an in-progress JSON object. The algorithm word and +/// lowercase digest are canonical, and the digest is rendered at the width required by `r.algo`. +void writeBlobRefFields(CasJsonWriter & out, bool & first, const BlobRef & r); + +/// Append the three flat `ManifestRef` fields `me`, `mb`, and `mo` to an in-progress JSON object. +/// `prefix` is prepended to each key, allowing the ref codecs to distinguish old and new owner +/// bindings (`ome`/`omb`/`omo` and `nme`/`nmb`/`nmo`) while part manifests and ordinary rows use an +/// empty prefix. The two unbounded `uint64_t` values are decimal JSON strings; the bounded ordinal +/// is a JSON number. All consumers use this exact spelling and representation. +void writeManifestRefFields(CasJsonWriter & out, bool & first, std::string_view prefix, const ManifestRef & r); + +/// Construct a `ManifestRef` from decoded field values and validate the complete domain range: +/// nonzero `writer_epoch` and `build_sequence`, and `manifest_ordinal` in +/// `[1, kMaxManifestOrdinal]`. The upper bound is checked before narrowing to the in-memory +/// `uint32_t` ordinal. `caller` and `what` identify the codec and field in `CORRUPTED_DATA` +/// exceptions. +ManifestRef manifestRefFromFields(uint64_t writer_epoch, uint64_t build_sequence, uint64_t manifest_ordinal, + std::string_view caller, std::string_view what); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md new file mode 100644 index 000000000000..abd32c43b499 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md @@ -0,0 +1,63 @@ +# CAS persisted formats — the living registry + +Every persisted CAS object is a text file: header line `{"type":"cas_","v":N}`, body +(one JSON object / sorted NDJSON records / raw payload zone), optional `{"n":…}` trailer. +Can-grow-large types are stored under a **`.zst` key suffix** and are ALWAYS one zstd frame +(checksum on; declared content size checked against the cap before allocation); always-small and +deterministic types are raw. `CasTextFormat.{h,cpp}` is the only code that knows this shape. + +The object inventory is text end to end — there are no binary CAS formats and no protobuf +dependency. The GC source-edge data plane (`cas_run`) is sorted NDJSON written and read as a +stream (no seek); its integrity check is the whole-file seal checksum. The part manifest is the +one `PayloadHybrid` object: text header + descriptor meta + sorted entry records + `{"n":…}` +trailer, followed by a banner-framed raw payload zone for inline file bytes. + +**Rule:** any change to a persisted format lands in the SAME commit as its row here. + +## Bucket map + +| Key (under the pool prefix) | Object | Codec | Writer | +|---|---|---|---| +| `_pool_meta` | pool identity + floors | `CasPoolMetaFormat` | pool create/admit | +| `cas/ns/stream//_log/…​.zst` | ref transaction log | `CasRefLogFormat` (`.zst`) | writer commit path | +| `cas/ns/stream//_snap/…​.zst` | complete ref table | `CasRefSnapshotFormat` (`.zst`) | writer/GC fold | +| `cas/ns/state//_ckpt` | mutable life checkpoint | `CasRefCkptFormat` | writer/GC fold | +| `cas/ns/state//_files/…​` | namespace-owned raw files | — | upper layers | +| `cas/manifests//-/.zst` | part manifest | `CasPartManifestFormat` | part build | +| blob keys (`CasLayout::blobKey`) | blob envelope + payload | `CasBlobEnvelopeFormat` | uploads | +| blob-meta keys (`CasLayout::blobMetaKey`) | freshness sidecar | `CasBlobMetaFormat` | dedup/GC | +| `gc/state`, `gc/hb` | GC state / leader heartbeat | `CasGcStateFormat` | GC | +| `gc/maintenance_state` | leak-only namespace-janitor cursor | `CasGcMaintenanceStateFormat` | future janitor | +| `gc/gen//attempt//outcomes/…​.zst` | outcome log | `CasGcOutcomesFormat` (`.zst`) | GC | +| `gc/gen//attempt//fold_seal` | fold seal (deterministic) | `CasFoldSealFormat` | GC | +| `gc/gen//…​/runs` | GC source-edge record-stream runs | `CasRecordStreamFormat` | GC | +| `gc/server-roots//{owner,epoch,mount}` | server-root singletons | `CasServerRootFormats` | mount | +| `roots/…` | raw passthrough (verbatim) | — (never interpreted) | upper layers | + +## Codec table + +Authoritative per-format traits (type string, family, strictness, compression policy, caps) live +in `CasFormat.cpp` (`TRAITS`), asserted complete by `gtest_cas_text_format.cpp`. Key naming: keys +2–5 chars; fixed-width `UInt128` identities = 32-char lowercase hex strings; blob digests = +algo-width hex (two chars per digest byte), rendered with their algo name (`sha256:ab12…`) wherever +a bare hex would be ambiguous; unbounded u64 = decimal strings; bounded counts/lengths/ms-timestamps += numbers; units documented here per object as codecs land. + +## Evolution rules (one screen) + +- `v` (header line) is the ONLY version field; reader gate: `v > G_BUILD` → + `UNKNOWN_FORMAT_VERSION`, checked before the body. +- Additive change = new tolerant key, no `v` bump; on MUTABLE objects the field is best-effort + until the pool floor rises (an old writer's fresh re-encode drops it). +- Breaking change = `v` bump + `changePoints` + write-down-to-floor; the floor raise is what + fences old builds out (mount gates: `min_reader_generation` forward, pool-meta `v` backward). +- Deterministic formats (`cas_fold_seal`, `cas_run`): strict keys, pinned raw, and the adoption + pin — on a `putDeterministicArtifact` conflict, re-encode at the `v` of the EXISTING object. +- A key prefixed `!` is critical: a reader that does not understand it fails closed. +- Padding zones (blob header pad, manifest banners) are deterministic and verified — no + unaccounted bytes in any object. +- `openObject` policy asymmetry: a compressed body under a raw-compression policy is rejected + (`CORRUPTED_DATA`), but a NON-zstd body under an `Always` policy is passed through verbatim as + canonical text — an intentional uncompressed-repair path, not an "`Always` ⇒ must be compressed" + enforcement. In practice the mismatch never arises: `Always` objects are read via a constructed + `.zst`-suffixed key, so a raw body is not GETtable at that key. From ef2448c0336bd5432dde6fd145527fae25fb3e11 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:34 +0200 Subject: [PATCH 15/30] CAS subsystem: Backend layer Object-storage access layer: the backend interface, the object-storage adapter, the in-memory backend for tests, capability probing and request control. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../ContentAddressed/Backend/CasBackend.h | 438 ++++++ .../Backend/CasInMemoryBackend.cpp | 426 ++++++ .../Backend/CasInMemoryBackend.h | 172 +++ .../Backend/CasInstrumentedBackend.cpp | 176 +++ .../Backend/CasInstrumentedBackend.h | 202 +++ .../Backend/CasObjectStorageBackend.cpp | 1209 +++++++++++++++++ .../Backend/CasObjectStorageBackend.h | 249 ++++ .../ContentAddressed/Backend/CasProbe.cpp | 320 +++++ .../ContentAddressed/Backend/CasProbe.h | 62 + .../Backend/CasRequestControl.cpp | 745 ++++++++++ .../Backend/CasRequestControl.h | 621 +++++++++ .../Backend/CasSentinelProbe.cpp | 110 ++ .../Backend/CasSentinelProbe.h | 55 + 13 files changed, 4785 insertions(+) create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequestControl.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequestControl.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasSentinelProbe.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasSentinelProbe.h diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h new file mode 100644 index 000000000000..3b284a452b17 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h @@ -0,0 +1,438 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::Cas +{ + +/// User metadata carried alongside an object (S3 x-amz-meta-*). The CA store uses exactly one entry, +/// "cas_owner" = "::" — the owner triple the GC watermark reads. +using ObjectMeta = std::map; + +/// A byte window requested from an object. An absent length means that the window extends to EOF. +/// Backends use the same semantics for materialized and forward-only reads: the offset is exact, +/// while a backend may expose an advisory end when its underlying read buffer cannot enforce one. +struct Range +{ + uint64_t offset = 0; + std::optional length; /// nullopt => to the end + bool whole() const { return offset == 0 && !length; } +}; + +/// Materialized object bytes together with the incarnation and user metadata observed by the read. +/// The token identifies the exact object version whose bytes are in `bytes`; callers may use it to +/// validate a subsequent token-conditional mutation. +struct GetResult +{ + String bytes; + Token token; /// token of the incarnation the bytes came from + ObjectMeta attributes; +}; + +/// A forward-only read of a WRITE-ONCE object (runs, seals): nothing is materialized by the seam. +/// MUTABLE objects (root shards, gc/state, mounts) MUST keep using `get` — their bytes may change +/// under an open stream. `token` identifies the incarnation the stream reads, same as `get`. +struct GetStreamResult +{ + std::unique_ptr stream; + Token token; +}; + +/// Metadata returned by `Backend::head`. For an absent key, `exists` is false and the other fields +/// retain their defaults; for a present key, `size`, `token`, and `attributes` describe one current +/// incarnation as observed by the backend. +struct HeadResult +{ + bool exists = false; + uint64_t size = 0; + Token token; + ObjectMeta attributes; +}; + +/// Outcome of a write-once create or a token-conditional overwrite. A precondition failure means +/// that the backend preserved the existing object; it is an expected result, not an exception. +enum class PutOutcome : uint8_t +{ + Done, /// object written; the returned PutResult.token is the new incarnation's token + PreconditionFailed, /// If-None-Match hit an existing key / If-Match mismatched — nothing changed +}; + +/// Outcome of a compare-and-set write. `Conflict` means that the expected token (or expected +/// absence) did not match and that the backend left the object unchanged. +enum class CasOutcome : uint8_t +{ + Committed, + Conflict, /// expected token (or absence) did not match — nothing changed +}; + +/// Result of a backend write: the outcome plus the resulting object token (previously a `Token * out_token` +/// out-parameter). `token` is set ONLY when the write actually landed an incarnation (a `Done`/`Committed` +/// outcome); on `PreconditionFailed`/`Conflict` nothing was written and `token` is left default-constructed, +/// exactly mirroring the old contract where callers only read `*out_token` on success. +template +struct WriteResultT +{ + Outcome outcome; + Token token; +}; + +using PutResult = WriteResultT; +using CasResult = WriteResultT; + +/// Result of deleting one exact incarnation. `TokenMismatch` and `NotFound` are deliberately +/// distinct: the former proves that another incarnation is now current, while the latter means +/// there is no object to remove. `created_delete_marker` exposes a storage-versioning behavior +/// that is incompatible with current-object reclamation. +struct DeleteOutcome +{ + enum class Kind : uint8_t { Deleted, TokenMismatch, NotFound } kind = Kind::NotFound; + /// TRUE if the backend reported a delete marker was created because versioning is enabled. The + /// capability probe rejects this for the current-object storage model: exact deletion must reclaim + /// the current object rather than archive a noncurrent version. + bool created_delete_marker = false; +}; + +/// A key returned by `Backend::list`. The `token` field is populated ONLY when the backend +/// returns TRUE from `supportsListTokens` — it identifies the key's current incarnation, matching +/// what `head` would return for the same key at that instant. Callers that do not need the token +/// (e.g. GC fence sweep, orphan sweep) ignore the field; GC discover uses it to skip unchanged +/// root shards. +struct ListedKey +{ + String key; + uint64_t size = 0; + std::optional token; /// present iff supportsListTokens() == true +}; +/// One page returned by `Backend::list`. `keys` contains only the requested prefix and the cursor +/// resumes strictly after the last returned key; an empty cursor marks the end of the enumeration. +struct ListPage +{ + std::vector keys; + String next_cursor; /// Last returned key; empty => no more pages. +}; + +/// Typed erasure evidence for one key or one prefix. `head`/ +/// `get` deliberately flatten every kind of miss (a clean absence, a missing bucket/container, a +/// permission failure, a transport fault) into one "not found" result, which is exactly right for +/// their callers (a plain read) but wrong for lifecycle recovery, which must never treat a +/// transport/permission failure as proof that data is gone. `ProbeOutcome` keeps the four cases +/// distinct: only a backend's OWN authoritative "not found" evidence earns `KeyAbsent` — a timeout, +/// a 5xx, or an unclassifiable error is ALWAYS `Indeterminate`, never promoted to absence. +enum class ProbeOutcome : uint8_t +{ + Present, /// the key (or, for a prefix probe, at least one object under it) exists + KeyAbsent, /// authoritative miss: the container is alive, the key itself is not there + ContainerAbsent, /// the bucket/prefix-parent itself is gone, not merely the key + AccessDenied, /// the probe was rejected on permissions — absence was never established + Indeterminate, /// a transport/timeout/unclassifiable error — absence was NEVER proven +}; + +/// Result of `Backend::probeSentinelRaw`. `body` carries the materialized bytes only when the outcome is +/// `Present`. +struct SentinelProbeResult +{ + ProbeOutcome outcome; + std::optional body; +}; + +/// Streaming conditional create (If-None-Match:* semantics). The caller writes the FULL object body +/// (envelope header + payload) into buffer, then calls finalize exactly once: +/// - Done ⇒ the object is durable; the returned PutResult.token is the new incarnation's token +/// - PreconditionFailed ⇒ the key already existed — NOTHING was changed (same contract as putIfAbsent) +/// finalize may throw on storage errors; PreconditionFailed is an OUTCOME, never an exception. +/// cancel (or destruction before finalize) abandons the upload: the key is never created by it. +/// +/// MISUSE/LIFETIME CONTRACT: after finalize or cancel the sink is DEAD — any further finalize, +/// cancel, or write into buffer is a programming error (finalize asserts on it in debug builds). +/// The caller must not call the underlying buffer's own finalize/cancel directly — only through +/// the sink. A sink is single-caller: it is NOT thread-safe (only Backend itself is), and it must +/// not outlive the Backend that created it. +class WriteSink +{ +public: + virtual ~WriteSink() = default; + virtual WriteBuffer & buffer() = 0; + virtual PutResult finalize() = 0; + virtual void cancel() noexcept = 0; +}; + +using WriteSinkPtr = std::unique_ptr; + +/// Token-aware storage seam used by the content-addressed pool. TOKEN SEMANTICS ARE THE CONTRACT: +/// - every present key has exactly one current incarnation identified by an opaque Token; +/// - putOverwrite/casPut succeed only against the expected current token (or expected absence); +/// - deleteExact removes ONLY the incarnation whose token matches — wrong token MUST be a +/// TokenMismatch with the object untouched (backends that silently ignore the condition are +/// rejected by `Cas::Probe`); +/// - conditional PUTs are protocol hygiene; casPut and deleteExact are SAFETY-critical. +/// +/// TOKEN ⟹ CONTENT PRECONDITION (read-path caches depend on this): a token must uniquely identify +/// the byte-content of the incarnation it labels — i.e. `head(k).token == prior get(k).token` MUST +/// imply the bytes are unchanged. The protocol's SAFETY only needs the contrapositive (changed +/// bytes ⟹ a new token, so a stale CAS/delete is rejected), but `Cas::Pool`'s read-path decode +/// cache (`readShardDecoded`) skips a re-`get`+decode on a token match, so a backend whose token +/// could REPEAT across different content would make it serve stale manifests (wrong results). Holds +/// for every backend in use: S3 ETag is content-derived; the emulated/in-memory backends mint a +/// strictly-monotonic sequence that is never reused. A backend with a weak/recycled token must NOT +/// be used as a Cas pool. The capability probe currently verifies conditional-operation behavior but +/// does not test token non-reuse across different contents, so this invariant remains a requirement +/// of every backend implementation. +/// +/// Most ops take/return whole `String` bodies — sufficient for manifests, trees, and probe/GC +/// objects. LARGE content blobs stream through `putIfAbsentStream` (see `WriteSink`); reads stay +/// String-based because blob payload reads go through the wiring's read stack, not this seam. +class Backend +{ +public: + virtual ~Backend() = default; + + /// Reads the selected bytes and their token, or returns nullopt when the key is absent. For a + /// mutable object, callers must use this materialized form so the body is fixed before parsing. + virtual std::optional get(const String & key, Range range) = 0; + std::optional get(const String & key) { return get(key, {}); } + + /// Forward-only stream over the object's `range` (default: whole object) for WRITE-ONCE objects + /// (runs, seals). The returned `stream` yields exactly the window's bytes and nothing is + /// materialized whole by the seam — the caller reads at its own pace. MUTABLE objects (root + /// shards, gc/state, mounts) MUST keep using `get`: their bytes can change under an open stream. + /// CAVEAT: the window END is advisory on storages where `setReadUntilPosition` is a hint + /// (LocalObjectStorage) — the stream may yield bytes past the window; consumers MUST bound their + /// own consumption (RunFileReader bounds to its data_end). The window START is always exact. + virtual std::optional getStream(const String & key, Range range) = 0; + std::optional getStream(const String & key) { return getStream(key, {}); } + + /// Returns the current incarnation's existence, size, token, and metadata without reading its + /// body. The result describes one point-in-time observation; a later operation must use the + /// returned token when it needs to protect against replacement. + virtual HeadResult head(const String & key) = 0; + + /// Creates `key` only when it is absent. `PreconditionFailed` leaves the existing object intact; + /// storage failures are reported as exceptions. On success, the returned token identifies the + /// newly created incarnation. + virtual PutResult putIfAbsent(const String & key, const String & bytes, const ObjectMeta & meta) = 0; + PutResult putIfAbsent(const String & key, const String & bytes) { return putIfAbsent(key, bytes, {}); } + /// Streaming variant of putIfAbsent — see WriteSink. Large content blobs use this; whole-String + /// ops remain for manifests, trees, probe and GC objects. + virtual WriteSinkPtr putIfAbsentStream(const String & key, const ObjectMeta & meta) = 0; + WriteSinkPtr putIfAbsentStream(const String & key) { return putIfAbsentStream(key, {}); } + + /// Replaces the current object only when its token equals `expected`. A mismatch leaves the + /// object unchanged and returns `PreconditionFailed`; the returned token is meaningful only on + /// `Done`. + virtual PutResult putOverwrite(const String & key, const String & bytes, const Token & expected, + const ObjectMeta & meta) = 0; + PutResult putOverwrite(const String & key, const String & bytes, const Token & expected) + { + return putOverwrite(key, bytes, expected, {}); + } + /// expected == nullopt => create-if-absent CAS (the first write of a root manifest). + /// A non-null expected token conditionally replaces that exact current incarnation. Conflicts + /// leave the object unchanged and are returned as an outcome rather than an exception. + virtual CasResult casPut(const String & key, const String & bytes, const std::optional & expected, + const ObjectMeta & meta) = 0; + CasResult casPut(const String & key, const String & bytes, const std::optional & expected) + { + return casPut(key, bytes, expected, {}); + } + + /// Deletes only the current incarnation identified by `token`. A token mismatch must leave the + /// object untouched; the result distinguishes that case from an already absent key. + virtual DeleteOutcome deleteExact(const String & key, const Token & token) = 0; + + /// Lists one page of keys under `prefix`, starting after `cursor` and returning at most `limit` + /// entries. `ListPage::next_cursor` is the only supported continuation state. + virtual ListPage list(const String & prefix, const String & cursor, size_t limit) = 0; + + /// Capability fact about the LIST seam: TRUE iff this backend can surface a per-key incarnation + /// token through `list` (i.e. each `ListedKey` carries a token that uniquely identifies the + /// current incarnation of that key, matching what `head` would return). + /// + /// Why this matters: S3 ETags are content-derived and are returned in list responses; the + /// in-memory backend mints a monotonic token it can also surface through `list`. A backend that + /// cannot surface per-key tokens through `list` MUST return FALSE. + /// + /// FALSE ⇒ GC `discover` must read every root-shard body to learn the current token (fail closed). + /// TRUE ⇒ `discover` may skip an unchanged root-shard body read when the listed token equals + /// the persisted folded token, saving a GET per unchanged shard. + virtual bool supportsListTokens() const = 0; + + /// Pool-level preconditions beyond per-op conditional semantics — checked by the capability + /// probe BEFORE the op battery. Default: nothing to check. The S3 backend fails closed here + /// when a generation-dialect (GCS) bucket has object versioning enabled: every token-exact + /// DELETE would archive a noncurrent generation instead of reclaiming storage, so GC + /// "reclaim" would silently stop reclaiming. + virtual void checkPoolPreconditions() {} + + /// Fail-closed precondition: a Native-mode backend MUST have a + /// working single-attempt conditional-write path before it coordinates a WRITABLE pool — silently + /// running CAS conditional writes under the disk's default (~500-attempt) transparent retry policy + /// is exactly the hazard this seam forbids. Checked by the capability probe alongside + /// checkPoolPreconditions. Default: nothing to check (EmulatedSingleProcess and non-S3 backends + /// are not gated here — see ObjectStorageBackend's override for the one backend that is). + virtual void checkConditionalWriteSingleAttemptSupport() {} + + /// Authoritative, cache-bypassing probe of one key — see `ProbeOutcome`. DEFAULT (used by every + /// backend without sharper raw-error evidence, e.g. `InMemoryBackend`): derived from `head`/`get` + /// alone, so it can only distinguish `Present` from `KeyAbsent`, and ANY exception from either + /// call is `Indeterminate` — never promoted to `KeyAbsent`. A backend able to surface real + /// container/permission evidence (the S3-native and Local paths of `ObjectStorageBackend`) + /// overrides this to sharpen the classification. + virtual SentinelProbeResult probeSentinelRaw(const String & key) + { + try + { + const HeadResult hr = head(key); + if (!hr.exists) + return {ProbeOutcome::KeyAbsent, std::nullopt}; + auto g = get(key); + /// Vanished between head and get: still a clean, authoritative miss, not an error. + if (!g) + return {ProbeOutcome::KeyAbsent, std::nullopt}; + return {ProbeOutcome::Present, std::move(g->bytes)}; + } + catch (...) + { + return {ProbeOutcome::Indeterminate, std::nullopt}; + } + } + + /// WRITE-ONCE conditional SERVER-SIDE COPY of `staging_key` to `blob_key` (`If-None-Match:*` on the + /// destination) — the S3-native staging promote's create primitive. `Done` + `token` = the + /// destination ETag (the new incarnation token, exactly the role the + /// streaming `putIfAbsentStream` PUT's ETag plays) when this call created `blob_key`; + /// `PreconditionFailed` when `blob_key` already existed — NOTHING was changed (same write-once + /// contract as `putIfAbsentStream`). No LIVE object is ever overwritten by this call. + /// + /// DEFAULT: fail closed (`NOT_IMPLEMENTED`) — a backend without a native, enforced conditional + /// server-side copy is NEVER selected for S3 staging (the mount-time probe fell back to Local + /// staging + `putIfAbsentStream`), so this must throw rather than silently degrade to an + /// unconditional overwrite. The caller must use local staging when this primitive is unavailable. + virtual PutResult promoteStaged(const String & /*staging_key*/, const String & /*blob_key*/) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Cas::Backend::promoteStaged (write-once server-side copy) is not implemented for this backend"); + } + + /// UNCONDITIONAL re-upload of the writer's OWN payload over `blob_key` under a FRESH-tagged + /// envelope header — the sanctioned condemned-object resurrection overwrite. Writes + /// `[fresh_header][payload]`, streaming `payload` from `reader`, and returns the fresh incarnation's + /// token. Blob bodies have no size cap, so a Native backend streams the payload and never + /// materializes it; the emulated backend materializes (its conditional ops are whole-`String` by + /// design) and serializes resurrections to bound the peak to one body at a time. + /// + /// The reader is the caller's: it is ALWAYS the writer's own source (a staging object or a local + /// staged file), NEVER a read of the condemned `blob_key`, and the caller has already skipped any + /// envelope header on it. `fresh_header` must carry a freshly-minted `incarnation_tag`, which is what + /// makes the resurrected body — and hence its ETag/token — differ from the condemned incarnation, so + /// a queued exact-token delete of that incarnation can never match the live resurrection + /// (`INV-NO-RETURN`). + /// + /// UNCONDITIONAL is deliberate. An `If-Match` on the condemned token would save a redundant + /// re-upload when another writer resurrects the same blob first, and would prevent nothing: two + /// racing resurrections write payload-identical bodies, no consumer reads a dep token's VALUE, and + /// durable references name content hashes rather than incarnations. + /// + /// The caller MUST have observed the current incarnation as `Condemned` (per-hash meta point-read) + /// before calling this. That observation is NOT re-checked at the write: two racing resurrections + /// of the same blob may both run, and the loser overwrites the winner's FRESH incarnation. That is + /// accepted, not prevented -- the payloads are content-identical by construction and durable + /// references name content hashes, so the overwrite rotates the envelope and token of an + /// equivalent body. What must never be overwritten is a live incarnation of DIFFERENT content, + /// and that is guaranteed by the content address itself, not by this call. + /// DEFAULT: fail closed (`NOT_IMPLEMENTED`), same rationale as `promoteStaged`. + /// `payload_size` is the payload byte count the caller verified at staging time. The write COUNTS + /// while streaming and MUST abort -- publishing nothing -- when the reader yields a different + /// number of bytes. With an unconditional write this is the last line of defence: a source + /// truncated after hashing would otherwise displace the condemned incarnation with a short body + /// that the content address does not match, and a post-write check can only detect it AFTER the + /// malformed incarnation became current (and can even inspect a racing writer's incarnation + /// instead of its own). + virtual Token resurrect(ReadBuffer & /*payload*/, uint64_t /*payload_size*/, const String & /*blob_key*/, + const String & /*fresh_header*/) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Cas::Backend::resurrect is not implemented for this backend"); + } +}; + +using BackendPtr = std::shared_ptr; + +/// Walk every key under `prefix` exactly once, resuming by the backend's explicit last-returned-key +/// cursor (`ListPage::next_cursor`, empty => done). This centralizes the pagination contract shared by +/// GC, fsck, and cleanup sweeps: each returned key is delivered once, and the backend's cursor is the +/// only state used to request the next page. +/// +/// `on_page_fetched`, if set, fires exactly once per physical `backend.list` call (including an +/// empty/undersized final page) — a GC-owned caller's hook for a page-level ProfileEvents counter, +/// without misattributing a non-GC caller (e.g. fsck) that leaves it unset. Trails `page_limit` +/// (rather than sitting before it) so the two existing callers that override `page_limit` +/// (`Gc::fold`, `CasFsck.cpp`'s `listAll`) can override `page_limit` without changing callback order. +inline void forEachListedKey(Backend & backend, const String & prefix, + const std::function & cb, + size_t page_limit = 1000, + const std::function & on_page_fetched = {}) +{ + String cursor; + for (;;) + { + const ListPage page = backend.list(prefix, cursor, page_limit); + if (on_page_fetched) + on_page_fetched(); + for (const ListedKey & k : page.keys) + cb(k); + if (page.next_cursor.empty()) + break; + cursor = page.next_cursor; + } +} + +/// The normalized verdict of a token-exact delete, unifying the DeleteOutcome::Kind three-way that GC +/// (blob + manifest delete) and the orphan-manifest sweep each mapped by hand. +enum class DeleteClass : uint8_t { Deleted, Absent, Replaced }; + +/// Converts a backend-specific delete outcome into the three states used by cleanup callers. The +/// default branch is fail-safe: an unknown value is treated as `Replaced`, so cleanup never reports +/// an unverified deletion as successful. +inline DeleteClass classifyDeleteOutcome(const DeleteOutcome & d) +{ + switch (d.kind) + { + case DeleteOutcome::Kind::Deleted: return DeleteClass::Deleted; + case DeleteOutcome::Kind::NotFound: return DeleteClass::Absent; + case DeleteOutcome::Kind::TokenMismatch: return DeleteClass::Replaced; + } + return DeleteClass::Replaced; /// unreachable; fail-safe toward "leave it" (never a false Deleted) +} + +/// Returns the stable lowercase label used when reporting a normalized delete result. Unknown enum +/// values are labeled `replaced`, matching `classifyDeleteOutcome`'s fail-safe behavior. +inline std::string_view deleteClassName(DeleteClass c) +{ + switch (c) + { + case DeleteClass::Deleted: return "deleted"; + case DeleteClass::Absent: return "absent"; + case DeleteClass::Replaced: return "replaced"; + } + return "replaced"; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.cpp new file mode 100644 index 000000000000..d8eca3d876d0 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.cpp @@ -0,0 +1,426 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int FILE_DOESNT_EXIST; +} +} + +namespace DB::Cas +{ + +namespace +{ + +/// Memory-buffered WriteSink: accumulates the body in a WriteBufferFromOwnString and delegates the +/// conditional publish to InMemoryBackend::putIfAbsent at finalize — the single mutex acquisition +/// inside putIfAbsent gives atomicity for free. Nothing is ever published on cancel/destruction. +class InMemoryWriteSink final : public WriteSink +{ +public: + InMemoryWriteSink(InMemoryBackend & backend, String key, ObjectMeta meta) + : backend_(backend) + , key_(std::move(key)) + , meta_(std::move(meta)) + { + } + + WriteBuffer & buffer() override { return buf_; } + + PutResult finalize() override + { + chassert(!done_); /// finalize after finalize/cancel is a misuse — see the WriteSink contract + done_ = true; + return backend_.putIfAbsent(key_, buf_.str(), meta_); + } + + void cancel() noexcept override + { + done_ = true; + buf_.cancel(); + } + + ~InMemoryWriteSink() override + { + if (!done_) + cancel(); + } + +private: + InMemoryBackend & backend_; + String key_; + ObjectMeta meta_; + WriteBufferFromOwnString buf_; + bool done_ = false; +}; + +/// The windowed slice of `data` for `range`, with the same clamping `get` documents: an offset at or +/// past EOF yields an empty result; an open-ended length runs to EOF. Shared by `get` and `getStream` +/// so the two stay in lockstep. +String sliceWindow(const String & data, Range range) +{ + const size_t offset = static_cast(range.offset); + if (offset >= data.size()) + return {}; + if (range.length.has_value()) + return data.substr(offset, static_cast(*range.length)); + return data.substr(offset); +} + +} + +Token InMemoryBackend::mintToken() +{ + Token t; + t.value = std::to_string(++token_seq_); + t.type = TokenType::Emulated; + return t; +} + +std::optional InMemoryBackend::get(const String & key, Range range) +{ + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it == store_.end()) + return std::nullopt; + + GetResult gr; + gr.bytes = sliceWindow(it->second.bytes, range); + gr.token = it->second.token; + gr.attributes = it->second.meta; + return gr; +} + +std::optional InMemoryBackend::getStream(const String & key, Range range) +{ + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it == store_.end()) + return std::nullopt; + + /// Copy the windowed bytes into an owning buffer — the in-memory backend has no separate storage + /// to stream from, so the "stream" reads from a private copy of exactly the requested window. + GetStreamResult sr; + sr.stream = std::make_unique(sliceWindow(it->second.bytes, range)); + sr.token = it->second.token; + return sr; +} + +HeadResult InMemoryBackend::head(const String & key) +{ + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it == store_.end()) + return HeadResult{}; + + HeadResult hr; + hr.exists = true; + hr.size = static_cast(it->second.bytes.size()); + hr.token = it->second.token; + hr.attributes = it->second.meta; + return hr; +} + +PutResult InMemoryBackend::putIfAbsent(const String & key, const String & bytes, const ObjectMeta & meta) +{ + std::lock_guard lock(mutex_); + + // One-shot injected ambiguous outcome: throw WITHOUT touching the store, modeling a request whose + // own attempt outcome never reached the caller (see the header doc for the classification this + // must produce). std::runtime_error, not DB::Exception, is deliberate: it dodges BOTH + // classification paths in BOTH build configurations -- dynamic_cast fails (so + // isDeterministicLocalFailure is never consulted), and classifyConditionalWriteResult falls through + // to its Unresolved default because it isn't an S3Exception. A DB::Exception would have been + // fragile: picking a code outside isDeterministicLocalFailure's set is a landmine for the next + // person who extends that set. + auto ambiguous_it = ambiguous_put_keys_.find(key); + if (ambiguous_it != ambiguous_put_keys_.end()) + { + ambiguous_put_keys_.erase(ambiguous_it); + throw std::runtime_error("InMemoryBackend: injected ambiguous putIfAbsent outcome for '" + key + "'"); + } + + if (store_.contains(key)) + return {PutOutcome::PreconditionFailed, {}}; + + Token t = mintToken(); + Object obj; + obj.bytes = bytes; + obj.token = t; + obj.meta = meta; + store_[key] = std::move(obj); + return {PutOutcome::Done, t}; +} + +WriteSinkPtr InMemoryBackend::putIfAbsentStream(const String & key, const ObjectMeta & meta) +{ + return std::make_unique(*this, key, meta); +} + +PutResult InMemoryBackend::putOverwrite(const String & key, const String & bytes, const Token & expected, const ObjectMeta & meta) +{ + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it == store_.end()) + return {PutOutcome::PreconditionFailed, {}}; + + if (enforce_tokens_ && it->second.token != expected) + return {PutOutcome::PreconditionFailed, {}}; + + Token t = mintToken(); + it->second.bytes = bytes; + it->second.token = t; + it->second.meta = meta; + return {PutOutcome::Done, t}; +} + +CasResult InMemoryBackend::casPut(const String & key, const String & bytes, const std::optional & expected, const ObjectMeta & meta) +{ + std::lock_guard lock(mutex_); + + // One-shot injected conflict + auto fail_it = fail_next_cas_.find(key); + if (fail_it != fail_next_cas_.end()) + { + fail_next_cas_.erase(fail_it); + return {CasOutcome::Conflict, {}}; + } + + auto it = store_.find(key); + bool exists = (it != store_.end()); + + if (!expected.has_value()) + { + // create-if-absent CAS + if (exists) + return {CasOutcome::Conflict, {}}; + Token t = mintToken(); + Object obj; + obj.bytes = bytes; + obj.token = t; + obj.meta = meta; + store_[key] = std::move(obj); + return {CasOutcome::Committed, t}; + } + else + { + // swap-if-current CAS + if (!exists) + return {CasOutcome::Conflict, {}}; + if (enforce_tokens_ && it->second.token != *expected) + return {CasOutcome::Conflict, {}}; + Token t = mintToken(); + it->second.bytes = bytes; + it->second.token = t; + it->second.meta = meta; + return {CasOutcome::Committed, t}; + } +} + +DeleteOutcome InMemoryBackend::applyDelete(const String & key, const Token & token) +{ + // Caller holds the mutex. + auto it = store_.find(key); + if (it == store_.end()) + { + DeleteOutcome d; + d.kind = DeleteOutcome::Kind::NotFound; + return d; + } + + if (enforce_tokens_ && it->second.token != token) + { + DeleteOutcome d; + d.kind = DeleteOutcome::Kind::TokenMismatch; + return d; + } + + store_.erase(it); + DeleteOutcome d; + d.kind = DeleteOutcome::Kind::Deleted; + d.created_delete_marker = simulate_delete_markers_; + return d; +} + +DeleteOutcome InMemoryBackend::deleteExact(const String & key, const Token & token) +{ + std::lock_guard lock(mutex_); + + if (hold_deletes_) + { + // Validate the key exists (and token matches if enforcing) before queuing, + // but don't remove yet — just enqueue. + auto it = store_.find(key); + if (it == store_.end()) + { + DeleteOutcome d; + d.kind = DeleteOutcome::Kind::NotFound; + return d; + } + if (enforce_tokens_ && it->second.token != token) + { + DeleteOutcome d; + d.kind = DeleteOutcome::Kind::TokenMismatch; + return d; + } + PendingDelete pd; + pd.key = key; + pd.token = token; + pending_deletes_.push_back(std::move(pd)); + DeleteOutcome d; + d.kind = DeleteOutcome::Kind::Deleted; + d.created_delete_marker = simulate_delete_markers_; + return d; + } + + return applyDelete(key, token); +} + +PutResult InMemoryBackend::promoteStaged(const String & staging_key, const String & blob_key) +{ + std::lock_guard lock(mutex_); + auto src = store_.find(staging_key); + if (src == store_.end()) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "InMemoryBackend::promoteStaged: staging object {} is absent", staging_key); + + /// Write-once: a present destination is the "lost the race" signal, not an overwrite. + if (store_.contains(blob_key)) + return {PutOutcome::PreconditionFailed, {}}; + + /// Server-side copy: the destination bytes ARE the staging bytes; a fresh monotone token stands in + /// for the destination ETag the real backend returns from the conditional copy. + const Token t = mintToken(); + Object obj; + obj.bytes = src->second.bytes; + obj.token = t; + store_[blob_key] = std::move(obj); + return {PutOutcome::Done, t}; +} + +Token InMemoryBackend::resurrect(ReadBuffer & payload, uint64_t payload_size, const String & blob_key, + const String & fresh_header) +{ + /// Drain the reader BEFORE taking the lock: the reader may be backed by another object in this + /// same store, and reading it under our own mutex would deadlock. + String body = fresh_header; + { + WriteBufferFromString out(body, AppendModeTag{}); + copyData(payload, out); + out.finalize(); + } + + if (body.size() - fresh_header.size() != payload_size) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "InMemoryBackend::resurrect: source yielded {} payload bytes for {}, declared {} -- nothing was published", + body.size() - fresh_header.size(), blob_key, payload_size); + + /// The fresh header makes the resurrected body differ from the condemned incarnation for the same + /// payload, so a delayed exact-token delete for the old incarnation cannot remove the resurrection + /// (`INV-NO-RETURN`). The condemned `blob_key` is never read. + std::lock_guard lock(mutex_); + const Token t = mintToken(); + Object obj; + obj.bytes = std::move(body); + obj.token = t; + store_[blob_key] = std::move(obj); + return t; +} + +ListPage InMemoryBackend::list(const String & prefix, const String & cursor, size_t limit) +{ + if (limit == 0) + return {}; + + std::lock_guard lock(mutex_); + ListPage page; + + // Cursor is the last key returned by the previous page. + auto it = cursor.empty() ? store_.lower_bound(prefix) : store_.upper_bound(cursor); + + size_t count = 0; + while (it != store_.end() && count < limit) + { + if (!it->first.starts_with(prefix)) + break; + + ListedKey lk; + lk.key = it->first; + lk.size = static_cast(it->second.bytes.size()); + lk.token = it->second.token; /// in-memory backend always surfaces the token (supportsListTokens == true) + page.keys.push_back(std::move(lk)); + ++count; + ++it; + } + + // Set next_cursor if there are more keys in this prefix + if (!page.keys.empty() && it != store_.end() && it->first.starts_with(prefix)) + page.next_cursor = page.keys.back().key; + + return page; +} + +void InMemoryBackend::setHoldDeletes(bool hold) +{ + std::lock_guard lock(mutex_); + hold_deletes_ = hold; +} + +size_t InMemoryBackend::pendingDeletes() const +{ + std::lock_guard lock(mutex_); + return pending_deletes_.size(); +} + +DeleteOutcome InMemoryBackend::landPendingDelete(size_t i) +{ + std::lock_guard lock(mutex_); + if (i >= pending_deletes_.size()) + { + DeleteOutcome d; + d.kind = DeleteOutcome::Kind::NotFound; + return d; + } + + PendingDelete pd = pending_deletes_[i]; + pending_deletes_.erase(pending_deletes_.begin() + static_cast(i)); + + // Apply the token check at LAND time — the object may have been modified since the delete was enqueued. + return applyDelete(pd.key, pd.token); +} + +void InMemoryBackend::failNextCasPut(const String & key) +{ + std::lock_guard lock(mutex_); + fail_next_cas_.insert(key); +} + +void InMemoryBackend::injectAmbiguousPutIfAbsent(const String & key) +{ + std::lock_guard lock(mutex_); + ambiguous_put_keys_.insert(key); +} + +void InMemoryBackend::setEnforceTokens(bool enforce) +{ + std::lock_guard lock(mutex_); + enforce_tokens_ = enforce; +} + +void InMemoryBackend::setSimulateDeleteMarkers(bool simulate) +{ + std::lock_guard lock(mutex_); + simulate_delete_markers_ = simulate; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h new file mode 100644 index 000000000000..5c5d5e906e57 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h @@ -0,0 +1,172 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Thread-safe, token-enforcing in-memory `Backend` implementation used by CAS tests. +/// +/// All successful writes mint a monotonically increasing token (`TokenType::Emulated`). +/// Tokens NEVER repeat across the lifetime of a backend instance. +/// +/// The backend also exposes fault-injection controls for probe tests and CAS correctness tests: +/// - `setHoldDeletes` / `landPendingDelete`: simulate async/delayed conditional deletes +/// - `failNextCasPut`: inject a one-shot conflict +/// - `setEnforceTokens(false)`: mimic a "dumb" backend that ignores token checks +/// - `setSimulateDeleteMarkers`: mimic S3 versioning-enabled buckets +/// +/// Not `final`: tests subclass it to distort single behaviors (e.g. clamp list page size to force +/// pagination) while delegating everything else to this base. +class InMemoryBackend : public Backend +{ +public: + InMemoryBackend() = default; + + /// Unhide the base convenience overloads (omitted Range/ObjectMeta/expected-token forms): the + /// overrides below would otherwise shadow them for callers holding a concrete backend type. + using Backend::get; + using Backend::getStream; + using Backend::putIfAbsent; + using Backend::putIfAbsentStream; + using Backend::putOverwrite; + using Backend::casPut; + + // ---- Backend interface ---- + + /// Returns the requested byte window, current token, and metadata, or `nullopt` when the key is absent. + std::optional get(const String & key, Range range) override; + + /// Returns a forward-only stream over the requested byte window, or `nullopt` when the key is absent. + /// The in-memory implementation copies the window into an owning read buffer while holding the + /// backend lock, so the returned stream remains independent of later backend mutations. + std::optional getStream(const String & key, Range range) override; + + /// Returns the current existence, size, token, and metadata without materializing the body. + HeadResult head(const String & key) override; + + /// The in-memory backend mints a monotonic token it surfaces through `list` — TRUE. + bool supportsListTokens() const override { return true; } + + /// Creates `key` only when it is absent. On success stores `bytes` and `meta` under a new token; + /// on a precondition failure leaves the existing object untouched. + PutResult putIfAbsent(const String & key, const String & bytes, const ObjectMeta & meta) override; + + /// Returns a single-use sink whose finalized body is published with the same atomic conditional + /// create semantics as `putIfAbsent`. Cancelling or destroying the sink before finalization does + /// not publish anything. + WriteSinkPtr putIfAbsentStream(const String & key, const ObjectMeta & meta) override; + + /// Replaces the existing object only when `expected` is its current token. Token enforcement can + /// be disabled with `setEnforceTokens` to model a backend that incorrectly ignores this condition. + PutResult putOverwrite(const String & key, const String & bytes, const Token & expected, + const ObjectMeta & meta) override; + + /// Performs create-if-absent when `expected` is empty, or replace-if-current-token otherwise. + /// Conflicts leave the store unchanged and are returned as an outcome rather than an exception. + CasResult casPut(const String & key, const String & bytes, const std::optional & expected, + const ObjectMeta & meta) override; + + /// Removes exactly the incarnation named by `token`, or queues that token check for a later + /// `landPendingDelete` when delete holding is enabled. A queued delete is reported as accepted, + /// but its token is rechecked when it is landed. + DeleteOutcome deleteExact(const String & key, const Token & token) override; + + /// Lists up to `limit` keys under `prefix` in map order. `cursor` is the last key from the previous + /// page; returned tokens identify the listed incarnations and `next_cursor` is set only when more + /// matching keys remain. + ListPage list(const String & prefix, const String & cursor, size_t limit) override; + + /// Same-store copy operations used to exercise the S3-staging contracts in memory. `promoteStaged` + /// copies the complete staging object to `blob_key` only when the destination is absent and returns + /// `PreconditionFailed` without changing it otherwise. The new token models the destination ETag. + /// + /// `resurrect` is the one sanctioned unconditional overwrite: it reads only the writer's + /// staging object, skips its envelope header, prepends `fresh_header`, and writes the resulting + /// body over `blob_key`. The fresh header makes the resurrected body and token different from the + /// condemned incarnation, so a delayed exact-token delete for that old incarnation cannot remove + /// the resurrection (`INV-NO-RETURN`). The caller must already have established that the + /// destination is condemned. + PutResult promoteStaged(const String & staging_key, const String & blob_key) override; + Token resurrect(ReadBuffer & payload, uint64_t payload_size, const String & blob_key, const String & fresh_header) override; + + // ---- Fault-injection controls ---- + + /// When true, `deleteExact` validates and enqueues deletes rather than applying them immediately. + /// The caller sees `Deleted` (the send was accepted), but the object remains until + /// `landPendingDelete`, where the token is checked again. + void setHoldDeletes(bool hold); + + /// Returns the number of currently held deletes. + size_t pendingDeletes() const; + + /// Applies and removes the held delete at index `i`. The token is evaluated against the current + /// object at land time; the queue entry is removed whether the result is `TokenMismatch` or + /// `Deleted`. An invalid index returns `NotFound`. + DeleteOutcome landPendingDelete(size_t i); + + /// Injects a one-shot artificial `Conflict` on the next `casPut` for `key`. + void failNextCasPut(const String & key); + + /// Injects a one-shot AMBIGUOUS outcome on the next `putIfAbsent` for `key`: instead of attempting + /// the write, that call throws a plain (non-`DB::Exception`) exception -- classified `Unresolved`, + /// never `DefiniteFailure`, by `classifyConditionalWriteResult` regardless of build flags -- and the + /// store is left exactly as it was. Models a request whose own HTTP attempt outcome is lost (a + /// timeout, a dropped connection) rather than a clean `PreconditionFailed`, for tests of controlled + /// ops (`CasRequestController::slotOccupy` and its callers) that must exercise the "ambiguous + /// attempt, resolve before deciding" path without a live network. One-shot, mirroring + /// `failNextCasPut`'s contract: consumed by the first matching `putIfAbsent` call, whether the key + /// was already present or not. + void injectAmbiguousPutIfAbsent(const String & key); + + /// Enables or disables token checks for delete, overwrite, and CAS operations. Disabling checks + /// models a backend that reports every expected token as matching. + void setEnforceTokens(bool enforce); + + /// When true, successful deletes report `created_delete_marker = true`, modelling a versioned S3 + /// bucket whose delete creates a marker instead of reclaiming the current object. + void setSimulateDeleteMarkers(bool simulate); + +private: + /// Complete in-memory incarnation state for one key. All fields are read or modified while + /// `mutex_` is held; replacing `token` marks a new incarnation even when the bytes are unchanged. + struct Object + { + String bytes; + Token token; + ObjectMeta meta; + }; + + /// Token captured when a held delete is queued. It is intentionally checked again at land time so + /// a replacement between send and land produces `TokenMismatch` rather than deleting the new object. + struct PendingDelete + { + String key; + Token token; + }; + + /// Mints the next process-local token. Tokens are strictly increasing and never reused by this + /// backend instance, which also makes token equality a safe content-cache identity check in tests. + Token mintToken(); + + /// Applies an exact-token delete while `mutex_` is already held. Used by immediate deletes and by + /// `landPendingDelete` after its queue entry has been removed. + DeleteOutcome applyDelete(const String & key, const Token & token); + + mutable std::mutex mutex_; + std::map store_; + uint64_t token_seq_ = 0; + + // Fault-injection state. These fields are protected by `mutex_` just like `store_`. + bool hold_deletes_ = false; + std::vector pending_deletes_; + std::set fail_next_cas_; + std::set ambiguous_put_keys_; + bool enforce_tokens_ = true; + bool simulate_delete_markers_ = false; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.cpp new file mode 100644 index 000000000000..81f69a83a11e --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.cpp @@ -0,0 +1,176 @@ +#include + +namespace ProfileEvents +{ +/// The CA per-namespace and per-operation events declared in `ProfileEvents.cpp`. +extern const Event CASBlobPut; +extern const Event CASBlobPutDeduplicated; +extern const Event CASBlobOverwrite; +extern const Event CASBlobCompareSwap; +extern const Event CASBlobCompareSwapConflict; +extern const Event CASBlobHead; +extern const Event CASBlobHeadMiss; +extern const Event CASBlobGet; +extern const Event CASBlobGetStream; +extern const Event CASBlobDelete; +extern const Event CASBlobList; + +extern const Event CASManifestPut; +extern const Event CASManifestPutDeduplicated; +extern const Event CASManifestOverwrite; +extern const Event CASManifestCompareSwap; +extern const Event CASManifestCompareSwapConflict; +extern const Event CASManifestHead; +extern const Event CASManifestHeadMiss; +extern const Event CASManifestGet; +extern const Event CASManifestGetStream; +extern const Event CASManifestDelete; +extern const Event CASManifestList; + +extern const Event CASRootPut; +extern const Event CASRootPutDeduplicated; +extern const Event CASRootOverwrite; +extern const Event CASRootCompareSwap; +extern const Event CASRootCompareSwapConflict; +extern const Event CASRootHead; +extern const Event CASRootHeadMiss; +extern const Event CASRootGet; +extern const Event CASRootGetStream; +extern const Event CASRootDelete; +extern const Event CASRootList; + +extern const Event CASGCPut; +extern const Event CASGCPutDeduplicated; +extern const Event CASGCOverwrite; +extern const Event CASGCCompareSwap; +extern const Event CASGCCompareSwapConflict; +extern const Event CASGCHead; +extern const Event CASGCHeadMiss; +extern const Event CASGCGet; +extern const Event CASGCGetStream; +extern const Event CASGCDelete; +extern const Event CASGCList; + +extern const Event CASServerPut; +extern const Event CASServerPutDeduplicated; +extern const Event CASServerOverwrite; +extern const Event CASServerCompareSwap; +extern const Event CASServerCompareSwapConflict; +extern const Event CASServerHead; +extern const Event CASServerHeadMiss; +extern const Event CASServerGet; +extern const Event CASServerGetStream; +extern const Event CASServerDelete; +extern const Event CASServerList; + +extern const Event CASOtherPut; +extern const Event CASOtherPutDeduplicated; +extern const Event CASOtherOverwrite; +extern const Event CASOtherCompareSwap; +extern const Event CASOtherCompareSwapConflict; +extern const Event CASOtherHead; +extern const Event CASOtherHeadMiss; +extern const Event CASOtherGet; +extern const Event CASOtherGetStream; +extern const Event CASOtherDelete; +extern const Event CASOtherList; +} + +namespace DB::Cas +{ + +/// Maps `(CasNs, CasOp)` to the corresponding `ProfileEvents::Event`. The table is row-major: the +/// outer index is the namespace and the inner index is the operation. Its rows and columns must stay +/// in lockstep with the `CasNs` and `CasOp` enum orderings. +static const ProfileEvents::Event cas_event_table[CAS_NS_COUNT][CAS_OP_COUNT] = +{ + /* Blob */ {ProfileEvents::CASBlobPut, ProfileEvents::CASBlobPutDeduplicated, ProfileEvents::CASBlobOverwrite, + ProfileEvents::CASBlobCompareSwap, ProfileEvents::CASBlobCompareSwapConflict, ProfileEvents::CASBlobHead, + ProfileEvents::CASBlobHeadMiss, ProfileEvents::CASBlobGet, ProfileEvents::CASBlobGetStream, + ProfileEvents::CASBlobDelete, ProfileEvents::CASBlobList}, + /* Manifest */ {ProfileEvents::CASManifestPut, ProfileEvents::CASManifestPutDeduplicated, ProfileEvents::CASManifestOverwrite, + ProfileEvents::CASManifestCompareSwap, ProfileEvents::CASManifestCompareSwapConflict, ProfileEvents::CASManifestHead, + ProfileEvents::CASManifestHeadMiss, ProfileEvents::CASManifestGet, ProfileEvents::CASManifestGetStream, + ProfileEvents::CASManifestDelete, ProfileEvents::CASManifestList}, + /* Root */ {ProfileEvents::CASRootPut, ProfileEvents::CASRootPutDeduplicated, ProfileEvents::CASRootOverwrite, + ProfileEvents::CASRootCompareSwap, ProfileEvents::CASRootCompareSwapConflict, ProfileEvents::CASRootHead, + ProfileEvents::CASRootHeadMiss, ProfileEvents::CASRootGet, ProfileEvents::CASRootGetStream, + ProfileEvents::CASRootDelete, ProfileEvents::CASRootList}, + /* Gc */ {ProfileEvents::CASGCPut, ProfileEvents::CASGCPutDeduplicated, ProfileEvents::CASGCOverwrite, + ProfileEvents::CASGCCompareSwap, ProfileEvents::CASGCCompareSwapConflict, ProfileEvents::CASGCHead, + ProfileEvents::CASGCHeadMiss, ProfileEvents::CASGCGet, ProfileEvents::CASGCGetStream, + ProfileEvents::CASGCDelete, ProfileEvents::CASGCList}, + /* Server */ {ProfileEvents::CASServerPut, ProfileEvents::CASServerPutDeduplicated, ProfileEvents::CASServerOverwrite, + ProfileEvents::CASServerCompareSwap, ProfileEvents::CASServerCompareSwapConflict, ProfileEvents::CASServerHead, + ProfileEvents::CASServerHeadMiss, ProfileEvents::CASServerGet, ProfileEvents::CASServerGetStream, + ProfileEvents::CASServerDelete, ProfileEvents::CASServerList}, + /* Other */ {ProfileEvents::CASOtherPut, ProfileEvents::CASOtherPutDeduplicated, ProfileEvents::CASOtherOverwrite, + ProfileEvents::CASOtherCompareSwap, ProfileEvents::CASOtherCompareSwapConflict, ProfileEvents::CASOtherHead, + ProfileEvents::CASOtherHeadMiss, ProfileEvents::CASOtherGet, ProfileEvents::CASOtherGetStream, + ProfileEvents::CASOtherDelete, ProfileEvents::CASOtherList}, +}; + +CasNs classifyCasNs(const String & key) +{ + if (key.find("/blobs/") != String::npos) + return CasNs::Blob; + /// Ref streams and namespace-owned state live under `cas/ns/`; part manifests are under + /// `cas/manifests//`. These paths must be classified before + /// the generic `roots/` and `Other` cases, otherwise the ref and manifest operation counters + /// silently accumulate in the wrong namespace. + if (key.find("/cas/ns/") != String::npos) + return CasNs::Root; + if (key.find("/cas/manifests/") != String::npos) + return CasNs::Manifest; + if (key.find("/roots/") != String::npos) + return CasNs::Root; + if (key.find("/gc/") != String::npos) + return CasNs::Gc; + return CasNs::Other; +} + +void incrementCasEvent(CasNs ns, CasOp op) +{ + ProfileEvents::increment(cas_event_table[static_cast(ns)][static_cast(op)]); +} + +namespace +{ + +/// Wraps an inner `WriteSink`. The namespace is captured at creation because the key is not available +/// at `finalize`; the `Put` versus `PutDeduplicated` outcome is emitted only after the inner sink returns. +/// Buffer access and cancellation delegate verbatim, while exceptions from the inner sink propagate. +class InstrumentedWriteSink final : public WriteSink +{ +public: + InstrumentedWriteSink(WriteSinkPtr inner_, CasNs ns_) : inner(std::move(inner_)), ns(ns_) {} + + WriteBuffer & buffer() override { return inner->buffer(); } + + /// Finalize the inner upload first, then count its returned outcome. No event is emitted if the + /// inner operation throws. + PutResult finalize() override + { + PutResult result = inner->finalize(); + incrementCasEvent(ns, result.outcome == PutOutcome::Done ? CasOp::Put : CasOp::PutDeduplicated); + return result; + } + + void cancel() noexcept override { inner->cancel(); } + +private: + WriteSinkPtr inner; + CasNs ns; +}; + +} + +WriteSinkPtr InstrumentedBackend::putIfAbsentStream(const String & key, const ObjectMeta & meta) +{ + WriteSinkPtr sink = inner->putIfAbsentStream(key, meta); + if (!sink) + return sink; + return std::make_unique(std::move(sink), classifyCasNs(key)); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h new file mode 100644 index 000000000000..b7d7e82a7f05 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h @@ -0,0 +1,202 @@ +#pragma once +#include +#include + +namespace DB::Cas +{ + +/// Per-namespace and per-operation instrumentation for the content-addressed storage seam. +/// +/// Every content-addressed storage operation flows through the abstract `Backend` seam. +/// `InstrumentedBackend` is a transparent decorator: it owns an inner `BackendPtr`, delegates every +/// operation, and increments a `ProfileEvent` keyed by the key's namespace and the operation's +/// outcome. The pool wraps its backend once in `Pool::open`, which includes operations issued by +/// background writers and GC as well as foreground calls, for both object-storage and in-memory +/// backends. This backend-level chokepoint is needed because background PUTs are not attributable +/// through the foreground request that scheduled them. + +/// Namespace of a CA key, classified by substring of the key path (6 classes; `Server` is currently +/// unreachable through this classifier — the per-server control subtree lives under +/// `/gc/server-roots//...` and classifies as Gc). +/// /blobs/.. → Blob +/// /cas/ns/.. → Root (immutable streams and point/path-addressed namespace state) +/// /cas/manifests/.. → Manifest +/// /roots/.. → Root (loose mountpoint objects) +/// /gc/.. → Gc +/// else (e.g. _pool_meta, _probe) → Other +enum class CasNs : uint8_t +{ + Blob = 0, + Manifest, + Root, + Gc, + Server, + Other, +}; +static constexpr size_t CAS_NS_COUNT = 6; + +/// Operation + outcome class (11 classes), mapped from the `Backend` method and its return value. +/// putIfAbsent / putIfAbsentStream finalize → Done ⇒ Put ; PreconditionFailed ⇒ PutDeduplicated +/// putOverwrite → Done ⇒ Overwrite ; PreconditionFailed ⇒ CasConflict +/// casPut → Committed ⇒ Cas ; Conflict ⇒ CasConflict +/// head → exists ⇒ Head ; !exists ⇒ HeadMiss (the 404 signal) +/// get → Get (all calls, hit or miss) +/// getStream → GetStream (all calls, hit or miss) +/// deleteExact → Delete (all outcomes) +/// list → List +enum class CasOp : uint8_t +{ + Put = 0, + PutDeduplicated, + Overwrite, + Cas, + CasConflict, + Head, + HeadMiss, + Get, + GetStream, + Delete, + List, +}; +static constexpr size_t CAS_OP_COUNT = 11; + +/// Classify a key into its namespace by substring. The order is significant where a more specific +/// layout such as `cas/ns/` must be recognized before a generic fallback; unknown key families +/// are intentionally counted as `Other`. +CasNs classifyCasNs(const String & key); + +/// Increment the `ProfileEvent` corresponding to `(ns, op)`. The row-major table is defined in the +/// implementation and must remain aligned with the `CasNs` and `CasOp` enum values. +void incrementCasEvent(CasNs ns, CasOp op); + +/// Transparent `Backend` decorator that records operation counts without changing the wrapped +/// backend's results, exceptions, or state transitions. The inner backend is owned by this object. +/// For streaming creates, namespace classification happens when the sink is created and the +/// `Put`/`PutDeduplicated` event is emitted only when `finalize` returns, because the outcome is unavailable +/// earlier. +class InstrumentedBackend final : public Backend +{ +public: + /// Unhide the base convenience overloads (omitted Range/ObjectMeta/expected-token forms): the + /// overrides below would otherwise shadow them for callers holding a concrete backend type. + using Backend::get; + using Backend::getStream; + using Backend::putIfAbsent; + using Backend::putIfAbsentStream; + using Backend::putOverwrite; + using Backend::casPut; + + explicit InstrumentedBackend(BackendPtr inner_) : inner(std::move(inner_)) {} + + /// Capability checks are deliberately uninstrumented: they do not represent storage operations. + void checkPoolPreconditions() override { inner->checkPoolPreconditions(); } + void checkConditionalWriteSingleAttemptSupport() override { inner->checkConditionalWriteSingleAttemptSupport(); } + + /// The typed sentinel probe is a diagnostic/authoritative read, not a routine storage operation — + /// deliberately uninstrumented (no ProfileEvent), like the capability checks above. MUST still be + /// forwarded explicitly: `Backend::probeSentinelRaw`'s generic default derives its classification from + /// THIS object's own `head`/`get` (virtual dispatch would otherwise resolve back to + /// `InstrumentedBackend`'s plain, non-typed overrides above), silently discarding whatever sharper + /// container/permission evidence the wrapped `inner` backend (e.g. `ObjectStorageBackend`'s S3/Local + /// classification) is able to provide. + SentinelProbeResult probeSentinelRaw(const String & key) override { return inner->probeSentinelRaw(key); } + + /// Delegate the read and count it after the inner call succeeds or returns absent. Exceptions + /// propagate unchanged and therefore do not produce a separate outcome event. + std::optional get(const String & key, Range range) override + { + auto result = inner->get(key, range); + incrementCasEvent(classifyCasNs(key), CasOp::Get); + return result; + } + + /// Delegate a forward-only read stream and count the request after the stream is acquired. + std::optional getStream(const String & key, Range range) override + { + auto result = inner->getStream(key, range); + incrementCasEvent(classifyCasNs(key), CasOp::GetStream); + return result; + } + + /// Count `Head` or `HeadMiss` from the returned presence flag after delegating to the backend. + HeadResult head(const String & key) override + { + HeadResult result = inner->head(key); + incrementCasEvent(classifyCasNs(key), result.exists ? CasOp::Head : CasOp::HeadMiss); + return result; + } + + /// Count a successful create as `Put` and an existing-key precondition result as `PutDeduplicated`. + PutResult putIfAbsent(const String & key, const String & bytes, const ObjectMeta & meta) override + { + PutResult result = inner->putIfAbsent(key, bytes, meta); + incrementCasEvent(classifyCasNs(key), result.outcome == PutOutcome::Done ? CasOp::Put : CasOp::PutDeduplicated); + return result; + } + + /// Return a sink that records the create outcome when its `finalize` is called. + WriteSinkPtr putIfAbsentStream(const String & key, const ObjectMeta & meta) override; + + /// Count a successful token-conditional overwrite as `Overwrite`; a precondition conflict is + /// counted as `CasConflict`. + PutResult putOverwrite(const String & key, const String & bytes, const Token & expected, + const ObjectMeta & meta) override + { + PutResult result = inner->putOverwrite(key, bytes, expected, meta); + incrementCasEvent(classifyCasNs(key), result.outcome == PutOutcome::Done ? CasOp::Overwrite : CasOp::CasConflict); + return result; + } + + /// Count a committed compare-and-swap as `Cas`; conflicts are counted as `CasConflict`. + CasResult casPut(const String & key, const String & bytes, const std::optional & expected, + const ObjectMeta & meta) override + { + CasResult result = inner->casPut(key, bytes, expected, meta); + incrementCasEvent(classifyCasNs(key), result.outcome == CasOutcome::Committed ? CasOp::Cas : CasOp::CasConflict); + return result; + } + + /// Delegate token-exact deletion and count every returned deletion outcome as `Delete`. + DeleteOutcome deleteExact(const String & key, const Token & token) override + { + DeleteOutcome outcome = inner->deleteExact(key, token); + incrementCasEvent(classifyCasNs(key), CasOp::Delete); + return outcome; + } + + /// Delegate one paginated listing and classify the prefix used for the request. + ListPage list(const String & prefix, const String & cursor, size_t limit) override + { + ListPage page = inner->list(prefix, cursor, limit); + incrementCasEvent(classifyCasNs(prefix), CasOp::List); + return page; + } + + /// This capability is a property of the wrapped backend, not an operation to count. + bool supportsListTokens() const override { return inner->supportsListTokens(); } + + /// Count a successful staged promotion as a create of the destination blob; an existing + /// destination is the same deduplication outcome as `putIfAbsent`. + PutResult promoteStaged(const String & staging_key, const String & blob_key) override + { + PutResult result = inner->promoteStaged(staging_key, blob_key); + /// A write-once server-side copy is a create attempt on the BLOB key: Done ⇒ Put, 412 ⇒ PutDeduplicated. + incrementCasEvent(classifyCasNs(blob_key), result.outcome == PutOutcome::Done ? CasOp::Put : CasOp::PutDeduplicated); + return result; + } + + /// Count a staged resurrection as an unconditional overwrite of the destination blob. The + /// wrapped backend remains responsible for its fresh-header and condemned-token guarantees. + Token resurrect(ReadBuffer & payload, uint64_t payload_size, const String & blob_key, const String & fresh_header) override + { + Token token = inner->resurrect(payload, payload_size, blob_key, fresh_header); + /// An unconditional resurrect re-upload overwrites the (condemned) BLOB key. + incrementCasEvent(classifyCasNs(blob_key), CasOp::Overwrite); + return token; + } + +private: + BackendPtr inner; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp new file mode 100644 index 000000000000..01eab7498a3b --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp @@ -0,0 +1,1209 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "config.h" + +#if USE_AWS_S3 +#include +#endif + +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int FILE_DOESNT_EXIST; + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::Cas +{ + +ObjectStorageBackend::ObjectStorageBackend(ObjectStoragePtr object_storage_, Mode mode_, uint64_t conditional_single_put_cap_) + : object_storage(std::move(object_storage_)) + , mode(mode_) + , conditional_single_put_cap(conditional_single_put_cap_) + , emu_root(object_storage->getCommonKeyPrefix()) +{ + if (mode == Mode::Native && object_storage->conditionalOpsUseGenerationTokens()) + native_token_type = TokenType::Generation; +} + +/// See Backend::checkPoolPreconditions. Only the Native, generation-dialect (GCS) combination has +/// anything to check: a token-exact DELETE on a versioned bucket archives a noncurrent generation +/// instead of reclaiming storage, so GC "reclaim" would silently stop reclaiming. +void ObjectStorageBackend::checkPoolPreconditions() +{ + if (mode != Mode::Native || native_token_type != TokenType::Generation) + return; + + const auto versioned = object_storage->isBucketVersioningEnabled(); + if (!versioned.has_value()) + { + /// The check itself could not be verified — either the GetBucketVersioning-equivalent call + /// failed (e.g. permissions) or the storage does not support answering it. We proceed on the + /// ASSUMPTION that versioning is off rather than fail-closing the mount on an unknown: a + /// confirmed Enabled below is what actually breaks reclaim, and an outright refusal to mount + /// whenever the check is inconclusive would be too aggressive. This is intentionally logged + /// (not silent) so an operator can confirm the bucket's real state. + LOG_WARNING(getLogger("CasObjectStorageBackend"), + "CAS on GCS: could not VERIFY the bucket-versioning precondition (the versioning check " + "request failed or is not supported by this backend) — proceeding on the assumption that " + "bucket versioning is OFF. If versioning is actually enabled, token-exact DELETEs will " + "archive noncurrent generations instead of reclaiming storage and GC will silently stop " + "reclaiming space. Please verify the bucket's versioning setting manually."); + return; + } + + if (*versioned) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "CAS on GCS: the bucket has object VERSIONING enabled. A token-exact DELETE on a " + "versioned bucket archives a noncurrent generation instead of reclaiming storage — GC " + "would silently stop reclaiming space. Disable versioning on the bucket (and prefer " + "soft-delete duration 0 for CAS pools) and retry the mount."); +} + +/// See Backend::checkConditionalWriteSingleAttemptSupport. This is a MOUNT-TIME gate, deliberately +/// separate from the ctor: narrow, targeted unit tests can keep constructing a raw Native-mode backend +/// over a non-S3 IObjectStorage (LocalObjectStorage) to exercise OTHER behaviors in isolation — the +/// established convention throughout this test suite (see e.g. gtest_cas_backend_generation.cpp). A +/// REAL writable mount, by contrast, always reaches this check: runCapabilityProbe (CasProbe.cpp) calls +/// it for every non-read-only Pool::open, so production never silently runs Native-mode conditional +/// writes under the disk's default (~500-attempt) transparent retry policy. +void ObjectStorageBackend::checkConditionalWriteSingleAttemptSupport() +{ + if (mode != Mode::Native) + return; + + /// The property checked is now backend CAPABILITY, not client presence: whether this object + /// storage can honor the SingleAttempt retry profile at all (S3ObjectStorage always can; a non-S3 + /// object storage like LocalObjectStorage cannot). + const bool single_attempt_supported = object_storage->supportsRetryProfile(ObjectStorageRetryProfile::SingleAttempt); + if (!single_attempt_supported) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "CAS Native-mode conditional writes require an object storage that supports the " + "SingleAttempt retry profile (RFC cas-s3-timeout-retry-control), but this one does not " + "(IObjectStorage::supportsRetryProfile returned false, or this build has no AWS S3 " + "support) — refusing to mount writable. Native mode is designed for an S3-like " + "conditional dialect only; a non-S3 object storage should use EmulatedSingleProcess."); +} + +/// ========================================================================================= +/// Native helpers +/// ========================================================================================= + +std::optional ObjectStorageBackend::nativeHead(const String & key) +{ + auto metadata = object_storage->tryGetObjectMetadata(key, /*with_tags=*/false); + if (!metadata) + return std::nullopt; + + HeadResult hr; + hr.exists = true; + hr.size = metadata->size_bytes; + hr.token = tokenForHead(metadata->etag); + hr.attributes = ObjectMeta(metadata->attributes.begin(), metadata->attributes.end()); + return hr; +} + +/// Finalize a conditional write (the condition rode on the buffer's WriteSettings) and map a +/// precondition loss to an OUTCOME — anything else propagates. +/// +/// A backend reports a lost condition as an `S3Exception` carrying the canonical S3 error code string +/// from the response XML `` (`S3Exception::getExceptionName`); a conditional-write 412 is +/// UNMODELED for the AWS SDK (its enum value is UNKNOWN), so `S3Exception::isPreconditionFailed` is the +/// typed signal — the `PreconditionFailed` name, or that token in the raw body for S3-compatible stores +/// (RustFS) whose non-AWS body the SDK cannot parse into a name. A `404 NoSuchKey` on an `If-Match` PUT +/// (the key was deleted out from under us) is treated identically: protocol callers handle 'mismatch' +/// and 'gone' the same way (re-validate), so both collapse onto `PreconditionFailed`. `NoSuchKey` IS +/// modeled by the SDK, and `WriteBufferFromS3` retries it internally surfacing the exhaustion with the +/// typed enum code (and no name), so the enum is matched as well as the name. The mapping is fail-safe in +/// direction: a misread error becomes a retryable PreconditionFailed/Conflict, never a false success. +/// +/// Native conditional writes require an S3-compatible integration environment for end-to-end +/// coverage. Unit tests cover the emulated semantics, the typed exception path, and this classifier +/// through the test-only `detail` declaration. +#if USE_AWS_S3 +PutOutcome detail::finalizeConditionalWrite(WriteBuffer & buf) +{ + try + { + buf.finalize(); + } + catch (const S3Exception & e) + { + if (e.isPreconditionFailed() + || e.getExceptionName() == "NoSuchKey" + || e.getS3ErrorCode() == Aws::S3::S3Errors::NO_SUCH_KEY) + return PutOutcome::PreconditionFailed; + throw; + } + return PutOutcome::Done; +} +#endif + +/// Build-dispatching shim for the write paths below: without the AWS SDK there is no S3Exception +/// to classify, so the errors of finalize simply propagate. +static PutOutcome finalizeConditionalWrite(WriteBuffer & buf) +{ +#if USE_AWS_S3 + return detail::finalizeConditionalWrite(buf); +#else + buf.finalize(); + return PutOutcome::Done; +#endif +} + +/// Instrument the same single `finalize` call used by both Native write paths without changing their +/// `Done`/`PreconditionFailed`-or-rethrow contract. A classified precondition loss is `Unresolved`, +/// not `Committed` or a definite exception, because the response does not prove who created or +/// replaced the object; the higher-level request controller may then resolve it with exact-key state. +static PutOutcome finalizeConditionalWriteInstrumented(WriteBuffer & buf) +{ + recordConditionalWriteAttemptStarted(); + try + { + const PutOutcome legacy = finalizeConditionalWrite(buf); + recordConditionalWriteOutcome( + legacy == PutOutcome::Done ? classifyConditionalWriteResult() : CasWriteOutcome::Unresolved); + return legacy; + } + catch (const std::exception & e) + { + recordConditionalWriteOutcome(classifyConditionalWriteResult(e)); + throw; + } +} + +/// Issue a conditional PUT (the condition rides on `ws`) and map a precondition loss — see +/// finalizeConditionalWrite. The condition is checked by the backend when the object is completed, +/// so the precondition loss always surfaces from the buffer's finalize, never from write. +PutResult ObjectStorageBackend::nativeConditionalPut(const String & key, const String & bytes, const WriteSettings & ws, const ObjectMeta & meta) +{ + std::optional attrs; + if (!meta.empty()) + attrs.emplace(meta.begin(), meta.end()); /// ObjectMeta is the same map type as ObjectAttributes + auto buf = object_storage->writeObject( + StoredObject(key), WriteMode::Rewrite, attrs, DBMS_DEFAULT_BUFFER_SIZE, ws); + buf->write(bytes.data(), bytes.size()); + if (finalizeConditionalWriteInstrumented(*buf) == PutOutcome::PreconditionFailed) + return {PutOutcome::PreconditionFailed, {}}; + + /// Record the token of the incarnation WE just wrote (model WCreate). The S3 write returns + /// its object ETag in the PutObject/CompleteMultipartUpload response, so no follow-up HEAD + /// is needed — this is ~73% of the CA backend's HEADs. A backend with no write-time ETag + /// (local files) returns nullopt and we fall back to the HEAD (a cheap local stat there). + Token token; + if (auto etag = buf->getResultObjectETag(); etag && !etag->empty()) + token = tokenForHead(*etag); + else + { + /// No write-time ETag (local files) or an (anomalous) empty one: fall back to the HEAD — + /// the pre-existing behavior, so an empty-ETag server is never worse than before. + auto hr = nativeHead(key); + token = hr ? hr->token : Token{}; + } + return {PutOutcome::Done, token}; +} + +namespace +{ + +/// True-streaming WriteSink for Native mode: the underlying object-storage write buffer was opened +/// with `If-None-Match: *` riding on its WriteSettings, so bytes stream through it directly and the +/// condition is checked when finalize completes the object — see finalizeConditionalWrite for the +/// outcome mapping. Nothing is ever published on cancel/destruction. +class NativeStreamingSink final : public WriteSink +{ +public: + NativeStreamingSink(ObjectStorageBackend & backend_, String key_, std::unique_ptr write_buf_) + : backend(backend_) + , key(std::move(key_)) + , write_buf(std::move(write_buf_)) + { + } + + WriteBuffer & buffer() override { return *write_buf; } + + PutResult finalize() override + { + chassert(!done); /// finalize after finalize/cancel is a misuse — see the WriteSink contract + done = true; + if (finalizeConditionalWriteInstrumented(*write_buf) == PutOutcome::PreconditionFailed) + { + /// Losing the condition is an ORDINARY outcome, not an error: another writer legitimately + /// took the slot. Abort HERE rather than leaving it to the buffer's destructor, which warns + /// "was neither finished nor aborted" on every occurrence -- and a server that writes that + /// to stderr fails the test around it -- while the uploaded parts stay billable until a + /// lifecycle rule reaps them. + write_buf->cancel(); + return {PutOutcome::PreconditionFailed, {}}; + } + + /// Record the token of the incarnation we just wrote (model WCreate). The S3 write + /// returns its object ETag in the response, so no follow-up HEAD is needed (the bulk of + /// the CA backend's HEADs). Backends with no write-time ETag (local) return nullopt and + /// we fall back to the HEAD (a cheap local stat there). + Token token; + if (auto etag = write_buf->getResultObjectETag(); etag && !etag->empty()) + token = backend.tokenForHead(*etag); + else + { + /// No write-time ETag (local) or an (anomalous) empty one: fall back to the HEAD. + auto hr = backend.head(key); + token = hr.exists ? hr.token : Token{}; + } + return {PutOutcome::Done, token}; + } + + void cancel() noexcept override + { + done = true; + write_buf->cancel(); + } + + ~NativeStreamingSink() override + { + if (!done) + cancel(); + } + +private: + ObjectStorageBackend & backend; + const String key; + std::unique_ptr write_buf; + bool done = false; +}; + +/// Memory-buffered WriteSink for EmulatedSingleProcess mode (unit tests only — buffering the whole +/// body is acceptable and documented): accumulates into a WriteBufferFromOwnString and delegates the +/// conditional publish to putIfAbsent at finalize, which provides atomicity under emu_mutex. Nothing +/// is ever published on cancel/destruction. +class EmulatedBufferedSink final : public WriteSink +{ +public: + EmulatedBufferedSink(Backend & backend_, String key_, ObjectMeta meta_) + : backend(backend_) + , key(std::move(key_)) + , meta(std::move(meta_)) + { + } + + WriteBuffer & buffer() override { return buf; } + + PutResult finalize() override + { + chassert(!done); /// finalize after finalize/cancel is a misuse — see the WriteSink contract + done = true; + return backend.putIfAbsent(key, buf.str(), meta); + } + + void cancel() noexcept override + { + done = true; + buf.cancel(); + } + + ~EmulatedBufferedSink() override + { + if (!done) + cancel(); + } + +private: + Backend & backend; + const String key; + const ObjectMeta meta; + WriteBufferFromOwnString buf; + bool done = false; +}; + +} + +/// True when an exception from `IObjectStorage::readObject` means "the object is simply not there". +/// Two surfaces: +/// 1. S3/RustFS: `S3Exception` with `S3Errors::NO_SUCH_KEY` (the modeled enum — the primary +/// signal) or `getExceptionName() == "NoSuchKey"` (the canonical XML `` string, present +/// when the SDK was able to parse it; mirrors `finalizeConditionalWrite`'s detection). +/// 2. Local / emulated: `DB::Exception` with `ErrorCodes::FILE_DOESNT_EXIST` (from +/// `ReadBufferFromFile` when `open(2)` returns ENOENT). +/// +/// Any other error (network, auth, throttle, corruption) propagates unchanged — fail-closed. +static bool isObjectNotFound(const std::exception & e) +{ +#if USE_AWS_S3 + if (const auto * s3e = dynamic_cast(&e)) + return s3e->getS3ErrorCode() == Aws::S3::S3Errors::NO_SUCH_KEY + || s3e->getExceptionName() == "NoSuchKey"; +#endif + if (const auto * dbe = dynamic_cast(&e)) + return dbe->code() == ErrorCodes::FILE_DOESNT_EXIST; + return false; +} + +/// Read `range` of the object at `path` as a TRUE ranged read: seek to the offset and bound the +/// read window. Seek the storage buffer to the requested offset and bound the returned bytes instead +/// of reading a whole snapshot run and slicing it afterward; snapshot runs can be gigabytes at scale, +/// while the caller's memory budget is O(block). +static String readObjectRanged(IObjectStorage & object_storage, const String & path, Range range, + uint64_t known_size = 0) +{ + auto buf = object_storage.readObject( + StoredObject(path), casSizedReadSettings(getReadSettings(), known_size), /*read_hint=*/std::nullopt); + String content; + if (range.whole()) + { + readStringUntilEOF(content, *buf); + return content; + } + + /// An offset at or past EOF yields an empty result, matching the range contract of the previous + /// whole-read implementation. + /// `seek` past the object size may throw depending on the storage, so fail-close the window + /// against the known size before touching the buffer position. + /// Native callers already HEAD the key, so passing its size avoids another metadata round trip. + /// A zero size means the caller does not know it and metadata must be fetched here. + const uint64_t object_size = known_size != 0 ? known_size + : object_storage.getObjectMetadata(path, /*with_tags=*/false).size_bytes; + if (range.offset >= object_size) + return {}; + + /// The readable window, clamped to EOF. `setReadUntilPosition` is only a hint (not every object + /// storage honors it — LocalObjectStorage does not), so the exact byte count below is what bounds + /// the read; the hint lets storages that DO honor it avoid over-fetching. + const uint64_t available = object_size - range.offset; + const uint64_t to_read = range.length.has_value() ? std::min(*range.length, available) : available; + + if (range.length.has_value()) + buf->setReadUntilPosition(range.offset + *range.length); + buf->seek(static_cast(range.offset), SEEK_SET); + + content.resize(to_read); + const size_t got = buf->read(content.data(), to_read); + content.resize(got); + return content; +} + +/// Open a forward-only stream over `range` of the object at `path`, positioned at the window's first +/// byte and bounded to its last. Mirrors +/// `readObjectRanged`'s seek + bound, but RETURNS the buffer instead of draining it — the caller reads +/// at its own pace, so nothing is materialized whole. Returns nullptr when the offset is at or past EOF +/// (the empty-window clamp), matching the ranged-get contract. +static std::unique_ptr openObjectRangedStream(IObjectStorage & object_storage, const String & path, Range range, + uint64_t known_size = 0) +{ + auto buf = object_storage.readObject( + StoredObject(path), casSizedReadSettings(getReadSettings(), known_size), /*read_hint=*/std::nullopt); + if (range.whole()) + return buf; + + /// Clamp exactly like `readObjectRanged`: an offset at or past EOF yields an empty stream, and + /// `seek` past the object size may throw depending on the storage, so fail-close against the known + /// size before touching the buffer position. + /// As in `readObjectRanged`, a caller-supplied size avoids another metadata round trip; zero means + /// that the size is unknown and must be fetched. + const uint64_t object_size = known_size != 0 ? known_size + : object_storage.getObjectMetadata(path, /*with_tags=*/false).size_bytes; + if (range.offset >= object_size) + return std::make_unique(std::string_view{}); + + /// `setReadUntilPosition` is only a hint (LocalObjectStorage does not honor it), but for a returned + /// stream it is the only bound available — the caller drains to EOF, so a storage that DOES honor + /// the hint stops at the window end, and one that does not over-reads only the trailing bytes. + if (range.length.has_value()) + buf->setReadUntilPosition(range.offset + *range.length); + buf->seek(static_cast(range.offset), SEEK_SET); + return buf; +} + +ReadSettings casSizedReadSettings(const ReadSettings & base, uint64_t known_size) +{ + if (known_size == 0) + return base; + return base.adjustBufferSize(known_size + CAS_FOLD_READ_SLACK_BYTES); +} + +/// ========================================================================================= +/// Emulated helpers (caller holds emu_mutex) +/// ========================================================================================= + +namespace +{ + +/// The mtime-quantum guard (emuMintToken) only needs a key's `emu_token_state` entry while a +/// same-quantum tie is still POSSIBLE for a FRESH recreate — i.e. while the just-deleted +/// incarnation's own etag (mtime-ns, see emuMintToken) is recent. Once it is comfortably behind +/// "now", no later recreate can land in the same mtime quantum, so retaining the entry serves no +/// purpose (codex-review-triage §3.18, Important #1). 2 seconds is far above any filesystem's mtime +/// tick coarseness while still bounding the map to the recently-deleted-key population. +constexpr uint64_t EMU_TOKEN_STALE_AGE_NS = 2'000'000'000ULL; +constexpr size_t EMU_TOKEN_EXPIRY_SWEEP_SIZE = 16; + +/// True iff `etag` parses as a plain nanosecond count (emuMintToken's `.first` is always the BARE +/// etag, never the `etag#N` disambiguated form) that is at least EMU_TOKEN_STALE_AGE_NS behind now. +/// An etag that fails to parse (e.g. a test double's non-numeric stub) is conservatively treated as +/// NOT stale — never erasing is always safe, merely un-bounded, so an unparseable value must not be +/// mistaken for a recent one. +bool etagComfortablyInThePast(const String & etag, uint64_t now_ns) +{ + if (etag.empty() || !std::all_of(etag.begin(), etag.end(), [](char c) { return c >= '0' && c <= '9'; })) + return false; + + uint64_t etag_ns = 0; + try + { + etag_ns = std::stoull(etag); + } + catch (...) + { + return false; + } + + return now_ns > etag_ns && (now_ns - etag_ns) >= EMU_TOKEN_STALE_AGE_NS; +} + +} + +String ObjectStorageBackend::emuPath(const String & key) const +{ + if (emu_root.empty()) + return key; + if (!emu_root.empty() && emu_root.back() == '/') + return emu_root + key; + return emu_root + "/" + key; +} + +uint64_t ObjectStorageBackend::emuNowNs() const +{ + if (emu_now_ns_for_test != 0) + return emu_now_ns_for_test; + return static_cast( + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count()); +} + +void ObjectStorageBackend::setEmuNowNsForTest(uint64_t now_ns) +{ + std::lock_guard lock(emu_mutex); + emu_now_ns_for_test = now_ns; +} + +size_t ObjectStorageBackend::emuTokenStateSizeForTest() const +{ + std::lock_guard lock(emu_mutex); + return emu_token_state.size(); +} + +void ObjectStorageBackend::emuPruneTokenState(uint64_t now_ns) +{ + for (size_t checked = 0; checked < EMU_TOKEN_EXPIRY_SWEEP_SIZE && !emu_token_expiry.empty(); ++checked) + { + const auto & candidate = emu_token_expiry.front(); + auto current = emu_token_state.find(candidate.key); + + /// A later mint (including a delete+recreate in the same mtime quantum) supersedes this exact + /// deleted state. Its queue record can be discarded immediately without touching the map. + if (current == emu_token_state.end() || current->second != candidate.token_state) + { + emu_token_expiry.pop_front(); + continue; + } + + /// Deletion time is monotonic within this mutex-protected FIFO. If its oldest record has not + /// crossed the safety window, every later matching record is too recent as well. + if (now_ns <= candidate.queued_at_ns || now_ns - candidate.queued_at_ns < EMU_TOKEN_STALE_AGE_NS) + break; + + /// The record has aged enough to inspect its etag. Unparseable or otherwise uncertain etags + /// stay in the map (fail safe), but their queue records cannot block pruning of later keys. + if (etagComfortablyInThePast(current->second.first, now_ns)) + emu_token_state.erase(current); + emu_token_expiry.pop_front(); + } +} + +bool ObjectStorageBackend::emuExists(const String & key) const +{ + return object_storage->exists(StoredObject(emuPath(key))); +} + +String ObjectStorageBackend::emuRead(const String & key, Range range) const +{ + return readObjectRanged(*object_storage, emuPath(key), range); +} + +Token ObjectStorageBackend::emuWrite(const String & key, const String & bytes, const ObjectMeta & meta) +{ + std::optional attrs; + if (!meta.empty()) + attrs.emplace(meta.begin(), meta.end()); /// ObjectMeta is the same map type as ObjectAttributes + auto buf = object_storage->writeObject(StoredObject(emuPath(key)), WriteMode::Rewrite, attrs); + buf->write(bytes.data(), bytes.size()); + buf->finalize(); + + const auto metadata = object_storage->tryGetObjectMetadata(emuPath(key), /*with_tags=*/false); + return emuMintToken(key, metadata ? metadata->etag : String{}, /*just_wrote=*/true); +} + +Token ObjectStorageBackend::emuObserveToken(const String & key) +{ + const auto metadata = object_storage->tryGetObjectMetadata(emuPath(key), /*with_tags=*/false); + return emuMintToken(key, metadata ? metadata->etag : String{}, /*just_wrote=*/false); +} + +Token ObjectStorageBackend::emuMintToken(const String & key, const String & etag, bool just_wrote) +{ + emuPruneTokenState(emuNowNs()); + + /// Anomalous: the object storage reported no etag at all (LocalObjectStorage always does; this + /// guards a hypothetical future/test double). Mint a fresh, UNPERSISTED value — never worse than + /// the old counter for this case, but never masquerading as a real etag-derived identity. + if (etag.empty()) + return Token{std::to_string(++emu_seq), TokenType::Emulated}; + + auto it = emu_token_state.find(key); + if (it != emu_token_state.end() && it->second.first == etag) + { + /// The etag has not advanced since the last token we minted for this key. For a read-only + /// observation that is expected (the object simply has not changed) and the SAME value must be + /// returned. For a just-completed WRITE it means this write's mtime landed in the same quantum + /// as the previous incarnation's — two DIFFERENT incarnations must still never mint identical + /// tokens, so bump a small per-key disambiguator (mtime-quantum guard, triage §3.18 19c step 4). + if (just_wrote) + ++it->second.second; + const String value = it->second.second == 0 ? etag : etag + "#" + std::to_string(it->second.second); + return Token{value, TokenType::Emulated}; + } + + /// The etag advanced (or this key is seen for the first time): the bare etag is the token, and any + /// previous disambiguator is dropped — a genuinely new incarnation starts clean. + emu_token_state[key] = {etag, 0}; + return Token{etag, TokenType::Emulated}; +} + +/// ========================================================================================= +/// Backend interface +/// ========================================================================================= + +std::optional ObjectStorageBackend::get(const String & key, Range range) +{ + if (mode == Mode::Native) + { + auto hr = nativeHead(key); + if (!hr) + return std::nullopt; + + /// The object may be deleted between the HEAD above and the GET below (a GC or concurrent + /// writer racing the read window). Catch the not-found signal and honor the `optional` + /// contract — callers such as `Pool::loadShardDecoded` already handle a nullopt return and + /// treat it as "raced a deletion, absent". Any other error (network, auth, corruption) + /// propagates unchanged — fail-closed by construction. + /// + /// A REPLACEMENT racing the same window (HEAD observes token A, GET reads the bytes of a + /// subsequently-written incarnation B) is likewise not a hazard: HEAD strictly precedes GET, so + /// the returned token is never NEWER than the returned bytes — a mixed pair is always + /// (bytes_newer, token_older), never the reverse. Every consumer of this token uses it as a + /// conditional precondition (`casPut`/`putOverwrite`/`deleteExact`), which fails closed EXACTLY + /// in the mixed case, so a stale token costs a retry, never lets a caller act on a + /// bytes/token pair that never coexisted. This also covers `known_size`: content-addressed blob + /// bodies are byte-identical across incarnations (a "replacement" only rotates envelope/token), + /// mutable control objects are read-modify-CAS loops that re-validate on conflict, and write-once + /// objects self-validate their contents on decode. + GetResult gr; + try + { + gr.bytes = readObjectRanged(*object_storage, key, range, hr->size); + } + catch (const std::exception & e) + { + if (isObjectNotFound(e)) + return std::nullopt; + throw; + } + gr.token = hr->token; + return gr; + } + + std::lock_guard lock(emu_mutex); + if (!emuExists(key)) + return std::nullopt; + + /// The emulated path holds emu_mutex across the exists-check and the read, so no concurrent + /// caller in this process can delete the file in between. External deletion (e.g. a test teardown + /// racing a read) is still handled: convert FILE_DOESNT_EXIST to nullopt rather than letting it + /// escape as an unexplained exception. + GetResult gr; + try + { + gr.bytes = emuRead(key, range); + } + catch (const std::exception & e) + { + if (isObjectNotFound(e)) + return std::nullopt; + throw; + } + gr.token = emuObserveToken(key); + return gr; +} + +std::optional ObjectStorageBackend::getStream(const String & key, Range range) +{ + if (mode == Mode::Native) + { + auto hr = nativeHead(key); + if (!hr) + return std::nullopt; + + /// Same HEAD-then-read race as `get`: the object may be deleted between the HEAD above and the + /// stream open below. Honor the `optional` contract on a not-found signal; any other error + /// (network, auth, corruption) propagates unchanged — fail-closed by construction. + GetStreamResult sr; + try + { + sr.stream = openObjectRangedStream(*object_storage, key, range, hr->size); + } + catch (const std::exception & e) + { + if (isObjectNotFound(e)) + return std::nullopt; + throw; + } + sr.token = hr->token; + return sr; + } + + std::lock_guard lock(emu_mutex); + if (!emuExists(key)) + return std::nullopt; + + /// The emulated path holds emu_mutex across the exists-check and the stream open, matching `get`. + /// External deletion still converts to nullopt rather than escaping as an unexplained exception. + GetStreamResult sr; + try + { + sr.stream = openObjectRangedStream(*object_storage, emuPath(key), range); + } + catch (const std::exception & e) + { + if (isObjectNotFound(e)) + return std::nullopt; + throw; + } + sr.token = emuObserveToken(key); + return sr; +} + +HeadResult ObjectStorageBackend::head(const String & key) +{ + if (mode == Mode::Native) + { + auto hr = nativeHead(key); + return hr ? *hr : HeadResult{}; + } + + std::lock_guard lock(emu_mutex); + if (!emuExists(key)) + return HeadResult{}; + + auto metadata = object_storage->tryGetObjectMetadata(emuPath(key), /*with_tags=*/false); + /// A path that exists on the Local filesystem but yields no object metadata is a directory, not + /// an object (`tryGetObjectMetadata` returns nullopt for a directory). HEAD must report it as + /// not-an-object (exists=false) — otherwise existsFile/getStorageObjects treat a pool sub-dir (e.g. + /// `store`, traversed by system.remote_data_paths) as a file and a later body read throws EISDIR. + if (!metadata) + return HeadResult{}; + HeadResult hr; + hr.exists = true; + hr.size = metadata->size_bytes; + hr.attributes = ObjectMeta(metadata->attributes.begin(), metadata->attributes.end()); + hr.token = emuObserveToken(key); + return hr; +} + +/// See Backend::probeSentinelRaw / CasBackend.h's ProbeOutcome for the semantics this classifies. +SentinelProbeResult ObjectStorageBackend::probeSentinelRaw(const String & key) +{ + if (mode == Mode::Native) + { + try + { + /// `getObjectMetadata` (unlike `tryGetObjectMetadata`/`nativeHead`) is the THROWING raw-HEAD + /// primitive — it does NOT collapse NO_SUCH_KEY/NO_SUCH_BUCKET/RESOURCE_NOT_FOUND into one + /// `nullopt` before we get a chance to classify the S3 error. Its result is discarded here; + /// only whether (and how) it throws matters — the body comes from `get` below. + object_storage->getObjectMetadata(key, /*with_tags=*/false); + + /// The raw HEAD proved the key present. Delegate the body read to the existing `get`, which + /// already HEADs again and reads — an extra round trip this authoritative, low-rate probe can + /// afford, in exchange for reusing its already-correct HEAD→GET race handling. Kept INSIDE + /// this try: a transient failure here must also classify Indeterminate, never escape unclassified. + auto g = get(key); + if (!g) + return {ProbeOutcome::KeyAbsent, std::nullopt}; /// raced a deletion right after the raw HEAD + return {ProbeOutcome::Present, std::move(g->bytes)}; + } +#if USE_AWS_S3 + catch (const S3Exception & e) + { + switch (e.getS3ErrorCode()) + { + case Aws::S3::S3Errors::NO_SUCH_KEY: + return {ProbeOutcome::KeyAbsent, std::nullopt}; + case Aws::S3::S3Errors::RESOURCE_NOT_FOUND: + /// A HEAD response carries no body, so the SDK cannot parse a `NoSuchKey` `` + /// and instead derives this generic code straight from the HTTP 404 status (see + /// `isNotFoundError`, `src/IO/S3/getObjectInfo.cpp`) — this is what a REAL S3 HEAD + /// on an absent key actually throws. The container/key distinction is deliberately + /// NOT attempted here (a bodyless 404 cannot carry it). + return {ProbeOutcome::KeyAbsent, std::nullopt}; + case Aws::S3::S3Errors::NO_SUCH_BUCKET: + return {ProbeOutcome::ContainerAbsent, std::nullopt}; + case Aws::S3::S3Errors::ACCESS_DENIED: + return {ProbeOutcome::AccessDenied, std::nullopt}; + default: + /// Everything else (timeouts, 5xx, throttling, an unmodeled code) is inconclusive — + /// NEVER promoted to KeyAbsent, per the IAM permutation table in spec §2. + return {ProbeOutcome::Indeterminate, std::nullopt}; + } + } +#endif + catch (...) + { + return {ProbeOutcome::Indeterminate, std::nullopt}; + } + } + + /// EmulatedSingleProcess (Local): stat the configured container directory FIRST — `emuExists`/`get` + /// alone cannot distinguish "this key is absent" from "the whole pool directory is gone" (Local + /// listing is best-effort and silently reports zero either way, see LocalObjectStorage::listObjects). + try + { + if (!object_storage->existsOrHasAnyChild(emu_root)) + return {ProbeOutcome::ContainerAbsent, std::nullopt}; + + auto g = get(key); + if (!g) + return {ProbeOutcome::KeyAbsent, std::nullopt}; + return {ProbeOutcome::Present, std::move(g->bytes)}; + } + catch (...) + { + return {ProbeOutcome::Indeterminate, std::nullopt}; + } +} + +/// Base WriteSettings for every Native conditional write. CAS-mutable keys (shard manifests, +/// gc/state, the registry) override check_objects_after_upload to `false` (see WriteSettings.h); +/// this was observed live against RustFS: a publish's manifest CAS raced the GC fence and the +/// mismatch terminated the server from the upload worker. +/// +/// On a generation-token store (GCS), a conditional write must ALSO never take the multipart path: +/// GCS enforces no preconditions on `CompleteMultipartUpload` (measured), so a lost +/// precondition on a multipart write would silently overwrite instead of failing. Force single-PUT +/// and raise the single-part cap to conditional_single_put_cap (RAM-buffered) to keep the fast path +/// available for bodies up to that size; a bigger body throws NOT_IMPLEMENTED from +/// WriteBufferFromS3::createMultipartUpload. +WriteSettings ObjectStorageBackend::conditionalWriteSettings() const +{ + WriteSettings ws; + ws.s3_check_objects_after_upload_override = false; + if (native_token_type == TokenType::Generation) + { + ws.s3_force_single_part_upload = true; + ws.s3_single_part_upload_max_bytes_override = conditional_single_put_cap; + } + /// Exactly one attempt at the WriteBufferFromS3 layer too: makeSinglepartUpload/ + /// completeMultipartUpload run their OWN retry loop above the S3 client, reissuing the identical + /// (conditional!) request on NO_SUCH_KEY — a client-level override alone does not bound it. Plain + /// size_t field, harmless (ignored) for a non-S3 write path. + ws.s3_max_unexpected_write_error_retries_override = 1; + /// Exactly one HTTP attempt for every conditional write: the object storage resolves the + /// profile to its own single-attempt client. A backend that cannot honor it is rejected for + /// writable Native mounts by checkConditionalWriteSingleAttemptSupport (fail closed). + ws.object_storage_retry_profile = ObjectStorageRetryProfile::SingleAttempt; + return ws; +} + +PutResult ObjectStorageBackend::putIfAbsent(const String & key, const String & bytes, const ObjectMeta & meta) +{ + if (mode == Mode::Native) + { + WriteSettings ws = conditionalWriteSettings(); + ws.object_storage_write_if_none_match = "*"; + return nativeConditionalPut(key, bytes, ws, meta); + } + + std::lock_guard lock(emu_mutex); + if (emuExists(key)) + return {PutOutcome::PreconditionFailed, {}}; + + return {PutOutcome::Done, emuWrite(key, bytes, meta)}; +} + +WriteSinkPtr ObjectStorageBackend::putIfAbsentStream(const String & key, const ObjectMeta & meta) +{ + if (mode == Mode::Native) + { + /// Same WriteSettings construction as putIfAbsent — the condition rides on the write buffer + /// and is checked when finalize completes the object. + WriteSettings ws = conditionalWriteSettings(); + ws.object_storage_write_if_none_match = "*"; + std::optional attrs; + if (!meta.empty()) + attrs.emplace(meta.begin(), meta.end()); /// ObjectMeta is the same map type as ObjectAttributes + auto buf = object_storage->writeObject( + StoredObject(key), WriteMode::Rewrite, attrs, DBMS_DEFAULT_BUFFER_SIZE, ws); + return std::make_unique(*this, key, std::move(buf)); + } + + return std::make_unique(*this, key, meta); +} + +PutResult ObjectStorageBackend::putOverwrite(const String & key, const String & bytes, const Token & expected, const ObjectMeta & meta) +{ + /// §3.18 №19: reject a wrong-dialect expected token before it ever reaches the wire (Native) or + /// the emu compare (Emulated) — see mintingTypeMatches. + if (!mintingTypeMatches(expected.type)) + return {PutOutcome::PreconditionFailed, {}}; + + if (mode == Mode::Native) + { + WriteSettings ws = conditionalWriteSettings(); + ws.object_storage_write_if_match = expected.value; + return nativeConditionalPut(key, bytes, ws, meta); + } + + std::lock_guard lock(emu_mutex); + if (!emuExists(key)) + return {PutOutcome::PreconditionFailed, {}}; + if (!tokenMatches(emuObserveToken(key), expected)) + return {PutOutcome::PreconditionFailed, {}}; + + return {PutOutcome::Done, emuWrite(key, bytes, meta)}; +} + +CasResult ObjectStorageBackend::casPut(const String & key, const String & bytes, const std::optional & expected, const ObjectMeta & meta) +{ + /// §3.18 №19: a create-if-absent CAS (expected == nullopt) has no token to validate; only the + /// swap form carries one, and it must match this backend's own minting dialect before anything + /// else runs. + if (expected.has_value() && !mintingTypeMatches(expected->type)) + return {CasOutcome::Conflict, {}}; + + if (mode == Mode::Native) + { + WriteSettings ws = conditionalWriteSettings(); + if (expected.has_value()) + ws.object_storage_write_if_match = expected->value; + else + ws.object_storage_write_if_none_match = "*"; + + /// The PUT-side outcomes (Done / PreconditionFailed) collapse onto CAS outcomes 1:1: a lost + /// condition — whether a mismatched If-Match or a 404 on an If-Match PUT — is a Conflict. + PutResult put = nativeConditionalPut(key, bytes, ws, meta); + return put.outcome == PutOutcome::Done + ? CasResult{CasOutcome::Committed, put.token} + : CasResult{CasOutcome::Conflict, {}}; + } + + std::lock_guard lock(emu_mutex); + const bool exists = emuExists(key); + + if (!expected.has_value()) + { + if (exists) + return {CasOutcome::Conflict, {}}; + } + else + { + if (!exists) + return {CasOutcome::Conflict, {}}; + if (!tokenMatches(emuObserveToken(key), *expected)) + return {CasOutcome::Conflict, {}}; + } + + return {CasOutcome::Committed, emuWrite(key, bytes, meta)}; +} + +DeleteOutcome ObjectStorageBackend::deleteExact(const String & key, const Token & token) +{ + /// §3.18 №19: same local dialect guard as putOverwrite/casPut — never forward a foreign-dialect + /// value as the removeObjectIfTokenMatches argument. + if (!mintingTypeMatches(token.type)) + { + DeleteOutcome d; + d.kind = DeleteOutcome::Kind::TokenMismatch; + return d; + } + + if (mode == Mode::Native) + { + /// `removeObjectIfTokenMatches` maps onto `DeleteOutcome` one-to-one. `NOT_IMPLEMENTED` from a + /// backend that does not enforce conditional removal propagates — fail-closed by construction. + auto result = object_storage->removeObjectIfTokenMatches(StoredObject(key), token.value); + DeleteOutcome d; + d.created_delete_marker = result.created_delete_marker; + switch (result.outcome) + { + case ConditionalRemoveOutcome::Removed: + d.kind = DeleteOutcome::Kind::Deleted; + break; + case ConditionalRemoveOutcome::TokenMismatch: + d.kind = DeleteOutcome::Kind::TokenMismatch; + break; + case ConditionalRemoveOutcome::NotFound: + d.kind = DeleteOutcome::Kind::NotFound; + break; + } + return d; + } + + std::lock_guard lock(emu_mutex); + DeleteOutcome d; + if (!emuExists(key)) + { + d.kind = DeleteOutcome::Kind::NotFound; + return d; + } + if (!tokenMatches(emuObserveToken(key), token)) + { + d.kind = DeleteOutcome::Kind::TokenMismatch; + return d; + } + + object_storage->removeObjectIfExists(StoredObject(emuPath(key))); + /// Keep the deleted incarnation's last-minted etag around ONLY while a same-mtime-quantum + /// collision with an immediate recreate is still possible (emuMintToken) — once it is + /// comfortably old, erase it so `emu_token_state` does not grow for the lifetime of the backend + /// instance (codex-review-triage §3.18, Important #1). + if (auto it = emu_token_state.find(key); it != emu_token_state.end()) + { + const uint64_t now_ns = emuNowNs(); + if (etagComfortablyInThePast(it->second.first, now_ns)) + emu_token_state.erase(it); + else + emu_token_expiry.push_back(EmuTokenExpiry{now_ns, key, it->second}); + } + d.kind = DeleteOutcome::Kind::Deleted; + return d; +} + +PutResult ObjectStorageBackend::promoteStaged(const String & staging_key, const String & blob_key) +{ + if (mode != Mode::Native) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "ObjectStorageBackend::promoteStaged is Native-mode only (EmulatedSingleProcess has no " + "server-side conditional copy and is never selected for S3 staging)"); + + /// WRITE-ONCE conditional server-side copy staging -> blob (`If-None-Match:*` on the destination), + /// via `IObjectStorage::copyObjectConditional`. `created` ⇒ the destination ETag is the new + /// incarnation token; `!created` ⇒ the destination already existed = the "lost the race" 412 signal. + /// Counted with the same attempt/outcome counters as every other conditional write + /// (`finalizeConditionalWriteInstrumented`'s contract): the copy is a conditional + /// create attempt too, and it is initiated by the controlled content-addressed upload path — an + /// uncounted attempt would hide SDK-versus-controller retry accounting. + /// A resolved `!created` is counted `Unresolved` (the 412 does not prove who created the occupant), + /// mirroring the PUT paths. + recordConditionalWriteAttemptStarted(); + ConditionalCopyResult res; + try + { + res = object_storage->copyObjectConditional( + StoredObject(staging_key), StoredObject(blob_key), getReadSettings(), WriteSettings{}); + } + catch (const std::exception & e) + { + recordConditionalWriteOutcome(classifyConditionalWriteResult(e)); + throw; + } + recordConditionalWriteOutcome(res.created ? classifyConditionalWriteResult() : CasWriteOutcome::Unresolved); + if (!res.created) + return {PutOutcome::PreconditionFailed, {}}; + return {PutOutcome::Done, Token{res.dest_etag, native_token_type}}; +} + +Token ObjectStorageBackend::resurrect(ReadBuffer & payload, uint64_t payload_size, const String & blob_key, + const String & fresh_header) +{ + if (mode != Mode::Native) + { + /// EmulatedSingleProcess (local object storage): same unconditional semantics. The body is + /// materialized -- the emulated conditional ops are whole-`String` by design -- so resurrections + /// are SERIALIZED process-wide by their own mutex: the fan-out may run N resurrect tasks at + /// once, and without this the peak would be the SUM of the bodies. One at a time bounds the + /// peak to the largest single body, the same guarantee the byte-weighted admission's exclusive + /// arm used to give. A dedicated mutex, not `emu_mutex`: the drain may read through the same + /// store, and `emu_mutex` guards individual ops inside it. + static std::mutex emulated_resurrect_mutex; + std::lock_guard resurrect_lock(emulated_resurrect_mutex); + String body = fresh_header; + { + WriteBufferFromString out(body, AppendModeTag{}); + copyData(payload, out); + out.finalize(); + } + if (body.size() - fresh_header.size() != payload_size) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "resurrect: source yielded {} payload bytes for {}, declared {} -- nothing was published", + body.size() - fresh_header.size(), blob_key, payload_size); + std::lock_guard lock(emu_mutex); + return emuWrite(blob_key, body, /*meta=*/{}); + } + + /// Unconditional overwrite of the condemned body (plain WriteSettings — no If-Match/If-None-Match). + /// This is safe by three independent structural properties, not merely "no time to add a + /// precondition": (1) the key is content-addressed, so every incarnation ever written under it is + /// byte-identical in its PAYLOAD — an overwrite here rotates only the envelope/token; (2) an + /// adopted dependency token VALUE is never a promote gate, only `has_value()` is consulted + /// (tokenless-on-ref promote), so no consumer can observe or react to the specific bytes of the old + /// token; (3) the fresh-tagged `fresh_header` guarantees the resurrected incarnation's token differs + /// from the condemned one, so every already-queued exact-token GC delete of the condemned + /// incarnation mismatches and misses (`INV-NO-RETURN`). An `If-Match` on the condemned token would + /// only save a redundant re-upload on a lost race, never prevent data loss. + /// + /// Plain `WriteSettings` also means no forced single part: a conditional write on a + /// generation-token store is capped because GCS drops preconditions on multipart completion, and + /// this write carries none, so it may take the multipart path on every backend. The payload is + /// streamed from `payload` and never materialized — blob bodies have no size cap. + auto out = object_storage->writeObject( + StoredObject(blob_key), WriteMode::Rewrite, /*attributes=*/std::nullopt, DBMS_DEFAULT_BUFFER_SIZE, WriteSettings{}); + out->write(fresh_header.data(), fresh_header.size()); + const size_t before = out->count(); + copyData(payload, *out); + const size_t streamed = out->count() - before; + if (streamed != payload_size) + { + /// Abort BEFORE finalize: the incomplete multipart upload is discarded and nothing becomes + /// current. This is what keeps the unconditional write fail-closed against a source truncated + /// after hashing -- a post-write check would run only after the short body had displaced the + /// condemned incarnation. + out->cancel(); + throw Exception(ErrorCodes::CORRUPTED_DATA, + "resurrect: source yielded {} payload bytes for {}, declared {} -- upload aborted, nothing published", + streamed, blob_key, payload_size); + } + out->finalize(); + + /// The plain write does not reliably surface the destination ETag across dialects, so HEAD the + /// fresh incarnation to learn its token. + const auto hr = nativeHead(blob_key); + if (!hr) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "ObjectStorageBackend::resurrect: blob {} is absent immediately after the resurrect " + "re-upload — failing closed", blob_key); + return hr->token; +} + +ListPage ObjectStorageBackend::list(const String & prefix, const String & cursor, size_t limit) +{ + /// Use the lazy object-storage iterator instead of `listObjects(..., max_keys=0)`: the latter + /// materialized the whole prefix, then sliced client-side, so a paginated walk re-fetched the full + /// subtree for every page. The backend cursor is "last key returned" (exclusive on resume). + /// + /// Some backends ignore `start_after`; filtering `key <= cursor` keeps the contract correct there, + /// only losing the resume optimization. S3 honors `start_after` and avoids the hot-path re-scan. + if (limit == 0) + return {}; + + const String physical_prefix = (mode == Mode::EmulatedSingleProcess) ? emuPath(prefix) : prefix; + const String strip = (mode == Mode::EmulatedSingleProcess) ? emuPath("") : String{}; + if (mode == Mode::EmulatedSingleProcess) + { + RelativePathsWithMetadata children; + object_storage->listObjects(physical_prefix, children, /*max_keys=*/0); + + /// Hold emu_mutex across the whole scan: emuMintToken below reads/updates emu_token_state, the + /// same per-key state get/head/put*/delete* mutate under this lock (see the "caller holds + /// emu_mutex" contract on the private emu* helpers). + std::lock_guard lock(emu_mutex); + + std::vector all; + all.reserve(children.size()); + for (const auto & child : children) + { + if (!child->relative_path.starts_with(physical_prefix)) + continue; + ListedKey lk; + lk.key = child->relative_path.substr(strip.size()); + lk.size = child->metadata ? child->metadata->size_bytes : 0; + /// §3.18 №18: mint DIRECTLY as TokenType::Emulated — do NOT call tokenForList, which always + /// stamps native_token_type (ETag/Generation) regardless of mode and would surface a token + /// of the wrong dialect for every Emulated consumer (head/get mint Emulated). + if (child->metadata) + lk.token = emuMintToken(lk.key, child->metadata->etag, /*just_wrote=*/false); + all.push_back(std::move(lk)); + } + std::sort(all.begin(), all.end(), [](const ListedKey & a, const ListedKey & b) { return a.key < b.key; }); + + ListPage page; + auto all_it = cursor.empty() + ? std::lower_bound(all.begin(), all.end(), prefix, [](const ListedKey & a, const String & s) { return a.key < s; }) + : std::upper_bound(all.begin(), all.end(), cursor, [](const String & s, const ListedKey & a) { return s < a.key; }); + while (all_it != all.end() && page.keys.size() < limit) + { + page.keys.push_back(*all_it); + ++all_it; + } + if (!page.keys.empty() && all_it != all.end()) + page.next_cursor = page.keys.back().key; + return page; + } + + const std::optional start_after = cursor.empty() + ? std::nullopt + : std::optional(cursor); + + ListPage page; + auto it = object_storage->iterate(physical_prefix, /*max_keys=*/0, /*with_tags=*/false, start_after); + for (; it->isValid(); it->next()) + { + const auto child = it->current(); + if (!child->relative_path.starts_with(physical_prefix)) + continue; + + ListedKey lk; + lk.key = child->relative_path.substr(strip.size()); + if (!cursor.empty() && lk.key <= cursor) + continue; + + lk.size = child->metadata ? child->metadata->size_bytes : 0; + /// Surface the per-key incarnation token (matching what `head` would return, see above) so the + /// `supportsListTokens() == true` capability is honest. A listing without an etag leaves the + /// token unset, which GC discover treats as Read (fail closed). The supportsListTokens()+ + /// empty-etag gate now lives in tokenForList. + if (child->metadata) + lk.token = tokenForList(child->metadata->etag); + + if (page.keys.size() == limit) + { + page.next_cursor = page.keys.back().key; + break; + } + page.keys.push_back(std::move(lk)); + } + + return page; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h new file mode 100644 index 000000000000..f06bafa969c2 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h @@ -0,0 +1,249 @@ +#pragma once +#include +#include +#include +#include +#include + +#include "config.h" + +namespace DB::Cas +{ + +/// Fold and point GETs commonly read tiny bodies (about 3.7 KiB on the measured workload), while the +/// default `ReadBufferFromS3` allocation is about 1 MiB. If the caller already knows the object size, +/// use `ReadSettings::adjustBufferSize` to request a buffer of `known_size + slack`, without exceeding +/// the caller's configured default. A zero `known_size` means that the size is unknown and preserves +/// the supplied settings unchanged. +constexpr uint64_t CAS_FOLD_READ_SLACK_BYTES = 4096; +ReadSettings casSizedReadSettings(const ReadSettings & base, uint64_t known_size); + +#if USE_AWS_S3 +namespace detail +{ +/// Finalize a conditional write (the condition rode on the buffer's WriteSettings) and map a +/// precondition loss to an OUTCOME — anything else propagates. This is the classifier for the +/// typed `S3Exception` signal; exposed here for unit tests only — production callers go through +/// `ObjectStorageBackend`. See the definition for the exact matching rules. +PutOutcome finalizeConditionalWrite(WriteBuffer & buf); +} +#endif + +/// Production Backend over IObjectStorage. +/// +/// Native mode (S3-like): conditions ride the existing plumbing — WriteSettings +/// object_storage_write_if_none_match / object_storage_write_if_match (consumed by WriteBufferFromS3) +/// and IObjectStorage::removeObjectIfTokenMatches. Tokens are backend ETags from getObjectMetadata. +/// Trust is NEVER assumed: Cas::Probe validates enforcement per pool at open. +/// +/// EmulatedSingleProcess mode (LocalObjectStorage — tests and local development ONLY): the object +/// storage has no conditional ops, so this adapter provides EXACT token semantics itself with a +/// process-wide mutex and an in-memory per-key token MINTED FROM the object's own etag (mtime-ns on +/// LocalObjectStorage) — see emuMintToken. Every emulated token IS the object's current etag (not +/// merely "seeded" for pre-existing keys); this is what keeps token-exact semantics correct ACROSS a +/// process restart, which a plain in-process counter cannot do (codex-review-triage §3.18, 19c): a +/// counter restarts at 0 and can re-mint a value colliding with a persisted pre-restart delete token +/// for a completely different incarnation, while a resurrected body's mtime is always later. Semantics +/// otherwise hold within ONE process only — exactly what unit tests need. +class ObjectStorageBackend final : public Backend +{ +public: + /// Unhide the base convenience overloads (omitted Range/ObjectMeta/expected-token forms): the + /// overrides below would otherwise shadow them for callers holding a concrete backend type. + using Backend::get; + using Backend::getStream; + using Backend::putIfAbsent; + using Backend::putIfAbsentStream; + using Backend::putOverwrite; + using Backend::casPut; + + enum class Mode { Native, EmulatedSingleProcess }; + + /// Construct a backend over `object_storage`. Native mode uses the storage's conditional + /// operations and native token dialect; `EmulatedSingleProcess` serializes operations locally for + /// tests and local development. The generation-token store limit applies only to Native mode: + /// generation stores must use a single PUT because their multipart completion path does not enforce + /// the precondition. + ObjectStorageBackend(ObjectStoragePtr object_storage_, Mode mode_, uint64_t conditional_single_put_cap_ = 1ULL << 30); + + /// Read an object or return `nullopt` if it is absent. Native mode HEADs first so the returned + /// token identifies the incarnation whose bytes are read; a not-found race is also reported as + /// `nullopt`, while unrelated storage errors propagate. + std::optional get(const String & key, Range range) override; + /// Open a forward-only ranged stream for a write-once object. The stream is not materialized in + /// memory; mutable objects must use `get` because their contents may change while it is open. + std::optional getStream(const String & key, Range range) override; + /// Return the current size, attributes, and incarnation token, or an absent `HeadResult`. + HeadResult head(const String & key) override; + /// S3 ETags are content-derived and surfaced in list responses — TRUE for ETag-token Native + /// and EmulatedSingleProcess modes. FALSE on a generation-token store (GCS): the XML LIST + /// surfaces MD5-style ETags in the response BODY, which the conditional dialect's header-level + /// rewrite cannot map to generations. A list-derived token would therefore be an invalid + /// `If-Match` token; generation stores deliberately omit it and make GC re-read each shard. + /// Consumers already treat absent list tokens as Read/fail-closed (GC discover re-reads every + /// shard — a cost, not a correctness change). + bool supportsListTokens() const override { return native_token_type != TokenType::Generation; } + + /// Create `key` only if it is absent. On a precondition failure the object is untouched and the + /// result has no token; on success the token identifies the newly written incarnation. + PutResult putIfAbsent(const String & key, const String & bytes, const ObjectMeta & meta) override; + + /// Native mode: true streaming — bytes flow straight into the object storage's write buffer with + /// `If-None-Match: *` riding on the request. EmulatedSingleProcess mode: memory-buffered delegation + /// to putIfAbsent (acceptable: this mode exists for unit tests only). + WriteSinkPtr putIfAbsentStream(const String & key, const ObjectMeta & meta) override; + /// Replace `key` only when its current token exactly equals `expected`; a mismatch leaves the + /// existing incarnation untouched. Storage exceptions propagate instead of being reported as a + /// successful or failed precondition. + PutResult putOverwrite(const String & key, const String & bytes, const Token & expected, const ObjectMeta & meta) override; + /// Perform a compare-and-set: `expected == nullopt` means create-if-absent. A conflict leaves the + /// object untouched; a committed result carries the new incarnation token. + CasResult casPut(const String & key, const String & bytes, const std::optional & expected, const ObjectMeta & meta) override; + /// Remove only the incarnation matching `token`, preserving the object on a mismatch and exposing + /// whether the storage created a delete marker. + DeleteOutcome deleteExact(const String & key, const Token & token) override; + /// Return a page after `cursor`; the next cursor is the last returned key and is empty at the end. + ListPage list(const String & prefix, const String & cursor, size_t limit) override; + + /// `promoteStaged` (S3-native staging, Native mode only — EmulatedSingleProcess has no server-side + /// conditional copy and is never selected for S3 staging, so it throws `NOT_IMPLEMENTED` there): + /// WRITE-ONCE conditional copy via `IObjectStorage::copyObjectConditional`. + /// `resurrect` (every mode): prepends `fresh_header` and UNCONDITIONALLY writes + /// `[fresh_header][payload]` to `blob_key` (fresh tag ⇒ distinct ETag from the condemned + /// incarnation, INV-NO-RETURN), then a fresh HEAD for the ETag. Native streams with plain + /// `WriteSettings` — no forced single part, no size ceiling on any dialect; EmulatedSingleProcess + /// materializes and SERIALIZES resurrections process-wide, bounding the peak to one body. + PutResult promoteStaged(const String & staging_key, const String & blob_key) override; + Token resurrect(ReadBuffer & payload, uint64_t payload_size, const String & blob_key, const String & fresh_header) override; + + /// Pool-level precondition: on a Native, generation-dialect (GCS) backend, reject the pool if the + /// bucket has object versioning enabled — see Backend::checkPoolPreconditions. + void checkPoolPreconditions() override; + + /// Fail-closed precondition for writable Native mode: require that the object storage supports the + /// SingleAttempt retry profile (ObjectStorageRetryProfile), which disables transparent + /// conditional-write retries. Without it, an SDK retry could cross the mount lease boundary or turn + /// an uncertain result into a misleading precondition failure. A non-S3 object storage used for + /// test construction reports no support; this check is the mount-time gate. No-op for + /// `EmulatedSingleProcess`. + void checkConditionalWriteSingleAttemptSupport() override; + + /// See Backend::probeSentinelRaw. Native: a raw HEAD via `IObjectStorage::getObjectMetadata` (the + /// THROWING variant — unlike `tryGetObjectMetadata`/`nativeHead`, it never swallows the S3 error), + /// classified by S3 error code. EmulatedSingleProcess (Local): stats the configured container + /// directory (`emu_root`) first — `ContainerAbsent` if it is gone — then the key. + SentinelProbeResult probeSentinelRaw(const String & key) override; + + /// The token kind this backend's object storage mints: TokenType::ETag for AWS-compatible + /// stores, TokenType::Generation when the storage runs the GCS conditional dialect (the + /// generation rides the ETag plumbing; the VALUE stays opaque either way). + TokenType nativeTokenType() const { return native_token_type; } + void setNativeTokenTypeForTest(TokenType t) { native_token_type = t; } + + /// ---- Token policy (single source of truth; see the .cpp) ---- + /// Mint the incarnation token for a key we just HEAD'd or wrote: the object ETag/generation + /// string carried under this backend's native dialect (native_token_type). + Token tokenForHead(const String & etag) const + { + return Token{etag, native_token_type}; + } + + /// The token to surface for a LISTED key: present iff this backend surfaces per-key list tokens + /// (supportsListTokens — FALSE on a generation store, where a list-derived token is a poisoned + /// If-Match) AND the listing carried a non-empty etag. Matches what tokenForHead would return. + std::optional tokenForList(const String & etag) const + { + if (!supportsListTokens() || etag.empty()) + return std::nullopt; + return Token{etag, native_token_type}; + } + + /// Whether an observed incarnation token satisfies an expected one: exact identity (value AND + /// type). Every conditional compare in this backend goes through here. + static bool tokenMatches(const Token & observed, const Token & expected) + { + return observed == expected; + } + + /// Build settings shared by every Native conditional write. They skip the racy post-upload + /// existence/size check, force single-part uploads for generation-token stores, and select the + /// SingleAttempt object-storage retry profile. + WriteSettings conditionalWriteSettings() const; + WriteSettings conditionalWriteSettingsForTest() const { return conditionalWriteSettings(); } + /// Override the emulated backend's wall clock for deterministic expiry tests. + void setEmuNowNsForTest(uint64_t now_ns); + /// Return the guarded per-key token-state size for expiry tests. + size_t emuTokenStateSizeForTest() const; + +private: + const ObjectStoragePtr object_storage; + const Mode mode; + TokenType native_token_type = TokenType::ETag; + /// GCS single-PUT budget for conditional writes (generation-token stores only); see ctor. + const uint64_t conditional_single_put_cap; + + /// EmulatedSingleProcess state: per-key {etag, disambiguator} — see emuMintToken. A successfully + /// deleted entry is retained only while its etag is recent enough that an immediate recreate could + /// land in the same mtime quantum. `deleteExact` erases already-old entries immediately and queues + /// recent ones for the bounded lazy sweep in emuMintToken, so a key need not be revisited to expire. + /// The queue records the exact state generation deleted; a subsequent re-mint makes the record + /// obsolete rather than allowing it to erase the live incarnation's token state. + mutable std::mutex emu_mutex; + std::map> emu_token_state; + struct EmuTokenExpiry + { + uint64_t queued_at_ns; + String key; + std::pair token_state; + }; + std::deque emu_token_expiry; + uint64_t emu_now_ns_for_test = 0; + /// Fallback nonce for the (anomalous) case where the object storage reports an EMPTY etag: mints a + /// fresh, unpersisted value each time — never worse than the old counter for that case, but never + /// masquerading as a real etag-derived identity either. + uint64_t emu_seq = 0; + + /// Look up Native metadata and convert the storage ETag or generation to this backend's token. + std::optional nativeHead(const String & key); + /// Write a body with the condition already encoded in `ws`, finalize it, classify a lost + /// precondition, and return the new token when the write succeeds. + PutResult nativeConditionalPut(const String & key, const String & bytes, const WriteSettings & ws, const ObjectMeta & meta); + + /// §3.18 №19 hardening: whether `t` is the dialect this backend itself mints (native_token_type + /// for Native mode, always TokenType::Emulated for EmulatedSingleProcess). Every conditional + /// mutation checks this BEFORE touching the wire (Native forwards only Token::value as the + /// If-Match/removeObjectIfTokenMatches argument, blind to Token::type) or comparing values + /// (Emulated) — a foreign-dialect token is rejected locally rather than trusted to the remote + /// backend, or to a value-space that was never designed to discriminate it. + bool mintingTypeMatches(TokenType t) const { return t == (mode == Mode::Native ? native_token_type : TokenType::Emulated); } + + /// ---- Emulated helpers (caller holds emu_mutex) ---- + /// + /// EmulatedSingleProcess resolves logical keys under the object storage's common key prefix (its + /// root), so each backend instance is physically isolated — a real object store likewise scopes keys + /// to a bucket/prefix. The token map is keyed by the LOGICAL key (prefix-independent). + const String emu_root; /// object_storage->getCommonKeyPrefix() captured at construction + String emuPath(const String & key) const; /// logical key -> physical object-storage path + + /// The caller holds `emu_mutex` for all five helpers below, preserving the exists/read and + /// observe/write checks as one process-local operation. + bool emuExists(const String & key) const; + String emuRead(const String & key, Range range) const; + /// Write a body as the new incarnation of `key` and return its freshly minted token (the + /// object's own post-write etag — see emuMintToken). + Token emuWrite(const String & key, const String & bytes, const ObjectMeta & meta); + /// Return the current emulated token for a key we just read/HEAD'd, reflecting its on-disk etag — + /// does NOT advance the same-etag disambiguator (that only applies to a just-completed write). + Token emuObserveToken(const String & key); + uint64_t emuNowNs() const; + /// Examine a fixed number of oldest deleted-state records, expiring only an exact current match. + void emuPruneTokenState(uint64_t now_ns); + /// Single source of truth for minting an emulated token from an observed `etag`: the wire value IS + /// the etag while it is the first thing minted for `key` at that etag, or `etag#N` once a SAME-etag + /// rewrite forces a disambiguator (`just_wrote` — see the mtime-quantum note in emu_token_state's + /// declaration and codex-review-triage §3.18 19c step 4). An empty `etag` (the storage could not + /// report one) falls back to a fresh, UNPERSISTED monotonic value from emu_seq. + Token emuMintToken(const String & key, const String & etag, bool just_wrote); +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp new file mode 100644 index 000000000000..6d2b31a74fb5 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp @@ -0,0 +1,320 @@ +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::Cas +{ + +void runCapabilityProbe(Backend & backend, const String & probe_prefix) +{ + // Probe key used for the primary battery steps. + // Sub-directory style ("probe_prefix/token") ensures that list(probe_prefix, …) works for both the + // in-memory backend (prefix match) and the LocalObjectStorage backend (directory listing). + const String key = probe_prefix + "/token"; + // Probe key used for the casPut chain. + const String cas_key = probe_prefix + "/cas"; + + // Best-effort cleanup — runs at function exit regardless of outcome. + // We capture the keys we need to clean up. + auto cleanup = [&]() noexcept + { + // Skip the delete when HEAD says the key is already gone (the happy path: step 8 deleted + // it). A deleteExact with the absent HeadResult's EMPTY token is a malformed conditional + // op — AWS S3 answers 400 InvalidArgument ("If-Match cannot be empty"), which lands as a + // scary AWSClient log line on every mount even though the catch swallows it. + for (const auto & k : {key, cas_key}) + { + try + { + const auto h = backend.head(k); + if (h.exists) + backend.deleteExact(k, h.token); + } + catch (...) {} /// NOLINT(bugprone-empty-catch) + } + }; + + try + { + // ---- Step 0: store-level preconditions (backend-specific; throws = mount refused). ---- + backend.checkPoolPreconditions(); + + // ---- Step 0b: conditional writes must use one underlying HTTP attempt. Transparent SDK + // retries can outlive the writer's mount lease and hide whether a conditional operation + // committed; CAS retries must instead be explicit and state-aware. Throws = mount refused. + // Keep this separate from Step 0 so each precondition remains independently unit-testable. ---- + backend.checkConditionalWriteSingleAttemptSupport(); + + // ---- Step 1: putIfAbsent fresh → Done; read-after-write returns the bytes. ---- + Token t1; + { + const auto res = backend.putIfAbsent(key, "probe-v1"); + t1 = res.token; + if (res.outcome != PutOutcome::Done) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: putIfAbsent on a fresh key returned PreconditionFailed — backend is unexpectedly occupied or broken"); + } + { + const auto g = backend.get(key); + if (!g.has_value() || g->bytes != "probe-v1") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: read-after-write failed — putIfAbsent succeeded but the object is not readable"); + } + + // ---- Step 2: putIfAbsent same key → PreconditionFailed; bytes intact. ---- + { + const auto outcome = backend.putIfAbsent(key, "should-not-land").outcome; + if (outcome != PutOutcome::PreconditionFailed) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: putIfAbsent on an existing key was not rejected (PreconditionFailed expected) — " + "backend does not enforce conditional create"); + const auto g = backend.get(key); + if (!g.has_value() || g->bytes != "probe-v1") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: putIfAbsent conflict was 'reported' but the original bytes were clobbered — " + "backend does not enforce conditional create"); + } + + // ---- Step 3: putOverwrite wrong token → PreconditionFailed; bytes intact. ---- + { + /// Wrong-token values are NUMERIC on purpose: a generation-dialect backend (GCS) + /// validates the If-Match FORMAT client-side and throws on a non-numeric value (an + /// ETag-kind token leaking into a generation dialect) — the probe's synthetic wrong + /// tokens must be format-valid for EVERY token kind, merely guaranteed-wrong. A huge + /// numeric is a wrong ETag on AWS (412), a wrong generation on GCS (412), and a wrong + /// sequence on the emulated backends (TokenMismatch). + /// + /// The TYPE must be the LIVE dialect (t1.type, just observed from this same backend), + /// never a hardcoded TokenType::Emulated: a backend that mints a different dialect + /// (e.g. Native/ETag) rejects a foreign-dialect token locally, before the wrong VALUE + /// ever reaches the wire — which would make this check pass vacuously against a + /// non-enforcing store instead of proving enforcement (codex-review-triage §3.18, + /// Critical: the №19 local dialect guard must not defeat this probe). + Token wrong_token{"900000000000000001", t1.type}; + const auto outcome = backend.putOverwrite(key, "clobbered", wrong_token).outcome; + if (outcome != PutOutcome::PreconditionFailed) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: putOverwrite with a wrong token was not rejected (PreconditionFailed expected) — " + "backend does not enforce conditional overwrite"); + const auto g = backend.get(key); + if (!g.has_value() || g->bytes != "probe-v1") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: putOverwrite with wrong token was 'rejected' but the original bytes were clobbered"); + } + + // ---- Step 4: putOverwrite correct token → Done; bytes replaced; token changed. ---- + Token t2; + { + const auto res = backend.putOverwrite(key, "probe-v2", t1); + t2 = res.token; + if (res.outcome != PutOutcome::Done) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: putOverwrite with the correct token was rejected — backend does not accept valid overwrite"); + if (t2 == t1) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: putOverwrite succeeded but did not mint a new token — tokens must change on every write"); + const auto g = backend.get(key); + if (!g.has_value() || g->bytes != "probe-v2") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: putOverwrite succeeded but the new bytes are not readable"); + } + + // ---- Step 5: casPut chain. ---- + // 5a: create-if-absent (nullopt expected). + Token ct1; + { + const auto res = backend.casPut(cas_key, "cas-s1", std::nullopt); + ct1 = res.token; + if (res.outcome != CasOutcome::Committed) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: casPut create-if-absent (nullopt expected) was not committed — " + "backend does not support CAS create-if-absent"); + } + // 5b: conflict on existing (nullopt expected, but key exists). + { + const auto outcome = backend.casPut(cas_key, "cas-s1x", std::nullopt).outcome; + if (outcome != CasOutcome::Conflict) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: casPut with nullopt expected against an existing key was not Conflict — " + "backend does not enforce create-if-absent semantics on casPut"); + } + // 5c: conflict on stale token. + { + Token stale{"900000000000000002", ct1.type}; /// numeric + live dialect: see step 3 + const auto outcome = backend.casPut(cas_key, "cas-s1y", stale).outcome; + if (outcome != CasOutcome::Conflict) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: casPut with a stale token was not Conflict — " + "backend does not enforce token-exact CAS"); + } + // Bytes must still be the original. + { + const auto g = backend.get(cas_key); + if (!g.has_value() || g->bytes != "cas-s1") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: casPut conflicts were reported but the original bytes were altered"); + } + // 5d: commit on current token. + { + const auto res = backend.casPut(cas_key, "cas-s2", ct1); + if (res.outcome != CasOutcome::Committed) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: casPut with the current token was not committed — " + "backend does not honor casPut with matching token"); + const auto g = backend.get(cas_key); + if (!g.has_value() || g->bytes != "cas-s2") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: casPut committed but new bytes are not readable"); + } + + // ---- Step 6: deleteExact wrong token → TokenMismatch AND the object still readable. ---- + { + Token wrong_token{"900000000000000003", t2.type}; /// numeric + live dialect: see step 3 + const auto d = backend.deleteExact(key, wrong_token); + if (d.kind != DeleteOutcome::Kind::TokenMismatch) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: deleteExact with a wrong token was not TokenMismatch — " + "delete with mismatching token was honored — backend does not enforce conditional deletes"); + const auto g = backend.get(key); + if (!g.has_value()) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: deleteExact with a wrong token was rejected (correctly) but the object was deleted anyway — " + "backend does not enforce conditional deletes"); + } + + // ---- Step 7: list(probe_prefix) contains the probe key (list-after-write). ---- + { + const auto page = backend.list(probe_prefix, "", 100); + bool found = false; + for (const auto & listed : page.keys) + { + if (listed.key == key) + { + found = true; + break; + } + } + if (!found) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: list-after-write failed — the probe key '{}' is not visible in the listing under prefix '{}'", + key, probe_prefix); + } + + // ---- Step 8: deleteExact correct token → Deleted; object gone; no delete marker; + // list no longer contains the key. ---- + { + const auto d = backend.deleteExact(key, t2); + if (d.kind != DeleteOutcome::Kind::Deleted) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: deleteExact with the correct token was not Deleted — backend rejected a valid token-exact delete"); + if (d.created_delete_marker) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: deleteExact succeeded but created a versioning delete marker — the bucket has " + "object VERSIONING enabled, and a content-addressed pool cannot run on a versioned bucket: " + "every GC delete would archive a noncurrent version instead of reclaiming storage (the bucket " + "grows forever), and the constantly-rewritten ref objects would pile up versions on every " + "commit. This is NOT ignorable and has no override. Use a bucket where versioning was NEVER " + "enabled — note that merely SUSPENDING versioning is not enough (deletes on a " + "versioning-suspended bucket still mint delete markers, so this probe will refuse again)"); + const auto g = backend.get(key); + if (g.has_value()) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: deleteExact succeeded (Deleted) but the object is still readable — backend delete is not effective"); + // List-after-delete. + const auto page = backend.list(probe_prefix, "", 100); + for (const auto & listed : page.keys) + { + if (listed.key == key) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: list-after-delete failed — the deleted probe key '{}' is still visible in the listing under prefix '{}'", + key, probe_prefix); + } + } + + // ---- Step 9: cleanup (best-effort; also deletes cas_key). ---- + // cas_key is still alive — clean it up via its current token. + { + const auto h = backend.head(cas_key); + if (h.exists) + backend.deleteExact(cas_key, h.token); + } + } + catch (...) + { + // Best-effort cleanup on failure path before re-throwing. + cleanup(); + throw; + } + + // Normal-exit cleanup (cas_key was cleaned inside the try; key was deleted in step 8). + // Call cleanup anyway to handle any partial state edge cases — it is a no-op if keys are gone. + cleanup(); +} + +bool probeConditionalCopy(IObjectStorage & object_storage, const String & probe_prefix) +{ + const String src_key = probe_prefix + "/src"; + const String dst_key = probe_prefix + "/dst"; + + // Best-effort cleanup — runs unconditionally, on every exit path (success, non-enforcing + // result, or exception), mirroring runCapabilityProbe's cleanup lambda. + auto cleanup = [&]() noexcept + { + for (const auto & key : {src_key, dst_key}) + { + try + { + object_storage.removeObjectIfExists(StoredObject(key)); + } + catch (...) {} /// NOLINT(bugprone-empty-catch) + } + }; + + try + { + // ---- Step 1: write a tiny throwaway source object. ---- + { + auto buf = object_storage.writeObject(StoredObject(src_key), WriteMode::Rewrite); + static constexpr char payload[] = "cas-conditional-copy-probe"; + buf->write(payload, sizeof(payload) - 1); + buf->finalize(); + } + + // ---- Step 2: conditional copy to a FRESH destination -> must be created. ---- + const auto first = object_storage.copyObjectConditional( + StoredObject(src_key), StoredObject(dst_key), ReadSettings{}, WriteSettings{}); + if (!first.created) + { + // The backend refused to create a fresh destination — broken or misbehaving in a + // different way than "does not enforce the precondition". Either way, fail closed. + cleanup(); + return false; + } + + // ---- Step 3: the SAME conditional copy onto the now-existing destination -> must be + // REJECTED (created == false). A backend that reports created == true here + // silently overwrote the destination — it does NOT enforce If-None-Match, and + // the S3-native staging promote path would be unsafe on it (see CasProbe.h). ---- + const auto second = object_storage.copyObjectConditional( + StoredObject(src_key), StoredObject(dst_key), ReadSettings{}, WriteSettings{}); + + cleanup(); + return !second.created; + } + catch (...) + { + // Any exception (including the default NOT_IMPLEMENTED thrown by a backend that does not + // override copyObjectConditional) is fail-closed: conditional copy is NOT supported. + cleanup(); + return false; + } +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.h new file mode 100644 index 000000000000..37313482be69 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.h @@ -0,0 +1,62 @@ +#pragma once +#include +#include +#include + +namespace DB::Cas +{ + +/// Run the capability battery against `backend`, using throwaway keys under `probe_prefix`. +/// +/// The probe validates the backend preconditions required by a writable content-addressed pool: +/// 1. Store-level safety checks pass, including the requirement that conditional writes use one +/// underlying HTTP attempt. Hidden SDK retries can outlive the writer's mount lease and obscure +/// whether a conditional operation committed; retries must therefore be explicit CAS state-machine +/// transitions rather than transparent client behavior. +/// 2. Conditional-create and conditional-overwrite are enforced (`putIfAbsent` prevents overwrites, +/// and `putOverwrite` rejects a wrong-token update). +/// 3. `casPut` supports create-if-absent, conflict-on-existing, conflict-on-stale, and commit-on-current. +/// 4. Conditional-delete is enforced (`deleteExact` with a wrong token is rejected and the object survives). +/// 5. Listing reflects both creation and deletion of a probe object. +/// 6. Successful deletion does not create a versioning delete marker. A content-addressed pool cannot +/// reclaim storage correctly from a versioned bucket: garbage-collection deletes would archive old +/// versions instead of removing objects, and repeated ref updates would accumulate versions. +/// +/// On any failed check, throws a DB::Exception(ErrorCodes::NOT_IMPLEMENTED) with a message naming the +/// specific failed check. This is fail-closed: a backend that does not pass the battery MUST NOT be +/// used to coordinate a content-addressed pool. +/// +/// Cleanup of probe keys is best-effort and runs unconditionally: after the battery completes, or on +/// the failure path immediately before the check-failing exception is rethrown. Cleanup itself suppresses +/// exceptions so that it cannot hide the capability-check failure. +void runCapabilityProbe(Backend & backend, const String & probe_prefix); + +/// Probe whether `object_storage` ENFORCES a write-once conditional server-side copy +/// (`IObjectStorage::copyObjectConditional`, `If-None-Match: *`) — an OPTIONAL capability, unlike +/// the mandatory battery above (`runCapabilityProbe`). It is meaningful only for a disk configured +/// with `staging_backend=s3`. S3-native staging promotes a temporary object into a content-addressed +/// blob with this copy; if the destination precondition is ignored, the copy can silently overwrite a +/// live blob. Such a backend is unsafe for S3-native staging, so the metadata layer must fall back to +/// local staging rather than refuse to mount. +/// +/// Goes directly through `IObjectStorage`, not through `Backend`/`CasProbe`'s battery — this keeps +/// the probe decoupled from the `Backend`'s content-addressed operations. The metadata layer uses the +/// result only to decide whether the optional S3-native staging path is safe to enable. +/// +/// Writes a tiny throwaway object at `/src`, conditionally copies it to +/// `/dst` (expects `created == true` — a fresh destination), then repeats the SAME +/// conditional copy onto the now-existing `dst` (expects `created == false` — the destination must +/// be REJECTED, proving the backend enforces `If-None-Match`). Returns `true` only when both +/// expectations hold. +/// +/// Fail-close: ANY exception (including the default `NOT_IMPLEMENTED` a backend throws when it does +/// not override `copyObjectConditional` at all) OR a non-enforcing result (the second copy also +/// reports `created == true`, i.e. the backend silently overwrote the destination) returns `false`. +/// This function never throws — the caller treats `false` as "fall back to local staging", never as +/// a mount failure. +/// +/// Cleanup of the probe objects (`src`, `dst`) is best-effort and runs unconditionally on normal and +/// exceptional exits, mirroring `runCapabilityProbe`. +bool probeConditionalCopy(IObjectStorage & object_storage, const String & probe_prefix); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequestControl.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequestControl.cpp new file mode 100644 index 000000000000..e8f9ef24ab4b --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequestControl.cpp @@ -0,0 +1,745 @@ +#include + +#include +#include +#include +#include + +#include "config.h" + +#if USE_AWS_S3 +#include +#endif + +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event CASConditionalWriteAttempts; + extern const Event CASConditionalWriteCommitted; + extern const Event CASConditionalWriteDefiniteFailure; + extern const Event CASConditionalWriteUnresolved; + extern const Event CASConditionalWriteFenceLostPostWrite; +} + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int CORRUPTED_DATA; + extern const int LOGICAL_ERROR; + extern const int NETWORK_ERROR; + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::Cas +{ + +CasWriteOutcome classifyConditionalWriteResult([[maybe_unused]] const std::exception & e) +{ +#if USE_AWS_S3 + /// `PreconditionFailed`/`NoSuchKey` (a lost If-None-Match/If-Match — see + /// ObjectStorageBackend::finalizeConditionalWrite for the exact matching), any 5xx + /// (InternalError/ServiceUnavailable/SlowDown/RequestTimeout), and any S3 error this function does + /// not recognize all fall through to the fail-safe default below: Unresolved. Only the WHITELIST + /// below proves the request was never applied. + if (const auto * s3e = dynamic_cast(&e)) + { + if (S3::isMalformedRequestError(*s3e) || S3::isEntityTooLargeError(*s3e) || S3::isAccessDeniedError(*s3e)) + return CasWriteOutcome::DefiniteFailure; + } +#endif + /// Poco::Net::NetException (connection loss) / Poco::TimeoutException (client-side timeout) and + /// every other error type: the request's fate is unproven — fail toward "resolve before + /// reissuing, never toward a false + /// DefiniteFailure. + return CasWriteOutcome::Unresolved; +} + +void recordConditionalWriteAttemptStarted() +{ + ProfileEvents::increment(ProfileEvents::CASConditionalWriteAttempts); +} + +void recordConditionalWriteOutcome(CasWriteOutcome outcome) +{ + switch (outcome) + { + case CasWriteOutcome::Committed: + ProfileEvents::increment(ProfileEvents::CASConditionalWriteCommitted); + return; + case CasWriteOutcome::DefiniteFailure: + ProfileEvents::increment(ProfileEvents::CASConditionalWriteDefiniteFailure); + return; + case CasWriteOutcome::Unresolved: + ProfileEvents::increment(ProfileEvents::CASConditionalWriteUnresolved); + return; + } +} + +namespace +{ + +uint64_t steadyClockNowMs() +{ + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count()); +} + +/// The default inter-attempt backoff sleep. NOT a race-fix sleep: it is deliberate, bounded, +/// fence-gated pacing of reissues toward a recovering object store, and it is injectable so tests +/// never wait on it. +void threadSleepMs(uint64_t ms) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); +} + +/// Deterministic caller/local bugs the create retry loop must surface immediately: +/// reissuing only replays the same failure — up to ~12 minutes of budget × putBlob's outer loop at +/// the defaults — and buries the root cause behind a retryable ABORTED. The set: +/// LOGICAL_ERROR — a local invariant violation (e.g. uploadFromSource's source-size check; pinned +/// by `CasPartWriteTxn.PutBlobWrongSizeFailsClosed`, which caught exactly this class) +/// NOT_IMPLEMENTED — a mode/capability guard (e.g. `promoteStaged` on a backend without a native +/// conditional server-side copy) — a deterministic configuration bug +/// BAD_ARGUMENTS — a deterministic encode/argument rejection (e.g. BAD_ARGUMENTS escaping +/// buildHeader's second, intended_ref-less encode) +/// CORRUPTED_DATA — integrity failure; retrying re-reads/re-streams the same bad bytes (the same +/// fail-fast rule the driver-side correctness markers enforce) +/// Fail-safe either way: a propagated exception is never a false Committed. +bool isDeterministicLocalFailure(int code) +{ + return code == ErrorCodes::LOGICAL_ERROR || code == ErrorCodes::NOT_IMPLEMENTED + || code == ErrorCodes::BAD_ARGUMENTS || code == ErrorCodes::CORRUPTED_DATA; +} + +} + +void validateCasRequestBudget(const CasRequestBudget & budget, uint64_t mount_lease_ttl_ms, uint64_t mount_renew_period_ms) +{ + if (budget.max_attempts < 1) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "CAS request budget rejected: max_attempts must be at least 1 (got {}) — zero would let " + "putIfAbsentControlled return Unresolved without ever sending an attempt.", + budget.max_attempts); + + /// Overflow-safe: `attempt_timeout_ms + lease_safety_margin_ms` could wrap uint64 for absurd config + /// values, which would make the sum spuriously small and the inequality below pass when it should + /// fail closed. Compare via subtraction against the (unsigned, so already non-negative) TTL instead + /// of computing the sum directly. + if (!(budget.attempt_timeout_ms < mount_lease_ttl_ms + && budget.lease_safety_margin_ms < mount_lease_ttl_ms - budget.attempt_timeout_ms)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "CAS request budget rejected: attempt_timeout_ms ({}) + lease_safety_margin_ms ({}) must be " + "strictly less than the mount lease TTL ({} ms). A writable mount refuses to open with " + "this budget.", + budget.attempt_timeout_ms, budget.lease_safety_margin_ms, mount_lease_ttl_ms); + /// STRICTLY less, and the strictness is the load-bearing half. `attempt_timeout_ms > + /// operation_deadline_ms` is the obvious error — a single attempt cannot outlast the logical + /// operation it belongs to. EQUALITY is the subtle one, and it is worse than useless: the deadline + /// is captured as `now + operation_deadline_ms` and every pre-send gate below asks + /// `now + attempt_timeout_ms > deadline_ms`, so equal values collapse that to `now_2 > now_1` and + /// ONE elapsed millisecond between the two clock reads refuses the operation having sent NOTHING. + /// The resulting behaviour is "mostly works, occasionally refuses with nothing sent", decided by + /// the scheduler rather than by the budget — exactly the flakiness this validation exists to catch, + /// and observed three times in tests before it was forbidden. A caller that wants one attempt says + /// `max_attempts = 1`; the equality adds only the race. + if (!(budget.attempt_timeout_ms < budget.operation_deadline_ms)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "CAS request budget rejected: attempt_timeout_ms ({}) must be strictly less than " + "operation_deadline_ms ({}) — equality turns the pre-send gate into a wall-clock race that " + "refuses after a single elapsed tick, having sent nothing. Use max_attempts to bound the " + "number of attempts.", + budget.attempt_timeout_ms, budget.operation_deadline_ms); + if (!(budget.retry_initial_backoff_ms <= budget.retry_max_backoff_ms)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "CAS request budget rejected: retry_initial_backoff_ms ({}) must not exceed " + "retry_max_backoff_ms ({}) — the capped-exponential backoff cap cannot sit below its own " + "starting value. Set both to 0 to disable inter-attempt backoff.", + budget.retry_initial_backoff_ms, budget.retry_max_backoff_ms); + + LOG_INFO(getLogger("CasRequestControl"), + "CAS request budget in effect: attempt_timeout_ms={} operation_deadline_ms={} max_attempts={} " + "lease_safety_margin_ms={} retry_initial_backoff_ms={} retry_max_backoff_ms={} " + "(mount_lease_ttl_ms={} mount_renew_period_ms={})", + budget.attempt_timeout_ms, budget.operation_deadline_ms, budget.max_attempts, + budget.lease_safety_margin_ms, budget.retry_initial_backoff_ms, budget.retry_max_backoff_ms, + mount_lease_ttl_ms, mount_renew_period_ms); +} + +namespace +{ +/// Shared by both public entry points below so the log line and the exception's message text can +/// never drift apart. Rate-limited (not per-distinct-`why` -- `LogSeriesLimiter` keys on the LOGGER +/// NAME only, so under a sustained outage where `why` keeps changing slightly, only the first message +/// in each window prints; this is the intended throttle, not a bug). Warning-level visibility is +/// intentional: this condition is expected to self-heal +/// (the caller retries), but an operator watching CAS logs directly should see it without having to +/// know to look at system.replication_queue. +void logCasWriteRetryLater(const String & why) +{ + LogSeriesLimiter log(getLogger("CasWriteRetryLater"), /*allowed_count=*/1, /*interval_s=*/30); + LOG_WARNING(log, "CAS write could not be committed ({}); retrying later", why); +} +} + +[[noreturn]] void throwCasWriteRetryLater(const String & why) +{ + logCasWriteRetryLater(why); + throw Exception(ErrorCodes::NETWORK_ERROR, "CAS write could not be committed ({}); retrying later", why); +} + +std::exception_ptr makeCasWriteRetryLaterExceptionPtr(const String & why) +{ + logCasWriteRetryLater(why); + return std::make_exception_ptr( + Exception(ErrorCodes::NETWORK_ERROR, "CAS write could not be committed ({}); retrying later", why)); +} + +[[noreturn]] void throwCasTransientUnavailable(const String & subject, const String & condition) +{ + /// The code is coarse (it shares a `system.errors` row with socket failures), so the MESSAGE must + /// carry the whole truth: which CA condition refused, and that the refusal is a state rather than + /// damage. Consumers key on the code; operators read this line. + /// + /// The shared suffix carries ONLY the classification, because that is the one claim true at every + /// site: retry-later is right even where the condition may turn out terminal, since the next attempt + /// re-decides against fresh state. Any promise about HOW the condition clears belongs in `condition`, + /// where the site that can actually prove it makes it -- `checkFenceOrThrow` provably cannot. + throw Exception(ErrorCodes::NETWORK_ERROR, + "{} -- {}; TRANSIENT unavailability, not damage", subject, condition); +} + +CasRequestController::CasRequestController(BackendPtr backend_, CasRequestBudget budget_, std::function now_ms_, + std::function sleep_ms_) + : backend(std::move(backend_)) + , budget(budget_) + , now_ms(now_ms_ ? std::move(now_ms_) : std::function(steadyClockNowMs)) + , sleep_ms(sleep_ms_ ? std::move(sleep_ms_) : std::function(threadSleepMs)) +{ +} + +void CasRequestController::setSleepFnForTest(std::function sleep_ms_) +{ + sleep_ms = sleep_ms_ ? std::move(sleep_ms_) : std::function(threadSleepMs); +} + +uint64_t CasRequestController::backoffBeforeAttempt(uint32_t next_attempt) const +{ + const uint64_t initial = budget.retry_initial_backoff_ms; + const uint64_t cap = budget.retry_max_backoff_ms; + if (initial == 0 || next_attempt < 2) + return 0; + /// Saturating `initial << doublings`: `initial > cap >> doublings` implies the unshifted product + /// already exceeds the cap, so return the cap without ever computing an overflowing shift. + const uint32_t doublings = next_attempt - 2; + if (doublings >= 63 || initial > (cap >> doublings)) + return cap; + return std::min(initial << doublings, cap); +} + +bool CasRequestController::pauseBeforeReissue(uint32_t completed_attempt, uint64_t deadline_ms, + const std::function & fence_ok, CasUnresolvedReason * out_reason) +{ + /// Fence BEFORE the sleep (the pre-attempt fence rule applies to the whole loop, not just the + /// attempt): a fence lost mid-backoff aborts the operation instantly — sleeping first would keep a + /// fenced writer alive for up to a full backoff cap after it lost its right to write. + if (!fence_ok()) + { + if (out_reason) + *out_reason = CasUnresolvedReason::FenceLostMidWay; + return false; + } + const uint64_t backoff = backoffBeforeAttempt(completed_attempt + 1); + if (backoff == 0) + return true; + /// Never serve a sleep the operation cannot afford: if the backoff plus one more attempt would + /// cross the operation deadline, give up NOW instead of sleeping into a guaranteed Unresolved. + if (now_ms() + backoff + budget.attempt_timeout_ms > deadline_ms) + { + if (out_reason) + *out_reason = CasUnresolvedReason::DeadlineMidWay; + return false; + } + sleep_ms(backoff); + return true; +} + +CasWriteOutcome CasRequestController::resolveByExactGet(std::string_view key, std::string_view expected_bytes, + Token * out_token) +{ + const String key_s{key}; + std::optional got; + try + { + got = backend->get(key_s); + } + catch (const std::exception &) + { + /// The GET itself failed (network, auth, ...): the object's identity cannot be proven either + /// way — an unresolved read leaves this Unresolved, exactly like an absent read. + return CasWriteOutcome::Unresolved; + } + + if (!got) + return CasWriteOutcome::Unresolved; /// absent -> another attempt may still be legal + + if (got->bytes == expected_bytes) + { + if (out_token) + *out_token = got->token; + return CasWriteOutcome::Committed; /// identical deterministic bytes -> the earlier attempt DID commit + } + + /// A DIFFERENT valid object at the exact key this create intended: a real conflict, not a retryable + /// ambiguity. Fail closed rather than silently treating it as Unresolved/DefiniteFailure. + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CasRequestController: exact-key resolution at '{}' observed a DIFFERENT object than the one " + "this attempt intended to create — a real conflict, not a retryable ambiguity", key_s); +} + +CasWriteOutcome CasRequestController::putIfAbsentControlled( + std::string_view key, std::string_view bytes, const std::function & fence_ok, Token * out_token, + CasUnresolvedReason * out_reason) +{ + const String key_s{key}; + const String bytes_s{bytes}; + const uint64_t deadline_ms = now_ms() + budget.operation_deadline_ms; + /// Diagnostic bookkeeping only -- nothing below branches on it (finding #37 defect 3). + uint32_t attempts_sent = 0; + /// Does an EARLIER attempt of THIS call remain unresolved -- sent, and neither proven applied nor + /// proven refused? Set on the one path that produces exactly that state: an attempt whose outcome + /// was ambiguous and whose exact-key resolve came back absent or unreadable. The request may have + /// been received; an absent read now proves nothing about what materializes later, which is the + /// whole reason the reissue loop exists. This is NOT diagnostic: it decides the CALL's verdict at + /// the DefiniteFailure arm below. A pre-attempt gate refusal never sets it -- those return without + /// sending, so they leave nothing that could land. + bool earlier_attempt_unresolved = false; + const auto unresolved = [&](CasUnresolvedReason reason) + { + if (out_reason) + *out_reason = reason; + return CasWriteOutcome::Unresolved; + }; + if (out_reason) + *out_reason = CasUnresolvedReason::NotUnresolved; + + for (uint32_t attempt = 1; attempt <= budget.max_attempts; ++attempt) + { + /// Gate BEFORE every attempt: the + /// local mount fence must still hold, and there must be enough of the operation's own deadline + /// left for one more attempt to plausibly complete. Neither check sends anything to the backend. + if (!fence_ok()) + return unresolved(attempts_sent == 0 ? CasUnresolvedReason::NoAttemptSent + : CasUnresolvedReason::FenceLostMidWay); + if (now_ms() + budget.attempt_timeout_ms > deadline_ms) + return unresolved(attempts_sent == 0 ? CasUnresolvedReason::NoAttemptSent + : CasUnresolvedReason::DeadlineMidWay); + ++attempts_sent; + + /// The committed incarnation's token, filled by whichever leg proves Committed below. + Token committed_token; + CasWriteOutcome attempt_outcome{}; + try + { + const PutResult put = backend->putIfAbsent(key_s, bytes_s); + /// PreconditionFailed here means only "the key already exists" — it does NOT prove who + /// created it (possibly OUR earlier unresolved attempt). Collapse it onto Unresolved so it + /// goes through the SAME resolve-before-reissue path as an ambiguous exception, never a + /// false DefiniteFailure/Committed. + attempt_outcome = put.outcome == PutOutcome::Done ? CasWriteOutcome::Committed : CasWriteOutcome::Unresolved; + if (put.outcome == PutOutcome::Done) + committed_token = put.token; + } + catch (const std::exception & e) + { + attempt_outcome = classifyConditionalWriteResult(e); + } + + if (attempt_outcome == CasWriteOutcome::DefiniteFailure) + { + /// THIS attempt is proven never applied — but the verdict belongs to the CALL. An earlier + /// attempt that is still unresolved may yet materialize at the key, and a caller reading + /// `DefiniteFailure` acts on "the key is unwritten": `CasRefLedger::commitRefChunk` clears + /// its apply-pending marker and reports the txn id never used, so the next append re-derives + /// that id and a late-landing predecessor becomes an acked-then-lost transaction. Ambiguity + /// dominates a definite refusal that came after it; the caller wedges and resolves the key + /// instead. No resolve and no retry either way — this attempt has nothing left to settle. + if (earlier_attempt_unresolved) + return unresolved(CasUnresolvedReason::DefiniteFailureAfterAmbiguity); + return CasWriteOutcome::DefiniteFailure; /// every attempt of this call was proven never applied + } + + if (attempt_outcome == CasWriteOutcome::Unresolved) + { + /// Resolve-before-reissue. May throw CORRUPTED_DATA (a real + /// conflict) straight out of this call — that is never a retry signal. + attempt_outcome = resolveByExactGet(key_s, bytes_s, &committed_token); + if (attempt_outcome == CasWriteOutcome::Unresolved) + { + /// This attempt is now one that may still land: it was sent, and the resolve settled + /// nothing. Recorded BEFORE the exhaustion checks below so it is set no matter which of + /// them ends the loop, and read by the DefiniteFailure arm of every later attempt. + earlier_attempt_unresolved = true; + /// Absent/unreadable: another attempt of the SAME (key, bytes) may be legal — after the + /// fence-gated capped-exponential backoff (pauseBeforeReissue). No pause after the LAST + /// attempt: the budget is spent, sleeping would only delay the Unresolved verdict. + /// + /// Both refusals report through `unresolved`, never a bare `return Unresolved`: this is + /// the ordinary way a busy lane exhausts itself, so leaving `out_reason` at its initial + /// `NotUnresolved` here made the ref lane's wedge message read "is UNCERTAIN (not + /// unresolved)" for the single most common wedge there is. + if (attempt == budget.max_attempts) + return unresolved(CasUnresolvedReason::AttemptsExhausted); + CasUnresolvedReason pause_reason = CasUnresolvedReason::AttemptsExhausted; + if (!pauseBeforeReissue(attempt, deadline_ms, fence_ok, &pause_reason)) + return unresolved(pause_reason); + continue; + } + } + + /// attempt_outcome == Committed here (either the attempt's own 2xx, or resolution found + /// identical bytes). Final fence check before reporting success: a fence lost here means the + /// write may have landed but this call must never claim it did. Count this "response observed + /// after the local fence" leg separately from the generic Unresolved classifier so a cross-epoch + /// fence loss is visible rather than folded into ordinary retry-budget exhaustion. + if (!fence_ok()) + { + ProfileEvents::increment(ProfileEvents::CASConditionalWriteFenceLostPostWrite); + return unresolved(CasUnresolvedReason::FenceLostPostWrite); + } + if (out_token) + *out_token = committed_token; + return CasWriteOutcome::Committed; + } + + return unresolved(CasUnresolvedReason::AttemptsExhausted); /// budget exhausted, no definite outcome +} + +CasCreateResult CasRequestController::conditionalCreateControlled( + std::string_view key, const std::function & attempt, const std::function & fence_ok) +{ + const String key_s{key}; + const uint64_t deadline_ms = now_ms() + budget.operation_deadline_ms; + + for (uint32_t attempt_no = 1; attempt_no <= budget.max_attempts; ++attempt_no) + { + /// Same pre-attempt gates as putIfAbsentControlled. + if (!fence_ok()) + return {CasCreateOutcome::Unresolved, {}}; + if (now_ms() + budget.attempt_timeout_ms > deadline_ms) + return {CasCreateOutcome::Unresolved, {}}; + + std::optional put; + try + { + put = attempt(); + } + catch (const std::exception & e) + { + /// A deterministic LOCAL bug surfaced by the attempt itself — a caller/config error, never + /// a wire ambiguity; reissuing would only replay it. Propagate unchanged: instant, loud, + /// exactly the pre-controller behavior (see isDeterministicLocalFailure for the set and the + /// per-code rationale). This deliberately differs from `putIfAbsentControlled`'s + /// everything-Unresolved: + /// that lane's byte-exact resolve makes retrying any unproven error harmless, while + /// retrying a broken source/mode/encode here is pure noise. Fail-safe either way — a + /// propagated exception is never a false Committed. + if (const auto * db_e = dynamic_cast(&e); db_e && isDeterministicLocalFailure(db_e->code())) + throw; + /// A whitelisted synchronous rejection PROVES the request was never applied: surface the + /// original exception — the blob lane's callers always saw the raw storage error's root + /// cause, and losing it behind an outcome enum here would only degrade diagnostics + /// Anything else is ambiguous: fall through to the occupancy + /// resolve below. + if (classifyConditionalWriteResult(e) == CasWriteOutcome::DefiniteFailure) + throw; + } + + if (put) + { + if (put->outcome == PutOutcome::PreconditionFailed) + return {CasCreateOutcome::Occupied, {}}; + + /// Done. Final fence check before reporting success: a fence + /// lost here means the write may have landed but this call must never claim it did. + if (!fence_ok()) + { + ProfileEvents::increment(ProfileEvents::CASConditionalWriteFenceLostPostWrite); + return {CasCreateOutcome::Unresolved, {}}; + } + return {CasCreateOutcome::Committed, put->token}; + } + + /// Ambiguous attempt: resolve by exact-key OCCUPANCY — one HEAD, never a body GET (the body + /// may be multi-GB, and reading a possibly-condemned occupant would flirt with the resurrect + /// invariant; the key's content-address IS the identity proof, see the header contract). + bool occupied = false; + bool head_answered = true; + try + { + occupied = backend->head(key_s).exists; + } + catch (const std::exception &) + { + /// The HEAD itself failed: occupancy unproven either way. Reissuing is still safe — an + /// occupant answers the reissued If-None-Match with PreconditionFailed (-> Occupied on + /// the next round) — so treat exactly like "absent" and let the budget bound the loop. + head_answered = false; + } + if (head_answered && occupied) + return {CasCreateOutcome::Occupied, {}}; + + if (attempt_no == budget.max_attempts || !pauseBeforeReissue(attempt_no, deadline_ms, fence_ok)) + return {CasCreateOutcome::Unresolved, {}}; + } + + return {CasCreateOutcome::Unresolved, {}}; /// attempt budget exhausted without a definite outcome +} + +CasOverwriteResult CasRequestController::putOverwriteControlled( + std::string_view key, std::string_view bytes, const Token & expected, const std::function & fence_ok) +{ + const String key_s{key}; + const String bytes_s{bytes}; + const uint64_t deadline_ms = now_ms() + budget.operation_deadline_ms; + + for (uint32_t attempt_no = 1; attempt_no <= budget.max_attempts; ++attempt_no) + { + if (!fence_ok()) + return {CasOverwriteOutcome::Unresolved, {}}; + if (now_ms() + budget.attempt_timeout_ms > deadline_ms) + return {CasOverwriteOutcome::Unresolved, {}}; + + std::optional put; + try + { + put = backend->putOverwrite(key_s, bytes_s, expected); + } + catch (const std::exception & e) + { + /// Same rethrow convention as conditionalCreateControlled: a deterministic local bug or + /// a whitelisted synchronous rejection PROVES no retry can help -- surface it unchanged. + if (const auto * db_e = dynamic_cast(&e); db_e && isDeterministicLocalFailure(db_e->code())) + throw; + if (classifyConditionalWriteResult(e) == CasWriteOutcome::DefiniteFailure) + throw; + /// Else ambiguous -- fall through to resolve below. + } + + if (put && put->outcome == PutOutcome::Done) + { + if (!fence_ok()) + { + ProfileEvents::increment(ProfileEvents::CASConditionalWriteFenceLostPostWrite); + return {CasOverwriteOutcome::Unresolved, {}}; + } + return {CasOverwriteOutcome::Committed, put->token}; + } + + /// Ambiguous: either a caught transient exception, or PreconditionFailed (which alone does + /// NOT prove a real conflict -- it may be our own earlier attempt's write landing under a + /// concurrent resolve). Resolve with one GET. + std::optional got; + try + { + got = backend->get(key_s); + } + catch (const std::exception &) + { + got.reset(); /// GET failed: still ambiguous, fall through to retry below. + } + + if (got && got->token == expected) + { + /// The token we CAS'd against is STILL current: our attempt never applied. Fall through + /// to the pause-and-reissue gate below (same key, bytes, expected). + } + else if (got && got->bytes == bytes_s) + { + if (!fence_ok()) + { + ProfileEvents::increment(ProfileEvents::CASConditionalWriteFenceLostPostWrite); + return {CasOverwriteOutcome::Unresolved, {}}; + } + return {CasOverwriteOutcome::Committed, got->token}; + } + else if (got) + { + /// A DIFFERENT token AND different bytes: a genuine competing write. Real conflict -- + /// never collapsed into Unresolved/DefiniteFailure, never thrown. + return {CasOverwriteOutcome::Conflict, {}}; + } + /// else: the GET itself failed or the key vanished -- still ambiguous, fall through to retry. + + if (attempt_no == budget.max_attempts || !pauseBeforeReissue(attempt_no, deadline_ms, fence_ok)) + return {CasOverwriteOutcome::Unresolved, {}}; + } + + return {CasOverwriteOutcome::Unresolved, {}}; /// attempt budget exhausted without a definite outcome +} + +CasOverwriteResult CasRequestController::putIfAbsentControlledMutable( + std::string_view key, std::string_view bytes, const std::function & fence_ok) +{ + const String key_s{key}; + const String bytes_s{bytes}; + const uint64_t deadline_ms = now_ms() + budget.operation_deadline_ms; + + for (uint32_t attempt_no = 1; attempt_no <= budget.max_attempts; ++attempt_no) + { + if (!fence_ok()) + return {CasOverwriteOutcome::Unresolved, {}}; + if (now_ms() + budget.attempt_timeout_ms > deadline_ms) + return {CasOverwriteOutcome::Unresolved, {}}; + + std::optional put; + try + { + put = backend->putIfAbsent(key_s, bytes_s); + } + catch (const std::exception & e) + { + /// Same rethrow convention as putOverwriteControlled/conditionalCreateControlled. + if (const auto * db_e = dynamic_cast(&e); db_e && isDeterministicLocalFailure(db_e->code())) + throw; + if (classifyConditionalWriteResult(e) == CasWriteOutcome::DefiniteFailure) + throw; + /// Else ambiguous -- fall through to resolve below. + } + + if (put && put->outcome == PutOutcome::Done) + { + if (!fence_ok()) + { + ProfileEvents::increment(ProfileEvents::CASConditionalWriteFenceLostPostWrite); + return {CasOverwriteOutcome::Unresolved, {}}; + } + return {CasOverwriteOutcome::Committed, put->token}; + } + + /// Ambiguous: either a caught transient exception, or PreconditionFailed (which alone does + /// NOT prove a real conflict -- it may be our own earlier attempt's write landing under a + /// concurrent resolve, or a racing writer creating the identical value). Resolve with one GET. + std::optional got; + try + { + got = backend->get(key_s); + } + catch (const std::exception &) + { + got.reset(); /// GET failed: still ambiguous, fall through to retry below. + } + + if (!got) + { + /// Still absent: our attempt never applied. Fall through to the pause-and-reissue gate + /// below (same key, bytes). + } + else if (got->bytes == bytes_s) + { + if (!fence_ok()) + { + ProfileEvents::increment(ProfileEvents::CASConditionalWriteFenceLostPostWrite); + return {CasOverwriteOutcome::Unresolved, {}}; + } + return {CasOverwriteOutcome::Committed, got->token}; + } + else + { + /// Present with DIFFERENT bytes: something else already occupies the key with a + /// different value. For a MUTABLE marker this is a normal outcome, not corruption -- + /// return it as a value, never thrown. + return {CasOverwriteOutcome::Conflict, {}}; + } + + if (attempt_no == budget.max_attempts || !pauseBeforeReissue(attempt_no, deadline_ms, fence_ok)) + return {CasOverwriteOutcome::Unresolved, {}}; + } + + return {CasOverwriteOutcome::Unresolved, {}}; /// attempt budget exhausted without a definite outcome +} + +SlotOccupyResult CasRequestController::slotOccupy( + std::string_view key, std::string_view bytes, const std::function & fence_ok) +{ + const String key_s{key}; + const String bytes_s{bytes}; + const uint64_t deadline_ms = now_ms() + budget.operation_deadline_ms; + + /// Pre-attempt gate ONLY -- the same two checks every controlled op runs before its first (here, + /// only) attempt: the mount fence must still hold, and there must be enough of the operation's own + /// deadline left for one attempt to plausibly complete. Neither check sends anything to the + /// backend, so a refusal here PROVES the key is untouched by this call. UNLIKE every sibling + /// controlled op, there is no post-write recheck below: fence_ok is evaluated once per attempt, and + /// the stronger post-I/O consistency check (fence generation together with wedge/txn identity) is + /// the CALLER's contract (Task 4/6's re-acquire-lock-and-checkFenceOrThrow step), not this raw + /// primitive's. + if (!fence_ok() || now_ms() + budget.attempt_timeout_ms > deadline_ms) + return {.kind = SlotOccupyResult::Kind::Unresolved, .occupant_bytes = {}, .occupant_token = {}, + .unresolved_reason = CasUnresolvedReason::NoAttemptSent}; + + std::optional put; + try + { + put = backend->putIfAbsent(key_s, bytes_s); + } + catch (const std::exception & e) + { + /// Same rethrow convention as putOverwriteControlled/putIfAbsentControlledMutable: a + /// deterministic local bug, or a whitelisted synchronous rejection that PROVES the request was + /// never applied, surfaces unchanged -- SlotOccupyResult::Kind has no DefiniteFailure member to + /// carry either one. Anything else is ambiguous: fall through to the raw resolve GET below, + /// exactly like a clean PreconditionFailed -- this primitive cannot and does not distinguish + /// the two. + if (const auto * db_e = dynamic_cast(&e); db_e && isDeterministicLocalFailure(db_e->code())) + throw; + if (classifyConditionalWriteResult(e) == CasWriteOutcome::DefiniteFailure) + throw; + } + + if (put && put->outcome == PutOutcome::Done) + return {.kind = SlotOccupyResult::Kind::Created, .occupant_bytes = {}, .occupant_token = {}, + .unresolved_reason = CasUnresolvedReason::NotUnresolved}; + + /// Ambiguous attempt or a clean conflict: resolve with exactly ONE raw exact GET -- no byte-compare, + /// no throw on a different occupant [codex finding 3: this is a DEDICATED slot operation, not + /// putIfAbsentControlled (which retries the same (key, bytes) internally) or resolveByExactGet + /// (which compares against an expected body and throws CORRUPTED_DATA on a mismatch) composed + /// together]. Adjudicating whether the occupant is "mine" is entirely the CALLER's job (the + /// CaCasMountCore `mine` contract), never this primitive's. + /// + /// WHOLE-OBJECT read, unlike conditionalCreateControlled's occupancy resolve (see that method's doc + /// in the header), which deliberately uses HEAD instead of GET because a blob body "may be + /// multi-GB". That reasoning does not apply here: slotOccupy is scoped by its callers (Task 4/6, + /// spec INV-2) to small, write-once CONTROL slots -- ref-log transactions and epoch seals -- whose + /// size is bounded by their own format's registry cap (the strict-grammar object caps + /// CasRefLogFormat/CasRefCkptFormat enforce on decode), never a data blob. slotOccupy itself stays + /// format-agnostic (it takes a raw key/bytes pair, per the "Interface handed to Stage B" contract in + /// the plan) and does not encode any format's cap here -- the size bound is a property of what + /// callers are allowed to pass it, enforced where the returned bytes are decoded, not by this seam. + std::optional got; + try + { + got = backend->get(key_s); + } + catch (const std::exception &) + { + got.reset(); /// the GET itself failed: still unresolved -- a one-shot primitive never retries + } + + if (!got) + /// The occupant that caused the conflict vanished before this GET (or the GET itself failed): + /// the outcome is unknowable right now -- NEVER a fabricated Created. + return {.kind = SlotOccupyResult::Kind::Unresolved, .occupant_bytes = {}, .occupant_token = {}, + .unresolved_reason = CasUnresolvedReason::AttemptsExhausted}; + + return {.kind = SlotOccupyResult::Kind::Occupied, .occupant_bytes = std::move(got->bytes), + .occupant_token = got->token, .unresolved_reason = CasUnresolvedReason::NotUnresolved}; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequestControl.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequestControl.h new file mode 100644 index 000000000000..730894cacd27 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequestControl.h @@ -0,0 +1,621 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Outcome of ONE HTTP attempt for a CAS conditional write (`If-None-Match`/`If-Match`), issued with +/// the generic S3 client's transparent retries disabled for that attempt. This is the seam the +/// `CasRequestController` is built on: it decides whether another attempt is legal and how an +/// uncertain result is resolved. +/// - Committed: the attempt's own request completed successfully (2xx) — the object is durable. +/// - DefiniteFailure: a synchronous rejection that PROVES the request was never applied server-side +/// — a WHITELISTED malformed-request / entity-too-large / access-denied error ONLY. Never +/// `PreconditionFailed`: a lost precondition means the key exists, not that the request failed. +/// - Unresolved: everything else — `PreconditionFailed`/`NoSuchKey`, a client-side timeout, a +/// connection loss, a 5xx, or any error this classifier does not recognize. The caller resolves +/// the exact key before deciding whether another attempt is legal; ambiguity always +/// resolves toward Unresolved, never toward a false DefiniteFailure or a false Committed. +enum class CasWriteOutcome : uint8_t +{ + Committed, + DefiniteFailure, + Unresolved, +}; + +/// WHY a controlled write came back `Unresolved`. It exists because `Unresolved` covers two materially +/// different states, and telling them apart is the difference between a five-minute triage and an hour +/// of it — and, since finding #37 defect 3, between a table that keeps its write availability and one +/// that loses it until remount. +/// +/// `NoAttemptSent` is the one that carries real information: both pre-attempt gates (the mount fence +/// and the operation deadline) reject BEFORE anything reaches the backend, so on the very first +/// iteration the key is PROVABLY unwritten — there is no ambiguity to resolve, only a lost right to +/// write. Every other reason leaves an object that may or may not be durable, which is what the +/// wedge/resolve machinery exists for. +/// +/// NOT purely diagnostic any more: `unresolvedProvesNothingWasSent` below turns this into the fact the +/// ref append lane acts on (`CasRefLedger::commitRefChunk`'s `Unresolved` arm), so ADDING A MEMBER HERE +/// IS A PROTOCOL DECISION — read that predicate before you do. +enum class CasUnresolvedReason : uint8_t +{ + NotUnresolved, /// the call did not return Unresolved + NoAttemptSent, /// a pre-attempt gate rejected on the FIRST iteration: nothing was ever sent + FenceLostMidWay, /// >= 1 attempt was sent, then the mount fence dropped + DeadlineMidWay, /// >= 1 attempt was sent, then the operation deadline left no room for another + FenceLostPostWrite,/// an attempt COMMITTED but the fence had dropped by the time it returned + AttemptsExhausted, /// the genuine case the "retry budget exhausted" wording describes + /// A LATER attempt was definitively refused while an EARLIER one of the same call is still + /// unresolved. The refusal proves only its own attempt never applied; the earlier one may still + /// materialize at the key, so the CALL cannot report `DefiniteFailure` (see + /// `putIfAbsentControlled`). Reported instead of the definite verdict, never alongside it. + DefiniteFailureAfterAmbiguity, +}; + +/// Does this `Unresolved` PROVE that no attempt ever reached the network — i.e. that the key is +/// unwritten and there is nothing for an exact-key resolution to settle? +/// +/// True for exactly ONE value, and that is the whole design: `NoAttemptSent` is reported only when a +/// pre-attempt gate rejected while `attempts_sent == 0`, so `backend->putIfAbsent` was never called +/// (see `putIfAbsentControlled`). Every other value — including `NotUnresolved`, which a caller can +/// still observe if some path returns `Unresolved` without recording a reason — leaves an object that +/// MAY be durable, and callers that protect themselves against that (the ref lane's append wedge) must +/// keep doing so. +/// +/// Written as an allow-list: a switch with no `default` and a trailing `return false`, so a member +/// added to `CasUnresolvedReason` later fails BOTH ways safely. The missing case is a `-Wswitch` build +/// error, which forces the contributor to classify it deliberately; and if that diagnostic is ever +/// silenced, the runtime answer for the unclassified member is "no, this does not prove anything", +/// which is the conservative side. Never turn this into a deny-list — a new reason must not be able to +/// claim "nothing was sent" by omission. +constexpr bool unresolvedProvesNothingWasSent(CasUnresolvedReason reason) +{ + switch (reason) + { + case CasUnresolvedReason::NoAttemptSent: + return true; + case CasUnresolvedReason::NotUnresolved: + case CasUnresolvedReason::FenceLostMidWay: + case CasUnresolvedReason::DeadlineMidWay: + case CasUnresolvedReason::FenceLostPostWrite: + case CasUnresolvedReason::AttemptsExhausted: + /// The whole point of this value is that an earlier attempt WAS sent and may still land. + case CasUnresolvedReason::DefiniteFailureAfterAmbiguity: + return false; + } + return false; +} + +/// Human-readable tail for an exception or log line, so the two states above stop reading alike. +constexpr std::string_view describeUnresolvedReason(CasUnresolvedReason reason) +{ + switch (reason) + { + case CasUnresolvedReason::NotUnresolved: return "not unresolved"; + case CasUnresolvedReason::NoAttemptSent: return "no attempt was sent (the mount fence or the " + "operation deadline rejected before the first " + "request) — the key is provably unwritten"; + case CasUnresolvedReason::FenceLostMidWay: return "the mount fence dropped after at least one " + "attempt had been sent"; + case CasUnresolvedReason::DeadlineMidWay: return "the operation deadline ran out after at least " + "one attempt had been sent"; + case CasUnresolvedReason::FenceLostPostWrite: return "an attempt committed but the mount fence had " + "dropped before it returned"; + case CasUnresolvedReason::AttemptsExhausted: return "the attempt budget was exhausted without a " + "definite outcome"; + case CasUnresolvedReason::DefiniteFailureAfterAmbiguity: + return "a later attempt was definitively refused, but " + "an earlier attempt of the same call is still " + "unresolved and may yet land"; + } + return "unspecified"; +} + +/// The success path: `buf.finalize()` returned without throwing. Always Committed — kept as a named, +/// counted entry point so both paths of a classify-then-record call site read the same way (see the +/// exception overload below). +constexpr CasWriteOutcome classifyConditionalWriteResult() +{ + return CasWriteOutcome::Committed; +} + +/// The exception path: classify what `buf.finalize()` threw for ONE CAS conditional-write HTTP +/// attempt, according to the CAS conditional-write operation classes. Pure — never rethrows, never +/// touches counters; see recordConditionalWriteOutcome for the counters hookup. +CasWriteOutcome classifyConditionalWriteResult(const std::exception & e); + +/// Records the start of one HTTP attempt for a CAS conditional write (the attempts counter). +void recordConditionalWriteAttemptStarted(); + +/// Records one attempt's terminal outcome (the per-class outcome counters). Callers pass the result of +/// whichever classifyConditionalWriteResult overload applies, or an outcome already known by +/// construction (e.g. the legacy `PutOutcome::PreconditionFailed` path, which today resolves without +/// throwing — see ObjectStorageBackend::nativeConditionalPut). +void recordConditionalWriteOutcome(CasWriteOutcome outcome); + +/// The three separate limits a CAS-owned retry controller enforces for ONE logical conditional-write +/// operation. Never represented by a single `request_timeout_ms` value — see `validateCasRequestBudget` +/// for the relationship a writable mount enforces at startup, and `CasRequestController` for the +/// runtime use. +struct CasRequestBudget +{ + /// Maximum client wait budgeted for one HTTP attempt. `CasRequestController` uses this ONLY as a + /// per-attempt scheduling check (an attempt is not started unless it could still finish inside the + /// operation deadline) — the actual socket-level wait is configured on the object storage's client + /// (the object storage backend's single-attempt client), not by this struct. + uint64_t attempt_timeout_ms = 5000; + /// Maximum wall-clock time for the COMPLETE logical operation — every attempt, every exact-key + /// resolution, and every inter-attempt backoff sleep — counted from the first call to + /// `putIfAbsentControlled`. A DURATION, not an absolute deadline: each call establishes its own + /// `now + operation_deadline_ms` bound. + /// + /// This deadline is the authoritative bound on how long a CAS conditional write keeps riding an S3 + /// disruption server-side before the caller sees an abort. 90s absorbs a ~60s object-store outage + /// with margin (see the arithmetic on `max_attempts` below) — PROVIDED the mount fence stays alive. + /// The fence, not this deadline, is + /// what binds under a TOTAL outage: lease renewals are conditional writes against the same store, + /// so when everything is unreachable the fence deadline freezes at `last_renew + mount_lease_ttl` + /// and `fence_ok` stops the loop ≈ TTL−attempt_timeout−margin (~23s) after the last successful + /// renewal — the required fail-closed behavior (never an attempt past the lease), not a + /// budget limitation. While renewals DO land (blips, throttling, partial outages — the renewer runs + /// on its own background thread and keeps extending the fence deadline), the op is NOT bounded by + /// the lease TTL and rides the full deadline here. + uint64_t operation_deadline_ms = 90000; + /// Maximum number of controlled attempts for one logical operation (the first attempt counts as 1). + /// Sized so the operation deadline above — never this count — is what binds under the observed + /// failure shape (~3s adaptive first-attempt PUT timeout per failed attempt + capped-exponential + /// backoff): 16 attempts × ~3s + Σ backoff (0.2+0.4+0.8+1.6+3.2 + 10×5 = 56.2s) ≈ 104s > 90s. + uint32_t max_attempts = 16; + /// Startup-only margin folded into `validateCasRequestBudget`'s inequality against the mount lease + /// TTL. Not consulted at runtime by the controller itself — the caller's `fence_ok` callback (backed + /// by the local write fence's own deadline) is what actually gates lease-relative timing per attempt. + uint64_t lease_safety_margin_ms = 2000; + /// Inter-attempt backoff (`cas_s3_retry_initial_backoff_ms` / + /// `cas_s3_retry_max_backoff_ms`): the sleep before reissuing + /// after an ambiguous attempt whose resolve observed the key absent, capped exponential — + /// `initial · 2^(reissues-1)`, never above `retry_max_backoff_ms`. 0 disables backoff (immediate + /// reissue — the pre-backoff behavior, and what most exhaustion-path unit tests configure). The + /// controller checks the fence BEFORE every sleep and never sleeps past the operation deadline. + uint64_t retry_initial_backoff_ms = 200; + uint64_t retry_max_backoff_ms = 5000; + + /// Recovery-level retry (`CasRefLedger::ensureRefTableRecovered`): a whole ref-table recovery + /// attempt (LIST + snapshot/log GETs + seal PUT) that fails with a transient NETWORK_ERROR is + /// retried, with capped-exponential backoff, until this total wall-clock budget is spent — then the + /// error propagates and the table's load fails for this touch (the `lazy_load_tables` database + /// setting makes the NEXT touch retry). This sits ON TOP of the per-request `operation_deadline_ms` + /// envelope above: one recovery attempt may itself burn ~90s inside a single seal PUT. Independent + /// of the mount-lease invariants validated in `validateCasRequestBudget` — not part of that + /// inequality set. + uint64_t recovery_retry_budget_ms = 120000; + uint64_t recovery_retry_initial_backoff_ms = 1000; + uint64_t recovery_retry_max_backoff_ms = 30000; +}; + +/// Startup validation: a writable mount refuses to open with an inconsistent budget rather than +/// silently falling back to an unbounded or unsafe retry policy. Throws +/// `BAD_ARGUMENTS` unless ALL hold: +/// attempt_timeout_ms + lease_safety_margin_ms < mount_lease_ttl_ms +/// attempt_timeout_ms < operation_deadline_ms (STRICTLY — see below) +/// retry_initial_backoff_ms <= retry_max_backoff_ms +/// +/// The middle one is strict on purpose. Equality does not mean "one attempt's worth of budget": the +/// deadline is captured as `now + operation_deadline_ms` and each pre-send gate asks +/// `now + attempt_timeout_ms > deadline_ms`, so equal values reduce it to `now_2 > now_1` and a single +/// elapsed millisecond refuses the operation having sent NOTHING. Bound the attempt COUNT with +/// `max_attempts`, never by starving the deadline. +/// `mount_renew_period_ms` takes no part in the inequality (the renewer keeps the fence deadline +/// refreshed well ahead of the TTL by construction) — it is accepted only so the effective-values log +/// line records the full picture in one place. +/// +/// A successor mounting over an unclean predecessor waits at least one lease TTL, plus its +/// materialization grace period, before trusting recovery listings. This is long enough for any +/// conditional PUT still in flight at the predecessor to either land or be abandoned by its own +/// exhausted retry budget. The predecessor's budget is constrained by +/// `attempt_timeout_ms + lease_safety_margin_ms < mount_lease_ttl_ms`, so no additional handover +/// check is needed here. +void validateCasRequestBudget(const CasRequestBudget & budget, uint64_t mount_lease_ttl_ms, uint64_t mount_renew_period_ms); + +/// Throw the recoverable "CAS write could not be committed, retry later" condition. +/// +/// WHY NETWORK_ERROR (this replaces an earlier ABORTED throw): +/// A content-addressed write can fail for a reason that is neither the caller's fault +/// nor permanent: the mount-lease / write fence was lost (e.g. a renewal PUT timed out +/// against a slow or throttling object store), or a conditional PUT exhausted its retry +/// budget mid-outage. The right response is "abandon this attempt, try again later" -- +/// which is precisely what a transient error means. +/// +/// It previously threw `ABORTED`, which was actively harmful to background merges: +/// `ReplicatedMergeMutateTaskBase` treats `ABORTED` as "merge deliberately cancelled +/// (shutdown / `DROP` / merges-blocker), not an error", so it neither records +/// `last_exception_time_ms` nor lets `ReplicatedMergeTreeQueue`'s exponential backoff +/// engage. Under a sustained store outage the queue re-executed the merge roughly every +/// 2 seconds, recomputing the whole (possibly multi-GiB) output part every time for the +/// entire outage -- hundreds of full recomputes, and invisible in system.replication_queue. +/// +/// `NETWORK_ERROR` is the best-fitting EXISTING code: +/// - it is NOT in the merge "retry silently, no backoff" exemption set (only `ABORTED` +/// and `PART_IS_TEMPORARILY_LOCKED` are), so the existing backoff -- capped by +/// `max_postpone_time_for_failed_replicated_merges_ms` -- engages automatically; +/// - it is already in ClickHouse's transient/retryable taxonomy +/// (`checkDataPart::isRetryableException` lists it beside `ABORTED`), so a part under +/// verification is not misread as corrupted; +/// - nothing on the merge / insert / replication commit path special-cases it in a way +/// that would misfire (ZooKeeper retriability keys on `Coordination::Exception`, a +/// different type), and it is not caught specially on the CAS write path. +/// +/// Honest caveat: `NETWORK_ERROR` is coarser than the true condition. For the +/// throttled-store / timed-out / lost-lease cases it is accurate; for a purely logical +/// fence loss (e.g. the namespace is being dropped) it slightly overstates "network". +/// The precise cause is always in the exception MESSAGE, never inferred from the code. +/// +/// If that imprecision ever matters -- operator confusion, or a future upstream change +/// that attaches merge-path handling to `NETWORK_ERROR` and reintroduces a collision -- +/// switch to a dedicated code (e.g. CAS_WRITE_RETRY_LATER) by changing the +/// single throw below. A dedicated code is honest and collision-proof by construction +/// (backoff still engages, since only `ABORTED` / `PART_IS_TEMPORARILY_LOCKED` are exempt); +/// the only extra work is one appended line in `ErrorCodes.cpp` and, optionally, adding it +/// to `checkDataPart::isRetryableException` and an HTTP-status mapping for the foreground +/// `INSERT` client. We deliberately kept `NETWORK_ERROR` for now to add zero new coupling to +/// generic ClickHouse code, consistent with the rest of the CAS layer. +/// +/// SCOPE: only the ESCAPING retry-later throws route here (fence lost / write outcome +/// uncertain / conditional-create Unresolved). The ABORTED values used as internal +/// control-flow signals (the condemned/vanished "re-upload from source" signal caught +/// inside putBlob), and the startup/decommission and generic live-lock-brake ABORTEDs, +/// keep their meaning and are NOT rerouted here. +[[noreturn]] void throwCasWriteRetryLater(const String & why); + +/// Same classification as `throwCasWriteRetryLater`, but returns the exception as a +/// `std::exception_ptr` for call sites that fail a pending future/promise (`CasRefLedger`'s +/// `complete_error`) rather than throw directly. Both entry points route through the SAME +/// construction internally, so the error code / message shape has exactly one place that decides it. +std::exception_ptr makeCasWriteRetryLaterExceptionPtr(const String & why); + +/// Throw the recoverable "this content-addressed disk cannot serve the request right now" condition. +/// Sibling of `throwCasWriteRetryLater`, same class for the same reasons (see the long rationale above), +/// differing only in what it describes: that one names a WRITE whose commit did not land, this one names +/// a DISK STATE that refused the request before it started -- on either plane. +/// +/// The class is load-bearing beyond CAS. `ReplicatedMergeTreePartCheckThread::checkPartImpl` rethrows +/// (leaving the part queued for a later check) exactly when `checkDataPart::isRetryableException` +/// recognises the error, and otherwise declares the part broken -- detach and re-fetch. +/// `INVALID_STATE` is absent from that classifier, so a lease blip used to read as part corruption +/// (BACKLOG `{#lease-blip-part-check-collapse}`). Re-coding the CA transients +/// was chosen over widening the upstream classifier because `INVALID_STATE` is broad: widening it would +/// also reclassify 18 unrelated TERMINAL sites, CA and non-CA alike. +/// +/// SCOPE, narrow by design: a refusal routes here when it either names an AUTO-RECOVERING disk condition, +/// or CANNOT ESTABLISH that its condition is terminal. `checkFenceOrThrow` is the second kind -- one guard +/// trips for a lease blip and for a FORGET decommission alike and it cannot tell them apart -- and it is in +/// scope for write-plane uniformity: its 32 sibling write-transient sites already mint this class, and an +/// unproven condition must be retried rather than consumed as damage. What is NEVER in scope is a refusal +/// whose condition is PROVEN terminal: `IdentityLost`, both `Vanished` flavours, a storage that is not +/// started, an unbootstrappable prefix, a proven-absent pool identity, a closed writer epoch -- all keep +/// `INVALID_STATE`. A proven-terminal state that read as retryable would make every consumer retry forever +/// against a disk that is never coming back. +/// +/// `subject` names the refusing disk or pool (e.g. "content-addressed disk 'ca'") and `condition` states +/// the CA condition truthfully, INCLUDING any promise about how it clears -- only the site knows whether +/// it can make one. What is appended HERE is the classification alone, so it cannot drift between call +/// sites. Unlike `throwCasWriteRetryLater` this deliberately does not log: these sites fire once per +/// refused operation (tens of thousands within a single observed lease gap) and every caller already +/// reports the exception it receives. +[[noreturn]] void throwCasTransientUnavailable(const String & subject, const String & condition); + +/// Outcome of a controlled CONTENT-ADDRESSED conditional create (`conditionalCreateControlled`): +/// - Committed: an attempt's own request completed (2xx) and the final fence check held — `token` +/// names the created incarnation. +/// - Occupied: the key holds an object — either a genuine `PreconditionFailed` (a racing twin) or +/// an earlier ambiguous attempt of THIS operation that actually landed. For a content-addressed +/// key these are THE SAME situation: the key embeds the content hash, so any occupant is the +/// intended content (the exact trust model the plain 412-adopt path already relies on) — the +/// caller runs its ordinary occupant machinery (adopt live / displace condemned). +/// - Unresolved: budget exhausted or fence lost without a definite outcome — the write may or may +/// not have landed; the caller must not ACK (a late-landing body is inert unreferenced debris for +/// the orphan sweep, exactly like a stageManifest Unresolved). +enum class CasCreateOutcome : uint8_t +{ + Committed, + Occupied, + Unresolved, +}; + +/// Result of one `CasRequestController::conditionalCreateControlled` operation. `token` is meaningful +/// only when `outcome` is `Committed`; it identifies the incarnation created by the successful attempt. +/// An `Occupied` result deliberately carries no token because the caller must use its normal occupant +/// handling, whether the occupant was created by a racing writer or by an earlier ambiguous attempt. +struct CasCreateResult +{ + CasCreateOutcome outcome = CasCreateOutcome::Unresolved; + Token token; /// set ONLY on Committed +}; + +/// Outcome of a controlled MUTABLE conditional overwrite (`putOverwriteControlled`) -- an If-Match +/// replace whose caller can, unlike a content-addressed create, supply the intended bytes for +/// GET-based resolution, because the payload here is deterministic (a pure function of the +/// caller's record), not freshly minted per attempt. +/// - Committed: an attempt's own request completed (2xx) and the final fence check held, or +/// resolution proved the intended bytes are already what's currently stored -- `token` names +/// that incarnation. +/// - Conflict: resolution proved the key's CURRENT token AND bytes both differ from what this +/// call intended -- a genuine competing write. Returned as a value, never thrown, never +/// collapsed into Unresolved/DefiniteFailure -- mirrors the existing uncontrolled +/// casMeta/CasResult contract (a conflict lets the caller reload and decide). +/// - Unresolved: budget exhausted, fence lost, or the current token still equals `expected` (the +/// attempt provably never applied) with the resolve unable to prove either outcome yet -- +/// caller must not ACK. +enum class CasOverwriteOutcome : uint8_t +{ + Committed, + Conflict, + Unresolved, +}; + +/// Result of one `CasRequestController::putOverwriteControlled` operation. `token` is meaningful +/// only when `outcome` is `Committed`. +struct CasOverwriteResult +{ + CasOverwriteOutcome outcome = CasOverwriteOutcome::Unresolved; + Token token; /// set ONLY on Committed +}; + +/// Result of one `CasRequestController::slotOccupy` operation — a WRITE-ONCE conditional create whose +/// body is content-addressed or otherwise not byte-comparable across separate CALLS the way +/// `putOverwriteControlled`'s deterministic marker is (each caller of `slotOccupy` — an epoch seal, a +/// wedge retry — mints its own attempt and decides for itself, from `Occupied`'s bytes, whether the +/// occupant is its own earlier write or something else entirely; see the adjudication note below). +/// - Created: this call's OWN conditional create committed — the key held nothing before it. +/// - Occupied: the key already holds an object, observed by ONE raw exact `GET` after the create +/// conflicted — `occupant_bytes`/`occupant_token` name exactly what is there NOW. The primitive +/// never compares these bytes against what this call attempted and never throws on a mismatch: +/// unlike `resolveByExactGet` (whose caller supplies ONE expected body across every retry of the +/// SAME logical attempt), `slotOccupy` never retries, so there is no "our earlier attempt" to +/// distinguish from a genuine foreign occupant — that adjudication (the `CaCasMountCore` `mine` +/// contract: an occupant is this caller's write only if the BYTES match, never a generation/shape +/// match alone) is entirely the CALLER's job. +/// - Unresolved: the outcome is unknowable right now — a pre-attempt gate refused (fence lost / +/// deadline exhausted, `unresolved_reason == NoAttemptSent`, nothing was sent — `unresolvedProvesNothingWasSent` +/// is TRUE only for this case), or the conditional create was itself ambiguous (a transient +/// exception) and the follow-up resolve GET found nothing (the occupant that caused the conflict +/// vanished before the GET, or the GET itself failed — BOTH of these report `unresolved_reason == +/// AttemptsExhausted`, for which `unresolvedProvesNothingWasSent` is FALSE) — NEVER fabricated into +/// a false `Created`. CALLERS: do not log a bare `describeUnresolvedReason(AttemptsExhausted)` for +/// this case — it reads "the attempt budget was exhausted", which is misleading for a primitive +/// with no retry budget, and it silently folds "the resolve GET found nothing" together with "the +/// occupant that caused the conflict was DELETED under a live epoch" (a GC-invariant alarm, not +/// routine contention) into the same generic wording. `SlotOccupyResult` carries no discriminator +/// between those two sub-cases — log the slot key plus "the resolve read found nothing"; the day a +/// caller NEEDS the split is the trigger for adding a dedicated `CasUnresolvedReason` value (a +/// gated protocol decision, not a drive-by). +struct SlotOccupyResult +{ + enum class Kind : uint8_t { Created, Occupied, Unresolved }; + Kind kind = Kind::Unresolved; + /// Occupied only: the occupant, fetched by exact GET after the conditional create conflicted. + String occupant_bytes; + Token occupant_token; + /// Unresolved only: why the attempt outcome is unknowable right now. + CasUnresolvedReason unresolved_reason{}; +}; + +/// CAS-owned retry controller: the only place that decides whether a conditional-write attempt may be +/// reissued. It does not touch a writer cache or return ACK. Callers update their cache and acknowledge +/// the operation only after this controller has resolved the outcome and performed its final fence +/// check, using the returned `CasWriteOutcome`. +class CasRequestController +{ +public: + /// `now_ms_`: monotonic-ish clock, defaulting to `std::chrono::steady_clock`; tests inject a fake + /// one to drive deadline behavior deterministically (no sleeps). + /// `sleep_ms_`: the inter-attempt backoff sleep, defaulting to a real `std::this_thread::sleep_for`; + /// tests inject a recorder/no-op to assert the backoff schedule without wall-clock waits. The + /// controller only ever sleeps BETWEEN attempts of one logical operation, on the calling thread, + /// with no Pool mutex held (every call site — the ref append lane's leader, `stageManifest`, blob + /// uploads, snapshot publishes — invokes the controller outside its locks; the append lane's + /// LEADERSHIP is deliberately held across the sleep: same-table appends must queue behind an + /// unresolved predecessor PUT anyway, preserving the writer's per-table ordering. + CasRequestController(BackendPtr backend_, CasRequestBudget budget_, std::function now_ms_ = {}, + std::function sleep_ms_ = {}); + + /// Controlled `putIfAbsent` with resolve-before-reissue. Performs at + /// most `budget.max_attempts` attempts of the exact SAME (key, bytes) — never a different key, never + /// a different body — bounded by `budget.operation_deadline_ms` measured from this call's own start, + /// with capped-exponential inter-attempt backoff (`retry_initial_backoff_ms`/`retry_max_backoff_ms`). + /// `fence_ok` is consulted before EVERY attempt (a false answer sends no further attempt), before + /// EVERY backoff sleep (a fence lost mid-loop aborts instantly, never after a pointless sleep), and + /// once more before a `Committed` return (a false answer there means the write may have landed but + /// this call reports `Unresolved`, never a false `Committed`). A sleep is + /// never entered when it (plus one more attempt) could not fit the operation deadline. An uncertain + /// attempt is resolved via `resolveByExactGet` before deciding whether to reissue. + /// Throws `CORRUPTED_DATA` if resolution ever observes DIFFERENT valid bytes at `key` — a real + /// conflict, never collapsed into `Unresolved`/`DefiniteFailure`. Returns `Unresolved` (never + /// throws) when the fence is lost or the budget is exhausted before a definite outcome is reached. + /// + /// THE VERDICT IS THE CALL'S, NOT THE LAST ATTEMPT'S. `DefiniteFailure` is returned only when EVERY + /// attempt this call sent was itself proven never applied. One attempt's whitelisted rejection + /// proves nothing about an EARLIER attempt of the same call that went ambiguous: that request may + /// have been received and may still materialize at `key` (an absent resolve GET is not evidence — + /// `unresolvedProvesNothingWasSent`). Any such attempt therefore dominates the result, which becomes + /// `Unresolved`/`DefiniteFailureAfterAmbiguity` — the wedge path — because a caller acting on + /// `DefiniteFailure` declares the key unwritten and reuses the id (`CasRefLedger::commitRefChunk`), + /// which an ambiguous predecessor can turn into an acked-then-lost transaction. Attempts a + /// pre-attempt gate refused never reach the backend, so they never make this call ambiguous. + /// `out_token` (optional): set ONLY on a `Committed` return, to the committed incarnation's token — + /// the attempt's own `PutResult` token, or the token the resolve GET observed when it proved an + /// earlier ambiguous attempt landed. Lets audit emitters (e.g. `PartWriteTxn::stageManifest`'s + /// `ManifestPut` event) keep the token without a follow-up HEAD. Untouched on any other return. + /// `out_reason` (optional): WHY an `Unresolved` was returned. Diagnostic only — the returned + /// outcome is unchanged, so no caller's decision depends on it. It exists because `Unresolved` + /// currently conflates two very different situations, and the resulting message + /// ("retry budget exhausted") is printed even where NOTHING was ever sent: finding #37 defect 3, + /// whose own note records that the opacity "plausibly fed 3 prior wrong analyses" — and it did so + /// again on 2026-07-24, when a sanitizer-slow unit test fenced itself and the text sent the CI + /// triage looking for a retry problem that did not exist. `NoAttemptSent` is the load-bearing + /// distinction: it means the key was provably never written, whereas the other reasons leave a + /// possibly-durable object behind. + CasWriteOutcome putIfAbsentControlled(std::string_view key, std::string_view bytes, + const std::function & fence_ok, Token * out_token = nullptr, + CasUnresolvedReason * out_reason = nullptr); + + /// One-shot exact-key resolution of an uncertain immutable create: + /// - identical bytes observed at `key` -> Committed (the earlier attempt DID commit) + /// - DIFFERENT bytes observed at `key` -> throws CORRUPTED_DATA (a real conflict, not a retry + /// signal — never silently treated as ambiguous) + /// - absent, or the GET itself fails -> Unresolved (another attempt may still be legal) + /// NEVER returns DefiniteFailure: an absent or unreadable key proves nothing about whether the + /// original request will eventually be provably non-applied, so resolution alone can never produce + /// that verdict. `out_token` (optional): set ONLY on `Committed`, to the observed incarnation's token. + CasWriteOutcome resolveByExactGet(std::string_view key, std::string_view expected_bytes, + Token * out_token = nullptr); + + /// Controlled conditional create for content-addressed write-once keys whose body CANNOT be + /// byte-compared across attempts — the blob-body `putIfAbsentStream` create and `promoteStaged`'s + /// conditional server-side copy (`PartWriteTxn::uploadFromSource`). Byte-exact resolve + /// (`resolveByExactGet`) is impossible there BY DESIGN: W-FRESH-TAG mints a fresh + /// `incarnation_tag` into the envelope header on every re-stream, so two attempts of the same + /// logical create legitimately differ in bytes. The identity authority is the KEY itself (it + /// embeds algo + content digest), so an uncertain attempt resolves by exact-key OCCUPANCY (one + /// HEAD — never a GET of a possibly-multi-GB body, and never a GET-revive): + /// - occupant present -> Occupied (definite; whether it is our own landed attempt or a twin is + /// immaterial for a content-addressed key — the caller's ordinary 412 machinery takes over) + /// - absent -> another attempt may be legal (fence/deadline/backoff-gated, same + /// schedule as putIfAbsentControlled); `attempt` re-streams from the caller's REPLAYABLE + /// source — a fresh re-upload, honoring the resurrect invariant + /// - the HEAD fails -> still ambiguous; reissue is safe (an occupant just answers the reissue + /// with PreconditionFailed -> Occupied on the next round) + /// `attempt` performs ONE conditional-create attempt of the same logical content and returns its + /// PutResult (Done/PreconditionFailed) or throws. A whitelisted DefiniteFailure classification + /// RETHROWS the original exception (the blob lane always surfaced the raw storage error's root + /// cause — unlike the ref lane's outcome mapping, nothing here needs the code collapsed), and a + /// deterministic LOCAL failure from the attempt — `LOGICAL_ERROR` (e.g. a source streaming a + /// different byte count than it declared), `NOT_IMPLEMENTED` (a mode/capability guard), + /// `BAD_ARGUMENTS` (a deterministic encode rejection), `CORRUPTED_DATA` (integrity) — propagates + /// unchanged too, on the FIRST attempt with no resolve and no backoff: a caller/config bug reissue + /// would only replay, never a wire ambiguity. + /// A Done attempt gets the final fence check before being reported Committed + /// Occupied needs none — it acks nothing of OUR write, and the + /// caller's occupant path gates its own adoption. + CasCreateResult conditionalCreateControlled(std::string_view key, + const std::function & attempt, + const std::function & fence_ok); + + /// Controlled If-Match overwrite with resolve-before-reissue, for a MUTABLE marker whose bytes + /// are deterministic so GET-based resolution can compare them (unlike a content-addressed + /// create's freshly-minted-per-attempt body). Performs at most `budget.max_attempts` attempts of + /// the exact SAME (key, bytes, expected token), bounded by `budget.operation_deadline_ms`, with + /// the same fence/backoff/deadline gates as `putIfAbsentControlled`. An ambiguous attempt + /// (`PreconditionFailed`, or a transient exception classified `Unresolved`) is resolved with ONE + /// GET at `key`: + /// - the current token still equals `expected` -> the attempt provably never applied; another + /// attempt of the SAME (key, bytes, expected) is legal (fence/backoff/deadline-gated) + /// - the current bytes equal `bytes` -> Committed (an earlier ambiguous attempt of + /// THIS call already landed); `token` is the observed incarnation + /// - neither -> Conflict: a genuine competing write + /// landed; returned as a value, never thrown + /// - the GET itself fails -> still ambiguous; reissue is safe + /// A whitelisted `DefiniteFailure` classification, or a deterministic LOCAL failure + /// (`isDeterministicLocalFailure`), RETHROWS the original exception -- mirrors + /// `conditionalCreateControlled`'s convention, never `putIfAbsentControlled`'s (that method + /// predates this convention). + CasOverwriteResult putOverwriteControlled(std::string_view key, std::string_view bytes, + const Token & expected, const std::function & fence_ok); + + /// Controlled put-if-absent for a MUTABLE marker whose bytes are deterministic, where an + /// EXISTING DIFFERENT value at the key is a normal outcome (Conflict), not corruption. This is + /// the create-side sibling of `putOverwriteControlled` and deliberately does NOT reuse + /// `putIfAbsentControlled`: that method's resolve (`resolveByExactGet`) throws `CORRUPTED_DATA` + /// on any different bytes at the key, which is correct for the ref-log lane's immutable, + /// content-addressed keys (a different value there truly is impossible-by-construction) but + /// wrong for a mutable state marker (e.g. a blob's freshness-meta sidecar), where a + /// pre-existing DIFFERENT value is an expected, non-corrupt state a racing writer or GC pass + /// left behind. Performs at most `budget.max_attempts` attempts of the exact SAME (key, bytes), + /// bounded by `budget.operation_deadline_ms`, with the same fence/backoff/deadline gates as + /// `putIfAbsentControlled`. An ambiguous attempt (`PreconditionFailed`, or a transient exception + /// classified `Unresolved`) is resolved with ONE GET at `key`: + /// - absent -> the attempt provably never applied; another attempt of + /// the SAME (key, bytes) is legal (fence/backoff/deadline-gated) + /// - present, bytes equal `bytes` -> Committed (an earlier ambiguous attempt of THIS call, + /// or a racing writer creating the identical value, already landed); `token` is the observed + /// incarnation + /// - present, bytes differ -> Conflict: something else already occupies the key with + /// a different value; returned as a value, never thrown + /// - the GET itself fails -> still ambiguous; reissue is safe + /// Same DefiniteFailure/deterministic-local-failure rethrow convention as `putOverwriteControlled`. + CasOverwriteResult putIfAbsentControlledMutable(std::string_view key, std::string_view bytes, + const std::function & fence_ok); + + /// A DEDICATED RAW slot-occupy primitive [codex finding 3]: exactly ONE fence/deadline-gated + /// conditional create of `bytes` at `key`; on conflict, exactly ONE raw exact `GET` of the + /// occupant. NEVER retries internally, NEVER lists, and NEVER composes `putIfAbsentControlled` + /// (which retries the SAME (key, bytes) internally) or `resolveByExactGet` (which compares against + /// an expected body and throws `CORRUPTED_DATA` on a mismatch) — both contradict "one conditional + /// create" and `Occupied(bytes, token)` respectively. This is the primitive every seal writer and + /// wedge retry uses (spec INV-2): each CALL is one bounded attempt, and a caller that wants to keep + /// trying calls this again later, under its OWN fence/deadline/backoff discipline. + /// + /// Pre-attempt gate ONLY: `fence_ok` and the operation deadline are checked ONCE, before the (only) + /// attempt — a refusal there sends nothing and reports `Unresolved`/`NoAttemptSent`, exactly like + /// every other controlled op's FIRST iteration. UNLIKE `putIfAbsentControlled` / + /// `conditionalCreateControlled` / `putOverwriteControlled` / `putIfAbsentControlledMutable`, there + /// is deliberately no POST-write fence recheck here: `fence_ok` is evaluated once per attempt, and + /// since this primitive makes exactly one attempt, admission-fence discipline across an outer + /// caller-driven retry (including verifying a `Created`/`Occupied` result is still relevant after + /// the I/O) is the CALLER's contract (Task 4/6's post-I/O recheck under its own state lock). + /// + /// CONSEQUENCE, stated bluntly because it is the OPPOSITE of every sibling op's behavior: a + /// `Created` or `Occupied` returned here may come from a call whose fence was lost WHILE the PUT or + /// GET was in flight — this primitive does not know and does not check. Acting on either result + /// (adopting, acknowledging, installing) without the caller's OWN post-I/O + /// `checkFenceOrThrow(admitted_generation)` under its own lock is a correctness bug, not a missed + /// diagnostic — see Task 4's `resolveWedgeOnce` and Task 6's recovery CAS-walk in the plan for the + /// exact recheck shape. + /// + /// A whitelisted synchronous rejection (`classifyConditionalWriteResult`'s `DefiniteFailure`) or a + /// deterministic local failure (`isDeterministicLocalFailure`) RETHROWS the original exception + /// unchanged — the same convention as `conditionalCreateControlled`/`putOverwriteControlled`/ + /// `putIfAbsentControlledMutable` (`SlotOccupyResult::Kind` has no `DefiniteFailure` member to carry + /// it). Any other exception, or a clean `PreconditionFailed`, is ambiguous and falls through to the + /// resolve GET identically — this primitive cannot and does not distinguish the two. + /// + /// Op-count contract (asserted by every `gtest_cas_slot_occupy.cpp` test): `Created` costs exactly + /// one backend op (the create); `Occupied` costs exactly two (the create, then the resolve GET); + /// `Unresolved` costs at most two (zero when a pre-attempt gate refuses, otherwise the create plus + /// a resolve GET that came up empty or failed). + SlotOccupyResult slotOccupy(std::string_view key, std::string_view bytes, + const std::function & fence_ok); + + /// Test-only: replace the inter-attempt backoff sleep (e.g. with a no-op) on an already-constructed + /// controller — for tests that reach the controller only through a fully-wired Pool/disk and cannot + /// pass the ctor parameter (see `Pool::setCasRetrySleepForTest`). Passing an empty function restores + /// the real sleep. Not thread-safe: call before driving any traffic through the controller. + void setSleepFnForTest(std::function sleep_ms_); + +private: + /// The gate between a completed ambiguous attempt and its reissue: fence check FIRST (a fence lost + /// mid-loop must abort before any sleep), then the capped-exponential backoff sleep — skipped + /// entirely (returning false, no sleep served) when the sleep plus one more attempt could not fit + /// the operation deadline. Returns true when the loop may proceed to the next attempt; the loop + /// top's own pre-attempt fence/deadline checks re-run AFTER the sleep. + /// + /// `out_reason` (optional) receives WHICH of the two refusals returned false, so a caller reporting + /// an `Unresolved` from here does not have to guess between them. Both are mid-way by construction: + /// this gate is only reached once an attempt has been sent. + bool pauseBeforeReissue(uint32_t completed_attempt, uint64_t deadline_ms, const std::function & fence_ok, + CasUnresolvedReason * out_reason = nullptr); + /// The backoff scheduled before attempt `next_attempt` (attempt 2 sleeps `retry_initial_backoff_ms`, + /// doubling per reissue), saturating at `retry_max_backoff_ms`. 0 when backoff is disabled. + uint64_t backoffBeforeAttempt(uint32_t next_attempt) const; + + BackendPtr backend; + CasRequestBudget budget; + std::function now_ms; + std::function sleep_ms; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasSentinelProbe.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasSentinelProbe.cpp new file mode 100644 index 000000000000..af52f2854bfd --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasSentinelProbe.cpp @@ -0,0 +1,110 @@ +#include +#include +#include +#include + +namespace DB::Cas +{ + +SentinelProbeResult probeSentinel(Backend & backend, const String & key) +{ + return backend.probeSentinelRaw(key); +} + +namespace +{ + +/// Is `key` structurally-valid `_probe/` capability-battery debris — i.e. an object strictly under the +/// reserved `/_probe/` subtree? `runCapabilityProbe` (invoked with `/_probe/` by +/// `Pool::open`) is the ONLY writer under `_probe/`; a content-addressed pool NEVER stores durable +/// data/control state there, so the whole subtree is ephemeral capability-probe scratch that a crash or a +/// concurrent fresh opener may leave behind ([D2]). Ignoring it can therefore never strand real data — +/// `pool_prefix` is exclusively CAS-owned. The trailing `/` in `probe_root` keeps a mere sibling +/// look-alike (`/_probe`, `/_probelike/…`) OUT of the reserved subtree, so it is still +/// treated as genuine residual and fails the bootstrap closed. +bool isProbeSubtreeDebris(const String & probe_root, const String & key) +{ + return key.starts_with(probe_root); +} + +} + +BootstrapResidual probePoolBootstrapResidual(Backend & backend, const Layout & layout) +{ + const String pool_meta_key = layout.poolMetaKey(); + const String catalog_key = layout.refCatalogKey(); + const String prefix = layout.poolPrefix() + "/"; + const String probe_root = layout.poolPrefix() + "/_probe/"; + + /// Classification is order-independent for correctness: every listed key is examined, and finding + /// `_pool_meta` anywhere is decisive. It relies on lexicographic LIST order only for COST — `_pool_meta` + /// sorts first under `/`, so a healthy pool short-circuits on the first page rather than + /// enumerating its whole content on every open. + bool has_residual = false; + bool has_catalog = false; + try + { + String cursor; + for (;;) + { + const ListPage page = backend.list(prefix, cursor, 1000); + for (const ListedKey & listed : page.keys) + { + if (listed.key == pool_meta_key) + return BootstrapResidual::PoolMetaPresent; /// decisive — the pool is authoritative + if (isProbeSubtreeDebris(probe_root, listed.key)) + continue; /// crash leftover / concurrent opener's battery — ignore ([D2]) + if (listed.key == catalog_key) + { + has_catalog = true; + continue; + } + has_residual = true; /// a non-`_probe` object, and no `_pool_meta` seen (so far) + } + if (page.next_cursor.empty()) + break; + cursor = page.next_cursor; + } + } + catch (...) + { + /// The LIST failed: absence was never proven. Never mint a fresh identity under uncertainty — + /// but log the swallowed error so the fail-closed startup refusal is diagnosable (the root cause + /// must not vanish just because the verdict is "Indeterminate"). + LOG_WARNING(getLogger("CasBootstrap"), + "Pool prefix '{}': the authoritative residual LIST failed; treating the bootstrap as " + "Indeterminate (fail-closed, will refuse to mint _pool_meta): {}", + prefix, getCurrentExceptionMessage(/*with_stacktrace=*/false)); + return BootstrapResidual::Indeterminate; + } + if (has_residual) + return BootstrapResidual::ResidualWithoutMeta; + if (!has_catalog) + return BootstrapResidual::EmptyOrProbeOnly; + + /// LIST is only a discovery hint. Before treating catalog-only residue as retryable, exact-read + /// the listed key and prove it is precisely the sole canonical empty authority. A missing object, + /// malformed body, alternate encoding, or nonempty catalog is ordinary residual data, never a + /// license to mint `_pool_meta`. + try + { + const auto got = backend.get(catalog_key); + if (!got) + return BootstrapResidual::ResidualWithoutMeta; + + RefCatalog catalog = decodeRefCatalog(got->bytes); + const String canonical_empty = encodeRefCatalog(RefCatalog{}); + if (catalog.entries.empty() && got->bytes == canonical_empty) + return BootstrapResidual::CanonicalEmptyCatalogOnly; + return BootstrapResidual::ResidualWithoutMeta; + } + catch (...) + { + LOG_WARNING(getLogger("CasBootstrap"), + "Pool prefix '{}': could not prove listed catalog '{}' is canonical empty; refusing bootstrap: {}", + prefix, catalog_key, getCurrentExceptionMessage(/*with_stacktrace=*/false)); + return BootstrapResidual::ResidualWithoutMeta; + } +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasSentinelProbe.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasSentinelProbe.h new file mode 100644 index 000000000000..c0d926f1c31d --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasSentinelProbe.h @@ -0,0 +1,55 @@ +#pragma once +#include +#include + +namespace DB::Cas +{ + +/// Authoritative, cache-bypassing probe of one key. NEVER conflates transport errors with absence: +/// timeouts / 5xx / connection errors => Indeterminate; permission errors => AccessDenied; +/// missing container/bucket/prefix-parent => ContainerAbsent; a clean authoritative miss => KeyAbsent. +/// +/// Free-function entry point (spec §2) — a thin dispatch to the backend's own typed-evidence +/// classification (`Backend::probeSentinelRaw`; see there for the per-backend semantics: the +/// S3-native raw HEAD error, the Local container-directory stat, or the generic head/get-based +/// default for a backend without sharper evidence). +SentinelProbeResult probeSentinel(Backend & backend, const String & key); + +/// Verdict of the zero-write startup bootstrap residual check ("Startup ordered vs the capability +/// probe"). Before a writable `Pool::open` runs +/// the MUTATING `_probe/` capability battery, it must decide whether it is safe to bootstrap a missing +/// `_pool_meta`. `pool_prefix` is EXCLUSIVELY CAS-owned: a fresh pool identity may be minted ONLY over a +/// genuinely empty prefix — never over residual data an incomplete erase left behind (that would strand +/// the old objects as orphans under a new, colliding identity, the "restart poisons a partially-erased +/// pool" hole this check closes). +enum class BootstrapResidual : uint8_t +{ + /// `/_pool_meta` exists → the pool is authoritative; proceed with the normal open/validate. + PoolMetaPresent, + /// No `_pool_meta`, and every listed object is structurally-valid `_probe/` debris (or the prefix is + /// empty) → safe to bootstrap a fresh pool. + EmptyOrProbeOnly, + /// No `_pool_meta`, and the only durable object is a byte-for-byte canonical empty + /// `cas/ref_catalog` (plus structurally-valid `_probe/` debris). This is the sole retryable + /// pre-meta bootstrap residue: a prior opener made the mandatory catalog durable but did not + /// publish `_pool_meta`. + CanonicalEmptyCatalogOnly, + /// No `_pool_meta`, but at least one non-`_probe` object exists → refuse to bootstrap (typed startup + /// failure, zero writes performed). + ResidualWithoutMeta, + /// The authoritative LIST itself failed → emptiness could not be proven → fail-closed (never mint a + /// fresh identity while residual data cannot be ruled out). + Indeterminate, +}; + +/// Zero-write authoritative classification of a pool prefix for the startup bootstrap decision. A single +/// paginated LIST of `layout.poolPrefix()`; each listed object is classified as the `_pool_meta` +/// sentinel, capability-battery debris under the reserved `/_probe/` subtree (a crash-mid-battery +/// leftover OR a concurrent fresh opener's in-flight battery — [D2]), or genuine residual CAS state. It +/// NEVER writes, and it IGNORES probe debris exactly so a normal restart after a crash-mid-battery still +/// bootstraps cleanly. On non-strong-LIST backends this is the single best-effort authoritative check the +/// weaker guarantee allows — still fail-closed on any residual object found. Used by `Pool::open` BEFORE +/// the capability battery so that no probe write ever precedes the emptiness proof. +BootstrapResidual probePoolBootstrapResidual(Backend & backend, const Layout & layout); + +} From 58d4f0b9ee024b44684f8d7d3835d27c3a69f116 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:34 +0200 Subject: [PATCH 16/30] CAS subsystem: Pool layer Pool identity and runtime: server root, mount lifecycle, pool metadata, the ref ledger and ref protocol (publish/confirm, recovery, snapshots), the part-write transaction (dedup gate, conditional create, promote), staging and plain objects. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../ContentAddressed/Pool/CasBlobMeta.cpp | 46 + .../ContentAddressed/Pool/CasBlobMeta.h | 65 + .../Pool/CasBlobUploadPool.cpp | 75 + .../ContentAddressed/Pool/CasBlobUploadPool.h | 43 + .../Pool/CasEventDispatcher.cpp | 56 + .../Pool/CasEventDispatcher.h | 64 + .../Pool/CasManifestReader.cpp | 170 + .../ContentAddressed/Pool/CasManifestReader.h | 101 + .../ContentAddressed/Pool/CasMountRuntime.cpp | 545 ++ .../ContentAddressed/Pool/CasMountRuntime.h | 431 ++ .../ContentAddressed/Pool/CasPartWriteTxn.cpp | 1463 +++++ .../ContentAddressed/Pool/CasPartWriteTxn.h | 402 ++ .../ContentAddressed/Pool/CasPlainObjects.cpp | 154 + .../ContentAddressed/Pool/CasPlainObjects.h | 111 + .../ContentAddressed/Pool/CasPool.cpp | 1851 ++++++ .../ContentAddressed/Pool/CasPool.h | 1159 ++++ .../ContentAddressed/Pool/CasPoolMeta.cpp | 168 + .../ContentAddressed/Pool/CasRefCatalog.cpp | 654 +++ .../ContentAddressed/Pool/CasRefCatalog.h | 351 ++ .../ContentAddressed/Pool/CasRefCkpt.cpp | 354 ++ .../ContentAddressed/Pool/CasRefCkpt.h | 172 + .../Pool/CasRefCowManifestSet.cpp | 119 + .../Pool/CasRefCowManifestSet.h | 127 + .../ContentAddressed/Pool/CasRefCowMap.cpp | 245 + .../ContentAddressed/Pool/CasRefCowMap.h | 209 + .../ContentAddressed/Pool/CasRefLedger.cpp | 5113 +++++++++++++++++ .../ContentAddressed/Pool/CasRefLedger.h | 1246 ++++ .../ContentAddressed/Pool/CasRefProtocol.cpp | 1108 ++++ .../ContentAddressed/Pool/CasRefProtocol.h | 794 +++ .../ContentAddressed/Pool/CasServerRoot.cpp | 1512 +++++ .../ContentAddressed/Pool/CasServerRoot.h | 774 +++ 31 files changed, 19682 insertions(+) create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobMeta.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobMeta.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobUploadPool.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobUploadPool.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasEventDispatcher.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasEventDispatcher.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasManifestReader.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasManifestReader.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPlainObjects.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPlainObjects.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPoolMeta.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCatalog.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCatalog.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowManifestSet.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowManifestSet.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowMap.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowMap.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefProtocol.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefProtocol.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.h diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobMeta.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobMeta.cpp new file mode 100644 index 000000000000..d1c44d2b168f --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobMeta.cpp @@ -0,0 +1,46 @@ +#include +#include + +#include + +namespace ProfileEvents +{ + extern const Event CASMetaPut; + extern const Event CASMetaCompareSwap; + extern const Event CASMetaDelete; +} + +namespace DB::Cas +{ + +std::optional loadMeta(Backend & backend, const Layout & layout, const BlobRef & ref) +{ + const String key = layout.blobMetaKey(ref); + auto got = backend.get(key); + if (!got) + return std::nullopt; + return LoadedMeta{.meta = decodeBlobMeta(got->bytes), .etag = got->token}; +} + +CasOverwriteResult putMetaIfAbsent(Pool & pool, const BlobRef & ref, const BlobMeta & meta) +{ + ProfileEvents::increment(ProfileEvents::CASMetaPut); + const String key = pool.layout().blobMetaKey(ref); + return pool.stagingPutIfAbsentMutable(key, encodeBlobMeta(meta)); +} + +CasOverwriteResult casMeta(Pool & pool, const BlobRef & ref, const Token & expected, const BlobMeta & meta) +{ + ProfileEvents::increment(ProfileEvents::CASMetaCompareSwap); + const String key = pool.layout().blobMetaKey(ref); + return pool.stagingConditionalOverwrite(key, encodeBlobMeta(meta), expected); +} + +DeleteOutcome deleteMetaExact(Backend & backend, const Layout & layout, const BlobRef & ref, const Token & expected) +{ + ProfileEvents::increment(ProfileEvents::CASMetaDelete); + const String key = layout.blobMetaKey(ref); + return backend.deleteExact(key, expected); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobMeta.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobMeta.h new file mode 100644 index 000000000000..87b77d1d8657 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobMeta.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace DB::Cas +{ + +class Pool; + +/// A decoded blob meta record together with the backend token observed for the same incarnation. +/// The token is returned with the decoded record because the next conditional update or exact delete +/// must be guarded by the version that was actually read; comparing encoded meta bytes would not +/// provide that protection. +struct LoadedMeta +{ + BlobMeta meta; + Token etag; +}; + +/// Shared lifecycle operations for the blob freshness marker used by the writer and GC. The key is +/// built from the complete `BlobRef`, so each algorithm uses its own digest representation and no +/// pool-wide digest width is threaded through these functions. The marker is a point-read hint rather +/// than the blob lifetime's linearization point: the blob body's incarnation tag and exact-token body +/// deletion provide the safety guarantee, while a stale marker can at most make a writer re-upload. +/// +/// `loadMeta` is used in the adopt path, so its backend must provide strong read-after-write +/// consistency: after a successful meta write, the one subsequent GET must observe that write. +/// Conditional updates and deletion use the backend token, not the encoded meta bytes. +/// +/// Returns the current decoded marker and its conditional token, or nullopt when the meta key is +/// absent. Decoding errors propagate as exceptions. +std::optional loadMeta(Backend & backend, const Layout & layout, const BlobRef & ref); + +/// Creates the marker only when its key is absent, controlled: a SlowDown/429/5xx on the attempt is +/// resolved-and-reissued within budget rather than escaping as a raw client error (triage: S22 RCA). +/// A precondition failure (another +/// writer already created the marker -- possibly with a DIFFERENT record, e.g. a stale `Condemned` +/// marker still present when a vanished body is freshly re-uploaded) is reported as +/// `CasOverwriteOutcome::Conflict`, never thrown -- this uses `putIfAbsentControlledMutable`, NOT the +/// ref-log lane's `putIfAbsentControlled` (that method's resolve throws `CORRUPTED_DATA` on any +/// different bytes at the key, which is correct for the ref-log's immutable content-addressed keys +/// but wrong for this mutable marker, where a pre-existing different value is an expected, non-corrupt +/// outcome). +CasOverwriteResult putMetaIfAbsent(Pool & pool, const BlobRef & ref, const BlobMeta & meta); + +/// Replaces the marker only when its current backend token equals `expected`, controlled (same +/// budgeted resolve-and-reissue as putMetaIfAbsent). A genuine conflict (current token AND bytes both +/// differ from what this call intended) is reported as `CasOverwriteOutcome::Conflict`, never thrown -- +/// exactly like the previous uncontrolled `CasResult` contract -- so the caller's existing +/// reload-and-retry loop (`writeResurrectMetaClean`) keeps working unchanged. +CasOverwriteResult casMeta(Pool & pool, const BlobRef & ref, const Token & expected, const BlobMeta & meta); + +/// Deletes only the marker incarnation identified by `expected`. A token mismatch leaves the current +/// marker untouched; `NotFound` is distinct from that case so callers can tell absence from a raced +/// replacement. The backend's complete `DeleteOutcome` is returned, including any storage-specific +/// delete-marker status. +DeleteOutcome deleteMetaExact(Backend & backend, const Layout & layout, const BlobRef & ref, const Token & expected); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobUploadPool.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobUploadPool.cpp new file mode 100644 index 000000000000..9573563a983a --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobUploadPool.cpp @@ -0,0 +1,75 @@ +#include + +#include +#include +#include + +#include +#include +#include + +namespace CurrentMetrics +{ + extern const Metric CASBlobUploadPoolThreads; + extern const Metric CASBlobUploadPoolThreadsActive; + extern const Metric CASBlobUploadPoolThreadsScheduled; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; +} + +namespace Cas +{ + +namespace +{ + std::mutex pool_mutex; + std::unique_ptr pool_instance; +} + +void initializeBlobUploadPool(size_t size) +{ + if (size == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "cas_blob_upload_pool_size must not be 0"); + + std::lock_guard lock(pool_mutex); + if (pool_instance) + throw Exception(ErrorCodes::LOGICAL_ERROR, "The CAS blob upload pool is initialized twice"); + + pool_instance = std::make_unique( + CurrentMetrics::CASBlobUploadPoolThreads, + CurrentMetrics::CASBlobUploadPoolThreadsActive, + CurrentMetrics::CASBlobUploadPoolThreadsScheduled, + size); +} + +ThreadPool & blobUploadPool() +{ + std::lock_guard lock(pool_mutex); + if (!pool_instance) + throw Exception(ErrorCodes::LOGICAL_ERROR, "The CAS blob upload pool is not initialized"); + + return *pool_instance; +} + +void shutdownBlobUploadPool() noexcept +{ + std::lock_guard lock(pool_mutex); + pool_instance.reset(); +} + +bool blobUploadPoolInitializedForTest() +{ + std::lock_guard lock(pool_mutex); + return pool_instance != nullptr; +} + + +} +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobUploadPool.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobUploadPool.h new file mode 100644 index 000000000000..1e8594b5594b --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasBlobUploadPool.h @@ -0,0 +1,43 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Server-wide pool for the parallel intra-part blob upload fan-out (stage-1 design §1, +/// "Parallel blob upload within a part"). Deliberately disjoint from +/// `IObjectStorage::getThreadPoolWriter`: an upload task may itself submit to the writer pool (S3 +/// multipart), so nesting the fan-out on that same pool would risk the classic same-pool +/// wait-on-self deadlock. The calling thread only submits tasks and joins them -- it never +/// occupies a pool slot itself -- so pool size 1 is a valid (fully serial) configuration, never a +/// deadlock risk. +/// +/// Fail-loud lifecycle: the server (or a test) must call `initializeBlobUploadPool` before any +/// `blobUploadPool` use. There is no lazy self-initialization on the production path. + +/// Throws `BAD_ARGUMENTS` if `size == 0`. Throws `LOGICAL_ERROR` if already initialized. +void initializeBlobUploadPool(size_t size); + +/// Throws `LOGICAL_ERROR` if the pool has not been initialized. The returned reference is only +/// valid while the pool stays initialized: callers must not race this against +/// `shutdownBlobUploadPool` (in the server, shutdown runs after query drain; tests own the order). +ThreadPool & blobUploadPool(); + +/// Idempotent: safe to call multiple times, and safe to call even if never initialized. Joins all +/// outstanding tasks before returning. +void shutdownBlobUploadPool() noexcept; + +/// For tests only: true once `initializeBlobUploadPool` has run, false before that call and after +/// `shutdownBlobUploadPool`. +bool blobUploadPoolInitializedForTest(); + + + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasEventDispatcher.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasEventDispatcher.cpp new file mode 100644 index 000000000000..39152156476f --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasEventDispatcher.cpp @@ -0,0 +1,56 @@ +#include + +#include + +#include + +namespace DB::Cas +{ + +void EventDispatcher::setSink(Sink sink_) +{ + std::lock_guard lock(mutex); + sink = std::move(sink_); + has_sink.store(static_cast(sink), std::memory_order_release); +} + +void EventDispatcher::emit(CasEvent event) +{ + std::unique_lock lock(mutex); + /// Enqueue before deciding who drains. A `bad_alloc` here escapes, but `draining` is untouched and + /// the deque's strong guarantee leaves the queue intact -- the dispatcher is never left wedged. + queue.push_back(std::move(event)); + + /// A drain loop already owns delivery (this call is reentrant from inside the sink, or a concurrent + /// emitter is draining). Enqueue-and-return: the running loop will deliver what we just pushed. + if (draining) + return; + + draining = true; + while (!queue.empty()) + { + CasEvent next = std::move(queue.front()); + queue.pop_front(); + /// The sink runs OUTSIDE `mutex`: a reentrant `emit` can take the lock, and a concurrent + /// emitter can enqueue, neither blocked on this delivery. + lock.unlock(); + try + { + /// `sink` is set pre-traffic and never swapped concurrently with delivery, so reading it + /// here without `mutex` is race-free. + if (sink) + sink(std::move(next)); + } + catch (...) + { + /// Contain the sink failure: dropping one audit event must not abandon the queued + /// remainder nor leave `draining` stuck true (which would silently mute all future events). + DB::tryLogCurrentException("CasEventDispatcher", + "Content-addressed audit event sink threw; the event was dropped"); + } + lock.lock(); + } + draining = false; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasEventDispatcher.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasEventDispatcher.h new file mode 100644 index 000000000000..4bfc8859f26a --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasEventDispatcher.h @@ -0,0 +1,64 @@ +#pragma once + +#include + +#include +#include +#include + +namespace DB::Cas +{ + +/// Serialized, reentrancy-safe delivery for `CasEvent`s. Every content-addressed component that emits +/// audit events (the `Pool`, the ref ledger, the manifest reader, the mount renewer) routes through +/// the ONE dispatcher owned by its `Pool` instead of calling an installed sink `std::function` +/// directly. That gives two properties the parallel intra-part upload fan-out (stage-1 design §1) +/// newly requires: +/// +/// - Serialization: concurrent emitters never run the sink at the same time, so a sink that appends +/// to state without a lock of its own (as every existing sink does) stays correct. +/// - Reentrancy safety: an emission performed FROM INSIDE a sink callback -- the sink calls a ledger +/// read that itself emits -- is queued and drained by the already-running dispatch loop instead of +/// recursing or self-deadlocking on the dispatcher mutex. +/// +/// Why one dispatcher and not a per-component locking wrapper: a wrapper that held its own mutex +/// across each component's sink call would establish a `state_mutex -> event_mutex` lock order and +/// deadlock the instant a reentrant sink took `state_mutex` again. The dispatcher never holds `mutex` +/// across the sink call, so reentrancy cannot deadlock it. It is the necessary-but-not-sufficient half +/// of the contract: ledger emission points are additionally restructured to emit AFTER releasing +/// `state_mutex`, so a sink's own reentrant ledger read can take that lock freshly. +class EventDispatcher +{ +public: + /// The sink type is `CasEventSink` (`std::function`) unchanged: every existing + /// sink stays valid as-is, and the by-value event preserves the move-the-`detail`-map-into-the-sink + /// idiom (`CasEvent.h`) rather than forcing a copy at a `const &` boundary. + using Sink = CasEventSink; + + /// Installs the delivery sink. Pre-traffic only, matching the contract of the setter it replaces + /// (`Pool::setEventSink`): intended for pre-open wiring or tests with no active mount thread. A + /// null sink disables delivery. + void setSink(Sink sink_); + + /// Whether a delivery sink is installed. Lock-free: `sink` is set pre-traffic and never swapped + /// concurrently with `emit`, so the query-frequency disabled hot path pays no mutex. + bool hasSink() const noexcept { return has_sink.load(std::memory_order_acquire); } + + /// Delivers `event`, serialized across threads and safe to call reentrantly from within the sink. + /// Does not propagate a sink exception: a throwing sink is contained per event so one bad event + /// neither abandons the queued remainder nor leaves the dispatcher wedged (a `draining` flag stuck + /// true would silently drop every future event). Takes the event by value so the completed record + /// is moved into the queue and then into the sink, never deep-copied on the emitter thread. The + /// only exception that can escape is an allocation failure while enqueuing, which happens before + /// any dispatcher state changes -- the invariant is never broken. + void emit(CasEvent event); + +private: + std::mutex mutex; + std::deque queue; /// guarded by `mutex` + bool draining = false; /// guarded by `mutex`; the draining thread owns delivery + Sink sink; /// swapped under `mutex`; read on the delivery path without it (no concurrent swap in traffic) + std::atomic has_sink{false}; /// lock-free mirror of `sink` presence for `hasSink` +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasManifestReader.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasManifestReader.cpp new file mode 100644 index 000000000000..778814997ca1 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasManifestReader.cpp @@ -0,0 +1,170 @@ +#include +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event CASPartFolderManifestGets; +} + +namespace CurrentMetrics +{ + extern const Metric CASManifestDecodeCacheBytes; + extern const Metric CASManifestDecodeCacheEntries; +} + +namespace DB +{ +namespace ErrorCodes +{ + extern const int FILE_DOESNT_EXIST; + extern const int CORRUPTED_DATA; + extern const int BAD_ARGUMENTS; +} +} + +namespace DB::Cas +{ + +CasManifestReader::CasManifestReader( + Backend & backend_, const Layout & layout_, const PoolMeta & meta_, + const CasEventSink & event_sink_, size_t manifest_decode_cache_bytes) + : backend(backend_), layout(layout_), meta(meta_), event_sink(event_sink_) +{ + if (manifest_decode_cache_bytes > 0) + manifest_cache = std::make_unique( + "LRU", CurrentMetrics::CASManifestDecodeCacheBytes, CurrentMetrics::CASManifestDecodeCacheEntries, + manifest_decode_cache_bytes, /*max_count=*/16384, ManifestDecodeCache::DEFAULT_SIZE_RATIO); +} + +size_t CasManifestReader::ManifestCacheKeyHash::operator()(const ManifestCacheKey & k) const +{ + /// Combine the manifest-id hash with the token's bytes + type. The token is part of the key so a + /// re-incarnation under the same id misses (the immutable bytes changed identity). + const size_t h1 = std::hash{}(k.manifest_id); + const size_t h2 = std::hash{}(k.token.value); + const size_t h3 = std::hash{}(static_cast(k.token.type)); + size_t h = h1; + h ^= h2 + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2); + h ^= h3 + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2); + return h; +} + +std::shared_ptr CasManifestReader::readManifestShared(const ManifestId & id) +{ + /// A live reference naming a missing manifest body is a dangling-reference violation + /// (`INV-NO-DANGLE`). Never substitute an empty manifest: callers must observe the missing object + /// as an exception. + const String key = layout.manifestKey(id); + + /// `HEAD` is mandatory even on a cache hit. It proves that the live reference still names an + /// existing object and supplies the token that identifies the immutable bytes being reused. + const HeadResult head = backend.head(key); + if (!head.exists) + { + if (event_sink) + { + CasEvent _ev1; + _ev1.type = CasEventType::ReadMissing; + _ev1.object_kind = CasEventObjectKind::Manifest; + _ev1.object_hash = manifestRefDebugString(id.ref); + _ev1.outcome = "missing"; + _ev1.reason = "live ref names manifest but its object is missing (INV-NO-DANGLE)"; + _ev1.detail = {{"code", "FILE_DOESNT_EXIST"}, {"site", "readManifest"}}; + event_sink(std::move(_ev1)); + } + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "live ref names manifest at {} but its object is missing — INV-NO-DANGLE", key); + } + + if (manifest_cache) + if (auto cached = manifest_cache->get(ManifestCacheKey{.manifest_id = id, .token = head.token})) + return cached; + + std::optional object = backend.get(key); + if (!object) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "manifest at {} vanished between head and get — INV-NO-DANGLE", key); + ProfileEvents::increment(ProfileEvents::CASPartFolderManifestGets); + + PartManifest body = decodePartManifest(openObject(FormatId::PartManifest, object->bytes)); + + /// The journal reference must equal the body's self-described `ref`; otherwise the reference + /// addresses a different object and the decoded bytes cannot be trusted. + if (!refMatchesBody(id.ref, body)) + { + if (event_sink) + { + CasEvent _ev2; + _ev2.type = CasEventType::CorruptDecode; + _ev2.object_kind = CasEventObjectKind::Manifest; + _ev2.object_hash = manifestRefDebugString(id.ref); + _ev2.outcome = "corrupt"; + _ev2.reason = "manifest body `ref` does not match the journal ManifestRef (refMatchesBody)"; + _ev2.detail = {{"code", "CORRUPTED_DATA"}, {"site", "readManifest"}}; + event_sink(std::move(_ev2)); + } + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS manifest at {} body ref does not match the journal ManifestRef — refMatchesBody", key); + } + + /// The body's `root_namespace_id` must equal the owning namespace. A mismatch is a + /// cross-namespace dangling reference and would give cleanup the wrong ownership authority. + if (!manifestNamespaceMatches(id.root_namespace, body)) + { + if (event_sink) + { + CasEvent _ev3; + _ev3.type = CasEventType::CorruptDecode; + _ev3.object_kind = CasEventObjectKind::Manifest; + _ev3.object_hash = manifestRefDebugString(id.ref); + _ev3.outcome = "corrupt"; + _ev3.reason = "manifest body root_namespace_id does not match the owning namespace (manifestNamespaceMatches)"; + _ev3.detail = {{"code", "CORRUPTED_DATA"}, {"site", "readManifest"}}; + event_sink(std::move(_ev3)); + } + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS manifest at {} body root_namespace_id does not match the owning namespace — manifestNamespaceMatches", key); + } + + auto decoded = std::make_shared(std::move(body)); + if (manifest_cache) + manifest_cache->set(ManifestCacheKey{.manifest_id = id, .token = head.token}, decoded); + return decoded; +} + +PartManifest CasManifestReader::readManifest(const ManifestId & id) +{ + return *readManifestShared(id); +} + +BlobLocation CasManifestReader::locate(const ManifestEntry & entry) const +{ + /// A ranged read into the content object: the payload starts at a constant offset for blobs + /// (the pool's fixed blob_header_len — no per-object header read). Inline carries no standalone + /// object location (there is no Subtree placement on a part manifest). + switch (entry.placement) + { + case EntryPlacement::Blob: + { + /// The entry carries the complete blob reference (algorithm and digest), so the key is + /// derived directly from it. The payload starts at the pool's fixed envelope length; + /// no object-specific header read is needed before the ranged payload read. + return BlobLocation{ + .key = layout.blobKey(entry.ref), + .offset = meta.blob_header_len, + .length = entry.blob_size, + }; + } + case EntryPlacement::Inline: + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "entry placement {} has no blob location", static_cast(entry.placement)); + } + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "entry placement {} has no blob location", static_cast(entry.placement)); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasManifestReader.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasManifestReader.h new file mode 100644 index 000000000000..af5d31776857 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasManifestReader.h @@ -0,0 +1,101 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// The object key and ranged payload window for a manifest entry stored as a separate blob. The +/// offset is measured from the beginning of the blob object and skips its fixed envelope; `length` +/// is the entry's raw file size. Inline entries do not have a `BlobLocation`. +struct BlobLocation +{ + String key; + uint64_t offset = 0; /// payload start within the object + uint64_t length = 0; +}; + +/// Reads and validates part manifests, caches immutable decodes, and translates blob entries into +/// ranged object reads. A read first obtains the object's current backend token, then reuses a +/// decode only for the matching `(ManifestId, Token)` pair; a cache miss performs a `GET` and +/// validates both the manifest reference and owning namespace before publication into the cache. +/// Missing or changing objects and failed identity checks are surfaced as exceptions, never as an +/// empty or partially trusted manifest. +/// +/// The reader receives its backend, immutable layout and pool metadata, and event sink by reference; +/// it has no `Pool` back-reference and owns no `Pool`-level mutex. The decode cache is a +/// byte-weighted `CacheBase` LRU whose synchronization is internal to `CacheBase`; a null cache +/// means caching is disabled (`manifest_decode_cache_bytes == 0`). +class CasManifestReader +{ +public: + /// Binds the reader to the pool environment. A positive cache budget creates the byte-weighted + /// LRU; zero disables caching while leaving the mandatory `HEAD` and validation sequence intact. + CasManifestReader( + Backend & backend_, const Layout & layout_, const PoolMeta & meta_, + const CasEventSink & event_sink_, size_t manifest_decode_cache_bytes); + + /// Reads a manifest by value using the fail-closed sequence described above. A missing body, + /// disappearance between `HEAD` and `GET`, decode failure, or either identity mismatch throws; + /// only a fully validated decode can enter the cache. + PartManifest readManifest(const ManifestId & id); + + /// Reads a manifest like `readManifest` but returns the immutable shared decode. This preserves + /// the cache's value on the part-folder path and avoids copying all manifest entries on success. + std::shared_ptr readManifestShared(const ManifestId & id); + + /// Computes the object key and payload window for a `Blob` entry without performing I/O. An + /// `Inline` entry, or any unsupported placement value, throws `BAD_ARGUMENTS` because it has no + /// standalone object to read. + BlobLocation locate(const ManifestEntry & entry) const; + + /// Test seam: retained bytes of the manifest decode cache (0 when disabled). + size_t manifestDecodeCacheBytes() const { return manifest_cache ? manifest_cache->sizeInBytes() : 0; } + +private: + /// The cache must include the backend token: a reused manifest identifier can refer to a new + /// object incarnation, and its immutable decoded bytes must not be reused across incarnations. + struct ManifestCacheKey + { + ManifestId manifest_id; + Token token; + bool operator==(const ManifestCacheKey &) const = default; + }; + + /// Hashes both identity components and the token type so cache lookup uses the same complete + /// identity as `ManifestCacheKey::operator==`. + struct ManifestCacheKeyHash + { + size_t operator()(const ManifestCacheKey & k) const; + }; + + /// Estimates retained decode memory from fixed object overhead plus entry path and inline-byte + /// storage. Weighting by bytes gives a server reading many parts an honest memory ceiling instead + /// of a count-only bound; the cache key still provides the fail-closed token semantics. + struct PartManifestWeight + { + /// Returns the approximate bytes retained for one decoded manifest by the cache. + size_t operator()(const PartManifest & m) const + { + size_t bytes = 256; + for (const auto & e : m.entries) + bytes += e.path.size() + e.inline_bytes.size() + 96; + return bytes; + } + }; + using ManifestDecodeCache = CacheBase; + + Backend & backend; + const Layout & layout; + const PoolMeta & meta; + const CasEventSink & event_sink; + std::unique_ptr manifest_cache; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.cpp new file mode 100644 index 000000000000..21fb2fdbcf96 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.cpp @@ -0,0 +1,545 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} +} + +namespace ProfileEvents +{ + extern const Event CASIdentityLost; + extern const Event CASDataRootVanished; +} + +namespace DB::Cas +{ + +namespace +{ +/// Wall-clock seconds since epoch — the `since` timestamp the lifecycle snapshot reports (spec §7). A +/// wall clock, deliberately unlike the fence's `CLOCK_BOOTTIME`: this is an operator-facing DateTime, not +/// an interval measured across a possible VM suspend. +int64_t wallClockNowSeconds() +{ + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); +} +} + +CasMountRuntime::CasMountRuntime( + BackendPtr backend_ptr_, + const Layout & layout_, + MountConfig config_, + String server_root_id_, + const CasEventSink & event_sink_, + CasRequestBudget cas_request_budget_, + std::function remount_attempt_) + : backend_ptr(std::move(backend_ptr_)) + , layout(layout_) + , config(std::move(config_)) + , server_root_id(std::move(server_root_id_)) + , event_sink(event_sink_) + , cas_request_budget(cas_request_budget_) + , remount_attempt(std::move(remount_attempt_)) +{ +} + +uint64_t CasMountRuntime::bootMs() +{ + struct timespec ts{}; + clock_gettime(CLOCK_BOOTTIME, &ts); + return static_cast(ts.tv_sec) * 1000 + static_cast(ts.tv_nsec) / 1000000; +} + +uint64_t CasMountRuntime::bootMsNow() const +{ + return config.boot_ms_fn ? config.boot_ms_fn() : bootMs(); +} + +void CasMountRuntime::waitSleep(uint64_t ms) const +{ + if (config.wait_sleep_fn) + config.wait_sleep_fn(ms); + else + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); +} + +bool CasMountRuntime::mayMutate() const +{ + return !mount_fence.lost.load(std::memory_order_acquire) + && bootMsNow() < mount_fence.deadline_boot_ms.load(std::memory_order_acquire); +} + +void CasMountRuntime::tripMountLost() +{ + mount_fence.lost.store(true, std::memory_order_release); + /// A durable-effect caller admitted under the incarnation this trip just ended must never conclude + /// the fence is fine again just because a LATER `armMountFence` happens to re-arm it (rev.7 [C2]). + fence_generation.fetch_add(1, std::memory_order_acq_rel); + /// The lease-loss event is exactly the `Live -> TransientNotLive` transition of the §1 state model. + /// Idempotent and terminal-safe (a compare-exchange from `Live` only). + noteLeaseLost(); +} + +void CasMountRuntime::checkFenceOrThrow(uint64_t admitted_generation) const +{ + /// [D5]: tell only what is known here. A tripped fence (or a bumped generation) means this node no + /// longer holds the mount incarnation the caller was admitted under -- but this same guard trips for a + /// transient lease blip AND for a deliberate terminal decommission (FORGET) or a lost identity, and this + /// code cannot tell them apart. So the CONDITION must NOT promise recovery ("temporarily unreachable" + /// would misdiagnose the terminal case); it names both possibilities and points at the authoritative + /// lifecycle. The CLASS is the write plane's uniform transient one (its 32 sibling write-transient sites + /// already mint it): under genuine ambiguity the refusal must be retried, never consumed as damage. + if (!mayMutate() || fenceGeneration() != admitted_generation) + throwCasTransientUnavailable( + fmt::format("content-addressed pool '{}'", server_root_id), + "mount fence tripped: the durable write is refused because this node no longer holds the mount " + "incarnation it was admitted under -- either a lease loss the disk auto-recovers from, or a " + "FORGET decommission / lost identity that does NOT recover; consult " + "system.cas_mounts for the disk's lifecycle before retrying"); +} + +bool CasMountRuntime::refAppendFenceOk() const +{ + /// `mayMutate` checks the latch and deadline. The additional budget check prevents starting a + /// controlled request that cannot plausibly finish, including its safety margin, before expiry. + if (mount_fence.lost.load(std::memory_order_acquire)) + return false; + const uint64_t now = bootMsNow(); + const uint64_t deadline = mount_fence.deadline_boot_ms.load(std::memory_order_acquire); + if (now >= deadline) + return false; + const uint64_t margin = cas_request_budget.attempt_timeout_ms + cas_request_budget.lease_safety_margin_ms; + return margin < deadline - now; +} + +void CasMountRuntime::setMountDeadline(uint64_t deadline_boot_ms) +{ + mount_fence.deadline_boot_ms.store(deadline_boot_ms, std::memory_order_release); +} + +void CasMountRuntime::armMountFence(UInt128 server_uuid, uint64_t writer_epoch, uint64_t deadline_boot_ms) +{ + mount_fence.server_uuid = server_uuid; + mount_fence.writer_epoch = writer_epoch; + mount_fence.deadline_boot_ms.store(deadline_boot_ms, std::memory_order_release); + /// A fresh lease incarnation is a fresh generation too: a durable-effect caller admitted under the + /// PRIOR incarnation must re-check and abort rather than ride this re-arm through (rev.7 [C2]). + fence_generation.fetch_add(1, std::memory_order_acq_rel); + if (arm_mount_fence_interposition_hook_for_test) + arm_mount_fence_interposition_hook_for_test(); + /// Open the gate LAST. A caller that observes `lost == false` with acquire semantics must also see + /// the fresh generation; publishing the latch first exposes one admission window in which the dead + /// generation looks live again. + mount_fence.lost.store(false, std::memory_order_release); +} + +uint64_t CasMountRuntime::minActive() +{ + std::lock_guard lk(builds_mutex); + return active_build_seqs.empty() ? next_build_seq : *active_build_seqs.begin(); +} + +uint64_t CasMountRuntime::peekNextBuildSeq() +{ + std::lock_guard lk(builds_mutex); + return next_build_seq; +} + +void CasMountRuntime::renewWatermarkOnce() +{ + /// A read-only runtime has no heartbeat to renew. Report that misuse instead of fabricating a keeper + /// or silently treating the call as successful. + if (!mount_keeper) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS heartbeat: renewWatermarkOnce on a read-only Pool"); + mount_keeper->renewOnce(); +} + +uint64_t CasMountRuntime::allocateBuildSeq() +{ + std::lock_guard lk(builds_mutex); + const uint64_t s = next_build_seq++; + active_build_seqs.insert(s); + return s; +} + +void CasMountRuntime::registerInflightBuild(uint64_t seq, const PartWriteTxnPtr & build) +{ + /// The caller owns the build's shared pointer. Keep only a weak reference here so the registry does + /// not extend the build lifetime; publication, abandonment, or destruction removes the entry. + std::lock_guard lk(builds_mutex); + inflight_builds[seq] = build; +} + +void CasMountRuntime::retireBuildSeq(uint64_t seq) +{ + std::lock_guard lk(builds_mutex); + active_build_seqs.erase(seq); + inflight_builds.erase(seq); +} + +void CasMountRuntime::cancelInflightBuildsForNamespace(const RootNamespace & ns) +{ + /// The removal callback is invoked only after the namespace-removal transaction is durable. Keep + /// cancellation outside `builds_mutex`; `cancelForNamespaceRemoval` changes the build's atomic + /// cancellation state and does not require the registry lock. + std::vector builds_to_check; + { + std::lock_guard lk(builds_mutex); + for (const auto & entry : inflight_builds) + if (auto build = entry.second.lock()) + builds_to_check.push_back(std::move(build)); + } + for (const auto & build : builds_to_check) + build->cancelForNamespaceRemoval(ns); +} + +void CasMountRuntime::mintRandomProcessEpoch() +{ + /// Mint a nonzero equality-only identity. Keep it away from the zero/unarmed and UINT64_MAX/retired + /// sentinels; 52 random bits are sufficient for the expected collision risk of this token. + constexpr uint64_t EPOCH_MASK = (1ULL << 52) - 1; + process_epoch.store( + (thread_local_rng() ^ (static_cast(thread_local_rng()) << 32)) & EPOCH_MASK, + std::memory_order_relaxed); + if (process_epoch.load(std::memory_order_relaxed) == 0) + process_epoch.store(1, std::memory_order_relaxed); +} + +void CasMountRuntime::setProcessEpoch(uint64_t v, std::memory_order order) +{ + process_epoch.store(v, order); +} + +void CasMountRuntime::setLiveWriterEpoch(uint64_t v) +{ + live_writer_epoch.store(v, std::memory_order_release); +} + +void CasMountRuntime::installKeeper(UInt128 our_uuid, uint64_t writer_epoch, const std::function & now_ms) +{ + /// The mount object already contains this runtime's live `(uuid, epoch)` body. Construct the keeper + /// to adopt that exact slot rather than triggering its double-start guard. The keeper reads the + /// build-watermark floor through `minActive` while preparing each renewal. + const uint64_t ttl_ms = static_cast(config.mount_lease_ttl_ms.count()); + mount_keeper = std::make_unique( + backend_ptr, layout, server_root_id, our_uuid, writer_epoch, + config.mount_lease_ttl_ms, now_ms, + [this] { return minActive(); }, + [this](CasEvent e) { emitEvent(std::move(e)); }, + std::chrono::milliseconds(cas_request_budget.lease_safety_margin_ms), + [this] { return bootMsNow(); }); + /// Install the fence callbacks before any background renewal can run: successful renewals extend the + /// local BOOTTIME deadline, while a superseded or foreign renewal latches the fence and starts recovery. + mount_keeper->setFenceCallbacks( + [this, ttl_ms](uint64_t attempt_boot_ms) { setMountDeadline(attempt_boot_ms + ttl_ms); }, + [this] + { + tripMountLost(); + /// Recover as a fresh incarnation; a fenced `(uuid, writer_epoch)` pair is never resurrected. + scheduleRemount(); + }); +} + +void CasMountRuntime::keeperStart() +{ + mount_keeper->start(); +} + +void CasMountRuntime::keeperRenewOnce() +{ + mount_keeper->renewOnce(); +} + +void CasMountRuntime::keeperReset() +{ + mount_keeper.reset(); +} + +void CasMountRuntime::keeperStartBackground(std::chrono::milliseconds period) +{ + mount_keeper->startBackground(period); +} + +void CasMountRuntime::keeperStopBackground() +{ + mount_keeper->stopBackground(); +} + +bool CasMountRuntime::isVanished() const +{ + const PoolLifecycle s = lifecycle(); + return s == PoolLifecycle::VanishedReplaced + || s == PoolLifecycle::VanishedForgotten; +} + +void CasMountRuntime::setLifecycleForTest(PoolLifecycle lc) +{ + /// Direct store, no precondition — the test harness pins an exact cell of the class × state table. + /// A `Vanished*` value also latches `vanished_intent` so the forced terminal state matches what a + /// natural `enterVanished` would leave behind (its truth semantics never depend on how it was reached). + /// Stamp `since` to match a naturally-reached state (release-store before the state store below, so a + /// snapshot reader that acquire-observes the forced state also observes the timestamp): 0 for `Live`, + /// now for every non-`Live` value. Keeps the forced cell of the class × state table indistinguishable + /// from a real transition for the introspection snapshot. + lifecycle_since_wall_s.store(lc == PoolLifecycle::Live ? 0 : wallClockNowSeconds(), std::memory_order_release); + pool_lifecycle.store(lc, std::memory_order_release); + if (lc == PoolLifecycle::VanishedReplaced + || lc == PoolLifecycle::VanishedForgotten) + { + vanished_intent.store(true, std::memory_order_release); + /// Keep the terminal-state guard consistent with the forced state, so a later `enterVanished` + /// (unusual, but not forbidden) is a clean no-op rather than re-storing / re-logging. + terminal_state_published.store(true, std::memory_order_release); + } +} + +void CasMountRuntime::noteLeaseLost() +{ + /// `Live -> TransientNotLive`, and nothing else. A compare-exchange FROM `Live` leaves every other + /// state untouched, so a terminal state is never downgraded and a repeated call is a no-op. This is + /// the only transition the keeper thread performs, and it needs no lock because of that discipline. + PoolLifecycle expected = PoolLifecycle::Live; + if (pool_lifecycle.compare_exchange_strong( + expected, PoolLifecycle::TransientNotLive, std::memory_order_acq_rel, std::memory_order_acquire)) + { + /// The `since` the lifecycle snapshot reports for `not_live` — the wall-clock instant this became + /// non-`Live`. Only the winning transition writes it (the guard above), so it is not re-stamped. + lifecycle_since_wall_s.store(wallClockNowSeconds(), std::memory_order_release); + } +} + +void CasMountRuntime::noteRemounted() +{ + /// `TransientNotLive -> Live` on a successful reclaim. A compare-exchange FROM `TransientNotLive` + /// never revives `IdentityLost` or a `Vanished` state ([D3]) and is a no-op if already `Live`. + PoolLifecycle expected = PoolLifecycle::TransientNotLive; + if (pool_lifecycle.compare_exchange_strong( + expected, PoolLifecycle::Live, std::memory_order_acq_rel, std::memory_order_acquire)) + { + /// Back to `Live`: the lifecycle snapshot reports no `since` (0) for a live pool. + lifecycle_since_wall_s.store(0, std::memory_order_release); + } +} + +void CasMountRuntime::enterIdentityLost() +{ + /// `TransientNotLive -> IdentityLost`, one way. The compare-exchange FROM `TransientNotLive` gives + /// the brief's "from TransientNotLive only" precondition, idempotency (a second call finds the state + /// already `IdentityLost` and its exchange fails), and safety against a concurrent keeper + /// `noteLeaseLost` (which only ever moves `Live -> TransientNotLive`, never away from it). It does NOT + /// set `vanished_intent` (that latch is reserved for the `Vanished*` idempotency/FORGET protocol); + /// rev.8 makes `IdentityLost` a fail-loud TERMINAL state through `remountTerminal()`, which folds it + /// into the observer-exit boundary alongside `vanished_intent`, so the remount/GC observer threads + /// self-exit rather than demote. + /// `since` for the `identity_lost` snapshot row — the wall-clock instant the observer proved the + /// sentinels gone. Stamped (release) BEFORE the CAS that publishes `IdentityLost`, so a reader that + /// acquire-observes `IdentityLost` is guaranteed to observe this timestamp too (the winning CAS's + /// release carries this prior store) — the same before-publish ordering `enterVanished` uses. Safe to + /// stamp before the CAS here, unlike the lock-free `noteLeaseLost`: this runs only from + /// `TransientNotLive` under `Pool::remount_mutex` (a `Vanished` pool bailed at the caller's + /// `isVanished()` gate and the caller guards `!= IdentityLost`), so the CAS wins deterministically and + /// the stamp can never land on a state we did not transition. + lifecycle_since_wall_s.store(wallClockNowSeconds(), std::memory_order_release); + + PoolLifecycle expected = PoolLifecycle::TransientNotLive; + if (!pool_lifecycle.compare_exchange_strong( + expected, PoolLifecycle::IdentityLost, std::memory_order_acq_rel, std::memory_order_acquire)) + return; + + ProfileEvents::increment(ProfileEvents::CASIdentityLost); + LOG_WARNING(getLogger("CasPool"), + "Content-addressed pool '{}' entered IdentityLost: the pool sentinels (_pool_meta and the owner " + "anchor) are authoritatively absent (both KeyAbsent). This is a fail-loud TERMINAL state: " + "store-class access now fails loud and this pool's remount + GC threads self-exit. " + "Recover by restart or SYSTEM CAS FORGET — a matching-sentinel restore does NOT " + "auto-revive this disk.", + server_root_id); +} + +void CasMountRuntime::enterVanished(PoolLifecycle which, const String & reason) +{ + /// Validate the target BEFORE mutating any state — `enterVanished` takes only the two `Vanished*` + /// values (`VanishedReplaced`/`VanishedForgotten`); fail loud on a call-site bug rather than store a + /// non-terminal value or mislabel it. + const char * label = nullptr; + switch (which) + { + case PoolLifecycle::VanishedReplaced: label = "replaced"; break; + case PoolLifecycle::VanishedForgotten: label = "forgotten"; break; + case PoolLifecycle::Live: + case PoolLifecycle::TransientNotLive: + case PoolLifecycle::IdentityLost: + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CasMountRuntime::enterVanished called with a non-terminal lifecycle value"); + } + + /// Publish the terminal-intent latch (spec §3). For a natural transition this is the FIRST publish; for + /// FORGET, `publishVanishedIntent` already set it at step 1. Either way it is published before the state + /// store below. + vanished_intent.store(true, std::memory_order_release); + + /// Idempotency guard for the STATE transition, keyed on a dedicated latch rather than + /// `vanished_intent` (which FORGET publishes early): the FIRST winner stores the state, records the + /// reason, and logs; a later call returns here without re-storing or re-logging. + if (terminal_state_published.exchange(true, std::memory_order_acq_rel)) + return; + + /// Record the reason BEFORE the state's release-store, so a reader that acquire-observes the terminal + /// state (e.g. `Pool::throwIfLifecycleTerminal`) also observes this string. Written exactly once. + vanished_reason = reason; + + /// The `since` the lifecycle snapshot reports for the `vanished` row — the wall-clock instant of the + /// terminal transition. Written before the `pool_lifecycle` release-store below, same as the reason, + /// so an acquire-observer of the terminal state also observes it. + lifecycle_since_wall_s.store(wallClockNowSeconds(), std::memory_order_release); + + /// An unconditional store is safe now: the guard above serializes terminal transitions, and no + /// non-terminal transition can move a `Vanished` state (their compare-exchanges are keyed on + /// `Live`/`TransientNotLive`), so this value is absorbing. + pool_lifecycle.store(which, std::memory_order_release); + + ProfileEvents::increment(ProfileEvents::CASDataRootVanished); + LOG_WARNING(getLogger("CasPool"), + "Content-addressed pool '{}' entered Vanished({}): {}. The disk stays registered but store-class " + "access now fails loud with a typed error (truth); restart re-registers the name.", + server_root_id, label, reason); +} + +void CasMountRuntime::publishVanishedIntent() +{ + /// spec §5 step 1: publish the terminal-intent latch WITHOUT settling the state. The keeper callback + /// (`scheduleRemount`) and the remount loop both consult `vanished_intent` at their step boundaries, so + /// this stops new remount scheduling and makes an in-flight remount loop bail at its next step — + /// bounding FORGET's subsequent joins to one step + one backend timeout. The state store + WARN follow + /// in `enterVanished` (step 6). Idempotent. + vanished_intent.store(true, std::memory_order_release); +} + +void CasMountRuntime::scheduleRemount() +{ + /// Count every entry before checking whether background work is enabled. Tests can therefore observe + /// the keeper's loss callback without depending on a recovery thread being spawned. + schedule_remount_calls_for_test.fetch_add(1, std::memory_order_relaxed); + if (!config.background_watermark) + return; + /// A terminal pool never claims/allocates/writes again (spec §3): the keeper callback must not arm a + /// recovery thread. `remountTerminal()` covers a published terminal `Vanished` intent (`vanished_intent`, + /// set by `publishVanishedIntent` at spec §5 step 1 for FORGET, or as `enterVanished`'s first step for a + /// natural transition, and subsuming every settled `Vanished*` state) AND `IdentityLost` (rev.8: now a + /// fail-loud terminal state — no demoted observer). + if (remount_shutting_down.load() || remount_running.load() || remountTerminal()) + return; + std::lock_guard g(remount_thread_mutex); + if (remount_shutting_down.load() || remount_running.load() || remountTerminal()) + return; + if (remount_thread.joinable()) + remount_thread.join(); /// Reap a previous recovery before starting a new one. + remount_running.store(true); + remount_thread = ThreadFromGlobalPool([this] + { + setThreadName(ThreadName::CAS_REMOUNT); + uint64_t backoff_ms = 1000; + /// Exit at any step boundary once the pool is (being driven) terminal (spec §3). `remountTerminal()` + /// bails on a published terminal `Vanished` intent (`vanished_intent` — by FORGET at step 1, before + /// it joins this thread, or as `enterVanished`'s first step for a natural transition, and subsuming + /// every settled `Vanished*` state) AND on `IdentityLost` (rev.8: a fail-loud terminal state whose + /// identity gate just set it — the thread self-exits at the next boundary, ending the observer). + while (!remount_stop.load() && !remountTerminal()) + { + if (remount_attempt()) + break; + std::unique_lock lk(remount_cv_mutex); + remount_cv.wait_for(lk, std::chrono::milliseconds(backoff_ms), + [this] { return remount_stop.load(); }); + backoff_ms = std::min(backoff_ms * 2, 30000); + } + remount_running.store(false); + }); +} + +bool CasMountRuntime::scheduleRemountForTest() +{ + scheduleRemount(); + std::lock_guard g(remount_thread_mutex); + return remount_thread.joinable(); +} + +void CasMountRuntime::beginShutdownForTest() +{ + std::lock_guard g(remount_thread_mutex); + remount_shutting_down.store(true); +} + +void CasMountRuntime::stopRemountThread() +{ + /// Refuse further recovery arming under the same mutex used by `scheduleRemount`, before joining. + /// Thus a keeper callback racing with teardown cannot re-arm the recovery thread after the join. + { + std::lock_guard g(remount_thread_mutex); + remount_shutting_down.store(true); + } + /// Stop recovery first; it could otherwise recreate the keeper while the heartbeat is being retired. + remount_stop.store(true); + remount_cv.notify_all(); + { + std::lock_guard g(remount_thread_mutex); + if (remount_thread.joinable()) + remount_thread.join(); + } +} + +void CasMountRuntime::finishTeardown(bool drained) +{ + /// On a drained teardown, `stop` writes an already-expired lease and the watermark farewell + /// (`min_active = UINT64_MAX`). This lets the same server reclaim immediately while retaining the + /// durable owner and epoch. A failure, such as another incarnation touching the slot, must not escape + /// destruction; log it and continue teardown. + if (mount_keeper) + { + if (drained) + { + try + { + mount_keeper->stop(); + } + catch (...) + { + tryLogCurrentException(getLogger("CasPool"), "CAS mount-lease: release during Pool teardown failed"); + } + } + else + { + /// If draining did not certify that every in-flight PUT resolved, a clean farewell would be + /// false evidence. Stop background renewal without a terminal operation so the successor uses + /// the slower but safe observation-based reclaim path. + LOG_WARNING(getLogger("CasPool"), + "CAS store shutdown with an unresolved ref-log PUT: skipping the clean-release marker; " + "the next mount will treat this end as unclean"); + mount_keeper->stopBackground(); + } + } + + /// The second join closes the residual window where a keeper loss callback observed the shutdown gate + /// late during the heartbeat stop operation. + { + std::lock_guard g(remount_thread_mutex); + if (remount_thread.joinable()) + remount_thread.join(); + } +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.h new file mode 100644 index 000000000000..81d9da0ba9b9 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.h @@ -0,0 +1,431 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +class PartWriteTxn; +using PartWriteTxnPtr = std::shared_ptr; + +/// The pool-level lifecycle condition (rev.7 §1) a `Pool` moves through as its shared backing changes +/// underfoot. It is distinct from the storage-level `Constructing/Started/ShutDown` lifecycle (a null +/// published pool -- before `startup`/after `shutdown`) the metadata storage tracks. Ordering of the +/// enumerators is not significant; membership tests do the work. +/// - `Live` — the steady state; the mount lease is (or was last) held. +/// - `TransientNotLive` — the lease was lost; access is uncertain and a self-remount retries. The §2 +/// `Present`+identity-match recovery rule fires only from here (or `Live`). +/// - `IdentityLost` — the pool sentinels are authoritatively absent (both KeyAbsent): +/// fail-loud and TERMINAL (rev.8). The remount/GC observer threads self-exit; +/// matching-sentinel reappearance does NOT auto-revive it ([D3]); recovery is a +/// restart or `SYSTEM CAS FORGET`. +/// - `Vanished*` — fully terminal truth: the data root was replaced by a foreign pool, or the +/// disk was decommissioned by `FORGET`. Store-class access fails loud from here. +enum class PoolLifecycle : uint8_t +{ + Live, + TransientNotLive, + IdentityLost, + VanishedReplaced, + VanishedForgotten, +}; + +/// Configuration owned by `CasMountRuntime`. `PoolConfig::mountConfig` projects the flat pool settings +/// into this value, keeping the pool's existing configuration interface unchanged while allowing this +/// lower-layer header to describe its own dependencies. +struct MountConfig +{ + std::chrono::milliseconds mount_lease_ttl_ms{30000}; + std::chrono::milliseconds mount_renew_period{10000}; + /// When false, tests drive `renewWatermarkOnce` explicitly. In production this flag enables both + /// the merged mount-lease/build-watermark heartbeat and self-remount recovery. + bool background_watermark = false; + std::function boot_ms_fn = {}; + std::function wait_sleep_fn = {}; +}; + +/// Local, in-memory write fence. It is deliberately not checked by reading the object store for every +/// write: the `MountLeaseKeeper` is the sole lease reader/renewer. A successful renewal translates the +/// durable `expires_at_ms` into `deadline_boot_ms`; a foreign owner, newer `writer_epoch`, or failed +/// renewal latches `lost`. Mutable operations are allowed only while the latch is clear and the local +/// deadline has not passed. The `writer_epoch` is the durable fencing token. +/// +/// The fence uses `CLOCK_BOOTTIME`, not `CLOCK_MONOTONIC`: monotonic time does not advance while a VM is +/// suspended, so a resumed sleeper would compute the same "not yet expired" verdict it had before the nap +/// even though wall time (and the GC leader's fence-out) moved far ahead — it could mutate the shared state +/// under a live writer. +/// `CLOCK_BOOTTIME` includes suspend time, so a resumed sleeper sees its fence expired. +/// Container pause is already safe under either clock (the process is frozen, so no local check runs). +struct MountFence +{ + UInt128 server_uuid{}; + uint64_t writer_epoch = 0; + /// Until something arms a real lease deadline, the permissive default allows mutations. UINT64_MAX = + /// unarmed (never expires); otherwise a CLOCK_BOOTTIME-milliseconds instant. + std::atomic deadline_boot_ms{std::numeric_limits::max()}; + std::atomic lost{false}; +}; + +/// Owns the live writer-incarnation mechanics shared by the pool's mount and recovery orchestration: +/// the `MountLeaseKeeper`, local `MountFence`, build watermark and in-flight build registry, +/// `live_writer_epoch`, unclean-boundary marker, and self-remount thread. `Pool` retains the higher-level +/// claim/recovery sequence and its `remount_mutex`; in particular, the runtime does not acquire or own +/// the ref-ledger locks. The runtime receives its backend, layout, configuration, event sink, request +/// budget, and a callback that performs one pool-level remount attempt, so it has no `Pool` back-reference. +/// `Pool` delegates preserve the existing callers and test seams. +class CasMountRuntime +{ +public: + CasMountRuntime( + BackendPtr backend_ptr_, + const Layout & layout_, + MountConfig config_, + String server_root_id_, + const CasEventSink & event_sink_, + CasRequestBudget cas_request_budget_, + /// One pool-level recovery attempt. The callback captures the owning `Pool` and is invoked only + /// after construction, from the recovery thread. + std::function remount_attempt_); + + /// ---- per-server watermark and identity ---- + /// `process_epoch` is random and nonzero for this pool incarnation. GC compares it for equality, + /// never ordering; a different value means that the previous writer incarnation is no longer live. + uint64_t epoch() const { return process_epoch.load(std::memory_order_acquire); } + uint64_t writerEpoch() const { return process_epoch.load(std::memory_order_acquire); } + /// The GC floor: the oldest in-flight build_seq, or next_build_seq when no build is active (so a + /// quiescent server's watermark floor advances to the next-to-be-allocated seq). Locks builds_mutex. + uint64_t minActive(); + /// Test/assertion accessor for the next-to-allocate build_seq under the lock. + uint64_t peekNextBuildSeq(); + /// Renew the merged mount heartbeat once, including its build-watermark floor. A read-only runtime + /// has no keeper and fails with a logical exception rather than fabricating a heartbeat. + void renewWatermarkOnce(); + + /// ---- local write fence ---- + /// Return whether a mutable operation may start under the locally observed lease state. + bool mayMutate() const; + /// Permanently latch the local fence as lost for this runtime incarnation. + void tripMountLost(); + /// Publish the BOOTTIME deadline from a successful lease renewal. + void setMountDeadline(uint64_t deadline_boot_ms); + /// Arm a new lease incarnation and clear any loss latched for the prior incarnation. + void armMountFence(UInt128 server_uuid, uint64_t writer_epoch, uint64_t deadline_boot_ms); + /// Test-only interposition at the publication boundary between the re-armed generation and the + /// live fence. A caller admitted from this hook must be refused: the old generation is already + /// dead, while the new generation is not live until `lost` is cleared. + void setArmMountFenceInterpositionHookForTest(std::function hook) + { + arm_mount_fence_interposition_hook_for_test = std::move(hook); + } + /// The fence clock: `CLOCK_BOOTTIME` in milliseconds (includes VM-suspend time, unlike + /// CLOCK_MONOTONIC — see `MountFence`). Consults the injected `config.boot_ms_fn` if set (tests), + /// otherwise `bootMs`. + uint64_t bootMsNow() const; + /// The real boot clock: `CLOCK_BOOTTIME` in milliseconds. Static so tests can compose it. + static uint64_t bootMs(); + + /// ---- fence-generation admission (rev.7 [C2]/[D1]) ---- + /// Bumped by EVERY `tripMountLost` (a fence loss) and EVERY `armMountFence` (a re-arm -- a fresh + /// lease incarnation, e.g. after a self-remount). A durable-effect caller captures this value once + /// at admission and compares it again immediately before its durable backend call: a DIFFERENT + /// value means the lease incarnation moved from under it since admission -- even when the fence + /// happens to be live again under a brand-new incarnation, the caller's write is stale and must not + /// land. See `checkFenceOrThrow`. + uint64_t fenceGeneration() const { return fence_generation.load(std::memory_order_acquire); } + + /// Fence-generation admission check for every durable CAS/PUT/DELETE (the plain-object surface, + /// staging-buffer finalize): the caller captures `fenceGeneration()` once at admission and passes it + /// back here immediately before its durable backend call -- and again before EVERY conditional-retry + /// iteration, not just the first attempt. Throws the typed transient refusal + /// (`throwCasTransientUnavailable`) when the fence is not currently held or the generation moved since + /// admission; the caller's write must never reach the backend in either case. + void checkFenceOrThrow(uint64_t admitted_generation) const; + + /// ---- pool lifecycle condition (rev.7 §1, spec §§1-3); enum at namespace scope below ---- + /// Atomic read of the current lifecycle (acquire). + PoolLifecycle lifecycle() const { return pool_lifecycle.load(std::memory_order_acquire); } + /// Whether the pool has reached one of the two fully-terminal `Vanished` values + /// (`VanishedReplaced` / `VanishedForgotten`). + bool isVanished() const; + /// Whether the terminal-intent latch (`vanished_intent`) is published — set by a natural + /// `enterVanished`, OR EARLY (spec §5 step 1) by FORGET's `publishVanishedIntent`, and NEVER by the + /// non-absorbing `IdentityLost` ([C1]). This is the EARLIEST terminal signal: it can already be true + /// while the state is still pre-terminal (mid-FORGET). Consulted alongside `isVanished()` by every + /// background worker that must self-exit the moment the pool is (being driven) terminal — the keeper + /// callback (`scheduleRemount`), the remount loop, and the GC scheduler. + bool vanishedIntentPublished() const { return vanished_intent.load(std::memory_order_acquire); } + + /// Non-terminal lease-loss transition: `Live -> TransientNotLive`. Idempotent and lock-free; a + /// compare-exchange FROM `Live` only, so it never downgrades a terminal state. `tripMountLost` + /// calls this (the lease-loss primitive), and the remount loop's identity gate calls it as its + /// first step so a direct/forced remount attempt has a valid non-terminal predecessor state. + void noteLeaseLost(); + /// Non-terminal recovery transition: `TransientNotLive -> Live`. Called after a self-remount + /// reclaimed a fresh incarnation. A compare-exchange FROM `TransientNotLive` only, so it NEVER + /// revives `IdentityLost`/`Vanished` ([D3]). + void noteRemounted(); + + /// One-way terminal transition to `IdentityLost`, from `TransientNotLive` only (a compare-exchange + /// FROM `TransientNotLive`, so it is idempotent and cannot fire from `Live`/`Vanished`). On the + /// transition it emits ONE WARN and one `CASIdentityLost` ProfileEvent. rev.8: `IdentityLost` is a + /// fail-loud TERMINAL state — `remountTerminal()` reports it, so the remount observer thread self-exits + /// (and the GC scheduler self-exits, through `Pool`) at its next boundary; there is no demoted observer. + /// It deliberately does NOT publish the `vanished_intent` latch (which is reserved for the `Vanished*` + /// idempotency/FORGET protocol); `remountTerminal()` widens the observer-exit boundary to include it. + /// Must be called under the caller's remount serialization (Pool::remount_mutex). + void enterIdentityLost(); + /// Test seam: force the lifecycle condition directly to `lc`, bypassing the natural transition + /// preconditions (used by the operation-gate tests to pin each class × state cell without driving a + /// full remount/erase sequence). For a `Vanished*` value it also latches `vanished_intent`, so the + /// forced state is indistinguishable from a naturally-reached one. Never used in production. + void setLifecycleForTest(PoolLifecycle lc); + + /// Publish the terminal-intent latch (`vanished_intent`) WITHOUT settling the lifecycle state. This is + /// spec §5 step 1 of `SYSTEM CAS FORGET`: publishing the latch FIRST makes the keeper + /// callback stop arming remounts and the remount loop bail at its next step boundary, so FORGET's + /// subsequent thread joins are bounded to one step + one backend timeout. The state store + WARN happen + /// later, in `enterVanished` at step 6. Idempotent; lock-free (a single release store). A natural + /// terminal transition does NOT call this — its `enterVanished` publishes the latch itself. + void publishVanishedIntent(); + + /// One-way transition to a fully-terminal `Vanished` value (spec §3). Publishes the terminal-intent + /// latch (so the keeper stops scheduling remounts and the remount loop exits at its next step + /// boundary) if it is not already published, records `reason`, stores the state, then emits ONE WARN + + /// one `CASDataRootVanished` ProfileEvent. Idempotent: the first terminal STATE transition wins (a + /// dedicated latch keyed separately from `vanished_intent`, because FORGET publishes that intent latch + /// early at step 1). `which` MUST be one of the two `Vanished*` values (`VanishedReplaced` or + /// `VanishedForgotten`). `reason` is retained and + /// surfaced verbatim in the `VanishedForgotten` [D5] error message (see `vanishedReason`). Threads exit + /// their own loops; the joins happen in `~Pool` for a natural transition, or synchronously in + /// `Pool::forgetDisk` for FORGET. Must be called under the caller's remount serialization + /// (Pool::remount_mutex). + void enterVanished(PoolLifecycle which, const String & reason); + + /// The reason string recorded by the winning `enterVanished`, or empty when none has run (a + /// forced-for-test terminal state, or a non-terminal pool). `Pool::throwIfLifecycleTerminal` reads it + /// to build the `VanishedForgotten` [D5] message (which carries the operator's decommission timestamp + /// authored by `forgetDisk`). Safe to read only AFTER observing a terminal state via `lifecycle()` + /// (acquire): the reason is written once, before the state's release-store, so a reader that + /// acquire-observes the terminal state also observes the reason (release/acquire handoff). + const String & vanishedReason() const { return vanished_reason; } + + /// Wall-clock second (`system_clock`, seconds since epoch) at which the pool ENTERED its current + /// non-`Live` lifecycle state, or 0 while `Live`. This is the `since` the non-gated + /// `system.cas_mounts` lifecycle snapshot (spec §7) reports. Written (release) at each + /// lifecycle edge — `noteLeaseLost`/`enterIdentityLost`/`enterVanished` set it to now, `noteRemounted` + /// clears it to 0 — and by `setLifecycleForTest`, so a forced state carries a `since` indistinguishable + /// from a naturally-reached one. + /// + /// Ordering vs the `pool_lifecycle` transition it accompanies: the TERMINAL edges (`enterVanished`, + /// `enterIdentityLost`) publish this store BEFORE the state store, so a reader that acquire-observes a + /// terminal state is guaranteed (release/acquire handoff) to observe this timestamp. The lock-free + /// lease-loss/remount edges (`noteLeaseLost`, `noteRemounted`) stamp it in the compare-exchange's + /// SUCCESS branch — after the CAS — because they may run on an already-terminal pool (`noteLeaseLost` is + /// called before the caller's `isVanished()` gate), where a pre-CAS stamp would clobber the terminal + /// `since`; a reader may therefore momentarily observe a just-entered `not_live` with `since` not yet + /// updated, a benign introspection artifact that converges within nanoseconds. + time_t lifecycleSinceWallS() const + { + return static_cast(lifecycle_since_wall_s.load(std::memory_order_acquire)); + } + + /// Extends `mayMutate` with a remaining-budget check. A ref-log attempt is refused unless the + /// current lease has room for its configured timeout and safety margin, so work is not started when + /// it cannot plausibly finish before the fence expires. + bool refAppendFenceOk() const; + + /// The `writer_epoch` of the live mount incarnation. Bumped by `tryRemountOnce` (self-remount after a + /// GC fence-out) — a `PartWriteTxn` minted under an older epoch fails closed on its next step. + uint64_t liveWriterEpoch() const { return live_writer_epoch.load(std::memory_order_acquire); } + + /// ---- build registry ---- + /// Allocate a strictly-increasing `build_seq` and add it to the active set. A sequence is never + /// reused or lowered, which lets the GC watermark advance monotonically. + uint64_t allocateBuildSeq(); + /// Register the in-flight build so `dropNamespace`'s post-durable cancellation can reach it (weak_ptr). + void registerInflightBuild(uint64_t seq, const PartWriteTxnPtr & build); + /// Remove a build_seq from the active set + inflight map; idempotent (safe from publish/abandon/dtor). + void retireBuildSeq(uint64_t seq); + /// After the namespace-removal transaction is durable, cancel every in-flight build targeting `ns`. + /// Live shared pointers are collected under `builds_mutex` and cancelled after releasing it, because + /// cancellation may take a different path and must not run under the registry lock. + void cancelInflightBuildsForNamespace(const RootNamespace & ns); + + /// ---- process epoch (identity) ---- + /// Mint the random nonzero process identity used by GC's equality check. + void mintRandomProcessEpoch(); + /// Set `process_epoch` to the durable `writer_epoch`. The caller supplies the memory order because + /// the initial writable claim and a later self-remount have different publication requirements. + void setProcessEpoch(uint64_t v, std::memory_order order); + /// Publish the live-incarnation `live_writer_epoch` with release ordering. + void setLiveWriterEpoch(uint64_t v); + + /// ---- mount-lease keeper (owned) ---- + /// Construct the `MountLeaseKeeper` adopting (our_uuid, writer_epoch) and wire its fence callbacks + /// (renew-ok refreshes the fence deadline; on-lost latches the fence + arms a self-remount) plus its + /// build-watermark `minActive` reader, and event sink. `keeperStart` is separate so pool claim + /// orchestration can catch `MountFencedException`, discard the keeper, allocate a fresh epoch, and + /// retry the claim. + void installKeeper(UInt128 our_uuid, uint64_t writer_epoch, const std::function & now_ms); + /// Adopt the already-claimed mount slot; on return the adoption is durable. + void keeperStart(); + /// Force one fresh conditional lease write on the already-adopted slot (fails closed, like any + /// other `renewOnce`, if the slot changed hands underfoot). Used to re-anchor the write-fence + /// arm after a materialization grace long enough to have consumed the lease TTL. + void keeperRenewOnce(); + /// Discard a keeper after a refused adoption so the caller can retry with a fresh epoch. + void keeperReset(); + /// Start periodic lease and watermark renewal. + void keeperStartBackground(std::chrono::milliseconds period); + /// Stop periodic renewal; safe to call more than once. + void keeperStopBackground(); + bool hasKeeper() const { return static_cast(mount_keeper); } + + + /// ---- self-remount recovery ---- + /// On a lost lease, arm a recovery thread when background operation is enabled. It retries the + /// pool-level remount callback with exponential backoff until success or teardown. + void scheduleRemount(); + /// Test seam: drive the arm/refuse path directly. Returns true iff a recovery thread is armed after. + bool scheduleRemountForTest(); + /// Test seam: latch the shutdown gate without joining or otherwise tearing down the runtime. + void beginShutdownForTest(); + /// Return how many times `scheduleRemount` was entered, including calls refused by the background + /// setting. This is useful for testing the keeper's loss callback without starting a real recovery. + uint64_t scheduleRemountCallCountForTest() const + { + return schedule_remount_calls_for_test.load(std::memory_order_relaxed); + } + + /// ---- teardown ---- + /// Stop and join the self-remount thread before retiring the keeper; otherwise it could recreate the + /// keeper while teardown is in progress. + void stopRemountThread(); + /// Retire the merged heartbeat. When `drained` is true, publish the clean farewell; otherwise stop + /// background renewal without writing a terminal marker, because unresolved writes must not be + /// certified as clean. Finish with a second recovery-thread join to close the final callback window. + void finishTeardown(bool drained); + + /// Sleep through the injected test hook when present; otherwise use the production thread sleep. + /// `Pool` claim observation and materialization grace waits share this seam so tests control both. + void waitSleep(uint64_t ms) const; + + /// Forward keeper events to the injected sink. The sink is held by reference so it observes the + /// owning pool's current event routing for the runtime's entire lifetime. + void emitEvent(CasEvent && e) const { if (event_sink) event_sink(std::move(e)); } + +private: + /// TRUE once the pool has reached — or is being driven toward — a state on which the self-remount + /// observer thread must stop: a published terminal `Vanished` intent (`vanished_intent` — set early by + /// FORGET, or by a natural `enterVanished`, and already subsuming every settled `Vanished*` state since + /// it is published before the state store) OR `IdentityLost` (rev.8: a fail-loud TERMINAL state — no + /// demoted observer; recovery is restart or FORGET). Consulted by `scheduleRemount` before arming and by + /// the remount loop at every step boundary. (The GC scheduler applies the same three-way test through + /// `Pool`, spec §9 rev.8 item 8.) + bool remountTerminal() const + { + return vanished_intent.load(std::memory_order_acquire) + || lifecycle() == PoolLifecycle::IdentityLost; + } + + /// ---- injected environment (no `Pool` back-reference); initialized first, in this order ---- + BackendPtr backend_ptr; + const Layout & layout; + MountConfig config; + String server_root_id; + const CasEventSink & event_sink; + CasRequestBudget cas_request_budget; + std::function remount_attempt; + + /// Per-server build watermark. `process_epoch` is a random + /// nonzero u64 minted once at open: GC checks it for EQUALITY (an object stamped with a different + /// epoch is from a dead incarnation), never for ordering. next_build_seq is a strictly-increasing + /// per-process counter (monotonicity is load-bearing — a seq is never reused or lowered); + /// active_build_seqs holds the seqs of in-flight builds, so `minActive` yields the GC floor. The floor + /// is published by the merged `mount_keeper` + /// beat (there is no standalone watermark object anymore). ATOMIC because a self-remount re-stamps it + /// (kept equal to `live_writer_epoch`) off the background remount thread while `epoch`/`writerEpoch` + /// may observe it; the ref-lane hot readers were moved to `liveWriterEpoch`, so this now backs only + /// the identity accessors. + std::atomic process_epoch{0}; + std::mutex builds_mutex; + uint64_t next_build_seq = 1; + std::set active_build_seqs; + /// In-flight builds keyed by `build_seq`. `dropNamespace` upgrades these weak pointers only after its + /// removal transaction is durable and cancels those targeting the removed namespace. The wiring owns + /// the shared pointers, so an expired entry is simply skipped. Guarded by `builds_mutex`. + std::map> inflight_builds; + + /// Mount-lease heartbeat. Constructed and started on a writable + /// open AFTER the owner/epoch/mount startup protocol; renews the mount lease async off the write + /// path and drives the local write fence (deadline on each successful renew, `tripMountLost` on a + /// superseded/foreign touch). Teardown stops it, whose `terminate` retires the lease (so a + /// same-server reopen can immediately reclaim). Null on a read-only open. + std::unique_ptr mount_keeper; + + std::atomic live_writer_epoch{0}; + std::mutex remount_thread_mutex; /// guards the thread handle below + std::atomic remount_running{false}; + std::atomic remount_stop{false}; + std::atomic remount_shutting_down{false}; /// latched at teardown top; scheduleRemount refuses to re-arm during teardown + std::condition_variable remount_cv; + std::mutex remount_cv_mutex; + ThreadFromGlobalPool remount_thread; + /// Counted entries into `scheduleRemount`; retained as a test-only observability seam. + std::atomic schedule_remount_calls_for_test{0}; + + /// Local write fence. The unarmed default (`deadline_boot_ms = UINT64_MAX`, `lost = false`) permits + /// mutation until a keeper supplies a real lease deadline or reports that the lease was lost. This + /// is the gate at the ref-append mutation chokepoint. + MountFence mount_fence; + + /// Fence-generation token (rev.7 [C2]): bumped by `tripMountLost` and `armMountFence`. See + /// `fenceGeneration`/`checkFenceOrThrow`. + std::atomic fence_generation{0}; + std::function arm_mount_fence_interposition_hook_for_test; + + /// The pool lifecycle condition (rev.7 §1). Starts `Live`. Non-terminal transitions + /// (`noteLeaseLost`/`noteRemounted`) are lock-free compare-exchanges guarded by their exact + /// predecessor state; the terminal transitions (`enterIdentityLost`/`enterVanished`) are serialized + /// by the caller's `Pool::remount_mutex` and made race-safe against the keeper thread's concurrent + /// `noteLeaseLost` by the compare-exchange/latch discipline in the .cpp. + std::atomic pool_lifecycle{PoolLifecycle::Live}; + /// Terminal-intent latch (spec §3), published before the state store — by `enterVanished` for a + /// natural transition, or EARLY (step 1) by `publishVanishedIntent` for FORGET. Only the fully-terminal + /// `Vanished*` transition sets it — `IdentityLost` deliberately does NOT (rev.8 folds IdentityLost into + /// the observer-exit boundary via `remountTerminal()` instead). Consulted (with `IdentityLost`) by + /// `remountTerminal()`, so a terminal pool's keeper callback never schedules a remount and the remount + /// loop bails at its next step boundary — no claim/allocate/write after the pool is (being driven) terminal. + std::atomic vanished_intent{false}; + /// Idempotency guard for the terminal STATE transition (`enterVanished`'s body). Distinct from + /// `vanished_intent`: FORGET publishes that intent latch at step 1, so it can no longer serve as the + /// "state transition already done" flag. The FIRST `enterVanished` to win this exchange stores the + /// state, records `vanished_reason`, and logs; every later call returns early. + std::atomic terminal_state_published{false}; + /// The reason recorded by the winning `enterVanished` (see `vanishedReason`). Written once, BEFORE the + /// `pool_lifecycle` release-store, and immutable thereafter — so a reader that acquire-observes a + /// terminal state also observes this string. Empty when no terminal transition has run. + String vanished_reason; + + /// Wall-clock second at which the current non-`Live` lifecycle state was entered; 0 while `Live` (see + /// `lifecycleSinceWallS`). Set at every lifecycle edge with a release-store ordered before the + /// `pool_lifecycle` transition it accompanies. + std::atomic lifecycle_since_wall_s{0}; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.cpp new file mode 100644 index 000000000000..e0c2fd1e210d --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.cpp @@ -0,0 +1,1463 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event CASBlobDeduplicationCacheHit; + extern const Event CASBlobHeadFirst; + extern const Event CASBlobBodyPutAvoided; + extern const Event CASBlobAdoptTrusted; + extern const Event CASMetaCreateClean; + extern const Event CASMetaAdoptBackfill; + extern const Event CASMetaResurrectClean; +} + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; + extern const int FILE_DOESNT_EXIST; + extern const int NOT_IMPLEMENTED; + extern const int ABORTED; + extern const int CORRUPTED_DATA; + extern const int LIMIT_EXCEEDED; +} +} + +namespace DB::Cas +{ + +namespace +{ + +/// Backpressure caps, enforced fail-closed in stageManifest before the body write returns. +constexpr uint64_t kMaxManifestEntries = 1048576; +constexpr uint64_t kMaxManifestEncodedBytes = 256ULL << 20; /// 256 MiB +constexpr uint64_t kMaxManifestInlineBytesTotal = 16ULL << 20; /// 16 MiB +constexpr uint64_t kMaxLargestInlineEntryBytes = 1ULL << 20; /// 1 MiB + +/// Two thread_local_rng draws composed into a UInt128. Used both to mint build ids and to mint, on +/// every upload/re-upload, a FRESH incarnation_tag (W-FRESH-TAG). +UInt128 mintU128() +{ + const UInt64 hi = thread_local_rng(); + const UInt64 lo = thread_local_rng(); + return (static_cast(hi) << 64) | lo; +} + +uint64_t nowMs() +{ + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); +} + +} + +/// The pool-wide content-hash convention, with the selected `algo`: `Cas::blobHashHexOneShot` uses the same +/// streaming/chunked convention the write path uses for `CityHash128` (`ContentAddressedWriteBuffers`: `getHashHex` -> +/// `BlobId` -> `hexToU128`) and the one-shot `XXH3_128bits` call for `XXH3_128` (defined to agree +/// with its own streaming digest). The core otherwise never re-hashes payloads; any copy-forward +/// re-verification must use this convention. A one-shot `CityHash128` instead of the chunked convention +/// diverges for payloads larger than one hash block and can report valid stored content as `CORRUPTED_DATA`. +/// +/// Returns the full `BlobRef` pair at `algo`'s own width (via `codecFor`) — never a bare digest. Exported +/// (declared in `CasPartWriteTxn.h`) so the wiring's inline-candidate hashing +/// (`ContentAddressedTransaction.cpp`) can mint the same way the +/// streaming blob path does, rather than reimplementing the hex round-trip inline. +BlobRef poolContentHash(BlobHashAlgo algo, std::string_view payload) +{ + return BlobRef{algo, codecFor(algo).fromHex(blobHashHexOneShot(algo, payload))}; +} + +BlobSource BlobSource::fromString(String bytes) +{ + BlobSource source; + source.size = bytes.size(); + /// Owning reader: `ReadBufferFromString` only borrows, and this factory outlives no single call + /// -- an owning copy per attempt removes the question entirely for a source that is small by + /// construction (this helper exists for tests and inline payloads). + source.open = [b = std::move(bytes)]() -> std::unique_ptr + { return std::make_unique(b); }; + return source; +} + +PartWriteTxn::PartWriteTxn(PoolPtr store_, UInt128 build_id_, + uint64_t build_seq_, uint64_t epoch_, PartWriteInfo info_) + : store(std::move(store_)) + , build_id(build_id_) + , build_seq(build_seq_) + , epoch(epoch_) + , info(std::move(info_)) +{ + /// A build began. `build_id`, `build_seq`, and `epoch` identify it for token-join attribution + /// against the GC delete rows. + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::BuildStart; + e.ref_name = info.intended_ref.value_or(""); + e.token = u128ToHex(build_id); + e.outcome = "started"; + e.reason = "beginPartWrite: build in-flight"; + e.detail = {{"build_seq", std::to_string(build_seq)}, {"epoch", std::to_string(epoch)}}; + }); +} + +PartWriteTxn::~PartWriteTxn() +{ + /// An attempted precommit whose terminal operation did not settle may still be a durable owner. + /// Transfer that exact cleanup duty to the mount and KEEP the build active: the next mutation of + /// this namespace first resolves the every-attempt wedge, then removes the owner if it exists, and + /// retires the sequence only after that same state observation proves the duty discharged. If the + /// process ends first, no false clean farewell is written and successor recovery seals the old + /// epoch before sweeping its stale precommit. + if (precommit_state == PrecommitState::Uncertain || precommit_state == PrecommitState::Durable) + { + store->enqueueWriterCleanupDuty( + precommit_target_ns, precommit_final_ref, precommit_manifest, build_seq); + return; + } + + /// No owner duty remains: promotion/abandon settled it, or no precommit append was attempted. + store->retireBuildSeq(build_seq); +} + +void PartWriteTxn::requireAlive() const +{ + if (!alive) + throw Exception(ErrorCodes::LOGICAL_ERROR, "PartWriteTxn has been abandoned; no further operations allowed"); + /// `dropNamespace` cancels in-flight builds once its removal transaction is + /// durable. A cancelled build must not promote/precommit a fresh owner into the just-removed namespace + /// (nor stage more debris there), so every further op fails closed here -- fast, before any backend + /// work, with a clear diagnostic (the append lane would reject a promote/precommit anyway). + if (cancelled.load(std::memory_order_acquire)) + throwCasWriteRetryLater( + "PartWriteTxn cancelled: its owning namespace was removed (dropNamespace) while this build was " + "in flight; restart the build only after the namespace is recreated"); + /// Self-remount (fence-out recovery) supersedes the mount incarnation this build was minted + /// under; its write fence already interrupted the build mid-flight, so every further step fails + /// closed and the caller restarts the build under the live epoch. + if (const uint64_t live = store->liveWriterEpoch(); epoch != live) + throwCasWriteRetryLater(fmt::format( + "PartWriteTxn (writer_epoch {}) belongs to a superseded mount incarnation (live epoch {}) — " + "the mount was fenced out and self-remounted; restart the build", epoch, live)); +} + +PutBlobResult PartWriteTxn::putBlob(const BlobRef & ref, BlobSource source) +{ + /// Serial API preserved for existing callers: run the transaction-detached primitive, then fold its + /// single result into `build` on this (the owning writer) thread. Semantics are byte-for-byte those + /// of the pre-carve `putBlob`. The fan-out path (spec §1) instead collects many results off-thread and + /// folds them together via `mergeBlobUploadResults`. `source.size` is the streaming byte authority, so + /// the request's `declared_size` mirrors it. + const uint64_t declared_size = source.size; + const BlobUploadResult r = uploadBlobDetached(BlobUploadRequest{ref, std::move(source), declared_size}); + deps[r.ref] = r.dep; + return PutBlobResult{r.ref, r.dep.size}; +} + +BlobUploadResult PartWriteTxn::uploadBlobDetached(const BlobUploadRequest & req) const +{ + requireAlive(); + + const PoolConfig & cfg = store->poolConfig(); + /// The caller (the write-mint site) already produced the full `BlobRef` pair (algo + digest), so there + /// is no hex round-trip here. `logical_ref` is the blob identity end-to-end: the dedup cache and every + /// downstream event render key off THIS value directly. + const BlobRef & logical_ref = req.ref; + const String key = store->layout().blobKey(req.ref); + const BlobSource & source = req.source; + + /// The source is RE-READABLE (the caller's `open` re-reads a staged temp file, or re-emits a + /// captured String): it can be invoked MULTIPLE times — the primary streaming PUT plus any INV-1 + /// re-upload — so we never materialize the whole blob into memory here. The byte count is verified + /// against `source.size` at each streaming write site (via the sink buffer's `count()`), not by a + /// full pre-materialization, so peak memory is bounded by the write-buffer, not the blob size. + + /// HEAD-before-PUT on a likely dedup hit (cache says present) or a large body (where a + /// wasted body-PUT that 412s is expensive — and on a store that early-closes a doomed conditional + /// PUT, the broken-pipe and retry storm caused by early rejection). A present HEAD ⇒ admit without streaming the body; + /// a stale/absent HEAD ⇒ fall through to the normal conditional upload. SAFE by construction: we + /// always genuinely observe present-at-round before skipping the body, so the cache can never cause + /// a dangle (a stale hit just HEADs 404 and uploads). The cache membership is read ONCE here: it both + /// arms the HEAD-first gate and distinguishes the `DeduplicationCacheHit` outcome from a size-triggered `HeadHit`. + const bool cache_hit = store->dedupCacheContains(logical_ref); + const bool head_first = + cache_hit + || (cfg.deduplication_head_first_min_bytes > 0 && source.size >= cfg.deduplication_head_first_min_bytes); + if (head_first) + { + ProfileEvents::increment(ProfileEvents::CASBlobHeadFirst); + const HeadResult hr = store->backend().head(key); + if (hr.exists) + { + ProfileEvents::increment(ProfileEvents::CASBlobBodyPutAvoided); + if (cache_hit) + ProfileEvents::increment(ProfileEvents::CASBlobDeduplicationCacheHit); + try + { + const BlobDepRecord dep = observeAndAdmit(ObjectKind::Blob, logical_ref, key, hr); + store->dedupCacheAdd(logical_ref); + return BlobUploadResult{req.ref, dep, + cache_hit ? BlobUploadOutcome::DeduplicationCacheHit : BlobUploadOutcome::HeadHit}; + } + catch (const Exception & e) + { + /// INV-1: HEAD-first path hit a condemned token → re-upload from our OWN source bytes. + if (e.code() != ErrorCodes::ABORTED) + throw; + /// Fall through to uploadFromSource below. + } + } + /// hr.exists == false OR condemned-ABORTED → fall through to uploadFromSource / fresh upload. + } + + /// Fresh-upload path + condemned-dedup recovery under INV-1. Try the primary upload via + /// uploadFromSource (which handles condemned-present via putOverwrite and absent via putIfAbsentStream + /// without any backend().get). Bounded loop guards against rare concurrent-condemnation churn. + constexpr int max_attempts = 8; + for (int attempt = 0; attempt < max_attempts; ++attempt) + { + try + { + const BlobUploadResult r = uploadFromSource(ObjectKind::Blob, logical_ref, key, source); + /// This hash is now known-present — future writers can HEAD-first and skip the body. + store->dedupCacheAdd(logical_ref); + return r; + } + catch (const Exception & e) + { + /// ABORTED from uploadFromSource is retryable — re-upload by re-streaming from our re-readable + /// source (bounded). Two cases produce it: + /// • a racing writer displaced the condemned token before our putOverwrite landed and + /// their fresh incarnation is itself already condemned (observeAndAdmit → ABORTED); or + /// • the object was GC-deleted during the post-412 revival re-observe: + /// reviveObserve converts FILE_DOESNT_EXIST → ABORTED so the vanish re-uploads here + /// rather than escaping FATAL. Both are rare races covered by the bounded loop. + if (e.code() != ErrorCodes::ABORTED || attempt + 1 == max_attempts) + throw; + } + } + + /// Unreachable: the loop either returns or rethrows on the final attempt. + throw Exception(ErrorCodes::LOGICAL_ERROR, "uploadBlobDetached: exhausted retries for {}", key); +} + +void PartWriteTxn::mergeBlobUploadResults(std::span results) +{ + /// Prevalidate EVERYTHING first, touching nothing but locals: a result without a token is + /// incomplete (every `uploadBlobDetached` branch sets one; a tokenless dep only ever arrives + /// through `adoptEvidence`'s separate direct-fold path, never through this method) -- a caller bug, + /// failed closed rather than merged as a hole. Two results for the SAME ref must carry an + /// IDENTICAL dep record (the fan-out's one-task-per-unique-ref invariant, spec §1); a conflict -- + /// most commonly a conflicting size -- means that invariant was violated upstream. + std::map seen; + for (const auto & r : results) + { + if (!r.dep.token.has_value()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "PartWriteTxn::mergeBlobUploadResults: incomplete result for {} (no token) -- every " + "uploadBlobDetached branch sets one; a tokenless dep must be recorded via adoptEvidence, " + "never merged here", blobIdOf(r.ref)); + const auto [it, inserted] = seen.try_emplace(r.ref, &r.dep); + if (!inserted && !(*it->second == r.dep)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "PartWriteTxn::mergeBlobUploadResults: dep records differ for {} (sizes {} vs {}) -- " + "the fan-out must launch exactly one task per unique ref and merge exactly one result; " + "records for one ref that differ in ANY field (size, token, kind, or adopted) are a " + "wiring error, so the sizes shown may be equal when the mismatch is in another field", + blobIdOf(r.ref), it->second->size, r.dep.size); + } + + /// Build-and-swap: apply every result into a COPY of the live `deps`. The live member is touched + /// ONLY by the final `swap` below, which runs after every result has applied -- so a mid-loop + /// exception (a `bad_alloc` from map-node allocation, or one injected by `setMergeHookForTest`'s + /// hook) leaves `deps` byte-for-byte as it was before this call. Applying an already-validated + /// duplicate a second time is idempotent (identical value, same key). + std::map candidate = deps; + size_t applied = 0; + for (const auto & r : results) + { + candidate[r.ref] = r.dep; + ++applied; + if (merge_hook_for_test) + merge_hook_for_test(applied); + } + deps.swap(candidate); +} + +bool PartWriteTxn::isTrustedAdopt(const BlobRef & ref) const +{ + /// §4: a leaf trusted at promote iff this build holds a TOKENLESS dep recorded by adoptEvidence + /// (a committed-source W-EVIDENCE adopt: the source pins it, in-degree >= 1, not condemnable). A + /// tokenless PENDING-upload dep (recordPendingBlobDep, adopted=false) is NOT trusted — it must be + /// tokened by putBlob before promote; reaching promote un-tokened is a staging bug (fail closed). + auto it = deps.find(ref); + return it != deps.end() && !it->second.token.has_value() && it->second.adopted; +} + +bool PartWriteTxn::depIsTokened(const BlobRef & ref) const +{ + /// Discriminator for B156b: a putBlob'd blob records a TOKENED dep (recreatable by retrying), an + /// adoptFromTree carry-forward records a TOKENLESS W-EVIDENCE dep (not recreatable — pinned by a + /// committed source). Returns false when this build has no dep for the ref (the caller decides + /// the default; not-tokened is the fail-loud, INV-NO-LOSS-safe choice). + auto it = deps.find(ref); + return it != deps.end() && it->second.token.has_value(); +} + +BlobDepRecord PartWriteTxn::observeAndAdmit(ObjectKind kind, const BlobRef & ref, const String & key) const +{ + /// EDGE-BEFORE-OBSERVE: the durable-precommit guard + /// lives in the 4-arg overload below, scoped to its ADOPT branch only — NOT here. A HEAD result + /// reached through this wrapper (e.g. `reviveObserve`'s post-race re-observe) can resolve to EITHER + /// the condemned/ABORTED branch or the adopt branch once the 4-arg overload point-reads the meta; + /// gating here (before that point-read) would wrongly block the condemned branch too, which + /// re-uploads under THIS build's own build_id via `uploadFromSource` and stays watermark-protected + /// pre-precommit. + const HeadResult hr = store->backend().head(key); + if (!hr.exists) + /// Object absent at observe time. The live caller of this overload is the revival re-observe in + /// `uploadFromSource` (`reviveObserve` below, retryable): a GC-deleted object at re-observe time + /// is a race under INV-3 — the caller HOLDS the source bytes, so it catches FILE_DOESNT_EXIST + /// here and re-throws it as ABORTED so the INSERT layer sees the uniform retryable error and + /// re-uploads from those bytes. This overload only ever raises FILE_DOESNT_EXIST; the caller + /// decides fail-closed vs retry. + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "PartWriteTxn::observeAndAdmit: object {} vanished (GC-deleted) before observe; " + "caller must re-upload/re-materialize from source", key); + return observeAndAdmit(kind, ref, key, hr); +} + +BlobDepRecord PartWriteTxn::observeAndAdmit(ObjectKind kind, const BlobRef & ref, const String & key, const HeadResult & hr) const +{ + /// `hr.exists` is guaranteed by the caller (the 3-arg wrapper checked it; the putBlob HEAD-first + /// path only calls this on a present HEAD). Avoids a redundant second HEAD on the dedup-hit path. + /// Logical (payload) size = object size minus the pool's fixed blob header. GUARD against + /// unsigned underflow: a truncated/corrupt object whose size is below the header length must surface + /// as CORRUPTED_DATA, never wrap to a huge value. Mirrors the GC path's `retiredLogicalSize`. + const uint64_t header_len = store->poolMeta().blob_header_len; + if (hr.size < header_len) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "PartWriteTxn: {} object {} size {} is below the pool blob header length {}", + kind == ObjectKind::Blob ? "blob" : "manifest", key, hr.size, header_len); + const uint64_t logical_size = hr.size - header_len; + + const CasEventObjectKind ev_kind = toEventKind(kind); + + /// The condemned decision is a per-hash META POINT-READ, not + /// the retired-view snapshot. `absent` meta means "not condemned" — GC always writes a `Condemned` + /// meta BEFORE it ever deletes a body, so an absent meta is exactly as live as a `Clean` one. + /// The meta ops layer is `BlobRef`-keyed directly (derives its codec from `ref.algo` + /// internally). Every event-log render below uses `blobIdOf(ref)` (":"), never a + /// bare hex. + const auto lm = loadMeta(store->backend(), store->layout(), ref); + const bool condemned = lm && lm->meta.state == MetaState::Condemned; + if (condemned) + { + /// INV-1 (revival-from-source): the observed token is condemned — we must NOT read the dying + /// object via backend().get. Throw ABORTED so the caller can re-upload from its OWN source bytes. + /// • putBlob (has BlobSource): catches ABORTED, calls uploadFromSource from held bytes. + /// • a bodyless observe (no source): propagates ABORTED (retryable; caller retries op). + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::BlobReuseResurrect; + e.object_kind = ev_kind; + e.object_hash = blobIdOf(ref); + e.token = hr.token.value; + e.round = 0; /// the round is no longer a writer concept (meta point-read replaces the retired view) + e.outcome = "condemned"; + e.reason = "observed token is condemned (meta point-read); caller must re-upload from source (INV-1)"; + }); + throw Exception(ErrorCodes::ABORTED, + "PartWriteTxn::observeAndAdmit: condemned token for {} — caller must re-upload from source bytes (INV-1)", + key); + } + + /// EDGE-BEFORE-OBSERVE: from here on we are about to + /// ADOPT the live (non-condemned) incarnation as our own dependency — safe ONLY under this build's + /// durable precommit closure, because an adopted blob carries the ORIGINAL writer's build_id, so the + /// newborn-debris watermark does not cover it. (The condemned branch above is unaffected — it + /// re-uploads under THIS build's own build_id via `uploadFromSource`, which stays watermark-protected + /// pre-precommit.) See the TLA+ order sabotage (Gate A). A4: a real throw, not chassert — chassert is + /// compiled out in release, and a wiring/retry bug that reached adopt without a durable precommit + /// would silently drop watermark protection (later dangling ref / data loss with no production + /// signal). + /// `Durable` and nothing else. `Uncertain` is a precommit that MAY be live and may equally not + /// exist at all, and "may" is not the closure this invariant needs: the adopted blob would carry + /// the original writer's build_id with only a possibly-absent edge protecting it. It fails closed + /// here exactly as `NotAttempted` does, and for the same reason. + if (precommit_state != PrecommitState::Durable) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "PartWriteTxn::observeAndAdmit: EDGE-BEFORE-OBSERVE invariant violated — adopting an existing " + "incarnation before this build's precommit is durable would admit {} ({}) under the original " + "writer's build_id with no newborn-debris watermark protection", + key, kind == ObjectKind::Blob ? "blob" : "manifest"); + + /// `!lm`: no meta yet for this hash (a pre-existing blob from before this protocol, or a lost race + /// with a concurrent fresh-uploader's own meta write). Best-effort create it as Clean so future + /// point-readers (writers and GC) never have to fall back to a HEAD-only guess. A Conflict here just + /// means a racing writer already created it — both agree on the same Clean steady state. + if (!lm) + { + ProfileEvents::increment(ProfileEvents::CASMetaAdoptBackfill); + putMetaIfAbsent(*store, ref, + BlobMeta{.state = MetaState::Clean, .condemn_round = 0, .size = logical_size}); + } + + /// Adopt the current incarnation — free, no bytes moved. + /// Reuse an ADOPTED existing incarnation's token as NOT condemned (per the meta point-read). + /// Was the CAREUSE adopt audit line. Token-join this against a later blob_delete + /// of the same hash/token to pin a reuse-of-an-object-being-deleted race. + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::BlobReuseAdopt; + e.object_kind = ev_kind; + e.object_hash = blobIdOf(ref); + e.token = hr.token.value; + e.round = 0; /// the round is no longer a writer concept (meta point-read replaces the retired view) + e.outcome = "adopt"; + e.reason = "observed token not condemned (meta point-read); adopted the live incarnation (no bytes moved)"; + }); + /// Build-neutral: RETURN the adopt dep (tokened, adopted=false) instead of folding it into `deps`. + return BlobDepRecord{kind, hr.token, logical_size, /*adopted=*/false}; +} + +BlobUploadResult PartWriteTxn::uploadFromSource(ObjectKind kind, const BlobRef & ref, const String & key, const BlobSource & source) const +{ + /// StagingPromoted vs FreshUpload discriminator for the write-once create terminals: the source + /// carries a server-side-copy descriptor iff the bytes already live in an S3 staging object. + const BlobUploadOutcome fresh_outcome + = source.server_side_copy_from ? BlobUploadOutcome::StagingPromoted : BlobUploadOutcome::FreshUpload; + + /// INV-1 (revival-from-source): re-upload a condemned or absent object from the writer's OWN + /// re-readable source — NEVER calls backend().get to read the dying object. W-FRESH-TAG: fresh + /// incarnation_tag and this build's build_id so the new incarnation is owned by THIS live build + /// (the source-based resurrection rule closes the prior ownership gap). The payload is STREAMED into the put sink + /// (`source.open`), never materialized into a full in-memory copy on the common + /// If-None-Match path; `source.open` is re-invoked on each attempt (it re-reads the staged + /// temp file), which is exactly what preserves INV-1 across retries. + const PoolMeta & meta = store->poolMeta(); + const PoolConfig & cfg = store->poolConfig(); + /// `ref` is the full blob identity (algo + digest) end-to-end -- the `.meta` API is + /// `BlobRef`-keyed directly, and the dep map + every event render below key off `ref` too. + + auto buildHeader = [&]() -> String + { + EnvelopeHeader header; + header.kind = kind; + header.incarnation_tag = mintU128(); + header.build_id = build_id; + /// ch = the real ClickHouse VERSION_INTEGER (diagnostic-only; no decision reads it) — the v3 + /// envelope drops writer_version/hash_algo/domain_id, so forensics ride on ch + bld. + header.provenance = Provenance{nowMs(), cfg.server_id, VERSION_INTEGER, info.op}; + if (kind == ObjectKind::Blob) + header.intended_ref = info.intended_ref; + /// The v3 codec pads to the pool's fixed header length and TRUNCATES a too-long intended_ref + /// internally (it is diagnostic-only), so the old drop-and-retry is gone — one encode call. + return encodeEnvelopeHeader(header, static_cast(meta.blob_header_len)); + }; + + const CasEventObjectKind ev_kind = toEventKind(kind); + + /// Revival-local wrapper for the post-412 re-observe (INV-3). On the post-412 path a + /// racing writer is assumed to have (re-)created the object, so we adopt its token via the 3-arg + /// observeAndAdmit. But the object can be GC-deleted in the window (present at the conditional PUT + /// → 412, gone at the subsequent HEAD), making the 3-arg overload throw FILE_DOESNT_EXIST. The + /// caller (putBlob's fresh-upload path, via `uploadFromSource`) HOLDS the source bytes, so a vanish + /// here is a retryable race: convert FILE_DOESNT_EXIST → ABORTED so putBlob's bounded retry loop + /// re-uploads from those bytes. Without this, FILE_DOESNT_EXIST escaped putBlob's ABORTED-only catch + /// as a FATAL INSERT failure — the sibling of the gate bug. + auto reviveObserve = [&](const String & k_) -> BlobDepRecord + { + try + { + return observeAndAdmit(kind, ref, k_); + } + catch (const Exception & e_) + { + if (e_.code() != ErrorCodes::FILE_DOESNT_EXIST) + throw; + throw Exception(ErrorCodes::ABORTED, + "uploadFromSource: object {} vanished (GC-deleted) during revival re-observe; " + "retry the operation — re-upload from held source bytes (INV-3)", k_); + } + }; + + /// Build-neutral: emit the BlobPut audit event and RETURN the fresh/resurrect dep record (tokened, + /// adopted=false) for the caller to compose into its `BlobUploadResult` — it folds nothing into `deps`. + auto makeDepAndEmit = [&](Token tok) -> BlobDepRecord + { + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::BlobPut; + e.object_kind = ev_kind; + e.object_hash = blobIdOf(ref); + e.token = tok.value; + e.round = 0; /// the round is no longer a writer concept (meta point-read replaces the retired view) + e.outcome = "ok"; + e.reason = "uploadFromSource: fresh incarnation streamed from writer's own re-readable source (INV-1)"; + e.detail = {{"size", std::to_string(source.size)}, {"build_id", u128ToHex(build_id)}}; + }); + return BlobDepRecord{kind, tok, source.size, /*adopted=*/false}; + }; + + /// Meta write for the RESURRECT (condemned-displacement) case: flip the now-stale Condemned meta + /// back to Clean now that a live incarnation has displaced the condemned body. `lm_before` is the + /// point-read taken just before the condemned decision — its etag is the CAS precondition (absent + /// when there is no prior point-read at all, the FRESH upload case below). Each outer attempt + /// routes through the Pool's request controller (putMetaIfAbsent/casMeta), which already absorbs + /// a transient transport error (SlowDown/429/5xx) within its own budget — this outer loop reacts + /// only to a genuine `Conflict` (the marker's current token AND bytes both differ from what this + /// call intended: a racing writer or GC re-condemning) by reloading and retrying against the + /// fresh state. `Unresolved` (the controller's own budget/fence exhausted for one attempt) is + /// retried the same way — reloading may simply observe the write actually landed. After + /// max_meta_attempts of this outer loop WITHOUT a Committed result, this is a persistent failure, + /// not a blip (each outer attempt already burned its own inner retry budget) — the RCA requires + /// this reach the caller as a controlled retry-later signal, never a silent skip: a dropped + /// freshness marker leaves stale state for the next point-reader. + auto writeResurrectMetaClean = [&](std::optional lm_before) + { + ProfileEvents::increment(ProfileEvents::CASMetaResurrectClean); + const BlobMeta clean{.state = MetaState::Clean, .condemn_round = 0, .size = source.size}; + constexpr int max_meta_attempts = 8; + for (int attempt = 0; attempt < max_meta_attempts; ++attempt) + { + const bool committed = lm_before + ? casMeta(*store, ref, lm_before->etag, clean).outcome == CasOverwriteOutcome::Committed + : putMetaIfAbsent(*store, ref, clean).outcome == CasOverwriteOutcome::Committed; + if (committed) + return; + lm_before = loadMeta(store->backend(), store->layout(), ref); + } + throwCasWriteRetryLater(fmt::format( + "writeResurrectMetaClean: freshness-meta transition to Clean for {} did not land within " + "{} attempts (each already budget-controlled) — the body incarnation is durable, but the " + "meta marker is stuck; retry", store->layout().blobMetaKey(ref), max_meta_attempts)); + }; + + /// Meta write for the FRESH (absent -> present) upload cases: the body just transitioned via + /// If-None-Match, so the freshness meta is created as Clean. Delegates to writeResurrectMetaClean + /// with no prior point-read: a pre-existing marker there is NOT always "a racing writer creating + /// the same Clean state" (the retired comment this replaced assumed) -- it can be a stale + /// Condemned marker left over from before this exact body vanished and was freshly re-uploaded + /// (proven reachable by CasPartWriteTxn.PutBlobResurrectVanishedReUploadsHeldBody), which must be + /// reconciled to Clean the same way a resurrect does, not silently ignored. + auto writeFreshMetaClean = [&]() + { + ProfileEvents::increment(ProfileEvents::CASMetaCreateClean); + writeResurrectMetaClean(std::nullopt); + }; + + /// Stream header + payload into a fresh putIfAbsentStream sink WITHOUT materializing the whole blob. + /// `source.open` re-reads the staged temp file (INV-1: the writer's own source, never the + /// dying object). The payload byte count is verified against `source.size` via the sink buffer's + /// `count()` (total bytes written so far) — the streaming equivalent of the old pre-materialized + /// size check, with no full in-memory copy. A mismatch is a LOGICAL_ERROR (a buggy/racing source). + /// + /// The whole conditional create rides the Pool's shared request controller + /// (`conditionalCreateControlled` — the retry controller's "any other + /// controller-bypassing conditional-write call site"): budgeted attempts + fence-gated backoff + + /// exact-key OCCUPANCY resolve, replacing the old bare single attempt whose whole S3-blip tolerance + /// was ONE ~3s adaptive-timeout attempt. Reissue is sound for BOTH primitives: the streaming PUT + /// re-invokes `source.open` (the REPLAYABLE source contract — a fresh re-upload, never a + /// GET-revive), and the server-side copy re-reads the intact staging object. Each re-stream mints a + /// fresh incarnation_tag (W-FRESH-TAG), so byte-exact resolve is impossible by design and the + /// controller resolves by occupancy instead: an occupant at this content-addressed key IS the + /// intended content (whether our own landed ambiguous attempt or a twin), surfaced as + /// PreconditionFailed so every existing gate branch below is UNCHANGED. + /// + /// The size-check LOGICAL_ERROR below stays instant and loud: `conditionalCreateControlled` + /// propagates every deterministic local failure — LOGICAL_ERROR, NOT_IMPLEMENTED (the promoteStaged + /// mode guard), BAD_ARGUMENTS (escaping buildHeader's second encode), CORRUPTED_DATA — from the + /// attempt unchanged (a caller/config bug reissue would only replay — pinned by + /// `CasPartWriteTxn.PutBlobWrongSizeFailsClosed` and the controller-level + /// `DeterministicLocalFailuresPropagateInstantly`). + auto streamIfAbsent = [&]() -> PutResult + { + const auto one_attempt = [&]() -> PutResult + { + /// S3-native staging promote: when the source + /// carries a server-side-copy descriptor, the write-once CREATE primitive is a conditional + /// server-side copy of the staging object to `key` (`If-None-Match:*`) instead of a + /// client-side streaming PUT. Same write-once contract (`Done` + dest-ETag token on created, + /// `PreconditionFailed` when `key` already exists). The staging object IS the promote source + /// — no envelope is streamed here. + if (source.server_side_copy_from) + return store->backend().promoteStaged(*source.server_side_copy_from, key); + + /// No owner metadata is needed: protection is the precommit edge — reachability, not `cas_owner`. + /// A throw mid-stream abandons the sink (its dtor cancels): nothing is ever published by a + /// failed attempt except via the storage's own late-landing ambiguity, which the controller + /// resolves. + WriteSinkPtr sink = store->backend().putIfAbsentStream(key); + WriteBuffer & out = sink->buffer(); + writeString(buildHeader(), out); + const size_t before = out.count(); + copyData(*source.open(), out); + const size_t written = out.count() - before; + if (written != source.size) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "uploadFromSource: source streamed {} bytes, declared {}", written, source.size); + return sink->finalize(); + }; + + const CasCreateResult res = store->stagingConditionalCreate(key, one_attempt); + switch (res.outcome) + { + case CasCreateOutcome::Committed: + return PutResult{PutOutcome::Done, res.token}; + case CasCreateOutcome::Occupied: + return PutResult{PutOutcome::PreconditionFailed, {}}; + case CasCreateOutcome::Unresolved: + break; + } + /// Unresolved = budget exhausted or fence lost without a definite outcome. Nothing referenced + /// this incarnation (deps/meta are recorded only on a definite outcome); a late-landing body is + /// inert debris behind the content-addressed key — a future writer of the same content adopts + /// or displaces it through the normal occupancy machinery. NETWORK_ERROR = the same retryable + /// abort class stageManifest and the ref lane map their exhausted budgets to. + throwCasWriteRetryLater(fmt::format( + "uploadFromSource: conditional create at '{}' is UNCERTAIN (retry budget exhausted or mount " + "fence lost) — nothing was acknowledged; retry re-uploads from the writer's own source (INV-1)", + key)); + }; + + /// Try the If-None-Match upload (object absent or race with another writer). + { + const PutResult res = streamIfAbsent(); + if (res.outcome == PutOutcome::Done) + { + const BlobDepRecord dep = makeDepAndEmit(res.token); + writeFreshMetaClean(); + return BlobUploadResult{ref, dep, fresh_outcome}; + } + } + + /// PreconditionFailed: an incarnation exists. HEAD it to check whether it is condemned or live. + /// We do NOT read the body (no backend().get) — we only need the token to decide: + /// • live (not condemned) → adopt; this is the standard dedup case. + /// • condemned → displace via putOverwrite(If-Match: current_token) by re-reading our + /// own source, so we never read the dying object. This is the + /// equivalent of the old `resurrect` minus the GET. + const HeadResult hr = store->backend().head(key); + if (!hr.exists) + { + /// Object vanished between putIfAbsentStream (412d) and our HEAD — a concurrent GC delete. + /// The object is now absent; re-stream from our re-readable source (bounded by caller). + const PutResult res2 = streamIfAbsent(); + if (res2.outcome == PutOutcome::Done) + { + const BlobDepRecord dep = makeDepAndEmit(res2.token); + writeFreshMetaClean(); + return BlobUploadResult{ref, dep, fresh_outcome}; + } + /// Still 412 after the vanish-and-retry: a racing writer re-created it. Adopt their token. + /// reviveObserve converts FILE_DOESNT_EXIST (deleted again in the window) → ABORTED (retryable). + return BlobUploadResult{ref, reviveObserve(key), BlobUploadOutcome::HeadMissAdopted}; + } + + /// The condemned decision is a per-hash META POINT-READ, not + /// the retired-view snapshot. `lm` (and its etag) is reused below to flip a condemned meta back to + /// Clean once the resurrect displacement lands. + const auto lm = loadMeta(store->backend(), store->layout(), ref); + const bool condemned = lm && lm->meta.state == MetaState::Condemned; + if (!condemned) + { + /// Live (not condemned): adopt the current incarnation — free, no bytes moved. + return BlobUploadResult{ref, observeAndAdmit(kind, ref, key, hr), BlobUploadOutcome::HeadMissAdopted}; + } + + /// Condemned: displace the condemned incarnation with our fresh source. + /// + /// rev.7 [C2] (backlog {#c2-resurrect-putoverwrite-fence-check}): the two displacement calls below + /// (`resurrect` / `putOverwrite`) are RAW backend writes with NO controller/fence coupling — + /// unlike `streamIfAbsent`, which rides the request controller's fence gate. They were the only + /// durable-effect writes left outside Task 4's fence-generation gate. Capture the mount fence + /// generation now, at the displacement DECISION, and re-check it (and `mayMutate()`) immediately before + /// whichever raw write we issue: a lease lost — or re-armed under a fresh incarnation — since we + /// observed the condemned state aborts with the typed transient refusal before any + /// stale-incarnation displacement can land. Mirrors `CasPlainObjects::casPutObject`'s + /// capture-at-admission-then-check-before-the-durable-write shape. + const uint64_t displace_admitted_generation = store->fenceGeneration(); + if (source.server_side_copy_from) + { + /// S3-native staging RESURRECT (INV-NO-RETURN): re-establish a fresh + /// incarnation by re-uploading OUR OWN staging PAYLOAD under a FRESHLY-tagged envelope header — + /// NEVER a read/copy of the condemned `key` (`feedback_ca_resurrect_invariant`). We reach here + /// ONLY after the per-hash meta point-read observed `Condemned` just above, so this overwrites a + /// condemned body -- or, on a lost race, an equivalent FRESH resurrection of it, which is + /// accepted: payloads are content-identical, so the overwrite rotates only envelope and token. + /// A live incarnation of DIFFERENT content is unreachable here by the content address itself. + /// + /// CRITICAL — a VERBATIM server-side copy of the create-time staging object would reproduce the + /// condemned incarnation's exact bytes ⇒ identical ETag ⇒ the queued exact-token delete of the + /// condemned incarnation would kill the live resurrection (data loss). `buildHeader` mints a + /// FRESH `incarnation_tag` distinct from the staging header's tag, so the resurrected body (and + /// hence its ETag) differs from the condemned incarnation regardless of edge-before-observe. The + /// backend reads the payload from the staging object skipping its own `blob_header_len` envelope + /// header and prepends this fresh header. + const String fresh_header = buildHeader(); + /// rev.7 [C2]: a raw `backend()` call with NO controller fence coupling (unlike + /// `promoteStaged`/`putIfAbsentStream` above, reached only through `stagingConditionalCreate`'s + /// controller+fence_ok). The `checkFenceOrThrow` re-checks the fence generation captured at the + /// displacement decision, closing the same gap Task 4 closed on the plain-object surface. + Token tok{}; + store->checkFenceOrThrow(displace_admitted_generation); + /// The staging object's own envelope header is skipped HERE, by whoever knows its shape: the + /// backend is handed a reader already positioned at the payload. + auto staged = store->backend().getStream(*source.server_side_copy_from); + if (!staged) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "CAS resurrect: staging object {} is absent", *source.server_side_copy_from); + staged->stream->ignore(meta.blob_header_len); + tok = store->backend().resurrect(*staged->stream, source.size, key, fresh_header); + const BlobDepRecord dep = makeDepAndEmit(tok); + writeResurrectMetaClean(lm); + return BlobUploadResult{ref, dep, BlobUploadOutcome::ResurrectedS3}; + } + + /// CRITICAL: we re-read the writer's OWN source (NOT backend().get) — no GET of the dying object. + /// W-FRESH-TAG: a fresh incarnation_tag minted inside buildHeader() ensures INV-NO-RETURN. + /// + /// UNCONDITIONAL, exactly like the staging arm above. An `If-Match` on the condemned token would + /// save a redundant re-upload when another writer resurrects the same blob first, and would prevent + /// nothing: two racing resurrections write payload-identical bodies, no consumer reads a dep token's + /// VALUE, and durable references name content hashes rather than incarnations. What protects the + /// resurrection is the fresh tag — it makes this body's ETag differ from the condemned one, so every + /// already-queued exact-token delete of that incarnation misses. + /// + /// Blob bodies have no size cap, so a body larger than memory has to remain writable here: a + /// Native backend streams the payload from the reader; the emulated (local) backend materializes + /// one body at a time, serialized inside `Backend::resurrect`. + auto payload = source.open(); + /// rev.7 [C2]: a raw, uncoupled backend call; fence-checked against the displacement-decision + /// generation immediately before the durable write. + store->checkFenceOrThrow(displace_admitted_generation); + /// `source.size` rides into the write, which counts while streaming and aborts WITHOUT publishing + /// on a mismatch -- with an unconditional overwrite, a post-write check would fire only after a + /// truncated body had already displaced the condemned incarnation (and could even inspect a racing + /// writer's fresh incarnation instead of ours). + const Token resurrected = store->backend().resurrect(*payload, source.size, key, buildHeader()); + + const BlobDepRecord dep = makeDepAndEmit(resurrected); + writeResurrectMetaClean(lm); + return BlobUploadResult{ref, dep, BlobUploadOutcome::ResurrectedLocal}; +} + +void PartWriteTxn::adoptEvidence(const ManifestEntry & entry) +{ + requireAlive(); + + /// W-EVIDENCE: record a TOKENLESS dependency — liveness evidence is the live source manifest, not a + /// token. Inline entries reference no standalone object, so they record nothing. NO backend call + /// (no HEAD, no GET, no PUT) — the caller already holds the resolved entry. Part manifests have only + /// Inline / Blob placements (no Subtree): only blobs are content-addressed. + if (entry.placement == EntryPlacement::Blob) + { + /// Carry `entry.ref` WHOLE (the pair, never re-derived) — this is what makes a + /// mixed-algo manifest's entries each dep-track under their OWN algo. §4: adopted=true marks this a + /// committed-source W-EVIDENCE dep, trusted at promote via the durable manifest edge (no probe). + deps[entry.ref] = BlobDepRecord{ObjectKind::Blob, std::nullopt, entry.blob_size, /*adopted=*/true}; + } +} + +void PartWriteTxn::recordPendingBlobDep(const BlobRef & ref, uint64_t size) +{ + requireAlive(); + deps[ref] = BlobDepRecord{ObjectKind::Blob, std::nullopt, size}; +} + +RootNamespace PartWriteTxn::manifestNamespace() const +{ + /// The wiring sets the owning namespace EXPLICITLY (PartWriteInfo::intended_namespace). This is the + /// authoritative source: a ref can itself contain `/` (the `detached/` fold), so we + /// must NOT recover the namespace by splitting intended_ref on the last `/` — that would yield a + /// spurious `/detached` namespace and the precommit namespace-match check would throw. + if (info.intended_namespace) + return *info.intended_namespace; + + /// Fallback (Core tests that set only the diagnostic intended_ref, where the ref has no `/`): the + /// owning namespace is intended_ref minus its last `/`-segment (the ref name). A CA namespace itself + /// contains `/` (e.g. "srv1/@cas@"), so split on the LAST slash only. + const String intended = info.intended_ref.value_or(""); + const size_t slash = intended.find_last_of('/'); + if (slash == String::npos || slash == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "stageManifest: intended_ref '{}' has no namespace/ref split", intended); + return RootNamespace{intended.substr(0, slash)}; +} + +ManifestId PartWriteTxn::stageManifest(std::vector entries) +{ + requireAlive(); + + /// Fail-closed caps — checked BEFORE the body write so no owner transition can ever name a + /// manifest that breaches a cap. Inline payload is read on every part-open and every owner + /// transition, so cap the total, not only per-entry. + if (entries.size() > kMaxManifestEntries) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "stageManifest: {} entries exceeds cap {}", entries.size(), kMaxManifestEntries); + uint64_t inline_total = 0; + for (const ManifestEntry & e : entries) + { + if (e.placement == EntryPlacement::Inline) + { + if (e.inline_bytes.size() > kMaxLargestInlineEntryBytes) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "stageManifest: inline entry '{}' of {} bytes exceeds cap {}", + e.path, e.inline_bytes.size(), kMaxLargestInlineEntryBytes); + inline_total += e.inline_bytes.size(); + } + } + if (inline_total > kMaxManifestInlineBytesTotal) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "stageManifest: total inline {} bytes exceeds cap {}", inline_total, kMaxManifestInlineBytesTotal); + + /// Mint the identity. `epoch` is the Pool's durable writer_epoch; `build_seq` is monotone inside + /// that epoch; `manifest_ordinal` is monotone inside this PartWriteTxn. Together with the owning namespace + /// this gives NoManifestIdReuse by construction, with no random manifest instance id. + if (next_manifest_ordinal > kMaxManifestOrdinal) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "stageManifest: manifest ordinal cap {} exceeded for build_seq {}", kMaxManifestOrdinal, build_seq); + const ManifestRef ref{epoch, build_seq, next_manifest_ordinal++}; + + /// Build the body. payload_digest is integrity/debug only — never a key, never dedup, never + /// in-degree. The body repeats its own ref + namespace for fail-closed RefMatchesBody / + /// ManifestNamespaceMatches at read/fold/promote time. + const RootNamespace owning_ns = manifestNamespace(); + PartManifest body; + body.ref = ref; + body.root_namespace_id = owning_ns; + body.entries = std::move(entries); + body.payload_digest = computePayloadDigest(body); + /// The cap is measured over the canonical text before sealing, not the compressed PUT bytes. + /// 256 MiB comfortably bounds the largest realistic manifest, including its JSON structure. + const String encoded_text = encodePartManifest(body); + if (encoded_text.size() > kMaxManifestEncodedBytes) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "stageManifest: encoded manifest {} bytes exceeds cap {}", encoded_text.size(), kMaxManifestEncodedBytes); + const String encoded = sealObject(FormatId::PartManifest, encoded_text); + + const ManifestId id{owning_ns, ref}; + const String key = store->layout().manifestKey(id); + + /// Body PUT through the Pool's shared request controller: + /// budgeted attempts + resolve-before-reissue, replacing the old bare single-attempt write whose + /// whole S3-blip tolerance was ONE ~3s adaptive-timeout attempt (a 19s object-store pause killed an + /// INSERT through it while every plain read/write path survived — v3 soak evidence). Reissuing this + /// conditional PUT is sound: the body bytes are fixed for the whole operation (`encoded` is built + /// once; `encodePartManifest` is canonical/deterministic), so `resolveByExactGet` can prove whether + /// an ambiguous attempt landed. Still NO preliminary HEAD. A DIFFERENT object at this key is a + /// ManifestId collision — the controller's resolve raises CORRUPTED_DATA (a proven conflict, + /// fail-closed before any owner transition can name this id), subsuming the old + /// PreconditionFailed->LOGICAL_ERROR mapping. + /// + /// fence_ok is the ref lane's own mount predicate (`refAppendFenceOk`: fence not lost + enough + /// lease left for one more attempt): staging runs on this writable Pool under that same mount + /// lease, and a fenced writer must not keep PUTting bodies ahead of a precommitAdd that would fail + /// the same fence anyway. There is no ref-table runtime here, so the lane's extra + /// `superseded_by_remount` term does not apply. + Token manifest_token; + const CasWriteOutcome put_outcome = store->stagingPutIfAbsent(key, encoded, &manifest_token); + if (put_outcome == CasWriteOutcome::DefiniteFailure) + throwCasWriteRetryLater(fmt::format( + "stageManifest: part-manifest PUT at '{}' definitively failed (non-retryable rejection); " + "nothing was named — the caller re-stages with a fresh ManifestId", key)); + /// Unresolved = budget exhausted (or fence lost) without a definite outcome. Unlike the ref-log + /// lane there is nothing to wedge: this id was never named by any owner transition + /// (`next_manifest_ordinal` is already past it, so no re-stage ever reuses the key), and a + /// late-landing body is inert unreferenced debris for the orphan-manifest sweep. NETWORK_ERROR = + /// the same retryable abort class the ref lane's exhausted budget maps to. + if (put_outcome == CasWriteOutcome::Unresolved) + throwCasWriteRetryLater(fmt::format( + "stageManifest: part-manifest PUT at '{}' is UNCERTAIN (retry budget exhausted) — " + "nothing conclusive was named; the caller re-stages with a fresh ManifestId", key)); + + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::ManifestPut; + e.namespace_ = owning_ns.string(); + e.object_kind = CasEventObjectKind::Manifest; + e.object_hash = manifestRefDebugString(id.ref); + e.token = manifest_token.value; + e.reason = "stageManifest: part-manifest body written"; + }); + + staged_manifests.push_back(id); + staged_manifest_ids.insert(id); /// A3 mint-tightening: `precommitAdd`'s only legal fresh-ownership source + return id; +} + +void PartWriteTxn::precommitAdd(const RootNamespace & target_ns, const String & final_ref_name, const ManifestId & id) +{ + requireAlive(); + + /// ManifestNamespaceMatches at the source: the precommit's manifest must belong to the target + /// namespace (its key is built from id.root_namespace). A cross-namespace precommit is a bug. + if (id.root_namespace != target_ns) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "precommitAdd: manifest namespace '{}' != target namespace '{}'", + id.root_namespace.string(), target_ns.string()); + + /// A3 mint-tightening (ABA barrier for the relink confirm's exact-`ManifestRef` equality): an + /// unowned `ManifestId` may enter ownership only from the transaction that freshly staged it. + /// Evaluated HERE (synchronously, off this transaction's own private `staged_manifest_ids` -- + /// never mutated concurrently, so no snapshot races) rather than read live inside the closure + /// below, because it does not depend on ledger state at all. The bool is captured BY VALUE into + /// the closure -- consistent with every other capture there -- rather than capturing `this`, + /// which would let a closure that outlives this stack (see the capture comment below) dereference + /// a dead transaction. Enforcement, however, is NOT done here: see the closure for why. + const bool id_staged_by_this_txn = staged_manifest_ids.contains(id); + + /// One `owner_transition` create-precommit op. + /// No body HEAD — a missing body is a legal fail-closed, non-activating intent, unchanged from the + /// old protocol. Precommit ownership carries NO separate build token: `id.ref` (writer_epoch, + /// build_sequence, manifest_ordinal) IS the build identity (the tuple + /// §Transaction Log Format: "there is no second build token"). `build_ops` is invoked from inside + /// the per-namespace flush, so it sees the table's CURRENT state including any earlier item of the + /// same batch; when that state is not `Live` (never born, or `Removed`), it prepends + /// `namespace_birth` in the SAME transaction (the birth transaction + /// normally also adds the first precommit"). + /// + /// THE INTENT IS RECORDED BEFORE THE APPEND, and that ordering is the whole point: an `Unresolved` + /// append MAY HAVE LANDED (`CasRefLedger.cpp`, the `Unresolved` arm says so explicitly), so a + /// `precommitAdd` that throws can still have made `id.ref` a live precommit owner. Setting the + /// fields afterwards left that case with an object which believed it had never precommitted: + /// `abandon` queued no removal and `cleanupStagedManifestDebrisBestEffort` -- deciding from the same + /// unset state -- writer-DELETED the body, so a wedge that later resolved as committed installed a + /// live precommit with no cleanup owner and no body. Same discipline, and same reason, as the ref + /// lane's preconstructed wedge. The three fields are plain assignments of already-materialized + /// values, so this cannot throw between the record and the append. + precommit_target_ns = target_ns; + precommit_final_ref = final_ref_name; + precommit_manifest = id.ref; + precommit_state = PrecommitState::Uncertain; + + store->appendRefOps(target_ns, MutationScope::ref(final_ref_name), + /// Capture everything the closure reads BY VALUE (the `store` handle, target namespace, ref name, + /// and manifest id) rather than `[&]`: defense-in-depth so a closure that ever outlives this + /// stack frame (e.g. if the append lane stranded its item) never dereferences a dead stack. The + /// ref-lane leadership guard is the real fix; this makes the closure self-contained regardless. + [pool = store, target_ns, final_ref_name, id, id_staged_by_this_txn](const RefTableState & state) -> std::vector + { + /// Idempotent re-add: the target ref is ALREADY committed to this EXACT manifest_ref (a + /// legitimate re-drive calling precommitAdd+promote again for content that is already + /// live -- see `promote`'s matching no-op guard). Nothing to append: re-adding a precommit + /// for an already-owned manifest would violate "no conflicting owner may name the same + /// manifest", which the state machine enforces for every OTHER case. + /// + /// A3 mint-tightening's enforcement lives HERE, gated on this SAME live-state read, rather + /// than unconditionally before the closure: a legitimate re-drive can be handed an id that + /// this exact `PartWriteTxn` object never staged (e.g. a fresh build re-precommitting + /// content a PRIOR, since-destroyed build already promoted -- `CasPromoteRepublish. + /// PromoteSameManifestIsIdempotent`). That case is not a fresh ownership grant: the manifest + /// is already `final_ref_name`'s live, currently-protected binding, so re-affirming it + /// creates no new ABA surface. Checking membership-or-committed-match atomically under the + /// SAME state read (rather than a separate pre-check against a possibly-stale resolve) is + /// what keeps this race-free: a snapshot taken before the closure could see "already + /// committed to id.ref" and let a later closure invocation append a fresh owner_transition + /// after a concurrent repoint moved the ref away from id.ref in between -- exactly the + /// re-ownership of a dropped identity this check exists to prevent. + if (const auto it = state.getCommitted().find(final_ref_name); + it != state.getCommitted().end() && it->second.manifest_ref == id.ref) + return {}; + if (!id_staged_by_this_txn) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "precommitAdd: manifest '{}' was not staged by this transaction and is not the current " + "committed manifest of ref '{}' -- refusing to re-own a foreign or previously-dropped identity", + manifestRefDebugString(id.ref), final_ref_name); + + std::vector ops; + if (state.getLifecycle() != RefLifecycle::Live) + { + /// Reaching an empty runtime here means catalog resolution admitted a new life. A + /// predecessor still `Removing` was refused before recovery, so rebirth needs no + /// physical marker or empty-prefix proof. + RefOp birth; + birth.kind = RefOpKind::NamespaceBirth; + ops.push_back(birth); + } + RefOp add; + add.kind = RefOpKind::OwnerTransition; + add.new_binding = RefOwnerBinding{RefOwnerKind::Precommit, final_ref_name, id.ref}; + ops.push_back(add); + return ops; + }, + RootMutationOrigin::Writer, RootMutationKind::Precommit); + + /// The append returned: the precommit binding is durably this build's, so the duty settles from + /// `Uncertain` to `Durable` and `observeAndAdmit`'s EDGE-BEFORE-OBSERVE gate opens. + precommit_state = PrecommitState::Durable; + + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::Precommit; + e.namespace_ = target_ns.string(); + e.ref_name = final_ref_name; + e.token = u128ToHex(build_id); + e.outcome = "ok"; + e.reason = "precommitAdd: build-intent owner add in the target shard (owner = precommit(build_id))"; + e.detail = {{"build_seq", std::to_string(build_seq)}}; + }); +} + +bool PartWriteTxn::promote(const RootNamespace & target_ns, const String & final_ref_name, UInt128 promote_build_id, const ManifestId & id, bool allow_repoint) +{ + requireAlive(); + + if (id.root_namespace != target_ns) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "promote: manifest namespace '{}' != target namespace '{}'", + id.root_namespace.string(), target_ns.string()); + + /// Read + validate the manifest body ONCE (O(manifest entries), one streaming read). Absent or + /// invalid ⇒ fail closed: a committed ref must never name a missing/mismatched manifest. + const String manifest_key = store->layout().manifestKey(id); + const auto body_got = store->backend().get(manifest_key); + if (!body_got) + throwCasWriteRetryLater(fmt::format( + "promote: manifest body absent at {} — failing closed (retry with a fresh ManifestId)", manifest_key)); + const PartManifest body = decodePartManifest(openObject(FormatId::PartManifest, body_got->bytes)); + if (!refMatchesBody(id.ref, body)) + throwCasWriteRetryLater(fmt::format("promote: RefMatchesBody failed for {}", manifest_key)); + if (!manifestNamespaceMatches(target_ns, body)) + throwCasWriteRetryLater(fmt::format("promote: ManifestNamespaceMatches failed for {}", manifest_key)); + + /// The copy-forward pre-pass is removed — the + /// in-closure blob revalidation below is now the SINGLE copy-forward site. Trade-off: the rare + /// condemned-tokenless copy-forward (a GET+PUT) now runs inside the append lane's flush, briefly + /// blocking the per-namespace batching queue; it is idempotent under a re-run (a retry sees its own + /// fresh token). The meta CAS is the only remaining coordination for this rare case. + + /// The intended-repoint operation is an atomic composition of `WDropRef` and `WPromote`: the + /// manifest the ref currently commits, + /// when this promote is retiring it via an intended repoint (`allow_repoint`) rather than the + /// ordinary first-time-commit path. Set inside the closure below; read after it returns to decide + /// whether the `RefRepoint` audit event fires. + std::optional repoint_old; + /// `created`: whether `final_ref_name` has NO committed row as of THIS builder's read of `state` + /// -- set as the very first statement of the closure (before any other branch), so it is correct + /// on every path: idempotent re-promote no-op (a committed row for `id.ref` already existed -> + /// false), an intended repoint (a committed row for a DIFFERENT manifest already existed -> false), + /// or a genuine first-time bind (no committed row -> true). Same in-closure-output pattern as + /// `repoint_old` immediately below; read by the caller only after `appendRefOps` returns. + bool created = false; + /// Recorded BEFORE the append for the same reason `precommitAdd` records its intent before its own: + /// the append is the point past which failure stops being proof of the negative. Everything above + /// this line is validation that rejects without publishing anything, so a throw there leaves + /// `NotAttempted` -- and a caller may safely conclude the ref was not committed. From here on it + /// may not. + commit_state = CommitState::Uncertain; + store->appendRefOps(target_ns, MutationScope::ref(final_ref_name), + /// Capture the closure's INPUTS by value (ref name, manifest id, promote build id, repoint flag, + /// and the manifest `body` it revalidates) rather than `[&]`, as defense-in-depth against a + /// closure outliving this stack. Unlike `precommitAdd`, this closure cannot be made fully + /// self-contained: `depIsTokened`/`isTrustedAdopt` read this build's `deps` member (so it must + /// keep `this`), and `repoint_old`/`created` are OUTPUTS read after the call returns (so they stay + /// references). The ref-lane leadership guard is the real fix; both residual by-reference captures + /// are safe because the guard guarantees the closure is never invoked after this frame unwinds. + [this, final_ref_name, id, promote_build_id, allow_repoint, body, &repoint_old, &created] + (const RefTableState & state) -> std::vector + { + created = !state.getCommitted().contains(final_ref_name); + + /// Idempotent re-promote: the target ref is ALREADY committed to this EXACT manifest_ref -- + /// a legitimate re-drive (a crash/retry between a prior promote and its caller's own + /// follow-up, or a direct repeat call) that must complete as a no-op, not require a live + /// precommit that a first successful call already consumed. Mirrors precommitAdd's matching + /// guard; the DIFFERENT-manifest case below remains the BUG-1a fail-closed leak guard. + if (const auto it = state.getCommitted().find(final_ref_name); + it != state.getCommitted().end() && it->second.manifest_ref == id.ref) + return {}; + + /// NO writer-side view refresh here: + /// Gate A): promote-time view freshness is not load-bearing — tokened leaves are edge-protected + /// (EDGE-BEFORE-OBSERVE) and the tokenless K3 gate below reads the live view, which the floor + /// guarantees contains every graduated entry (in EVERY view >= condemn round + 1). + + /// The `WPromote` owner guard (`owner[m] = bld`): a promote is a PURE owner MOVE that emits + /// NO blob delta (Δ=0) — it restores no blob in-degree. It is therefore only sound when the + /// precommit is STILL the live owner of the ref: if an abandon or GC reclaim already appended + /// a removal of the precommit binding, the blobs' in-degree was already decremented and a Δ=0 + /// move would re-publish a committed ref over to-be-deleted blobs ⇒ a dangling committed + /// manifest (INV_NO_DANGLE). `RefTableState::getPrecommits` materializes live ownership directly: + /// `id.ref` alone identifies the build (there is no second + /// build token"), so an exact-binding lookup answers the same question directly. + if (!state.getPrecommits().contains({final_ref_name, id.ref})) + throwCasWriteRetryLater(fmt::format( + "promote: precommit owner binding for ref '{}' (build {}) was removed (abandon or GC " + "reclaim) and is no longer the live owner — failing closed; the build must restart " + "(WPromote owner==bld)", + final_ref_name, u128ToHex(promote_build_id))); + + /// Blob-leaf revalidation. TOKENED leaves are + /// edge-protected — EDGE-BEFORE-OBSERVE: the precommit closure was durable BEFORE putBlob + /// observed them, so a condemnation in the putBlob→promote window cannot graduate (the next fold + /// sees the edge, d >= 1, spared), and putBlob's gate already validated them against the installed + /// view under that edge. They are NOT re-checked here. A NON-tokened leaf is EITHER a + /// committed-source W-EVIDENCE adopt (adoptEvidence ⇒ adopted=true) or a no-dep / pending-upload + /// staging bug. There is NO per-file probe on this path: an adopted leaf is TRUSTED via the + /// durable manifest edge (the live source pins the blob, in-degree >= 1, not condemnable) and this + /// build's precommit edge is durable — matching the relink trust model (ordinary + /// ReplicatedMergeTree interserver trust). A genuinely-absent adopted blob is an invariant + /// violation caught by fsck, not here. + for (const ManifestEntry & e : body.entries) + { + if (e.placement != EntryPlacement::Blob) + continue; + if (depIsTokened(e.ref)) + continue; /// edge-protected (EDGE-BEFORE-OBSERVE); putBlob validated under the durable edge + /// §4 manifest-trust: a tokenless adoptEvidence leaf is trusted — no HEAD, no loadMeta, no + /// copy-forward; the durable manifest edge is the liveness evidence. EDGE-BEFORE-TRUST: this + /// build's precommit edge was durably appended (`precommitAdd`, the `Precommit` + /// `OwnerTransition` above) BEFORE we get here, and the owner-liveness check at the top of this + /// closure ("WPromote owner==bld") already re-proved it is the LIVE owner — so the dst manifest + /// is a live precommit owner input and GC's fold pins every blob it names at in-degree >= 1 + /// (the barrier-activated create-precommit +1). GC is the sole deleter and respects + /// in-degree, so a trusted-promote leaf cannot have been condemned/deleted. The backstop for + /// the (production-unreachable) genuinely-absent case is fsck's reachable-but-absent scan + /// (`CasFsck.cpp`, `++report.dangling`), NOT this gate. A tokenless PENDING-upload dep + /// (adopted=false) or a no-dep leaf never reaches promote un-resolved legitimately: it is a + /// staging bug (a pending upload that never completed) and fails closed (LOGICAL_ERROR). + if (!isTrustedAdopt(e.ref)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "promote: blob leaf {} has no tokened and no adopted dep at commit — a staging bug " + "(a pending upload never completed); failing closed", + store->layout().blobKey(e.ref)); + ProfileEvents::increment(ProfileEvents::CASBlobAdoptTrusted); + EventEmitter{*store}.emit([&](CasEvent & ev) + { + ev.type = CasEventType::BlobReuseAdopt; + ev.object_kind = CasEventObjectKind::Blob; + ev.object_hash = blobIdOf(e.ref); + ev.outcome = "adopt"; + ev.reason = "manifest-trust"; /// distinguishable trusted-adopt class (empty token) + }); + } + + /// BUG 1a: refuse to overwrite a live committed ref that already names a DIFFERENT manifest — + /// that would orphan the old manifest (its owner-removal `-1` is never emitted) UNLESS the + /// caller opted into an intended repoint (`allow_repoint`, modeled as the `WDropRef`+`WPromote` + /// composition) -- a standalone + /// write/remove on an already-committed part. Without the flag + /// this enforces the model's `RefFreeFor` guard (`WPromote` requires it) exactly as before. A + /// re-promote of the SAME manifest_ref is idempotent and allowed regardless (the state + /// machine's own promote precondition below would reject it as "precommit absent" anyway once + /// idempotent republish skips this call — see `republishRef`). Fail-closed with ABORTED (not + /// LOGICAL_ERROR): a conflicting durable state the caller handles, never a must-not-happen + /// invariant. + if (const auto it = state.getCommitted().find(final_ref_name); + it != state.getCommitted().end() && !(it->second.manifest_ref == id.ref)) + { + if (!allow_repoint) + throwCasWriteRetryLater(fmt::format( + "promote: ref '{}' already names a different committed manifest — refusing to overwrite " + "(unique-ref invariant; use republishRef for an intended repoint)", final_ref_name)); + repoint_old = it->second.manifest_ref; + } + + /// Promotion is a PURE OWNER MOVE: the SAME manifest_ref T moves from + /// precommit to committed in one atomic transaction, together with the SetPublishedAt op + /// that stamps `published_at_ms` (the initial stamp arrives via a separate set_published_at + /// op, in the same transaction or a later one -- here, the same one). It emits NO blob + /// deltas; the activating `+1` came from GC's barrier-activation of the create-precommit op. + std::vector ops; + /// An intended repoint additionally retires the OLD committed + /// binding in this SAME ref-log record -- a separate OwnerTransition (old=Committed(repoint_old), + /// new=absent), since one RefOwnerBinding cannot carry both the retired-committed and the + /// promoted-precommit manifests at once (CasRefLogCodec.h). Together with the transition op + /// below, this record atomically composes two verified model shapes -- this removal is `WDropRef`'s + /// and the transition is `WPromote`'s; `applyRefLogTxn`'s whole-record scratch apply makes the + /// composition a sound refinement (no intra-record intermediate state is observable; + /// corrected note). NOT `WRepoint` -- that action requires an unowned destination, while this + /// trigger always promotes a live precommit. Ref still moves directly + /// from the old manifest to the new one. Mirrors `publishCommittedTransition`'s old-removal + /// shape (cas_test_helpers.h) minus the add-precommit step, which already landed earlier as + /// this build's own `precommitAdd`. + if (repoint_old) + { + RefOp old_removal; + old_removal.kind = RefOpKind::OwnerTransition; + old_removal.old_binding = RefOwnerBinding{RefOwnerKind::Committed, final_ref_name, *repoint_old}; + ops.push_back(old_removal); + } + RefOp transition; + transition.kind = RefOpKind::OwnerTransition; + transition.old_binding = RefOwnerBinding{RefOwnerKind::Precommit, final_ref_name, id.ref}; + transition.new_binding = RefOwnerBinding{RefOwnerKind::Committed, final_ref_name, id.ref}; + ops.push_back(transition); + + RefOp set_published_at; + set_published_at.kind = RefOpKind::SetPublishedAt; + set_published_at.ref_name = final_ref_name; + set_published_at.expected_manifest_ref = id.ref; + set_published_at.published_at_ms = nowMs(); + ops.push_back(set_published_at); + return ops; + }, + RootMutationOrigin::Writer, RootMutationKind::Promote); + + commit_state = CommitState::Durable; + /// The owed terminal operation is discharged. `Settled`, not `NotAttempted`: the manifest body now + /// belongs to a committed ref, so it stays out of `cleanupStagedManifestDebrisBestEffort`'s reach. + precommit_state = PrecommitState::Settled; + store->retireBuildSeq(build_seq); + try + { + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::BuildPublish; + e.namespace_ = target_ns.string(); + e.ref_name = final_ref_name; + e.object_hash = manifestRefDebugString(id.ref); + e.token = u128ToHex(promote_build_id); + e.outcome = "promoted"; + e.reason = "promote: atomic owner move precommit(build_id) -> ref(final_ref_name) after fail-closed reval"; + e.detail = {{"build_seq", std::to_string(build_seq)}}; + }); + } + catch (...) + { + tryLogCurrentException(getLogger("CasPartWriteTxn"), "CAS event emission after durable promote"); + } + + /// The low-level audit event for an intended repoint. The ProfileEvent counter + LOG_WARNING are + /// owned by `CachedPartFolderAccess::repointRef`, + /// the user-facing primitive that calls into this promote -- adding them here too would double-fire + /// once the caller-level instrumentation is installed. This event alone is per-call-site accurate regardless of which caller opted + /// into `allow_repoint`. + if (repoint_old) + { + try + { + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::RefRepoint; + e.namespace_ = target_ns.string(); + e.ref_name = final_ref_name; + e.object_kind = CasEventObjectKind::Manifest; + e.object_hash = manifestRefDebugString(id.ref); + e.outcome = "repointed"; + e.reason = "promote(allow_repoint=true): committed ref retargeted from an old manifest to a " + "new one in one ref-log record -- standalone write/remove on a committed part"; + e.detail = {{"old_manifest", manifestRefDebugString(*repoint_old)}}; + }); + } + catch (...) + { + tryLogCurrentException(getLogger("CasPartWriteTxn"), "CAS event emission after durable promote"); + } + } + return created; +} + +void PartWriteTxn::abandon() +{ + if (cancelled.load(std::memory_order_acquire)) + { + /// `dropNamespace` cancelled this build once its removal transaction became durable: that same + /// transaction already removed EVERY precommit binding for the (now Removed) namespace, so + /// re-appending a precommit removal here is redundant AND impossible (`owner_transition` on a + /// non-Live namespace is rejected by the state machine -- and `requireAlive` would throw first + /// anyway). Do ONLY the best-effort staged-debris cleanup + seq retire (both idempotent, both on + /// this build's OWN thread); leave the precommit body, if any, for GC's + /// delete-after-sealed-decrements exactly as the normal path does. + alive = false; + precommit_state = PrecommitState::Settled; + store->retireBuildSeq(build_seq); + cleanupStagedManifestDebrisBestEffort(); + /// Audit emission is best-effort: a throwing sink (e.g. a bad_alloc growing the system-log + /// queue, or a Context/log-shutdown edge) must never turn the already-durable cleanup above + /// into a reported failure. Mirrors `promote`'s post-durable emit guard. + try + { + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::BuildAbort; + e.token = u128ToHex(build_id); + e.outcome = "cancelled"; + e.reason = "cancelForNamespaceRemoval: owning namespace removed (dropNamespace); best-effort " + "deleted staged _manifests debris; precommit body (if any) left for GC"; + e.detail = {{"build_seq", std::to_string(build_seq)}, {"staged", std::to_string(staged_manifests.size())}}; + }); + } + catch (...) + { + tryLogCurrentException(getLogger("CasPartWriteTxn"), "CAS event emission after durable abandon"); + } + return; + } + + requireAlive(); + + /// BUG 2 / TLA+ `WAbandonPrecommit`: if this build made a manifest a LIVE precommit owner input + /// (`precommitAdd` ran), abandoning it must NOT writer-delete that body. Instead append an exact + /// precommit-removal ref-log transaction, mirroring EXACTLY the removal + /// shape `Pool::dropRef` and the fenced-successor stale-precommit sweep use (an old_binding naming + /// the exact precommit, no new_binding). GC then folds the `-1` blob decrements and deletes the + /// body only after they are sealed (delete-after-sealed-decrements). Deleting a live precommit body + /// here would strand GC's fold barrier (live precommit, missing body → clamp forever) or lose the + /// activating `+1`. This is the correctness-bearing step of abandon, so it runs through + /// `appendRefOps` (the reliable append lane), not best-effort. It precedes the best-effort debris + /// deletion below so a partial cleanup can never leave the precommit binding live without its + /// body's GC release queued. + /// + /// `Uncertain` is treated EXACTLY as `Durable` here, and that is the point of the state existing: a + /// precommit that may be live owes the same removal a precommit that is live owes. The one + /// difference is the closure's tolerance below. + if (precommit_state == PrecommitState::Uncertain || precommit_state == PrecommitState::Durable) + { + /// A removal whose `old_binding` names an ABSENT precommit is rejected by the state machine + /// (`RefTableState::applyOwnerTransition`, `RemovePrecommit`: "exact precommit binding to remove + /// is absent") -- so the removal is NOT unconditionally idempotent, and an `Uncertain` append + /// that in fact never landed would make every `abandon` of this build fail forever, destructor + /// retries included. Under `Uncertain` the closure therefore reads the CURRENT table state and + /// emits NO op when the binding is not there: same read, same flush, no race. Under `Durable` + /// the binding is known to have been appended, so an absent one is a genuine anomaly and the + /// strict form is kept -- a fail-closed error is the right report for it. + const bool tolerate_absent = precommit_state == PrecommitState::Uncertain; + store->appendRefOps(precommit_target_ns, MutationScope::ref(precommit_final_ref), + /// By-value capture of the values the closure reads (ref name, manifest and the tolerance + /// flag) rather than `[&]`: defense-in-depth so the closure is self-contained if it ever + /// outlives this stack. + [ref_name = precommit_final_ref, manifest = precommit_manifest, tolerate_absent] + (const RefTableState & state) -> std::vector + { + if (tolerate_absent && !state.getPrecommits().contains({ref_name, manifest})) + return {}; + RefOp op; + op.kind = RefOpKind::OwnerTransition; + op.old_binding = RefOwnerBinding{RefOwnerKind::Precommit, ref_name, manifest}; + return {op}; + }, + RootMutationOrigin::Writer, RootMutationKind::Abandon); + precommit_state = PrecommitState::Settled; + + try + { + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::PrecommitRemoved; + e.namespace_ = precommit_target_ns.string(); + e.ref_name = precommit_final_ref; + e.object_kind = CasEventObjectKind::Root; + e.object_hash = manifestRefDebugString(precommit_manifest); + e.reason = "abandon: precommit binding removed"; + }); + } + catch (...) + { + tryLogCurrentException(getLogger("CasPartWriteTxn"), "CAS event emission after durable abandon"); + } + } + + /// `alive` flips to false only AFTER the correctness-bearing precommit removal above is durable: if + /// `appendRefOps` threw, `alive` stays true and `precommit_state` keeps owing the removal, so a + /// caller that catches the failure can retry `abandon()` on this same object. A retry after an + /// ambiguous already-appended failure re-validates `old_binding` and errors (or, under the + /// `Uncertain` tolerance above, no-ops) -- it never corrupts. + alive = false; + + /// No longer in-flight: retire the seq so the per-server active-build floor (`min_active`) can advance + /// (idempotent). This runs AFTER the precommit removal above (mirrors `PartWriteTxn::promote`, which retires + /// after its commit) so the build stays active until its precommit binding's removal is durable: + /// retiring first would advance `min_active` past a build whose precommit binding is still live in the + /// ref log, letting a freshness-window consumer judge the manifest build-dead while an un-removed + /// precommit still names it. Ordering removal-before-retire keeps that happens-before clean. + store->retireBuildSeq(build_seq); + + cleanupStagedManifestDebrisBestEffort(); + + /// The build was abandoned; its uploads become GC-reclaimable debris. Best-effort audit emission: + /// the durable work (precommit removal + seq retire) is already done, so a throwing sink must not + /// propagate a failure out of a successful abandon. + try + { + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::BuildAbort; + e.token = u128ToHex(build_id); + e.outcome = "abandoned"; + e.reason = "abandon: appended a precommit-removal event for the live precommit (body left for GC); " + "best-effort deleted the never-precommitted _manifests debris; remainder reaped by the orphan sweep"; + e.detail = {{"build_seq", std::to_string(build_seq)}, {"staged", std::to_string(staged_manifests.size())}}; + }); + } + catch (...) + { + tryLogCurrentException(getLogger("CasPartWriteTxn"), "CAS event emission after durable abandon"); + } +} + +void PartWriteTxn::cleanupStagedManifestDebrisBestEffort() +{ + /// Best-effort writer cleanup of THIS build's pre-precommit/staged `_manifests` debris. The common case + /// is writer cleanup; a missed object is benign — the namespace-scoped orphan sweep reclaims it. Exact-token delete only; never + /// throws. SKIP the manifest that became a live precommit owner: its body is a live precommit input + /// whose deletion is GC's job after the sealed decrement (never writer-delete it). + /// + /// "Became a live precommit owner" is decided from the ATTEMPT, not from a confirmed append: any + /// state other than `NotAttempted` -- including `Uncertain`, where the precommit may or may not + /// exist -- protects the body. Deleting it under uncertainty is the unrecoverable direction (a + /// precommit that turns out to be live and whose body is gone clamps GC's fold barrier forever), + /// while keeping it is not: an unreferenced body is ordinary orphan-sweep debris. + const bool precommit_attempted = precommit_state != PrecommitState::NotAttempted; + for (const ManifestId & id : staged_manifests) + { + if (precommit_attempted && id.ref == precommit_manifest && id.root_namespace == precommit_target_ns) + continue; /// the (possibly) live precommit body — left for GC (delete-after-sealed-decrements) + try + { + const String key = store->layout().manifestKey(id); + const HeadResult hr = store->backend().head(key); + if (hr.exists) + store->backend().deleteExact(key, hr.token); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// Best-effort cleanup: the GC backstop sweep is the durable guarantee. + } + } +} + +void PartWriteTxn::cancelForNamespaceRemoval(const RootNamespace & removed_ns) +{ + /// Only cancel a build that actually targets the removed namespace. `manifestNamespace` reads only + /// the immutable `info` set at construction, so this is safe to call from `dropNamespace`'s thread + /// while the build's own thread runs. A build whose owning namespace cannot be determined (a + /// malformed diagnostic-only PartWriteInfo) is left alone — the append lane still fails closed on any op + /// it later attempts against the now-Removed namespace. + std::optional mine; + try + { + mine = manifestNamespace(); + } + catch (...) + { + return; + } + if (mine != removed_ns) + return; + cancelled.store(true, std::memory_order_release); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.h new file mode 100644 index 000000000000..2ef5a44df0af --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.h @@ -0,0 +1,402 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB { class WriteBuffer; } + +namespace DB::Cas +{ + +/// Re-readable source for one content-addressed blob upload. +/// +/// `open` returns a FRESH reader over exactly `size` logical bytes and may be called more than once: an +/// upload can race with another writer or with GC, in which case the transaction retries from the writer's +/// own source and each attempt must read from the beginning. `server_side_copy_from` is set only for an S3 +/// staging object; the ordinary create then promotes it by a server-side copy instead of streaming through +/// ClickHouse, and `open` reads that same staging object when a resurrection has to stream it. The staging +/// object must remain available through a condemned-object resurrection. +struct BlobSource +{ + uint64_t size = 0; + std::function()> open; /// yields exactly `size` bytes, from the start + /// When set, the blob's bytes already live in an S3 staging object with this key, and `putBlob` promotes it by a + /// WRITE-ONCE conditional SERVER-SIDE COPY (`Backend::promoteStaged`) instead of streaming + /// `open` — and resurrects a condemned incarnation by an unconditional server-side copy + /// from the SAME staging object (`Backend::resurrect`), never a read of the condemned blob + /// (revival must always be a fresh write from the source). Unset (the default, `StagingBackend::Local`) ⇒ the local + /// streaming path is byte-for-byte unchanged and `open` is the source. + std::optional server_side_copy_from; + /// Build a re-readable source backed by an owned string; intended for small payloads and tests. + static BlobSource fromString(String bytes); +}; + +/// putBlob's return value: the `BlobRef` it was addressed by (the write mint's algo + digest pair) +/// plus the admitted logical size. +struct PutBlobResult +{ + BlobRef ref; + uint64_t size = 0; +}; + +/// One blob dependency this build contributes — EXACTLY the record `putBlob` folds into `deps`. +/// A token identifies an incarnation uploaded by this transaction and must be retained through +/// promotion; a tokenless entry relies on the durable source-manifest edge instead. `adopted` +/// distinguishes trusted committed-source evidence (`adoptEvidence`) from a pending upload that has +/// not yet been completed. CAS-owned public value type so a transaction-detached upload can RETURN +/// its complete dep effect instead of folding it as a side effect (spec §1: "no branch may leave its +/// dep effect behind as a side effect"). +struct BlobDepRecord +{ + ObjectKind kind = ObjectKind::Blob; + std::optional token; /// nullopt = live-source evidence + uint64_t size = 0; + bool adopted = false; /// true only for `adoptEvidence` + + bool operator==(const BlobDepRecord &) const = default; +}; + +/// Which branch of the upload primitive admitted a blob. Carried out of `uploadBlobDetached` so the +/// merge/fan-out layers (and tests) can assert the branch taken without inferring it from the dep. +enum class BlobUploadOutcome +{ + DeduplicationCacheHit, /// dedup cache said present; HEAD-first confirmed a live incarnation; adopted + HeadHit, /// size-triggered HEAD-first found a present live incarnation; adopted + HeadMissAdopted, /// the write-once create 412'd on a live incarnation (or a racing writer's); adopted + FreshUpload, /// the write-once conditional create streamed a fresh local body + StagingPromoted, /// the write-once conditional server-side copy promoted an S3 staging object + ResurrectedLocal, /// a condemned incarnation displaced by a fresh local `putOverwrite` + ResurrectedS3, /// a condemned incarnation displaced by a fresh server-side copy from staging +}; + +/// Public, CAS-owned input to `uploadBlobDetached`; the transaction's private dep representation is not +/// exposed. `source` mirrors `putBlob`'s re-readable `BlobSource` (local streaming / S3 staging copy). +/// `declared_size` is the value the fan-out layer groups and conflict-checks on; it mirrors +/// `source.size`, which stays the authority for the per-attempt streaming byte check. +struct BlobUploadRequest +{ + BlobRef ref; + BlobSource source; + uint64_t declared_size = 0; +}; + +/// Complete result of one detached upload: the addressed ref, the COMPLETE dep effect the upload +/// contributes (no side channel), and the branch outcome. +struct BlobUploadResult +{ + BlobRef ref; + BlobDepRecord dep; + BlobUploadOutcome outcome = BlobUploadOutcome::FreshUpload; +}; + +/// Hash `payload` with `algo` using the same convention as the streaming blob writer and return the complete +/// `BlobRef` identity. The algorithm travels with the digest; callers must not reconstruct a blob identity from +/// a bare digest or from an independently supplied digest width. +BlobRef poolContentHash(BlobHashAlgo algo, std::string_view payload); + +/// Coordinates one part write from manifest staging through blob admission and ref publication. The transaction +/// owns the in-memory dependency set and the identities of manifests staged by this build; `Pool` owns the +/// durable object store and ref-log operations. A transaction is normally used by one writer thread, while +/// `cancelForNamespaceRemoval` may set its cancellation flag from the namespace-removal thread. +/// +/// The durable write order is `stageManifest` → `precommitAdd` → `putBlob` → `promote`. The precommit edge +/// must be durable before any existing blob incarnation is adopted, because the edge is what protects that +/// incarnation from GC while this build is in flight. `promote` then moves the same manifest owner binding +/// atomically from precommit to committed. A failed or abandoned transaction never resumes after a process +/// restart; its precommit is removed by the live owner or a fenced successor, and GC reclaims the resulting +/// debris only after the corresponding ref-log decrements are durable. +class PartWriteTxn +{ +public: + /// Start a build and emit its durable in-flight-build attribution. The identity arguments identify the + /// build for ownership and GC fencing; `info_` is retained as immutable build context. + PartWriteTxn(PoolPtr store_, UInt128 build_id_, + uint64_t build_seq_, uint64_t epoch_, PartWriteInfo info_); + + /// Retire this build's sequence so the pool's active-build watermark can advance. This is idempotent when + /// `promote` or `abandon` already retired the sequence, and also covers destruction during unwinding. + ~PartWriteTxn(); + + /// Every upload attempt mints a fresh random `incarnation_tag`. + /// New content: streaming PUT If-None-Match:*; on PreconditionFailed ⇒ the cold-reuse rule + /// (observe current token; condemned ⇒ uploadFromSource — re-upload from the writer's source + /// bytes; else adopt — free). + /// Ordering: `putBlob` is always called after `precommitAdd` (the wiring order is + /// `stageManifest` → `precommitAdd` → `putBlob` → `promote`). Its + /// ADOPT paths observe an existing incarnation, so they are safe only under this build's durable + /// precommit closure — enforced by a fail-closed throw (LOGICAL_ERROR, not a `chassert`, which is + /// compiled out in release) in observeAndAdmit. A FRESH upload before precommit is legal + /// (newborn-debris watermark), but production never does it. + PutBlobResult putBlob(const BlobRef & ref, BlobSource source); + + /// Transaction-DETACHED upload primitive (spec §1). Runs the SAME durable, ordering-sensitive pool + /// effects `putBlob` runs — the HEAD-first dedup gate, the write-once conditional create, condemned + /// resurrection (INV-1: never GET a condemned object), the freshness-meta `Clean` transition, + /// dedup-cache reads/inserts, event emission, ProfileEvents — but folds NOTHING into `build` + /// (`deps`), returning the complete dep effect + branch outcome as a value instead. It is therefore + /// safe to run off the owning writer thread while `PartWriteTxn` stays single-writer for `build`. + /// `putBlob` = this primitive + a single-result `deps` fold on the calling thread. + BlobUploadResult uploadBlobDetached(const BlobUploadRequest & req) const; + + /// Applies a fan-out's `uploadBlobDetached` results into `deps` on the CALLING thread, after the + /// fan-out's join -- so this is an owning-writer-thread API exactly like `putBlob`, and MUST NOT be + /// called from a pool task. Merge failure must not leave a partially merged build (spec §1): every + /// result is prevalidated FIRST -- completeness (a result's dep must carry a token; every branch of + /// `uploadBlobDetached` sets one, so a tokenless result is a caller bug, not a valid dep-only-evidence + /// state, which is folded through `adoptEvidence` instead) and duplicate-grouping consistency + /// (two results for the same `BlobRef` must carry an identical dep record; a conflict -- most + /// commonly a conflicting size -- means the fan-out's one-task-per-unique-ref invariant was + /// violated) -- BEFORE any result is applied. Application then runs against a COPY of `deps` (a + /// "build"), so a mid-application exception (including one raised by `setMergeHookForTest`'s hook, or + /// a `bad_alloc` from map-node allocation) never touches the live `deps`; the copy is committed by a + /// single no-throw `swap` only after every result has applied. The build is therefore either fully + /// merged or byte-for-byte untouched -- never partially merged. + void mergeBlobUploadResults(std::span results); + + /// Test-only fault-injection seam (inert in production): invoked after each result has applied to + /// the in-progress merge copy, with the count of results applied so far (1-based). A hook that + /// throws (e.g. to model a `bad_alloc` mid-merge) aborts `mergeBlobUploadResults` before its final + /// swap, so the live `deps` stays untouched -- the seam exists to prove that all-or-nothing property + /// under injected failure at every application point, not just the first or the last. + void setMergeHookForTest(std::function hook) { merge_hook_for_test = std::move(hook); } + + /// Test-only DEEP snapshot of this build's recorded deps, keyed by `BlobRef`. A plain copy of the + /// private `deps` map -- lets a test assert the whole build is byte-for-byte untouched after a + /// rejected or aborted merge, rather than probing one ref at a time via `depIsTokened`. + std::map depsSnapshotForTest() const { return deps; } + + /// Return whether this build holds a TOKENED Blob dep for `ref` (`putBlob` ⇒ + /// tokened) versus a tokenless evidence dep (`adoptEvidence` ⇒ tokenless)? False also when this + /// build has no dep for the ref at all. + bool depIsTokened(const BlobRef & ref) const; + + /// Record a TOKENLESS evidence blob dep directly from a `ManifestEntry` — no HEAD or backend + /// call. Lets staging adopt sites record the dep by hash without asserting presence before + /// precommit; the promote gate observes/resurrects it post-precommit. Inline entries record nothing. + void adoptEvidence(const ManifestEntry & entry); + + /// Record a TOKENLESS pending blob dep by ref (without a HEAD) for a blob whose bytes are staged locally and + /// will be putBlob'd post-precommit. putBlob later overwrites it with the tokened dep on upload. + void recordPendingBlobDep(const BlobRef & ref, uint64_t size); + + /// Mint a root-local part `ManifestId`, write its body under + /// `cas/manifests////000001.zst` via the pool's shared request + /// controller. It uses budgeted attempts with resolve-before-reissue and performs no preliminary HEAD, + /// because `manifest_ordinal` is monotone within this build. It enforces manifest-size caps before the body + /// write returns and therefore before any owner transition is published. The body is not retained after a + /// successful write; on retry the caller re-stages from source. Every call uses a fresh manifest ordinal. + /// The id is recorded for best-effort `abandon` cleanup. + ManifestId stageManifest(std::vector entries); + + /// Add this transaction's precommit owner intent; there is no `_precommits` namespace. One + /// `appendRefOps` call appending an OwnerTransition `RefOp` (new_binding = {Precommit, + /// final_ref_name, id.ref}) to final_ref_name's ref-log entry, so the later promote is an atomic + /// owner move over that same entry. Needs NO body-exists HEAD as a safety authority: GC and + /// promotion handle a missing precommit manifest body by failing closed (a missing-body precommit + /// is a non-activating, non-promotable intent). + /// + /// A3 mint-tightening: `id` must either be one THIS transaction minted via `stageManifest`, or + /// already be `final_ref_name`'s current committed manifest (the idempotent re-drive -- see the + /// closure in the .cpp). Any other id names a manifest this transaction never staged and does not + /// currently own -- most commonly one a DIFFERENT, since-abandoned/dropped transaction once staged + /// -- and granting it fresh ownership here would let a later exact-`ManifestRef` equality check + /// (the relink confirm) compare true against a token whose blobs may already be reclaimed (an ABA). + /// Rejected with `LOGICAL_ERROR`: this is a programming-invariant violation, never an operational one. + void precommitAdd(const RootNamespace & target_ns, const String & final_ref_name, const ManifestId & id); + + /// Atomically promote the precommit to the committed ref with one `appendRefOps` call on the target ref's + /// ref-log entry. + /// 1. tokened leaves are already protected by the durable precommit edge, so no writer-side retired-view + /// refresh is needed; + /// 2. stream-read the precommit manifest body; validate RefMatchesBody / ManifestNamespaceMatches; + /// 3. the NON-tokened blob leaves (tokened leaves are edge-protected, not re-checked): a committed-source + /// adoptEvidence leaf is TRUSTED via the durable manifest edge — NO per-file HEAD/loadMeta probe (§4 + /// manifest-trust: the live source pins the blob, in-degree >= 1); a genuinely + /// absent adopted blob is an invariant violation caught by fsck, not here; + /// 4. a body-absent precommit or a lost owner-liveness ⇒ ABORTED; a non-tokened, non-adopted leaf (no + /// tokened dep and no committed-source adopt — a staging bug) ⇒ LOGICAL_ERROR (fail closed); + /// 5. atomically replace precommit(build_id) owner with committed(final_ref_name) owner by appending + /// ONE pure-move `RefOp` (old_binding={Precommit,final_ref_name,T}, new_binding={Committed,final_ref_name,T}, + /// same manifest_ref T) and setting refs[final_ref_name]; + /// 6. promotion never emits blob deltas. A missing-body precommit + /// is non-activating and was rejected at step 4 (the writer re-stages with a fresh ManifestId). + /// + /// PROCESS-RESTART INVARIANT: a `PartWriteTxn` is a plain in-memory C++ object owned by the wiring's + /// `ContentAddressedTransaction` — it is NEVER persisted and NEVER resumed across a process + /// restart. There is no "replay a precommit" code path anywhere in the core: `promote` is called + /// synchronously, in-process, strictly AFTER every referenced blob's `putBlob` (which for S3 + /// staging drives `promoteStaged`'s conditional copy) has already returned successfully. If the + /// process exits between `precommitAdd` and `promote` (e.g. between staging a blob and its + /// server-side-copy promote completing), the `PartWriteTxn` object is simply lost with it: nothing ever + /// "wakes up" that precommit and finishes promoting it. The precommit's owner binding is left as a + /// dead intent in the ref log and is REMOVED (never promoted) by an exact precommit-removal ref-log + /// transaction -- the current writer's own `PartWriteTxn::abandon` if it is still mounted, otherwise a + /// fenced successor's stale-precommit sweep. `GC` folds the resulting + /// `-1` manifest edge but never detects or removes a dead precommit itself. So the + /// hazard — "promote a precommit whose copy did not complete" — has no code + /// path to occur through: promotion is not a recoverable/resumable operation, only a synchronous + /// one that either completes within the writing process or never happens at all. + /// `allow_repoint` opts into an intended + /// repoint of a committed ref that already names a DIFFERENT manifest -- a standalone write/remove + /// on an already-committed part (the committed-publish machinery's missing piece; the promote guard's + /// own error text already named it: "use republishRef for an intended repoint"). Default `false` + /// preserves the existing unique-ref guard byte-for-byte: a committed ref naming a different manifest + /// still throws ABORTED. With `true`, the guard is skipped and the old committed binding is retired + /// in the SAME ref-log record as the ordinary precommit->committed promotion, plus a + /// `CasEventType::RefRepoint` audit event -- every effective repoint is loud by construction. + /// + /// Returns `created`: whether `final_ref_name` had NO committed row before this call. Derived + /// INSIDE the `appendRefOps` builder (the same in-closure-output pattern the builder already uses + /// for `repoint_old`) as `!state.getCommitted().contains(final_ref_name)`, evaluated once at the + /// top of the closure -- so it is correct on every path the closure can take: a first-time bind + /// (true), a repoint of an existing binding (false), and the idempotent re-promote no-op (false, + /// since a committed row for this exact manifest already existed). `build_ops` runs at most once, + /// on the flush leader, so this single evaluation is authoritative. + bool promote(const RootNamespace & target_ns, const String & final_ref_name, UInt128 build_id, const ManifestId & id, bool allow_repoint = false); + + /// Retire the build sequence so the GC watermark floor can advance; staged manifest debris is best-effort + /// cleaned, with the orphan sweep as the durable backstop. + void abandon(); + + /// Called by `Pool::dropNamespace` for every in-flight build once its namespace-removal transaction is + /// durable. If + /// this build's owning namespace equals `removed_ns`, mark it cancelled so every further operation + /// fails closed at `requireAlive` (ABORTED); a build in any other namespace is left untouched. + /// Cross-thread safe: reads only the immutable `info` (the owning namespace) and stores ONE atomic -- + /// it touches no other member, so the build's own thread may keep running concurrently. Staged debris + /// is cleaned best-effort when the build's own thread later runs `abandon` (or via the GC backstop). + void cancelForNamespaceRemoval(const RootNamespace & removed_ns); + + UInt128 buildId() const { return build_id; } + /// The strictly increasing per-process sequence used by the active-build watermark. + uint64_t buildSeq() const { return build_seq; } + + /// Lifecycle of THIS build's create-precommit `owner_transition`. + /// + /// It is a state and not a bool because `appendRefOps` has three outcomes and only two of them are + /// knowledge: an `Unresolved` `PUT` MAY HAVE LANDED (`CasRefLedger.cpp`, the `Unresolved` arm), so a + /// `precommitAdd` that threw can still have made this build's manifest a LIVE precommit owner. The + /// intent is therefore recorded BEFORE the ambiguous append -- the same "preconstruct before the + /// PUT" discipline the ref lane's wedge uses -- and settled either way when it returns. + /// + /// `NotAttempted` -- `precommitAdd` never reached its append; nothing can be owned, and this + /// build's staged manifest bodies are ordinary writer debris. + /// `Uncertain` -- the append was ATTEMPTED and its outcome is unknown. The build OWES the + /// precommit removal exactly as if it were durable, and its manifest body is + /// NOT writer-deletable: it may be a live precommit input whose deletion would + /// strand GC's fold barrier. + /// `Durable` -- the append returned, so the precommit binding is this build's. + /// `Settled` -- the owed terminal operation (`promote` or `abandon`) has discharged the duty. + /// The body stays non-deletable: after a promote it belongs to a committed ref, + /// and after an abandon it is GC's to reclaim after the sealed decrement. + enum class PrecommitState : uint8_t { NotAttempted, Uncertain, Durable, Settled }; + PrecommitState precommitState() const { return precommit_state; } + + /// Lifecycle of THIS build's promote (precommit -> committed) append, recorded for the same reason + /// and in the same way as `precommitState`: a promote whose append threw may still have COMMITTED + /// the ref. The distinction is load-bearing one layer up -- the interserver relink maps a promote + /// that definitely did not commit to "fetch the bytes from the same source instead", and doing that + /// after a commit that actually landed would publish the same part twice. + /// + /// `NotAttempted` -- promote failed (or was never called) strictly BEFORE its append: a rejected + /// validation, an absent body, a lost owner liveness. Proof of the negative. + /// `Uncertain` -- the append was attempted and its outcome is unknown. Conservative: the ref + /// lane collapses `DefiniteFailure` and a pre-attempt refusal into the same + /// retry-later class as a genuinely ambiguous `PUT`, so both are reported here + /// as uncertain. That costs a retry, never correctness. + /// `Durable` -- the append returned; the ref is committed. + enum class CommitState : uint8_t { NotAttempted, Uncertain, Durable }; + CommitState commitState() const { return commit_state; } + +private: + /// Keyed on the full `BlobRef` pair (algorithm + digest), because a bare digest is not a blob identity. + /// This remains an ordered `std::map` (not `unordered_map`): `BlobRef` already provides `operator<=>`, so + /// no hasher is needed here. `BlobRefHash` is for unordered dedup-cache/set consumers elsewhere. The + /// dependencies are blob-only, so `ObjectKind` is not part of the key. + using DepKey = BlobRef; + + /// Apply the cold-reuse rule: HEAD the key; absent ⇒ FILE_DOESNT_EXIST; + /// condemned-at-current-token ⇒ throw ABORTED (caller must re-upload from its own source bytes); + /// else RETURN the adopt dep record (current token, admitted logical size). Build-neutral: it folds + /// nothing into `deps` (its callers compose the returned record into a `BlobUploadResult`). + BlobDepRecord observeAndAdmit(ObjectKind kind, const BlobRef & ref, const String & key) const; + /// Overload for callers that already hold a fresh, present HeadResult for `key` (the putBlob + /// HEAD-before-PUT path), avoiding a redundant second HEAD. `hr.exists` MUST be true. + BlobDepRecord observeAndAdmit(ObjectKind kind, const BlobRef & ref, const String & key, const HeadResult & hr) const; + /// INV-1 (revival-from-source): revive a condemned or absent object by re-uploading from the writer's + /// OWN re-readable source without reading the dying object (no backend().get). On a Native backend the + /// source is STREAMED into the put sink (header + `source.open`); the emulated backend materializes + /// one body at a time inside `Backend::resurrect`; + /// `source.open` may be re-invoked on each conditional-write attempt (it re-reads the staged + /// temp file / re-emits the captured String), so it is taken by const ref and not consumed. Build-neutral: + /// RETURNS the complete `BlobUploadResult` (dep + branch outcome); it folds nothing into `deps`. + BlobUploadResult uploadFromSource(ObjectKind kind, const BlobRef & ref, const String & key, const BlobSource & source) const; + + /// The build's owning root namespace, derived from PartWriteInfo::intended_ref ("ns/ref" — the ref is the + /// last `/`-segment; the namespace is everything before it). Sets a manifest body's root_namespace_id. + RootNamespace manifestNamespace() const; + + /// Reject operations after `abandon`, namespace cancellation, or writer-epoch fencing. These checks happen + /// before backend work so a stale transaction cannot publish a new owner or stage more debris. + void requireAlive() const; + /// Best-effort exact-token delete of THIS build's staged `_manifests` debris; the precommit body (if + /// any) is SKIPPED -- left for GC's delete-after-sealed-decrements. Never throws (the namespace-scoped orphan + /// sweep is the durable backstop). Shared by the normal and the namespace-removal-cancelled `abandon` + /// paths; only ever called on the build's OWN thread. + void cleanupStagedManifestDebrisBestEffort(); + + /// A leaf is trusted at promote iff this build holds a TOKENLESS dep recorded by + /// `adoptEvidence` (adopted=true) — a committed-source evidence adopt. The live source pins the blob + /// (in-degree >= 1, not condemnable) and this build's precommit edge is durable, so the durable manifest + /// edge is the liveness evidence: no HEAD, no loadMeta, no copy-forward. A tokened dep (edge-protected, + /// handled by `depIsTokened`), a tokenless PENDING-upload dep (adopted=false), or NO dep at all (a + /// staging bug) is NOT trusted — it fails closed. The single gate for the promote non-tokened leaf. + bool isTrustedAdopt(const BlobRef & ref) const; + + PoolPtr store; + UInt128 build_id{}; + uint64_t build_seq{}; /// per-process monotone sequence + uint64_t epoch{}; /// owning Pool's process_epoch + uint32_t next_manifest_ordinal = 1; /// per-build monotone manifest ordinal + PartWriteInfo info; + bool alive = true; + /// Set by `cancelForNamespaceRemoval` from `Pool::dropNamespace`'s thread + /// once this build's owning namespace is durably removed. Atomic because it is WRITTEN cross-thread + /// and READ by `requireAlive` on the build's own thread. Once cancelled, every further op fails closed. + std::atomic cancelled{false}; + PrecommitState precommit_state = PrecommitState::NotAttempted; + CommitState commit_state = CommitState::NotAttempted; + + /// The precommit's target, recorded by `precommitAdd` BEFORE its append and never cleared + /// afterwards. Two consumers with different lifetimes read them: `abandon` needs them while the + /// duty is owed (`Uncertain`/`Durable`), and `cleanupStagedManifestDebrisBestEffort` needs them for + /// as long as this object lives, because a body that was ever the target of a precommit attempt + /// must never be writer-deleted -- which is why the terminal operations move `precommit_state` to + /// `Settled` rather than resetting these back to their unset values. + RootNamespace precommit_target_ns; + String precommit_final_ref; + ManifestRef precommit_manifest; + + std::vector staged_manifests; /// for best-effort abandon cleanup + + /// Every `ManifestId` this transaction has minted via `stageManifest`, checked by `precommitAdd`: + /// an unowned id may enter ownership only from the transaction that freshly staged it. Kept separate from + /// `staged_manifests` above -- that vector's role (best-effort abandon cleanup) is unrelated and + /// could change independently; this set exists purely as the ABA barrier's identity check, so it + /// stays correct even if the cleanup vector's contents or lifetime ever change. + std::set staged_manifest_ids; + + std::map deps; /// dependencies recorded by this build (blobs only) + + /// Backing state for `setMergeHookForTest`; empty (no-op) in production. + std::function merge_hook_for_test; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPlainObjects.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPlainObjects.cpp new file mode 100644 index 000000000000..511ff5cc2516 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPlainObjects.cpp @@ -0,0 +1,154 @@ +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ABORTED; +} +} + +namespace DB::Cas +{ + +namespace +{ + constexpr size_t MAX_CAS_ATTEMPTS = 100; +} + +void CasPlainObjects::casPutObject(const String & full_key, const String & bytes) +{ + /// The read determines whether this is a conditional create or replacement. The token is only + /// valid for the incarnation returned by that head, so a precondition failure means another + /// writer won the race and the loop must observe the new incarnation before trying again. + /// + /// SINGLE-APPENDER INVARIANT: `bytes` is frozen by the caller before this loop starts (see the + /// append-base note at `ContentAddressedTransaction::writeFile`'s Append branch); the loop only + /// re-reads the TOKEN on conflict, never the base content. This is correct only while nothing + /// concurrently appends to the same key — a losing retry would overwrite the winner's bytes with a + /// stale, pre-conflict payload (a lost update). Implement a real `casAppendObject` (re-reading the + /// base content, not just the token, inside the loop) before adding any concurrent appender. + /// + /// rev.7 [C2]: the fence generation captured at admission is re-checked immediately before EVERY + /// durable PUT below, not just the first attempt. A mismatch (the mount lease was lost, or re-armed + /// under a fresh incarnation, since admission) aborts with the typed transient error before the backend + /// is ever touched. + const uint64_t admitted_generation = fence_generation_fn(); + + for (size_t attempt = 0; attempt < MAX_CAS_ATTEMPTS; ++attempt) + { + HeadResult head = backend.head(full_key); + check_fence_or_throw_fn(admitted_generation); + if (!head.exists) + { + if (backend.putIfAbsent(full_key, bytes).outcome == PutOutcome::Done) + return; + } + else + { + if (backend.putOverwrite(full_key, bytes, head.token).outcome == PutOutcome::Done) + return; + } + /// `PreconditionFailed` means the observed state changed under us; re-head and retry. + } + throw Exception(ErrorCodes::ABORTED, "object CAS contention on '{}'", full_key); +} + +std::optional CasPlainObjects::casGetObject(const String & full_key) +{ + std::optional result = backend.get(full_key); + if (!result) + return std::nullopt; + return result->bytes; +} + +void CasPlainObjects::casRemoveObject(const String & full_key) +{ + /// Delete only the incarnation observed by the preceding head. A token mismatch leaves the + /// replacement untouched and is retried against a fresh observation; absence is a successful + /// no-op. + /// + /// rev.7 [C2]: same fence-generation admission as `casPutObject` -- the admitted generation is + /// re-checked immediately before every durable delete. + const uint64_t admitted_generation = fence_generation_fn(); + + for (size_t attempt = 0; attempt < MAX_CAS_ATTEMPTS; ++attempt) + { + const HeadResult head = backend.head(full_key); + if (!head.exists) + return; + check_fence_or_throw_fn(admitted_generation); + const DeleteOutcome outcome = backend.deleteExact(full_key, head.token); + if (outcome.kind == DeleteOutcome::Kind::Deleted || outcome.kind == DeleteOutcome::Kind::NotFound) + return; + /// `TokenMismatch` means a concurrent rewrite; re-head and retry. + } + throw Exception(ErrorCodes::ABORTED, "object CAS contention on '{}' (runaway live-lock brake)", full_key); +} + +void CasPlainObjects::putNamespaceFile(const NamespaceLifeId & life, const String & name, const String & bytes) +{ + casPutObject(layout.namespaceFileKey(life, name), bytes); +} + +std::optional CasPlainObjects::getNamespaceFile(const NamespaceLifeId & life, const String & name) +{ + return casGetObject(layout.namespaceFileKey(life, name)); +} + +std::vector CasPlainObjects::listNamespaceFiles(const NamespaceLifeId & life) +{ + const String prefix = layout.namespaceFilesPrefix(life); + std::vector names; + String cursor; + while (true) + { + ListPage page = backend.list(prefix, cursor, /*limit*/ 1000); + for (const ListedKey & listed : page.keys) + { + /// Strip the storage prefix so callers receive the bare flat file name. + if (listed.key.starts_with(prefix)) + names.push_back(listed.key.substr(prefix.size())); + } + if (page.next_cursor.empty()) + break; + cursor = page.next_cursor; + } + /// Backends are not required to return pages in the same order, so make the public result + /// deterministic instead of relying on `InMemoryBackend` ordering. + std::sort(names.begin(), names.end()); + return names; +} + +void CasPlainObjects::removeNamespaceFile(const NamespaceLifeId & life, const String & name) +{ + casRemoveObject(layout.namespaceFileKey(life, name)); +} + +void CasPlainObjects::putMountpointObject(const String & key, const String & bytes) +{ + casPutObject(layout.mountpointObjectKey(key), bytes); +} + +std::optional CasPlainObjects::getMountpointObject(const String & key) +{ + return casGetObject(layout.mountpointObjectKey(key)); +} + +bool CasPlainObjects::mountpointObjectExists(const String & key) +{ + /// Use metadata rather than a body GET because a path probe may resolve to a directory, such as + /// the `store` pool subdirectory traversed by `system.remote_data_paths`. The local backend + /// treats a directory as not an object, so this returns false instead of attempting a body read + /// that would raise a filesystem exception for a directory. + return backend.head(layout.mountpointObjectKey(key)).exists; +} + +void CasPlainObjects::removeMountpointObject(const String & key) +{ + casRemoveObject(layout.mountpointObjectKey(key)); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPlainObjects.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPlainObjects.h new file mode 100644 index 000000000000..d1eae78d6e57 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPlainObjects.h @@ -0,0 +1,111 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Provides the pool's plain-object surface: loose, non-content-addressed objects whose key is +/// chosen by the caller. This covers namespace files under `cas/ns/state//_files/` -- keyed +/// by the namespace LIFE, never by its bare name -- and mountpoint objects mirrored by path. The object +/// bodies are raw passthrough bytes; this component does not decode them as CAS metadata. +/// +/// The component holds references to the shared `Backend` and `Layout` only. It owns no pool mutex +/// and has no pool back-reference, allowing `Pool` to retain thin forwarding methods with the same +/// external interface. The private helpers implement the shared head-plus-conditional-write and +/// head-plus-exact-delete protocols used by both object families. A conditional outcome means that +/// the observed incarnation changed, so the helper re-reads the head and retries; the fixed bound +/// prevents an unexpected continuous conflict from becoming an unbounded operation and reports +/// `ABORTED` when it is reached. +/// +/// Every durable write/delete on this surface is fence-generation-gated (rev.7 [C2]): `Pool` injects +/// two callbacks that reach its `mount_runtime` (declared AFTER this member, hence constructed +/// after it -- these callbacks capture `Pool` itself and are invoked only at runtime, post- +/// construction, exactly like `ref_ledger`'s callbacks in `CasPool.cpp`, so referencing a +/// not-yet-constructed sibling member through them is safe). +class CasPlainObjects +{ +public: + CasPlainObjects( + Backend & backend_, const Layout & layout_, + std::function fence_generation_fn_, + std::function check_fence_or_throw_fn_) + : backend(backend_), layout(layout_) + , fence_generation_fn(std::move(fence_generation_fn_)) + , check_fence_or_throw_fn(std::move(check_fence_or_throw_fn_)) + { + } + + /// Stores the raw bytes under ONE LIFE's `_files/` prefix. Existing files are replaced + /// conditionally using the object incarnation observed by `Backend::head`; a storage failure or an + /// exhausted conflict-retry bound is propagated as an exception. + /// + /// `life` is supplied by the caller and never re-derived here, so this surface issues no catalog + /// request of its own. A stale writer therefore targets its own old incarnation's key and cannot + /// write into a newer life's prefix. + void putNamespaceFile(const NamespaceLifeId & life, const String & name, const String & bytes); + + /// Reads a namespace file of ONE LIFE without interpreting its body. Returns `nullopt` when the + /// object is absent and propagates backend read failures. A stale reader may see stale bytes or + /// `NotFound`, never a newer incarnation's data: its key names the life it was given. + std::optional getNamespaceFile(const NamespaceLifeId & life, const String & name); + + /// Enumerates the file names directly below ONE LIFE's `_files/` prefix. Fetches all paginated + /// backend results, strips the prefix, and returns names in sorted order independent of backend + /// listing order. + std::vector listNamespaceFiles(const NamespaceLifeId & life); + + /// Removes the current OBJECT incarnation of one of a life's files, if any (the object token, not + /// the namespace incarnation, which `life` fixes). A concurrent replacement is never removed + /// accidentally: the exact-delete helper re-reads and retries with the new token. + void removeNamespaceFile(const NamespaceLifeId & life, const String & name); + + /// Stores raw bytes for a loose mountpoint file at the path-derived object key. The key is + /// validated and constructed by `Layout`; this method applies the same conditional overwrite + /// protocol as namespace files. + void putMountpointObject(const String & key, const String & bytes); + + /// Reads a path-mirrored mountpoint object as raw bytes. Returns `nullopt` for an absent object + /// and propagates backend read failures. + std::optional getMountpointObject(const String & key); + + /// Checks only object metadata, not the body. A directory at the path-derived key is therefore + /// reported as absent, matching object-store semantics and avoiding a filesystem exception from + /// attempting to read a directory as an object. + bool mountpointObjectExists(const String & key); + + /// Removes the current path-mirrored mountpoint-object incarnation, if present, using exact-token + /// deletion so a concurrent rewrite remains intact. + void removeMountpointObject(const String & key); + +private: + /// Creates or conditionally replaces one raw object. The method re-heads after a conditional + /// conflict and throws `ABORTED` after the bounded retry loop cannot establish a stable token. + /// Fence-generation-gated (rev.7 [C2]): captures the fence generation at admission for the call's + /// whole retry loop; every iteration re-checks it immediately before its durable PUT. + void casPutObject(const String & full_key, const String & bytes); + + /// Reads one raw object by its complete backend key and returns `nullopt` when it is absent. A read, + /// not a durable-effect operation -- NOT fence-gated (rev.7 [C2] scopes the gate to durable writes). + std::optional casGetObject(const String & full_key); + + /// Removes one raw object by exact token. Absence is a successful no-op; a token mismatch causes + /// a fresh head and retry, while a bounded retry failure throws `ABORTED`. Fence-generation-gated + /// the same way as `casPutObject`. + void casRemoveObject(const String & full_key); + + Backend & backend; + const Layout & layout; + + /// ---- fence-generation admission (injected by `Pool`; see the class doc comment) ---- + std::function fence_generation_fn; + std::function check_fence_or_throw_fn; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp new file mode 100644 index 000000000000..d7334ecc2c96 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp @@ -0,0 +1,1851 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; + extern const int ABORTED; + extern const int BAD_ARGUMENTS; + extern const int CORRUPTED_DATA; + extern const int FILE_DOESNT_EXIST; + extern const int LIMIT_EXCEEDED; + extern const int LOGICAL_ERROR; + extern const int INVALID_STATE; +} +} + +namespace ProfileEvents +{ + extern const Event CASPartFolderManifestGets; + extern const Event CASRemountHeldTransient; + extern const Event CASRefBatchFlushes; + extern const Event CASRefBatchedMutations; + extern const Event CASRefBatchScopeCuts; + extern const Event CASRefQueueWaitMicroseconds; + extern const Event CASRefRecoveryRestarts; + extern const Event CASRefRecoveryRetries; + extern const Event CASRefAppendWedged; + extern const Event CASRefAppendUnwedged; + extern const Event CASRefAppendDefiniteFailure; + extern const Event CASRefSweepDeferred; + extern const Event CASRefSweepRearmed; + extern const Event CASRefStalePrecommitsReclaimed; + extern const Event CASRefTableEvictions; + extern const Event CASRefSnapshotPutBytes; + extern const Event CASRefSnapshotTailLogs; + extern const Event CASRefSnapshotPublishDispatched; + extern const Event CASRefSnapshotPublishBackoff; + extern const Event CASDeduplicationCacheHits; + extern const Event CASDeduplicationCacheMisses; +} + +namespace CurrentMetrics +{ + extern const Metric CASDeduplicationCacheBytes; + extern const Metric CASDeduplicationCacheEntries; +} + +namespace DB::Cas +{ + +namespace +{ + +/// The verdict of the pool-lifecycle identity gate (step 0 of `tryRemountOnce`, spec §2). Exactly one +/// `Recover` path falls through to the existing fresh-incarnation recovery; every other verdict is +/// resolved by the gate itself (a terminal transition, or staying transient to retry). +enum class LifecycleGateVerdict : uint8_t +{ + Recover, /// `_pool_meta` present + identity matches: proceed with the existing recovery. + Replaced, /// `_pool_meta` present + a FOREIGN pool_id: `Vanished(replaced)` immediately. + IdentityLost, /// both sentinels (`_pool_meta` + owner) authoritatively absent: fail-loud terminal. + StayTransient, /// a probe error, an undecodable meta, or any ambiguous observation: retry as today. +}; + +struct LifecycleGate +{ + LifecycleGateVerdict verdict; + String reason; /// human-readable detail for the WARN / typed error (only meaningful for Replaced). +}; + +/// Authoritative, cache-bypassing evaluation of the §2 verdict table. Reads ONLY; never claims, +/// allocates, or writes. `expected_pool_id`/`expected_blob_header_len` are the identity this Pool +/// established at open — the comparison is over those two fields ONLY ([B6]); `algos_used` and +/// `min_reader_generation` are legally mutable and are deliberately not compared (the format gate is the +/// decode itself succeeding). +LifecycleGate probePoolLifecycleGate( + Backend & backend, const Layout & layout, const String & srid, + UInt128 expected_pool_id, uint64_t expected_blob_header_len) +{ + const SentinelProbeResult meta_probe = probeSentinel(backend, layout.poolMetaKey()); + switch (meta_probe.outcome) + { + case ProbeOutcome::Present: + { + /// Format gate = a successful, compatible decode. A present-but-undecodable body proves + /// neither identity nor a foreign pool_id, so it stays transient rather than being declared + /// replaced (throw-when-uncertain). + PoolMeta fresh; + try + { + if (!meta_probe.body) + return {LifecycleGateVerdict::StayTransient, "_pool_meta probed Present without a body"}; + fresh = decodePoolMeta(*meta_probe.body); + } + catch (...) + { + return {LifecycleGateVerdict::StayTransient, "_pool_meta present but could not be decoded"}; + } + if (fresh.pool_id == expected_pool_id && fresh.blob_header_len == expected_blob_header_len) + return {LifecycleGateVerdict::Recover, {}}; + return {LifecycleGateVerdict::Replaced, + fmt::format("data root replaced by a foreign pool (pool_id {} != {})", + u128ToHex(fresh.pool_id), u128ToHex(expected_pool_id))}; + } + case ProbeOutcome::KeyAbsent: + { + /// `_pool_meta` is authoritatively gone. Require the OTHER sentinel (the owner anchor) to be + /// conclusively absent too before declaring identity lost — any surviving sentinel, or an + /// undecidable owner probe, keeps us transient (throw-when-uncertain). rev.8: BOTH sentinels + /// authoritatively absent ⇒ `IdentityLost` (a fail-loud terminal state), regardless of whatever + /// else remains under the prefix. Erasure is never PROVEN by the system — only asserted by the + /// operator's `FORGET` — so there is no prefix-emptiness leg here. + const SentinelProbeResult owner_probe = probeSentinel(backend, layout.ownerKey(srid)); + if (owner_probe.outcome != ProbeOutcome::KeyAbsent) + return {LifecycleGateVerdict::StayTransient, + "_pool_meta absent but the owner sentinel was not conclusively absent"}; + return {LifecycleGateVerdict::IdentityLost, + "pool sentinels (_pool_meta + owner) authoritatively absent"}; + } + default: /// ContainerAbsent / AccessDenied / Indeterminate — absence was never proven. + return {LifecycleGateVerdict::StayTransient, "pool-meta probe inconclusive"}; + } +} + +} + +Pool::Pool(BackendPtr backend_, PoolConfig config_, PoolMeta meta_) + : pool_backend(std::move(backend_)) + , config(std::move(config_)) + , meta(std::move(meta_)) + /// Seed the monotone admitted-algo cache from the pool state `createOrValidate` already + /// established (fresh create, steady-state member, or a just-completed admission union) -- + /// register-before-first-write means this Pool's own `writeAlgo()` is ALWAYS a + /// member by the time the constructor runs. + , admitted_algos(meta.algos_used) + /// `Layout` no longer captures a pool algo -- every blob key is built from a + /// `BlobRef` (algo + digest) directly, so the constructor takes only the pool prefix. + , pool_layout(config.pool_prefix) + /// Plain-object surface component: binds to this Pool's own backend + layout (declared after + /// both, so this reference-holding member is constructed last and destroyed first) plus two + /// fence-generation callbacks reaching `mount_runtime` (declared AFTER `plain_objects`, hence + /// constructed after it -- these callbacks capture `this` and are invoked only at runtime, + /// post-construction, exactly like `ref_ledger`'s callbacks below, so referencing a + /// not-yet-constructed sibling member through them is safe). + , plain_objects( + *pool_backend, pool_layout, + [this] { return mount_runtime.fenceGeneration(); }, + [this] (uint64_t gen) { mount_runtime.checkFenceOrThrow(gen); }) + /// Manifest reader component: backend/layout/meta by reference + the event-sink reference. The + /// sink is installed by the factory before writable mounting starts. Owns the decode cache, + /// built from the same config bytes the Pool ctor used before. + , manifest_reader(*pool_backend, pool_layout, meta, event_sink_, config.manifest_decode_cache_bytes) + /// Ref-log / ref-table subsystem. Injected with backend/layout + the + /// RefLedgerConfig slice + the event-sink reference + the pool `cas_request_budget` + the RAW mount + /// `boot_ms_fn` (for its retry controller), plus callbacks into the mount/watermark state that lives + /// on `mount_runtime` (reached through Pool delegates). The callbacks capture `this`; they are + /// invoked only at runtime (post-construction), so referencing `mount_runtime` (declared AFTER + /// `ref_ledger`, hence constructed after it) is safe -- exactly as the pre-3.5 layout referenced the + /// mount raw-members that also followed `ref_ledger`. Declared/constructed BEFORE `mount_runtime`, + /// preserving the original member order verbatim (see the header note). + , ref_ledger( + pool_backend, pool_layout, config.refLedgerConfig(), event_sink_, config.cas_request_budget, + config.server_root_id, + config.boot_ms_fn, + [this] { return liveWriterEpoch(); }, + [this] { return refAppendFenceOk(); }, + [this] { return mount_runtime.fenceGeneration(); }, + [this] (uint64_t gen) { mount_runtime.checkFenceOrThrow(gen); }, + [this] { return bootMsNow(); }, + [this] { return mayMutate(); }, + [this] (const String & key, const String & reason, const std::optional & offending_ns) + { reportImpossibleInterference(key, reason, offending_ns); }, + [this] { return std::static_pointer_cast(shared_from_this()); }, + [this] (const RootNamespace & ns) { cancelInflightBuildsForNamespace(ns); }) + /// Mount / write-fence / build-watermark / self-remount runtime. Injected with + /// backend/layout + the `MountConfig` slice + `server_root_id` + the event-sink reference + the pool + /// `cas_request_budget` + the `remount_attempt` callback (== `Pool::tryRemountOnce`, whose claim/ + /// recovery ORCHESTRATION stays on Pool). The callback captures `this`; it is invoked only at runtime + /// (post-construction). Declared/constructed AFTER `ref_ledger`, preserving the original member order + /// verbatim (mount destroyed first, ledger last; both orders proven safe -- see the header note). + , mount_runtime( + pool_backend, pool_layout, config.mountConfig(), config.server_root_id, event_sink_, + config.cas_request_budget, + [this] { return tryRemountOnce(); }) +{ + if (config.deduplication_cache_bytes > 0) + dedup_cache = std::make_unique( + "LRU", CurrentMetrics::CASDeduplicationCacheBytes, CurrentMetrics::CASDeduplicationCacheEntries, + config.deduplication_cache_bytes, DeduplicationCache::NO_MAX_COUNT, DeduplicationCache::DEFAULT_SIZE_RATIO); +} + +bool Pool::isAlgoAdmitted(BlobHashAlgo algo) const +{ + const auto v = static_cast(algo); + std::lock_guard lock(admitted_algos_mutex); + return std::binary_search(admitted_algos.begin(), admitted_algos.end(), v); +} + +std::vector Pool::refreshAdmittedAlgos() +{ + /// A direct GET+decode of `_pool_meta`, not a re-run of `createOrValidate`'s admission logic -- + /// this Pool's OWN algo is already admitted, so all this + /// needs is the CURRENT authoritative `algos_used`, unioned into the monotone cache. + const auto existing = pool_backend->get(pool_layout.poolMetaKey()); + + std::lock_guard lock(admitted_algos_mutex); + if (existing) + { + const PoolMeta fresh = decodePoolMeta(existing->bytes); + for (uint8_t v : fresh.algos_used) + if (!std::binary_search(admitted_algos.begin(), admitted_algos.end(), v)) + { + admitted_algos.push_back(v); + std::sort(admitted_algos.begin(), admitted_algos.end()); + } + } + return admitted_algos; +} + +bool Pool::dedupCacheContains(const BlobRef & ref) const +{ + /// raw lookup counters on the presence cache itself, disabled + /// (nullptr `dedup_cache`) means neither counter moves -- the short-circuit below never reaches the + /// probe. `PartWriteTxn::putBlob` calls this seam up to twice on a genuine hit (once to pick the + /// HEAD-first branch, once more just to attribute `CASBlobBodyPutAvoided` to the cache -- see + /// CasPartWriteTxn.cpp), so `CASDeduplicationCacheHits` counts LOOKUPS, not distinct blobs or putBlob calls. A hit + /// does not itself skip the HEAD that follows in putBlob's HEAD-first branch -- it steers the call + /// onto that cheap branch instead of an unconditional body stream; the body PUT is what a hit + /// actually avoids. + if (!dedup_cache) + return false; + if (dedup_cache->contains(ref)) + { + ProfileEvents::increment(ProfileEvents::CASDeduplicationCacheHits); + return true; + } + ProfileEvents::increment(ProfileEvents::CASDeduplicationCacheMisses); + return false; +} + +void Pool::dedupCacheAdd(const BlobRef & ref) +{ + if (dedup_cache) + dedup_cache->set(ref, std::make_shared()); +} + +/// ==== mount-runtime delegates ==== The mount lease keeper, the local write +/// fence, the per-server build watermark, the live-incarnation epoch, and the self-remount recovery +/// thread live in the `mount_runtime` member (Pool/CasMountRuntime.h); Pool keeps these thin public +/// forwarders so the wiring, PartWriteTxn, Gc, the ref-ledger callbacks, and every test call site are unchanged. +uint64_t Pool::bootMs() +{ + return CasMountRuntime::bootMs(); +} + +uint64_t Pool::bootMsNow() const +{ + return mount_runtime.bootMsNow(); +} + +bool Pool::mayMutate() const +{ + return mount_runtime.mayMutate(); +} + +void Pool::tripMountLost() +{ + mount_runtime.tripMountLost(); +} + +bool Pool::refAppendFenceOk() const +{ + return mount_runtime.refAppendFenceOk(); +} + +void Pool::setMountDeadline(uint64_t deadline_boot_ms) +{ + mount_runtime.setMountDeadline(deadline_boot_ms); +} + +void Pool::armMountFence(UInt128 server_uuid, uint64_t writer_epoch, uint64_t deadline_boot_ms) +{ + mount_runtime.armMountFence(server_uuid, writer_epoch, deadline_boot_ms); +} + +String Pool::lifecycleReasonDetail(PoolLifecycle lc) const +{ + /// The [D5] per-reason detail (spec §1) — named once here so the typed error and the introspection + /// snapshot always agree. No `content-addressed pool '' ` prefix (callers add it if they want one). + switch (lc) + { + case PoolLifecycle::Live: + case PoolLifecycle::TransientNotLive: + return {}; + case PoolLifecycle::IdentityLost: + return "identity lost — the pool sentinels are absent; access fails loud. Recover by restart or " + "SYSTEM CAS FORGET (a matching-sentinel restore does not auto-revive it)."; + case PoolLifecycle::VanishedReplaced: + return "data root replaced by a foreign pool (pool_id mismatch) — our generation is gone; " + "restart re-registers the name."; + case PoolLifecycle::VanishedForgotten: + { + /// The forgotten detail carries the operator's decommission TIMESTAMP, threaded through + /// `enterVanished`'s reason by `forgetDisk` (`vanishedReason()`). A forced-for-test + /// `VanishedForgotten` (no real FORGET ran) has no stored reason, so fall back to the static + /// [D5] text — it still names the sub-state and keeps "erasure was NOT verified". + const String & reason = mount_runtime.vanishedReason(); + if (!reason.empty()) + return reason; + return "decommissioned by SYSTEM CAS FORGET — erasure was NOT verified; if this " + "was a mistake the data may be intact (restart re-registers the name)."; + } + } + return {}; /// unreachable — every `PoolLifecycle` value is handled above (`-Wswitch` enforces it). +} + +void Pool::throwIfLifecycleTerminal() const +{ + /// The typed error carries the sub-state in its message so a wrong diagnosis is impossible from the + /// first error line (spec §1 [D5]). `Live`/`TransientNotLive` proceed here — the transient class is + /// still gated only by the write fence in this task (the full six-class gate is Task 8). + const PoolLifecycle lc = mount_runtime.lifecycle(); + if (lc == PoolLifecycle::Live || lc == PoolLifecycle::TransientNotLive) + return; + throw Exception(ErrorCodes::INVALID_STATE, + "content-addressed pool '{}' {}", config.server_root_id, lifecycleReasonDetail(lc)); +} + +Pool::LifecycleSnapshot Pool::lifecycleSnapshot() const +{ + /// Non-gated, I/O-free (spec §7). Read the lifecycle ONCE (acquire), then the detail/`since` it + /// implies. `lifecycleReasonDetail` reads `vanishedReason()` only for a terminal state we have already + /// acquire-observed here, and `since` was release-stored before the same transition — so this coherent + /// triple never mixes a terminal state with a pre-terminal detail/timestamp. + LifecycleSnapshot snap; + snap.lifecycle = mount_runtime.lifecycle(); + snap.detail = lifecycleReasonDetail(snap.lifecycle); + snap.since = mount_runtime.lifecycleSinceWallS(); + return snap; +} + +PoolPtr Pool::open(BackendPtr backend, PoolConfig config) +{ + /// Wrap the pool backend once, transparently, so EVERY CA S3 op — probe, pool-meta, + /// writer, GC, watermark — flows through the per-namespace/op ProfileEvents chokepoint. The + /// decorator only delegates and counts; it changes no behavior (read-only opens stay write-free). + backend = std::make_shared(std::move(backend)); + + /// FAIL-CLOSED: the capability probe throws NOT_IMPLEMENTED on any failed check, and + /// PoolMeta::createOrValidate is pool-authoritative — the config constants apply only at creation. + Layout layout(config.pool_prefix); + bool initialize_empty_catalog = false; + /// The probe writes and deletes throwaway keys to verify conditional-op enforcement. A read-only + /// open must never mutate the pool it inspects; fsck only reads, so skip it. (Pool meta below is + /// read-only when the pool already exists; a missing pool meta on a read-only backend fails closed.) + if (!config.read_only) + { + /// (0) [C4][D2] Zero-write residual check FIRST — before ANY probe write. `pool_prefix` is + /// EXCLUSIVELY CAS-owned; `createOrValidate` below may mint a fresh `_pool_meta` only over a + /// genuinely empty prefix. The MUTATING capability battery must run AFTER this proof (it writes + /// `_probe/` debris, which would itself make the prefix look non-empty), and the emptiness + /// classification IGNORES structurally-valid `_probe/` debris so a normal restart after a + /// crash-mid-battery still bootstraps cleanly. This closes the "restart poisons a + /// partially-erased pool" hole: a missing `_pool_meta` over residual data now fails startup loud + /// with zero writes, instead of minting a fresh identity on top of the old objects. + switch (probePoolBootstrapResidual(*backend, layout)) + { + case BootstrapResidual::PoolMetaPresent: + break; /// authoritative existing pool; its catalog is mandatory below. + case BootstrapResidual::EmptyOrProbeOnly: + case BootstrapResidual::CanonicalEmptyCatalogOnly: + initialize_empty_catalog = true; + break; /// proven-new or canonical catalog-only pre-meta bootstrap state. + case BootstrapResidual::ResidualWithoutMeta: + { + /// RECREATION QUIESCE. Reaching here means the prefix holds objects but no authoritative + /// `_pool_meta` -- the shape a pool recreation leaves behind. Before telling the operator + /// anything about residual data, ask the one question whose answer changes the remedy: is + /// a writer still entitled to this prefix? Refusing an OLD-FORMAT open fences nothing -- + /// a server that mounted before the erase is still running, still holds its lease, and + /// still has queued writes; if the operator answers the residual message below by + /// clearing the prefix and recreating, that writer's next flush lands its old-format + /// transactions inside the NEW pool. So a non-terminal slot fails closed with its own + /// message: stop the writer first. + /// + /// This gate is the PRIMARY defence, not a nicety, because the mount fence cannot be + /// relied on to catch a straggler afterwards. Clearing the prefix also destroys the + /// durable writer-epoch counter, so a recreation by the SAME server uuid is handed the + /// very `(uuid, epoch)` the survivor still holds -- and the two are then indistinguishable + /// to the lease protocol, which reads the survivor's renewal as its own keeper adopting a + /// refreshed body. The fence only bites when the recreating mount is DISTINGUISHABLE (a + /// different server uuid, or a surviving epoch counter): then the survivor's next renewal + /// finds a slot it cannot hold and its local fence latches shut. + /// + /// What catches the ambiguous case is INV-1, after the fact: the straggler's append lands + /// at `{E, 1}`-relative ids in a table the recreated pool sees as empty, and the first + /// recovery of that namespace refuses the stream as non-contiguous rather than absorbing + /// it. That is a loud post-mortem, not prevention -- which is why the refusal here, BEFORE + /// anything is cleared, is the one that matters. + /// + /// Only on this arm: `EmptyOrProbeOnly` proves there is no slot object to read (a mount + /// lease is itself residual), and `PoolMetaPresent` is not a recreation at all -- neither + /// pays for the scan. + const std::vector held = probeNonTerminalMountSlots(*backend, layout); + if (!held.empty()) + { + String detail; + for (const NonTerminalMountSlot & slot : held) + detail += fmt::format("\n server root '{}': {}", slot.server_root_id, slot.detail); + throw Exception(ErrorCodes::INVALID_STATE, + "content-addressed pool '{}' (prefix '{}'): missing _pool_meta, but {} mount " + "lease(s) under this prefix are still held — refusing to recreate the pool while " + "a writer may still be using it. Stop (or decommission) the holder(s) so their " + "mount slots become terminal, then retry; do NOT clear the prefix first, which " + "would leave the surviving writer appending into the new pool.{}", + config.server_root_id, config.pool_prefix, held.size(), detail); + } + throw Exception(ErrorCodes::INVALID_STATE, + "content-addressed pool '{}' (prefix '{}'): missing _pool_meta over a non-empty pool " + "prefix — refusing to bootstrap over residual data; recreate the pool or restore " + "_pool_meta. The pool prefix is exclusively CAS-owned.", + config.server_root_id, config.pool_prefix); + } + case BootstrapResidual::Indeterminate: + throw Exception(ErrorCodes::INVALID_STATE, + "content-addressed pool '{}' (prefix '{}'): could not authoritatively list the pool " + "prefix to prove it is safe to bootstrap — refusing to create _pool_meta while " + "residual data cannot be ruled out (fail-closed).", + config.server_root_id, config.pool_prefix); + } + + if (!config.skip_access_check) + { + /// Give each mount a PER-MOUNT UNIQUE probe key prefix so two servers mounting the SAME + /// shared pool concurrently never collide on the (formerly fixed) `/_probe/token` / + /// `/_probe/cas` keys. Without this, the loser of the `putIfAbsent` race aborts startup + /// with PreconditionFailed (and the winner's cleanup delete can cascade into the loser). With a + /// fresh random 128-bit id per `Pool::open`, each mounter validates conditional-op support + /// independently. A crashed mount leaves harmless `_probe//...` debris under the `_probe/` + /// namespace only (never the content planes) — acceptable. + const UInt128 probe_uid = (static_cast(thread_local_rng()) << 64) | thread_local_rng(); + runCapabilityProbe(*backend, config.pool_prefix + "/_probe/" + u128ToHex(probe_uid)); + } + else + { + /// skip_access_check: skip the access-check-class probe I/O (store preconditions + the + /// `_probe/` round trip, both folded into runCapabilityProbe above) but NOT the + /// single-attempt conditional-write gate — see `PoolConfig::skip_access_check`. Run it + /// directly so a Native-mode backend with no working single-attempt client still fails + /// closed at open instead of silently corrupting CAS state under blind retries later. + /// The skipped store-precondition check also covers GCS bucket-versioning/delete-marker + /// detection — that risk is purely environmental (slower GC reclaim, not data loss) and + /// gets re-checked the next time this pool is opened without skip_access_check. + backend->checkConditionalWriteSingleAttemptSupport(); + } + } + /// The catalog is mandatory for every minted pool. Make it durable before `_pool_meta`: otherwise + /// an acknowledgement-loss or definite catalog-write failure could strand an authoritative meta + /// that makes every later open refuse the absent catalog. The catalog-only residual proof above is + /// the narrowly-defined retry path when this opener (or a concurrent opener) completed this step + /// but did not reach the pool-meta create. + if (initialize_empty_catalog) + CasRefCatalog::initializeEmptyForNewPool(*backend, layout); + /// `allow_mint` = writable open only: a writable `Pool::open` reaches here having just passed the + /// zero-write residual proof above, so minting a missing `_pool_meta` is safe. A read-only/observe + /// open never ran that proof (and there is no truly-read-only backend — `openPoolView` opens the same + /// writable object storage and only sets `read_only`), so it must NEVER mint: an absent meta fails + /// closed instead (spec §2 [C4][D2]). + PoolMeta meta = PoolMeta::createOrValidate( + *backend, layout, config.blob_header_len, config.gc_shards, config.blob_hash_algo, config.blob_hash_allow_new, + /*allow_mint=*/!config.read_only); + config.gc_shards = meta.gc_shards; + const BlobHashAlgo write_algo = config.blob_hash_algo; /// `config` is moved-from just below + + /// Private ctor: make_shared cannot reach it. + PoolPtr store(new Pool(std::move(backend), std::move(config), std::move(meta))); + store->setEventSink(std::move(store->config.event_sink)); + + /// Register-before-first-write, belt-and-braces: `createOrValidate` above already + /// admitted/validated the write algo, so the freshly-seeded cache must already contain it -- a + /// violation here would mean a build/write could reach this Pool naming an algo that was never + /// durably admitted (the invariant this whole design rests on). + chassert(store->isAlgoAdmitted(write_algo)); + + /// Per-server watermark: mint the random NONZERO `process_epoch` + /// once per Pool (GC checks it for equality only -- a different epoch == a dead incarnation). The + /// masking/redraw detail lives in `CasMountRuntime::mintRandomProcessEpoch`. + store->mount_runtime.mintRandomProcessEpoch(); + + /// W-ANCHOR: the per-server watermark must be durable BEFORE any object PUT. A read-only open + /// must never mutate the pool (the probe is skipped above for the same reason), so the watermark + /// — which rides inside the `gc/server-roots//mount` lease object — is only + /// constructed and anchored on a writable open. + if (!store->config.read_only) + mountWritable(store, store->config.server_id, MountClaimPolicy::WaitForExpiry); + + return store; +} + +void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy policy) +{ + /// === Mount-safety startup protocol === + /// STRICT ORDER: validate id → claim owner (identity) → allocate durable writer_epoch → claim + /// the mount lease (liveness) + arm the local write fence → anchor the watermark. owner / epoch + /// / mount / watermark are BOOTSTRAP-CONTROL writes: they establish the very right to write and + /// run BEFORE the write fence gates ordinary data/ref/manifest mutations. Fail closed throughout. + /// Shared by `open` (`policy = WaitForExpiry`, `our_uuid = config.server_id`) and + /// `openForDecommission` (`policy = NoWait`, `our_uuid` = the impersonated victim owner uuid; + /// -- the two differ only in WHO they mount + /// as and whether a non-`Claimed`/`FencedSelf` mount result gets `open`'s bounded observation wait + /// or an immediate refusal. + const String & srid = store->config.server_root_id; + + /// 1. The server_root_id is a clean relative path (mirrors the config-read validation; cheap to + /// re-check here so a Pool opened directly in tests is held to the same contract). + validateServerRootId(srid); + + const ObserveRefCatalog observe_catalog = [s = store.get()]() + { + CasRefCatalog::Snapshot snapshot = CasRefCatalog::read(*s->pool_backend, s->pool_layout); + snapshot.life_index.throwIfAmbiguous("CAS server-root mount safety"); + return snapshot.catalog; + }; + /// Existing matching owner and epoch objects take fast paths that do not need an emptiness + /// observation. Validate the mandatory authority object unconditionally before either fast path + /// can reach a slot mutation; the callback remains available for fresh conflict rechecks below. + (void)observe_catalog(); + + /// 2. Owner anchor — IDENTITY (clock-free). A foreign uuid fails closed; an absent owner over a + /// non-empty subtree is CORRUPTED_DATA; a fresh empty root is claimed. + claimOwnerOrThrow(*store->pool_backend, store->pool_layout, srid, our_uuid, observe_catalog); + + /// Wall-clock `now_ms`, hoisted above the writer_epoch allocation below: the absent-epoch + /// branch's `DecommissionRecovery` policy needs it to judge a surviving mount's liveness before + /// the mount-lease claim (step 4) gets its own use of the same clock. + const auto now_ms = []() -> uint64_t + { + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + }; + + /// 3. Durable-monotone writer_epoch — CAS-bump the sticky `epoch` object. THE BRIDGE: this + /// durable value REPLACES the random `process_epoch` for identity, so the watermark + every + /// manifest ref carries it (the random mint above stays for the read-only + /// path, which never reaches here). The epoch-aware sweep reads this value. + /// Mutable: a GC fence of our fresh lease during open (expiry mid-open racing a GC round) is + /// recoverable — a fence costs an epoch, so the fence-recovery loop below re-allocates a fresh + /// writer_epoch and re-claims (the TLA+-checked `NoPermanentWedge` invariant). + /// `epoch_policy`: `openForDecommission`'s `NoWait` gates the absent-epoch branch's mount-probe + /// on a TERMINAL (not live) surviving mount instead of authoritative absence (Phase C) — passed + /// uniformly to every call below, including the fence-recovery re-allocations, where it is inert + /// because those run with the epoch object already present. + const EpochMintPolicy epoch_policy = (policy == MountClaimPolicy::NoWait) + ? EpochMintPolicy::DecommissionRecovery + : EpochMintPolicy::NormalMount; + uint64_t writer_epoch = allocateWriterEpoch( + *store->pool_backend, store->pool_layout, srid, epoch_policy, now_ms(), observe_catalog); + store->mount_runtime.setProcessEpoch(writer_epoch, std::memory_order_relaxed); + + /// 4. Mount lease — LIVENESS. Decide over the current mount object using the wall-clock `now_ms` + /// hoisted above. + const uint64_t ttl_ms = static_cast(store->config.mount_lease_ttl_ms.count()); + + /// CAS request budget: a + /// writable mount refuses to open with a budget that could let a controlled attempt outlive the + /// mount lease it is fenced under. Throws BAD_ARGUMENTS and aborts open on an inconsistent + /// budget; logs the effective values once on success. The controller gates Pool's ref-mutation + /// paths; this validates the config invariant up front, before any attempt runs. + validateCasRequestBudget(store->config.cas_request_budget, ttl_ms, + static_cast(store->config.mount_renew_period.count())); + + /// Poll twice per renew period so a live holder's renewal is always observed within the + /// observation window. Derived from existing config — no new knob. + const uint64_t poll_interval_ms = std::max( + 1, static_cast(store->config.mount_renew_period.count()) / 2); + /// routes through `mount_runtime.waitSleep` (which itself routes through + /// `config.wait_sleep_fn` when a test injected one) rather than a bare `sleep_for` directly, so + /// a test intercepting `wait_sleep_fn` observes every wait `open` can block on -- and since the + /// post-reclaim materialization grace was retired, this observation poll is the only one left. + const auto sleep_ms = [s = store.get()](uint64_t ms) { s->mount_runtime.waitSleep(ms); }; + /// Operator-visible log the moment startup decides to watch a stale-looking self-mount (the + /// disk-open path blocks up to ~threshold_ms here, so a silent block would be confusing). May + /// fire more than once per open: the observation restarts (and re-logs) every time the watched + /// lease's write-token changes before the full threshold elapses. + const auto on_wait_start = [&srid](const MountLease & held, uint64_t threshold_ms) + { + LOG_INFO(getLogger("CasPool"), + "CAS mount '{}': a stale-looking mount lease is held by uuid={} epoch={} pid={} " + "hostname={} (expires_at_ms={}); observing its write-token for up to ~{} ms before " + "reclaiming. If a second server is genuinely live, its renewals will keep restarting " + "the observation and startup will eventually abort as a live double-start.", + srid, u128ToHex(held.server_uuid), held.writer_epoch, held.pid, held.hostname, + held.expires_at_ms, threshold_ms); + }; + + /// Mount-slot writer audit (the "foreign writer" instrument): route every mount-slot + /// write/conflict event through the Pool's own sink. The factory installs the configured sink + /// before this mount protocol starts, including before any renewal thread can emit. + /// `s` outlives the lambda: it is captured by raw pointer into the keeper, a member of + /// `Pool` destroyed before the `Pool` itself. + const auto emit_mount_event = [s = store.get()](CasEvent e) { s->emitEvent(std::move(e)); }; + + Pool * raw = store.get(); + + /// Crash-recovery (`WaitForExpiry` only): a hard-killed prior incarnation leaves a stale, + /// unreleased mount lease. Rather than aborting, OBSERVE that lease's write-token (never its + /// stamped `expires_at_ms` against our wall clock) until it has held stable for the full + /// rate-bound threshold, then reclaim it; a genuinely live second server keeps renewing the + /// token and is (after bounded restarts) reported as LiveDoubleStart. The reclaim is + /// token-guarded (see `claimMountAwaitingExpiry`), so a live twin is never stolen from. + /// `NoWait` skips this observation entirely (see the policy branch below). + /// + /// Fence-recovery loop: if the GC fences our own fresh lease while we are opening + /// (the lease expired mid-open — e.g. a slow first beat — and a GC round fenced it), that is a + /// RECOVERABLE state, not a wedge: a fence costs an epoch, so allocate a fresh writer_epoch and + /// re-claim. Bounded so a pathological fence storm still fails closed. The fence can surface two + /// ways: `claimMount` observes an already-fenced own slot (`FencedSelf`), or the keeper's adopt + /// races a fence between its GET and CAS (`MountFencedException` from `start()`). + /// which certificate of death (if any) justified the reclaim FINALLY adopted below + /// (the last iteration's `claim` before `break` -- `claim` itself is loop-scoped). Read after the + /// loop to classify (and log) an unclean reclaim. + MountPriorState claimed_prior = MountPriorState::None; + /// The pre-I/O boot-clock instant of the claim attempt FINALLY adopted below -- survives the + /// `break` so the arm below can detect a claim that consumed the lease TTL and re-anchor before + /// arming (rev.4 Phase B, round-3 finding 2). + uint64_t claim_anchor_boot_ms = 0; + constexpr int max_fence_recoveries = 3; + for (int fence_recovery = 0; ; ++fence_recovery) + { + MountClaimResult claim; + if (policy == MountClaimPolicy::WaitForExpiry) + { + claim = claimMountAwaitingExpiry( + *store->pool_backend, store->pool_layout, srid, our_uuid, writer_epoch, + [&now_ms]() { return now_ms(); }, [raw] { return raw->bootMsNow(); }, + ttl_ms, poll_interval_ms, sleep_ms, on_wait_start, emit_mount_event); + } + else + { + /// NoWait (decommission gate): a single unobserved attempt -- no bounded wait-and-retry + /// for a stale-looking lease to lapse. Anything but Claimed/FencedSelf below is refused + /// immediately. + claim = claimMount(*store->pool_backend, store->pool_layout, srid, our_uuid, writer_epoch, + now_ms(), ttl_ms, /*proven_dead_token=*/{}, emit_mount_event); + } + if (claim.kind == MountClaimResult::FencedSelf) + { + if (fence_recovery >= max_fence_recoveries) + throw Exception(ErrorCodes::ABORTED, + "CAS mount '{}': our own mount lease was GC-fenced repeatedly during open " + "({} recoveries exhausted) — a fresh writer_epoch kept being fenced before we " + "could adopt it. This should not persist; investigate GC fence-out timing.", + srid, max_fence_recoveries); + writer_epoch = allocateWriterEpoch( + *store->pool_backend, store->pool_layout, srid, epoch_policy, now_ms(), observe_catalog); + store->mount_runtime.setProcessEpoch(writer_epoch, std::memory_order_relaxed); + continue; + } + if (claim.kind != MountClaimResult::Claimed) + { + if (policy == MountClaimPolicy::NoWait) + /// No FORCE variant, no wait-and-observe: the decommission gate treats any live-looking + /// or foreign-owner lease as an immediate refusal. + throw Exception(ErrorCodes::ABORTED, + "CAS decommission '{}': pool member is alive or contended — mount lease held by " + "uuid={} epoch={} pid={} hostname={} (expires_at_ms={}). Refusing (no FORCE variant " + "exists; stop the server or wait for its lease to lapse).", + srid, u128ToHex(claim.body.server_uuid), claim.body.writer_epoch, claim.body.pid, + claim.body.hostname, claim.body.expires_at_ms); + /// LiveDoubleStart (waited out the bound → a live twin) or ForeignOwner → fail closed + /// with the actionable, multi-line startup error. + throw Exception(ErrorCodes::ABORTED, "{}", mountDoubleStartMessage(srid, claim.body)); + } + claimed_prior = claim.prior; + + /// The mount object now holds OUR live (uuid, epoch) body. `installKeeper` constructs the keeper + /// -- which ADOPTS that very (uuid, epoch) slot rather than self-tripping the double-start guard -- + /// AND wires its `minActive` build-watermark reader, its event sink, and the fence-coupling + /// callbacks (renew-ok refreshes the monotonic deadline; on-lost latches the fence + arms a + /// self-remount), all captured on `mount_runtime` (see `CasMountRuntime::installKeeper`). + /// `keeperStart` is separate so this claim orchestration can catch `MountFencedException` and + /// retry with a fresh epoch. + store->mount_runtime.installKeeper(our_uuid, writer_epoch, now_ms); + claim_anchor_boot_ms = store->bootMsNow(); /// pre-I/O anchor of the claim attempt + try + { + store->mount_runtime.keeperStart(); + } + catch (const MountFencedException &) + { + /// The GC fenced our fresh lease between the keeper's adopt GET and CAS. Recoverable: + /// drop this keeper, take a fresh epoch, and re-claim. + if (fence_recovery >= max_fence_recoveries) + throw; + store->mount_runtime.keeperReset(); + writer_epoch = allocateWriterEpoch( + *store->pool_backend, store->pool_layout, srid, epoch_policy, now_ms(), observe_catalog); + store->mount_runtime.setProcessEpoch(writer_epoch, std::memory_order_relaxed); + continue; + } + break; + } + + /// A reclaim over a predecessor whose death was NOT proven clean may still have a conditional PUT + /// from that predecessor in flight -- `Fenced` and `UncleanObserved` are exactly the two + /// `MountPriorState`s with no such proof (`Clean`, drained farewell, and `None`, a fresh mount / + /// same-epoch refresh with nothing to hand over, are the proven ones). + /// An EXHAUSTIVE switch, not a positive allowlist -- a future `MountPriorState` + /// enumerator with no proof of clean death must fail the BUILD (a missing `-Wswitch` case), never + /// silently fall through to "clean". + /// + /// THIS NO LONGER WAITS. The `materialization_grace_ms` (`T_mat`) sleep that used to sit here bought + /// one thing: time for that straggler to land (or exhaust its retries) BEFORE this incarnation began + /// trusting its recovery LISTINGS. Recovery does not trust listings any more. It walks each ref + /// stream arithmetically from `_ckpt` and closes every dead epoch with an in-band `EpochSeal` at + /// `{E, T+1}`, written as a conditional create -- so the straggler's own conditional create loses to + /// an occupied slot no matter when it arrives, and a wait can only make startup slower, never safer. + /// What survives is the CLASSIFICATION and saying it out loud: an unclean predecessor is worth an + /// operator-visible line, and the exhaustive switch is worth keeping as the build-time guard. + bool unclean_reclaim = false; + switch (claimed_prior) + { + case MountPriorState::None: + case MountPriorState::Clean: + unclean_reclaim = false; + break; + case MountPriorState::Fenced: + case MountPriorState::UncleanObserved: + unclean_reclaim = true; + break; + } + if (unclean_reclaim) + { + LOG_INFO(getLogger("CasPool"), + "Content-addressed mount {} follows a predecessor whose death was not proven clean " + "(writer_epoch {}). Opening without a grace period: a still-in-flight conditional PUT from " + "that predecessor is fenced by the recovery seal, whenever it arrives.", srid, writer_epoch); + } + + /// Arm the local write fence: cache (uuid, epoch) and set the boottime deadline at the claim + /// attempt's anchor + ttl (NOT `bootMsNow()` here -- arming from a post-I/O instant would authorize + /// mutations under a deadline the durable lease never actually backs). From here ordinary ref + /// mutations (appendRefOps) are fence-gated via mayMutate. + const uint64_t ttl_ms_u = static_cast(store->config.mount_lease_ttl_ms.count()); + if (store->bootMsNow() >= claim_anchor_boot_ms + ttl_ms_u) + { + /// The claim path outlived the lease TTL: its anchor can no longer authorize an armed fence (a + /// successor may have legally started reclaiming). Re-anchor with ONE fresh conditional lease + /// write -- it fails closed (Phase A classification) if anything took the slot meanwhile -- and + /// arm from the new attempt's anchor (rev.4 Phase B, round-3 finding 2). + /// + /// The unbounded operator-configured wait this guard was written for (`T_mat`) is gone, so + /// reaching it now means the CLAIM ITSELF -- `keeperStart`'s GET+CAS -- outran the whole lease + /// TTL, which `validateCasRequestBudget` already refuses to configure. It stays because a stalled + /// socket can still outlive a budget, and its recovery is one conditional write that fails closed; + /// it is LOUD rather than fatal because a slow open under a healthy protocol is not a reason to + /// refuse to start. + LOG_WARNING(getLogger("CasPool"), + "Content-addressed mount {}: the mount claim consumed the lease TTL ({} ms) before the write " + "fence could be armed; re-writing the lease first", srid, ttl_ms_u); + claim_anchor_boot_ms = store->bootMsNow(); + store->mount_runtime.keeperRenewOnce(); + } + store->armMountFence(our_uuid, writer_epoch, claim_anchor_boot_ms + ttl_ms_u); + /// Gate the background renewer with `background_watermark`: it runs only in production + /// (`background_watermark` = context != nullptr && !read_only), never in unit tests — which + /// drive renewOnce (or renewWatermarkOnce) explicitly and rely on the armed sub-TTL deadline, + /// never on the loop. The keeper itself is still started above (it must claim/adopt the mount + + /// arm the fence on every writable open); only the renewal thread is conditional. The merged + /// heartbeat renews at `mount_renew_period` — one beat now renews the lease and the floor. + if (store->config.background_watermark) + store->mount_runtime.keeperStartBackground(store->config.mount_renew_period); + + store->mount_runtime.setLiveWriterEpoch(writer_epoch); +} + +PoolPtr Pool::openForDecommission(BackendPtr backend, PoolConfig config, const String & victim_srid) +{ + backend = std::make_shared(std::move(backend)); + validateServerRootId(victim_srid); + + config.server_root_id = victim_srid; + config.read_only = false; + config.skip_access_check = true; /// the pool exists (the calling disk validated it); no probe writes + /// The admin claim must be RENEWED like any writable mount: the host disk may be observe-only + /// (background_watermark=false), but an unrenewed claim (TTL ~30s) aborts any long drain midway. + config.background_watermark = true; + + Layout layout(config.pool_prefix); + + /// Impersonate the victim: decommission acts as "the next incarnation of that server". The claim + /// below is then EXACTLY the crash-recovery reclaim semantics (`MountClaimPolicy::NoWait`): a + /// fenced/terminated/clean-farewell lease reclaims; a live lease refuses immediately (no bounded + /// observation wait -- see `mountWritable`). Owner anchor absent + mount absent = nothing to + /// decommission. + std::optional victim_uuid = readOwnerUuid(*backend, layout, victim_srid); + if (!victim_uuid) + { + if (const auto mount = backend->get(layout.mountKey(victim_srid))) + victim_uuid = decodeMountLease(mount->bytes).server_uuid; /// partial hand-cleanup: adopt from the lease + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "CAS decommission '{}': unknown pool member (no owner anchor and no mount lease). " + "Nothing to decommission; if victim objects linger without a slot, run cas-fsck.", + victim_srid); + } + config.server_id = *victim_uuid; + + backend->checkConditionalWriteSingleAttemptSupport(); + /// Decommission operates on an EXISTING pool member (an owner anchor / mount lease was just found), so + /// `_pool_meta` must already be present. It never bootstraps: `allow_mint=false` so an absent meta + /// (a partially-erased pool whose owner anchor survives) fails closed with INVALID_STATE rather than + /// minting a fresh identity here (spec §2 [C4][D2]). + PoolMeta meta = PoolMeta::createOrValidate( + *backend, layout, config.blob_header_len, config.gc_shards, config.blob_hash_algo, config.blob_hash_allow_new, + /*allow_mint=*/false); + config.gc_shards = meta.gc_shards; + const BlobHashAlgo write_algo = config.blob_hash_algo; /// `config` is moved-from just below + + /// Private ctor: make_shared cannot reach it. + PoolPtr store(new Pool(std::move(backend), std::move(config), std::move(meta))); + store->setEventSink(std::move(store->config.event_sink)); + + /// Register-before-first-write, belt-and-braces: same invariant `open` asserts. + chassert(store->isAlgoAdmitted(write_algo)); + + /// No random `process_epoch` mint here: `open` pays that prologue because its read-only path + /// never reaches `mountWritable` and so needs SOME nonzero epoch, but this factory is + /// writer-only -- `mountWritable` below unconditionally overwrites `process_epoch` with the + /// freshly allocated durable `writer_epoch` before anything could observe the zero-initialized + /// default. + mountWritable(store, *victim_uuid, MountClaimPolicy::NoWait); + return store; +} + +Pool::~Pool() +{ + /// Teardown order is load-bearing and unchanged from the pre-3.5 inline sequence (only the + /// mount/remount mechanics were relocated into `mount_runtime`): + /// + /// 1. Stop + join the self-remount recovery thread FIRST (it may otherwise re-create the keeper + /// below us). `stopRemountThread` latches `remount_shutting_down` under the thread mutex before + /// the join, so a keeper on_lost firing during teardown can never re-arm the thread after we join. + mount_runtime.stopRemountThread(); + + /// 2. The farewell marker the keeper's `stop()` writes is a + /// certificate that no in-flight ref-log conditional PUT from this incarnation can land after it -- + /// a successor treats it as proof of a clean death (`MountPriorState::Clean`, no observation wait + /// needed). Writing it without an actual drain would be a protocol-safety bug: an uncertain PUT this + /// incarnation is still resolving could land AFTER the successor already reclaimed and started + /// mutating. `drainRefLanesForShutdown` is the drain; bounded by one attempt's worth of budget plus + /// the lease safety margin -- long enough for an in-flight attempt to resolve, never unbounded. It + /// stays on `Pool` (mediating the mount↔ledger coupling), sequenced between the two mount-runtime + /// teardown steps exactly as before. + const bool ref_lanes_drained = ref_ledger.drainRefLanesForShutdown( + config.cas_request_budget.attempt_timeout_ms + config.cas_request_budget.lease_safety_margin_ms); + const bool drained = ref_lanes_drained && !writerCleanupDutiesPending(); + + /// 3. Retire the merged heartbeat: `finishTeardown` runs the keeper's terminal op on a clean drain + /// (stamping the lease already-expired + folding in the watermark farewell so a SAME-server reopen + /// reclaims immediately) or the fail-closed no-terminal-op on an unresolved PUT, then does the + /// belt-and-suspenders remount-thread re-join. See `CasMountRuntime::finishTeardown`. + mount_runtime.finishTeardown(drained); +} + +void Pool::forgetDisk(const std::function & stop_and_join_gc, const String & reason) +{ + /// Hazard C6: FORGET joins the self-remount thread (and, via `stop_and_join_gc`, the GC threads), so it + /// MUST run on the admin/query thread — never a pool thread, whose join of itself would deadlock. The + /// guard is a programming-error assertion (a self-join hangs; it never corrupts), so a chassert is the + /// right severity, not a release fail-close. + const ThreadName tn = getThreadName(); + chassert(tn != ThreadName::CAS_REMOUNT && tn != ThreadName::CAS_GC_SCHEDULER + && tn != ThreadName::CAS_GC_HEARTBEAT + && "SYSTEM CAS FORGET must not run on a CAS pool thread (self-join deadlock)"); + + /// Idempotent: an already-terminal `Vanished` pool (a second FORGET, or a pool that naturally vanished + /// as replaced) is already the terminal truth — nothing to force, and re-running the teardown + /// would double-retire the keeper. `IdentityLost`/`TransientNotLive`/`Live` all proceed (FORGET is + /// their escape hatch). Reading `isVanished()` here without the lock is safe: only a terminal transition + /// sets it, terminal states are absorbing, and a natural transition that wins concurrently below merely + /// makes our own `enterVanished` a no-op (first terminal transition wins). + if (mount_runtime.isVanished()) + return; + + /// (1) Publish the terminal-intent latch FIRST (spec §5). The keeper callback stops arming remounts and + /// the remount loop bails at its next step boundary, so every join below is bounded to one step + one + /// backend timeout. + mount_runtime.publishVanishedIntent(); + + /// (2) Trip the local fence — the deliberate decommission act (allowed on a live disk). No durable- + /// effect write admits past this point (the fence-generation gate), and a live pool moves to + /// `TransientNotLive`, so store-class access already fails loud during the teardown window below. + mount_runtime.tripMountLost(); + + /// (3+4) Stop the GC scheduler (clears its leadership and JOINS its worker + heartbeat threads) BEFORE + /// the Pool-side teardown, so no round writes `gc/state` under a disk we are decommissioning. Injected + /// because the scheduler is owned above the Pool (a no-op in unit / read-only / clickhouse-disks + /// contexts that run none). Runs OUTSIDE `remount_mutex` (spec §3 join discipline). + if (stop_and_join_gc) + stop_and_join_gc(); + + /// (5a) Stop + join the self-remount thread. `stopRemountThread` latches the shutdown gate under the + /// thread mutex before joining, and the thread is already bailing on the intent latch (step 1) — so the + /// join is bounded and a keeper callback racing teardown can never re-arm it. Outside `remount_mutex`. + mount_runtime.stopRemountThread(); + + /// A remount attempt already IN FLIGHT when step 1 published the intent completes its current step + /// before the loop bails (the "one step + one backend timeout" bound of §5), and a successful reclaim in + /// that window re-arms the local fence (`lost = false`). Now that the remount thread is JOINED and can + /// never run again, re-latch the fence so the terminal `mayMutate() == false` holds regardless of any + /// such raced reclaim. Idempotent; the durable mount lease the reclaim wrote is retired by the + /// `finishTeardown` below (it operates on whatever keeper is current — the reclaimed one). + mount_runtime.tripMountLost(); + + /// (5b) Drain the ref lanes (bounded by one attempt's budget + safety margin) to learn whether a clean + /// farewell is EARNED — exactly the `~Pool` rule. + const bool ref_lanes_drained = ref_ledger.drainRefLanesForShutdown( + config.cas_request_budget.attempt_timeout_ms + config.cas_request_budget.lease_safety_margin_ms); + const bool drained = ref_lanes_drained && !writerCleanupDutiesPending(); + + /// (3+5c) Retire the merged heartbeat: a clean-release farewell ONLY if the lanes provably drained, + /// otherwise stop background renewal with NO terminal marker so the lease expires by observation (never + /// an unearned clean farewell). Also does the belt-and-suspenders remount rejoin. Outside `remount_mutex`. + mount_runtime.finishTeardown(drained); + + /// The pool object OUTLIVES this FORGET (it stays registered, `Vanished(forgotten)`, until DROP/restart), + /// so `~Pool` will re-run the same teardown. Drop the keeper now so that later teardown finds none and + /// skips it: `MountLeaseKeeper::stop`'s terminal op is single-shot (`doTerminate` throws a `LOGICAL_ERROR` + /// on a second call — an ASan-abort at construction), so a keeper that already terminated here must not be + /// terminated again. `keeperReset` is safe now: every keeper-touching thread (renewal, remount) is joined. + mount_runtime.keeperReset(); + + /// (6) Publish the terminal state + WARN, under remount serialization — matching the natural-transition + /// contract. Every pool thread is already joined, so taking `remount_mutex` here cannot self-deadlock. + /// `reason` is the [D5] message (with the operator's decommission timestamp) that + /// `throwIfLifecycleTerminal` surfaces to store-class callers. + { + std::lock_guard g(remount_mutex); + mount_runtime.enterVanished(PoolLifecycle::VanishedForgotten, reason); + } +} + +/// The plain-object surface (namespace files + mountpoint objects) is implemented by the stateless +/// `plain_objects` component; these are thin delegates preserving the API. +void Pool::putNamespaceFile(const NamespaceLifeId & life, const String & name, const String & bytes) +{ + plain_objects.putNamespaceFile(life, name, bytes); +} + +std::optional Pool::getNamespaceFile(const NamespaceLifeId & life, const String & name) +{ + return plain_objects.getNamespaceFile(life, name); +} + +std::vector Pool::listNamespaceFiles(const NamespaceLifeId & life) +{ + return plain_objects.listNamespaceFiles(life); +} + +uint64_t Pool::minActive() +{ + return mount_runtime.minActive(); +} + +uint64_t Pool::peekNextBuildSeq() +{ + return mount_runtime.peekNextBuildSeq(); +} + +bool Pool::tryRemountOnce() +{ + std::lock_guard serialize(remount_mutex); + + const String & srid = config.server_root_id; + const UInt128 our_uuid = config.server_id; + const auto now_ms = []() -> uint64_t + { + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + }; + const uint64_t ttl_ms = static_cast(config.mount_lease_ttl_ms.count()); + const uint64_t poll_interval_ms = std::max( + 1, static_cast(config.mount_renew_period.count()) / 2); + + /// Best-effort round for the MountRemount audit event only (diagnostic, never correctness-relevant): + /// `currentGcRound` is a live `gc/state` GET, which may itself fail on the very backend trouble that + /// is causing this remount attempt to fail — never let that escalate into an uncaught throw out of + /// a function whose contract is "returns bool, never throws". + const auto round_for_event = [this]() -> uint64_t + { + try { return currentGcRound(); } catch (...) { return 0; } + }; + + /// ==== Step 0 (rev.7 §2): pool lifecycle identity gate — BEFORE any claim/allocate/mount write ==== + /// A remount attempt means the lease is presumed lost, so first ensure we are at least transient + /// (production reaches here already transient via `tripMountLost`; a direct/forced call may still be + /// `Live`). Then authoritatively probe the pool sentinels and dispatch per the §2 verdict table. Only + /// `Recover` (a present `_pool_meta` whose identity matches, in a non-`IdentityLost` state) falls + /// through to the existing recovery below; every other verdict resolves here and returns false. + mount_runtime.noteLeaseLost(); + /// A fully-terminal `Vanished` pool never probes/claims/writes again. + if (mount_runtime.isVanished()) + return false; + { + const LifecycleGate gate = probePoolLifecycleGate( + *pool_backend, pool_layout, config.server_root_id, meta.pool_id, meta.blob_header_len); + switch (gate.verdict) + { + case LifecycleGateVerdict::Recover: + /// [D3] no auto-revival: a matching-sentinel observation while `IdentityLost` does NOT + /// bring the disk back — the state is terminal; only a restart recovers. + if (mount_runtime.lifecycle() == PoolLifecycle::IdentityLost) + return false; + break; /// fall through to the existing fresh-incarnation recovery. + case LifecycleGateVerdict::Replaced: + /// NOT while a FORGET is in progress (spec §9 rev.8 item 7). `forgetDisk` publishes the + /// terminal-intent latch at step 1 (`publishVanishedIntent`), then joins the remount thread; + /// a `tryRemountOnce` already IN FLIGHT — one that passed the step-0 `isVanished()` gate + /// BEFORE the intent was published, which that gate therefore cannot catch — could otherwise + /// settle `Vanished(replaced)` mid-FORGET, stranding FORGET's own + /// `enterVanished(VanishedForgotten)` (first terminal STATE transition wins) and mislabeling + /// the operator-visible reason. The bail lives HERE, at the terminal settle, so the + /// Recover/`armMountFence` reclaim path is untouched — its mid-FORGET fence re-arm is the + /// SEPARATE hazard `forgetDisk`'s post-join re-trip (trip#2) guards. Post-excision this is the + /// ONLY surviving mid-FORGET natural-terminal race (the old erasure-proof promotion is gone). + if (mount_runtime.vanishedIntentPublished()) + return false; + mount_runtime.enterVanished(PoolLifecycle::VanishedReplaced, gate.reason); + return false; + case LifecycleGateVerdict::IdentityLost: + /// Both sentinels authoritatively absent. Enter `IdentityLost` once (from `TransientNotLive`); + /// a repeat probe while already `IdentityLost` is a no-op. rev.8: `IdentityLost` is a + /// fail-loud TERMINAL state — the remount thread self-exits at its next boundary (see + /// `CasMountRuntime::remountTerminal`), so there is no demoted observer. + if (mount_runtime.lifecycle() != PoolLifecycle::IdentityLost) + mount_runtime.enterIdentityLost(); + return false; + case LifecycleGateVerdict::StayTransient: + /// Uncertain — remain transient and let the recovery loop retry. The probe's own reason is + /// the only account of WHY, and it is the difference between a transient store hiccup and + /// a pool this build can never open again: an undecodable `_pool_meta` is what a pool + /// predating the contiguous-ref-stream format floor looks like from here (`decodePoolMeta` + /// throws, the gate catches it), and silently retrying forever would leave that pool's + /// operator with a fenced mount and nothing to read. Say it, and count it. + ProfileEvents::increment(ProfileEvents::CASRemountHeldTransient); + LOG_WARNING(getLogger("CasPool"), + "content-addressed pool '{}' (prefix '{}'): remount held TRANSIENT — {}. The mount " + "stays fenced closed and the remount loop will retry. If this repeats, the pool " + "metadata is unreadable to this build (a pool below its format floor must be " + "recreated) rather than merely unavailable.", + config.server_root_id, config.pool_prefix, gate.reason); + return false; + } + } + + /// The same startup protocol as Pool::open steps 2-4, as a FRESH incarnation (the old one is + /// dead by the fence-out contract and its keeper never re-mints). Open THROWS on any failure + /// (startup is fail-closed); the remount RETURNS false instead — the recovery loop retries. + try + { + const ObserveRefCatalog observe_catalog = [this]() + { + CasRefCatalog::Snapshot snapshot = CasRefCatalog::read(*pool_backend, pool_layout); + snapshot.life_index.throwIfAmbiguous("CAS server-root remount safety"); + return snapshot.catalog; + }; + /// The present-owner/present-epoch fast path would otherwise skip the observer entirely. + /// Require the catalog before allocating a new epoch or replacing the mount slot. + (void)observe_catalog(); + claimOwnerOrThrow(*pool_backend, pool_layout, srid, our_uuid, observe_catalog); + const uint64_t writer_epoch = allocateWriterEpoch( + *pool_backend, pool_layout, srid, EpochMintPolicy::NormalMount, 0, observe_catalog); + + /// Mount-slot writer audit: `this` is already fully open (setEventSink ran long ago), so + /// unlike the initial `open`, every event fired below reaches the real sink immediately. + const auto emit_mount_event = [this](CasEvent e) { emitEvent(std::move(e)); }; + + const auto sleep_ms = [](uint64_t ms) { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); }; + const MountClaimResult claim = claimMountAwaitingExpiry( + *pool_backend, pool_layout, srid, our_uuid, writer_epoch, + now_ms, [this] { return bootMsNow(); }, ttl_ms, poll_interval_ms, sleep_ms, + [&srid](const MountLease & held, uint64_t threshold_ms) + { + LOG_INFO(getLogger("CasPool"), + "CAS self-remount '{}': observing a stale-looking mount's write-token (uuid={} " + "epoch={} expires_at_ms={}) for up to ~{} ms before reclaiming", + srid, u128ToHex(held.server_uuid), held.writer_epoch, held.expires_at_ms, threshold_ms); + }, + emit_mount_event); + if (claim.kind != MountClaimResult::Claimed) + { + LOG_WARNING(getLogger("CasPool"), + "CAS self-remount '{}': mount not claimable ({}); will retry", srid, + claim.kind == MountClaimResult::ForeignOwner ? "foreign owner — never taking over" + : "a live twin holds the lease"); + return false; + } + + /// NO WAIT HERE, AND THE ASSERT IS WHAT REPLACES IT. This used to pay `materialization_grace_ms` + /// whenever this incarnation's own ref lanes had not provably settled before the fence tripped: + /// an unresolved (still-wedged) ref-log conditional `PUT` from the dying epoch could otherwise + /// land after recovery began trusting its listings. Recovery no longer trusts listings — it walks + /// arithmetically and closes every epoch below the live one with an in-band `EpochSeal` at + /// `{E, T+1}`, so the wedged `PUT` loses its own conditional create to that seal and the wait + /// bought nothing but latency on every fence recovery. + /// + /// That replacement has ONE precondition, and it is this one: the incarnation we are about to + /// install must outrank the dying one. If the epoch did not strictly advance, the straggler's + /// slot is not "below the live epoch", nothing seals it, and the hole the wait used to paper over + /// reopens silently. `allocateWriterEpoch` mints from a durable monotone counter, so this holds by + /// construction — which is exactly why it is worth asserting rather than assuming: it fails the + /// build's own tests the moment someone reuses an epoch across a remount. + chassert(writer_epoch > mount_runtime.liveWriterEpoch()); + + /// Swap the keeper for the new incarnation. The old keeper's renewal loop already stopped on + /// its failed renew; never run its terminal op (the slot now belongs to the new claim). + /// `installKeeper` re-wires the SAME fence/min-active/event callbacks on `mount_runtime` as the + /// initial `open` did -- so the granular mechanics here are the exact same primitives, in the + /// exact same order, as `mountWritable`'s. + if (mount_runtime.hasKeeper()) + mount_runtime.keeperStopBackground(); + mount_runtime.installKeeper(our_uuid, writer_epoch, now_ms); + /// Pre-I/O anchor of this remount's claim attempt (mirrors `mountWritable`'s + /// `claim_anchor_boot_ms`, captured at the identical point -- right after `installKeeper`, + /// right before the keeper's own adopt write). No wait can land between this anchor and the arm + /// below, so no TTL-consumed redo is needed here: the anchor alone suffices (rev.4 Phase B, + /// round-3 finding 2; the redo lives in `mountWritable`, which keeps it for a claim that stalls). + /// `quiesceRefTablesForRemount` below IS a wait, but a bounded one -- bounded by the same + /// `cas_request_budget` that `validateCasRequestBudget` already guarantees fits under `ttl_ms`. + const uint64_t remount_anchor_boot_ms = mount_runtime.bootMsNow(); + mount_runtime.keeperStart(); + + /// Re-establish the ref-protocol incarnation BEFORE re-arming the fence. Order is load-bearing: + /// `keeperStart` refreshes the lease deadline + /// but does NOT clear `lost`, so the fence stays closed here and no append/publish can race the + /// swap. + /// 1. Bump the live epoch so every subsequent `allocateRefTxnId` sorts strictly above any older + /// (dead-incarnation or twin) durable log. Do this BEFORE `armMountFence` so there is no window + /// where the gate is open while the epoch is still stale. Keep `process_epoch` (the identity + /// accessors) equal to it. + mount_runtime.setLiveWriterEpoch(writer_epoch); + mount_runtime.setProcessEpoch(writer_epoch, std::memory_order_release); + /// 2. CANCEL OR JOIN every in-flight ref-table recovery, and BLOCK here until none is left (spec + /// §3: "self-remount cancels or waits out recovery before rearming"). A recovery admitted under + /// the outgoing incarnation WRITES -- its seal CAS-walk mints epoch seals and advances the + /// `_ckpt` -- so it must be stopped at this boundary rather than caught one site at a time + /// after the incarnation has already changed underneath it. Strictly before the quiesce below + /// so a cancelled walk unwinds while its runtime is still attached, and strictly before the + /// re-arm so no old-generation write can straddle it. + ref_ledger.cancelRecoveriesAndAwaitQuiescence(); + /// 3. Drain publishers and drop the cached tables so each re-recovers under the new epoch on next + /// touch (and any leader still holding an orphaned runtime fails closed). While the fence is lost. + ref_ledger.quiesceRefTablesForRemount(); + /// 4. Re-open the gate. Anchored at the claim attempt's pre-I/O instant (`remount_anchor_boot_ms`), + /// never at response time -- see the comment above `keeperStart()`. From here appends allocate + /// ids under the new epoch and touch fresh runtimes. + mount_runtime.armMountFence(our_uuid, writer_epoch, remount_anchor_boot_ms + ttl_ms); + if (config.background_watermark) + mount_runtime.keeperStartBackground(config.mount_renew_period); + + LOG_INFO(getLogger("CasPool"), + "CAS self-remount '{}': recovered as writer_epoch {} (fresh incarnation; older builds fail closed)", + srid, writer_epoch); + EventEmitter{*this}.emit([&](CasEvent & e) + { + e.type = CasEventType::MountRemount; + e.round = round_for_event(); + e.outcome = "ok"; + e.reason = "self-remount recovered a fresh mount incarnation after fence-out / renewal failure"; + e.detail = {{"writer_epoch", std::to_string(writer_epoch)}, + {"server_root_id", srid}}; + }); + /// Recovery succeeded: `TransientNotLive -> Live` (never revives a terminal state — but the gate + /// above guarantees we only reach here from a non-terminal state anyway). + mount_runtime.noteRemounted(); + return true; + } + catch (...) + { + tryLogCurrentException(getLogger("CasPool"), "CAS self-remount attempt failed; will retry"); + EventEmitter{*this}.emit([&](CasEvent & e) + { + e.type = CasEventType::MountRemount; + e.round = round_for_event(); + e.outcome = "failed"; + e.reason = "self-remount attempt failed; the recovery loop retries with backoff"; + e.detail = {{"server_root_id", srid}, + {"error", getCurrentExceptionMessage(/*with_stacktrace*/ false)}}; + }); + return false; + } +} + +/// The self-remount recovery thread + the merged-heartbeat renew live in `mount_runtime` +/// (Pool/CasMountRuntime.h); these are thin delegates. `mount_runtime`'s `remount_attempt` callback is +/// bound to `Pool::tryRemountOnce` (the claim/recovery orchestration that stays on Pool). +bool Pool::scheduleRemountForTest() +{ + return mount_runtime.scheduleRemountForTest(); +} + +void Pool::beginShutdownForTest() +{ + mount_runtime.beginShutdownForTest(); +} + +void Pool::renewWatermarkOnce() +{ + mount_runtime.renewWatermarkOnce(); +} + +void Pool::retireBuildSeq(uint64_t seq) +{ + mount_runtime.retireBuildSeq(seq); +} + +void Pool::enqueueWriterCleanupDuty( + const RootNamespace & ns, const String & ref_name, const ManifestRef & manifest, uint64_t build_seq) noexcept +{ + try + { + auto duty = std::make_shared(WriterCleanupDuty{ + .ref_name = ref_name, + .manifest = manifest, + .build_seq = build_seq, + }); + std::lock_guard lock(writer_cleanup_mutex); + writer_cleanup_queues[ns].pending.push_back(std::move(duty)); + writer_cleanup_cv.notify_all(); + } + catch (...) + { + /// The build deliberately remains active. Advancing `min_active` after losing the only cleanup + /// duty would make an uncertain owner grant look dead; pinning the floor until process exit is + /// the safe failure direction, and successor recovery handles the durable remnant. + writer_cleanup_queue_failed.store(true, std::memory_order_release); + try + { + tryLogCurrentException( + getLogger("CasPool"), + "CAS writer cleanup duty could not be queued; retaining the build in the active watermark"); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// `noexcept` destructor path: the sticky bit above is the safety mechanism. + } + } +} + +bool Pool::writerCleanupDutiesPending() const +{ + if (writer_cleanup_queue_failed.load(std::memory_order_acquire)) + return true; + std::lock_guard lock(writer_cleanup_mutex); + return std::any_of( + writer_cleanup_queues.begin(), writer_cleanup_queues.end(), + [](const auto & item) { return !item.second.pending.empty(); }); +} + +void Pool::drainWriterCleanupDuties(const RootNamespace & ns) +{ + { + std::unique_lock lock(writer_cleanup_mutex); + writer_cleanup_cv.wait(lock, [&] + { + const auto it = writer_cleanup_queues.find(ns); + return it == writer_cleanup_queues.end() || !it->second.draining; + }); + + const auto it = writer_cleanup_queues.find(ns); + if (it == writer_cleanup_queues.end() || it->second.pending.empty()) + return; + it->second.draining = true; + } + + try + { + while (true) + { + std::shared_ptr duty; + { + std::lock_guard lock(writer_cleanup_mutex); + const auto it = writer_cleanup_queues.find(ns); + chassert(it != writer_cleanup_queues.end() && it->second.draining); + if (it->second.pending.empty()) + { + writer_cleanup_queues.erase(it); + writer_cleanup_cv.notify_all(); + return; + } + duty = it->second.pending.front(); + } + + ref_ledger.appendRefOps( + ns, + MutationScope::ref(duty->ref_name), + [ref_name = duty->ref_name, manifest = duty->manifest] + (const RefTableState & state) -> std::vector + { + /// Absence is a conclusive settlement, not an error: the original uncertain grant + /// was rejected, a promote atomically consumed it, or another exact owner-removal + /// path already discharged it. Presence owes one exact removal. + if (!state.getPrecommits().contains({ref_name, manifest})) + return {}; + RefOp op; + op.kind = RefOpKind::OwnerTransition; + op.old_binding = RefOwnerBinding{RefOwnerKind::Precommit, ref_name, manifest}; + return {op}; + }, + RootMutationOrigin::Writer, + RootMutationKind::Abandon); + + /// Removal/absence is now durable in the same state observation. Only now may the active + /// build floor advance beyond the manifest's build sequence. + retireBuildSeq(duty->build_seq); + + std::lock_guard lock(writer_cleanup_mutex); + const auto it = writer_cleanup_queues.find(ns); + chassert(it != writer_cleanup_queues.end() && it->second.draining); + chassert(!it->second.pending.empty() && it->second.pending.front() == duty); + it->second.pending.pop_front(); + } + } + catch (...) + { + std::lock_guard lock(writer_cleanup_mutex); + const auto it = writer_cleanup_queues.find(ns); + if (it != writer_cleanup_queues.end()) + it->second.draining = false; + writer_cleanup_cv.notify_all(); + throw; + } +} + +PartWriteTxnPtr Pool::beginPartWrite(PartWriteInfo info) +{ + /// Mint a globally-unique build id from two thread_local_rng draws (random u128). + const UInt64 hi = thread_local_rng(); + const UInt64 lo = thread_local_rng(); + const UInt128 build_id = (static_cast(hi) << 64) | lo; + + /// Strictly-increasing per-process build_seq carried by the `PartWriteTxn`. The `PartWriteTxn` is + /// added to the active set here and retired on publish/abandon/dtor, so minActive — the GC floor + /// the Pool-owned watermark renews — tracks in-flight builds. The build registry lives on + /// `mount_runtime`. + const uint64_t seq = mount_runtime.allocateBuildSeq(); + bool registered = false; + SCOPE_EXIT({ if (!registered) retireBuildSeq(seq); }); + + auto build = std::make_shared(shared_from_this(), build_id, seq, liveWriterEpoch(), std::move(info)); + /// Register for `dropNamespace`'s post-durable build cancellation. weak_ptr: + /// the wiring owns the returned shared_ptr; `retireBuildSeq` (publish/abandon/dtor) removes the entry. + mount_runtime.registerInflightBuild(seq, build); + registered = true; + return build; +} + +/// The manifest read path (readManifest / readManifestShared / locate) + its decode cache live in +/// the `manifest_reader` component (Pool/CasManifestReader.h); these are thin delegates. +std::shared_ptr Pool::readManifestShared(const ManifestId & id) +{ + return manifest_reader.readManifestShared(id); +} + +PartManifest Pool::readManifest(const ManifestId & id) +{ + return manifest_reader.readManifest(id); +} + +BlobLocation Pool::locate(const ManifestEntry & entry) const +{ + return manifest_reader.locate(entry); +} + +namespace +{ +/// a tolerant, read-only peek at the +/// `cas_ref_log` TEXT object (codecs-v3 phase 3) WITHOUT `decodeRefLogTxn`'s expected-value cross-check +/// -- the whole point of this diagnostic is that the body is NOT expected to match this key's identity. +/// It `openObject`s the stored `.zst`, skips the header line, and reads `ns`/`we`/`rs` off the meta +/// line (`we`/`rs` are decimal u64 strings). Never validates the header `v`, never reads past the meta +/// line (the ops are irrelevant to identifying the writer), and swallows any truncation/garbage: this +/// is a background diagnostic only, never a decode anything else depends on. +struct ForeignRefLogHeaderPeek +{ + String ns; + uint64_t writer_epoch = 0; + uint64_t ref_sequence = 0; +}; + +std::optional peekForeignRefLogHeader(const String & bytes) +{ + try + { + const String text = openObject(FormatId::RefLog, bytes); + ReadBufferFromMemory in(text.data(), text.size()); + const uint64_t line_cap = traitsFor(FormatId::RefLog).line_cap; + readLine(in, line_cap, "cas_ref_log"); /// header line -- skip + const String meta = readLine(in, line_cap, "cas_ref_log"); + ReadBufferFromMemory m(meta.data(), meta.size()); + JsonObjectReader r(m, KeyStrictness::Tolerant, "cas_ref_log"); + ForeignRefLogHeaderPeek peek; + bool saw_ns = false; + bool saw_we = false; + bool saw_rs = false; + String key; + while (r.nextKey(key)) + { + if (key == "ns") { peek.ns = r.readString(); saw_ns = true; } + else if (key == "we") { peek.writer_epoch = r.readU64String(); saw_we = true; } + else if (key == "rs") { peek.ref_sequence = r.readU64String(); saw_rs = true; } + else r.skipUnknown(key); + } + if (!saw_ns || !saw_we || !saw_rs) + return std::nullopt; + return peek; + } + catch (...) + { + return std::nullopt; + } +} +} + +void Pool::reportImpossibleInterference(const String & key, const String & reason, + const std::optional & offending_ns) +{ + LOG_ERROR(getLogger("CasPool"), + "CAS anomaly policy: impossible foreign interference for server_root '{}' (namespace='{}', key='{}'): " + "{} -- fencing this mount closed and scheduling a remount", + config.server_root_id, offending_ns.value_or(String{}), key, reason); + + EventEmitter{*this}.emit([&](CasEvent & e) + { + e.type = CasEventType::ForeignInterference; + if (offending_ns) + e.namespace_ = *offending_ns; + e.reason = reason; + e.detail = {{"key", key}, {"server_root_id", config.server_root_id}}; + }); + + /// Incidental-only detection has a fail-closed reaction -- the SAME on_lost + /// mechanics a foreign/superseded lease renewal already drives (the keeper's `setFenceCallbacks` + /// lambda). The fence + self-remount now live on `mount_runtime`. + mount_runtime.tripMountLost(); + mount_runtime.scheduleRemount(); + + /// Diagnosis off the critical path: a background task may spend a FEW + /// requests -- never the caller's thread, and never blocking this call's own return. + /// `shared_from_this()` keeps the Pool alive for the thread's lifetime (mirrors + /// `maybeScheduleSnapshotPublish`'s dispatch). + auto self = shared_from_this(); + try + { + ThreadFromGlobalPool([self, key] + { + setThreadName(ThreadName::CAS_ANOMALY_DIAG); + try + { + const auto got = self->pool_backend->get(key); + if (!got) + { + LOG_ERROR(getLogger("CasPool"), + "CAS anomaly diagnostics: the offending object at '{}' had already vanished by the " + "time the background diagnostic GET ran", key); + return; + } + if (const auto peek = peekForeignRefLogHeader(got->bytes)) + LOG_ERROR(getLogger("CasPool"), + "CAS anomaly diagnostics: offending object at '{}' ({} bytes) decodes as a ref-log " + "header: namespace='{}', writer_epoch={}, ref_sequence={}", + key, got->bytes.size(), peek->ns, peek->writer_epoch, peek->ref_sequence); + else + LOG_ERROR(getLogger("CasPool"), + "CAS anomaly diagnostics: offending object at '{}' ({} bytes) does not decode as a " + "ref-log header -- raw and unidentified", key, got->bytes.size()); + } + catch (...) + { + tryLogCurrentException(getLogger("CasPool"), + "CAS anomaly diagnostics: background GET failed for '" + key + "'"); + } + }).detach(); + } + catch (...) + { + /// Pool exhaustion: best-effort diagnostics must never block the caller's own fail-closed throw. + tryLogCurrentException(getLogger("CasPool"), "CAS anomaly diagnostics dispatch failed to launch for '" + key + "'"); + } +} + +uint64_t Pool::currentGcRound() const +{ + /// Read `gc/state` once (no CAS loop — a point-in-time read is sufficient; a concurrent + /// GC advance only makes the returned round larger, which is strictly more conservative for the + /// `precommitAdd` self-floor). Returns 0 when absent (pool never GC'd — no round to floor to). + const auto state_bytes = pool_backend->get(pool_layout.gcStateKey()); + if (!state_bytes) + return 0; + return decodeGcState(state_bytes->bytes).round; +} + +void Pool::removeNamespaceFile(const NamespaceLifeId & life, const String & name) +{ + plain_objects.removeNamespaceFile(life, name); +} + +void Pool::putMountpointObject(const String & key, const String & bytes) +{ + plain_objects.putMountpointObject(key, bytes); +} + +std::optional Pool::getMountpointObject(const String & key) +{ + return plain_objects.getMountpointObject(key); +} + +bool Pool::mountpointObjectExists(const String & key) +{ + return plain_objects.mountpointObjectExists(key); +} + +void Pool::removeMountpointObject(const String & key) +{ + plain_objects.removeMountpointObject(key); +} + +NamespaceListing Pool::listNamespaces(const String & prefix) +{ + /// The catalog is the logical namespace authority. Physical stream/state keys expose only an + /// opaque life id and therefore cannot mint a namespace during discovery. + std::unordered_set found; + std::vector skipped; + const CasRefCatalog::Snapshot cut = CasRefCatalog::read(*pool_backend, pool_layout); + for (const CatalogEntry & entry : cut.catalog.entries) + { + try + { + if (const auto life = cut.life_index.resolve(entry.incarnation); + life && life->ns.string().starts_with(prefix)) + found.insert(life->ns.string()); + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::CORRUPTED_DATA) + throw; + skipped.push_back(UnattributableNamespaceKey{ + pool_layout.refCatalogKey() + "#" + renderIncarnation(entry.incarnation), e.message()}); + } + } + + return NamespaceListing{{found.begin(), found.end()}, std::move(skipped)}; +} + +std::vector Pool::listMirroredChildren(const String & prefix) +{ + /// Namespace children come from the catalog. `roots/` is still listed for loose mountpoint files, + /// whose logical paths retain path identity. + std::unordered_set children; + const CasRefCatalog::Snapshot cut = CasRefCatalog::read(*pool_backend, pool_layout); + for (const CatalogEntry & entry : cut.catalog.entries) + { + if (!entry.ns.string().starts_with(prefix)) + continue; + const std::string_view rest(entry.ns.string().data() + prefix.size(), entry.ns.string().size() - prefix.size()); + const size_t slash = rest.find('/'); + const std::string_view segment = slash == std::string_view::npos ? rest : rest.substr(0, slash); + if (!segment.empty()) + children.emplace(segment); + } + + const String roots_full = pool_layout.rootsPrefix() + prefix; + { + String cursor; + while (true) + { + ListPage page = pool_backend->list(roots_full, cursor, /*limit*/ 1000); + for (const ListedKey & listed : page.keys) + { + const String & key = listed.key; + if (!key.starts_with(roots_full)) + continue; + const std::string_view rest(key.data() + roots_full.size(), key.size() - roots_full.size()); + const size_t slash = rest.find('/'); + const std::string_view seg = slash == std::string_view::npos ? rest : rest.substr(0, slash); + if (!seg.empty()) + children.emplace(seg); + } + if (page.next_cursor.empty()) + break; + cursor = page.next_cursor; + } + } + return {children.begin(), children.end()}; +} + + +/// ==== ref-ledger delegates ==== The ref-log / ref-table subsystem lives in the +/// `ref_ledger` member (Pool/CasRefLedger.h); Pool keeps these thin public forwarders so the wiring, +/// PartWriteTxn, Gc, and every test call site is unchanged. + +void Pool::setCasRetrySleepForTest(std::function sleep_fn) +{ + ref_ledger.setCasRetrySleepForTest(std::move(sleep_fn)); +} + +std::optional Pool::resolveRef(const RootNamespace & ns, const String & ref_name, bool allow_stale, ResolveAudit audit) +{ + return ref_ledger.resolveRef(ns, ref_name, allow_stale, audit); +} + +std::map Pool::listRefs(const RootNamespace & ns) +{ + return ref_ledger.listRefs(ns); +} + +bool Pool::hasAnyRefWithPrefix(const RootNamespace & ns, std::string_view prefix) +{ + return ref_ledger.hasAnyRefWithPrefix(ns, prefix); +} + +void Pool::dropRef(const RootNamespace & ns, const String & ref_name) +{ + mutateRefsAfterWriterCleanup(ns, [&] + { + ref_ledger.dropRef(ns, ref_name); + }); +} + +void Pool::updateRefPublishedAt(const RootNamespace & ns, const String & ref_name, + std::function mutator) +{ + mutateRefsAfterWriterCleanup(ns, [&] + { + ref_ledger.updateRefPublishedAt(ns, ref_name, std::move(mutator)); + }); +} + +DropNamespaceStats Pool::dropNamespace(const RootNamespace & ns) +{ + return mutateRefsAfterWriterCleanup(ns, [&] + { + return ref_ledger.dropNamespace(ns); + }); +} + +DropNamespaceStats Pool::dropNamespace(const NamespaceLifeId & life) +{ + return mutateRefsAfterWriterCleanup(life.ns, [&] + { + return ref_ledger.dropNamespace(life); + }); +} + +NamespaceLifeId Pool::namespaceLife(const RootNamespace & ns) +{ + return ref_ledger.namespaceLife(ns); +} + +std::optional Pool::namespaceFilesLifeIfReadable(const RootNamespace & ns) +{ + return ref_ledger.namespaceFilesLifeIfReadable(ns); +} + +bool Pool::namespaceStillLogicallyPresent(const RootNamespace & ns) +{ + return ref_ledger.namespaceStillLogicallyPresent(ns); +} + +void Pool::invalidateRemovedCatalogLife(const NamespaceLifeId & life) +{ + ref_ledger.invalidateRemovedCatalogLife(life); +} + +void Pool::reconcileRefCatalogCut(const CasRefCatalog::Snapshot & catalog_cut) +{ + ref_ledger.reconcileCatalogCut(catalog_cut); +} + +RefTxnId Pool::appendRefOps(const RootNamespace & ns, MutationScope scope, + std::function(const RefTableState &)> build_ops, + RootMutationOrigin origin, RootMutationKind kind, + bool skip_stale_precommit_sweep) +{ + return mutateRefsAfterWriterCleanup(ns, [&] + { + return ref_ledger.appendRefOps( + ns, std::move(scope), std::move(build_ops), origin, kind, skip_stale_precommit_sweep); + }); +} + +bool Pool::tryPublishSnapshotAndAdvanceCheckpointOnce(const RootNamespace & ns) +{ + return mutateRefsAfterWriterCleanup(ns, [&] + { + return ref_ledger.tryPublishSnapshotAndAdvanceCheckpointOnce(ns); + }); +} + +size_t Pool::wedgedRefLaneCount() +{ + return ref_ledger.wedgedRefLaneCount(); +} + +CasWriteOutcome Pool::stagingPutIfAbsent(std::string_view key, std::string_view bytes, Token * out_token) +{ + return ref_ledger.stagingPutIfAbsent(key, bytes, out_token); +} + +CasCreateResult Pool::stagingConditionalCreate(std::string_view key, const std::function & attempt) +{ + return ref_ledger.stagingConditionalCreate(key, attempt); +} + +CasOverwriteResult Pool::stagingConditionalOverwrite(std::string_view key, std::string_view bytes, const Token & expected) +{ + return ref_ledger.stagingConditionalOverwrite(key, bytes, expected); +} + +CasOverwriteResult Pool::stagingPutIfAbsentMutable(std::string_view key, std::string_view bytes) +{ + return ref_ledger.stagingPutIfAbsentMutable(key, bytes); +} + +void Pool::cancelInflightBuildsForNamespace(const RootNamespace & ns) +{ + /// Delegate to `mount_runtime`. Invoked by + /// `ref_ledger` through the `cancel_inflight_builds` callback once its removal transaction is durable, + /// so that in-flight local builds targeting the removed namespace are cancelled. + mount_runtime.cancelInflightBuildsForNamespace(ns); +} + +uint64_t Pool::refRecoveryRestartsForTest(const RootNamespace & ns) +{ + return ref_ledger.refRecoveryRestartsForTest(ns); +} + +bool Pool::refLaneWedgedForTest(const RootNamespace & ns) +{ + return ref_ledger.refLaneWedgedForTest(ns); +} + +String Pool::wedgedKeyForTest(const RootNamespace & ns) +{ + return ref_ledger.wedgedKeyForTest(ns); +} + +void Pool::forceWedgeForTest(const RootNamespace & ns, uint64_t writer_epoch, uint64_t ref_sequence, + const String & key, const String & bytes, + std::optional admitted_generation) +{ + ref_ledger.forceWedgeForTest(ns, writer_epoch, ref_sequence, key, bytes, admitted_generation); +} + +uint64_t Pool::wedgedAdmittedGenerationForTest(const RootNamespace & ns) +{ + return ref_ledger.wedgedAdmittedGenerationForTest(ns); +} + +std::optional Pool::lastEpochSealForTest(const RootNamespace & ns) +{ + return ref_ledger.lastEpochSealForTest(ns); +} + +void Pool::setLastEpochSealForTest(const RootNamespace & ns, const std::optional & seal) +{ + ref_ledger.setLastEpochSealForTest(ns, seal); +} + +RefLaneState Pool::laneStateForTest(const RootNamespace & ns) +{ + return ref_ledger.laneStateForTest(ns); +} + +bool Pool::needsStalePrecommitSweepForTest(const RootNamespace & ns) +{ + return ref_ledger.needsStalePrecommitSweepForTest(ns); +} + +void Pool::waitForSnapshotPublishSettleForTest(const RootNamespace & ns) +{ + ref_ledger.waitForSnapshotPublishSettleForTest(ns); +} + +int Pool::pendingSnapshotPublishesForTest(const RootNamespace & ns) +{ + return ref_ledger.pendingSnapshotPublishesForTest(ns); +} + +std::optional Pool::newestPublishedSnapshotIdForTest(const RootNamespace & ns) +{ + return ref_ledger.newestPublishedSnapshotIdForTest(ns); +} + +bool Pool::refTableRecoveredForTest(const RootNamespace & ns) +{ + return ref_ledger.refTableRecoveredForTest(ns); +} + +bool Pool::refRecoveryCancelRequestedForTest(const RootNamespace & ns) +{ + return ref_ledger.refRecoveryCancelRequestedForTest(ns); +} + +void Pool::cancelRefRecoveriesAndAwaitQuiescence() +{ + ref_ledger.cancelRecoveriesAndAwaitQuiescence(); +} + +size_t Pool::tailSinceSnapshotCountForTest(const RootNamespace & ns) +{ + return ref_ledger.tailSinceSnapshotCountForTest(ns); +} + +size_t Pool::committedOverlayEntriesForTest(const RootNamespace & ns) +{ + return ref_ledger.committedOverlayEntriesForTest(ns); +} + +std::set> Pool::livePrecommitsForTest(const RootNamespace & ns) +{ + return ref_ledger.livePrecommitsForTest(ns); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h new file mode 100644 index 000000000000..5016d5283979 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h @@ -0,0 +1,1159 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + + +/// Configuration supplied when opening a content-addressed pool. The fields remain flat for the +/// compatibility of existing wiring and tests, while `refLedgerConfig` and `mountConfig` project +/// the fields owned by those subsystems into typed values passed by value. The projections avoid an +/// include cycle and make it explicit that the subsystem objects do not retain a reference to this +/// configuration object. +struct PoolConfig +{ + String pool_prefix; + UInt128 server_id{}; /// owner token (ServerUUID) — provenance + watermark + /// Explicit, configured identity of the layout subtree this server owns. Required + validated + /// (clean relative path); `ServerUUID`/`server_id` is demoted to an + /// owner token. Validated via `Cas::validateServerRootId`. + String server_root_id; + uint64_t blob_header_len = 256; /// creation-time only; the pool is authoritative on reopen + /// CAS mixed-algo pools: the + /// NODE-LOCAL algo this Pool writes NEW content with (`Pool::writeAlgo()`). NOT durable pool + /// state -- two live nodes may intentionally write with different (already-admitted) algos, so + /// no single truthful pool-wide value exists. `PoolMeta::createOrValidate` accepts it with no + /// write when it is already a member of the pool's `algos_used`; otherwise it is admitted via + /// the CAS-union below (opt-in, `blob_hash_allow_new`) or refused (BAD_ARGUMENTS, the default -- + /// a changed config alone must never silently turn a pool mixed). Default `CityHash128` keeps + /// every existing pool's hash byte-for-byte unchanged. + BlobHashAlgo blob_hash_algo = BlobHashAlgo::CityHash128; + /// Opt-in for `blob_hash_algo` to be ADMITTED into the pool's `algos_used` when it is not + /// already a member. Consulted only on + /// the FIRST open that would otherwise refuse -- once admitted, `algos_used` membership alone is + /// the steady-state check and this flag is not needed again for the same algo. Default `false` + /// (fail-closed, matching the default-refuse behavior). + bool blob_hash_allow_new = false; + /// Dedup cache: byte ceiling for the per-disk known-present blob-hash LRU set. 0 disables the + /// cache (every create misses → HEAD-before-PUT only). A hint cache; correctness never depends on + /// it (a stale hit is caught by the mandatory HEAD in putBlob). + uint64_t deduplication_cache_bytes = 64ULL << 20; /// 64 MiB + /// HEAD-before-PUT: on a dedup-cache MISS, a blob whose body is >= this many bytes is written + /// HEAD-first (a cheap HEAD avoids streaming a body that would 412). 0 disables the size trigger. + uint64_t deduplication_head_first_min_bytes = 1ULL << 20; /// 1 MiB + /// Part-folder cache: byte bound for the manifest DECODE cache. The old cache + /// was count-bounded only (16384 entries) — decoded manifests carry inline bytes, so the worst + /// case was multi-GB. 0 disables decode caching (every read decodes fresh — diagnostic mode). + uint64_t manifest_decode_cache_bytes = 128ULL << 20; + /// How many superseded snapshot generations to retain. After committing + /// generation G, generations <= G - this are pruned (bounded per round). 0 = keep ALL + /// (debug/forensics — replay GC's in-degree view as-of a past round). Default 3 = the safety + /// margin covering any in-flight/resuming leader (a leader more than `keep` generations behind + /// has lost its lease; its round-commit CAS fails). + uint64_t gc_snapshot_generations_to_keep = 3; + /// Blob target shards for GC. Default 1 (a single shard, i.e. no fan-out). Creation-time only; + /// the pool is authoritative on reopen. This is the BLOB-HASH-prefix reducer axis. + uint64_t gc_shards = 1; + /// Cursor-paced orphan part-manifest sweep. The LIST budget bounds cold-prefix enumeration + /// per completed GC round; the delete budget separately bounds exact-token destructive work. + uint64_t manifest_sweep_list_budget_keys = 1000; + uint64_t manifest_sweep_delete_budget_keys = 100; + /// Per-round blob-deletion work envelope: caps how many entries the fold's graduation + /// (condemned -> delete_pending) and redelete (exact-token delete of a prior delete_pending row) + /// arms move out of the durable retired pipeline in one round. Excess entries are carried + /// unchanged in `still_retired` and retried next round (never dropped). 0 = unbounded. + /// Default UNBOUNDED. A count cap here is not backpressure: it throttles the consumer while the + /// producer (inserts, merges) is unaware of it, and the excess is carried in `still_retired`, which + /// the next round reads in full -- so a round's cost grows with the debt while its useful work stays + /// capped. Under sustained load that is a feedback loop, not a delay. + uint64_t gc_round_graduation_budget = 0; + uint64_t gc_round_redelete_budget = 0; + /// Orphan-manifest sweep: caps on the expensive step the LIST/nomination budgets never covered — + /// building a namespace's protection view (catalog-authoritative table recovery + committed-tail + /// walk). `sweep_namespace_budget` bounds how many DISTINCT namespaces one page may build a view + /// for; `sweep_recovery_op_budget` bounds the total ref-log GET/decode ops the committed-tail walk + /// may spend across every namespace, cumulative for the round. Exhausting either retains every + /// remaining candidate of the affected namespace on this page (fail-closed; retention is always + /// safe) rather than deciding it without a complete protection view. + uint64_t gc_round_sweep_namespace_budget = 20; + uint64_t gc_round_sweep_recovery_op_budget = 5000; + /// Ref-object cleanup (covered log/snapshot exact deletes) and generation-prefix wholesale delete + /// (superseded-generation prune) caps, cumulative for the round. `prefix_wholesale_budget` is the + /// round's shared remainder every prune `deletePrefixWholesale` call draws from, so no single call + /// passes an unbounded `bounded_remaining`. + uint64_t gc_round_ref_cleanup_budget = 5000; + uint64_t gc_round_prefix_wholesale_budget = 20000; + /// The post-CAS hand-off reclaim draws from its OWN reserve, never from `prefix_wholesale_budget` + /// above: the prune safely under-serves and retries next round via its cursor, but the hand-off is a + /// one-shot event with no reclaimer besides `fsck`, so a prune-heavy round must never be able to + /// starve it to zero. + /// Default UNBOUNDED, and this one is not a tuning choice. The hand-off is one-shot: a generation it + /// cannot fully reclaim in its round is never revisited, because the parent-seal difference that + /// triggers it does not recur. A cap on work that has no second chance does not defer the work, it + /// leaks it -- the same shape as the manifest-cleanup cap that was removed outright. + uint64_t gc_round_handoff_prefix_wholesale_budget = 0; + /// `GcOutcomes` per-round entry cap across the redelete/spared audit log, cumulative for the round. + /// Bounds only the audit-log write -- the settlement decision it records already happened + /// unconditionally in the fold. 0 = unbounded. + /// Default UNBOUNDED: nothing is retried on exhaustion because there is nothing left to retry -- the + /// decision already happened -- so the only thing a cap drops is the audit row explaining it, and it + /// drops exactly the rows of the busiest rounds, which are the ones an investigation needs. + uint64_t gc_round_outcome_entry_budget = 0; + /// Frontier probes: how many KNOWN-BUT-UNHINTED namespaces one round may walk to prove their + /// frontier. A namespace this round's LIST hint still mentions is walked regardless (the round owes + /// its edges anyway), and a HELD one is always walked (its hold must be retried by exact key, spec + /// §5), so this bounds only the extra exact `GET`s the universe union introduced -- normally zero, + /// because ordinary active namespaces normally remain hinted. Running out is NOT an error: the + /// unprobed namespaces are simply unproven, which + /// suppresses all destruction for the round. 0 => probe none (the exhaustion path, which tests + /// drive directly). + /// + /// Default effectively unbounded, and it must be spelled as a huge number rather than `0`: unlike + /// every other budget here, `0` means "probe nothing", not "no cap" -- the tests drive that path + /// deliberately, so the sentinel cannot simply be redefined. Exhaustion is the worst failure shape + /// of any budget in this struct, because it does not defer work: unprobed namespaces are unproven, + /// and one unproven namespace suppresses ALL destruction for the round. A count that is fine for ten + /// namespaces silently becomes a permanent GC stop for a pool with enough tables. + /// The cost of removing the cap is round LENGTH (extra exact `GET`s, normally zero because active + /// namespaces stay hinted) -- which is bounded by nothing today, since rounds have no time deadline. + uint64_t gc_frontier_probe_budget = std::numeric_limits::max(); + /// skip-unchanged: a GC round may DEFER + /// (re-adopt the sealed in-degree generation instead of rebuilding it) when fewer than this many + /// shards changed since the last fold AND no destructive decision is due. Default 1 = fold as soon + /// as anything changed (batching off; only idle rounds defer). > 1 batches small deltas. + uint64_t gc_fold_threshold = 1; + /// Liveness bound for batching: force a FOLD after this many consecutive DEFER rounds even below + /// the threshold. Inert at gc_fold_threshold == 1 (an idle defer has nothing to fold). Default 8. + uint64_t gc_fold_max_defer_rounds = 8; + /// Diagnostic-only threshold for a `Removing` catalog life whose terminal cleanup evidence has + /// not appeared. Test/config struct seam only; no user-facing setting is registered. + uint64_t gc_stuck_removal_rounds = 10; + /// gc-rebuild: max in-memory edges per gc-shard batch during rebuildBaseline + /// (~32 B each => default ~256 MB); each full batch folds into the next attempt number with the + /// previous attempt's runs as priors, so memory is O(budget), never O(edges). + uint64_t rebuild_edge_budget = 8000000; + /// Bounded pool size for the per-hash freshness + /// meta writes GC schedules at condemn/spare/delete (mass-DROP: a round condemning ~1M blobs would + /// take hours sequential). Every job internally catches its own exceptions (never wedges the round; + /// feedback_ca_gc_never_throw_on_404) and `Gc::runRegularRound` waits for the round's whole batch + /// before the round's single gc/state CAS, so the meta writes are durable before that CAS commits. + uint64_t gc_meta_pool_size = 16; + bool background_watermark = false; /// tests drive renewOnce explicitly; gates the merged heartbeat's background thread + /// Installed on the pool before a writable mount can start its renewal thread. + CasEventSink event_sink = {}; + + /// Mount-lease TTL: how long a freshly-renewed mount lease is valid. The local + /// write fence's monotonic deadline is `renew_time + this`, so a superseded/paused writer is fenced + /// once `this` elapses with no successful renew. The background renewer runs every + /// `mount_renew_period` (default ttl/3) so a healthy mount renews well before expiry. + std::chrono::milliseconds mount_lease_ttl_ms{30000}; + std::chrono::milliseconds mount_renew_period{10000}; /// = ttl/3 by default + bool read_only = false; /// observe-only open: skip the mutating capability probe; reads only + + /// Boot-time "start now, fix later": skip the access-check-class part of the capability probe + /// (the `_probe/` read/write/delete/list round trip and the store-precondition check) while + /// STILL opening writable — a mistyped bucket / transient DNS blip at mount should not hard-fail + /// the disk when the operator asked to defer the access check (U#5), mirroring `checkAccess`'s + /// `skip_access_check` gate for other disk types. Does NOT skip the single-attempt conditional- + /// write gate (`checkConditionalWriteSingleAttemptSupport`, cas-s3-timeout-retry-control): + /// that guards every conditional write this writable mount will ever issue against running under + /// the disk's ~500-attempt transparent retry policy, a correctness hazard rather than a preflight + /// convenience, so `Pool::open` still runs it unconditionally whenever the mount is writable. + bool skip_access_check = false; + + /// The CAS retry controller's budget (cas-s3-timeout-retry-control), validated against + /// `mount_lease_ttl_ms` at writable open (`Pool::open` calls `validateCasRequestBudget`) — an + /// inconsistent budget refuses the mount rather than silently retrying unsafely. Defaults are + /// consistent with the default `mount_lease_ttl_ms` above; a caller that raises the lease TTL may + /// keep these defaults, but a caller that LOWERS it must revisit this budget too. + CasRequestBudget cas_request_budget{}; + + /// The write-fence deadline clock (CLOCK_BOOTTIME milliseconds; see `MountFence`). Empty = the real + /// boot clock (`Pool::bootMs`); injected by tests to drive the fence deadline deterministically. + std::function boot_ms_fn = {}; + + /// Test hook for open/remount waits: `Pool::waitSleep` -- the mount-claim observation loop's poll -- + /// routes through this function when set instead of a real `std::this_thread::sleep_for`, so a test + /// observes every wait without actually blocking. Empty (the production default) sleeps for real. + /// + /// That poll is now the ONLY wait either path can block on. The post-reclaim materialization grace + /// (`T_mat`) that used to sit beside it is gone: it existed so a straggler conditional `PUT` from a + /// dying epoch would land (or exhaust its retries) before the successor started trusting its recovery + /// LISTINGS, and recovery does not trust listings any more -- it walks the stream arithmetically and + /// closes every dead epoch with an in-band `EpochSeal` at `{E, T+1}`, written as a conditional + /// create. The straggler's own conditional create then LOSES against that occupied slot, whenever it + /// arrives. Waiting for a race that the protocol already decides is not caution, it is latency. + std::function wait_sleep_fn = {}; /// test hook for open/remount waits + + /// a table becomes a publish candidate once its retained + /// tail -- every applied txn strictly above the newest published snapshot, no age filter -- exceeds + /// either threshold (their count / the sum of their encoded bytes), or right after recovery replays + /// a tail already above one (the mount-time trigger). Publication is background and never blocks an + /// append (see `Pool::maybeScheduleSnapshotPublish`). The grace-age holdback this trigger used to + /// apply is gone by design: the recovery seal decides a late-arriving predecessor write outright -- + /// its conditional create loses to the seal already occupying the slot -- so a young txn has nothing + /// left to wait out. + /// The count default trades write-side PUT volume against read-side cold-fold cost: every publish + /// re-encodes and PUTs the FULL snapshot, so a low threshold under sustained load degenerates into + /// a near-continuous full-snapshot PUT stream, while on the read side a cold fold pays one GET per + /// log the newest snapshot does not cover -- 256 bounds that at 256 extra GETs, each far cheaper + /// than the snapshot churn it avoids. + uint64_t snapshot_log_count_threshold = 256; + uint64_t snapshot_log_bytes_threshold = 1ULL << 20; /// 1 MiB + /// bounded per-table backoff arming a dispatch cooldown after + /// a NON-Committed publish outcome (an S3 timeout / uncertain PUT). Without it, a saturated backend + /// turns every ref read into a re-dispatched full-snapshot encode+PUT (the read-triggered PUT storm), + /// since a non-Committed publish deliberately does not prune the tail (that would be data loss) and so + /// leaves the threshold trigger latched. The interval doubles from `initial` up to `max` per + /// consecutive failure and resets on the next durable publish; combined with the single-in-flight + /// gate, it bounds publish dispatch to O(failures), not O(reads). + uint64_t snapshot_publish_backoff_initial_ms = 200; + uint64_t snapshot_publish_backoff_max_ms = 30000; + /// Bounded per-table cooldown between FAILED stale-precommit sweep attempts (the dangling-precommit + /// hazard). A failed/partial sweep re-arms + /// `needs_stale_precommit_sweep` instead of consuming the once-per-mount shot (one attempt burned in + /// the post-restart error window used to leave a dead incarnation's precommit bindings -- and the + /// manifests they protect from the GC orphan sweep -- live forever on a long-lived mount); this + /// cooldown keeps the retry from storming a saturated backend, exactly like the publish backoff. + /// Doubles from `initial` up to `max` per consecutive failure; reset by a verified-clean sweep. + uint64_t precommit_sweep_backoff_initial_ms = 200; + uint64_t precommit_sweep_backoff_max_ms = 30000; + + /// resident-memory ceiling for the writer's + /// whole-table ref cache (`CasRefLedger::ref_name_slots`). This implementation has no row overlay, so eviction is + /// WHOLE-TABLE: when the summed estimated weight of cached tables exceeds this, whole tables are + /// dropped (never rows) and the next touch re-recovers them from the durable snapshot+log objects + /// Evicting the table drops the entire object; the next access repeats + /// recovery"). A table with a wedged append lane, a nonempty pending queue, or any in-flight + /// caller/publish (its un-persisted lane/queue state is not reconstructable) is never evicted, and + /// neither is the table whose recovery just triggered the pass -- so the effective floor is one + /// table. 0 = unbounded (eviction disabled). The estimate is the base snapshot body size plus the + /// retained log-tail bytes; both are already tracked, so a mutation costs no extra encode. + uint64_t ref_table_cache_bytes = 256ULL << 20; /// 256 MiB + + /// Projection accessors: build the per-owner typed slice from + /// the flat fields above, for BY-VALUE injection into the ref-ledger / mount-runtime components. + /// The fields stay flat here so every external caller (wiring, tests) that sets them is unchanged; + /// the slices are derived, not stored. `boot_ms_fn` is intentionally in `MountConfig` and reaches + /// the ref-ledger as a ctor callback (not duplicated into `RefLedgerConfig`). + RefLedgerConfig refLedgerConfig() const + { + return RefLedgerConfig{ + .server_root_id = server_root_id, + .gc_shards = gc_shards, + .snapshot_log_count_threshold = snapshot_log_count_threshold, + .snapshot_log_bytes_threshold = snapshot_log_bytes_threshold, + .snapshot_publish_backoff_initial_ms = snapshot_publish_backoff_initial_ms, + .snapshot_publish_backoff_max_ms = snapshot_publish_backoff_max_ms, + .precommit_sweep_backoff_initial_ms = precommit_sweep_backoff_initial_ms, + .precommit_sweep_backoff_max_ms = precommit_sweep_backoff_max_ms, + .ref_table_cache_bytes = ref_table_cache_bytes, + }; + } + + MountConfig mountConfig() const + { + return MountConfig{ + .mount_lease_ttl_ms = mount_lease_ttl_ms, + .mount_renew_period = mount_renew_period, + .background_watermark = background_watermark, + .boot_ms_fn = boot_ms_fn, + .wait_sleep_fn = wait_sleep_fn, + }; + } +}; + + +struct PartWriteInfo +{ + std::optional intended_ref; /// "ns/ref" forensics for the envelope (diagnostic) + /// The owning root namespace, set EXPLICITLY by the wiring. When present it is authoritative for + /// the manifest's owning namespace (PartWriteTxn::manifestNamespace), so a ref that itself contains '/' + /// (the `detached/` fold) is staged in the TABLE namespace — NOT in a spurious + /// `/detached` namespace produced by splitting intended_ref on the last '/'. Absent ⇒ fall + /// back to splitting intended_ref on the last '/' (the diagnostic-only path used by Core tests). + std::optional intended_namespace; + ProvenanceOp op = ProvenanceOp::Other; +}; + +class PartWriteTxn; +using PartWriteTxnPtr = std::shared_ptr; +class Gc; +class Pool; +using PoolPtr = std::shared_ptr; + +/// One listed key `Pool::listNamespaces` could not attribute to a namespace, with the refusal message +/// that stopped it. Behind Stage B's format bump the only such key is a ref object or a namespace file +/// that names no LIFE (the un-incarnated Stage A shape), which the `Layout` parsers refuse by name. +struct UnattributableNamespaceKey +{ + String key; + String reason; + + bool operator==(const UnattributableNamespaceKey &) const = default; +}; + +/// What one `Pool::listNamespaces` enumeration observed: the namespaces it attributed, and the keys it +/// could not attribute to any namespace at all. +/// +/// The two halves are separate because a short `namespaces` list is not the same fact as a clean one, +/// and the enumeration is not the place to decide what the difference means. A namespace disappears +/// from `namespaces` only if EVERY key that would have named it is in `skipped`, since attribution is +/// per key -- but "only if" is not "never", and each consumer's stakes differ: a browse probe can +/// answer conservatively, while a caller that RETIRES a slot on the strength of an empty list cannot +/// treat an incomplete universe as a drained one. +struct NamespaceListing +{ + std::vector namespaces; + std::vector skipped; +}; + +/// The façade for one content-addressed pool. `open` first validates the backend's conditional-write +/// capabilities and the durable pool metadata, and refuses to mount when either check fails. A +/// writable instance owns the mount lease and write fence; the read path uses immutable manifests +/// and does not participate in GC token ownership. The façade delegates plain-object access, +/// manifest reads, ref-log mutation, and mount lifecycle to the corresponding member components. +class Pool : public std::enable_shared_from_this +{ + /// PartWriteTxn/Gc reach the ref-log lane only through Pool's PUBLIC surface now (the ref subsystem moved + /// to the `ref_ledger` member): PartWriteTxn's staging PUTs go through `stagingPutIfAbsent`/ + /// `stagingConditionalCreate` (which encapsulate the controller call + fence), its ref mutations + /// through the public `appendRefOps` delegate; Gc uses the public + /// `wedgedRefLaneCount`. No `friend` needed -- both prior friendships were removed when the + /// ref-ledger became a member component. + +public: + /// Construct a pool after backend capabilities and durable pool metadata have been validated. + /// For a writable configuration, `open` also claims the configured mount before returning; a + /// read-only configuration returns an instance that performs no mutating startup operations. + static PoolPtr open(BackendPtr backend, PoolConfig config); + /// Admin writer mount of the VICTIM `server_root_id`, for `SYSTEM CAS DROP POOL + /// MEMBER`. Impersonates the victim's + /// owner uuid (`readOwnerUuid`, or -- when the owner anchor itself is missing -- recovered from a + /// lingering mount lease) and mounts writable under `MountClaimPolicy::NoWait`: a live victim + /// lease is an immediate `ABORTED` refusal (no wait-and-observe, no FORCE variant), unlike the + /// bounded reclaim wait a normal `open` pays. Throws `BAD_ARGUMENTS` when there is nothing to + /// decommission (no owner anchor and no mount lease for `victim_srid`). + static PoolPtr openForDecommission(BackendPtr backend, PoolConfig config, const String & victim_srid); + /// Stop the mount-renewal and remount activity, drain ref-log work, and release the owned + /// backend-facing components in their dependency order. Destruction is also the clean-farewell + /// path for a writable mount, so it must complete before the owning backend is released. + ~Pool(); + + /// ---- per-server watermark surface ---- + /// process_epoch: random nonzero per Pool (process). GC checks epoch EQUALITY, never ordering. + uint64_t epoch() const { return mount_runtime.epoch(); } + /// The durable-monotone writer_epoch allocated at writable open. On a + /// writable Pool this is the value bridged into `process_epoch` (so the watermark + the manifest + /// manifest ref carries it); on a read-only open the random `process_epoch` is unchanged and + /// no durable epoch is allocated. A self-remount re-establishes this to the fresh incarnation's + /// writer_epoch (kept equal to `liveWriterEpoch`). The epoch-aware sweep reads this value. + uint64_t writerEpoch() const { return mount_runtime.writerEpoch(); } + /// The GC floor: the oldest in-flight build_seq, or next_build_seq when no build is active (so a + /// quiescent server's watermark floor advances to the next-to-be-allocated seq). Locks builds_mutex. + uint64_t minActive(); + /// Test/assertion accessor for the next-to-allocate build_seq under the lock. + uint64_t peekNextBuildSeq(); + /// Renew the merged heartbeat once (bump seq, refresh min_active from the live callback, stamp a + /// fresh expires_at_ms). The build-watermark floor rides this beat. In production this is driven by + /// the background renewer (background_watermark). + void renewWatermarkOnce(); + + /// ---- local write fence ---- + /// A purely local, in-memory check — NEVER a per-write S3 read. True iff the fence has not latched + /// `lost` and the monotonic deadline has not passed. Permissive until armed: a Pool that has not + /// armed the fence (the default deadline is steady_clock::time_point::max()) always allows mutations. + bool mayMutate() const; + /// Latch the fence to lost (once lost, stays lost). Called by the renewer on a superseded + /// or foreign observation; the gated mutate chokepoints then fail closed. + void tripMountLost(); + /// Refresh the write-fence deadline (a CLOCK_BOOTTIME-milliseconds instant; release). + /// keeper renew calls this on success. + void setMountDeadline(uint64_t deadline_boot_ms); + /// Arm the fence at startup: set (uuid, epoch, deadline), clear `lost`. + void armMountFence(UInt128 server_uuid, uint64_t writer_epoch, uint64_t deadline_boot_ms); + void setArmMountFenceInterpositionHookForTest(std::function hook) + { + mount_runtime.setArmMountFenceInterpositionHookForTest(std::move(hook)); + } + /// The fence clock: CLOCK_BOOTTIME in milliseconds (includes VM-suspend time, unlike + /// CLOCK_MONOTONIC — see `MountFence`). Consults the injected `config.boot_ms_fn` if set (tests), + /// otherwise `bootMs`. + uint64_t bootMsNow() const; + /// The real boot clock: CLOCK_BOOTTIME in milliseconds. Static so tests can compose it. + static uint64_t bootMs(); + + /// ---- fence-generation admission (rev.7 [C2]/[D1]; owned by `mount_runtime`) ---- + /// Bumped on every `tripMountLost`/`armMountFence`. Forwarders used directly by the S3-native + /// staging-buffer finalize (`ContentAddressedTransaction::writeFile`) -- the durable-effect site + /// outside `CasPlainObjects` that needs to capture-then-recheck a fence-generation token across an + /// async, potentially long-running upload. `CasPlainObjects` reaches the same primitives via + /// injected callbacks (see its own constructor). + uint64_t fenceGeneration() const { return mount_runtime.fenceGeneration(); } + /// Throws the typed transient refusal (`throwCasTransientUnavailable`) unless the fence is currently + /// held AND still at `admitted_generation`. + void checkFenceOrThrow(uint64_t admitted_generation) const { mount_runtime.checkFenceOrThrow(admitted_generation); } + + /// ---- pool lifecycle condition (rev.7 §1; owned by `mount_runtime`) ---- + /// Atomic read of the current lifecycle condition. Thin forwarder; safe to call from any thread. + PoolLifecycle lifecycle() const { return mount_runtime.lifecycle(); } + /// Whether the pool has reached one of the two fully-terminal `Vanished` values + /// (`VanishedReplaced` / `VanishedForgotten`). + bool isVanished() const { return mount_runtime.isVanished(); } + /// Whether the terminal-intent latch is published — a natural `enterVanished`, OR FORGET's early + /// (spec §5 step 1) `publishVanishedIntent`, and NEVER `IdentityLost` ([C1]). See + /// `CasMountRuntime::vanishedIntentPublished`. The GC scheduler consults this ALONGSIDE `isVanished()` + /// to self-exit its loops the instant the pool is (being driven) terminal, at the earliest signal. + bool vanishedIntentPublished() const { return mount_runtime.vanishedIntentPublished(); } + /// The store()-class lifecycle gate: throws the typed `INVALID_STATE` error, whose message names the + /// terminal sub-state, when the pool has entered `IdentityLost` or any `Vanished` state; returns + /// silently while `Live`/`TransientNotLive`. This is the minimal "nothing silently proceeds" hook the + /// metadata storage's `poolAccess()` calls after its null-pool (`throwStorageNotStarted`) check. The + /// FULL six-class operation gate — which additionally throws in the transient state and answers + /// truth-absent on removes/enumeration — is `checkOpAdmitted`; this covers only the terminal states. + void throwIfLifecycleTerminal() const; + + /// A non-gated, I/O-free lifecycle snapshot for `system.cas_mounts` (spec §7, + /// Factory class). Reads only the runtime's atomics — NO backend op — so it is truthful in EVERY + /// state, including the terminal ones the store()-class surface refuses. `detail` is the same [D5] + /// reason text `throwIfLifecycleTerminal` throws (empty while `Live`/`TransientNotLive`), which spec §1 + /// requires appear verbatim in the snapshot; `since` is the wall-clock second the current non-`Live` + /// state was entered (0 while `Live`). The metadata-storage layer maps `lifecycle` to the operator + /// vocabulary and derives the enum-clean sub-state word separately (see `CasLifecycleSnapshot`). + struct LifecycleSnapshot + { + PoolLifecycle lifecycle = PoolLifecycle::Live; + String detail; + time_t since = 0; + }; + LifecycleSnapshot lifecycleSnapshot() const; + + /// `SYSTEM CAS FORGET` — the operator force-Vanish (spec §5). Drives THIS pool to + /// `Vanished(forgotten)` with the fence-first protocol, node-locally, regardless of the current + /// lifecycle (it works precisely on a NOT-live disk — a stuck transient/`IdentityLost` pool). In order: + /// (1) publish the terminal-intent latch FIRST (so the remount loop / keeper callback bail at their + /// next step boundary, bounding the joins below); (2) trip the local fence (the deliberate + /// decommission act, allowed on a live disk); (3+4) stop the GC scheduler via `stop_and_join_gc` — + /// injected because the scheduler is owned above the Pool, a no-op in contexts that run none — and stop + /// + join the self-remount thread; (5) drain the ref lanes (bounded) and retire the keeper WITHOUT an + /// unearned clean farewell (the lease expires by observation unless the lanes provably drained); then + /// (6) publish `Vanished(forgotten)` carrying `reason` (the [D5] message with the operator's decommission + /// timestamp). Idempotent: an already-`Vanished` pool returns immediately (first terminal transition + /// wins). MUST run on the admin/query thread, never a pool (remount/GC) thread — the joins would + /// otherwise self-deadlock (hazard C6). + void forgetDisk(const std::function & stop_and_join_gc, const String & reason); + + /// Test seam: force the pool lifecycle condition directly to `lc` (see + /// `CasMountRuntime::setLifecycleForTest`). Lets the operation-gate tests pin each class x state cell + /// on a metadata-storage-owned pool without driving a full remount/erase sequence. Never used in + /// production. + void setLifecycleForTest(PoolLifecycle lc) { mount_runtime.setLifecycleForTest(lc); } + + /// Test seam: publish the terminal-intent latch WITHOUT settling a terminal state (spec §5 step 1 of + /// FORGET), so a test can exercise the "FORGET intent published, state still pre-terminal" window — the + /// step-0 remount-observer bail (M1) and the GC scheduler's earliest-signal self-exit (C1). Never used + /// in production; FORGET reaches `publishVanishedIntent` through `forgetDisk`. + void publishVanishedIntentForTest() { mount_runtime.publishVanishedIntent(); } + + /// ---- write side ---- + PartWriteTxnPtr beginPartWrite(PartWriteInfo info); /// W-HEARTBEAT durable before return + /// Remove a build_seq from the active set; idempotent (safe from publish/abandon/dtor). Public + /// PartWriteTxn-facing surface: a `PartWriteTxn` retires its own seq on finalize/abandon/dtor (previously reached + /// via `friend class PartWriteTxn`, removed when the ref-ledger became a member component. + void retireBuildSeq(uint64_t seq); + + /// Transfer a destroyed transaction's unresolved precommit-release duty to the mount. The build + /// sequence remains active until a later mutation resolves the namespace's every-attempt wedge and + /// proves the exact precommit absent or appends its exact removal. `noexcept`: a transaction + /// destructor must fail closed by retaining the active build, never terminate while trying to + /// allocate queue bookkeeping. + void enqueueWriterCleanupDuty( + const RootNamespace & ns, const String & ref_name, const ManifestRef & manifest, uint64_t build_seq) noexcept; + + /// ---- read side ---- + /// `audit` defaults to `Emit` so every existing caller keeps emitting `RefResolve` unchanged; see + /// `ResolveAudit`'s doc comment (`CasRefLedger.h`) for the one `Deferred` call site. + std::optional resolveRef(const RootNamespace & ns, const String & ref_name, bool allow_stale = false, + ResolveAudit audit = ResolveAudit::Emit); + /// Gate 1 of the relink confirm -- a thin forward to the ref ledger, whose declaration carries the + /// rules (`CasRefLedger::confirmExactRef`). Read-only and object-store-I/O-free by contract. + ConfirmAnswer confirmExactRef(const RootNamespace & ns, const String & ref_name, + const ManifestRef & manifest_ref) const + { + return ref_ledger.confirmExactRef(ns, ref_name, manifest_ref); + } + /// Read the single immutable part manifest named by `id`. Derives the key via CasLayout::manifestKey, + /// decodes the body, and fails CLOSED: a committed ref naming a missing body throws FILE_DOESNT_EXIST + /// (INV-NO-DANGLE surfaced on the read path); a body whose `ref` ≠ id.ref (refMatchesBody) or whose + /// `root_namespace_id` ≠ id.root_namespace (manifestNamespaceMatches) throws CORRUPTED_DATA — the + /// ref is addressing the wrong object, or a cross-namespace dangle. Token-gated decode cache below. + PartManifest readManifest(const ManifestId & id); + /// Identical to `readManifest` (same mandatory HEAD, same fail-closed validation, same decode + /// cache) but returns the SHARED immutable decode the manifest cache holds — no per-call copy. + /// The wiring read path uses this variant. + std::shared_ptr readManifestShared(const ManifestId & id); + BlobLocation locate(const ManifestEntry & entry) const; /// Blob placement only + std::map listRefs(const RootNamespace & ns); + /// Pure existence probe: whether any committed ref name starts with `prefix`, without + /// materializing `listRefs`'s full map. Empty `prefix` means "any ref at all". + bool hasAnyRefWithPrefix(const RootNamespace & ns, std::string_view prefix); + /// Catalog-authoritative namespaces with the given logical prefix, returned unordered. + /// + /// The enumeration REPORTS the keys it could not attribute and DECIDES NOTHING about them: it + /// neither aborts nor silently drops them, because the right answer differs per consumer and only + /// the consumer knows its own stakes. See `NamespaceListing`. + NamespaceListing listNamespaces(const String & prefix); + + /// Scoped LIST of the mirrored subtree: the distinct next-path-segment names under + /// `roots/` (a loose LIST used by browse only; callers re-check `listRefs`/`getFileSize` + /// before showing an entry). Not authoritative — logical discovery uses the ref catalog. `prefix` + /// is a server-relative or shadow-relative path ending in '/'. + std::vector listMirroredChildren(const String & prefix); + + /// ---- ref lifecycle ---- + void dropRef(const RootNamespace & ns, const String & ref_name); /// one owner_transition removal txn + void updateRefPublishedAt(const RootNamespace & ns, const String & ref_name, + std::function mutator); /// one set_published_at txn + /// The catalog transition to `Removing` happens first, then one ref-log transaction naming every + /// owner's exact removal followed by `remove_namespace`. Performs no physical deletion: GC records + /// folded terminal evidence and later removes the exact catalog row, while the perpetual janitor + /// reclaims dead-life stream, checkpoint, and namespace-file bytes independently. + DropNamespaceStats dropNamespace(const RootNamespace & ns); + /// Decommission-only exact-life overload; never re-resolves by namespace name. + DropNamespaceStats dropNamespace(const NamespaceLifeId & life); + + /// The catalog life this namespace's objects are keyed under, for a WRITER, and the only resolution + /// that CREATES one: minted if the catalog names none (a namespace's first namespace file births it + /// exactly as its first ref op would). Resolved once per table-open and cached, so this is not a + /// per-operation catalog request. + NamespaceLifeId namespaceLife(const RootNamespace & ns); + + /// The life a READER or a REMOVER of this namespace's files must use, or `nullopt` when it has no + /// readable files -- a never-created namespace, one mid-creation and a dropped table all answer + /// alike. NEVER creates a namespace: an uncataloged one is answered from a catalog-only lookup that + /// writes nothing, so a probe or an `if_exists` unlink against a table that was never opened cannot + /// admit an entry into the pool-wide catalog. Replaces the older "is it removed?" predicate at every + /// namespace-file read: the life and the readability come from one observation, so a reader cannot + /// pair one with the other's stale answer, and an unreadable namespace yields no life to read with + /// rather than a wrong one. See `CasRefLedger`'s declaration. + std::optional namespaceFilesLifeIfReadable(const RootNamespace & ns); + + /// Thin forward to `CasRefLedger::namespaceStillLogicallyPresent`, whose declaration carries the + /// state matrix. The sole caller is `ContentAddressedMetadataStorage::existsDirectory`'s + /// `DirShape::TableDir` case. + bool namespaceStillLogicallyPresent(const RootNamespace & ns); + + /// GC callback after a proved exact catalog deletion; see `CasRefLedger` for the in-place cached + /// runtime invalidation contract. + void invalidateRemovedCatalogLife(const NamespaceLifeId & life); + + /// Reconciles cached removal-closed ref runtimes against a complete catalog cut. + void reconcileRefCatalogCut(const CasRefCatalog::Snapshot & catalog_cut); + + /// ==== writer ref-log append lane ==== + /// + /// The ONE entry point every ref mutation funnels through -- Pool's own dropRef/updateRefPublishedAt + /// above, and (as a friend) PartWriteTxn's precommitAdd/promote/abandon. This is the SOLE ref-persistence + /// lane now: the legacy per-(ns,shard) mutable manifest format was removed once GC/sweep/fsck/inspect + /// were rewired onto the snapshot+log ref protocol. + /// + /// `build_ops(state)` is invoked from the per-namespace flush leader with the table's CURRENT cached + /// state (reflecting every earlier item of the SAME batch already applied) -- exactly the atomicity + /// the old per-shard closure got from running inside the shard's own CAS loop. It may perform + /// arbitrary caller-side I/O (PartWriteTxn's blob revalidation) and throw to reject ONLY this item; a + /// LOGICAL_ERROR/ABORTED/etc it throws propagates to the item's own caller without touching any + /// other queued item. It returns the ops this call contributes to the batch's one transaction. + /// `scope` reuses `MutationScope` (Ref(name) may co-batch; WholeShard runs solo -- used here for + /// `namespace_birth`, which the flush forces automatically whenever the cached state is not `Live`). + /// + /// Wedge semantics: at most one unresolved `PUT` per table. An + /// `Unresolved` outcome wedges this namespace's lane -- no later id is allocated until that SAME + /// (key, bytes) reaches a conclusive outcome. There are exactly three, and the middle one is the + /// reason a wedge is no longer a one-way door: + /// it resolves DURABLE -- either an earlier attempt landed, or the bounded retry's own + /// conditional create lands it now -- and is applied to the cache before the next id; + /// a successor's `EpochSeal` occupies the key, which PROVES our bytes never landed and never can: + /// the wedge clears, its callers fail permanently (they were never acknowledged), and the lane + /// resumes only under a later writer epoch; + /// or the process unmounts. + /// Every item in the failing batch receives the SAME uncertainty exception (`NETWORK_ERROR`, the + /// retry-later class); items already wedged from an EARLIER flush are retried by the NEXT call into + /// this namespace's queue -- at most one bounded attempt per flush, under the fence generation the + /// wedge was ADMITTED under, never the current one. + /// + /// `skip_stale_precommit_sweep`: + /// suppresses the hoisted `maybeSweepStalePrecommits` call below for THIS call only. Set ONLY by + /// `dropNamespace`'s own removal call: that call's `build_ops` already names every current precommit + /// binding (stale or not) for removal via `RemoveNamespace`, so the ordinary maintenance sweep is + /// redundant there -- and, left enabled, would race it: the sweep runs FIRST (hoisted at this + /// function's top) and reclaims any epoch-stale binding in its OWN separate transaction, so + /// `dropNamespace`'s later `build_ops` would see it already gone and undercount + /// `DropNamespaceStats::precommits`. No other caller passes `true` -- the sweep's behavior for + /// ordinary writers is unchanged. + RefTxnId appendRefOps(const RootNamespace & ns, MutationScope scope, + std::function(const RefTableState &)> build_ops, + RootMutationOrigin origin, RootMutationKind kind, + bool skip_stale_precommit_sweep = false); + + /// the synchronous core of one publish attempt -- copies + /// the live `RefTableState` ONCE under `state_mutex` (candidate `X` = the state's `greatest_applied` + /// at that instant, no replay), encodes it, and `putIfAbsentControlled`s it off the lock. Returns true iff + /// a NEW snapshot was confirmed durable this call (false covers "nothing eligible yet", "nothing + /// new to cover", and every non-Committed outcome -- all harmless per the Failure Handling table: + /// "Snapshot create fails: keep all logs; writer recovery remains unchanged"). Public so tests can + /// drive one attempt deterministically without depending on the background dispatch's timing; + /// production reaches it only through `maybeScheduleSnapshotPublish`. + bool tryPublishSnapshotAndAdvanceCheckpointOnce(const RootNamespace & ns); + + /// ---- verbatim namespace files (format_version.txt, ...) — plain keys, never content-addressed ---- + /// Every one of them names ONE LIFE of the namespace (directive §2): the caller passes the life it + /// already holds, and none of these issues a catalog request to obtain one. + void putNamespaceFile(const NamespaceLifeId & life, const String & name, const String & bytes); + std::optional getNamespaceFile(const NamespaceLifeId & life, const String & name); + std::vector listNamespaceFiles(const NamespaceLifeId & life); + /// Exact-token delete of one verbatim file (no-op when absent). Verbatim files are never + /// content-addressed, so a mid-life delete (a pruned mutation entry, a stale tmp) must reclaim + /// the object NOW - the reachability GC never scans them. + void removeNamespaceFile(const NamespaceLifeId & life, const String & name); + + /// ---- plain mountpoint objects ---- + /// A loose disk file (the startup write probe; anything written outside a `@cas@` archive) is a + /// plain object at its mirrored path `roots/`. No manifest, no journal, no dedup. GC never + /// scans these (it deletes only content and folds only registered namespaces); they are owned by + /// their path and removed only by `removeMountpointObject`. + void putMountpointObject(const String & key, const String & bytes); + std::optional getMountpointObject(const String & key); + /// Existence check for a loose mountpoint object WITHOUT reading its body. Directory-safe: a HEAD + /// routes through the backend's metadata path (a directory reports as not-an-object), so probing + /// a directory-shaped pool path (e.g. `store`, system.remote_data_paths traversal) returns false + /// instead of a body read that would throw "Is a directory" (EISDIR). + bool mountpointObjectExists(const String & key); + void removeMountpointObject(const String & key); + + /// Internal surface for PartWriteTxn (same TU family; not for the wiring): + const PoolConfig & poolConfig() const { return config; } + const PoolMeta & poolMeta() const { return meta; } + const Layout & layout() const { return pool_layout; } + Backend & backend() { return *pool_backend; } + /// The owning `BackendPtr` itself (not just a reference into it): the decommission slot-retirement + /// decommission step (`CasDecommission.cpp`) must keep the backend alive across `admin.reset()` -- the graceful + /// close that stamps the mount's farewell -- to physically delete the control objects afterward. A + /// bare `Backend &` from `backend()` would dangle the instant the owning `Pool` is destroyed. + BackendPtr poolBackendPtr() const { return pool_backend; } + + /// Staging PUT surface for `PartWriteTxn`: both wrap the ref-ledger's retry controller + /// AND the ref-lane fence predicate, so `PartWriteTxn` reaches neither directly (the `friend` is gone). + /// Behavior-identical to the previously-inlined controller+fence at CasPartWriteTxn.cpp stageManifest / + /// uploadFromSource; thin delegates to `ref_ledger`. + CasWriteOutcome stagingPutIfAbsent(std::string_view key, std::string_view bytes, Token * out_token = nullptr); + CasCreateResult stagingConditionalCreate(std::string_view key, const std::function & attempt); + /// Same retry/fence policy as `stagingConditionalCreate`, for a mutable If-Match overwrite. + CasOverwriteResult stagingConditionalOverwrite(std::string_view key, std::string_view bytes, const Token & expected); + /// Same retry/fence policy as `stagingPutIfAbsent`, for a mutable marker where an existing + /// DIFFERENT value at the key is a normal Conflict outcome, not corruption. + CasOverwriteResult stagingPutIfAbsentMutable(std::string_view key, std::string_view bytes); + + /// CAS mixed-algo pools: + /// the NODE-LOCAL algo this Pool mints NEW content with (`PoolConfig::blob_hash_algo` -- never + /// durable pool state, see the field comment). Every write-mint site uses this, never a bare + /// `poolMeta()` field (the pool no longer records one truthful write algo). + BlobHashAlgo writeAlgo() const { return config.blob_hash_algo; } + + /// Whether `algo` is a member of the pool's `algos_used`, per this Pool's MONOTONE in-memory + /// cache (seeded from `algos_used` at open time, unioned by `refreshAdmittedAlgos` -- never + /// shrinks). This is the validation-protocol fast path: a hit needs no I/O. A miss + /// for an algo this build KNOWS about must be followed by `refreshAdmittedAlgos()` before + /// concluding the algo is genuinely not admitted (a long-running fold can overlap a later + /// registration by another node) -- callers at the manifest-read boundary do this. + bool isAlgoAdmitted(BlobHashAlgo algo) const; + + /// Re-reads `_pool_meta` and unions its CURRENT `algos_used` into the in-memory admitted-algo + /// cache (mutex-guarded; monotone -- a concurrent shrink is impossible since `algos_used` is + /// itself append-only). Returns the refreshed cache as a sorted vector, for callers that want to + /// render it (error messages, diagnostics). THE stale-cache-race fix: call this + /// on every admission-check miss, not just once at open. + std::vector refreshAdmittedAlgos(); + + /// The writer_epoch of the LIVE mount incarnation. Bumped by `tryRemountOnce` (self-remount + /// after a GC fence-out) — a `PartWriteTxn` minted under an older epoch fails closed on its next step. + uint64_t liveWriterEpoch() const { return mount_runtime.liveWriterEpoch(); } + + /// Test seam: publish a new live-incarnation writer epoch WITHOUT running a self-remount -- the + /// epoch half of what `tryRemountOnce` does alongside its fence re-arm. Lets a test drive an epoch + /// transition's WRITER-side effects (INV-2's `prev_epoch_seal` on the first append of the new epoch) + /// without the claim machinery and, deliberately, without `quiesceRefTablesForRemount`, so the + /// cached ref runtimes survive the transition and the effect under test is isolated from recovery. + void setLiveWriterEpochForTest(uint64_t writer_epoch) { mount_runtime.setLiveWriterEpoch(writer_epoch); } + + /// Self-remount after a GC fence-out (liveness counterpart of the fence-out safety rule): the + /// OLD incarnation may never write again (the keeper never re-mints), but a FRESH incarnation — + /// durable writer_epoch bump + mount reclaim + re-armed write fence — is exactly what a server + /// restart would create, so a live server may create it in place. Runs the same claim machinery as + /// `Pool::open`. Orchestration stays here; the owned mount primitives it drives (keeper swap, + /// epoch bump, fence re-arm) live on `mount_runtime`. Returns false (and changes nothing durable + /// beyond the epoch bump) when the + /// mount cannot be claimed (foreign owner / a genuinely live twin) — the caller retries. Safe to + /// call concurrently (serialized internally); also the synchronous test seam. + bool tryRemountOnce(); + + /// Test seam: drive the (private) self-remount arm/refuse path directly — in production the + /// keeper's on_lost callback calls `scheduleRemount`, otherwise reachable only via the background + /// renewer's cadence. Returns true iff a recovery thread is armed after the call. + bool scheduleRemountForTest(); + /// Test seam: how many times `scheduleRemount` has been ENTERED, counted + /// unconditionally as its very first statement -- BEFORE the `background_watermark` early-return, so + /// this increments even under the default `background_watermark = false` (no thread ever spawns; a + /// test never pays for a real self-remount attempt racing this Pool's own still-live keeper, which + /// -- confirmed while building this seam -- reliably takes 30+ seconds per call and is not something + /// a fast unit test should be driving). Positively pins that a production call site (e.g. + /// `reportImpossibleInterference`) actually invoked `scheduleRemount`, as opposed to merely observing + /// `mayMutate() == false` (which `tripMountLost` alone already accounts for). + uint64_t scheduleRemountCallCountForTest() const { return mount_runtime.scheduleRemountCallCountForTest(); } + /// Test seam: latch `remount_shutting_down` exactly as `~Pool()` does at its top, WITHOUT tearing + /// the Pool down, so a test can assert `scheduleRemount` refuses to spawn once teardown has begun. + void beginShutdownForTest(); + + + /// Known-present blob-hash cache. A HINT only — correctness never + /// depends on it: a hit just makes putBlob go HEAD-first, and a stale hit is caught by that HEAD. + /// No-ops when disabled (deduplication_cache_bytes == 0). Keyed on the full `BlobRef` pair: + /// a bare digest is never the blob identity, and the same digest value under two algos is two + /// different objects. + bool dedupCacheContains(const BlobRef & ref) const; + void dedupCacheAdd(const BlobRef & ref); + /// Test seam: retained bytes of the manifest decode cache (0 when disabled). + size_t manifestDecodeCacheBytesForTest() const { return manifest_reader.manifestDecodeCacheBytes(); } + + /// ---- event audit (system.cas_log) ---- + /// The wiring injects a sink (CasEvent -> SystemLog row) when the log is configured; null sink + /// (unit tests, log disabled) makes emitEvent a no-op single branch. PartWriteTxn/Gc reach this via + /// their owning Pool. `reason`/`detail` on the event carry the decision's full rationale. + /// Intended only for pre-open wiring or tests with no active mount thread; later installation races emitters. + /// Every component that emits (this `Pool`, the ref ledger, the manifest reader, the mount + /// renewer) holds a reference to `event_sink_`, so routing that ONE `std::function` through the + /// single `event_dispatcher_` funnels every emitter into serialized, reentrancy-safe delivery + /// (stage-1 §1, Task 2). The forwarder is installed only when a real sink is present so + /// `hasEventSink`/`event_sink_` stays a truthful "delivery enabled" predicate and the disabled hot + /// path still skips constructing events entirely. + void setEventSink(CasEventSink sink) + { + event_dispatcher_.setSink(std::move(sink)); + if (event_dispatcher_.hasSink()) + event_sink_ = [this](CasEvent e) { event_dispatcher_.emit(std::move(e)); }; + else + event_sink_ = {}; + } + /// Rvalue-only: forces every call site to `std::move` its (dead-after) `CasEvent` local, so a + /// site a future edit forgets to update is a COMPILE ERROR here rather than a silent deep copy. + void emitEvent(CasEvent && e) const { if (event_sink_) event_sink_(std::move(e)); } + /// Cheap predicate so query-frequency hooks can skip constructing the CasEvent (+ its detail map) + /// entirely when the log is disabled (sink null) — a true no-op on the production hot path. + bool hasEventSink() const noexcept { return static_cast(event_sink_); } + + /// Read the current GC round from `gc/state`. Returns 0 when `gc/state` is absent (pool + /// never GC'd). Best-effort: its one remaining caller is `tryRemountOnce`'s MountRemount audit + /// event, which reports round 0 on any read failure rather than let the error escalate. + uint64_t currentGcRound() const; + +private: + + /// Construct the in-memory façade from validated backend, configuration, and pool metadata. + /// `open` performs the checks and then moves these values here so no partially validated pool is + /// exposed to callers. + Pool(BackendPtr backend_, PoolConfig config_, PoolMeta meta_); + + /// Mount-claim policy for `mountWritable`. + enum class MountClaimPolicy : uint8_t + { + WaitForExpiry, /// normal server open — waits out a stale self-lease + NoWait, /// decommission gate — a live lease is an immediate ABORTED refusal + }; + + /// The writable-mount startup tail shared by `open` and `openForDecommission`: owner claim → + /// writer_epoch → mount claim (+fence-recovery loop) → `MountLeaseKeeper` start → watermark + /// anchor. `our_uuid` is the identity to mount as -- `config.server_id` for a normal open, the + /// victim's owner uuid for decommission (impersonation). `policy` changes only what happens when + /// the mount claim does not resolve `Claimed`/`FencedSelf`: `WaitForExpiry` observes a stale- + /// looking lease and refuses (`mountDoubleStartMessage`) only once it proves genuinely live; + /// `NoWait` refuses immediately, with no observation wait. + static void mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy policy); + + /// The single serialized, reentrancy-safe event funnel (Task 2). Declared BEFORE `event_sink_` so + /// it constructs first and destructs last: the forwarder stored in `event_sink_` references it, and + /// reverse-order destruction retires the forwarder before the dispatcher it captures. + EventDispatcher event_dispatcher_; + /// Null means delivery is disabled and `emitEvent` is a no-op. When a real sink is installed this + /// holds a thin forwarder into `event_dispatcher_` (set by `setEventSink`); every other component + /// references this member, so all emitters share the one dispatcher. + CasEventSink event_sink_; + + /// ==== ref-ledger callbacks that stay on Pool (thin delegates onto `mount_runtime`) ==== The whole + /// ref-log / ref-table subsystem moved to the `ref_ledger` member (Pool/CasRefLedger.h) and the mount/ + /// watermark/build-registry state to `mount_runtime` (Pool/CasMountRuntime.h); these remain here + /// because the ledger is injected with them as callbacks (`fence_ok_fn` / `cancel_inflight_builds` / + /// `on_impossible_interference`) at construction and they bind to `Pool`. + + /// Delegate to `mount_runtime`: the build registry (`inflight_builds`/`builds_mutex`) moved there. + /// Cancel every in-flight build targeting `ns` once its removal transaction is durable; this + /// cancels local builds. Injected into `ref_ledger` as the + /// `cancel_inflight_builds` callback. + void cancelInflightBuildsForNamespace(const RootNamespace & ns); + + /// Delegate to `mount_runtime`: the write fence moved there. pre-attempt fence check: extends + /// `mayMutate` with the REMAINING budget check -- an attempt is not even started unless there is + /// enough of the mount lease left for one more attempt_timeout plus the lease safety margin. Passed + /// as `fence_ok` to every `CasRequestController` call the ref-log writer path makes. + bool refAppendFenceOk() const; + + /// incidental-detection reaction for a foreign-interference + /// anomaly -- a signal that arrives on an operation the writer already performs (never a dedicated + /// probe) and that is impossible under legitimate single-writer operation once the mount lease + /// makes `key` exclusively ours: foreign bytes observed at our own wedge key, or the wedge hard + /// contract itself violated at new-id-allocation time. LOG_ERROR with full context, emit a + /// `ForeignInterference` CasEvent, then fence this mount closed and arm the SAME bounded + /// self-remount a foreign/superseded lease renewal already drives (`tripMountLost`/ + /// `scheduleRemount` -- see the keeper's `on_lost` callback). Diagnosis is strictly off the + /// critical path: ONE background GET of `key` (best-effort, single attempt), decoded as far as its + /// ref-log header parses, logged -- never blocking or throwing on the caller's thread. Does NOT + /// itself throw: every call site raises its OWN `LOGICAL_ERROR` immediately after this returns, so + /// the message can name the specific contract that broke. + void reportImpossibleInterference(const String & key, const String & reason, + const std::optional & offending_ns = {}); + + +public: + /// Test seams: observe resident recovery/wedge state without a private-member friend hack. These + /// observers never resolve a name, recover a table, or materialize a runtime. + uint64_t refRecoveryRestartsForTest(const RootNamespace & ns); + bool refLaneWedgedForTest(const RootNamespace & ns); + /// (I1) The object key of the current wedge for `ns`, or empty when the lane is not wedged -- lets a + /// test land a DIFFERENT object at the exact wedged key to exercise resolve-time CORRUPTED_DATA. + String wedgedKeyForTest(const RootNamespace & ns); + /// test seam: force this table's wedge to a synthetic value directly under `state_mutex`, + /// bypassing every production trigger. The ONLY way to construct the provably-unreachable state the + /// release-mode wedge-contract guard in `flushRefBatch` defends against (a wedge still present at + /// the new-id-allocation point) -- combine with `setRefPreCarveHookForTest` to install it AFTER the + /// top-of-flush wedge-resolution check has already run clean but BEFORE the batch is carved. + void forceWedgeForTest(const RootNamespace & ns, uint64_t writer_epoch, uint64_t ref_sequence, + const String & key, const String & bytes, + std::optional admitted_generation = std::nullopt); + /// test seam: the fence generation the current wedge was ADMITTED under (0 when not wedged) -- the + /// value every later retry of that wedge is gated on. See `CasRefLedger::RefAppendAttempt`. + uint64_t wedgedAdmittedGenerationForTest(const RootNamespace & ns); + /// test seams: this table's `prev_epoch_seal` source -- the seal that closed its previous writer + /// epoch, `nullopt` at genesis. The setter stands in for the recovery CAS-walk that produces it. + std::optional lastEpochSealForTest(const RootNamespace & ns); + void setLastEpochSealForTest(const RootNamespace & ns, const std::optional & seal); + /// Test seam: this table's append lane state. + RefLaneState laneStateForTest(const RootNamespace & ns); + + /// Whether this resident table still owes a stale-precommit sweep (armed by recovery; re-armed by a + /// failed attempt; cleared permanently only by a verified-clean sweep). + bool needsStalePrecommitSweepForTest(const RootNamespace & ns); + + /// Number of ref-append lanes currently wedged (an uncertain PUT exhausted its retry budget and + /// the lane blocks until the same key resolves durable or is conclusively rejected). Per-disk GC + /// health for system.cas_mounts. O(live tables); takes each runtime state lock. + size_t wedgedRefLaneCount(); + + /// test seam: blocks until every background snapshot-publish attempt dispatched so far for + /// `ns` has settled. Needed only by tests that exercise the REAL background dispatch (production + /// concurrency); tests that just want deterministic publish-logic coverage call + /// `tryPublishSnapshotAndAdvanceCheckpointOnce` directly instead. + void waitForSnapshotPublishSettleForTest(const RootNamespace & ns); + + /// test seam: the count of in-flight background snapshot-publish attempts for resident `ns` (the + /// single-in-flight gate holds this at <= 1). + int pendingSnapshotPublishesForTest(const RootNamespace & ns); + + /// test seam: the id of the newest snapshot this resident runtime has confirmed durable (recovered + /// or published), or `nullopt` if none. + std::optional newestPublishedSnapshotIdForTest(const RootNamespace & ns); + + /// test seam: whether `ns` has a RECOVERED cached runtime, WITHOUT forcing a recovery to find out. + bool refTableRecoveredForTest(const RootNamespace & ns); + /// test seam: whether the self-remount barrier's cancellation request is visible for `ns`. + bool refRecoveryCancelRequestedForTest(const RootNamespace & ns); + + /// The self-remount's cancel-or-join barrier over in-flight ref-table recoveries (spec §3). + /// `tryRemountOnce` runs it immediately before quiescing the tables and re-arming the mount fence; + /// exposed so the barrier itself can be driven directly by a test. + void cancelRefRecoveriesAndAwaitQuiescence(); + + /// test seam: count of applied txns retained above `newestPublishedSnapshotIdForTest` (the + /// tail a snapshot candidate would replay from). + size_t tailSinceSnapshotCountForTest(const RootNamespace & ns); + size_t committedOverlayEntriesForTest(const RootNamespace & ns); + + /// test seam: the ledger's live precommit view for `ns` (see + /// `CasRefLedger::livePrecommitsForTest`) -- the durable-but-unpromoted owner bindings, which is + /// what an abandoned/aborted build must leave empty. + std::set> livePrecommitsForTest(const RootNamespace & ns); + + /// test seam: whether any writer cleanup duty is still owed (see `writerCleanupDutiesPending`) -- + /// the direct signal that a settlement failure retained its duty for retry, independent of any + /// build-floor side effect that could have the same shape for an unrelated reason. + bool writerCleanupDutiesPendingForTest() const { return writerCleanupDutiesPending(); } + + /// Test-only hook: called by `flushRefBatch` + /// right before it carves a batch, i.e. AFTER the table is already recovered -- the one otherwise + /// untestable timing window `BlockingGetBackend`-style backend tricks cannot reach, since a warm + /// flush performs no I/O between becoming leader and carving. A test blocks here to let a second + /// caller's item join `rt->pending` before the carve, forcing deterministic co-batching. + void setRefPreCarveHookForTest(std::function hook) { ref_ledger.setRefPreCarveHookForTest(std::move(hook)); } + + /// Test-only: pre-tenure fault seam for the append-lane leadership acquisition; forwards to + /// `CasRefLedger::setRefPreTenureHookForTest` (see it for the baton-safety contract). + void setRefPreTenureHookForTest(std::function hook) { ref_ledger.setRefPreTenureHookForTest(std::move(hook)); } + void setAppendAfterRuntimeCaptureHookForTest(std::function hook) + { + ref_ledger.setAppendAfterRuntimeCaptureHookForTest(std::move(hook)); + } + void setReadBeforeStateLockHookForTest(std::function hook) + { + ref_ledger.setReadBeforeStateLockHookForTest(std::move(hook)); + } + void setReadableCatalogAfterObservationHookForTest(std::function hook) + { + ref_ledger.setReadableCatalogAfterObservationHookForTest(std::move(hook)); + } + void setWedgeBeforeSlotOccupyHookForTest(std::function hook) + { + ref_ledger.setWedgeBeforeSlotOccupyHookForTest(std::move(hook)); + } + void setNamespacePresenceProbeAfterFirstReadHookForTest(std::function hook) + { + ref_ledger.setNamespacePresenceProbeAfterFirstReadHookForTest(std::move(hook)); + } + void setNamespacePresenceProbeAfterTerminalProvenHookForTest(std::function hook) + { + ref_ledger.setNamespacePresenceProbeAfterTerminalProvenHookForTest(std::move(hook)); + } + uint64_t recoveryInstallCountForTest() const { return ref_ledger.recoveryInstallCountForTest(); } + + /// Test-only: fault seam for the ref-flush two-phase carve/validation protocol; forwards to + /// `CasRefLedger::setCarveHookForTest` (see it for the phase-point contract). + void setCarveHookForTest(std::function hook) + { + ref_ledger.setCarveHookForTest(std::move(hook)); + } + + /// Test-only: negative control for the post-durable install region; forwards to + /// `CasRefLedger::setInstallRegionProbeForTest` (see it for what an allocating probe must do). + void setInstallRegionProbeForTest(std::function probe) + { + ref_ledger.setInstallRegionProbeForTest(std::move(probe)); + } + void setSnapshotAfterCaptureHookForTest(std::function hook) + { + ref_ledger.setSnapshotAfterCaptureHookForTest(std::move(hook)); + } + void setSnapshotBeforeCkptCasHookForTest(std::function hook) + { + ref_ledger.setSnapshotBeforeCkptCasHookForTest(std::move(hook)); + } + + /// Test-only: replace the request controller's inter-attempt backoff sleep (e.g. with a no-op) — + /// for tests that drive a persistent conditional-write fault to budget exhaustion through a fully + /// wired Pool/disk and must not serve the production capped-exponential sleeps for real (see + /// `CasRequestController::setSleepFnForTest`). Call before driving traffic; empty restores the + /// real sleep. + void setCasRetrySleepForTest(std::function sleep_fn); + + /// Queue depth for the ref-append-lane tests (mirrors `shardQueuePendingForTest`): how many + /// `appendRefOps` callers are enqueued for `ns` right now. + size_t refQueuePendingForTest(const RootNamespace & ns) { return ref_ledger.refQueuePendingForTest(ns); } + + /// Test seam: whether `ns` currently has an active append-lane leader (the baton). Mirrors + /// `refQueuePendingForTest`; used to assert the baton is not stranded on a pre-tenure fault. + bool refLeaderActiveForTest(const RootNamespace & ns) { return ref_ledger.refLeaderActiveForTest(ns); } + + /// Test seam: how many concurrent `ensureRefTableRecovered` callers for `ns` are + /// PARKED right now waiting on the leader's in-flight recovery (see `RefTableRuntime:: + /// recovery_waiters_for_test`) -- lets a test `yield()`-poll for "a second caller actually reached + /// the wait" deterministically, mirroring `refQueuePendingForTest` above. + uint64_t refRecoveryWaitersForTest(const RootNamespace & ns) { return ref_ledger.refRecoveryWaitersForTest(ns); } + + /// cache-eviction test seams: how many whole ref tables are cached right now, and whether a + /// specific table's runtime is currently materialized (recovered) in the cache -- a table that was + /// evicted reports false until its next touch re-recovers it. + size_t refTablesCachedCountForTest() { return ref_ledger.refTablesCachedCountForTest(); } + bool refTableCachedForTest(const RootNamespace & ns) { return ref_ledger.refTableCachedForTest(ns); } + uint64_t refTableRuntimeIdentityForTest(const RootNamespace & ns) + { + return ref_ledger.refTableRuntimeIdentityForTest(ns); + } + uint64_t refTableRuntimeAdmittedFenceGenerationForTest(const RootNamespace & ns) + { + return ref_ledger.refTableRuntimeAdmittedFenceGenerationForTest(ns); + } + std::optional refTableLifeForTest(const RootNamespace & ns) + { + return ref_ledger.refTableLifeForTest(ns); + } + + /// Recovery-publication inventory seams (forward to `CasRefLedger`): the seeded admission budgets, + /// the recovered base snapshot's encoded body size and the tail-since-snapshot byte sum. + uint64_t refSnapshotBudgetForTest(const RootNamespace & ns) { return ref_ledger.refSnapshotBudgetForTest(ns); } + uint64_t refRemovalBudgetForTest(const RootNamespace & ns) { return ref_ledger.refRemovalBudgetForTest(ns); } + uint64_t refBaseSnapshotBytesForTest(const RootNamespace & ns) { return ref_ledger.refBaseSnapshotBytesForTest(ns); } + uint64_t refTailBytesSinceSnapshotForTest(const RootNamespace & ns) { return ref_ledger.refTailBytesSinceSnapshotForTest(ns); } +private: + struct WriterCleanupDuty + { + String ref_name; + ManifestRef manifest; + uint64_t build_seq = 0; + }; + + struct WriterCleanupQueue + { + std::deque> pending; + bool draining = false; + }; + + /// Drain `ns` before admitting its next ordinary mutation. Only one caller drains a namespace at a + /// time; concurrent callers wait so none can overtake a cleanup whose build still holds the active + /// watermark floor. The drain calls `ref_ledger` directly to avoid re-entering this Pool wrapper. + void drainWriterCleanupDuties(const RootNamespace & ns); + bool writerCleanupDutiesPending() const; + + /// The single admission seam for durable ref mutations exposed by `Pool`. Keeping drain-before-call + /// here makes a new forwarding entry point visibly choose between servicing writer cleanup and + /// deliberately bypassing it; the cleanup implementation itself uses `ref_ledger` directly. + template + decltype(auto) mutateRefsAfterWriterCleanup(const RootNamespace & ns, Mutation && mutation) + { + drainWriterCleanupDuties(ns); + return std::forward(mutation)(); + } + + BackendPtr pool_backend; + PoolConfig config; + PoolMeta meta; + + mutable std::mutex writer_cleanup_mutex; + std::condition_variable writer_cleanup_cv; + std::map writer_cleanup_queues; + /// Sticky fail-close bit for the destructor's allocation-failure path. If a duty could not enter + /// the queue, no mount teardown may claim a clean farewell even though the guarded map is empty. + std::atomic writer_cleanup_queue_failed{false}; + + /// CAS mixed-algo pools: monotone in-memory cache of `algos_used`, seeded from + /// `meta.algos_used` at open. Guards `isAlgoAdmitted`/`refreshAdmittedAlgos` -- ITS OWN mutex, + /// not `meta`'s (there is no other mutable access to `meta` post-open; this avoids taking a + /// wider lock than the cache needs). Kept sorted (a plain vector; membership is a handful of + /// entries, no need for a set). + mutable std::mutex admitted_algos_mutex; + std::vector admitted_algos; + + /// Known-present cache: a bytes-bounded LRU set of blob hashes confirmed present in the pool. + /// Value is a 1-byte presence marker; DedupWeight charges a fixed per-entry byte estimate so the + /// configured `deduplication_cache_bytes` is an honest memory ceiling. nullptr ⇔ disabled. + /// Marker stored for a blob hash known to be present. The value has no payload; the cache key is + /// the complete `BlobRef`, including its hash algorithm. + struct DedupPresent {}; + + /// Fixed memory estimate used by `CacheBase` to enforce the configured byte ceiling. It is an + /// estimate rather than an allocation measurement, but keeps cache growth bounded predictably. + struct DedupWeight + { + size_t operator()(const DedupPresent &) const { return 64; } + }; + using DeduplicationCache = CacheBase; + std::unique_ptr dedup_cache; + Layout pool_layout; + /// The plain-object surface (namespace files + loose mountpoint objects), extracted from Pool. + /// Stateless over `Backend &` + `const Layout &`; declared AFTER + /// pool_backend and pool_layout so it is constructed after (and destroyed before) both. + CasPlainObjects plain_objects; + /// The manifest read path + decode cache + locate, extracted from Pool. + /// Injected with backend/layout/meta + the event-sink reference; owns the decode cache (whose + /// synchronization is CacheBase-internal). Declared after event_sink_, pool_backend, meta, and + /// pool_layout so it is constructed after (and destroyed before) all four. + CasManifestReader manifest_reader; + /// The writer ref-log / ref-table subsystem, extracted from Pool. Owns the + /// whole-table ref cache, the append lane + wedge protocol, snapshot publication, stale-precommit + /// sweep, cache-budget eviction, the remount/shutdown drain, and the CAS retry controller -- with + /// the two ref mutexes. Declared AFTER event_sink_, pool_backend, meta, pool_layout, plain_objects + /// and manifest_reader so it is constructed after (and destroyed before) every dependency it is + /// injected with; its callbacks reach mount/watermark state now owned by `mount_runtime` (declared + /// AFTER this member), but they capture `Pool` and run only at runtime after the Pool is fully + /// constructed -- exactly as in the pre-3.5 layout, where the mount raw-members these callbacks reach + /// were also declared after `ref_ledger`. `~Pool` still calls `ref_ledger.drainRefLanesForShutdown` + /// explicitly, sequenced between `mount_runtime.stopRemountThread()` and + /// `mount_runtime.finishTeardown()` exactly as before. + CasRefLedger ref_ledger; + /// The mount / write-fence / build-watermark / self-remount runtime, extracted + /// from Pool. Owns the `MountLeaseKeeper`, the local `MountFence`, the per-server + /// build watermark (`process_epoch` + the `builds_mutex`-guarded seq/registry) and its in-flight-build + /// map, the live-incarnation `live_writer_epoch`, the unclean-epoch high-water-mark, and the + /// self-remount recovery thread (with its own thread-lifecycle locks). Injected with backend/layout + + /// the `MountConfig` slice + `server_root_id` + the event-sink reference + the pool `cas_request_budget` + /// + a `remount_attempt` callback (== `Pool::tryRemountOnce`, which STAYS on Pool: the claim/recovery + /// ORCHESTRATION drives these owned primitives). + /// + /// Declared AFTER `ref_ledger` -- preserving the pre-3.5 relative order VERBATIM (the mount raw-members + /// this component replaces all sat after `ref_ledger`), so `mount_runtime` is destroyed FIRST and + /// `ref_ledger` LAST. Both orders were proven equally safe -- `~Pool` + /// quiesces both subsystems before ANY member dtor runs (stopRemountThread -> + /// ref_ledger.drainRefLanesForShutdown -> mount_runtime.finishTeardown), and the ledger's async paths + /// pin `Pool::shared_from_this`, so no ledger->mount callback can fire during destruction in either + /// order. Both safe ⇒ this is a pure behavior-preserving relocation, so the ORIGINAL order is kept and + /// NO member-order change is introduced. Declared after event_sink_, pool_backend, config and + /// pool_layout so it is constructed after every dependency it is injected with. + CasMountRuntime mount_runtime; + + /// Serializes `tryRemountOnce` (whose claim/recovery ORCHESTRATION stays on Pool). STAYS here with + /// its guarded critical section: the self-remount thread-lifecycle locks + fence + /// atomics + build registry moved to `mount_runtime`, but the top-level remount serialization guards + /// the Pool-side orchestration, so it stays on Pool. + std::mutex remount_mutex; + + /// The single home of the [D5] per-lifecycle reason detail (spec §1) — the human-readable text that + /// names the ACTUAL sub-state, WITHOUT the `content-addressed pool '' ` prefix. Both + /// `throwIfLifecycleTerminal` (which prefixes it and throws) and the non-gated `lifecycleSnapshot` + /// (which surfaces it verbatim in the system table) read it, so the error message and the introspection + /// row can never drift. Empty for `Live`/`TransientNotLive` (no terminal detail); for + /// `VanishedForgotten` it prefers the stored `vanishedReason()` (carrying the operator's decommission + /// timestamp) and falls back to the static [D5] text when none was recorded (a forced-for-test state). + String lifecycleReasonDetail(PoolLifecycle lc) const; + + /// NOTE (M-C2): the ref-log is never trimmed here — trimming needs GC's fold state + /// (`last_folded_ref_id`, INV-JOURNAL-COVERAGE), which is GC state landing in M-C3. +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPoolMeta.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPoolMeta.cpp new file mode 100644 index 000000000000..b658d9458b12 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPoolMeta.cpp @@ -0,0 +1,168 @@ +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; + extern const int INVALID_STATE; +} +} + +namespace DB::Cas +{ + +namespace +{ + +/// Two `thread_local_rng` u64 draws composed into a 128-bit id. +UInt128 mintPoolId() +{ + const UInt128 hi = thread_local_rng(); + const UInt128 lo = thread_local_rng(); + return (hi << 64) | lo; +} + +/// Whether `config_algo` is already a registered member of `pool.algos_used`. Membership, not the +/// opt-in flag, is the steady-state check. `algos_used` is kept sorted, so this is a binary search. +bool isAlgoAdmittedIn(const PoolMeta & pool, BlobHashAlgo config_algo) +{ + const auto v = static_cast(config_algo); + return std::binary_search(pool.algos_used.begin(), pool.algos_used.end(), v); +} + +/// Renders `algos_used` as "ch128, sha256" for the refusal message below. +String joinAlgoNames(const std::vector & algos_used) +{ + String out; + for (size_t i = 0; i < algos_used.size(); ++i) + { + if (i != 0) + out += ", "; + out += blobHashAlgoName(static_cast(algos_used[i])); + } + return out; +} + +/// Fail-closed on a non-admitted algo without the opt-in flag: admission is EXPLICIT +/// opt-in; the default stays fail-closed -- a changed config alone must never silently turn a pool +/// mixed. Never touches the pool. +[[noreturn]] void throwNotAdmitted(const PoolMeta & pool, BlobHashAlgo config_algo) +{ + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "CAS pool blob_hash mismatch: pool has {{{}}}; config requests {}; set " + "1 to admit a new algo into this pool", + joinAlgoNames(pool.algos_used), blobHashAlgoName(config_algo)); +} + +/// The relaxed admission check (replaces an earlier fail-close that required the single pool algo to match): +/// `pm`/`token` are the most-recently-read `_pool_meta` state (present, decoded, valid). Already a +/// member of `algos_used` => OK, no write (steady state). Not a member and `!allow_new` => +/// `BAD_ARGUMENTS` (the pool is never touched). Not a member and `allow_new` => CAS-union `config_algo` +/// into `algos_used` (recomputed from the FRESH value on every retry -- union-only, so there is no +/// ABA) and raises `min_reader_generation` to THIS build's own floor (`G_BUILD`, `CasFormat.h`) in +/// the SAME write (first registration of a schema-3-bearing algo also raises +/// `min_reader_generation` -- a build that cannot decode schema-3 settlement state has an OLDER +/// `G_BUILD` and is correctly refused by the startup gate once a future generation bump lands here). +/// On a CAS conflict, re-read and retry the whole decision (a concurrent admitter may have unioned a +/// DIFFERENT algo, or the very one we wanted, in the meantime). +PoolMeta admitOrValidate( + Backend & backend, const String & key, PoolMeta pm, Token token, + BlobHashAlgo config_algo, bool allow_new) +{ + for (;;) + { + if (isAlgoAdmittedIn(pm, config_algo)) + return pm; + + if (!allow_new) + throwNotAdmitted(pm, config_algo); + + PoolMeta next = pm; + next.algos_used.push_back(static_cast(config_algo)); + std::sort(next.algos_used.begin(), next.algos_used.end()); + next.min_reader_generation = G_BUILD; + + const CasResult res = backend.casPut(key, encodePoolMeta(next), token); + if (res.outcome == CasOutcome::Committed) + return next; + + auto fresh = backend.get(key); + if (!fresh) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS pool meta: '{}' vanished mid-admission (conflicting write then a concurrent delete)", key); + pm = decodePoolMeta(fresh->bytes); + token = fresh->token; + /// loop: re-evaluate membership against the FRESH pm (never re-encode the stale `next`) + } +} + +} + +PoolMeta PoolMeta::createOrValidate( + Backend & backend, const Layout & layout, uint64_t blob_header_len, uint64_t gc_shards, + BlobHashAlgo blob_hash_algo, bool allow_new, bool allow_mint) +{ + /// The passed config is the caller's responsibility — reject bad values before any I/O. + validatePoolBlobHeaderLen(blob_header_len, ErrorCodes::BAD_ARGUMENTS, "pool meta"); + if (gc_shards == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "CAS pool meta: gc_shards must be >= 1"); + /// Defense against a garbage `static_cast` past the caller's own boundary: `blobHashAlgoName` + /// throws BAD_ARGUMENTS for anything `BlobHashAlgo` does not actually admit. + blobHashAlgoName(blob_hash_algo); + + const String key = layout.poolMetaKey(); + + /// Present => the pool is authoritative; ignore the passed config's blob_header_len and run the + /// flag-gated admission check rather than the old single-value fail-close. + if (auto existing = backend.get(key)) + { + PoolMeta pm = decodePoolMeta(existing->bytes); + return admitOrValidate(backend, key, std::move(pm), existing->token, blob_hash_algo, allow_new); + } + + /// Absent => mint a pool id and try to create the object with `algos_used = {blob_hash_algo}`. + /// Every pool this build creates is schema-3-shaped from birth (schemas 1/2 do not exist in this + /// build at all), so the reader-generation floor is stamped at THIS build's `G_BUILD` at + /// creation, not left at 0. + /// + /// BOOTSTRAP GATE (spec §2 [C4][D2]): minting is permitted ONLY on the verified bootstrap path. A + /// non-bootstrap caller (a read-only/observe open, `openForDecommission`) passes `allow_mint=false` + /// and fails closed here — never minting a fresh identity outside that path (an observe scan that + /// minted would poison the next writable mount's residual check). The residual EMPTINESS proof itself + /// cannot live here: it must precede ANY write (spec's "zero-write residual check FIRST, before ANY + /// probe write"), and by the time `createOrValidate` runs the capability battery has already written + /// to the prefix, so `Pool::open` runs `probePoolBootstrapResidual` up front and only then passes + /// `allow_mint=true`. `pool_prefix` is exclusively CAS-owned. + if (!allow_mint) + throw Exception(ErrorCodes::INVALID_STATE, + "CAS pool meta: _pool_meta absent — refusing to mint outside the verified bootstrap path; " + "run a writable mount (or recreate the pool)"); + + PoolMeta pm; + pm.pool_id = mintPoolId(); + pm.blob_header_len = blob_header_len; + pm.gc_shards = gc_shards; + pm.min_reader_generation = G_BUILD; + pm.algos_used = {static_cast(blob_hash_algo)}; + + if (backend.casPut(key, encodePoolMeta(pm), /*expected*/ std::nullopt).outcome == CasOutcome::Committed) + return pm; + + /// Lost the race: the winner's object MUST be present now. The loser UNIONS its algo via the SAME + /// flag-gated admission path as a reopen, instead of the old unconditional fail-close. + auto winner = backend.get(key); + if (!winner) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS pool meta: create-if-absent reported Conflict but '{}' is absent on re-read", key); + PoolMeta winner_pm = decodePoolMeta(winner->bytes); + return admitOrValidate(backend, key, std::move(winner_pm), winner->token, blob_hash_algo, allow_new); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCatalog.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCatalog.cpp new file mode 100644 index 000000000000..b9b952a7321a --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCatalog.cpp @@ -0,0 +1,654 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +namespace + +{ + +CasRefCatalog::Snapshot readOptionalForBootstrap(Backend & backend, const Layout & layout) +{ + const auto got = backend.get(layout.refCatalogKey()); + if (!got) + { + RefCatalog empty; + return CasRefCatalog::Snapshot{ + .catalog = empty, .token = std::nullopt, .life_index = CatalogLifeIndex(empty)}; + } + RefCatalog catalog = decodeRefCatalog(got->bytes); + return CasRefCatalog::Snapshot{ + .catalog = catalog, .token = got->token, .life_index = CatalogLifeIndex(catalog)}; +} + +} + +CasRefCatalog::Snapshot CasRefCatalog::read(Backend & backend, const Layout & layout) +{ + Snapshot snapshot = readOptionalForBootstrap(backend, layout); + if (!snapshot.token) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "Mandatory CAS ref catalog '{}' is absent -- refusing to interpret opaque life " + "objects as an empty ownership universe", + layout.refCatalogKey()); + return snapshot; +} + +CasRefCatalog::Snapshot CasRefCatalog::initializeEmptyForNewPool(Backend & backend, const Layout & layout) +{ + RefCatalog empty; + const String canonical_empty = encodeRefCatalog(empty); + const PutResult put = backend.putIfAbsent(layout.refCatalogKey(), canonical_empty); + if (put.outcome == PutOutcome::Done) + return Snapshot{.catalog = empty, .token = put.token, .life_index = CatalogLifeIndex(empty)}; + + /// A second opener can win after both proved the prefix empty. Decode its exact object before + /// accepting the race; conflict is never a license to continue with an assumed empty catalog or + /// arbitrary decoded body. + const auto got = backend.get(layout.refCatalogKey()); + if (!got) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref catalog '{}' disappeared after bootstrap create conflict", + layout.refCatalogKey()); + RefCatalog catalog = decodeRefCatalog(got->bytes); + if (!catalog.entries.empty() || got->bytes != canonical_empty) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref catalog '{}' conflicts with bootstrap's required canonical empty catalog", + layout.refCatalogKey()); + return Snapshot{.catalog = std::move(catalog), .token = got->token, .life_index = CatalogLifeIndex(empty)}; +} + +std::optional CasRefCatalog::lifeIfCataloged( + Backend & backend, const Layout & layout, const RootNamespace & ns) +{ + const Snapshot snap = read(backend, layout); + for (const CatalogEntry & entry : snap.catalog.entries) + if (entry.ns.string() == ns.string() && entry.state != NsState::Creating) + return snap.life_index.resolve(entry.incarnation); + return std::nullopt; +} + +std::vector CasRefCatalog::liveUniverse(Backend & backend, const Layout & layout) +{ + const Snapshot snap = read(backend, layout); + snap.life_index.throwIfAmbiguous("CAS live namespace discovery"); + std::vector universe; + universe.reserve(snap.catalog.entries.size()); + for (const CatalogEntry & entry : snap.catalog.entries) + { + if (entry.state == NsState::Creating) + continue; + universe.push_back(NamespaceLifeId::fromCatalogEntry(entry.ns, entry.incarnation)); + } + return universe; +} + +namespace +{ + +/// Live-lock brake, the same shape and for the same reason as `publishCkpt`'s/`allocateWriterEpoch`'s +/// on their own contended token-CAS singletons: the catalog is ONE object mutated by every lifecycle +/// transition of every namespace in the pool, so persistent contention is a real, not theoretical, +/// exit condition to plan for. +constexpr size_t kMaxCatalogCasAttempts = 100; + +/// Shared body of `casUpdate`/`casAdmitEntry`. `encode` turns a freshly `mutate`d candidate into the +/// bytes to write: the plain path just grammar-checks (`encodeRefCatalog`), the admitting path also +/// runs both admission predicates (`checkCatalogAdmission`) first. Retries on `Conflict` against a +/// FRESH read, exactly like `PoolMeta::admitOrValidate` -- never re-encoding the stale candidate. +RefCatalog casUpdateImpl( + Backend & backend, const Layout & layout, + const std::function & mutate, + const std::function & encode) +{ + const String key = layout.refCatalogKey(); + CasRefCatalog::Snapshot snap = CasRefCatalog::read(backend, layout); + + for (size_t attempt = 0; attempt < kMaxCatalogCasAttempts; ++attempt) + { + snap.life_index.throwIfAmbiguous("CAS ref catalog mutation"); + RefCatalog candidate = mutate(snap.catalog); + const String bytes = encode(candidate); + const CasResult res = backend.casPut(key, bytes, snap.token); + if (res.outcome == CasOutcome::Committed) + return candidate; + + snap = CasRefCatalog::read(backend, layout); + /// `read` treats authoritative absence after a conflict as corruption. Therefore no retry + /// can turn a vanished mandatory catalog into a one-update replacement authority. + } + + throwCasWriteRetryLater(fmt::format( + "CAS ref catalog '{}' did not converge after {} attempts", key, kMaxCatalogCasAttempts)); +} + +/// Thrown from inside a `casUpdate` `mutate` closure to signal a refusal that must STOP the attempt +/// rather than be treated as a `Conflict` to retry: `casUpdateImpl` propagates whatever `mutate` +/// throws straight out, uncaught, which is exactly the behavior these three need. Retrying any of them +/// against a freshly re-read catalog would just re-decide against an entry that is, by definition, no +/// longer `observed` -- token-exactness means the FIRST mismatch is final, not a reason to loop. +/// Each is caught by its own exact type right where it is thrown; deriving from `std::exception` +/// is only so the throw itself is well-formed, never so a caller catches these by base class. +struct CatalogFenceMovedMarker : std::exception {}; +struct CatalogEntryMismatchMarker : std::exception {}; +struct CatalogCreatorStillLiveMarker : std::exception {}; + +/// Two `thread_local_rng` draws composed into a `UInt128`, the same pattern already used throughout +/// this tree to mint build ids and incarnation tags (`CasPartWriteTxn.cpp`'s `mintU128`, +/// `ContentAddressedTransaction.cpp`'s `incarnation_tag`). Retried on the astronomically unlikely `0` +/// draw: unlike those callers, this value must never be zero (`CatalogEntry::incarnation == 0` is +/// always invalid -- "0 never names a life"), and this is the one mint site in that family with a +/// grammar rule to uphold. +UInt128 mintFreshIncarnation() +{ + UInt128 v = 0; + while (v == 0) + v = (static_cast(thread_local_rng()) << 64) | thread_local_rng(); + return v; +} + +/// Shared by every function below that needs "the current entry for this namespace, if any" -- +/// keeping ONE lookup rather than three independently-written `find_if`s that could drift apart on +/// what counts as a match. +std::vector::const_iterator findEntry(const RefCatalog & catalog, const RootNamespace & ns) +{ + return std::find_if(catalog.entries.begin(), catalog.entries.end(), + [&](const CatalogEntry & e) { return e.ns.string() == ns.string(); }); +} + +/// Thrown from `createNamespaceStep1`'s own `mutate` (below) when a FRESH read -- the first one, or +/// any `Conflict` retry's re-read -- already carries an entry for the namespace being admitted. Never +/// thrown by the public `casAdmitEntry`: that function keeps its documented "already-present is a +/// caller bug, let `encodeRefCatalog` abort" contract for its many single-namespace-per-catalog test +/// callers -- it has no production caller at all; `createNamespaceStep1` below duplicates its +/// admission shape rather than calling it, precisely so this recheck can be added without weakening +/// `casAdmitEntry` itself. `createNamespace` alone needs the other answer, because ITS +/// "already present" can be a sibling opener's OWN in-flight step 1 landing between createNamespace's +/// pre-check read and this loop's read -- a race the design already names and resumes through +/// `Superseded`, not a caller bug. +struct CatalogEntryAlreadyPresentMarker : std::exception {}; + +/// Fires once, synchronously, right before `createNamespaceStep1`'s own first catalog read -- i.e. +/// after `createNamespace`'s pre-check read already observed no entry. Lets a test land a sibling +/// opener's full `createNamespace` call in that exact window, driving the interleaving +/// `CatalogEntryAlreadyPresentMarker` exists to catch instead of relying on real thread scheduling. +/// Empty (no-op) in production, mirroring every other `*_hook_for_test` in this tree. +std::function create_namespace_step1_pre_read_hook_for_test; + +/// Step 1 of `createNamespace`, split out so it can recheck presence on EVERY catalog read this loop +/// performs (the first one, and any `Conflict` retry's re-read), not only the snapshot-in-time read +/// `createNamespace` itself already did before calling in. That single upfront read cannot see a +/// sibling opener's OWN step 1 landing between it and this loop's read; without the recheck here, this +/// loop would blindly insert a second row for the same namespace and let `encodeRefCatalog`'s +/// canonical-order/no-duplicate grammar check abort the process with `LOGICAL_ERROR` for what is, at +/// this call site only, an ordinary race outcome. +RefCatalog createNamespaceStep1( + Backend & backend, const Layout & layout, uint64_t gc_shards, const CatalogEntry & entry) +{ + /// Moved into a local before invoking, not called on the global directly: a hook that reassigns + /// `create_namespace_step1_pre_read_hook_for_test` from inside its own body (a test driving a + /// one-shot interleaving) would otherwise reassign the very `std::function` object whose `operator()` + /// is executing it -- undefined behavior, not merely untidy. The local copy is a distinct object the + /// hook body cannot reach. + if (create_namespace_step1_pre_read_hook_for_test) + { + std::function hook_to_run; + std::swap(hook_to_run, create_namespace_step1_pre_read_hook_for_test); + hook_to_run(); + } + + const auto mutate = [&entry](const RefCatalog & cur) -> RefCatalog + { + if (findEntry(cur, entry.ns) != cur.entries.end()) + throw CatalogEntryAlreadyPresentMarker{}; + RefCatalog next = cur; + const auto it = std::lower_bound(next.entries.begin(), next.entries.end(), entry, + [](const CatalogEntry & a, const CatalogEntry & b) { return a.ns.string() < b.ns.string(); }); + next.entries.insert(it, entry); + return next; + }; + return casUpdateImpl(backend, layout, mutate, + [&entry, gc_shards, &layout](const RefCatalog & c) + { + return checkCatalogAdmission(c, gc_shards, layout, entry.ns); + }); +} + +} + +RefCatalog CasRefCatalog::casUpdate( + Backend & backend, const Layout & layout, const std::function & mutate) +{ + const auto identity_preserving_mutate = [&](const RefCatalog & current) -> RefCatalog + { + RefCatalog candidate = mutate(current); + if (candidate.entries.size() != current.entries.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CasRefCatalog::casUpdate cannot add or delete catalog entries -- use casAdmitEntry, " + "deleteCompletedRemoving, or cancelStalledCreating"); + for (size_t i = 0; i < current.entries.size(); ++i) + { + if (candidate.entries[i].ns != current.entries[i].ns + || candidate.entries[i].incarnation != current.entries[i].incarnation) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CasRefCatalog::casUpdate cannot replace catalog identity at row {} -- namespace " + "and incarnation are immutable outside the narrow admission/deletion APIs", + i); + } + return candidate; + }; + return casUpdateImpl( + backend, layout, identity_preserving_mutate, [](const RefCatalog & c) { return encodeRefCatalog(c); }); +} + +RefCatalog CasRefCatalog::casAdmitEntry( + Backend & backend, const Layout & layout, uint64_t gc_shards, const CatalogEntry & entry) +{ + if (entry.state == NsState::Removing) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CasRefCatalog::casAdmitEntry cannot admit namespace '{}' directly as Removing -- " + "removal is an exact transition of an existing Live row", + entry.ns.string()); + + /// The mutation shape is FIXED (insert `entry` at its canonical position) rather than a + /// caller-supplied lambda -- see the header comment on why that is the point, not an + /// inconvenience. A namespace that already has an entry is not de-duplicated here: the insert + /// makes the candidate carry two adjacent equal-ns rows, and `encodeRefCatalog`'s own + /// canonical-order/no-duplicate check (run inside `checkCatalogAdmission` below) rejects that + /// shape -- one place owns that rule, not two. + const auto mutate = [&entry](const RefCatalog & cur) -> RefCatalog + { + RefCatalog next = cur; + const auto it = std::lower_bound(next.entries.begin(), next.entries.end(), entry, + [](const CatalogEntry & a, const CatalogEntry & b) { return a.ns.string() < b.ns.string(); }); + next.entries.insert(it, entry); + return next; + }; + return casUpdateImpl(backend, layout, mutate, + [&entry, gc_shards, &layout](const RefCatalog & c) + { + return checkCatalogAdmission(c, gc_shards, layout, entry.ns); + }); +} + +CasRefCatalog::BeginRemovingOutcome CasRefCatalog::beginRemoving( + Backend & backend, const Layout & layout, const CatalogEntry & observed, + uint64_t removal_started_round, uint64_t admitted_generation, + const std::function & check_fence_or_throw) +{ + if (observed.state != NsState::Live || observed.creator || observed.removal_started_round) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CasRefCatalog::beginRemoving: namespace '{}' is not an exact Live entry", + observed.ns.string()); + + const auto mutate = [&](const RefCatalog & cur) -> RefCatalog + { + try { check_fence_or_throw(admitted_generation); } + catch (...) { throw CatalogFenceMovedMarker{}; } + + const auto it = findEntry(cur, observed.ns); + if (it == cur.entries.end() || *it != observed) + throw CatalogEntryMismatchMarker{}; + + RefCatalog next = cur; + CatalogEntry & entry = next.entries[it - cur.entries.begin()]; + entry.state = NsState::Removing; + entry.removal_started_round = removal_started_round; + return next; + }; + + try + { + casUpdateImpl(backend, layout, mutate, [](const RefCatalog & c) { return encodeRefCatalog(c); }); + } + catch (const CatalogFenceMovedMarker &) + { + return BeginRemovingOutcome::FencedOut; + } + catch (const CatalogEntryMismatchMarker &) + { + const Snapshot current = read(backend, layout); + const auto it = findEntry(current.catalog, observed.ns); + if (it != current.catalog.entries.end() + && it->incarnation == observed.incarnation + && it->state == NsState::Removing) + return BeginRemovingOutcome::AlreadyRemoving; + return BeginRemovingOutcome::EntryChanged; + } + return BeginRemovingOutcome::Transitioned; +} + +CasRefCatalog::CompletedRemovingDeleteResult CasRefCatalog::deleteCompletedRemoving( + Backend & backend, const Layout & layout, const CatalogEntry & observed, + const CasFoldSeal & authoritative_parent, uint64_t admitted_generation, + const std::function & check_fence) +{ + if (observed.state != NsState::Removing || !observed.removal_started_round) + return { + .outcome = CompletedRemovingDeleteOutcome::ProofRefused, + .invalidated_life = std::nullopt, + .catalog_snapshot = std::nullopt}; + + const auto parent_it = authoritative_parent.ref_lives.find(observed.incarnation); + if (parent_it == authoritative_parent.ref_lives.end() + || !parent_it->second.cleanup_evidence + || parent_it->second.coverage.hold) + return { + .outcome = CompletedRemovingDeleteOutcome::ProofRefused, + .invalidated_life = std::nullopt, + .catalog_snapshot = std::nullopt}; + + return deleteCompletedRemovingAtSnapshot( + backend, layout, read(backend, layout), observed, authoritative_parent, + admitted_generation, check_fence); +} + +CasRefCatalog::CompletedRemovingDeleteResult CasRefCatalog::deleteCompletedRemovingAtSnapshot( + Backend & backend, const Layout & layout, Snapshot catalog_snapshot, + const CatalogEntry & observed, const CasFoldSeal & authoritative_parent, + uint64_t admitted_generation, + const std::function & check_fence) +{ + if (observed.state != NsState::Removing || !observed.removal_started_round) + return { + .outcome = CompletedRemovingDeleteOutcome::ProofRefused, + .invalidated_life = std::nullopt, + .catalog_snapshot = std::nullopt}; + + const auto parent_it = authoritative_parent.ref_lives.find(observed.incarnation); + if (parent_it == authoritative_parent.ref_lives.end() + || !parent_it->second.cleanup_evidence + || parent_it->second.coverage.hold) + return { + .outcome = CompletedRemovingDeleteOutcome::ProofRefused, + .invalidated_life = std::nullopt, + .catalog_snapshot = std::nullopt}; + + const NamespaceLifeId old_life + = NamespaceLifeId::fromCatalogEntry(observed.ns, observed.incarnation); + const auto resolved_result = [&](CompletedRemovingDeleteOutcome outcome) + { + const auto current_it = findEntry(catalog_snapshot.catalog, observed.ns); + const bool old_life_still_cataloged = current_it != catalog_snapshot.catalog.entries.end() + && current_it->incarnation == observed.incarnation; + return CompletedRemovingDeleteResult{ + .outcome = outcome, + .invalidated_life = old_life_still_cataloged + ? std::nullopt + : std::optional{old_life}, + .catalog_snapshot = std::move(catalog_snapshot)}; + }; + + for (size_t attempt = 0; attempt < kMaxCatalogCasAttempts; ++attempt) + { + catalog_snapshot.life_index.throwIfAmbiguous("CAS completed-removal deletion"); + const auto observed_it = findEntry(catalog_snapshot.catalog, observed.ns); + if (observed_it == catalog_snapshot.catalog.entries.end() || *observed_it != observed) + return resolved_result(CompletedRemovingDeleteOutcome::EntryChanged); + + bool fence_lost = check_fence(admitted_generation) == LeaderFenceStatus::Moved; + + std::optional cas_result; + std::exception_ptr attempt_failure; + if (!fence_lost) + { + RefCatalog candidate = catalog_snapshot.catalog; + candidate.entries.erase(candidate.entries.begin() + (observed_it - catalog_snapshot.catalog.entries.begin())); + try + { + cas_result = backend.casPut( + layout.refCatalogKey(), encodeRefCatalog(candidate), catalog_snapshot.token); + } + catch (...) + { + attempt_failure = std::current_exception(); + } + } + + /// The response to a conditional erase is not authority for what became durable. Resolve + /// every attempted erase, and a pre-CAS fence refusal, through one complete catalog read. + /// This snapshot is also the next retry/selection cut, so no second read separates them. + catalog_snapshot = read(backend, layout); + + if (!fence_lost) + fence_lost = check_fence(admitted_generation) == LeaderFenceStatus::Moved; + if (fence_lost) + return resolved_result(CompletedRemovingDeleteOutcome::FencedOut); + + const auto current_it = findEntry(catalog_snapshot.catalog, observed.ns); + const bool old_life_still_cataloged = current_it != catalog_snapshot.catalog.entries.end() + && current_it->incarnation == observed.incarnation; + if (!old_life_still_cataloged) + return resolved_result(current_it == catalog_snapshot.catalog.entries.end() + ? CompletedRemovingDeleteOutcome::Deleted + : CompletedRemovingDeleteOutcome::EntryChanged); + + if (attempt_failure) + std::rethrow_exception(attempt_failure); + if (cas_result && cas_result->outcome == CasOutcome::Committed) + throwCasWriteRetryLater(fmt::format( + "CAS ref catalog erase for namespace '{}' reported committed, but a complete resolution read " + "still observed incarnation {}", + observed.ns.string(), u128ToHex(observed.incarnation))); + /// A token conflict that leaves the exact old row present retries from this mandatory + /// resolution snapshot. The fence is checked again immediately before the next CAS. + } + + throwCasWriteRetryLater(fmt::format( + "CAS ref catalog erase for namespace '{}' did not converge after {} attempts", + observed.ns.string(), kMaxCatalogCasAttempts)); +} + +CasRefCatalog::StalledCreatingCancelOutcome CasRefCatalog::cancelStalledCreating( + Backend & backend, const Layout & layout, const CatalogEntry & observed, + const std::function & is_creator_fence_terminal, + uint64_t admitted_generation, const std::function & check_fence_or_throw) +{ + if (observed.state != NsState::Creating || !observed.creator) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CasRefCatalog::cancelStalledCreating: namespace '{}' is not a Creating entry with a " + "creator fence", + observed.ns.string()); + + const auto mutate = [&](const RefCatalog & cur) -> RefCatalog + { + try { check_fence_or_throw(admitted_generation); } + catch (...) { throw CatalogFenceMovedMarker{}; } + + const auto it = findEntry(cur, observed.ns); + if (it == cur.entries.end() || *it != observed) + throw CatalogEntryMismatchMarker{}; + if (!is_creator_fence_terminal(*observed.creator)) + throw CatalogCreatorStillLiveMarker{}; + + RefCatalog next = cur; + next.entries.erase(next.entries.begin() + (it - cur.entries.begin())); + return next; + }; + + try + { + casUpdateImpl(backend, layout, mutate, [](const RefCatalog & c) { return encodeRefCatalog(c); }); + } + catch (const CatalogFenceMovedMarker &) { return StalledCreatingCancelOutcome::FencedOut; } + catch (const CatalogEntryMismatchMarker &) { return StalledCreatingCancelOutcome::EntryChanged; } + catch (const CatalogCreatorStillLiveMarker &) { return StalledCreatingCancelOutcome::CreatorFenceStillLive; } + return StalledCreatingCancelOutcome::Cancelled; +} + +CasRefCatalog::NamespaceCreationOutcome CasRefCatalog::completeCreation( + Backend & backend, const Layout & layout, const CatalogEntry & observed, + uint64_t admitted_generation, const std::function & check_fence_or_throw, + const CkptDeadline & deadline) +{ + if (observed.state != NsState::Creating || !observed.creator) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CasRefCatalog::completeCreation: namespace '{}' is not a Creating entry with a creator " + "fence -- steps 2/3 only ever run over one of those", observed.ns.string()); + + /// Step 2 (spec §3): INV-4's first `_ckpt` writer for this incarnation, and the only writer that + /// will ever know its genesis epoch -- see `Pool/CasRefCkpt.h`'s `publishCkpt` doc for the merge + /// discipline this rides on unchanged. `FencedOut` here ends the attempt: nothing durable changed. + const RefCkpt contribution{.life_epoch = std::optional{observed.creator->writer_epoch}, + .committed_through = std::nullopt, + .checkpoint_snapshot_id = std::nullopt, .last_epoch_seal = std::nullopt}; + if (publishCkpt(backend, layout, NamespaceLifeId::fromCatalogEntry(observed.ns, observed.incarnation), + contribution, admitted_generation, check_fence_or_throw, + deadline) == CkptPublishOutcome::FencedOut) + return NamespaceCreationOutcome::FencedOut; + + /// Step 3. `mutate` is the fence re-check point `casUpdate`'s header doc names -- checked FIRST, + /// mirroring `publishCkpt`'s own "after the read, before the CAS, on every attempt" placement, so a + /// caller stale on BOTH axes between step 2 and here is reported `FencedOut`, never `Superseded` + /// (both are truthful refusals of a CAS that was never sent; this is only which one speaks first). + const auto mutate = [&](const RefCatalog & cur) -> RefCatalog + { + try { check_fence_or_throw(admitted_generation); } + catch (...) { throw CatalogFenceMovedMarker{}; } /// typed, not propagated -- publishCkpt's own precedent + + const auto it = findEntry(cur, observed.ns); + if (it == cur.entries.end() || *it != observed) + throw CatalogEntryMismatchMarker{}; + + RefCatalog next = cur; + next.entries[static_cast(it - cur.entries.begin())].state = NsState::Live; + next.entries[static_cast(it - cur.entries.begin())].creator = std::nullopt; + return next; + }; + + try + { + casUpdate(backend, layout, mutate); + } + catch (const CatalogFenceMovedMarker &) { return NamespaceCreationOutcome::FencedOut; } + catch (const CatalogEntryMismatchMarker &) { return NamespaceCreationOutcome::Superseded; } + return NamespaceCreationOutcome::Live; +} + +CasRefCatalog::NamespaceCreationOutcome CasRefCatalog::createNamespace( + Backend & backend, const Layout & layout, uint64_t gc_shards, + const RootNamespace & ns, const CreatorFence & creator, + uint64_t admitted_generation, const std::function & check_fence_or_throw, + const CkptDeadline & deadline) +{ + /// Read-first, per the Task 2 review's own note on `casAdmitEntry`: a namespace that already + /// carries an entry is THIS function's job to reject with a clear message, not `casAdmitEntry`'s + /// duplicate-namespace grammar refusal (which would report a `LOGICAL_ERROR` about canonical order + /// -- true, but useless to a caller trying to understand why its create failed). A concurrent + /// insert of the SAME namespace between this read and step 1 is still caught -- `casAdmitEntry`'s + /// own grammar check is the backstop, not the only check. + const Snapshot snap = read(backend, layout); + const auto existing = findEntry(snap.catalog, ns); + if (existing != snap.catalog.entries.end()) + { + /// `Creating` is not this function's problem to solve (the class-level doc above says so) -- + /// it is exactly the race `resolveNamespaceLife`'s own loop is built to absorb: sibling openers + /// of the SAME namespace (e.g. concurrent per-part freeze threads of one query, which share one + /// mount's fence) can all observe "no entry" before any of them lands step 1, then race into + /// this call. Reporting `Superseded` sends the loser back through the loop, where it re-reads + /// and takes the documented resume path (its own fence: `completeCreation`; a foreign one: + /// `reconcileStaleCreator`) instead of aborting the server for an outcome the design already + /// names and handles. `Live`/`Removing` stay a `LOGICAL_ERROR`: `namespaceLife`'s caller filters + /// `Live` before ever reaching here and refuses `Removing` outright, so seeing either here means + /// a caller bypassed that dispatch, not a race. + if (existing->state == NsState::Creating) + return NamespaceCreationOutcome::Superseded; + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CasRefCatalog::createNamespace: namespace '{}' already carries a catalog entry (state " + "'{}') -- a stalled Creating entry is resumed through reconcileStaleCreator + " + "completeCreation, never a fresh createNamespace call; an existing Live or Removing " + "namespace must complete its current lifecycle before a fresh creation can be admitted", + ns.string(), nsStateToWord(existing->state)); + } + + const CatalogEntry entry{.ns = ns, .state = NsState::Creating, + .incarnation = mintFreshIncarnation(), .creator = creator}; + /// The read above is a snapshot in time, not a lock: a sibling opener of the SAME namespace that + /// also observed "no entry" can land its own step 1 between that read and this one. `casAdmitEntry` + /// itself cannot be the backstop for that shape -- it retries its own `Conflict`s by blindly + /// re-inserting `entry` into whatever it freshly reads, and a duplicate-namespace insert reaches + /// `encodeRefCatalog`'s grammar check as an unconditional `LOGICAL_ERROR` abort. `createNamespaceStep1` + /// is the same admission, but rechecks presence on every read this loop performs (not just the one + /// above) and reports the race as `Superseded` instead. + try + { + createNamespaceStep1(backend, layout, gc_shards, entry); /// step 1 + } + catch (const CatalogEntryAlreadyPresentMarker &) + { + return NamespaceCreationOutcome::Superseded; + } + return completeCreation(backend, layout, entry, admitted_generation, check_fence_or_throw, deadline); +} + +void CasRefCatalog::setCreateNamespaceStep1PreReadHookForTest(std::function hook) +{ + create_namespace_step1_pre_read_hook_for_test = std::move(hook); +} + +CasRefCatalog::ReconcileCreatorOutcome CasRefCatalog::reconcileStaleCreator( + Backend & backend, const Layout & layout, const CatalogEntry & observed, const CreatorFence & new_creator, + const std::function & is_creator_fence_terminal, + uint64_t admitted_generation, const std::function & check_fence_or_throw) +{ + if (observed.state != NsState::Creating || !observed.creator) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CasRefCatalog::reconcileStaleCreator: namespace '{}' is not a Creating entry with a " + "creator fence -- nothing to reconcile", observed.ns.string()); + + /// Review I6: the fence re-check is checked FIRST, on every fresh read this CAS retries -- the same + /// placement `completeCreation` uses for exactly the same reason (see that function's own doc). + /// Token-exactness (the catalog's own entry, by full value) comes next: it is the cheaper, purely + /// local comparison, and a mismatch here means the question "is the OLD creator's fence terminal" is + /// moot -- `observed` no longer describes anything live to reconcile. + const auto mutate = [&](const RefCatalog & cur) -> RefCatalog + { + try { check_fence_or_throw(admitted_generation); } + catch (...) { throw CatalogFenceMovedMarker{}; } /// typed, not propagated -- completeCreation's own precedent + + const auto it = findEntry(cur, observed.ns); + if (it == cur.entries.end() || *it != observed) + throw CatalogEntryMismatchMarker{}; + if (!is_creator_fence_terminal(*observed.creator)) + throw CatalogCreatorStillLiveMarker{}; + + RefCatalog next = cur; + next.entries[static_cast(it - cur.entries.begin())].creator = new_creator; + return next; + }; + + try + { + casUpdate(backend, layout, mutate); + } + catch (const CatalogFenceMovedMarker &) { return ReconcileCreatorOutcome::FencedOut; } + catch (const CatalogEntryMismatchMarker &) { return ReconcileCreatorOutcome::EntryChanged; } + catch (const CatalogCreatorStillLiveMarker &) { return ReconcileCreatorOutcome::CreatorFenceStillLive; } + return ReconcileCreatorOutcome::Reconciled; +} + +void CasRefCatalog::checkPublicationAdmittedOrThrow(const RefCatalog & catalog, const RootNamespace & ns) +{ + const auto it = findEntry(catalog, ns); + if (it != catalog.entries.end() && it->state == NsState::Creating) + throwCasWriteRetryLater(fmt::format( + "CAS ref catalog: namespace '{}' is still Creating -- no ref writes are admitted until " + "its creation completes or is reconciled away", ns.string())); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCatalog.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCatalog.h new file mode 100644 index 000000000000..6eca5f4c985f --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCatalog.h @@ -0,0 +1,351 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// The `cas/ref_catalog` object (spec INV-3) as seen from the pool side: reading the current +/// catalog, and the generic token-CAS retry primitive every lifecycle transition rides. This class +/// builds ONLY that primitive -- the actual lifecycle steps (the three-conditional-write creation +/// sequence, the removal terminal-record-then-entry-delete sequence) are later tasks' job, built ON +/// TOP of `casUpdate`/`casAdmitEntry`. +class CasRefCatalog +{ +public: + /// The catalog snapshot as read from the backend: the decoded object plus the token an update + /// must present to `casPut`. Operational reads always return a token because the catalog is a + /// mandatory control object after pool bootstrap. + struct Snapshot + { + RefCatalog catalog; + std::optional token; + CatalogLifeIndex life_index; + }; + + /// Reads and decodes the mandatory current catalog. Absence is corruption, never an empty + /// authority set: without the catalog, opaque life keys cannot prove ownership. + static Snapshot read(Backend & backend, const Layout & layout); + + /// Materializes the explicit empty catalog for a prefix already proven new by + /// `probePoolBootstrapResidual`. This is the only absence-tolerant catalog operation: no + /// existing-pool caller can accidentally turn authoritative absence into an empty catalog. A + /// concurrent bootstrap winner is accepted only after its object is read and decoded. + static Snapshot initializeEmptyForNewPool(Backend & backend, const Layout & layout); + + /// The catalog's life for `ns` if a `Live`/`Removing` entry names it, else `nullopt` -- ONE catalog + /// read and, crucially, NO WRITE OF ANY KIND. This is the resolution a READ or a REMOVAL uses: it + /// answers "this namespace does not exist" instead of making it exist, which is what + /// `CasRefLedger::resolveNamespaceLife` would do (it mints for an absent entry, correctly, on behalf + /// of a writer). A caller must not substitute that one here: a read or an unlink against a + /// never-opened table would then perform a catalog CAS and a `_ckpt` publish, growing the single + /// pool-wide catalog object -- which is under a capacity-admission predicate -- for a namespace + /// nobody ever created. + /// + /// `Creating` is excluded for the same reason `liveUniverse` excludes it: no publication can exist + /// under an entry still being created, so there is nothing to resolve to and nothing to read. + static std::optional lifeIfCataloged( + Backend & backend, const Layout & layout, const RootNamespace & ns); + + /// Every `Live`/`Removing` life the catalog currently names, from this call's own catalog `GET`. + /// This helper is for independent readers such as `CasFsck`; a GC fold instead keeps the immutable + /// post-LIST snapshot attached to its scan and reuses that exact cut throughout the round. + /// `Creating` is excluded: spec §3, no publication can exist yet. + static std::vector liveUniverse(Backend & backend, const Layout & layout); + + /// The generic token-CAS retry loop shared by every catalog mutation, mirroring + /// `PoolMeta::admitOrValidate`'s loop: read the current snapshot, apply `mutate` to obtain the + /// CANDIDATE next catalog, `casPut` it against the mandatory object's observed token, and on + /// `Conflict` re-read and re-apply `mutate` to the + /// FRESH snapshot -- never re-encoding the stale candidate. `mutate` must return a canonically + /// ordered, grammar-valid candidate; `encodeRefCatalog` (called internally) enforces that. + /// + /// Bounded (the same live-lock brake `publishCkpt`/`allocateWriterEpoch` use on their own + /// contended token-CAS singletons): after 100 conflicting attempts it gives up and raises the + /// typed retryable error `throwCasWriteRetryLater`, naming the key and the attempt count, rather + /// than spinning forever against a pathologically busy catalog. + /// + /// A re-read that finds the object genuinely ABSENT after it was previously observed present is + /// corruption, not a fresh bootstrap. The required `read` throws before another CAS attempt, so + /// no mutation can replace every other namespace with a one-update catalog. + /// + /// This primitive runs NO admission check: Constraint 13 (removal is never refused) means + /// whether a candidate must clear the additive predicate is the CALLER's decision, not this + /// loop's. A caller mutating an entry's state without growing the catalog (a removal transition) + /// uses this directly. + /// + /// THE FENCE OBLIGATION (Task 3 carry-over from the Task 2 review): this loop has no fence + /// parameter and performs no fence check of its own -- `publishCkpt`'s "AFTER the read, BEFORE the + /// CAS, on every attempt" discipline has no equivalent built in here. The seam a fenced caller + /// MUST use is `mutate` itself: it runs, fresh, after EVERY read this loop performs (the very + /// first one and every one after a `Conflict`), immediately before the candidate it returns is + /// encoded and `casPut`. A caller that needs its own write fenced (any catalog mutation minted + /// under a mount incarnation -- which is every one Task 3 onward adds) MUST throw from inside + /// `mutate`, checking on EVERY invocation, not once before calling `casUpdate`: checking once + /// before the call fences against the read this loop is *about* to perform, not the one it just + /// did, and a `Conflict` retry performs an entirely new read `mutate` is never told about except + /// by being called again. `completeCreation`'s own `mutate` (below) is the first production + /// caller to ride this seam, and does so by wrapping its `check_fence_or_throw` call at the top of + /// its `mutate`, exactly where `publishCkpt` places the identical check. + static RefCatalog casUpdate( + Backend & backend, const Layout & layout, + const std::function & mutate); + + /// Admits exactly ONE new namespace into the catalog under INV-3's two-predicate gate, inserting + /// `entry` at its canonical (ns-sorted) position and running the SAME bounded `casUpdate` retry + /// loop. Takes the entry to insert rather than an arbitrary mutation, by design: an admission + /// entry point that accepted a free-form candidate could be handed a REMOVAL by a future caller + /// that reads as correct, silently reopening Constraint 13 (removal is never refused) behind a + /// name that says "admitting". A namespace `entry.ns` already carries an entry is a bug in the + /// caller (Task 3's creation lifecycle owns checking that first) and surfaces as + /// `encodeRefCatalog`'s own canonical-order/no-duplicate grammar check, inside + /// `checkCatalogAdmission`. + static RefCatalog casAdmitEntry( + Backend & backend, const Layout & layout, uint64_t gc_shards, const CatalogEntry & entry); + + enum class BeginRemovingOutcome : uint8_t + { + Transitioned, + AlreadyRemoving, + EntryChanged, + FencedOut, + }; + + /// Exact `Live -> Removing` transition. The immutable observed row is compared by full value on + /// every catalog retry, and the mount fence is checked after every fresh read and before its CAS. + /// A row already `Removing` under the same namespace/life resolves an ambiguous or concurrent + /// transition positively; no caller may change its recorded start round afterward. + static BeginRemovingOutcome beginRemoving( + Backend & backend, const Layout & layout, const CatalogEntry & observed, + uint64_t removal_started_round, uint64_t admitted_generation, + const std::function & check_fence_or_throw); + + /// Outcome of the only fold-authorized catalog deletion. A refusal never writes the catalog. + enum class CompletedRemovingDeleteOutcome : uint8_t + { + Deleted, + ProofRefused, + EntryChanged, + FencedOut, + }; + + /// Authority result for completed-removal erases. Only an explicit `Moved` is a fence outcome; + /// exceptions mean authority could not be evaluated and propagate to the caller. + enum class LeaderFenceStatus : uint8_t + { + Held, + Moved, + }; + + struct CompletedRemovingDeleteResult + { + CompletedRemovingDeleteOutcome outcome; + /// Present only when a mandatory fresh catalog read proves that the exact observed life is no + /// longer cataloged, whether this actor's erase committed or another actor removed/replaced it. + std::optional invalidated_life; + /// The complete mandatory resolution snapshot after an attempted erase. The GC drain feeds + /// this directly into its next deterministic selection; proof refusal performs no read and + /// leaves it absent. + std::optional catalog_snapshot; + + bool operator==(CompletedRemovingDeleteOutcome expected) const { return outcome == expected; } + }; + + /// Exact-CAS-deletes `observed` only when it is a complete `Removing` row and the authoritative + /// adopted parent carries cleanup evidence, but no hold, in the row keyed by the same opaque life + /// id. The whole parent seal is consumed so a caller cannot separate the life id from its proof or + /// reduce the proof to a caller-computed boolean. The leader fence is checked after every fresh + /// catalog read and before every attempted CAS. + static CompletedRemovingDeleteResult deleteCompletedRemoving( + Backend & backend, const Layout & layout, const CatalogEntry & observed, + const CasFoldSeal & authoritative_parent, uint64_t admitted_generation, + const std::function & check_fence); + + /// Same exact deletion, using the caller's complete selected catalog snapshot and token for the + /// one CAS attempt. Its mandatory resolution snapshot is returned in the result so a catalog-only + /// drain can select the next row without an intervening read. + static CompletedRemovingDeleteResult deleteCompletedRemovingAtSnapshot( + Backend & backend, const Layout & layout, Snapshot catalog_snapshot, + const CatalogEntry & observed, const CasFoldSeal & authoritative_parent, + uint64_t admitted_generation, + const std::function & check_fence); + + /// Outcome of exact stalled-creation cancellation, the only other exported deletion shape. + enum class StalledCreatingCancelOutcome : uint8_t + { + Cancelled, + CreatorFenceStillLive, + EntryChanged, + FencedOut, + }; + + /// Exact-CAS-deletes one observed `Creating` row only after its complete creator fence is proven + /// terminal. This performs no `_ckpt` or other physical cleanup; debris belongs to the janitor. + static StalledCreatingCancelOutcome cancelStalledCreating( + Backend & backend, const Layout & layout, const CatalogEntry & observed, + const std::function & is_creator_fence_terminal, + uint64_t admitted_generation, const std::function & check_fence_or_throw); + + /// === Task 3: the §3 creation lifecycle, built on the two primitives above === + + /// Outcome of the two-step tail every creation attempt ends in (`_ckpt` publish + `Creating -> + /// Live` CAS) -- shared by a fresh `createNamespace` and a reconciler that just adopted a stalled + /// entry via `reconcileStaleCreator`, since both resume from the identical point (an OBSERVED + /// `Creating` entry with a live creator identity that is now THIS caller's own). + enum class NamespaceCreationOutcome : uint8_t + { + Live, /// the entry reached `Live`; `_ckpt` is durable with this creator's `writer_epoch` + /// as `life_epoch` (spec INV-4: the genesis epoch, recorded nowhere else). + FencedOut, /// this caller's OWN admitted generation moved before the `_ckpt` publish or the + /// `Creating -> Live` CAS. Nothing more was written; the caller's own mount + /// incarnation is gone, so it cannot be the one to retry. + Superseded, /// the catalog entry no longer equals what this caller observed -- a concurrent + /// reconciler stole it, or a race already carried it to `Live`/`Removing`. Nothing + /// was written; a DIFFERENT actor now owns whatever happens to this namespace next. + }; + + /// Outcome of `reconcileStaleCreator` alone (see below) -- kept distinct from + /// `NamespaceCreationOutcome` because the two refusal reasons here are not interchangeable with + /// "fenced" / "superseded": one is "not yet permitted" (retry later, unconditionally on the SAME + /// entry), the other is "someone else already moved this entry" (retrying against the SAME + /// `observed` value can never succeed; the caller must re-read first). + enum class ReconcileCreatorOutcome : uint8_t + { + Reconciled, /// `creator` is now `new_creator`; the caller may proceed as if it had + /// just run step 1 itself, over the SAME (unchanged) incarnation. + CreatorFenceStillLive, /// `is_creator_fence_terminal` said no -- the stalled creator might + /// still complete this itself. Not written; retry later against a FRESH + /// terminality read, not immediately. + EntryChanged, /// the catalog's current entry for `observed.ns` no longer equals + /// `observed` -- token-exactness failed. Not written; the caller must + /// re-read the catalog before trying again. + FencedOut, /// review I6: this caller's OWN admitted generation moved before the CAS + /// -- nothing was written, and the caller's own mount incarnation is gone, + /// so it cannot be the one to retry. Mirrors `NamespaceCreationOutcome:: + /// FencedOut`; without this check a deposed mount could still steal a + /// `Creating` entry onto its own dead fence before the following + /// `completeCreation` refuses it -- the catalog would be mutated by an + /// actor this subsystem otherwise never lets touch it. + }; + + /// The full, fresh §3 sequence for a namespace that carries NO catalog entry yet: mints a random + /// nonzero incarnation (spec: "fresh_random_128"), runs step 1 (`casAdmitEntry` inserting `{ns, + /// Creating, incarnation, creator}`), then steps 2+3 via `completeCreation` below. + /// + /// Per the Task 2 review's own note on `casAdmitEntry` ("a namespace `entry.ns` already carries an + /// entry is a bug in the caller -- Task 3's creation lifecycle owns checking that first"): this + /// function reads the catalog FIRST rather than handing `casAdmitEntry` a doomed insert and letting + /// its own grammar check report a confusing duplicate-namespace message. A namespace already + /// `Creating` is not this function's problem to solve -- that is exactly what `reconcileStaleCreator` + /// + `completeCreation` are for, so this reports `Superseded` (never `LOGICAL_ERROR`) and sends the + /// caller back through its own resume loop: sibling openers of the same namespace that all observed + /// "no entry" before any of them landed step 1 race in here exactly this way. A namespace already + /// `Live`/`Removing` IS a caller bug (recreating an existing name is removal's business, not + /// creation's) and still throws `LOGICAL_ERROR` naming the observed state. + static NamespaceCreationOutcome createNamespace( + Backend & backend, const Layout & layout, uint64_t gc_shards, + const RootNamespace & ns, const CreatorFence & creator, + uint64_t admitted_generation, const std::function & check_fence_or_throw, + const CkptDeadline & deadline); + + /// Fires once, synchronously, right after `createNamespace`'s own pre-check read observed no + /// entry and right before its step 1 performs its own (first) catalog read -- the exact window a + /// sibling opener of the same namespace can land its own step 1 in. Lets a test drive that + /// interleaving deterministically instead of relying on real thread scheduling. Empty (no-op) hook + /// in production; a stateless class-scope hook (rather than an instance member) because + /// `CasRefCatalog` itself carries no state. + static void setCreateNamespaceStep1PreReadHookForTest(std::function hook); + + /// Steps 2 (`_ckpt` publish) + 3 (`Creating -> Live` CAS) alone, given an entry the caller already + /// owns as `observed` -- either the entry `createNamespace`'s own step 1 just inserted, or one a + /// caller just reconciled onto itself via `reconcileStaleCreator`. Exposed separately (rather than + /// folded invisibly into `createNamespace`) because reconciliation resumes exactly HERE, never + /// re-running step 1. + /// + /// Step 2: `publishCkpt` with a contribution carrying `observed.creator->writer_epoch` as + /// `life_epoch` -- INV-4's genesis record; `observed.creator` must be present (i.e. `observed.state + /// == Creating`), enforced with `LOGICAL_ERROR` since a caller reaching here with anything else is + /// this module's own bug, not a race. A `FencedOut` from `publishCkpt` ends the attempt here. + /// + /// Step 3: `CasRefCatalog::casUpdate`'s `mutate` is the fence re-check point (see the class-level + /// note below) -- `check_fence_or_throw(admitted_generation)` runs FIRST, on every fresh read this + /// retry loop performs, exactly like `publishCkpt`'s own re-check; a throw from it is caught and + /// reported as `FencedOut`, nothing else. ONLY THEN is the fresh entry for `observed.ns` compared + /// against `observed` by FULL VALUE equality (`CatalogEntry::operator==`) -- the value-CAS that + /// plays the role `publishCkpt`'s object token plays for `_ckpt`, since one catalog object holds + /// every namespace's entry and there is no separate per-entry token to CAS against. A mismatch + /// (stolen by a concurrent reconciler, or already carried to `Live`/`Removing`) is `Superseded`, + /// caught before any CAS is attempted -- not a retry against fresh state, because retrying here + /// would mean re-deciding against an entry that is no longer `observed`, which is precisely what + /// token-exactness forbids. Ordering the fence check before the entry check is deliberate (mirrors + /// `publishCkpt`); a caller that manages to make BOTH stale sees `FencedOut`, not `Superseded` -- + /// both are truthful refusals of a CAS that was never sent. + static NamespaceCreationOutcome completeCreation( + Backend & backend, const Layout & layout, const CatalogEntry & observed, + uint64_t admitted_generation, const std::function & check_fence_or_throw, + const CkptDeadline & deadline); + + /// Stale-`Creating` reconciliation (spec INV-3: "stalled creators occupy entries until + /// fence-terminal reconciliation"; TLA Task 3 obligation 1: "the call-site is where + /// token-exactness is enforced"). `observed` must be a `Creating` entry this caller read a moment + /// ago (`LOGICAL_ERROR` otherwise -- a caller mistake, not a race). Refuses, WITHOUT writing + /// anything, unless BOTH hold against a FRESH catalog read: + /// - `is_creator_fence_terminal(*observed.creator)` -- injected rather than reaching into + /// `CasServerRoot` directly, so this module (and its tests) stay independent of the mount-lease + /// machinery; the real predicate a production caller wires in is `isCreatorFenceTerminal` + /// (`Pool/CasServerRoot.h`, called as `isCreatorFenceTerminal(backend, layout, + /// fence.server_root_id, fence.writer_epoch)` -- it takes those two scalars, not the whole + /// `CreatorFence`, so the mount layer stays independent of the ref-catalog format), built from + /// `writer_epoch` plus the mount-terminality certificates + /// `probeNonTerminalMountSlots`/`computeHeartbeatFloor` already use -- NEVER from + /// `CreatorFence::fence_generation`. That field IS persisted (Task 2 serializes it into the + /// catalog entry), so it reaches the object store fine; what it is NOT is comparable across + /// actors: it mirrors `CasMountRuntime::fence_generation`, an in-process atomic that each mount + /// bumps from its OWN zero on every open, so a different actor's counter (or the SAME actor's + /// after a restart) starts over at the same values and answers a different question than "is + /// the incarnation that minted this entry still alive"; + /// - the catalog's CURRENT entry for `observed.ns` still equals `observed` exactly + /// (token-exactness: a concurrent reconciler, or the original creator finishing on its own, + /// invalidates this immediately). + /// On success, CASes `creator` to `new_creator` -- `state` and `incarnation` are UNCHANGED, so the + /// caller resumes with `completeCreation(backend, layout, {..., .creator = new_creator}, ...)` over + /// the SAME incarnation, never a fresh one (rebirth under a fresh incarnation is Task 5/removal's + /// business, not a live reconciliation's). + /// + /// `admitted_generation`/`check_fence_or_throw` (review I6): re-checked FIRST on every fresh read + /// this CAS retries, exactly like `completeCreation`'s own placement -- a caller whose OWN mount + /// fence has already moved must not be the one to steal a `Creating` entry onto its own (now dead) + /// fence, even though the following `completeCreation` would go on to refuse it as `FencedOut` + /// anyway: by then the catalog would already have been mutated by a deposed actor, the one posture + /// this subsystem otherwise refuses everywhere else. + static ReconcileCreatorOutcome reconcileStaleCreator( + Backend & backend, const Layout & layout, const CatalogEntry & observed, + const CreatorFence & new_creator, + const std::function & is_creator_fence_terminal, + uint64_t admitted_generation, const std::function & check_fence_or_throw); + + /// Spec §3: "`Creating` forbids publication -- no ref writes admitted while the entry is + /// Creating." Throws `throwCasWriteRetryLater`'s class (transient: `Creating` resolves once the + /// creator finishes or is reconciled away) if `catalog`'s entry for `ns` is `Creating`; a no-op for + /// every other case -- no entry, `Live`, or `Removing` -- since this is ONLY the birth-lifecycle + /// gate on the catalog's own `Creating` state, never a general existence/removal check (that role + /// moves onto the catalog in Task 4/Task 6). Takes an already-read `RefCatalog` rather than + /// `Backend`/`Layout`, so a caller that is about to append anyway (and so already holds a fresh + /// read for its OWN purposes) pays no second GET here. + /// + /// Production publication is catalog-governed by construction: an ordinary `appendRefOps` first + /// resolves the namespace lifecycle, refuses a foreign live `Creating` row without writing, and + /// cannot publish the initial stream object until creation has published `_ckpt` and moved the row + /// to `Live`. This helper states the same admission rule for callers that already own a catalog cut; + /// it is not the production append path's enforcement seam. + static void checkPublicationAdmittedOrThrow(const RefCatalog & catalog, const RootNamespace & ns); +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.cpp new file mode 100644 index 000000000000..af43807648bf --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.cpp @@ -0,0 +1,354 @@ +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int INVALID_STATE; +} +} + +namespace DB::Cas +{ + +namespace +{ + +/// Live-lock brake, the same shape and for the same reason as `CasPlainObjects`': the deadline is the +/// real bound, and this only stops an unexpected continuous conflict from spinning until it elapses. +constexpr size_t MAX_CKPT_CAS_ATTEMPTS = 100; + +/// The per-field semantic maximum for an OPTIONAL field: a present value beats an absent one (an +/// absence is "this writer knew nothing", never "this writer says none"), and two present values +/// resolve by the field's own order -- for `RefTxnId` that is writer_epoch then ref_sequence, the +/// intended timeline even across an epoch restart that resets the sequence. +template +std::optional maxKnown(const std::optional & a, const std::optional & b) +{ + if (!a) + return b; + if (!b) + return a; + return std::max(*a, *b); +} + +std::optional mergeCommittedThrough(const RefCkpt & a, const RefCkpt & b) +{ + if (!a.committed_through) + return b.committed_through; + if (!b.committed_through) + return a.committed_through; + if (a.committed_through->writer_epoch == b.committed_through->writer_epoch) + return std::max(*a.committed_through, *b.committed_through); + + const RefCkpt & higher = *a.committed_through < *b.committed_through ? b : a; + const RefCkpt & lower = *a.committed_through < *b.committed_through ? a : b; + if (lower.committed_through->writer_epoch + 1 != higher.committed_through->writer_epoch + || !higher.last_epoch_seal + || *higher.last_epoch_seal < *lower.committed_through + || *higher.committed_through < *higher.last_epoch_seal) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS _ckpt: cross-epoch committed_through requires the immediately next writer epoch and " + "a seal covering the lower frontier without exceeding the higher frontier"); + return higher.committed_through; +} + +/// `life_epoch` MAY NOT DECREASE, and this is the whole of that rule. It lives HERE, at the publish +/// site, and deliberately not inside `mergeCkpt`: the merge is commutative, which is the stated reason +/// the two writers need no ordering between them, and a commutative function cannot even express this +/// rule -- it does not know which of its two arguments is the durable one. Here that distinction +/// exists, because `durable` came from this attempt's read and `contribution` is what the caller wants +/// to add. +/// +/// The refusal is narrow ON PURPOSE, and the narrowness is what makes it stronger rather than weaker +/// than a rule against any disagreement. Two present-and-different values are ORDINARY: the two writers +/// that know a `life_epoch` derive it from different epochs that legitimately differ -- +/// `completeCreation` from the catalog creator's `writer_epoch`, `commitRefChunk`'s birth chunk from the +/// `NamespaceBirth` record's -- so a resumed creation (`reconcileStaleCreator` handing a stalled +/// `Creating` entry to a later actor, same incarnation) and a plain restart between CREATE TABLE and the +/// first INSERT both raise the value honestly. Refusing THAT would wedge the namespace permanently: +/// `_ckpt` has no repair path and no writer may delete it outside namespace removal, so every retry +/// would re-read the old value, re-contribute the new one and re-throw. +/// +/// A DECREASE is the case that cannot happen honestly, which is precisely why it is worth detecting. +/// `writer_epoch` is durable-monotone per server root (`allocateWriterEpoch` CAS-bumps +/// `/gc/server-roots//epoch`), and every live namespace is rooted at its own member's +/// `server_root_id`, so a namespace's creator and any actor that later reconciles it share ONE monotone +/// counter and a live actor's epoch always exceeds a terminal one's. A contribution below what is +/// durable therefore means a writer whose epoch is already superseded got its contribution through -- +/// a fence violation, the class this subsystem cares most about. The semantic maximum absorbs it +/// silently, which is why the check cannot be left to the merge even setting commutativity aside. +/// +/// (That argument is per-server-root. It would need revisiting if one namespace could ever be created +/// by two DIFFERENT server roots, whose epoch counters are independent and so unordered.) +/// +/// It is a PREDICATE rather than a check that throws, because the caller has to consult the mount fence +/// between detecting this and classifying it: a writer the fence is about to refuse has landed nothing +/// anywhere, and reporting corruption for it would turn an expected transient control signal into a +/// permanent verdict. Only a STILL-ADMITTED writer contributing a superseded epoch is the violation this +/// detects. +bool lifeEpochWouldDecrease(const RefCkpt & durable, const RefCkpt & contribution) +{ + return durable.life_epoch && contribution.life_epoch && *contribution.life_epoch < *durable.life_epoch; +} + +/// The verdict for a decrease by a writer that IS still admitted. +/// +/// The message says the object cannot be repaired in place, because that is the part an operator cannot +/// work out from the numbers and cannot afford to guess: nothing rewrites `_ckpt` downwards, and no +/// writer deletes it outside namespace removal, so if the durable value is the wrong one then every +/// honest writer from here on contributes something lower and hits this same refusal forever. The +/// refusal is still right -- silently adopting a suspect genesis epoch would corrupt the epoch-seal +/// grammar for the life of the namespace -- but a fail-closed state with no in-place exit has to say so +/// where it fires, not leave the operator to discover it by retrying. +[[noreturn]] void throwLifeEpochDecrease(const RefCkpt & durable, const RefCkpt & contribution, const String & key) +{ + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS {}: life_epoch may not decrease -- {} is durable and a still-admitted writer contributed {}. " + "Writer epochs are monotone per server root, so a lower contribution means a superseded writer's " + "work reached this object; refusing rather than taking the maximum. This object has NO in-place " + "repair: it is never rewritten downwards and is deleted only by namespace removal, so if {} is " + "itself the wrong value then every later writer will hit this same refusal and the namespace " + "cannot be written again until it is recreated", + key, *durable.life_epoch, *contribution.life_epoch, *durable.life_epoch); +} + +} + +RefCkpt mergeCkpt(const RefCkpt & a, const RefCkpt & b) +{ + RefCkpt merged; + /// `life_epoch` is merged like every other field, by the same semantic maximum: taking it from + /// either side by name is how a writer that knows nothing about it would erase it. In the steady + /// state one side knows it and the other does not, and the max keeps the one that does. (It is not + /// a namespace-lifetime constant -- see the field's own doc in `Formats/CasRefCkptFormat.h` -- but + /// the values it legitimately takes only ever RISE, which is what makes the max right here and what + /// lets `publishCkpt` refuse the fall separately.) + merged.life_epoch = maxKnown(a.life_epoch, b.life_epoch); + merged.committed_through = mergeCommittedThrough(a, b); + merged.checkpoint_snapshot_id = maxKnown(a.checkpoint_snapshot_id, b.checkpoint_snapshot_id); + merged.last_epoch_seal = maxKnown(a.last_epoch_seal, b.last_epoch_seal); + /// Contributions may omit independent facts, but the resulting durable shape may not. In + /// particular, if one writer supplies `life_epoch` and another supplies a later frontier, their + /// merge must carry the chain evidence before `publishCkpt` can encode it. + if (merged.committed_through) + checkRefCkptInvariants(merged, "_ckpt merge"); + return merged; +} + +RecoveryGrounding chooseRecoveryGrounding(const std::optional & catalog_state, + const std::optional & ckpt) +{ + if (!catalog_state || catalog_state->state == NsState::Creating) + throw Exception(ErrorCodes::INVALID_STATE, "CAS recovery grounding: namespace is absent or Creating"); + if (catalog_state->state != NsState::Live && catalog_state->state != NsState::Removing) + throw Exception(ErrorCodes::INVALID_STATE, "CAS recovery grounding: namespace has an unsupported lifecycle state"); + if (!ckpt || !ckpt->life_epoch) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery grounding: a Live or Removing namespace requires a readable _ckpt with life_epoch"); + + checkRefCkptInvariants(*ckpt, "recovery grounding"); + RecoveryGrounding result; + result.committed_through = ckpt->committed_through; + if (!result.committed_through) + { + if (ckpt->checkpoint_snapshot_id || ckpt->last_epoch_seal) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery grounding: a checkpoint without committed_through cannot name a snapshot or epoch seal"); + return result; + } + + if (ckpt->checkpoint_snapshot_id && ckpt->last_epoch_seal + && *ckpt->checkpoint_snapshot_id == *ckpt->last_epoch_seal) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery grounding: checkpoint_snapshot_id must not name last_epoch_seal"); + + if (ckpt->checkpoint_snapshot_id) + result.base = ckpt->checkpoint_snapshot_id; + if (result.base) + { + if (result.base->ref_sequence == std::numeric_limits::max()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery grounding: checkpoint base has no representable successor"); + result.walk_from = RefTxnId{result.base->writer_epoch, result.base->ref_sequence + 1}; + } + else if (!result.base) + { + result.walk_from = RefTxnId{*ckpt->life_epoch, 1}; + } + return result; +} + +std::optional readCkpt(Backend & backend, const Layout & layout, const NamespaceLifeId & life) +{ + std::optional got = backend.get(layout.refCkptKey(life)); + if (!got) + return std::nullopt; + /// Materialized read, then decode: the object is MUTABLE, so the body must be fixed before it is + /// parsed, and the token must be the one that labels exactly these bytes. + return CkptSample{decodeRefCkpt(got->bytes), got->token}; +} + +CkptPublishOutcome publishCkpt(Backend & backend, const Layout & layout, const NamespaceLifeId & life, + const RefCkpt & contribution, uint64_t admitted_generation, + const std::function & check_fence_or_throw, + const CkptDeadline & deadline) +{ + const String key = layout.refCkptKey(life); + std::optional current; + bool have_current = false; + + for (size_t attempt = 0; attempt < MAX_CKPT_CAS_ATTEMPTS; ++attempt) + { + if (deadline.now_ms() >= deadline.deadline_ms) + break; + + /// Read the WHOLE body every attempt. A retry after a conflict must merge against what is + /// there NOW: reusing the previous attempt's reading is precisely the read-modify-write with + /// the merge left out, one round later. + if (!have_current) + { + current = readCkpt(backend, layout, life); + have_current = true; + } + + /// The one rule the commutative merge cannot state, and it has to be decided HERE, before the + /// merge: the semantic maximum turns a decrease into a body identical to the stored one, which + /// the identical-skip below would return as a successful no-op. Detecting it after the merge + /// would therefore detect nothing. + /// + /// The FENCE decides which of the two verdicts it gets, so it is consulted first. A writer the + /// fence is about to refuse has landed nothing anywhere and gets `FencedOut` -- the same + /// transient control signal every other refusal in this function returns rather than throws. A + /// writer that is still admitted and yet contributing a superseded epoch is the fence violation + /// this detects, and that one is corruption. + if (current && lifeEpochWouldDecrease(current->ckpt, contribution)) + { + try + { + check_fence_or_throw(admitted_generation); + } + catch (...) + { + return CkptPublishOutcome::FencedOut; + } + throwLifeEpochDecrease(current->ckpt, contribution, key); + } + + /// ANY writer may create the object; none of them may invent a field. An absent `_ckpt` is + /// created from the contribution as it stands, so a publisher that knows only the checkpoint + /// creates one that knows only the checkpoint, and the birth transaction's `life_epoch` merges + /// into it whenever it arrives -- in either order, because the merge is a per-field maximum. + const RefCkpt merged = current ? mergeCkpt(current->ckpt, contribution) : contribution; + + /// Nothing new: return WITHOUT a CAS. This is not an optimization -- both writers publish on + /// every snapshot and every seal, and most of those carry a checkpoint the object already has, + /// so issuing the write anyway would mint a fresh token per no-op and turn every other writer's + /// in-flight CAS into a conflict, for a body byte-identical to the one already stored. + if (current && merged == current->ckpt) + { + try + { + check_fence_or_throw(admitted_generation); + } + catch (...) + { + return CkptPublishOutcome::FencedOut; + } + return CkptPublishOutcome::IdenticalSkip; + } + + /// AFTER the read, BEFORE the CAS, on EVERY attempt (spec §3). A generation that moved since + /// admission means this writer's lease incarnation is gone and the body it just merged is + /// stale, so the CAS must never be sent -- and because the check precedes it, nothing was. + try + { + check_fence_or_throw(admitted_generation); + } + catch (...) + { + /// Typed, not propagated: the caller asked "did this land", and "the fence moved, so + /// nothing was sent" is an answer, not a failure of the operation. Only the fence check is + /// wrapped, so nothing else can be mistaken for it. + return CkptPublishOutcome::FencedOut; + } + + const std::optional expected = + current ? std::optional{current->token} : std::nullopt; + /// Encode before entering the ambiguity catch. Allocation or invariant failures happen before + /// any request is sent and must propagate as themselves, not trigger a needless resolution GET. + const String merged_bytes = encodeRefCkpt(merged); + try + { + if (backend.casPut(key, merged_bytes, expected).outcome == CasOutcome::Committed) + return CkptPublishOutcome::Published; + } + catch (...) + { + /// A thrown CAS response does not say whether the object changed. Never retry its bytes + /// from memory: first point-read the exact mutable object, including its fresh token. If + /// that observation includes this contribution under the semantic join, the write is + /// resolved durable; otherwise that exact observation is the only valid base for a retry. + try + { + current = readCkpt(backend, layout, life); + have_current = true; + } + catch (...) + { + throwCasWriteRetryLater("CAS _ckpt for namespace '" + life.ns.string() + + "': a CAS response was ambiguous and the mandatory exact-read resolution failed (" + + getCurrentExceptionMessage(/*with_stacktrace*/ false) + ")"); + } + + try + { + check_fence_or_throw(admitted_generation); + } + catch (...) + { + return CkptPublishOutcome::FencedOut; + } + + if (current && lifeEpochWouldDecrease(current->ckpt, contribution)) + throwLifeEpochDecrease(current->ckpt, contribution, key); + + const RefCkpt resolved_merge = current ? mergeCkpt(current->ckpt, contribution) : contribution; + if (current && resolved_merge == current->ckpt) + return CkptPublishOutcome::Published; + + /// `current` is the exact observation made after the ambiguous response. The next loop + /// iteration retries the SAME contribution against its token (or expected absence), with + /// no blind CAS and no redundant intervening GET. + continue; + } + /// `Conflict`: the incarnation we read is no longer current, so another writer's merge landed + /// first. Nothing of ours was written; re-read and merge against the winner. + current.reset(); + have_current = false; + } + + /// Fail closed. Every attempt was all-or-nothing, so there is no partial state -- only an + /// unpublished contribution, which the caller must be told about rather than left to assume. + throwCasWriteRetryLater("CAS _ckpt for namespace '" + life.ns.string() + + "': persistent CAS contention, the checkpoint contribution was not published"); +} + +MissingBaseVerdict classifyMissingSampledBase(const Token & sampled_token, const std::optional & current_token) +{ + if (current_token && !(*current_token == sampled_token)) + return MissingBaseVerdict::RestartRecovery; + return MissingBaseVerdict::Corrupted; +} + +bool snapshotDeletableUnderCkpt(const RefTxnId & snapshot_id, const std::optional & checkpoint_snapshot_id) +{ + return checkpoint_snapshot_id.has_value() && snapshot_id < *checkpoint_snapshot_id; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.h new file mode 100644 index 000000000000..5bf3782503aa --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.h @@ -0,0 +1,172 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// The pure, catalog-authoritative input to both recovery entry points. The checkpoint supplies every +/// recovery boundary; `LIST` cannot supply a base, life epoch, frontier, or stopping condition. +struct RecoveryGrounding +{ + std::optional base; + std::optional walk_from; + std::optional committed_through; +}; + +/// Choose recovery's finite exact range from catalog lifecycle authority and `_ckpt`. Throws +/// `INVALID_STATE` for absent/Creating names and `CORRUPTED_DATA` when a Live/Removing life lacks +/// the checkpoint or genesis fact it must have. This helper performs no I/O and never trusts LIST. +RecoveryGrounding chooseRecoveryGrounding(const std::optional & catalog_state, + const std::optional & ckpt); + +/// THE update and read algorithms for one namespace's `_ckpt` object (spec INV-4). The object itself +/// and its codec live in `Formats/CasRefCkptFormat.h`; this header is what WRITES it and what a reader +/// consults when the base it sampled turns out to be gone. + +/// The one semantic-maximum merge, shared by BOTH `_ckpt` writers (the snapshot publisher and the +/// sealer). Per field: the greater `life_epoch`, the greater present `checkpoint_snapshot_id`, the +/// greater present `last_epoch_seal`; an absent optional loses to a present one, and two absents stay +/// absent. +/// +/// There is deliberately ONE of these rather than a surgical per-writer update. A `_ckpt` write is a +/// whole-body read-modify-write, so a writer that wrote back only the field it knows about would carry +/// its STALE reading of the other field along with it and silently regress the other writer's +/// progress. That regression is not hypothetical: it is TLC counterexample `_sab_sealclobbersbase`, +/// where a sealer writing its sampled body back verbatim drops a concurrently published base and the +/// next recovery loses an ACKED transaction. +/// +/// Compatible contributions still merge commutatively, but the committed frontier is deliberately not +/// an unconstrained CRDT maximum: a cross-epoch pair must be numerically adjacent and carry its seal +/// evidence. That makes arbitrary regrouping of a corrupt historical set invalid, while the actual +/// publish protocol remains simple: each token-CAS merges one contribution with the one durable body it +/// just read. +/// +/// It is therefore NOT where `life_epoch`'s may-not-decrease rule lives, and that is a placement +/// decision rather than an omission: a commutative function does not know which of its arguments is the +/// durable one, so it cannot tell a decrease from an increase. That rule belongs to `publishCkpt`, which +/// does know (see `checkLifeEpochDoesNotDecrease` in the `.cpp`). +RefCkpt mergeCkpt(const RefCkpt & a, const RefCkpt & b); + +/// What one `publishCkpt` call did. +enum class CkptPublishOutcome : uint8_t +{ + Published, /// the merged body is durable -- this call's CAS committed it + IdenticalSkip, /// the contribution added nothing to what was already there; NO write was issued + FencedOut, /// the admitted fence generation moved before the CAS; NOTHING was written +}; + +/// The retry bound for `publishCkpt`: an absolute point on a monotonic millisecond clock, plus that +/// clock. Both are required and must be the SAME clock -- the caller passes its own injectable boot +/// clock (`CasRefLedger`'s `boot_ms_fn`), so a test drives the exhaustion arm deterministically +/// instead of sleeping, and a VM suspend cannot shorten the window. +struct CkptDeadline +{ + std::function now_ms; + uint64_t deadline_ms = 0; +}; + +/// Merge `contribution` into `ns`'s `_ckpt` and make the result durable. +/// +/// One attempt is: GET the object -> decode it -> merge -> (identical? return without a CAS) -> +/// re-check the fence -> token-CAS. A CAS conflict means another writer's read-modify-write landed +/// between our GET and our CAS, so the whole attempt repeats against the NEW body -- never against the +/// one we already read, which is the point of re-reading rather than retrying the same bytes. +/// A THROWN CAS response is ambiguous rather than a conflict: exact-read the object, validate its body +/// and token, then check admission again. If the durable body semantically includes the contribution, +/// the write is resolved; otherwise retry the same contribution against that exact-read token. An +/// unreadable resolution fails retry-later, and no path issues two CAS attempts without an intervening +/// exact observation. +/// +/// An ABSENT object is created from `contribution` as it stands. Every writer may create it and none +/// may complete it: a publisher that knows only the checkpoint creates one that knows only the +/// checkpoint, and the field a different writer knows merges in whenever it arrives, in either order. +/// That is the whole reason each field is optional rather than defaulted. +/// +/// FENCE DISCIPLINE (spec §3, the same value at every site of the trio): `check_fence_or_throw` is +/// re-run on EVERY attempt, AFTER that attempt's read and immediately BEFORE its CAS -- not once at +/// entry. A generation that moved means the mount lease incarnation changed since this work was +/// admitted, so this writer's body is stale even if the fence happens to be live again; the CAS must +/// not be sent. That refusal is returned as `FencedOut` rather than thrown: it is an expected, +/// transient control signal (the same class the request controller reports as `Unresolved`), and the +/// snapshot publisher that calls this sits after a durable PUT where an exception would be worse than +/// a value. NOTHING has been written when it is returned -- the check precedes the CAS. +/// +/// FAILS CLOSED, never open: +/// - an existing `_ckpt` that does not decode PROPAGATES `CORRUPTED_DATA` and is never overwritten. +/// It is the only record of recovery's base and of what cleanup may delete; replacing it with a +/// body derived from `contribution` alone would erase the base while leaving a well-formed object +/// behind -- corruption laundered into something a reader would trust. +/// - a contribution whose `life_epoch` is BELOW the durable one raises `CORRUPTED_DATA`, checked after +/// that attempt's read and before its merge, so no body is built and no CAS is sent. This is the one +/// refusal that HAS to live here rather than in `mergeCkpt`: only this function knows which side is +/// durable. It is reported as corruption ONLY for a writer the fence still admits -- one the fence +/// is about to refuse gets `FencedOut` like every other refusal here, since it landed nothing. +/// - exhausting the deadline (or the live-lock brake) under persistent conflict throws the +/// retry-later class. No partial state exists to clean up: every attempt either committed the +/// complete merged body or changed nothing. +/// +/// `admitted_generation` is the fence generation the CALLER captured when its work was admitted, and +/// `check_fence_or_throw` is the callback the pool wires from `CasMountRuntime::checkFenceOrThrow` +/// (the ledger never owns a `CasMountRuntime`; it receives the pair the way `CasPlainObjects` does). +CkptPublishOutcome publishCkpt(Backend & backend, const Layout & layout, const NamespaceLifeId & life, + const RefCkpt & contribution, uint64_t admitted_generation, + const std::function & check_fence_or_throw, + const CkptDeadline & deadline); + +/// One observation of a namespace's `_ckpt`: the decoded body and the incarnation TOKEN it was read +/// at. The token is what the missing-base revalidation adjudicates against, so a reader that keeps +/// only the body cannot apply the rule. +struct CkptSample +{ + RefCkpt ckpt; + Token token; +}; + +/// Point-read of `life`'s `_ckpt`. `nullopt` means the object is absent (a namespace whose creation has +/// not published one yet); a present-but-undecodable object throws `CORRUPTED_DATA`. +std::optional readCkpt(Backend & backend, const Layout & layout, const NamespaceLifeId & life); + +/// The verdict of INV-4's three-way revalidation, for the one leg that is not simply "it is there". +enum class MissingBaseVerdict : uint8_t +{ + RestartRecovery, /// the checkpoint moved under us -- re-read `_ckpt` and start over from the new base + Corrupted, /// the base is gone under a checkpoint that still names it -- fail closed +}; + +/// Adjudicate a sampled recovery anchor that turned out to be unavailable, by comparing the `_ckpt` +/// token this recovery sampled against the token a fresh re-read observes. The anchor is the +/// checkpoint-named snapshot and its retained same-id non-seal log witness; the caller supplies this +/// verdict after either exact GET is absent. +/// +/// - token ADVANCED -> `RestartRecovery`. Cleanup legitimately advanced the checkpoint and deleted +/// the previous anchor while we were reading. Nothing is wrong; restart from the newer base +/// (bounded by the caller's own restart budget). +/// - token UNCHANGED -> `Corrupted`. The checkpoint still names an object that is not there, and +/// the deletion gate makes that unreachable in an honest run: the named snapshot and matching log +/// are both retained. Something deleted a live anchor. +/// - `_ckpt` itself ABSENT on the re-read -> `Corrupted` for the same reason, and more bluntly: the +/// namespace has a base we sampled and no checkpoint at all. `_ckpt` is deleted only as part of +/// namespace removal, which cannot be racing a live recovery of that same namespace. +/// +/// Pure, so it is decided the same way at every call site; the caller raises `CORRUPTED_DATA` on +/// `Corrupted` with its own context. +MissingBaseVerdict classifyMissingSampledBase(const Token & sampled_token, const std::optional & current_token); + +/// INV-4's snapshot-deletion gate: a snapshot is deletable only STRICTLY BELOW the checkpoint. Strict +/// rather than at-or-below because the checkpoint names the snapshot a recovery is entitled to fetch +/// by exact key; deleting THAT one is what turns a stale-but-harmless pointer into the corruption +/// `classifyMissingSampledBase` has to report (TLC counterexample `_sab_staleckptcorruption`). +/// +/// Fail-closed on `nullopt`: a namespace with no checkpoint yet has NOTHING deletable, because no +/// snapshot has been established as a covering base. +bool snapshotDeletableUnderCkpt(const RefTxnId & snapshot_id, const std::optional & checkpoint_snapshot_id); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowManifestSet.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowManifestSet.cpp new file mode 100644 index 000000000000..cc3e448bee11 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowManifestSet.cpp @@ -0,0 +1,119 @@ +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace ProfileEvents +{ + extern const Event CASRefMaterializeInPlace; + extern const Event CASRefMaterializeCopy; +} + +namespace DB::Cas +{ + +bool RefCowManifestSet::contains(const ManifestRef & m) const +{ + const auto it = overlay.find(m); + if (it != overlay.end()) + return it->second; + return base->contains(m); +} + +void RefCowManifestSet::insert(const ManifestRef & m) +{ + /// Unconditional membership guard, in EVERY build (not a `chassert`): a duplicate insert means the + /// index has drifted from `committed`/`precommits`, and if it silently bumped `net_delta` the index + /// would report a manifest present that a single `erase` could then hide while another owner still + /// names it -- corrupting the add-precommit uniqueness invariant and GC's `+1/-1` edge accounting. + /// Fail closed instead. The caller's own uniqueness check is what enforces the invariant; this is the + /// last line that turns a maintaining-code bug into a caught exception rather than silent drift. + if (contains(m)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefCowManifestSet: inserting a manifest that already has an owner -- the owned-manifest " + "index has drifted from committed/precommits (a bug in the maintaining code)"); + const auto it = overlay.find(m); + if (it != overlay.end()) + it->second = true; /// was a tombstone shadowing a base member -- revive it + else + overlay.emplace(m, true); + ++net_delta; +} + +void RefCowManifestSet::erase(const ManifestRef & m) +{ + /// Same fail-closed rationale as `insert`: erasing an absent manifest would drift `net_delta` and, + /// worse, could shadow a still-live owner. Throw in every build rather than silently corrupting. + if (!contains(m)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefCowManifestSet: erasing a manifest with no current owner -- the owned-manifest index " + "has drifted from committed/precommits (a bug in the maintaining code)"); + const auto it = overlay.find(m); + if (it != overlay.end()) + { + if (base->contains(m)) + it->second = false; /// keep shadowing the base member + else + overlay.erase(it); /// pure-overlay member: nothing left to shadow + } + else + { + overlay.emplace(m, false); /// tombstone a base-only member + } + --net_delta; +} + +void RefCowManifestSet::materialize() +{ + if (overlay.empty()) + return; + /// Same two-path shape and exception-coherence contract as `RefCowMap::materialize` (see its + /// comment for the full argument). Uniquely-owned base: fold each overlay entry into `*base` IN + /// PLACE -- O(overlay), no O(N) `unordered_set` copy. Incrementally coherent: the base mutation (the + /// only throw point -- an `unordered_set` insert's alloc/rehash, which is strong) runs first, then + /// the non-throwing `net_delta` adjustment and `overlay.erase` commit the entry. Any escape leaves + /// (base, overlay, net_delta) exactly coherent and a later `materialize` resumable. + if (base.use_count() == 1) + { + ProfileEvents::increment(ProfileEvents::CASRefMaterializeInPlace); + for (auto it = overlay.begin(); it != overlay.end(); ) + { + if (it->second) + { + const bool inserted = base->insert(it->first).second; /// throw point (alloc/rehash, strong) + if (inserted) + --net_delta; /// a member absent from base counted +1 in net_delta; now it lives in base + } + else + { + base->erase(it->first); /// a tombstone only ever shadows a base member: non-throwing erase + ++net_delta; /// counted -1 in net_delta; now actually removed from base + } + it = overlay.erase(it); /// non-throwing: retire this entry only after its base mutation stuck + } + return; /// net_delta arithmetic above lands it back at 0 (base now holds the whole merged view) + } + /// A copy still shares this base, so fold into a FRESH one and swap -- strong guarantee, the shared + /// holder stays byte-unchanged. + ProfileEvents::increment(ProfileEvents::CASRefMaterializeCopy); + auto fresh = std::make_shared(*base); + for (const auto & [m, present] : overlay) + { + if (present) + fresh->insert(m); + else + fresh->erase(m); + } + base = std::move(fresh); + overlay.clear(); + net_delta = 0; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowManifestSet.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowManifestSet.h new file mode 100644 index 000000000000..097ebeec68bd --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowManifestSet.h @@ -0,0 +1,127 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// A copy-cheap membership set of every `ManifestRef` that currently has an owner (a committed row +/// or a precommit binding). `RefTableState` (Pool/CasRefProtocol.h) uses it to hold the +/// add-precommit uniqueness invariant ("no conflicting owner may name the same manifest") as a +/// structure instead of `manifestAlreadyOwned`'s old linear scan over `committed` + `precommits`. +/// +/// Same copy-on-write shape as `RefCowMap` (Pool/CasRefCowMap.h): copies share an immutable `base` +/// set (a `shared_ptr` refcount bump, no per-element copy) and differ only in a per-copy `overlay`, +/// so the copy-then-mutate-then-swap pattern `applyRefLogTxn`'s scratch copy uses stays O(overlay +/// size), never O(table size) -- exactly the regression `RefCowMap` was already built to avoid, and +/// a plain `std::set` member would have reintroduced here. Membership-only: unlike +/// `RefCowMap` there is no ordered (or any) iteration, because nothing in `RefTableState` ever needs +/// to enumerate owned manifests, only ask "does any owner already name this one". Not thread-safe; +/// same ownership rules as `RefCowMap` (callers retain the state lock or detached-copy ownership +/// rules of the state that contains it). +/// +/// `base` is `std::unordered_set` (O(1) lookup, the point of this class at large table size), but +/// `overlay` is deliberately `std::map`, not `std::unordered_map`, even though it holds the exact +/// same key type: `overlay` is copied on EVERY `RefTableState` scratch copy (unlike `base`, which is +/// shared), and libstdc++'s `unordered_map` copy constructor allocates a real bucket array even for +/// an empty source (measured ~30ns/copy via `.claude/tools/cppexpr.sh`, versus effectively free for +/// an empty `std::map` -- the same reason `RefCowMap`'s own `overlay` is a `std::map`, not an +/// `unordered_map`). The common case is an empty-or-few-entries overlay between flushes, so this +/// keeps the added cost of `owned_manifests` on `BM_ScratchCopy` to roughly one more `shared_ptr` +/// copy, not one more `shared_ptr` copy plus a hidden allocation. +/// +/// `insert`/`erase` throw `CORRUPTED_DATA` on a violated precondition (absence / presence, +/// respectively) in EVERY build rather than merely `chassert`-ing it: the ref table's own uniqueness +/// invariant already guarantees both, so a violation here means the index itself has drifted from +/// `committed`/`precommits`. A `chassert` would let that drift through silently in a release build, +/// after which a single `erase` could report a manifest absent while another owner still names it -- +/// corrupting the add-precommit uniqueness invariant and GC's `+1/-1` manifest-edge accounting +/// downstream. Failing closed keeps `net_delta` from ever drifting. This is the same class of bug +/// `RefTableState::debugAssertBodyCounters` cross-checks in debug/sanitizer builds; the throw extends +/// the guarantee to release builds too. +class RefCowManifestSet +{ +public: + /// Bucket hashing comes from the existing `std::hash` specialization + /// (Primitives/CasTypes.h), picked up by default. Membership hashing only -- this set is never + /// exposed to attacker-chosen keys, only to manifest refs this process itself allocated, so + /// adversarial collision resistance is not a concern here. + using Base = std::unordered_set; + + RefCowManifestSet() = default; + + /// True iff `m` currently has an owner: present in the merged base+overlay view. An overlay + /// tombstone reports absent even when `base` still has `m`. + bool contains(const ManifestRef & m) const; + + /// Records `m` as owned. `m` must be absent from the merged view or this throws `CORRUPTED_DATA` + /// (in every build) -- the caller's own uniqueness check is what actually enforces the invariant; + /// this only guards against the index drifting away from it, failing closed rather than silently. + void insert(const ManifestRef & m); + + /// Records `m` as no longer owned. `m` must be present in the merged view or this throws + /// `CORRUPTED_DATA` (in every build), same rationale as `insert`. + void erase(const ManifestRef & m); + + /// `base->size() + net_delta`, O(1). + size_t size() const { return static_cast(static_cast(base->size()) + net_delta); } + bool empty() const { return size() == 0; } + + /// Folds `overlay` into `base` and clears the overlay. Call this at the same state-install point + /// `RefCowMap::materialize()` is called from (once per ref-log flush, never once per batch item). + /// If the overlay is already empty, this is a no-op. + /// + /// When `base` is uniquely owned (`use_count() == 1`, the production flush case), the overlay is + /// folded into `*base` IN PLACE -- O(overlay), no O(N) `unordered_set` copy. When a copy still + /// shares `base`, a fresh merged base is built and swapped in, so the shared holder's view stays + /// byte-unchanged. The full ownership-and-coherence safety argument is `RefCowMap::materialize`'s + /// (the `use_count()` of 1 is stable against both a concurrent increment and any cross-thread + /// release, every such release being lock-ordered under `state_mutex`; the in-place fold is coherent + /// at every throw point; `base` is a non-const `shared_ptr` so no `const_cast` is needed). Both paths + /// leave an empty overlay and `net_delta == 0`. + void materialize(); + + /// Member-wise swap, guaranteed non-throwing AND allocation-free, with the same contract and the + /// same install-time purpose as `RefCowMap::swap` (Pool/CasRefCowMap.h): `shared_ptr::swap` and + /// `std::map::swap` exchange pointers only, `net_delta` is a POD. The swapped-out set keeps its + /// former base reference until it is destroyed, so a caller that folds the installed set must + /// destroy the swapped-out one first. + void swap(RefCowManifestSet & other) noexcept + { + base.swap(other.base); + overlay.swap(other.overlay); + std::swap(net_delta, other.net_delta); + } + + /// Test-only: current overlay entry count (0 right after `materialize()`). + size_t overlayEntriesForTest() const { return overlay.size(); } + /// Test-only: `base`'s `shared_ptr::use_count()` -- a copy that shares `base` (no per-element + /// allocation) bumps this by exactly one. + int64_t baseUseCountForTest() const { return base.use_count(); } + /// Test-only: identity of the current `base` allocation. `materialize()` on a uniquely-owned base + /// folds the overlay in place and leaves this unchanged; on a base still shared with a copy it + /// swaps in a fresh base, changing it. Lets a test tell the fast (in-place) path from the copy path. + const void * baseIdentityForTest() const { return base.get(); } + +private: + /// Non-const so `materialize()` can fold the overlay into `*base` in place when it is the sole + /// owner (see `materialize`'s doc for the safety argument). It is never mutated while shared. + std::shared_ptr base = std::make_shared(); + /// `true` = an overlay addition (present, whether or not `base` also has it); `false` = a + /// tombstone shadowing a `base` member. An overlay-only member that is erased is removed from + /// this map outright rather than tombstoned (nothing left to shadow), mirroring `RefCowMap`. + /// `std::map`, not `std::unordered_map`: see the class doc comment above -- this is what keeps an + /// empty overlay's copy cost negligible. + std::map overlay; + /// `size() = base->size() + net_delta`, maintained in lock-step by `insert`/`erase` so + /// `size()`/`empty()` stay O(1). Counts live overlay changes relative to `base`. + int64_t net_delta = 0; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowMap.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowMap.cpp new file mode 100644 index 000000000000..67cd1c007e74 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowMap.cpp @@ -0,0 +1,245 @@ +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event CASRefMaterializeInPlace; + extern const Event CASRefMaterializeCopy; +} + +namespace DB::Cas +{ + +std::pair RefCowMap::const_iterator::operator*() const +{ + return at_overlay + ? std::pair(overlay_it->first, *overlay_it->second) + : std::pair(base_it->first, base_it->second); +} + +void RefCowMap::const_iterator::normalize() +{ + /// Drop overlay tombstones (and the base row each one shadows) until the next live overlay + /// entry, or exhaustion. A tombstone is only actually consumed once it is next-in-merge-order + /// (its key <= base_it's current key, the same tie-break the final at_overlay check below + /// uses) -- a tombstone further ahead than base_it must stay put, or base_it would later walk + /// straight past its (still-hidden) target with nothing left in overlay to hide it. + while (overlay_it != overlay_end && !overlay_it->second.has_value() + && (base_it == base_end || overlay_it->first <= base_it->first)) + { + const String key = overlay_it->first; + ++overlay_it; + if (base_it != base_end && base_it->first == key) + ++base_it; + } + /// Overlay wins ties: a live overlay entry at the same key as `base_it` is an override. + at_overlay = (overlay_it != overlay_end) && (base_it == base_end || overlay_it->first <= base_it->first); +} + +RefCowMap::const_iterator & RefCowMap::const_iterator::operator++() +{ + if (at_overlay) + { + const String key = overlay_it->first; + ++overlay_it; + if (base_it != base_end && base_it->first == key) + ++base_it; /// this overlay entry shadowed a base row of the same key: consume it too + } + else + { + ++base_it; + } + normalize(); + return *this; +} + +RefCowMap::const_iterator RefCowMap::begin() const +{ + const_iterator it; + it.base_it = base->begin(); + it.base_end = base->end(); + it.overlay_it = overlay.begin(); + it.overlay_end = overlay.end(); + it.normalize(); + return it; +} + +RefCowMap::const_iterator RefCowMap::end() const +{ + const_iterator it; + it.base_it = base->end(); + it.base_end = base->end(); + it.overlay_it = overlay.end(); + it.overlay_end = overlay.end(); + it.at_overlay = false; + return it; +} + +RefCowMap::const_iterator RefCowMap::find(const String & key) const +{ + const auto ov = overlay.find(key); + if (ov != overlay.end()) + { + if (!ov->second.has_value()) + return end(); /// tombstoned: not present + const_iterator it; + it.base_it = base->lower_bound(key); /// first base key >= this one: keeps the iterator mergeable + it.base_end = base->end(); + it.overlay_it = ov; + it.overlay_end = overlay.end(); + it.at_overlay = true; + return it; + } + const auto b = base->find(key); + if (b == base->end()) + return end(); + const_iterator it; + it.base_it = b; + it.base_end = base->end(); + it.overlay_it = overlay.lower_bound(key); /// first overlay key >= this one: keeps the iterator mergeable + it.overlay_end = overlay.end(); + it.at_overlay = false; + return it; +} + +const RefCommittedRow & RefCowMap::at(const String & key) const +{ + const auto it = find(key); + if (it == end()) + throw std::out_of_range("RefCowMap::at: key not found: " + key); + return it->second; +} + +void RefCowMap::insertLive(const String & key, RefCommittedRow row) +{ + const auto ov = overlay.find(key); + if (ov != overlay.end()) + { + if (!ov->second.has_value()) + ++net_delta; /// tombstoned (dead) -> live again + ov->second = std::move(row); + } + else + { + if (!base->contains(key)) + ++net_delta; /// brand new key, absent from base too + overlay.emplace(key, std::move(row)); + } +} + +std::pair RefCowMap::emplace(String key, RefCommittedRow row) +{ + if (contains(key)) + return {find(key), false}; + insertLive(key, std::move(row)); + return {find(key), true}; +} + +std::pair RefCowMap::insert_or_assign(String key, RefCommittedRow row) +{ + const bool was_present = contains(key); + insertLive(key, std::move(row)); + return {find(key), !was_present}; +} + +size_t RefCowMap::erase(const String & key) +{ + const auto ov = overlay.find(key); + if (ov != overlay.end()) + { + if (!ov->second.has_value()) + return 0; /// already tombstoned: no-op + if (base->contains(key)) + ov->second.reset(); /// keep shadowing the base row + else + overlay.erase(ov); /// pure-overlay key: nothing left to shadow + --net_delta; + return 1; + } + if (!base->contains(key)) + return 0; + overlay.emplace(key, std::nullopt); /// tombstone a base-only row + --net_delta; + return 1; +} + +RefCowMap::iterator RefCowMap::erase(const_iterator pos) +{ + if (pos == end()) + return end(); + const String key = pos->first; + ++pos; + erase(key); + return pos; +} + +bool RefCowMap::operator==(const RefCowMap & other) const +{ + if (size() != other.size()) + return false; + auto a = begin(); + auto b = other.begin(); + for (; a != end() && b != other.end(); ++a, ++b) + if (a->first != b->first || !(a->second == b->second)) + return false; + return a == end() && b == other.end(); +} + +void RefCowMap::materialize() +{ + if (overlay.empty()) + return; + /// See the header for the full ownership-safety argument. The exception-coherence argument for the + /// in-place path is here, next to the code it governs. + /// + /// Uniquely-owned base (the production flush case): fold each overlay entry into `*base` IN PLACE -- + /// O(overlay), no O(N) copy. The fold is INCREMENTALLY COHERENT: for every entry the base mutation + /// (its ONLY throw point -- a `std::map` node insert or a `RefCommittedRow` copy under memory + /// pressure) runs FIRST, and the matching `net_delta` adjustment and `overlay.erase` are both + /// non-throwing and run only AFTER it succeeds. So at every point an exception can escape, the + /// (base, overlay, net_delta) triple is exactly coherent: the merged view is unchanged, `size()` is + /// exact, and a later `materialize` resumes cleanly from the un-erased overlay tail. This is a + /// STRONGER guarantee than the copy path's (which only had to be strong for the shared holder): the + /// state is coherent even mid-fold, which matters because `materialize` runs AFTER the transaction is + /// durably PUT and applied -- an allocation throw here must never leave live bookkeeping wrong. (A + /// throwing key-assign can leave a partially-updated base row, but its still-present overlay entry + /// shadows that key in the merged view, so the partial row stays invisible until it is re-folded.) + if (base.use_count() == 1) + { + ProfileEvents::increment(ProfileEvents::CASRefMaterializeInPlace); + for (auto it = overlay.begin(); it != overlay.end(); ) + { + if (it->second) + { + const bool inserted = base->insert_or_assign(it->first, *it->second).second; /// throw point + if (inserted) + --net_delta; /// a key absent from base counted +1 in net_delta; now it lives in base + } + else + { + base->erase(it->first); /// a tombstone only ever shadows a base member: non-throwing erase + ++net_delta; /// counted -1 in net_delta; now actually removed from base + } + it = overlay.erase(it); /// non-throwing: retire this entry only after its base mutation stuck + } + return; /// net_delta arithmetic above lands it back at 0 (base now holds the whole merged view) + } + /// A copy still shares this base, so fold into a FRESH one and swap -- the shared holder must stay + /// byte-unchanged. Strong guarantee: a mid-fold throw discards `fresh` and leaves this container (and + /// every sharer) exactly as it was. + ProfileEvents::increment(ProfileEvents::CASRefMaterializeCopy); + auto fresh = std::make_shared(*base); + for (const auto & [key, maybe_row] : overlay) + { + if (maybe_row) + (*fresh)[key] = *maybe_row; + else + fresh->erase(key); + } + base = std::move(fresh); + overlay.clear(); + net_delta = 0; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowMap.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowMap.h new file mode 100644 index 000000000000..567dc044aef5 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCowMap.h @@ -0,0 +1,209 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// A value-semantic ordered map from `ref_name` to `RefCommittedRow`, designed as a drop-in +/// replacement for the `std::map` held by `RefTableState::committed`. +/// Copies share an ordered base and copy only a per-copy overlay, so the +/// copy-then-mutate-then-swap operations used by `CasRefLedger` and `CasRefProtocol` cost +/// O(touched rows), rather than copying every committed reference. The base is immutable WHILE SHARED +/// (a copy exists, `use_count() > 1`); when uniquely owned it may be folded into IN PLACE by +/// `materialize()` under the caller's exclusive access. The map is not thread-safe; callers retain the +/// same state lock or detached-copy ownership rules as the state it contains. +/// +/// - Keyed reads (`find`/`contains`/`at`/`count`) check `overlay` first (a tombstone there means +/// "removed"; a present entry means "overridden"), falling back to `base`. +/// - Point writes (`emplace`/`insert_or_assign`/`erase`) only ever touch `overlay`. +/// - Ordered iteration (`begin`/`end`) merges `base` and `overlay` in sorted key order, applying +/// overlay overrides/tombstones -- used only by the table's cold full-scan paths (`snapshotOf`, +/// `CasRefLedger::listRefs`, `dropNamespace`, `CasFsck`/`CasGc` owner-set builders) once the map +/// is integrated into the ref table. This merge preserves the canonical bytewise `ref_name` +/// order required by snapshot encoding. +/// - `materialize()` folds `overlay` into `base` (in place when uniquely owned, else into a fresh +/// base), leaving an empty overlay before the next flush's trial copies begin. Ref-table integration +/// must perform this at the state-install point, not once per batch item, so the hot copy path remains +/// proportional to the rows touched by the flush. +/// +/// Iterators are read-only even when obtained from a non-const map. A caller that needs to change a +/// row must copy it and use `insert_or_assign`; exposing mutable references would allow a write to +/// bypass the overlay and modify neither the owning map's accounting nor its copy-on-write state. +class RefCowMap +{ +public: + using Base = std::map; + +private: + using Overlay = std::map>; + +public: + /// A read-only forward iterator over the merged base-and-overlay view, in sorted key order. + /// `iterator` is an alias of `const_iterator`, so erasing through an iterator cannot expose a + /// mutable reference into the immutable base. + class const_iterator + { + public: + const_iterator() = default; + + /// Returns the current key and row. The references remain valid while the source map's + /// base and overlay entries used by this iterator are not erased or otherwise replaced. + std::pair operator*() const; + + /// Temporary proxy that gives a merged pair-of-references iterator the usual `it->member` + /// syntax without exposing mutable storage. + struct ArrowProxy + { + std::pair value; + const std::pair * operator->() const { return &value; } + }; + ArrowProxy operator->() const { return ArrowProxy{**this}; } + + /// Advances to the next live entry, skipping tombstones and consuming shadowed base rows. + const_iterator & operator++(); + + bool operator==(const const_iterator & other) const + { + return base_it == other.base_it && overlay_it == other.overlay_it; + } + bool operator!=(const const_iterator & other) const { return !(*this == other); } + + private: + friend class RefCowMap; + + /// Advances the two sorted source iterators past tombstones and selects the next source; + /// overlay entries win when both sources contain the same key. + void normalize(); + + Base::const_iterator base_it{}; + Base::const_iterator base_end{}; + Overlay::const_iterator overlay_it{}; + Overlay::const_iterator overlay_end{}; + bool at_overlay = false; + }; + using iterator = const_iterator; + + RefCowMap() = default; + + /// Returns an iterator to the first live entry in the merged view. + const_iterator begin() const; + + /// Returns the past-the-end iterator for the merged view. + const_iterator end() const; + + /// Looks up `key`, consulting overlay entries before the shared base. A tombstone is reported + /// as absent, and a successful iterator refers to the overlay row when one overrides the base. + const_iterator find(const String & key) const; + + bool contains(const String & key) const { return find(key) != end(); } // NOLINT(readability-container-contains): this is the container's contains implementation. + size_t count(const String & key) const { return contains(key) ? 1 : 0; } + /// Returns the row for `key`, or throws `std::out_of_range` when the key is absent or tombstoned. + const RefCommittedRow & at(const String & key) const; + + size_t size() const { return static_cast(static_cast(base->size()) + net_delta); } + bool empty() const { return size() == 0; } + + /// Inserts `row` only when `key` is absent from the merged view. The new row is stored in the + /// overlay; the returned flag reports whether insertion happened. + std::pair emplace(String key, RefCommittedRow row); + + /// Inserts or replaces `key` in the overlay. The returned flag is true only when the merged + /// view did not already contain the key. + std::pair insert_or_assign(String key, RefCommittedRow row); + + /// Removes `key` from the merged view. A base row is retained behind an overlay tombstone; + /// an overlay-only row can be removed outright. Returns one when a live row was removed. + size_t erase(const String & key); + + /// Removes the row referenced by `pos` and returns the following iterator. `pos` must belong to + /// this map and be dereferenceable, as with the corresponding `std::map` operation; `end()` is + /// accepted as a no-op for compatibility with existing callers. + iterator erase(const_iterator pos); + + /// Compares the live merged views, including both keys and committed-row contents; the + /// representation of base and overlay storage does not affect the result. + bool operator==(const RefCowMap & other) const; + + /// Folds `overlay` into `base` and clears the overlay. Call this after installing a completed + /// state, once per ref-log flush and never once per batch item. If the overlay is already empty, + /// this is a no-op. + /// + /// When `base` is uniquely owned (`use_count() == 1`, the production flush case: the live table's + /// base is not shared with any outstanding scratch copy at the install point), the overlay is + /// folded into `*base` IN PLACE -- O(overlay), no O(N) base copy. When a copy still shares `base` + /// (`use_count() > 1`), a fresh merged base is built and swapped in, so the shared holder's view + /// stays byte-unchanged. Both paths leave an empty overlay and `net_delta == 0`. The in-place path + /// is additionally coherent at every intermediate throw point (see the exception-coherence argument + /// in `CasRefCowMap.cpp`), so an allocation failure mid-fold -- possible because this runs AFTER a + /// durable commit -- can never leave `size()` or the merged view wrong. + /// + /// The in-place path is sound because this container is not thread-safe by contract (Pool/ + /// CasRefProtocol.h: callers serialize all access through the state lock, or own a detached copy). + /// A `use_count()` of 1 observed by the sole owner is STABLE for the duration of the fold, against + /// both a concurrent refcount INCREMENT and a concurrent DECREMENT: + /// - No increment: every other holder reaches a copy by copying THIS container, which needs access + /// the caller's exclusivity denies while `materialize` runs -- so no new sharer can appear. + /// - No racing decrement: every copy of a live state that lives on a DIFFERENT thread (the + /// background snapshot publisher's `candidate_state`) is BOTH created and destroyed under the same + /// `state_mutex` this fold holds (see + /// `CasRefLedger::tryPublishSnapshotAndAdvanceCheckpointOnce`, which resets its copy under the lock + /// rather than at function return). So no cross-thread `shared_ptr` release can run + /// concurrently with -- and form a data race against -- this relaxed `use_count()` load. The + /// flush's OWN same-thread scratch copy (`working`) is released BEFORE this fold by program order + /// on the one thread, which is a happens-before all its own. + /// `base` is a `shared_ptr` (non-const) so the in-place fold needs no `const_cast`: the pointee + /// was never const-qualified at construction, so mutating it is defined. The base is still never + /// mutated while shared -- the `use_count() > 1` branch is what guarantees that. Iterators handed out + /// by `begin`/`find` are read-only and short-lived by contract (see the class-level note on iterator + /// validity); `materialize` runs at the install point where no iterator into `base` is live, so an + /// in-place fold cannot invalidate an outstanding one. + void materialize(); + + /// Member-wise swap, guaranteed non-throwing AND allocation-free: `shared_ptr::swap` exchanges two + /// pointers, `std::map::swap` exchanges the trees' internal pointers (the allocator is + /// `std::allocator`, so it is always-equal and the swap is `noexcept`), and `net_delta` is a POD. + /// This is what lets a completed candidate state be installed inside `DENY_ALLOCATIONS_IN_SCOPE` + /// after its transaction is durable -- see `RefTableState::swap` and `CasRefLedger::commitRefChunk`. + /// Note the deliberate consequence: the swapped-out map still OWNS its former base reference, so the + /// caller must destroy it before folding the installed map (`materialize` takes its O(overlay) + /// in-place path only while `base` is uniquely owned). + void swap(RefCowMap & other) noexcept + { + base.swap(other.base); + overlay.swap(other.overlay); + std::swap(net_delta, other.net_delta); + } + + /// Test-only: current overlay row count (0 right after `materialize()`). + size_t overlayEntriesForTest() const { return overlay.size(); } + /// Test-only: `base`'s `shared_ptr::use_count()` -- a copy that shares `base` (no per-row + /// allocation) bumps this by exactly one. + int64_t baseUseCountForTest() const { return base.use_count(); } + /// Test-only: identity of the current `base` allocation. `materialize()` on a uniquely-owned base + /// folds the overlay in place and leaves this unchanged; on a base still shared with a copy it + /// swaps in a fresh base, changing it. Lets a test tell the fast (in-place) path from the copy path. + const void * baseIdentityForTest() const { return base.get(); } + +private: + /// Records a live overlay value and updates `net_delta` according to whether it replaces a + /// tombstone, overrides the base, or introduces a key absent from both sources. + void insertLive(const String & key, RefCommittedRow row); + + /// Non-const so `materialize()` can fold the overlay into `*base` in place when it is the sole + /// owner (see `materialize`'s doc for the safety argument). It is never mutated while shared: + /// every write goes to `overlay`, and only the uniquely-owned branch of `materialize` touches it. + std::shared_ptr base = std::make_shared(); + Overlay overlay; + /// size() = base->size() + net_delta, maintained in lock-step by every overlay-mutating op so + /// size()/empty() stay O(1). `net_delta` counts live overlay changes relative to `base`. + int64_t net_delta = 0; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.cpp new file mode 100644 index 000000000000..b2a43bdbf718 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.cpp @@ -0,0 +1,5113 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int FILE_DOESNT_EXIST; + extern const int INVALID_STATE; + extern const int LIMIT_EXCEEDED; + extern const int LOGICAL_ERROR; + extern const int NETWORK_ERROR; + extern const int S3_ERROR; + extern const int POCO_EXCEPTION; + extern const int UNKNOWN_FORMAT_VERSION; + extern const int SOCKET_TIMEOUT; + extern const int CANNOT_READ_FROM_SOCKET; + extern const int TIMEOUT_EXCEEDED; +} +} + +namespace ProfileEvents +{ + extern const Event CASRefBatchFlushes; + extern const Event CASRefBatchedMutations; + extern const Event CASRefBatchScopeCuts; + extern const Event CASRefQueueWaitMicroseconds; + extern const Event CASRefRecoveryRestarts; + extern const Event CASRefRecoveryRetries; + extern const Event CASRefAppendWedged; + extern const Event CASRefAppendPreAttemptRefused; + extern const Event CASRefAppendUnwedged; + extern const Event CASRefAppendDefiniteFailure; + extern const Event CASRefAppendSealRejected; + extern const Event CASRefAppendOccupantUnreadable; + extern const Event CASRefNeedsRecovery; + extern const Event CASRefSweepDeferred; + extern const Event CASRefSweepRearmed; + extern const Event CASRefStalePrecommitsReclaimed; + extern const Event CASRefTableEvictions; + extern const Event CASRefSnapshotPutBytes; + extern const Event CASRefSnapshotTailLogs; + extern const Event CASRefSnapshotPublishDispatched; + extern const Event CASRefSnapshotPublishBackoff; + extern const Event CASRefCheckpointPublished; + extern const Event CASRefCheckpointIdenticalSkip; + extern const Event CASRefCheckpointNotAdvanced; + extern const Event CASRefRecoveryEpochSealed; + extern const Event CASRefRecoveryEpochSealAdopted; + extern const Event CASRefRecoveryStragglerAdopted; + extern const Event CASRefRecoveryCancelled; + extern const Event CASRefRecoveryStreamHole; +} + +namespace DB::Cas +{ + +namespace +{ +/// Classifies whether an exception thrown out of a ref-table recovery attempt (checkpoint/snapshot/log +/// GETs, or the seal PUT) is a TRANSIENT object-store transport failure worth retrying, +/// vs. a terminal condition (corruption, decode failure, logic error, resource limit) that must fail +/// fast. The recovery reads call the backend directly (not through `ref_request_controller`), so a +/// transient blip surfaces as the object storage's native code -- `S3_ERROR` for the S3 backend, or a +/// socket/timeout/Poco transport code -- NOT the `NETWORK_ERROR` that only the seal PUT's controller +/// re-mints. Retrying only `NETWORK_ERROR` would leave the LIST/GET legs unprotected, which is exactly +/// the exact-read path the recovery retry boundary protects. +bool isTransientRecoveryError(int code) +{ + return code == ErrorCodes::NETWORK_ERROR + || code == ErrorCodes::S3_ERROR + || code == ErrorCodes::POCO_EXCEPTION + || code == ErrorCodes::SOCKET_TIMEOUT + || code == ErrorCodes::CANNOT_READ_FROM_SOCKET + || code == ErrorCodes::TIMEOUT_EXCEEDED; +} + +/// What sits at a ref-log key that our own conditional create just lost to. +enum class Occupant : uint8_t +{ + NotOccupied, /// the create won, or the outcome is unresolved -- nothing was read + Ours, /// byte-for-byte the transaction this attempt intended: an earlier attempt landed + SuccessorSeal, /// this namespace's epoch-closing record, written by a successor (spec INV-2) + Foreign, /// none of the above -- impossible under mount-lease exclusivity +}; + +/// The `mine | successor's seal | foreign` adjudication both write sites owe (the primitive that read +/// the occupant deliberately does none of it). "Mine" means BYTE EQUALITY -- never a shape or +/// generation match, which is the aliasing the phase-0 model rejected. +/// +/// The `catch` is narrow ON PURPOSE, and it is the whole reason this is a function rather than three +/// lines inline. A blanket `catch (...)` here would launder a TRANSIENT failure -- an allocation +/// failure or a memory-limit hit while decoding -- into a `Foreign` verdict, and `Foreign` fences the +/// mount closed and raises a foreign-interference alarm. A perfectly ordinary successor handover plus a +/// memory blip would then read as corruption. Only the two codes that actually mean "these bytes are +/// not a well-formed ref-log transaction of this namespace at this id" are absorbed: the decode layer +/// normalises every malformed-input class to `CORRUPTED_DATA` and passes `UNKNOWN_FORMAT_VERSION` +/// through (an object this build cannot read is still not a seal it can consume). Everything else +/// propagates, leaving the caller's lane exactly as it was. +std::optional chainLinkFor(const RefTxnId & id, const std::optional & last_epoch_seal) +{ + return id.ref_sequence == 1 && last_epoch_seal + && last_epoch_seal->writer_epoch + 1 == id.writer_epoch + ? last_epoch_seal : std::nullopt; +} + +/// The epoch-closing transaction INV-2 places at `{E, T+1}`: exactly one `EpochSeal` op, no table +/// content, and the chain link on -- and only on -- sequence 1, where the grammar requires it. +/// +/// The seal carries nothing about the table because its entire effect is POSITIONAL: it occupies the +/// one key a dying predecessor's in-flight PUT would have taken, so the store's write-once create is +/// what fences the ghost, rather than a detector noticing it afterwards. +RefLogTxn makeEpochSealTxn(const RootNamespace & ns, const RefTxnId & id, const std::optional & prev_epoch_seal) +{ + RefLogTxn seal; + seal.ns = ns.string(); + seal.txn_id = id; + RefOp op; + op.kind = RefOpKind::EpochSeal; + seal.ops.push_back(op); + /// Through `chainLinkFor`, NOT a local `ref_sequence == 1` test, so this is not a fifth site of a + /// rule whose four-site drift is what let the writer preview a transaction it would never send. The + /// two conditions happen to coincide for every id the walk can reach here -- its first dead epoch + /// always already holds the birth transaction, so that seal sits at sequence >= 2, and every later + /// dead epoch's seal is at sequence 1 with the immediately preceding epoch's held seal -- but + /// "happen to coincide" is exactly the property that rots silently. One rule, one caller shape. + seal.prev_epoch_seal = chainLinkFor(id, prev_epoch_seal); + return seal; +} + +/// The `prev_epoch_seal` a transaction at `id` carries, given what this table knows about the seal that +/// closed its previous epoch. INV-2's grammar in one place, because there are now FOUR constructions of +/// the same transaction -- the real one and its three previews -- and the read side REJECTS a mismatch: +/// a preview built without the link previews a transaction the writer would never send. +/// +/// The epoch comparison is not belt-and-braces, it is the DEPOSED-LANE case. A successor that seals an +/// EMPTY epoch writes its record at `{E, 1}`, and a lane still live at E re-derives exactly `{E, 1}`. +/// Stamping the seal there would produce a self-pointer, which `validateEpochSealGrammarStructural` +/// refuses at ENCODE: the lane would fail with a self-inflicted `CORRUPTED_DATA` on every attempt and +/// never reach the collision that is supposed to fence it. Stamping nothing lets the attempt go out and +/// meet the seal, which is the intended conclusive rejection. +Occupant classifyRefLogOccupant(const RootNamespace & ns, const RefTxnId & id, const String & occupant_bytes, + const String & expected_bytes) +{ + if (occupant_bytes == expected_bytes) + return Occupant::Ours; + try + { + return refLogTxnIsEpochSeal(decodeRefLogTxn(openObject(FormatId::RefLog, occupant_bytes), ns.string(), id)) + ? Occupant::SuccessorSeal : Occupant::Foreign; + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::CORRUPTED_DATA && e.code() != ErrorCodes::UNKNOWN_FORMAT_VERSION) + throw; + return Occupant::Foreign; + } +} +} + +CasRefLedger::CasRefLedger( + BackendPtr backend_ptr, + const Layout & layout_, + RefLedgerConfig config_, + const CasEventSink & event_sink_, + CasRequestBudget cas_request_budget_, + String server_root_id_, + std::function controller_boot_ms_fn, + std::function live_epoch_fn_, + std::function fence_ok_fn_, + std::function fence_generation_fn_, + std::function check_fence_or_throw_, + std::function boot_ms_now_fn_, + std::function may_mutate_, + std::function &)> on_impossible_interference_, + std::function()> pin_owner_, + std::function cancel_inflight_builds_) + : backend(*backend_ptr) + , layout(layout_) + , config(std::move(config_)) + , event_sink(event_sink_) + , cas_request_budget(cas_request_budget_) + , server_root_id(std::move(server_root_id_)) + , live_epoch_fn(std::move(live_epoch_fn_)) + , fence_ok_fn(std::move(fence_ok_fn_)) + , fence_generation_fn(std::move(fence_generation_fn_)) + , check_fence_or_throw(std::move(check_fence_or_throw_)) + , boot_ms_now_fn(std::move(boot_ms_now_fn_)) + , may_mutate(std::move(may_mutate_)) + , on_impossible_interference(std::move(on_impossible_interference_)) + , pin_owner(std::move(pin_owner_)) + , cancel_inflight_builds(std::move(cancel_inflight_builds_)) +{ + /// The ref-log writer path uses the same retry controller and clock seam as the mount's local + /// write fence, so deadline-sensitive tests exercise both paths with one monotonic clock. + /// The raw mount `boot_ms_fn` -- the SAME fake-clock seam the local write fence uses -- is reused + /// here rather than adding a second clock knob; both are monotonic-ms clocks and tests that need + /// deterministic deadline behavior already inject it. + ref_request_controller = std::make_unique(backend_ptr, cas_request_budget, controller_boot_ms_fn); + + /// Default backoff sleep for the recovery retry loop (`ensureRefTableRecovered`): sleep in short + /// slices and stop early if the mount fence drops (shutdown / lease loss), so teardown never waits + /// out a full 30s backoff. This is deliberate, bounded backoff against external object-store I/O + /// failure -- NOT masking a race -- exactly like `CasRequestControl`'s own inter-attempt + /// `threadSleepMs`; the slice loop additionally makes it interruptible, which that one is not. + recovery_retry_sleep_fn = [this](uint64_t total_ms) + { + constexpr uint64_t slice_ms = 200; + uint64_t slept = 0; + while (slept < total_ms && fence_ok_fn()) + { + const uint64_t chunk = std::min(slice_ms, total_ms - slept); + sleepForMilliseconds(chunk); + slept += chunk; + } + }; +} + +CasWriteOutcome CasRefLedger::stagingPutIfAbsent(std::string_view key, std::string_view bytes, Token * out_token) +{ + /// The ref lane's mount predicate (`fence_ok_fn` == `Pool::refAppendFenceOk`, with no per-table + /// runtime term) gates every attempt, matching the other staged writes. + return ref_request_controller->putIfAbsentControlled(key, bytes, fence_ok_fn, out_token); +} + +CasCreateResult CasRefLedger::stagingConditionalCreate(std::string_view key, const std::function & attempt) +{ + /// The supplied attempt is controlled by the same retry and mount-fence policy as other staged + /// writes. + return ref_request_controller->conditionalCreateControlled(key, attempt, fence_ok_fn); +} + +CasOverwriteResult CasRefLedger::stagingConditionalOverwrite(std::string_view key, std::string_view bytes, const Token & expected) +{ + /// The supplied write is controlled by the same retry and mount-fence policy as other staged + /// writes. + return ref_request_controller->putOverwriteControlled(key, bytes, expected, fence_ok_fn); +} + +CasOverwriteResult CasRefLedger::stagingPutIfAbsentMutable(std::string_view key, std::string_view bytes) +{ + return ref_request_controller->putIfAbsentControlledMutable(key, bytes, fence_ok_fn); +} + +void CasRefLedger::setCasRetrySleepForTest(std::function sleep_fn) +{ + ref_request_controller->setSleepFnForTest(sleep_fn); + recovery_retry_sleep_fn = std::move(sleep_fn); +} + + +std::optional CasRefLedger::resolveRef(const RootNamespace & ns, const String & ref_name, bool /*allow_stale*/, + ResolveAudit audit) +{ + /// The read side of the snapshot+log protocol has one authoritative cached table for this mounted + /// writer. The `allow_stale` staleness-tolerance knob no longer selects anything: this mounted writer is the + /// ONLY writer of `ns`'s ref state (no external CAS token to go stale against, unlike the old + /// per-shard decode cache), so the recovered-and-cached `RefTableState` is always this process's + /// authoritative view. Kept as a parameter so existing callers compile unchanged. + const auto rt = acquireReadableRefTableRuntime(ns); + /// A namespace the catalog does not name resolves nothing and is not born by being asked: a ref that + /// could exist would have needed a write to put it there, and that write would have birthed the + /// namespace first. The two maintenance calls below are skipped with it, and provably lose nothing -- + /// see the note in `listRefs`. + if (!rt) + return std::nullopt; + ensureRefTableRecovered(ns, *rt); + /// A table this mount only ever READS (never mutates) would otherwise + /// never have its just-replayed tail/precommits checked -- `appendRefOps`'s own hoisted checks only + /// fire for a table this mount WRITES to. Both are cheap (lock + comparison) on the warm path (the + /// flag/threshold is already false after the table's first touch this mount); the sweep, if it DOES + /// fire, runs synchronously here (safe: this call is not nested inside any queue leader's stack). + /// Insulated (unlike appendRefOps's own hoisted call): a READ must not fail because a piggybacked + /// maintenance action hit an uncertain PUT -- see `sweepStalePrecommitsForRead`. + sweepStalePrecommitsForRead(ns, rt); + maybeScheduleSnapshotPublish(ns, rt); + + if (read_before_state_lock_hook_for_test) + read_before_state_lock_hook_for_test(); + + /// Capture the resolved edge under `state_mutex`, but emit AFTER releasing it (Task 2): the audit + /// sink may re-enter a ledger read that itself takes `state_mutex` (e.g. `resolveRef`), so emitting + /// while holding the lock self-deadlocks that reentrant read on the same thread. The reentrancy-safe + /// dispatcher additionally serializes delivery, but the same-thread relock is prevented here by the + /// lock discipline, not the dispatcher. + ManifestRef resolved_ref; + uint64_t resolved_published_at_ms = 0; + std::optional pending_event; + { + std::lock_guard lock(rt->state_mutex); + const auto it = rt->state.getCommitted().find(ref_name); + if (it == rt->state.getCommitted().end()) + return std::nullopt; + + const RefCommittedRow & row = it->second; + resolved_ref = row.manifest_ref; + resolved_published_at_ms = row.published_at_ms; + /// A resolved ref points to its manifest (the read-path entry point). `object_hash` is the manifest + /// instance id the ref names; pairs with a later readManifest ReadMissing if that body is gone. + /// `Deferred` (used only by `CachedPartFolderAccess::resolve` on the `getView` call path) skips this + /// emit; the caller decides, once it knows whether the access as a whole did real resolve work, + /// whether to emit the identical event itself — see `ResolveAudit`'s doc comment. + if (audit == ResolveAudit::Emit && hasEventSink()) + { + CasEvent _ev0; + _ev0.type = CasEventType::RefResolve; + _ev0.namespace_ = ns.string(); + _ev0.ref_name = ref_name; + _ev0.object_kind = CasEventObjectKind::Manifest; + _ev0.object_hash = manifestRefDebugString(row.manifest_ref); + _ev0.outcome = "resolved"; + _ev0.reason = "read-side resolve of a ref to its part manifest"; + pending_event = std::move(_ev0); + } + } + if (pending_event) + emitEvent(std::move(*pending_event)); + return Resolved{ + .manifest_id = ManifestId{.root_namespace = ns, .ref = resolved_ref}, + .manifest_size = 0, + .published_at_ms = resolved_published_at_ms, + }; +} + +std::map CasRefLedger::listRefs(const RootNamespace & ns) +{ + /// The whole ref set is a map iteration over this namespace's recovered-and-cached `RefTableState`: + /// an empty but existing namespace still pays one exact recovery pass and zero further requests; + /// a warm namespace costs nothing at all (replacing the old per-shard LIST-then-HEAD-present-shards + /// dance, since there is no longer a shard fan-out to rediscover on every call). A namespace that was + /// never born costs one catalog GET and stops there. + const auto rt = acquireReadableRefTableRuntime(ns); + /// A namespace the catalog does not name has no refs to list, and listing them is the wrong event to + /// bring one into existence on. + /// + /// Returning here also skips the two maintenance calls below, and that is not a lost obligation: + /// neither would do anything. The stale-precommit sweep runs only when `needs_stale_precommit_sweep` + /// is armed, which recovery and commit are the only things that arm; the snapshot publish is admitted + /// only for a `Live` lifecycle over a non-empty tail, and an unrecovered runtime is neither. + if (!rt) + return {}; + ensureRefTableRecovered(ns, *rt); + /// Apply the same read-side maintenance policy as `resolveRef`; see `sweepStalePrecommitsForRead`. + sweepStalePrecommitsForRead(ns, rt); + maybeScheduleSnapshotPublish(ns, rt); + + std::map result; + std::lock_guard lock(rt->state_mutex); + for (const auto [ref_name, row] : rt->state.getCommitted()) + result.emplace(ref_name, Resolved{ + .manifest_id = ManifestId{.root_namespace = ns, .ref = row.manifest_ref}, + .manifest_size = 0, + .published_at_ms = row.published_at_ms, + }); + return result; +} + +bool CasRefLedger::hasAnyRefWithPrefix(const RootNamespace & ns, std::string_view prefix) +{ + /// Same non-minting recovery/maintenance preamble as `listRefs`; see there for what each shape of + /// namespace -- never born, empty, warm -- costs. + const auto rt = acquireReadableRefTableRuntime(ns); + if (!rt) + return false; + ensureRefTableRecovered(ns, *rt); + sweepStalePrecommitsForRead(ns, rt); + maybeScheduleSnapshotPublish(ns, rt); + + std::lock_guard lock(rt->state_mutex); + for (const auto [ref_name, row] : rt->state.getCommitted()) + if (prefix.empty() || std::string_view(ref_name).starts_with(prefix)) + return true; + return false; +} + + +ConfirmAnswer CasRefLedger::confirmExactRef(const RootNamespace & ns, const String & ref_name, + const ManifestRef & manifest_ref) const +{ + /// Gate 1 of the relink confirm (spec §confirm-primitive). A `Yes` authorizes a REMOTE receiver to + /// promote a manifest whose blobs are protected only by this writer's committed binding of that + /// exact manifest, so a `Yes` is an assertion about the durable table, not about this cache. Every + /// rule below exists to make that assertion true; the answer to anything a rule cannot establish is + /// `Unknown`, which costs the receiver a retry and costs correctness nothing. + /// + /// Two structural properties, both load-bearing: + /// + /// ZERO object-store I/O. This runs on an interserver request, so anything it could be made to + /// do is something a remote peer can make this writer do. It therefore reads only what is already + /// resident, never recovers, never resolves a wedge, and -- see the `find` below -- never even + /// materializes a runtime. Deliberately absent for the same reason: `ensureRefTableRecovered`, + /// `sweepStalePrecommitsForRead` and `maybeScheduleSnapshotPublish`, the three maintenance calls + /// `resolveRef` performs and all three of which can do I/O. + /// + /// ONE snapshot across BOTH lane mutexes. `pending`/`leader_active` live under + /// `ref_queue_mutex`, the rows and the wedge under `state_mutex`, and the whole point of the + /// rules is their CONJUNCTION -- read at different instants they would prove nothing. The lock + /// ORDER is the one the rest of this file already establishes (`enforceRefTableCacheBudget` + /// nests `state_mutex` under `ref_queue_mutex`, and nothing anywhere takes them the other way + /// round). Because admission (`appendRefOps`' `pending.push_back`) happens under + /// `ref_queue_mutex`, an append is either entirely before this snapshot -- and then visible as a + /// pending item -- or entirely after it. There is no interleaving in which a removal is admitted + /// and this function still answers `Yes`. + /// + /// What a `Yes` does NOT prove, stated so nobody has to rediscover it: that this runtime's + /// recovered view is a COMPLETE replay of the durable log. Completeness is recovery's contract, not + /// this function's, and it cannot be re-established here without I/O. Rules 2-4 exclude every way + /// this MOUNT can have fallen behind its own durable writes; a recovery that silently observed less + /// than it should have is a different defect, in a different component. + std::lock_guard qlock(ref_queue_mutex); + + /// Rule 2 (residency). Direct slot lookup, never a catalog observation or exact-runtime acquisition: + /// a read-only query must not let a peer grow this writer's cache or make the next reader pay for a + /// recovery it invented. A cold or evicted table is simply unknown here. + const auto it = ref_name_slots.find(ns.string()); + if (it == ref_name_slots.end()) + return ConfirmAnswer::Unknown; + if (!it->second.current) + return ConfirmAnswer::Unknown; + RefTableRuntime & rt = *it->second.current; + + /// `try_to_lock`, not a blocking acquire: `ensureRefTableRecovered` holds `state_mutex` across its + /// whole exact replay, so blocking here would make a confirm WAIT on someone else's recovery -- + /// up to the full retry envelope -- while holding `ref_queue_mutex`, which is pool-wide append + /// admission. That is the zero-I/O contract broken by proxy: the query would not issue a request, + /// it would merely be paid for by one, and it would stall every table's lane meanwhile. Failing to + /// take the lock is just one more ambiguity, so it answers like every other one. (Same technique, + /// and same non-blocking rationale, as `enforceRefTableCacheBudget`'s candidate loop.) + std::unique_lock slock(rt.state_mutex, std::try_to_lock); + if (!slock.owns_lock()) + return ConfirmAnswer::Unknown; + + /// Rule 2 (warm). An unrecovered or mid-recovery runtime has an EMPTY `state`, which would read as + /// "the ref does not exist" -- knowledge it does not have. `superseded_by_remount` is the same + /// class: this runtime was detached by a self-remount and its view belongs to a dead incarnation. + /// Recovery publishes atomically (`installRecoveryResult` sets `recovered` LAST under this mutex), + /// so there is no half-recovered view to catch in between. + if (!rt.recovered || rt.recovery_in_progress + || rt.catalog_life_invalidated.load(std::memory_order_acquire) + || rt.superseded_by_remount.load(std::memory_order_acquire)) + return ConfirmAnswer::Unknown; + + /// Rule 3 (lane quiescent). A wedge is "an object that may be durable and is not applied" -- it may + /// BE the removal being asked about. A pending item or an active leader tenure is a mutation this + /// table has already admitted; mid-tenure, a chunked flush has committed some of its transactions + /// and not others, and `leader_active` spans the whole tenure, so that partially-durable window is + /// covered too. None of the three says anything about WHICH ref is affected, so all three are + /// table-scoped refusals. + if (rt.lane_state != RefLaneState::Ready || !rt.pending.empty() || rt.leader_active) + return ConfirmAnswer::Unknown; + + /// Rule 5 (exact row equality) -- the only rule that can answer `No` at all. On a table that passed + /// rules 2-4 the committed map is this writer's view, so a missing row or a different `ManifestRef` + /// is a real disagreement rather than an ambiguity about this cache. It is NOT a proof about the + /// DURABLE table: the fence has not been checked yet (rule 6, below, states why that order is + /// deliberate and why it is sound). Equality is exact and total: + /// mint-tightening (spec §A3) guarantees a repoint or a recreation mints a fresh `ManifestRef`, so + /// there is no ABA to defend against here. + const auto & committed = rt.state.getCommitted(); + const auto row = committed.find(ref_name); + if (row == committed.end() || !(row->second.manifest_ref == manifest_ref)) + return ConfirmAnswer::No; + + /// Rule 6 (mount fence), LAST and still under both locks -- the order the spec fixes. Everything + /// above describes what this process believes; this is the check that it is still entitled to + /// believe it: a fenced-out mount is no longer the namespace's single writer, so another writer may + /// already have repointed the ref. Being last means a token that does not match is reported as `No` + /// even under a lost fence: `No` and `Unknown` are the same outcome for the caller (both are + /// `SourceProofFailed`, spec §failure-taxonomy), and only `Yes` is gated on the fence. + /// Both monotone runtime-invalidations are folded in exactly as the mutation gates fold them: a + /// retired or remount-superseded runtime can never authorize a remote promotion. + if (!fence_ok_fn() + || rt.catalog_life_invalidated.load(std::memory_order_acquire) + || rt.superseded_by_remount.load(std::memory_order_acquire)) + return ConfirmAnswer::Unknown; + + return ConfirmAnswer::Yes; +} + +std::shared_ptr CasRefLedger::lookupRefTableRuntime(const RootNamespace & ns) const +{ + std::lock_guard lock(ref_queue_mutex); + const auto it = ref_name_slots.find(ns.string()); + return it == ref_name_slots.end() ? nullptr : it->second.current; +} + +std::shared_ptr CasRefLedger::acquireRefTableRuntime( + const NamespaceLifeId & life, uint64_t admitted_generation) +{ + check_fence_or_throw(admitted_generation); + + std::shared_ptr result; + bool generation_moved = false; + bool identity_conflict = false; + { + std::lock_guard lock(ref_queue_mutex); + generation_moved = fence_generation_fn() != admitted_generation; + if (!generation_moved) + { + const auto it = ref_name_slots.find(life.ns.string()); + if (it != ref_name_slots.end() && it->second.current) + { + const auto & current = it->second.current; + if (current->life == life + && current->admitted_fence_generation == admitted_generation + && !current->catalog_life_invalidated.load(std::memory_order_acquire) + && !current->superseded_by_remount.load(std::memory_order_acquire)) + result = current; + else + identity_conflict = true; + } + else + { + result = std::make_shared( + next_ref_runtime_id.fetch_add(1, std::memory_order_relaxed) + 1, + life, + admitted_generation); + ref_name_slots.emplace(life.ns.string(), RefNameSlot{.current = result}); + } + } + } + + if (generation_moved) + check_fence_or_throw(admitted_generation); + if (identity_conflict) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': the cached runtime identity changed while publishing catalog life {}; " + "retry from a fresh catalog observation", + life.ns.string(), renderIncarnation(life.incarnation))); + return result; +} + +std::shared_ptr CasRefLedger::acquireReadableRefTableRuntime( + const RootNamespace & ns) +{ + /// A resident runtime is the process's already-held immutable life handle. Hot readers deliberately + /// pay no catalog request here: after rebirth the handle may return predecessor bytes or NotFound, + /// but its exact physical id can never alias successor bytes. A genuinely fresh logical-name + /// admission is the cold path below and resolves the current catalog life before publishing a + /// runtime. + if (auto current = lookupRefTableRuntime(ns)) + { + check_fence_or_throw(current->admitted_fence_generation); + { + std::lock_guard queue_lock(ref_queue_mutex); + if (current->removal_admission_closed) + return nullptr; + } + if (current->catalog_life_invalidated.load(std::memory_order_acquire) + || current->superseded_by_remount.load(std::memory_order_acquire)) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': its cached life was detached; retry against a fresh observation", + ns.string())); + return current; + } + + const uint64_t admitted_generation = fence_generation_fn(); + check_fence_or_throw(admitted_generation); + const CasRefCatalog::Snapshot first_catalog = CasRefCatalog::read(backend, layout); + check_fence_or_throw(admitted_generation); + first_catalog.life_index.throwIfAmbiguous("CAS cold readable runtime admission"); + const auto it = std::find_if(first_catalog.catalog.entries.begin(), first_catalog.catalog.entries.end(), + [&](const CatalogEntry & entry) { return entry.ns == ns; }); + if (it == first_catalog.catalog.entries.end() || it->state != NsState::Live) + return nullptr; + const NamespaceLifeId life = NamespaceLifeId::fromCatalogEntry(it->ns, it->incarnation); + + if (readable_catalog_after_observation_hook_for_test) + readable_catalog_after_observation_hook_for_test(); + + /// The backend token, unlike a process-local invalidation counter, observes catalog mutations by + /// every actor that shares this pool. Validate both the token and the decoded canonical value: + /// neither a token-reuse defect nor an unrelated catalog write may let a life derived from the + /// first cut become the first resident runtime. This second GET is deliberately immediately before + /// the queue-locked fence/slot recheck in `acquireRefTableRuntime`; the held-handle warm path above + /// pays none. + const CasRefCatalog::Snapshot second_catalog = CasRefCatalog::read(backend, layout); + check_fence_or_throw(admitted_generation); + const bool catalog_changed + = second_catalog.token != first_catalog.token || second_catalog.catalog != first_catalog.catalog; + if (catalog_changed) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': its catalog changed while a cold reader observed life {}; " + "retry from a fresh catalog observation", + ns.string(), renderIncarnation(life.incarnation))); + + return acquireRefTableRuntime(life, admitted_generation); +} + +std::shared_ptr CasRefLedger::acquireMutableRefTableRuntime( + const RootNamespace & ns) +{ + const NamespaceLifeId life = namespaceLife(ns); + if (const auto current = lookupRefTableRuntime(ns)) + { + if (current->life == life) + return current; + } + const uint64_t admitted_generation = fence_generation_fn(); + return acquireRefTableRuntime(life, admitted_generation); +} + +void CasRefLedger::invalidateRemovedCatalogLife(const NamespaceLifeId & life) +{ + std::shared_ptr rt; + { + std::lock_guard lock(ref_queue_mutex); + const auto it = ref_name_slots.find(life.ns.string()); + if (it == ref_name_slots.end()) + return; + rt = it->second.current; + if (!rt) + return; + } + + /// Invalidation must neither materialize a cache entry nor perform catalog I/O on GC's thread. + /// Re-check the exact resident life while holding its state lock: a delayed reconciliation for a + /// predecessor must not invalidate a successor. Publishing this bit makes existing holders inert; + /// the slot is then detached and a later name-based caller may publish a distinct runtime. + { + std::lock_guard state_lock(rt->state_mutex); + if (rt->life != life) + return; + rt->catalog_life_invalidated.store(true, std::memory_order_release); + } + { + /// Detach by POINTER identity, not by logical name. A concurrent fresh lookup may already + /// have installed a successor for the same name; a delayed predecessor invalidation must never + /// erase that successor's cache slot. + std::lock_guard queue_lock(ref_queue_mutex); + const auto it = ref_name_slots.find(life.ns.string()); + if (it != ref_name_slots.end() && it->second.current == rt) + ref_name_slots.erase(it); + } + rt->cv.notify_all(); + rt->recovery_cv.notify_all(); + rt->publish_settle_cv.notify_all(); +} + +void CasRefLedger::reconcileCatalogCut(const CasRefCatalog::Snapshot & catalog_cut) +{ + catalog_cut.life_index.throwIfAmbiguous("CAS resident ref-runtime reconciliation"); + + std::vector> closed_runtimes; + { + std::lock_guard queue_lock(ref_queue_mutex); + for (const auto & [_, slot] : ref_name_slots) + if (slot.current && slot.current->removal_admission_closed) + closed_runtimes.push_back(slot.current); + } + + for (const auto & rt : closed_runtimes) + { + const NamespaceLifeId & resident_life = rt->life; + const std::optional catalog_life + = catalog_cut.life_index.resolve(resident_life.incarnation); + if (!catalog_life || *catalog_life != resident_life) + invalidateRemovedCatalogLife(resident_life); + } +} + +void CasRefLedger::checkRecoveryStillAdmitted(const RootNamespace & ns, RefTableRuntime & rt, + bool & cancelled) const +{ + /// Polled at EVERY I/O boundary of the walk, because every one of them is a point at which this + /// recovery may already have lost the right to continue -- and the walk WRITES, so "continue" is not + /// a read-only proposition. + /// + /// Cancellation is checked FIRST and reported as its own outcome: a self-remount asking recovery to + /// stop is an orderly hand-off, not a failure of the store, and the caller must not re-drive it + /// through the transient-retry loop the way it would an S3 blip. + if (rt.recovery_cancel_requested.load(std::memory_order_acquire)) + { + cancelled = true; + ProfileEvents::increment(ProfileEvents::CASRefRecoveryCancelled); + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}' was cancelled by a self-remount before the mount " + "fence was re-armed; nothing was written and nothing installed — the next touch recovers under " + "the fresh incarnation", ns.string())); + } + + if (rt.catalog_life_invalidated.load(std::memory_order_acquire)) + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': catalog retirement invalidated life {} " + "while recovery I/O was in flight — nothing further is written and nothing is installed", + ns.string(), renderIncarnation(rt.life.incarnation))); + + /// The remount's OTHER publication, ordered before the fence re-arm: this runtime is detached, so + /// whatever it recovers belongs to a dead incarnation's cache. Checked separately from the + /// cancellation because the two are independent facts, exactly as `resolveWedgeOnce` checks them. + if (rt.superseded_by_remount.load(std::memory_order_acquire)) + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': this cached table was superseded by a self-remount " + "mid-recovery — retry against the fresh mount incarnation", ns.string())); + + /// The FENCE is deliberately NOT checked here, and the omission is the point. `checkFenceOrThrow` + /// asks two things at once -- "is the fence held right now" and "is the generation still mine" -- and + /// the first has no business gating a READ. Most of this walk is reads, and a mount that has + /// transiently lost its lease can still honestly serve them from durable data; refusing at every GET + /// would turn a lease blip into "this table cannot be read at all". + /// + /// The fence gates exactly the three sites that spend it, which is the trio: every `slotOccupy` + /// (through its own `admitted_fence_ok`), the `_ckpt` CAS (inside `publishCkpt`), and the install. + /// A walk that keeps reading after the generation moved simply wastes its own I/O and is then refused + /// at the first of those -- bounded, and strictly better than refusing the reads themselves. +} + +std::optional CasRefLedger::runRecoveryWalkOnce( + const RootNamespace & ns, RefTableRuntime & rt, uint64_t admitted_generation, uint64_t live_epoch, + const std::optional & retained_attempt, std::optional & hole_detail, + bool & cancelled) +{ + /// Spec §4, one attempt. Runs with NO lock held: everything below is either read-only I/O or a + /// conditional create at a key this namespace owns, and the replayed candidate is PRIVATE until the + /// caller installs it. `recovery_in_progress` (not `state_mutex`) is what keeps a second caller for + /// this same table from racing an independent walk -- see its doc comment. + /// + /// Runtime construction already fixed the exact catalog life, so every key this walk builds remains + /// under the predecessor even if the same logical name is concurrently rebound. + const NamespaceLifeId life = rt.life; + + /// ---- Step 2: immutable runtime authority and checkpoint ---- + /// The runtime was admitted for this exact life before entering recovery, so this walk must not take + /// another catalog cut. Retirement invalidates the runtime through `catalog_life_invalidated`, which + /// `checkRecoveryStillAdmitted` and the recovery write/install fences observe below. + const CatalogEntry catalog_entry{ + .ns = life.ns, + .state = NsState::Live, + .incarnation = life.incarnation}; + const std::optional sampled_ckpt = readCkpt(backend, layout, life); + std::optional accepted_ckpt_sample = sampled_ckpt; + checkRecoveryStillAdmitted(ns, rt, cancelled); + + /// ---- Step 3: the finite grounding and exact base ---- + /// `chooseRecoveryGrounding` is the single policy boundary. The immutable checkpoint alone names + /// the base and inclusive frontier; recovery does not enumerate its own stream. + RecoveryGrounding grounding = chooseRecoveryGrounding( + catalog_entry, + sampled_ckpt ? std::optional{sampled_ckpt->ckpt} : std::nullopt); + std::optional base_id = grounding.base; + + std::optional base_snapshot; + uint64_t base_snapshot_bytes = 0; + if (base_id) + { + try + { + CheckpointSnapshotBase base = readCheckpointSnapshotBase(backend, layout, life, sampled_ckpt->ckpt); + base_snapshot = std::move(base.snapshot); + base_snapshot_bytes = base.bytes; + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::CORRUPTED_DATA || !sampled_ckpt) + throw; + + /// A newer checkpoint may have atomically re-anchored recovery while cleanup retired this + /// old anchor's snapshot or witness log. Restart from that newer immutable sample; an + /// unchanged checkpoint turns every helper failure (missing, malformed, or seal) into the + /// fail-closed corruption it describes. + const std::optional current = readCkpt(backend, layout, life); + if (classifyMissingSampledBase(sampled_ckpt->token, + current ? std::optional(current->token) : std::nullopt) + == MissingBaseVerdict::RestartRecovery) + return std::nullopt; + throw; + } + } + checkRecoveryStillAdmitted(ns, rt, cancelled); + + /// The committed replay range comes only from the grounding. Writer recovery additionally probes + /// the single arithmetic successor: it is the durable-but-not-yet-frontiered transaction left by + /// a lost checkpoint response. With no committed transaction that successor is genesis itself. + /// Deliberately no enumerated-log fallback exists here. + std::optional walk_from = grounding.walk_from; + if (!walk_from && !grounding.committed_through) + walk_from = RefTxnId{*sampled_ckpt->ckpt.life_epoch, 1}; + + RefReplayBuilder builder(std::move(base_snapshot), base_snapshot_bytes); + std::optional private_frontier = base_id; + + /// The chain link, threaded through the whole walk: the greatest seal this recovery has SEEN, + /// whether it read it out of the durable tail, adopted it from a concurrent recoverer, or minted it. + /// It is what a sequence-1 seal must name, and what the table's next sequence-1 append must name. + std::optional last_epoch_seal = sampled_ckpt->ckpt.last_epoch_seal; + + /// Applies one decoded transaction to the private candidate, accounting its resident footprint to + /// the streaming-recovery memory probe for exactly the span it is held (no-op in production). + const auto apply_one = [&](RefLogTxn && txn, uint64_t encoded_bytes) + { + const int64_t footprint = static_cast(decodedRefLogTxnFootprint(txn)); + reportReplayMemoryDelta(footprint); + SCOPE_EXIT({ reportReplayMemoryDelta(-footprint); }); + if (refLogTxnIsEpochSeal(txn)) + last_epoch_seal = txn.txn_id; + private_frontier = txn.txn_id; + builder.applyOne(std::move(txn), encoded_bytes); + }; + + const auto check_recovery_write_admitted = [this, &rt](uint64_t expected_generation) + { + check_fence_or_throw(expected_generation); + if (rt.catalog_life_invalidated.load(std::memory_order_acquire)) + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': catalog retirement invalidated life {} " + "before its checkpoint contribution", + rt.life.ns.string(), renderIncarnation(rt.life.incarnation))); + }; + const auto publish_recovered_frontier = [&](const RefLogTxn & txn) + { + const RefCkpt contribution{ + .life_epoch = std::nullopt, + .committed_through = txn.txn_id, + .checkpoint_snapshot_id = std::nullopt, + .last_epoch_seal = refLogTxnIsEpochSeal(txn) + ? std::optional{txn.txn_id} : txn.prev_epoch_seal}; + if (publishCkptContribution(life, contribution, admitted_generation, check_recovery_write_admitted) + == CkptPublishOutcome::FencedOut) + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': the mount incarnation moved before the " + "checkpoint could record recovered txn {}-{}; nothing is installed", + ns.string(), txn.txn_id.writer_epoch, txn.txn_id.ref_sequence)); + + /// `publishCkptContribution` correctly merges a concurrent winner, but recovery's private + /// candidate cannot silently inherit that winner's farther frontier. If it moved beyond this + /// one successor between lookahead and our CAS, restart from the exact newer checkpoint so the + /// installed state covers every transaction its frontier certifies. + std::optional exact = readCkpt(backend, layout, life); + if (!exact || !exact->ckpt.committed_through || *exact->ckpt.committed_through < txn.txn_id) + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': exact checkpoint read after publishing " + "recovered txn {}-{} did not certify that transaction; nothing is installed", + ns.string(), txn.txn_id.writer_epoch, txn.txn_id.ref_sequence)); + if (*exact->ckpt.committed_through != txn.txn_id) + return false; + + /// This recovery itself may advance `_ckpt`. The just-read token and decoded body are the + /// latest authority cut the private candidate has validated, so the final install boundary + /// compares against this sample rather than the original one. + accepted_ckpt_sample = std::move(exact); + return true; + }; + + const std::optional sampled_frontier = grounding.committed_through; + + /// ---- Steps 5 and 6: the arithmetic tail and the seal CAS-walk, as ONE loop ---- + /// They are the same walk seen from two sides. Reading `{E, S}` and finding it present is the tail; + /// finding it ABSENT is a decision point: the live epoch's stream simply ends there, while a DEAD + /// epoch's must be closed at that exact slot before this table may be trusted. Writing them as one + /// loop is not brevity -- it is what makes "the seal goes where the ghost's PUT would have gone" + /// true by construction rather than by two functions agreeing on an index. + if (walk_from) + { + uint64_t epoch = walk_from->writer_epoch; + uint64_t sequence = walk_from->ref_sequence; + size_t slot_attempts_this_epoch = 0; + + for (;;) + { + checkRecoveryStillAdmitted(ns, rt, cancelled); + const RefTxnId id{epoch, sequence}; + + if (const auto got = backend.get(layout.refLogKey(life, id))) + { + /// `runRecoveryWalkOnce` is the writer recovery entry point even after a process + /// restart, when no in-memory attempt survives. A readable birth checkpoint with no + /// committed frontier makes its first `{epoch,1}` log the same one unfrontiered writer + /// successor as `F+1`; both are recovered from the exact checkpoint and stream alone. + const bool above_sampled_frontier = sampled_ckpt + && (!sampled_frontier || *sampled_frontier < id); + if (above_sampled_frontier && retained_attempt && retained_attempt->txn_id == id) + { + /// `NeedsRecovery` retains the writer's complete attempted bytes exactly for this + /// adjudication. A storage violation can replace a write-once log object between the + /// failed frontier publish and recovery; accepting another ordinary transaction here + /// would acknowledge history the admitted writer never created. A successor seal is + /// the one conclusive, already-defined loss case: it closes the old writer's epoch + /// and is replayed below as the durable stream record. + const Occupant occupant = classifyRefLogOccupant(ns, id, got->bytes, retained_attempt->bytes); + if (occupant == Occupant::Foreign) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref-table recovery for namespace '{}': a DIFFERENT object occupies retained " + "writer txn {}-{} above the exact checkpoint frontier; recovery must not publish " + "or install it as that writer's history", + ns.string(), id.writer_epoch, id.ref_sequence); + } + RefLogTxn txn = decodeRefLogTxn(openObject(FormatId::RefLog, got->bytes), ns.string(), id); + const bool is_seal = refLogTxnIsEpochSeal(txn); + const std::optional next_committed_id + = sampled_frontier && id <= *sampled_frontier + ? nextRefLogIdWithinCommittedFrontier(id, is_seal, *sampled_frontier) + : std::nullopt; + const RefLogTxn frontier_txn = txn; + apply_one(std::move(txn), got->bytes.size()); + if (above_sampled_frontier) + { + /// Before changing the sampled checkpoint, prove that this is the ONLY object above + /// it. The append lane cannot allocate a second unfrontiered id. If the checkpoint + /// moved while we inspected the second slot, restart from that exact newer cut; + /// otherwise two successors are durable corruption and F+1 must not be laundered + /// into the frontier first. + const RefTxnId following_id = is_seal + ? RefTxnId{id.writer_epoch + 1, 1} + : RefTxnId{id.writer_epoch, id.ref_sequence + 1}; + if (backend.get(layout.refLogKey(life, following_id))) + { + const std::optional current = readCkpt(backend, layout, life); + if (!sampled_ckpt || !current || current->token != sampled_ckpt->token) + return std::nullopt; + const String frontier_description = sampled_frontier + ? fmt::format("{}-{}", sampled_frontier->writer_epoch, sampled_frontier->ref_sequence) + : "with only a life epoch"; + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref-table recovery for namespace '{}': exact checkpoint {} " + "had two durable successors through {}-{} while its token remained unchanged; " + "the append lane permits at most one unfrontiered transaction", + ns.string(), frontier_description, + following_id.writer_epoch, following_id.ref_sequence); + } + + /// The sole successor is valid in the private candidate. Publish its frontier under + /// this recovery's current admission before exposing the candidate. + if (!publish_recovered_frontier(frontier_txn)) + return std::nullopt; + /// Publishing F+1 makes this transaction admitted history, not the end of the + /// writer walk. A cold writer can still be above this epoch and must seal its + /// now-dead stream at the following slot before the recovered table is installed. + if (is_seal) + { + ++epoch; + sequence = 1; + slot_attempts_this_epoch = 0; + } + else + ++sequence; + continue; + } + if (next_committed_id) + { + epoch = next_committed_id->writer_epoch; + sequence = next_committed_id->ref_sequence; + slot_attempts_this_epoch = 0; + } + else if (is_seal) + { + /// This epoch is closed. Its stream cannot continue, so the next durable id of this + /// namespace is sequence 1 of the next epoch -- including when that epoch is at or + /// above our own live one, which is what a mount deposed by a higher-epoch successor + /// sees. Reading on is honest (the transactions ARE this namespace's) and harmless: + /// our own appends then collide with the seal and are conclusively rejected, which is + /// exactly how INV-2 tells a deposed writer it has been deposed. + ++epoch; + sequence = 1; + slot_attempts_this_epoch = 0; + } + else + ++sequence; + continue; + } + + /// ---- Absent. Hole, end of the live stream, or a dead epoch to close ---- + if (sampled_frontier && id <= *sampled_frontier) + { + /// This id belongs to the inclusive committed range. A 404 cannot shorten that range: + /// re-read the exact mutable checkpoint to distinguish a concurrent frontier movement + /// from durable-data loss under an unchanged authority token. + const std::optional current = readCkpt(backend, layout, life); + if (!current || current->token != sampled_ckpt->token) + return std::nullopt; + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref-table recovery for namespace '{}': committed log id {}-{} is absent while " + "the exact checkpoint frontier {}-{} and its token remain unchanged", + ns.string(), id.writer_epoch, id.ref_sequence, + sampled_frontier->writer_epoch, sampled_frontier->ref_sequence); + } + + /// Only the exact checkpoint's recorded seal may witness a same-epoch hole. LIST log names + /// are diagnostics only; an omitted or stale name cannot change a correctness verdict. + if (sampled_ckpt->ckpt.last_epoch_seal + && sampled_ckpt->ckpt.last_epoch_seal->writer_epoch == epoch + && sequence < sampled_ckpt->ckpt.last_epoch_seal->ref_sequence + && backend.get(layout.refLogKey(life, *sampled_ckpt->ckpt.last_epoch_seal))) + { + ProfileEvents::increment(ProfileEvents::CASRefRecoveryStreamHole); + hole_detail = fmt::format( + "id {}-{} is absent while the exact checkpoint records same-epoch seal {}-{} — the " + "ref-log stream is dense by construction (INV-1), so this is a hole, not the end of " + "the stream", + id.writer_epoch, id.ref_sequence, + sampled_ckpt->ckpt.last_epoch_seal->writer_epoch, + sampled_ckpt->ckpt.last_epoch_seal->ref_sequence); + return std::nullopt; + } + + if (epoch >= live_epoch) + break; /// the LIVE epoch's stream ends here: this is where the next append goes + + /// A seal closes the epoch of a LIVE stream, and a namespace that is Removed (or never + /// born) has none: its terminal record already closed its history, and `applyOp` refuses a + /// seal over it -- correctly, since such an object would be a statement about a stream that + /// does not exist. Both sides of that rule live here and in `applyOp`, and they have to + /// agree: minting a seal this build then cannot replay would leave a durable object that + /// makes the namespace permanently unrecoverable. + /// + /// Advance to the next epoch WITHOUT writing. Skipping the write is not skipping the walk: + /// a namespace removed at epoch 5 and RECREATED at epoch 7 still has durable transactions + /// above, and stopping here would silently truncate them. The recreation's chain link comes + /// from its own `life_epoch` (its birth is sequence 1 of its genesis epoch, where the + /// grammar forbids a `prev_epoch_seal`), not from a seal over the dead life. + if (builder.lifecycle() != RefLifecycle::Live) + { + ++epoch; + sequence = 1; + slot_attempts_this_epoch = 0; + continue; + } + + /// A DEAD epoch, unclosed. Everything that can throw is prepared BEFORE the conditional + /// create, so a failure here happens while the slot is provably untouched. + if (++slot_attempts_this_epoch > kRefRecoveryMaxSlotAttemptsPerEpoch) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref-table recovery for namespace '{}': the seal slot of dead epoch {} was taken by a " + "straggler {} times running. INV-1 permits at most one in-flight append per table per " + "writer, so a store that keeps materializing objects underneath this walk is not a race " + "this recovery may keep chasing", + ns.string(), epoch, slot_attempts_this_epoch - 1); + + const RefLogTxn seal_txn = makeEpochSealTxn(ns, id, last_epoch_seal); + /// The CONTEXTUAL half of the seal grammar, checked only when this recovery actually LEARNED + /// the namespace's `life_epoch`. There is no `value_or` here on purpose (task 5's interface + /// note): a substituted 0 would demand a `prev_epoch_seal` on every sequence-1 transaction + /// and reject a genesis birth. Unknown `life_epoch` therefore means the structural grammar + /// alone -- which `encodeRefLogTxn` enforces unconditionally on the very next line -- and the + /// walk's own construction rule, which is provably equivalent here (see `makeEpochSealTxn`). + if (sampled_ckpt && sampled_ckpt->ckpt.life_epoch) + validateEpochSealGrammarContextual(seal_txn, *sampled_ckpt->ckpt.life_epoch); + const String seal_bytes = sealObject(FormatId::RefLog, encodeRefLogTxn(seal_txn)); + + /// Presented on EVERY attempt: the generation this recovery was admitted under, never the + /// current one. A seal written by an incarnation that no longer owns the namespace is a write + /// from a dead mount, and refusing pre-attempt leaves the slot provably untouched. + const auto admitted_fence_ok = [this, &rt, admitted_generation] + { + return fence_ok_fn() + && !rt.catalog_life_invalidated.load(std::memory_order_acquire) + && !rt.superseded_by_remount.load(std::memory_order_acquire) + && fence_generation_fn() == admitted_generation; + }; + + const SlotOccupyResult occupied = + ref_request_controller->slotOccupy( + layout.refLogKey(life, id), seal_bytes, admitted_fence_ok); + + switch (occupied.kind) + { + case SlotOccupyResult::Kind::Created: + { + /// The epoch is ours to close and now IS closed. Apply our own seal to the candidate: + /// it is a durable transaction of this stream like any other, and the next recovery + /// will read it back exactly where we put it. + RefLogTxn applied = seal_txn; + apply_one(std::move(applied), seal_bytes.size()); + ProfileEvents::increment(ProfileEvents::CASRefRecoveryEpochSealed); + if (!publish_recovered_frontier(seal_txn)) + return std::nullopt; + ++epoch; + sequence = 1; + slot_attempts_this_epoch = 0; + break; + } + case SlotOccupyResult::Kind::Occupied: + { + /// Someone reached this slot first. A DECODE FAILURE here propagates: an object at a + /// key this namespace owns that is not a transaction of this namespace at this id is + /// corruption or a protocol breach, and the one thing recovery must not do is guess + /// past it. + RefLogTxn occupant = decodeRefLogTxn( + openObject(FormatId::RefLog, occupied.occupant_bytes), ns.string(), id); + const bool occupant_is_seal = refLogTxnIsEpochSeal(occupant); + const RefLogTxn frontier_txn = occupant; + apply_one(std::move(occupant), occupied.occupant_bytes.size()); + if (!publish_recovered_frontier(frontier_txn)) + return std::nullopt; + if (occupant_is_seal) + { + /// A concurrent recoverer closed this epoch (or our own earlier attempt did, and + /// its acknowledgment was lost). Either way the epoch is closed by a seal that is + /// as good as ours -- adopt it and continue. Contesting a peer's CORRECT write is + /// how two recoverers of the same table turn a designed race into an incident. + ProfileEvents::increment(ProfileEvents::CASRefRecoveryEpochSealAdopted); + ++epoch; + sequence = 1; + slot_attempts_this_epoch = 0; + } + else + { + /// A STRAGGLER: an ordinary transaction of the dead epoch landed at `T+1` between + /// our read and our create. Adopt it, advance `T` by exactly ONE, and try the seal + /// again at the NEW `T+1`. Never mint `T+2` around it: ids are state-derived + /// (INV-1/INV-2), and writing past an occupied slot puts a hole in the durable + /// stream that no later reader can distinguish from a lost object. + ProfileEvents::increment(ProfileEvents::CASRefRecoveryStragglerAdopted); + ++sequence; + } + break; + } + case SlotOccupyResult::Kind::Unresolved: + { + /// The store will not say whether our seal landed. There is no honest way to continue: + /// exposing the table would publish a dead epoch that may or may not be closed, and + /// re-deriving the slot later needs a fresh read anyway. Fail this attempt into the + /// caller's transient-retry loop, which either succeeds on a later attempt or spends + /// its budget and leaves the table unrecovered. + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': the epoch seal at {}-{} is UNRESOLVED " + "({}); the table stays unrecovered rather than being exposed with a dead epoch that " + "may or may not be closed", + ns.string(), id.writer_epoch, id.ref_sequence, + unresolvedProvesNothingWasSent(occupied.unresolved_reason) + ? "nothing was sent" : "the outcome of the attempt is unknown")); + } + } + } + } + + /// The sealer's checkpoint contribution is durable before the authority cut is validated. This is + /// still one contribution for the current walk; the following exact read certifies the same private + /// frontier rather than assuming the CAS result and the candidate stayed aligned. + if (last_epoch_seal) + { + checkRecoveryStillAdmitted(ns, rt, cancelled); + const RefCkpt contribution{.life_epoch = std::nullopt, + .committed_through = last_epoch_seal, + .checkpoint_snapshot_id = std::nullopt, + .last_epoch_seal = last_epoch_seal}; + if (publishCkptContribution(life, contribution, admitted_generation, check_recovery_write_admitted) + == CkptPublishOutcome::FencedOut) + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': the mount incarnation moved before the " + "checkpoint could record the epoch seal {}-{}; nothing was written and nothing is installed", + ns.string(), last_epoch_seal->writer_epoch, last_epoch_seal->ref_sequence)); + + const std::optional exact = readCkpt(backend, layout, life); + if (!exact || exact->ckpt.committed_through != private_frontier || exact->ckpt.last_epoch_seal != last_epoch_seal) + return std::nullopt; + accepted_ckpt_sample = exact; + } + + /// Final authority validation is the recovery linearization point. The last exact log probe fixed + /// the private cut, but another actor could have changed `_ckpt` immediately afterwards. Install + /// only when both the exact object token and its complete decoded body remain equal to the latest + /// authority sample this private candidate accepted. + const std::optional final_ckpt = readCkpt(backend, layout, life); + if (!final_ckpt || !accepted_ckpt_sample + || final_ckpt->token != accepted_ckpt_sample->token + || final_ckpt->ckpt != accepted_ckpt_sample->ckpt) + return std::nullopt; + checkRecoveryStillAdmitted(ns, rt, cancelled); + + RecoveryResult result = std::move(builder).finish(); + + /// `finish` returns the candidate WITHOUT materializing: `stateFromSnapshot` loads every committed + /// row and owned-manifest entry into the COW OVERLAY, and no tail transaction materializes either. + /// This state is the table's long-lived working state, so fold both `committed` and `owned_manifests` + /// into fresh shared bases ONCE here -- rather than making the first flush's scratch copy (and every + /// per-item/shape-check copy on it) deep-copy an N-row overlay. The O(N) fold rides inside recovery, + /// which is already O(N). + result.state.materializeCommitted(); + /// Stale-precommit cleanup is dispatched once, from `appendRefOps`'s top level (never from here -- + /// this call may itself be nested inside a queue leader's flush stack). + result.needs_stale_precommit_sweep = true; + result.last_epoch_seal = last_epoch_seal; + /// Per-table admission budgets pre-subtract this table's own wire overhead (`4 + ns.size()`, once in + /// a snapshot body and once in a removal txn body) plus a fixed safety margin from the raw hard + /// limits, once, here. + const uint64_t overhead = 4 + ns.string().size() + kRefAdmissionSafetyMargin; + result.snapshot_budget = overhead < ref_snapshot_max_bytes ? ref_snapshot_max_bytes - overhead : 0; + result.removal_budget = overhead < ref_removal_max_bytes ? ref_removal_max_bytes - overhead : 0; + + return result; +} + +NamespaceLifeId CasRefLedger::resolveNamespaceLife( + const RootNamespace & ns, uint64_t admitted_generation, uint64_t live_epoch, + bool * lifecycle_refusal) +{ + /// Bounded exactly like `CasRefCatalog::casUpdateImpl`'s own live-lock brake, but against THIS + /// loop's re-read cycle only -- every primitive called below already bounds its OWN retry against + /// the catalog's single contended object. A duel between two openers (one creating, one + /// reconciling a stale creator) converges in a handful of rounds; this guards only against a + /// pathologically un-converging sequence of them. + static constexpr size_t kMaxResolveAttempts = 32; + const CkptDeadline deadline{boot_ms_now_fn, boot_ms_now_fn() + cas_request_budget.operation_deadline_ms}; + const CreatorFence our_fence{server_root_id, live_epoch, admitted_generation}; + + for (size_t attempt = 0; attempt < kMaxResolveAttempts; ++attempt) + { + const CasRefCatalog::Snapshot snap = CasRefCatalog::read(backend, layout); + const auto it = std::find_if(snap.catalog.entries.begin(), snap.catalog.entries.end(), + [&](const CatalogEntry & e) { return e.ns.string() == ns.string(); }); + + if (it == snap.catalog.entries.end()) + { + /// No entry at all: this open is the namespace's first-ever opener. `createNamespace` + /// mints a fresh incarnation and carries it all the way to `Live` (or reports why it could + /// not); either way it does not hand the incarnation back, so a `Live` outcome re-reads the + /// catalog on the next loop iteration to learn it -- one extra GET, paid once per birth, + /// never per write. + const auto outcome = CasRefCatalog::createNamespace( + backend, layout, config.gc_shards, ns, our_fence, + admitted_generation, check_fence_or_throw, deadline); + if (outcome == CasRefCatalog::NamespaceCreationOutcome::FencedOut) + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': the mount incarnation moved while " + "birthing its catalog entry; nothing was written and nothing installed", ns.string())); + continue; /// Live or Superseded: re-read (Superseded means a DIFFERENT actor won birth) + } + + if (it->state == NsState::Live) + return NamespaceLifeId::fromCatalogEntry(it->ns, it->incarnation); + + if (it->state == NsState::Removing) + { + if (lifecycle_refusal) + *lifecycle_refusal = true; + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}' is Removing: creation waits for its terminal fold and catalog " + "removal to complete; retry later", ns.string())); + } + + /// `Creating`, with a strict-grammar creator fence (`CatalogEntry`'s own invariant guarantees + /// `it->creator` is set whenever `state == Creating`). If it names THIS mount's own currently + /// live fence, an earlier attempt of this SAME open landed step 1 (`casAdmitEntry`) but not + /// steps 2/3 -- e.g. a transient failure inside `completeCreation`'s own `publishCkpt` retry -- + /// and resuming is simply re-running steps 2/3 over the entry as observed just now. Reasoning + /// about fence terminality for our OWN live fence would never terminate (we are, by definition, + /// not dead), so this case is checked FIRST and unconditionally, before any terminality probe. + if (it->creator->server_root_id == server_root_id && it->creator->writer_epoch == live_epoch) + { + const auto outcome = CasRefCatalog::completeCreation( + backend, layout, *it, admitted_generation, check_fence_or_throw, deadline); + if (outcome == CasRefCatalog::NamespaceCreationOutcome::FencedOut) + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': the mount incarnation moved while " + "resuming its own stalled creation; nothing was written and nothing installed", + ns.string())); + continue; /// Live or Superseded: re-read either way + } + + /// A DIFFERENT actor's `Creating` entry. It may still be mid-flight (retry later, against a + /// fresh read -- never busy-loop this instant) or provably dead, in which case reconciliation + /// steals it onto our own fence and this open resumes `completeCreation` itself. + const auto reconcile_outcome = CasRefCatalog::reconcileStaleCreator( + backend, layout, *it, our_fence, + [this](const CreatorFence & f) { return isCreatorFenceTerminal(backend, layout, f.server_root_id, f.writer_epoch); }, + admitted_generation, check_fence_or_throw); + switch (reconcile_outcome) + { + case CasRefCatalog::ReconcileCreatorOutcome::FencedOut: + /// Review I6: our OWN mount fence moved before the steal CAS -- nothing was written, and + /// this mount is the wrong actor to retry (its incarnation is gone). + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': the mount incarnation moved while " + "reconciling a stalled foreign creator; nothing was written and nothing installed", + ns.string())); + case CasRefCatalog::ReconcileCreatorOutcome::Reconciled: + { + CatalogEntry resumed = *it; + resumed.creator = our_fence; + const auto outcome = CasRefCatalog::completeCreation( + backend, layout, resumed, admitted_generation, check_fence_or_throw, deadline); + if (outcome == CasRefCatalog::NamespaceCreationOutcome::FencedOut) + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': the mount incarnation moved while " + "completing a reconciled creation; nothing was written and nothing installed", + ns.string())); + continue; /// Live or Superseded: re-read either way + } + case CasRefCatalog::ReconcileCreatorOutcome::CreatorFenceStillLive: + if (lifecycle_refusal) + *lifecycle_refusal = true; + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': its catalog entry is still Creating " + "under a creator fence that is not yet provably dead; retry later", ns.string())); + case CasRefCatalog::ReconcileCreatorOutcome::EntryChanged: + continue; /// token-exactness failed: someone else already moved this entry; re-read + } + } + + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': its catalog entry did not converge to a resolvable " + "incarnation after {} attempts", ns.string(), kMaxResolveAttempts)); +} + +void CasRefLedger::ensureRefTableRecovered(const RootNamespace & ns, RefTableRuntime & rt) +{ + { + std::unique_lock lock(rt.state_mutex); + /// Every touch, warm or cold, + /// marks this table most-recently-used so `enforceRefTableCacheBudget` evicts idle tables first. + rt.last_touch_tick = ref_table_access_tick.fetch_add(1, std::memory_order_relaxed) + 1; + + /// `NeedsRecovery` is a hard lane fence: a transaction is known durable but this cache could not + /// install it. Replaying the durable stream is the only transition back to `Ready`. + const auto needs_rerecovery = [&rt] + { + return rt.lane_state == RefLaneState::NeedsRecovery; + }; + if (rt.recovered && !needs_rerecovery()) + return; + + /// A concurrent second caller waits here rather than racing an independent walk against the first + /// caller's unlocked I/O. + while (rt.recovery_in_progress) + { + ++rt.recovery_waiters_for_test; + rt.recovery_cv.wait(lock); + --rt.recovery_waiters_for_test; + } + if (rt.recovered && !needs_rerecovery()) + return; /// the caller we waited on already finished it + + rt.recovery_in_progress = true; + /// Cleared + broadcast on every exit from here, success or exception -- so a parked waiter is never + /// left hanging, and so the self-remount barrier's JOIN completes on a failed attempt as surely as + /// on a successful one. + SCOPE_EXIT({ + rt.recovery_in_progress = false; + rt.recovery_cv.notify_all(); + }); + + /// ---- Step 1: capture the admitted generation, ONCE ---- + /// The trio (spec §3, codex finding 7): this ONE value is what the walk's every `slotOccupy` and its + /// `_ckpt` CAS present, and what the install below presents one final time. One capture point, three + /// checks, no re-derivation -- a value re-read midway would let a recovery that lost the mount + /// "recover" its right to write by observing a fresh incarnation it was never admitted under. + /// + /// Captured for the WHOLE call, not per attempt, for the same reason: the transient-retry loop below + /// exists for object-store blips, and a generation that moved is not one. The loop refuses to + /// re-drive under a moved generation (below), so the budget is never burned on a doomed retry. + const uint64_t admitted_generation = rt.admitted_fence_generation; + check_fence_or_throw(admitted_generation); + /// Preserve this runtime's exact writer identity across the unlocked walk. The runtime stays in + /// `NeedsRecovery` until the same lock installs a result, so no later append can replace it here. + const std::optional retained_attempt = rt.append_attempt; + /// The live writer epoch, likewise captured once: it decides WHICH epochs are dead and therefore what + /// the walk may seal. Re-reading it mid-walk would let the boundary move under the decision. + const uint64_t live_epoch = live_epoch_fn(); + + /// Outer transient-retry loop (Layer 1 of the stuck-table-load fix): a whole recovery attempt that + /// fails with a TRANSIENT object-store transport error (`isTransientRecoveryError` -- `S3_ERROR`/ + /// socket/timeout from the direct LIST/GET backend calls, or a controller `NETWORK_ERROR`) is retried + /// with capped-exponential backoff until `recovery_retry_budget_ms` is spent, instead of propagating + /// and failing this table's async load permanently. Non-transient errors (corruption, decode, a moved + /// fence, logic, resource limits) fail fast; so do the two LATCHED terminal cases below. + const uint64_t recovery_start_ms = boot_ms_now_fn(); + uint64_t recovery_retry_num = 0; + bool vanish_brake_tripped = false; + bool cancelled = false; + for (;;) + { + try + { + std::optional hole_detail; + for (uint64_t attempt = 0; ; ++attempt) + { + if (attempt > 0) + { + if (attempt > kRefRecoveryMaxRestarts) + { + /// Terminal, NOT a transient object-store outage: latch so the outer retry loop + /// rethrows immediately instead of re-driving this brake for the whole budget. + vanish_brake_tripped = true; + if (hole_detail) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref-table recovery for namespace '{}' found a hole in the durable ref-log " + "stream that persisted across {} re-reads: {}", + ns.string(), attempt - 1, *hole_detail); + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}' restarted {} times (a selected snapshot " + "kept vanishing between the checkpoint that named it and its GET) — giving up; this " + "bound is a runaway brake against a pathological cleanup race, not an expected " + "steady state", ns.string(), attempt - 1)); + } + ++rt.recovery_restarts; + ProfileEvents::increment(ProfileEvents::CASRefRecoveryRestarts); + } + + /// Each attempt derives its own restart reason, so a prior hole is not misreported as a + /// selected-base vanish after the next attempt takes a different path. + hole_detail.reset(); + + /// The whole walk runs UNLOCKED. Nothing it touches is shared: the candidate is private + /// and `rt` is read only through atomics the poll consults. Unlocking is what keeps a + /// recovery's full I/O envelope from stalling every reader of this table -- and what lets + /// the self-remount barrier take the lock promptly to JOIN us. + lock.unlock(); + std::optional walked; + try + { + walked = runRecoveryWalkOnce( + ns, rt, admitted_generation, live_epoch, retained_attempt, hole_detail, cancelled); + } + catch (...) + { + /// Re-acquire BEFORE letting anything propagate: the SCOPE_EXIT above mutates + /// `recovery_in_progress` and notifies `recovery_cv`, and it MUST run with + /// `state_mutex` held -- unwinding through it unlocked would be a data race on the + /// plain bool and an unlocked notify. + lock.lock(); + throw; + } + lock.lock(); + + if (!walked) + continue; /// restart requested (vanished base, or a hole worth one more reading) + + /// ---- Step 8: the install recheck, the LAST member of the trio ---- + /// Under the re-acquired lock and IMMEDIATELY before the install, present the admitted + /// generation one final time. Everything above ran on I/O that took an unbounded amount + /// of time to come back, and a recovery whose window straddled a fence bump describes a + /// mount incarnation that no longer owns this namespace. It must publish NOTHING: the + /// table stays unrecovered and the next touch recovers it properly. + check_fence_or_throw(admitted_generation); + if (rt.catalog_life_invalidated.load(std::memory_order_acquire)) + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': catalog retirement invalidated life {} " + "before recovery install — nothing is installed", + ns.string(), renderIncarnation(rt.life.incarnation))); + if (rt.superseded_by_remount.load(std::memory_order_acquire)) + throwCasWriteRetryLater(fmt::format( + "CAS ref-table recovery for namespace '{}': this cached table was superseded by a " + "self-remount while the walk was in flight — nothing is installed", + ns.string())); + + /// One atomic publication under `state_mutex`: `installRecoveryResult` copies every seeded + /// field from the result and sets `recovered` LAST, so no waiter (woken only by the + /// function-scope SCOPE_EXIT's `recovery_cv` notify, which runs after this returns) ever + /// observes a partially-installed table. + installRecoveryResult(rt, std::move(*walked)); + recovery_install_count_for_test.fetch_add(1, std::memory_order_relaxed); + break; + } + break; /// recovery succeeded -> exit the outer retry loop + } + catch (...) + { + /// `catch (...)`, not `catch (const Exception &)`: recovery exact-GET failures can surface as + /// a raw object-storage transport exception (even a non-`DB::Exception` Poco timeout), which + /// a `catch (const Exception &)` would not even see. `getCurrentExceptionCode()` normalises + /// every exception (DB, Poco, std) to a code so the transient classifier can decide. + const int code = getCurrentExceptionCode(); + /// `cancelled` is latched by the poll itself: a self-remount's cancellation uses the + /// retry-later class (the caller should retry, against the FRESH incarnation), so without the + /// latch this loop would read it as a transient blip and re-drive the very work the remount + /// just stopped. + if (vanish_brake_tripped || cancelled || !isTransientRecoveryError(code)) + throw; /// a latched terminal case, or a non-transient failure -- fail fast + + const uint64_t elapsed_ms = boot_ms_now_fn() - recovery_start_ms; + /// Fail closed BEFORE sleeping: budget spent, mount fence lost, this runtime superseded by a + /// self-remount, or the incarnation that admitted this recovery has moved (retrying under a + /// generation that is already stale can only ever be refused at the install). + if (elapsed_ms >= cas_request_budget.recovery_retry_budget_ms + || !fence_ok_fn() + || rt.catalog_life_invalidated.load(std::memory_order_acquire) + || rt.superseded_by_remount.load(std::memory_order_acquire) + || fence_generation_fn() != admitted_generation) + throw; + + /// Saturating `initial << recovery_retry_num` (mirrors `CasRequestController::backoffBefore + /// Attempt`): `initial > cap >> n` implies the unshifted product already exceeds the cap, so + /// return the cap without ever computing an overflowing/UB shift for large retry counts. + const uint64_t init_backoff = cas_request_budget.recovery_retry_initial_backoff_ms; + const uint64_t cap_backoff = cas_request_budget.recovery_retry_max_backoff_ms; + const uint64_t backoff_ms = (recovery_retry_num >= 63 || init_backoff > (cap_backoff >> recovery_retry_num)) + ? cap_backoff + : std::min(cap_backoff, init_backoff << recovery_retry_num); + ++recovery_retry_num; + ProfileEvents::increment(ProfileEvents::CASRefRecoveryRetries); + LOG_WARNING(getLogger("CasRefLedger"), + "CAS ref-table recovery for namespace '{}' hit a transient object-store error " + "(code {}: {}); retry #{} after {}ms backoff (elapsed {}ms / budget {}ms)", + ns.string(), code, getCurrentExceptionMessage(/*with_stacktrace=*/false), + recovery_retry_num, backoff_ms, elapsed_ms, cas_request_budget.recovery_retry_budget_ms); + + lock.unlock(); + /// Re-acquire the lock before letting any exception from the sleep unwind, so the SCOPE_EXIT + /// (which mutates `recovery_in_progress` + notifies `recovery_cv` and MUST run under + /// `state_mutex`) never runs unlocked -- same obligation as the walk's window above. + try + { + recovery_retry_sleep_fn(backoff_ms); + } + catch (...) + { + lock.lock(); + throw; + } + lock.lock(); + /// The fence/supersession/budget can all change during the unlocked sleep -- re-check before + /// starting the next full attempt so we never re-drive recovery on an orphaned runtime, past + /// the budget, or under a lost or moved fence (the sliced sleep may have woken early on fence + /// loss). + if (boot_ms_now_fn() - recovery_start_ms >= cas_request_budget.recovery_retry_budget_ms + || !fence_ok_fn() + || rt.catalog_life_invalidated.load(std::memory_order_acquire) + || rt.superseded_by_remount.load(std::memory_order_acquire) + || fence_generation_fn() != admitted_generation) + throw; + /// loop: re-run recovery from a fresh checkpoint read, exact replay and walk + } + } + } + + /// A NEW table was just materialized; enforce the whole-table cache budget, protecting this one + /// The pass runs OUTSIDE `rt.state_mutex` (that scope closed above) so + /// the pass -- which acquires `ref_queue_mutex` and try-locks other tables' `state_mutex` -- never + /// nests this table's `state_mutex` under `ref_queue_mutex`. + enforceRefTableCacheBudget(ns); +} + +void CasRefLedger::installRecoveryResult(RefTableRuntime & rt, RecoveryResult && result) +{ + /// One place that seeds a recovered table's runtime, copying EVERY `RecoveryResult` field so the + /// publication cannot drift from the struct. `recovered` is set LAST: the caller holds `state_mutex` + /// throughout and the function-scope SCOPE_EXIT notifies `recovery_cv` only after this returns, so a + /// parked waiter re-checking `recovered` under the same lock sees a fully-installed table or none. + rt.state = std::move(result.state); + rt.newest_snapshot_id = result.newest_snapshot_id; + /// The chain link the CAS-walk ended on -- the `prev_epoch_seal` this table's next sequence-1 append + /// must name. This is the PRODUCTION producer of `last_epoch_seal` (the two writer-side arms record + /// only what they happened to observe): a real epoch change arrives with a self-remount, which + /// discards every cached runtime, so the fresh one gets its link from exactly here. + rt.last_epoch_seal = result.last_epoch_seal; + rt.tail_count_since_snapshot.store(result.tail_count, std::memory_order_relaxed); + rt.tail_bytes_since_snapshot.store(result.tail_bytes, std::memory_order_relaxed); + rt.base_snapshot_bytes.store(result.base_snapshot_bytes, std::memory_order_relaxed); + rt.snapshot_budget = result.snapshot_budget; + rt.removal_budget = result.removal_budget; + rt.needs_stale_precommit_sweep = result.needs_stale_precommit_sweep; + rt.append_attempt.reset(); + rt.lane_state = RefLaneState::Ready; + rt.recovered = true; /// set LAST +} + +void CasRefLedger::cancelRecoveriesAndAwaitQuiescence() +{ + /// Snapshot the runtimes (the copies keep them alive across the wait, exactly as + /// `quiesceRefTablesForRemount` does). + std::vector> tables; + { + std::lock_guard qlock(ref_queue_mutex); + tables.reserve(ref_name_slots.size()); + for (auto & [name, slot] : ref_name_slots) + if (slot.current) + tables.push_back(slot.current); + } + + /// Publish the request to EVERY table first, then wait -- never table-by-table. Requesting and + /// waiting in one pass would let a recovery start on table B while we are still parked on table A, + /// and we would then join work that began after the cancellation was already under way. + for (auto & rt : tables) + rt->recovery_cancel_requested.store(true, std::memory_order_release); + + for (auto & rt : tables) + { + std::unique_lock slock(rt->state_mutex); + rt->recovery_cv.wait(slock, [&] { return !rt->recovery_in_progress; }); + } + + /// Released once nothing is in flight. Clearing here rather than after the fence re-arm keeps this a + /// self-contained barrier with no obligation on the caller to unwind: the window it opens (a recovery + /// starting between here and the re-arm) is closed twice over by the re-arm's own generation bump and + /// by `quiesceRefTablesForRemount`'s `superseded_by_remount`, both of which the walk polls. + for (auto & rt : tables) + rt->recovery_cancel_requested.store(false, std::memory_order_release); +} + + +void CasRefLedger::enforceRefTableCacheBudget(const RootNamespace & keep_ns) +{ + if (config.ref_table_cache_bytes == 0) + return; /// 0 = unbounded: eviction disabled + + /// Evicted runtimes are held alive here until AFTER every lock is released, so a runtime whose sole + /// owner is its map slot is never destroyed while we still hold its `state_mutex` (that would destroy + /// a locked mutex). + std::vector> evicted; + { + std::lock_guard qlock(ref_queue_mutex); + + /// Relaxed atomic reads: the `total` loop below reads this for EVERY table, including hot ones a + /// concurrent append lane is mutating under `state_mutex` only (a cross-lock read). The gated + /// candidate loop reads it too, but only for `use_count()==1` tables (no concurrent writer). + const auto weightOf = [](const RefTableRuntime & rt) + { + return rt.base_snapshot_bytes.load(std::memory_order_relaxed) + + rt.tail_bytes_since_snapshot.load(std::memory_order_relaxed); + }; + + uint64_t total = 0; + for (const auto & [name, slot] : ref_name_slots) + if (slot.current) + total += weightOf(*slot.current); + if (total <= config.ref_table_cache_bytes) + return; + + /// Idle candidates, least-recently-touched first. Idle == the map holds the SOLE `shared_ptr` + /// (`use_count() == 1`: no in-flight caller, queued append, leader, or background publish holds a + /// copy), no active queue leader, an empty pending queue, and not the just-recovered `keep_ns`. + /// The `use_count() == 1` gate is what makes append-lane split-brain impossible: any thread that + /// fetched this runtime keeps it non-evictable for as long as it holds the copy. + struct Cand { String name; uint64_t tick; uint64_t weight; }; + std::vector cands; + for (const auto & [name, slot] : ref_name_slots) + { + if (name == keep_ns.string()) + continue; + const auto & rt = slot.current; + if (!rt) + continue; + if (rt.use_count() != 1 || rt->leader_active || !rt->pending.empty()) + continue; + cands.push_back(Cand{name, rt->last_touch_tick, weightOf(*rt)}); + } + std::sort(cands.begin(), cands.end(), + [](const Cand & a, const Cand & b) { return a.tick < b.tick; }); + + for (const Cand & c : cands) + { + if (total <= config.ref_table_cache_bytes) + break; + auto it = ref_name_slots.find(c.name); + if (it == ref_name_slots.end()) + continue; + std::shared_ptr & rt = it->second.current; + if (!rt) + continue; + { + /// `use_count() == 1` guarantees no other thread holds the runtime, so this try_lock + /// cannot fail; the RAII scope releases `state_mutex` before `rt` is moved out. A wedged + /// append lane is never evicted -- its uncertain in-flight PUT is not reconstructable from + /// the durable objects, and re-recovery could re-allocate an id: + /// Linearization forbids this). + std::unique_lock slock(rt->state_mutex, std::try_to_lock); + if (!slock.owns_lock() || rt->lane_state != RefLaneState::Ready) + continue; + } + if (rt.use_count() != 1 || rt->leader_active || !rt->pending.empty()) + continue; /// re-check under the still-held ref_queue_mutex + total -= c.weight; + evicted.push_back(std::move(rt)); /// keep alive past the erase and lock release + ref_name_slots.erase(it); + ProfileEvents::increment(ProfileEvents::CASRefTableEvictions); + } + } + /// `evicted` destructs the dropped runtimes here, with no lock held. +} + + +void CasRefLedger::quiesceRefTablesForRemount() +{ + /// Snapshot the current runtimes (copies keep them alive across the drain). New dispatches are + /// already suppressed while the fence is lost (`maybeScheduleSnapshotPublish`'s fence guard), so the + /// only publishers to drain are those dispatched before the fence dropped. + std::vector> tables; + { + std::lock_guard qlock(ref_queue_mutex); + tables.reserve(ref_name_slots.size()); + for (auto & [name, slot] : ref_name_slots) + if (slot.current) + tables.push_back(slot.current); + } + + /// Wait for every in-flight background publisher to finish so none is mid-PUT when its runtime is + /// detached. A publisher observes the lost fence (`fence_ok` false) and returns without committing, + /// then decrements `pending_snapshot_publishes` under `state_mutex` and signals `publish_settle_cv`. + for (auto & rt : tables) + { + std::unique_lock slock(rt->state_mutex); + rt->publish_settle_cv.wait(slock, + [&] { return rt->pending_snapshot_publishes.load(std::memory_order_relaxed) == 0; }); + } + + /// Detach every cached table. Mark it superseded FIRST (release, and before the caller re-arms the + /// fence): a leader that raced in and holds one of these orphaned runtimes then fails closed at the + /// `flushRefBatch` gate rather than allocating an id against a stale cache under the re-armed fence. + /// Queued callers self-drain -- each `flushRefBatch` for a superseded runtime completes its whole + /// carved batch with a retry error, so no caller hangs; the next touch creates a fresh runtime that + /// re-recovers from the durable snapshot+log objects under `live_writer_epoch`. Dropping the map slot + /// discards each runtime's in-memory wedge, and nothing about that drop needs to be certified here: + /// the undecided `PUT` the wedge describes is settled by the durable protocol rather than by + /// bookkeeping this process carries across the boundary. Recovery closes the dead epoch with an + /// in-band `EpochSeal` written as a conditional create, so the wedged write either already landed + /// (and the arithmetic walk reads it) or loses its own create to the seal. + std::vector> detached; + { + std::lock_guard qlock(ref_queue_mutex); + detached.reserve(ref_name_slots.size()); + for (auto & [name, slot] : ref_name_slots) + { + auto & rt = slot.current; + if (!rt) + continue; + rt->superseded_by_remount.store(true, std::memory_order_release); + rt->cv.notify_all(); /// wake any waiter so it re-leads and fails closed against the flag + detached.push_back(rt); + } + ref_name_slots.clear(); + } + /// `detached` releases the map's references here (with no lock held); each runtime lives on only as + /// long as an in-flight leader/caller still holds it. +} + + +uint64_t CasRefLedger::refRecoveryRestartsForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return 0; + std::lock_guard lock(rt->state_mutex); + return rt->recovery_restarts; +} + +bool CasRefLedger::refLaneWedgedForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return false; + std::lock_guard lock(rt->state_mutex); + return rt->lane_state == RefLaneState::Wedged; +} + +String CasRefLedger::wedgedKeyForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return {}; + std::lock_guard lock(rt->state_mutex); + return rt->append_attempt ? rt->append_attempt->key : String{}; +} + +uint64_t CasRefLedger::wedgedAdmittedGenerationForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return 0; + std::lock_guard lock(rt->state_mutex); + return rt->append_attempt ? rt->append_attempt->admitted_fence_generation : 0; +} + +std::optional CasRefLedger::lastEpochSealForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return std::nullopt; + std::lock_guard lock(rt->state_mutex); + return rt->last_epoch_seal; +} + +void CasRefLedger::setLastEpochSealForTest(const RootNamespace & ns, const std::optional & seal) +{ + const auto rt = acquireMutableRefTableRuntime(ns); + ensureRefTableRecovered(ns, *rt); + std::lock_guard lock(rt->state_mutex); + rt->last_epoch_seal = seal; +} + +void CasRefLedger::forceWedgeForTest(const RootNamespace & ns, uint64_t writer_epoch, uint64_t ref_sequence, + const String & key, const String & bytes, + std::optional admitted_generation) +{ + const auto rt = acquireMutableRefTableRuntime(ns); + ensureRefTableRecovered(ns, *rt); + /// Read outside `state_mutex`: it is an atomic load on the mount runtime, and taking it here keeps + /// the seam's default identical to what a wedge born at this instant would carry. + const uint64_t generation = admitted_generation.value_or(fence_generation_fn()); + std::lock_guard lock(rt->state_mutex); + rt->append_attempt = RefAppendAttempt{RefTxnId{writer_epoch, ref_sequence}, key, bytes, generation}; + rt->lane_state = RefLaneState::Wedged; +} + +RefLaneState CasRefLedger::laneStateForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return RefLaneState::Closed; + std::lock_guard lock(rt->state_mutex); + return rt->lane_state; +} + +bool CasRefLedger::needsStalePrecommitSweepForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return false; + std::lock_guard lock(rt->state_mutex); + return rt->needs_stale_precommit_sweep; +} + + +size_t CasRefLedger::wedgedRefLaneCount() +{ + std::vector> runtimes; + { + std::lock_guard g(ref_queue_mutex); + runtimes.reserve(ref_name_slots.size()); + for (const auto & [_, slot] : ref_name_slots) + if (slot.current) + runtimes.push_back(slot.current); + } + size_t wedged = 0; + for (const auto & rt : runtimes) + { + std::lock_guard lock(rt->state_mutex); + if (rt->lane_state == RefLaneState::Wedged) + ++wedged; + } + return wedged; +} + + +bool CasRefLedger::drainRefLanesForShutdown(uint64_t wait_budget_ms) +{ + /// Latch FIRST, then snapshot under `ref_queue_mutex` (see the `shutting_down` member comment): this + /// ordering is what makes the check in `appendRefOps` -- performed inside the SAME critical section + /// as its `pending.push_back` -- race-free against the snapshot below, for both an already-cached + /// table and one whose very first touch races this call. + shutting_down.store(true, std::memory_order_release); + + std::vector> runtimes; + { + std::lock_guard g(ref_queue_mutex); + runtimes.reserve(ref_name_slots.size()); + for (const auto & [_, slot] : ref_name_slots) + if (slot.current) + runtimes.push_back(slot.current); + } + + /// Wait for every table's queue to go idle (no pending item, no active leader), bounded overall by + /// `wait_budget_ms` -- `cv.wait_until` slices against one shared deadline, never a sleep. All the + /// runtimes share the one `ref_queue_mutex` that guards `pending`/`leader_active` (see the + /// `RefTableRuntime` field comments), so a single `lk` covers every table in the loop below; each + /// table's OWN `cv` is what its leader/appendRefOps notifies on a state change, so the wait must + /// target that specific `cv`, one table at a time. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(wait_budget_ms); + bool timed_out = false; + { + std::unique_lock lk(ref_queue_mutex); + for (const auto & rt : runtimes) + { + while (!(rt->pending.empty() && !rt->leader_active)) + { + if (rt->cv.wait_until(lk, deadline) == std::cv_status::timeout + && !(rt->pending.empty() && !rt->leader_active)) + { + timed_out = true; + break; + } + } + if (timed_out) + break; + } + } + + /// A queue going idle does NOT by itself prove no PUT is in flight: a wedge is recorded (under + /// `state_mutex`) strictly BEFORE the wedged item's caller is completed and the leader bookkeeping + /// reset (see `flushRefBatch`'s `Unresolved` case), so this check -- performed AFTER the wait above + /// -- observes it whenever the queue-idle wait itself raced a wedge. Every table is checked + /// regardless of `timed_out`, purely for a complete diagnostic; the return value already fails + /// closed on either condition alone. + bool any_wedge = false; + for (const auto & rt : runtimes) + { + std::lock_guard lock(rt->state_mutex); + if (rt->lane_state == RefLaneState::Writing || rt->lane_state == RefLaneState::Wedged) + any_wedge = true; + } + + return !timed_out && !any_wedge; +} + + +RefTxnId CasRefLedger::appendRefOps(const RootNamespace & ns, MutationScope scope, + std::function(const RefTableState &)> build_ops, + RootMutationOrigin origin, RootMutationKind kind, + bool skip_stale_precommit_sweep) +{ + if (kind == RootMutationKind::DropNamespace) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS namespace '{}': the generic append surface cannot acquire removal ownership; " + "use the exact `dropNamespace` lifecycle operation", + ns.string()); + const auto rt = acquireMutableRefTableRuntime(ns); + if (append_after_runtime_capture_hook_for_test) + append_after_runtime_capture_hook_for_test(); + return appendRefOpsOnRuntime( + ns, rt, std::move(scope), std::move(build_ops), origin, kind, skip_stale_precommit_sweep, + /*terminal_removal_authorized=*/false); +} + + +RefTxnId CasRefLedger::appendRefOpsOnRuntime( + const RootNamespace & ns, const std::shared_ptr & rt, MutationScope scope, + std::function(const RefTableState &)> build_ops, + RootMutationOrigin origin, RootMutationKind kind, bool skip_stale_precommit_sweep, + bool terminal_removal_authorized) +{ + const auto refuse_if_removing = [&] + { + std::lock_guard lock(ref_queue_mutex); + if (rt->removal_admission_closed && !terminal_removal_authorized) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}' is Removing: positive ref mutation admission is closed while " + "its terminal fold and catalog removal complete; retry later", ns.string())); + }; + /// Check before recovery/maintenance so a known-Removing runtime cannot spend an object-store + /// mutation on behalf of an operation it is already required to refuse. + refuse_if_removing(); + /// Hoisted here (rather than left to `flushRefBatch`'s own idempotent call) so both + /// triggers below run on the CALLING thread, strictly BEFORE this call enqueues its own item or + /// becomes a queue leader -- `maybeSweepStalePrecommits`'s own nested `appendRefOps` calls are + /// therefore always a fresh top-level invocation, never nested inside a leader's flush stack + /// (which would deadlock the leader against itself). + ensureRefTableRecovered(ns, *rt); + if (!skip_stale_precommit_sweep) + maybeSweepStalePrecommits(ns, rt); + maybeScheduleSnapshotPublish(ns, rt); + + auto item = std::make_shared(); + item->scope = std::move(scope); + item->build_ops = std::move(build_ops); + item->origin = origin; + item->kind = kind; + item->terminal_removal_authorized = terminal_removal_authorized; + + const auto enqueued_at = std::chrono::steady_clock::now(); + std::unique_lock lk(ref_queue_mutex); + /// Refuse admission once a clean-release drain has begun (`drainRefLanesForShutdown`). + /// Checked in the SAME critical section as the `pending.push_back` below -- the pairing that makes + /// this race-free against the drain's snapshot-and-wait (see the `shutting_down` member comment). + if (shutting_down.load(std::memory_order_acquire)) + throwCasWriteRetryLater(fmt::format( + "CAS store is shutting down — refusing to append ref-log transactions for server_root '{}'", + config.server_root_id)); + if (rt->removal_admission_closed && !terminal_removal_authorized) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}' is Removing: positive ref mutation admission is closed while its " + "terminal fold and catalog removal complete; retry later", ns.string())); + rt->pending.push_back(item); + + while (!item->done) + { + if (!rt->leader_active) + { + /// The set of items THIS leader is responsible for: its own enqueued `item` plus every item a + /// flush carves out of `pending` (recorded by `flushRefBatch` as it carves). Whatever the + /// leader loop does below -- return normally, or throw at ANY point including BEFORE it ever + /// carves -- every one of these items must leave here `done` (its waiter woken), never left + /// stranded in `pending` for a future leader to carve after this caller's stack (and its + /// `build_ops` closure) is gone: a use-after-free that concurrent per-part commit makes + /// immediate. `completeOwnedItemsAndReleaseLeadership` enforces that on EVERY exit and folds in + /// the `leader_active` release the old catch used to own. Items NOT owned by this leader (other + /// callers' still-queued items) are untouched -- they stay validly owned by their blocked + /// callers. + /// + /// Build the responsibility set (its own `item`) BEFORE publishing the baton, so becoming + /// leader contains NO throwing operation once `leader_active` is set: the only allocation is + /// this first `push_back`, done here while still holding `lk` and NOT yet leader. If it throws + /// (a `bad_alloc` at the pre-tenure point; codex stage-1 review, Important), the baton is never + /// taken -- but `item` is already in `pending` (pushed above), so it must be un-enqueued before + /// propagating, else a future leader would carve an item whose `build_ops` closure died with + /// this unwinding caller (the same use-after-free the exit guard prevents post-publication). + /// Publishing the baton and reaching the exit guard is then a pure no-throw sequence. + std::vector> owned_items; + try + { + if (ref_pre_tenure_hook_for_test) + ref_pre_tenure_hook_for_test(); + owned_items.push_back(item); + } + catch (...) + { + std::erase(rt->pending, item); + throw; + } + + rt->leader_active = true; + lk.unlock(); + std::exception_ptr flush_exception; + try + { + runRefQueueLeader(ns, rt, item, owned_items); + } + catch (...) + { + flush_exception = std::current_exception(); + } + /// Single exit authority (normal AND exceptional): complete every still-incomplete owned + /// item with `flush_exception` (nullptr on the normal path -> a fail-closed LOGICAL_ERROR) + /// and release leadership. This does NOT rethrow. Under chunked flush the leader's OWN item + /// may already have succeeded in an earlier committed chunk, and a later exception -- from a + /// subsequent chunk, the reseed, or chunk-N processing -- must NOT be handed to this caller + /// whose mutation is already durable (tenure exception containment, spec §3): the guard + /// leaves such an item `done` with no error, and the loop re-check + tail below return its + /// `committed_id`. An item that genuinely failed carries `item->error` and the tail rethrows + /// it, exactly as the old unconditional rethrow did for the single-chunk case. + completeOwnedItemsAndReleaseLeadership(ns, rt, owned_items, flush_exception); + lk.lock(); + } + else + { + rt->cv.wait(lk); + } + } + lk.unlock(); + + ProfileEvents::increment(ProfileEvents::CASRefQueueWaitMicroseconds, + std::chrono::duration_cast( + std::chrono::steady_clock::now() - enqueued_at).count()); + if (item->error) + std::rethrow_exception(item->error); + return item->committed_id; +} + + +void CasRefLedger::runRefQueueLeader(const RootNamespace & ns, const std::shared_ptr & rt, + const std::shared_ptr & own, + std::vector> & owned_items) +{ + /// Fairness baton pass: serve flushes only until the caller's OWN item is done, then hand off to a + /// woken waiter. + while (true) + { + { + std::lock_guard g(ref_queue_mutex); + if (own->done) + return; + } + flushRefBatch(ns, rt, owned_items); + } +} + +void CasRefLedger::completeOwnedItemsAndReleaseLeadership( + const RootNamespace & ns, const std::shared_ptr & rt, + const std::vector> & owned_items, + std::exception_ptr flush_exception) +{ + std::lock_guard g(ref_queue_mutex); + for (const auto & owned : owned_items) + { + if (!owned->done) + { + owned->error = flush_exception + ? flush_exception + : std::make_exception_ptr(Exception(ErrorCodes::LOGICAL_ERROR, + "CAS ref-log append for namespace '{}': the append-lane leader exited without " + "completing an owned queue item -- failing it closed rather than leaving it stranded " + "in the pending queue for a future leader to carve", ns.string())); + owned->done = true; + } + /// Never leave an owned item in `pending`: a stranded item would be carved by a future leader + /// which would then invoke its (now dangling) `build_ops` closure -- the use-after-free this + /// guard exists to prevent. Carved items were already popped during the carve, so this is a + /// no-op for them; it only matters for an item the leader owned but never got to carve. + std::erase(rt->pending, owned); + } + rt->leader_active = false; + rt->cv.notify_all(); +} + +void CasRefLedger::requireRecovery(RefTableRuntime & rt, const RootNamespace & ns, std::string_view region) noexcept +{ + const bool entering = rt.lane_state != RefLaneState::NeedsRecovery; + rt.lane_state = RefLaneState::NeedsRecovery; + if (!entering) + return; + ProfileEvents::increment(ProfileEvents::CASRefNeedsRecovery); + try + { + LOG_ERROR(getLogger("CasPool"), + "CAS ref table '{}' NEEDS RECOVERY at {}: a transaction is known durable but could not be " + "installed in this cached table. New writes, snapshots, and confirmations are fenced until " + "recovery replays the durable log.", + ns.string(), region); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// The state transition above is the safety mechanism; logging must not replace the original + /// post-durable exception. + } +} + +CasRefLedger::WedgeResolutionResult +CasRefLedger::resolveWedgeOnce(const RootNamespace & ns, const std::shared_ptr & rt) +{ + WedgeResolutionResult result; + + /// WHY the lane is not proceeding, decided under `state_mutex` and rendered into a message AFTER it + /// is released. Building an exception is `fmt::format` plus a stack-trace capture, neither of which + /// belongs under a lock that readers and the snapshot publisher contend for. + enum class Reason : uint8_t + { + None, + FenceMoved, /// the mount incarnation moved since this attempt was admitted + CatalogLifeRetired, /// exact catalog retirement detached this immutable life + Superseded, /// a self-remount detached this runtime + WedgeReplaced, /// the result belongs to a wedge that is no longer installed + RefusedPreAttempt, /// `slotOccupy` sent nothing + ResolveFoundNothing, /// it sent an attempt and the follow-up read came up empty + StaleState, /// the table advanced under a proven-durable object we cannot install + }; + Reason reason = Reason::None; + std::optional invalid_lane_state; + + /// ---- Read the wedge and prepare EVERYTHING that can throw, before any I/O (spec §A1, site 2) ---- + RefAppendAttempt wedge; + std::optional candidate; + RefTxnId candidate_base_id; + { + std::lock_guard lock(rt->state_mutex); + if (rt->lane_state == RefLaneState::Ready) + return result; /// NoWedge -- the ordinary flush pays nothing for any of this + if (rt->lane_state != RefLaneState::Wedged || !rt->append_attempt) + { + invalid_lane_state = rt->lane_state; + } + else + { + wedge = *rt->append_attempt; + candidate.emplace(rt->state); + candidate_base_id = rt->state.getGreatestApplied(); + } + } + + if (invalid_lane_state) + { + result.kind = WedgeResolution::StillWedged; + if (*invalid_lane_state == RefLaneState::NeedsRecovery) + { + result.survivor_error = makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for namespace '{}': lane recovery is still in progress; retry after " + "recovery completes", + ns.string())); + } + else + { + result.survivor_error = std::make_exception_ptr(Exception( + ErrorCodes::INVALID_STATE, + "CAS ref-log append for namespace '{}': terminal lane state {} does not permit wedge " + "resolution", + ns.string(), static_cast(*invalid_lane_state))); + } + return result; + } + + /// Decode and apply BEFORE the I/O. The wedge carries the encoded body, so this costs no extra + /// round trip -- only the decode and the overlay build, both of which can throw (allocation) and + /// both of which MUST NOT run after the object is proven durable. A throw here happens while the + /// outcome is still unknown and the wedge is still set, i.e. it is indistinguishable from "the + /// resolution has not been attempted yet": the lane stays wedged and a later flush retries the whole + /// thing. It propagates to `appendRefOps`' catch, which completes every survivor. + /// + /// The candidate is deliberately NOT cached in the wedge across attempts: a wedge can live until a + /// remount, and retaining a full state copy for that long is a real memory cost on a path that is + /// rare by construction. Recomputing it per attempt is the cheaper trade. + const RefLogTxn wedged_txn = decodeRefLogTxn( + openObject(FormatId::RefLog, wedge.bytes), ns.string(), wedge.txn_id); + applyRefLogTxn(*candidate, wedged_txn); + + /// ---- ONE bounded attempt, admitted under the wedge's ORIGINAL generation ---- + /// Never the CURRENT generation: a retry that "passes" because the mount was re-armed under a new + /// lease incarnation is a write from an incarnation that never admitted this transaction. Refusing + /// pre-attempt leaves the key provably untouched, which is the only state a later recovery can + /// reason about. + const auto admitted_fence_ok = [this, &rt, admitted = wedge.admitted_fence_generation] + { + return fence_ok_fn() + && !rt->catalog_life_invalidated.load(std::memory_order_acquire) + && !rt->superseded_by_remount.load(std::memory_order_acquire) + && fence_generation_fn() == admitted; + }; + + SlotOccupyResult occupied; + try + { + if (wedge_before_slot_occupy_hook_for_test) + wedge_before_slot_occupy_hook_for_test(); + occupied = ref_request_controller->slotOccupy(wedge.key, wedge.bytes, admitted_fence_ok); + } + catch (...) + { + /// `ambiguous-then-definite`, the model-proven control. `slotOccupy` rethrows only a definite + /// refusal of THIS attempt (a whitelisted synchronous rejection, or a deterministic local + /// failure) -- and a definite refusal of a LATER attempt proves nothing whatsoever about the + /// EARLIER ambiguous one, which may still be in flight or may already have landed. So the lane + /// stays wedged: unwedging here is exactly how an acked-then-lost transaction gets written + /// around. The id is not consumed either, so the next attempt re-derives the SAME one. + result.kind = WedgeResolution::StillWedged; + result.survivor_error = makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for namespace '{}': the bounded retry of wedged txn {}-{} was definitively " + "refused ({}), which says nothing about the earlier ambiguous attempt — the lane stays wedged", + ns.string(), wedge.txn_id.writer_epoch, wedge.txn_id.ref_sequence, + getCurrentExceptionMessage(/*with_stacktrace*/ false))); + return result; + } + + /// ---- Classify the occupant OFF the lock: pure, and the decode allocates ---- + /// The three-way `mine | successor's seal | foreign` adjudication is the CALLER's job by + /// construction (`slotOccupy` never compares bytes), and "mine" means BYTE EQUALITY -- never a + /// shape or generation match, which is the aliasing the phase-0 model rejected. + const Occupant occupant = occupied.kind == SlotOccupyResult::Kind::Occupied + ? classifyRefLogOccupant(ns, wedge.txn_id, occupied.occupant_bytes, wedge.bytes) + : Occupant::NotOccupied; + const bool exact_attempt_is_durable + = occupied.kind == SlotOccupyResult::Kind::Created || occupant == Occupant::Ours; + /// Caller holds `state_mutex`. Keeping the identity predicate in one place is part of the safety + /// rule: adding a frontier must not create yet another subtly different notion of "same attempt". + const auto same_wedge_under_lock = [&] + { + return rt->append_attempt + && rt->lane_state == RefLaneState::Wedged + && rt->append_attempt->txn_id == wedge.txn_id + && rt->append_attempt->bytes == wedge.bytes + && rt->append_attempt->admitted_fence_generation == wedge.admitted_fence_generation; + }; + + /// A durable ref-log object is not yet admissible history. Exactly as on the ordinary committed + /// append path, publish its frontier under the SAME admission before the cached table can install + /// it, return to `Ready`, or wake a surviving caller. This deliberately runs only for `Created` or + /// byte-identical `Ours`: a successor seal is conclusive evidence that OUR transaction did not land + /// and retains the rejection path below without publishing our frontier. + bool same_wedge_before_frontier = false; + { + std::lock_guard lock(rt->state_mutex); + same_wedge_before_frontier = same_wedge_under_lock(); + } + if (exact_attempt_is_durable && same_wedge_before_frontier) + { + const auto check_wedge_admitted = [this, &rt, &same_wedge_under_lock](uint64_t expected_generation) + { + check_fence_or_throw(expected_generation); + if (rt->catalog_life_invalidated.load(std::memory_order_acquire) + || rt->superseded_by_remount.load(std::memory_order_acquire)) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': its captured runtime was retired before wedged-frontier publication", + rt->life.ns.string())); + + bool same_wedge = false; + { + std::lock_guard lock(rt->state_mutex); + same_wedge = same_wedge_under_lock(); + } + if (!same_wedge) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': the captured wedge changed before frontier publication", + rt->life.ns.string())); + }; + + const RefCkpt frontier{ + .life_epoch = std::nullopt, + .committed_through = wedge.txn_id, + .checkpoint_snapshot_id = std::nullopt, + .last_epoch_seal = refLogTxnIsEpochSeal(wedged_txn) + ? std::optional{wedge.txn_id} : wedged_txn.prev_epoch_seal}; + + CkptPublishOutcome frontier_outcome = CkptPublishOutcome::FencedOut; + try + { + frontier_outcome = publishCkptContribution( + rt->life, frontier, wedge.admitted_fence_generation, check_wedge_admitted); + } + catch (...) + { + result.kind = WedgeResolution::StillWedged; + result.survivor_error = std::current_exception(); + std::lock_guard lock(rt->state_mutex); + if (same_wedge_under_lock()) + requireRecovery(*rt, ns, "wedged-frontier publication"); + return result; + } + if (frontier_outcome == CkptPublishOutcome::FencedOut) + { + result.kind = WedgeResolution::StillWedged; + result.survivor_error = makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for namespace '{}': wedged txn {}-{} is durable, but its admitted " + "fence moved before checkpoint-frontier publication; the lane needs recovery", + ns.string(), wedge.txn_id.writer_epoch, wedge.txn_id.ref_sequence)); + std::lock_guard lock(rt->state_mutex); + if (same_wedge_under_lock()) + requireRecovery(*rt, ns, "wedged-frontier publication fence"); + return result; + } + } + + /// ---- POST-I/O RECHECK, then act, in ONE hold of `state_mutex` ---- + /// Everything above ran on an I/O result that took an unbounded amount of time to come back. Before + /// ANY consequence follows from it -- adopting, acknowledging, unwedging, failing the survivors -- + /// this runtime must still be the one the attempt was admitted for. Two independent things can have + /// made it not: the mount fence moved (a loss, or a re-arm under a fresh lease incarnation), or the + /// wedge itself was replaced. Both are checked; neither implies the other. + { + std::lock_guard lock(rt->state_mutex); + + /// The generation this attempt was admitted under, presented back. `checkFenceOrThrow` reports a + /// moved incarnation by throwing; it is CAUGHT here rather than propagated, because the caller's + /// retry classification keys on the retry-later error class and a routine lease blip must not + /// reach it as a hard failure. Nothing is installed and nothing is unwedged either way, which is + /// the whole meaning of INERT here. + bool fence_moved = false; + try + { + check_fence_or_throw(wedge.admitted_fence_generation); + } + catch (...) + { + fence_moved = true; + } + + /// The remount half of the same question, checked separately because the two are independent + /// facts even though today's ordering makes one imply the other: `quiesceRefTablesForRemount` + /// detaches this runtime BEFORE the fence is re-armed, so a detached runtime always has a moved + /// generation and the check above would already have caught it. Relying on that ordering + /// silently is how a future edit to the remount sequence turns into a stale install. + const bool superseded = rt->superseded_by_remount.load(std::memory_order_acquire); + const bool catalog_life_retired = rt->catalog_life_invalidated.load(std::memory_order_acquire); + + /// All three components of the identity, because none of them alone identifies the attempt: two + /// attempts of one table can share an id and a generation and describe DIFFERENT bytes, and + /// installing one attempt's candidate because the other's key resolved is the acked-then-lost + /// class itself. One leader per table makes this unreachable today; it is checked, not assumed, + /// because the cost is a comparison and the failure mode is silent data loss. + const bool same_wedge = same_wedge_under_lock(); + + if ((fence_moved || superseded) && same_wedge && exact_attempt_is_durable) + { + requireRecovery(*rt, ns, "wedged attempt resolved after its fence moved"); + reason = Reason::StaleState; + result.kind = WedgeResolution::StillWedged; + } + else if (fence_moved || catalog_life_retired || superseded || !same_wedge) + { + reason = fence_moved ? Reason::FenceMoved + : (catalog_life_retired ? Reason::CatalogLifeRetired + : (superseded ? Reason::Superseded : Reason::WedgeReplaced)); + result.kind = WedgeResolution::StillWedged; + } + else if (occupied.kind == SlotOccupyResult::Kind::Unresolved) + { + /// Still uncertain. Do NOT clear the wedge: it describes an object that may well be durable, + /// and clearing it on a failed read is the one thing this path must never do. There is no + /// deadline reset and no background loop -- the next caller of this namespace, or a remount, + /// retries. That is register R6's ACCEPTED behaviour: a permanently quiet wedged namespace + /// waits, which costs nothing, because the wedged operation was never acknowledged. + reason = unresolvedProvesNothingWasSent(occupied.unresolved_reason) + ? Reason::RefusedPreAttempt : Reason::ResolveFoundNothing; + result.kind = WedgeResolution::StillWedged; + } + else if (occupant == Occupant::SuccessorSeal) + { + /// THE conclusive rejection (spec INV-2). The ref-log key is write-once and a successor put + /// its epoch-closing record there, so our bytes provably never landed and never can. The + /// operation was never acknowledged, so nothing is lost by failing it permanently -- and it + /// must be permanent, not "retry later": no later attempt in this epoch can ever succeed. + /// + /// The seal IS this namespace's epoch-closing record, so it is also the `prev_epoch_seal` + /// that the first transaction of a LATER epoch must name. In this runtime that record is + /// mostly introspection: a real epoch change arrives with a self-remount, which discards + /// this runtime, and the fresh one gets its chain link from recovery's CAS-walk (Task 6). + /// It is recorded anyway because it is durable evidence this runtime holds and nothing else + /// would, and because `commitRefChunk` consumes it the moment the live epoch does advance + /// past the seal's. + /// + rt->append_attempt.reset(); + rt->last_epoch_seal = wedge.txn_id; + rt->lane_state = RefLaneState::Closed; + result.kind = WedgeResolution::Rejected; + } + else if (occupant == Occupant::Foreign) + { + /// Impossible under mount-lease exclusivity. The terminal state carries the verdict; no + /// uncertain attempt remains to be retried. + rt->append_attempt.reset(); + rt->lane_state = RefLaneState::Faulted; + result.kind = WedgeResolution::Corrupted; + } + else if (!(rt->state.getGreatestApplied() == candidate_base_id)) + { + /// ADOPTION, refused. Only this leader mutates `rt->state`, and the attempt above ran + /// without the lock, so this compares the state against the snapshot the candidate was built + /// from. A debug build also `chassert`s it inside the region below; this is the RELEASE-mode + /// counterpart, because the window it guards is a full network round trip and a silent swap + /// would discard whatever advanced the table. + /// + /// The object is proven durable and this runtime cannot record it. No later id may be + /// allocated from this cache; recovery is the only legal successor. + requireRecovery(*rt, ns, "wedge-resolution install"); + reason = Reason::StaleState; + result.kind = WedgeResolution::StillWedged; + } + else + { + /// ---- ADOPTION: `Created`, or `Occupied` with our own bytes ---- + /// Receives the resolved wedge so it is destroyed OUTSIDE the region: clearing `rt->append_attempt` + /// in place would free its two `String` bodies there, and the region's contract is that it + /// touches no allocator at all. + std::optional displaced_wedge; + static_assert(std::is_nothrow_swappable_v>, + "the wedge hand-off below must be non-throwing: it runs after the wedged object is proven " + "durable, where a throw would re-apply the transaction on the next resolution"); + /// One of the two post-durable install regions (spec §A2; the other is `commitRefChunk`'s + /// own, further below, sharing this same probe). The `catch` cannot fire while §A1 holds -- + /// the body below allocates nothing -- and is what makes a violation of §A1 VISIBLE rather + /// than silent: the attempt proved the object durable, so an install that does not complete + /// leaves this table's cached state missing it. It rethrows unchanged, so the lane's error + /// handling is unchanged; the explicit lane state makes the missing install visible. + try + { + DENY_ALLOCATIONS_IN_SCOPE; + /// The negative control (`setInstallRegionProbeForTest`), fired with the guard already + /// armed, exactly as in `commitRefChunk`'s region. + if (install_region_probe_for_test) + install_region_probe_for_test(); + /// The debug-build twin of the refusal above. `chassert` stringifies its condition, so + /// it reads a short local rather than the comparison itself: a long condition would heap + /// allocate ON FAILURE inside the very region that must not allocate. + chassert(same_wedge); + /// The install, allocation-free by construction: a member-wise swap of pointers and + /// PODs, two atomic increments, and a second swap of pointers. The object is durable, so + /// the transaction MUST be recorded -- and recording it MUST be inseparable from clearing + /// the wedge, or a failure between them leaves the transaction applied with the wedge + /// still set and the next resolution re-applies it. A wedge-resolved transaction is a + /// commit like any other: it joins the applied-above-newest-snapshot tail counters + /// exactly as the ordinary commit arm's does, or the snapshot-publish threshold and the + /// resident-weight estimate undercount by one transaction per resolved wedge until the + /// next recovery reseeds. + rt->state.swap(*candidate); + rt->tail_count_since_snapshot.fetch_add(1, std::memory_order_relaxed); + rt->tail_bytes_since_snapshot.fetch_add(wedge.bytes.size(), std::memory_order_relaxed); + rt->append_attempt.swap(displaced_wedge); + rt->lane_state = RefLaneState::Ready; + } + catch (...) + { + requireRecovery(*rt, ns, "wedge-resolution install"); + throw; + } + /// `candidate` now holds the DISPLACED state, which still shares the COW bases `rt->state` + /// uses; destroying it here restores unique base ownership so the fold below keeps its + /// O(overlay) in-place path instead of rebuilding the whole base. Both `reset`s only destroy: + /// they allocate nothing and cannot throw. + candidate.reset(); + displaced_wedge.reset(); + /// Fold the just-installed overlay back into the base right here, exactly as the ordinary + /// commit arm does at its install point, so `rt->state` returns to "base + empty overlay" and + /// the next flush's trial copies stay cheap. Cheap: no scratch copy shares the base at this + /// point in the flush (`working` is not taken until later), so this is the O(overlay) in-place + /// fold. Coherent-on-throw (see `CasRefCowMap.cpp`), and SWALLOWING, symmetrically with the + /// ordinary commit arm: the transaction is durable, installed and unwedged before this runs, + /// so a mid-fold allocation failure merely defers the fold to the next flush -- it must not + /// unwind past a completed install. + try + { + rt->state.materializeCommitted(); + } + catch (...) + { + tryLogCurrentException(getLogger("CasPool"), fmt::format( + "CAS ref-log append for namespace '{}': wedged txn {}-{} resolved durable and was " + "installed, but the post-install overlay fold failed and was retained coherently for " + "the next flush", + ns.string(), wedge.txn_id.writer_epoch, wedge.txn_id.ref_sequence)); + } + result.kind = WedgeResolution::Adopted; + } + } + + /// ---- Everything that allocates or reacts, now that `state_mutex` is released ---- + switch (result.kind) + { + case WedgeResolution::Adopted: + ProfileEvents::increment(ProfileEvents::CASRefAppendUnwedged); + break; + case WedgeResolution::Rejected: + result.survivor_error = std::make_exception_ptr(Exception(ErrorCodes::INVALID_STATE, + "CAS ref-log append for namespace '{}': writer epoch {} was CLOSED by a successor's epoch " + "seal at {}-{}, which conclusively rejects the wedged transaction (it was never " + "acknowledged). This mount's append lane resumes only under a later epoch", + ns.string(), wedge.txn_id.writer_epoch, + wedge.txn_id.writer_epoch, wedge.txn_id.ref_sequence)); + break; + case WedgeResolution::Corrupted: + on_impossible_interference(wedge.key, + fmt::format("ref-log wedge resolution for namespace '{}' txn {}-{} observed a foreign object " + "at the wedged slot: neither this attempt's own bytes nor an epoch seal of this namespace", + ns.string(), wedge.txn_id.writer_epoch, wedge.txn_id.ref_sequence), + ns.string()); + result.survivor_error = std::make_exception_ptr(Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref-log append for namespace '{}': impossible foreign interference observed at the " + "wedged slot '{}' — the mount is fenced closed and a remount is scheduled; the lane is " + "deliberately left wedged for inspection. See the anomaly diagnostics log", + ns.string(), wedge.key)); + break; + case WedgeResolution::StillWedged: + { + const String txn = fmt::format("{}-{}", wedge.txn_id.writer_epoch, wedge.txn_id.ref_sequence); + String why; + switch (reason) + { + case Reason::FenceMoved: + why = fmt::format( + "CAS ref-log append for namespace '{}': the resolution of txn {} returned under a " + "DIFFERENT mount incarnation than the one that admitted it — the result is inert " + "and the lane keeps its wedge for whoever recovers it under the live incarnation", + ns.string(), txn); + break; + case Reason::CatalogLifeRetired: + why = fmt::format( + "CAS ref-log append for namespace '{}': catalog retirement detached life {} " + "before the bounded retry could be adopted — the result is inert and the " + "successor life is untouched", + ns.string(), renderIncarnation(rt->life.incarnation)); + break; + case Reason::Superseded: + why = fmt::format( + "CAS ref-log append for namespace '{}': the resolution of txn {} returned for a " + "table that a self-remount had already detached — the result is inert; the fresh " + "incarnation re-derives this table from the durable log", + ns.string(), txn); + break; + case Reason::WedgeReplaced: + why = fmt::format( + "CAS ref-log append for namespace '{}': the resolution of txn {} returned for a " + "wedge that is no longer installed — the result is inert and the lane keeps " + "whatever wedge it has", + ns.string(), txn); + break; + case Reason::RefusedPreAttempt: + /// The admission-generation half of a pre-attempt refusal never reaches this message: + /// the recheck above presents the same generation and reports `FenceMoved` first. + /// What is left are the two causes that leave the generation intact. + why = fmt::format( + "CAS ref-log append for namespace '{}': the bounded retry of wedged txn {} was " + "refused BEFORE any request was sent — the mount lease is not healthy enough to " + "start a write, or the operation deadline is exhausted — so the slot '{}' is " + "provably untouched by this attempt and the lane stays wedged", + ns.string(), txn, wedge.key); + break; + case Reason::ResolveFoundNothing: + /// Never a bare `describeUnresolvedReason` for this primitive (the `SlotOccupyResult` + /// doc's explicit call-site rule): `AttemptsExhausted` reads "the retry budget ran + /// out", which is nonsense for a primitive with no retry budget, and it folds two very + /// different observations together. Both are named instead. + why = fmt::format( + "CAS ref-log append for namespace '{}': the bounded retry of wedged txn {} was sent " + "and the resolve read found NOTHING at slot '{}' — either the read itself failed, " + "or the occupant that rejected the create was DELETED under a live epoch, which is " + "a GC invariant alarm rather than routine contention. The lane stays wedged until " + "the SAME slot resolves durable or a conclusive rejection is observed", + ns.string(), txn, wedge.key); + break; + case Reason::StaleState: + why = fmt::format( + "CAS ref-log append for namespace '{}': wedged txn {} is DURABLE but this table " + "advanced under it during the resolution, so it cannot be installed without " + "discarding that advance — the lane NEEDS RECOVERY and refuses later writes until " + "replay re-derives the cache from the durable log", + ns.string(), txn); + break; + case Reason::None: + why = fmt::format("CAS ref-log append for namespace '{}': txn {} stays wedged", + ns.string(), txn); + break; + } + result.survivor_error = makeCasWriteRetryLaterExceptionPtr(why); + break; + } + case WedgeResolution::NoWedge: + break; + } + return result; +} + +void CasRefLedger::flushRefBatch(const RootNamespace & ns, const std::shared_ptr & rt, + std::vector> & owned_items) +{ + /// One flush = one carved batch through one attempted append. Contract: every ORDINARY outcome + /// (validation reject, DefiniteFailure, Unresolved/wedge, Committed) lands in the affected items so + /// waiters always wake, and this does NOT throw for any of them. Neither `commitRefChunk` nor + /// `resolveWedgeOnce` throws past the point where its object is proven durable -- both installs are + /// allocation-free by construction (spec §A1) -- and `resolveWedgeOnce` reports every ORDINARY + /// outcome, including a fence that moved under it, through its own result rather than by throwing. + /// The paths that can still throw are the wedge-resolution candidate build, which runs BEFORE the + /// bounded retry and therefore before anything is proven; a TRANSIENT failure while decoding a + /// foreign occupant (deliberately not laundered into a verdict); and an allocation failure in this + /// function's own bookkeeping (e.g. the chunk-boundary reseed). All are contained by + /// `appendRefOps`' catch, which completes every still-unfinished survivor and restores the leader + /// bookkeeping, so no caller hangs. + auto complete_error = [&](const std::vector> & items, std::exception_ptr e) + { + std::lock_guard g(ref_queue_mutex); + for (const auto & it : items) + { + it->error = e; + it->done = true; + } + rt->cv.notify_all(); + }; + auto carve_all_pending = [&]() -> std::vector> + { + std::lock_guard g(ref_queue_mutex); + std::vector> all(rt->pending.begin(), rt->pending.end()); + rt->pending.clear(); + return all; + }; + + try + { + ensureRefTableRecovered(ns, *rt); + } + catch (...) + { + complete_error(carve_all_pending(), std::current_exception()); + return; + } + + /// The local write fence ensures that a superseded or paused writer cannot race the live one. + /// Fails the WHOLE queue -- every caller would have gotten the same refusal alone. + if (!may_mutate()) + { + complete_error(carve_all_pending(), makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS mount lost / lease expired — refusing to append ref-log transactions for server_root '{}'", + config.server_root_id))); + return; + } + + /// Self-remount re-incarnation: this runtime was detached by a + /// `quiesceRefTablesForRemount` swap, so its cache is a stale (pre-remount) view. Fail the whole + /// carved batch closed -- allocating an id / applying against this orphaned runtime under the + /// re-armed fence would split-brain against the fresh runtime the next touch re-recovers. The + /// superseded flag is ordered before the fence re-arm (release/acquire through `mayMutate`), so + /// reaching this AFTER passing `mayMutate` above proves the swap happened. + if (rt->superseded_by_remount.load(std::memory_order_acquire)) + { + complete_error(carve_all_pending(), makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for server_root '{}': this cached table was superseded by a self-remount — " + "retry against the fresh mount incarnation", + config.server_root_id))); + return; + } + + /// Resolve an outstanding wedge FIRST (spec INV-1): "It does not start a later ref-log PUT for + /// that table until the earlier result is resolved." One bounded attempt per flush, and the only + /// outcome that lets this flush continue is an adoption -- everything else either leaves the lane + /// uncertain or closes it deliberately, and in both cases every queued caller is told so. + { + const WedgeResolutionResult resolution = resolveWedgeOnce(ns, rt); + switch (resolution.kind) + { + case WedgeResolution::NoWedge: + break; + case WedgeResolution::Adopted: + /// The adoption's tail bump may have crossed the snapshot-publish threshold, and this + /// flush can still return early below WITHOUT reaching the post-commit scheduler -- an + /// empty carve, or an all-no-op survivor batch, both return before it. Trigger it HERE + /// so a resolved wedge never leaves the table over-threshold until some later unrelated + /// mutation happens to arrive. Idempotent with the post-commit call below (the + /// single-in-flight gate), and off-lock as that call requires. + maybeScheduleSnapshotPublish(ns, rt); + break; + case WedgeResolution::Rejected: + case WedgeResolution::StillWedged: + case WedgeResolution::Corrupted: + complete_error(carve_all_pending(), resolution.survivor_error); + return; + } + } + + /// Test-only (see `setRefPreCarveHookForTest`): a no-op in production. + if (ref_pre_carve_hook_for_test) + ref_pre_carve_hook_for_test(); + + /// Carve a compatible batch. `lifecycle != Live` forces a solo carve: + /// `namespace_birth` must run alone, and the flush already KNOWS the table's current lifecycle + /// before carving (unlike a per-item property, which would need speculative undo). + RefTableState working; + bool table_live = false; + /// Captured in the SAME hold as `working`, for the same reason the id is derived there: a preview + /// must describe the transaction the writer will actually send, and INV-2's read side rejects one + /// that carries the wrong chain link as hard as it rejects a hole. + std::optional preview_epoch_seal; + { + std::lock_guard lock(rt->state_mutex); + working = rt->state; + table_live = rt->state.getLifecycle() == RefLifecycle::Live; + preview_epoch_seal = rt->last_epoch_seal; + } + + /// The supersession gate again, and this is NOT the same check as the one at the top of the flush. + /// A self-remount does not wait for leaders, so it can land in the window between them -- and after + /// it does, this runtime's cached view belongs to a dead incarnation while `live_epoch_fn` already + /// reports the NEW epoch. Everything derived below is then a transaction of an epoch this state has + /// no chain link for, which INV-2's read side (correctly) calls corruption. + /// + /// Checking here is what keeps that from being how an ORDINARY remount is reported. The condition is + /// a routine, retryable fact about the world -- the fresh runtime the next touch recovers has both + /// the epoch and the link -- so it must surface as the retry-safe supersession error, not as a + /// `CORRUPTED_DATA` about a stale premise this lane was never entitled to use. + if (rt->superseded_by_remount.load(std::memory_order_acquire)) + { + complete_error(carve_all_pending(), makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for server_root '{}': this cached table was superseded by a self-remount " + "before its batch was carved — retry against the fresh mount incarnation", + config.server_root_id))); + return; + } + + /// THE DEPOSED LANE, recognised BEFORE spending a request on it (spec INV-2). + /// + /// The shape: this table's next id is sequence 1 of the live epoch, and the only seal this runtime + /// holds is of THAT epoch -- a successor closed it while we still believed we were writing in it. + /// There is then no legal transaction to construct at all. Stamping the seal we hold would be a + /// self-pointer the ENCODER refuses; stamping nothing leaves an uncertified epoch crossing the + /// READER refuses. Both are dead ends, and both are provable from what this lane already knows. + /// + /// The OUTCOME here must be the same one the collision produced, not merely "an error". Before this + /// gate the lane sent its create, met the successor's seal at the key, and took the conclusive + /// rejection arm; skipping the request must not skip the CONCLUSION. So this is a permanent + /// rejection in the same class and the same words -- the operation was never acknowledged, no later + /// attempt in this epoch can ever succeed, and the lane resumes only under a later epoch, which is + /// what tells the caller (and the operator) that this mount has been deposed rather than merely + /// delayed. A retry-later class here would be the real bug: every caller would re-derive the same + /// impossible transaction forever, and the deposition would never be visible anywhere. + /// + /// Deliberately NOT a remount trigger, exactly as the collision arm is not: a successor closing our + /// epoch is a legitimate handover, and the mount lease is what resolves it. See `resolveWedgeOnce`'s + /// `SuccessorSeal` arm, whose reasoning this mirrors. + if (table_live) + { + const RefTxnId next_id = working.nextTxnId(live_epoch_fn()); + if (next_id.ref_sequence == 1 && !chainLinkFor(next_id, preview_epoch_seal)) + { + /// The two causes are reported SEPARATELY. They are different facts about the world and the + /// message must not assert the one it did not observe: holding a seal OF this epoch is + /// positive evidence that a successor closed it, while holding no seal at all says only that + /// this runtime has no chain link -- a deposition may or may not have happened. The behaviour + /// is identical either way (a `Live` table at sequence 1 with no usable link can construct + /// nothing legal), so only the diagnosis differs, and only the diagnosis is at risk of + /// being wrong. + const String cause = preview_epoch_seal + ? fmt::format("the only seal it holds, {}-{}, is of that SAME epoch — a successor already " + "closed it", preview_epoch_seal->writer_epoch, preview_epoch_seal->ref_sequence) + : String("it holds no epoch seal at all, so it has no chain link to name"); + complete_error(carve_all_pending(), std::make_exception_ptr(Exception(ErrorCodes::INVALID_STATE, + "CAS ref-log append for namespace '{}': writer epoch {} cannot be opened by this mount — its " + "next transaction would be {}-{}, sequence 1 of an epoch it holds no closing seal BELOW: {}. " + "Nothing legal can be written here and nothing was sent. This mount's append lane resumes " + "only under a later epoch", + ns.string(), next_id.writer_epoch, next_id.writer_epoch, next_id.ref_sequence, cause))); + ProfileEvents::increment(ProfileEvents::CASRefAppendSealRejected); + return; + } + } + + /// Two-phase carve (spec §2). The old carve popped from `pending` while interleaving the allocating + /// `seen_refs`/`batch` growth and only recorded the batch into `owned_items` afterwards, so any throw + /// after the first pop stranded already-popped items -- neither in `pending` nor in `owned_items` -- + /// and their waiters hung forever. Instead: + /// PLAN (may throw, mutates NOTHING): under `ref_queue_mutex`, scan `pending` WITHOUT popping and + /// build the selection count, reserving every container (`batch`, `owned_items`) that the publish + /// below grows. A throw here leaves `pending`/`owned_items` byte-for-byte unchanged, so the + /// leadership-exit guard completes only the leader's own item and the untouched followers stay + /// queued for a later leader. + /// PUBLISH (no-throw): still under the SAME continuous `ref_queue_mutex` hold (no TOCTOU by + /// construction), pop the selected front items and append them to `batch` and `owned_items` using + /// only non-throwing operations (capacity pre-reserved; `shared_ptr` copies and `deque::pop_front` + /// never throw). ProfileEvents increments are deferred past the plan so the plan is literally + /// non-mutating. + std::vector> batch; + { + std::lock_guard g(ref_queue_mutex); + const size_t cap = table_live ? kMaxRefBatch : 1; + + /// --- PLAN --- + std::set seen_refs; + size_t selected = 0; /// contiguous front items to carve + bool scope_cut = false; /// a duplicate ref name ended the selection early + for (const auto & candidate : rt->pending) + { + if (selected >= cap) + break; + if (candidate->scope.kind == MutationScope::Kind::WholeShard) + { + /// A whole-shard mutation carves solo -- it may only be the FIRST (and then only) item. + if (selected != 0) + break; + if (carve_hook_for_test) + carve_hook_for_test(CarvePhaseForTest::PlanBatchGrow); + batch.reserve(1); + ++selected; + break; + } + if (carve_hook_for_test) + carve_hook_for_test(CarvePhaseForTest::PlanSeenRefs); + if (!seen_refs.insert(candidate->scope.ref_name).second) + { + scope_cut = true; + break; + } + if (carve_hook_for_test) + carve_hook_for_test(CarvePhaseForTest::PlanBatchGrow); + batch.reserve(selected + 1); + ++selected; + } + /// Reserve the leader's responsibility set for the whole selection BEFORE any pop, so the publish + /// append below cannot throw. `owned_items` already holds the leader's own item (recorded by + /// `appendRefOps`), which the carve re-adds as it re-appears at the front of `pending` -- a + /// harmless idempotent double-listing the guard tolerates, matching the pre-fix behavior. + if (carve_hook_for_test) + carve_hook_for_test(CarvePhaseForTest::PlanReserveOwned); + owned_items.reserve(owned_items.size() + selected); + + /// --- PUBLISH (no-throw) --- + if (carve_hook_for_test) + carve_hook_for_test(CarvePhaseForTest::PublishPop); + for (size_t i = 0; i < selected; ++i) + { + batch.push_back(rt->pending.front()); /// shared_ptr copy, capacity reserved + owned_items.push_back(rt->pending.front()); /// same item into the responsibility set + rt->pending.pop_front(); + } + + /// Deferred past the plan (spec §2) so the plan phase performs no observable mutation. + if (scope_cut) + ProfileEvents::increment(ProfileEvents::CASRefBatchScopeCuts); + } + if (batch.empty()) + return; /// raced: everything was carved by a previous flush of this leader + + /// Per-item validation, in order, against `working` (per-request undo via `item_scratch`): + /// business preconditions (thrown by `build_ops` itself) and the pre-encode admission budget both + /// fail ONLY the offending item; survivors' ops accumulate into `final_ops` for ONE CHUNK. When + /// admitting the next item's ops would exceed `ref_txn_max_ops`, the accumulated chunk is committed + /// as a complete ref-log transaction and validation continues into a fresh chunk against the + /// reseeded live state (spec §3 chunked flush): one tenure may emit several transactions. + std::vector final_ops; + std::vector> survivors; + /// Every preview below stamps its throwaway transaction with `nextRefTxnId` of the state it is about + /// to be applied to, under `live_epoch_fn()` -- the SAME rule and the same epoch source + /// `allocateRefTxnId` uses for the real id, so a preview can never be rejected for an id shape the + /// persisted transaction would have been given. Deriving each preview id from ITS OWN state, rather + /// than carrying a running counter across the loop, is what makes failure isolation hold under + /// INV-1: an item that throws part-way through its per-op previews leaves `working` untouched, and + /// the next item's preview is still the successor of `working` rather than of the abandoned item's + /// last trial id. These ids are never persisted or compared outside this loop. + for (size_t item_index = 0; item_index < batch.size(); ++item_index) + { + const auto & it = batch[item_index]; + + /// Step 1: build this item's ops and apply the counts-only per-item caps. `build_ops` runs at + /// most once per item, HERE -- the overflowing item's ops are built once and reused in the fresh + /// chunk it lands in (the at-most-once contract holds across a chunk boundary). A failure here + /// (a business precondition thrown by `build_ops`, or an over-cap item/op) fails ONLY this item; + /// the chunk in progress and the remaining items are untouched. + std::vector item_ops; + bool removal_class = false; + try + { + item_ops = it->build_ops(working); + + /// Counts-only admission caps (spec §3), checked before any op is touched further so an + /// oversized item or op never reaches `working` or the state-machine preview below and + /// fails ALONE -- neighbors in this same batch are unaffected. Removal-class items are + /// exempt from both: they share the larger `ref_removal_max_bytes` byte budget instead + /// (`checkBudget`, `CasRefLogFormat.cpp`) and are already carved as singletons (`WholeShard` + /// scope forces a solo carve above). `refLogTxnIsRemovalClass` is the ONE canonical + /// discriminator (built ops contain `RemoveNamespace`) shared with the codec's own + /// `checkBudget` -- `WholeShard` scope alone is NOT a substitute (the stale-precommit + /// reclaim sweep is also `WholeShard`-scoped but is not removal-class). + removal_class = refLogTxnIsRemovalClass(item_ops); + if (!removal_class) + { + if (item_ops.size() > ref_txn_max_ops) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "ref mutation on namespace '{}' has {} operations, exceeding the normal-class " + "per-item op-count cap {} — refusing before any object is created", + ns.string(), item_ops.size(), ref_txn_max_ops); + for (const RefOp & op : item_ops) + { + const size_t op_bytes = encodedOpSize(op); + if (op_bytes > ref_op_max_bytes) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "ref mutation on namespace '{}' contains an op of encoded size {}, exceeding " + "the normal-class per-op cap {} — refusing before any object is created", + ns.string(), op_bytes, ref_op_max_bytes); + } + } + } + catch (...) + { + complete_error({it}, std::current_exception()); + continue; + } + + /// Step 2: chunk boundary (spec §3). If admitting this item's ops would push the current + /// (non-empty) chunk over `ref_txn_max_ops`, COMMIT the accumulated chunk now as a COMPLETE + /// ref-log transaction and start a fresh one. Removal-class items are always solo-carved + /// (`WholeShard` scope), so `final_ops` is empty when one is processed and this branch never + /// fires for them. + if (!removal_class && !final_ops.empty() + && final_ops.size() + item_ops.size() > ref_txn_max_ops) + { + /// Release the scratch `working` so `commitRefChunk`'s post-commit overlay fold is in place + /// (the E5 fast path), exactly as the single-chunk path does before its commit arm. + working = RefTableState{}; + const bool committed = commitRefChunk(ns, rt, final_ops, survivors); + if (!committed) + { + /// Failure isolation (spec §3): chunk N's survivors were already failed inside + /// `commitRefChunk`. Fail THIS item and the entire not-yet-attempted remainder too, so no + /// owned item is left stranded (its waiter would hang and its `build_ops` closure become + /// unsafe). Earlier chunks that already committed keep their callers' success -- an + /// unresolved wedge from `commitRefChunk` therefore contains ONLY this chunk. + std::vector> remainder(batch.begin() + item_index, batch.end()); + complete_error(remainder, makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for namespace '{}': a preceding chunk of this multi-transaction " + "flush did not commit — this item was not attempted and can be retried", ns.string()))); + return; + } + /// Reseed `working` from the now-live state: the speculative `working` with trial ids from + /// the just-committed chunk is discarded, so a later zero-op item is completed against the + /// REAL committed id, never a trial id that never persisted. The preview ids need no reseed + /// of their own -- each is derived from the state it is applied to, so re-seating `working` + /// re-seats them. A throw at the boundary -- the injected `ChunkReseed` fault, or a genuine + /// reseed allocation failure -- propagates to `appendRefOps`' tenure-containment catch, which + /// preserves the already-committed chunk's callers' success. + if (carve_hook_for_test) + carve_hook_for_test(CarvePhaseForTest::ChunkReseed); + { + std::lock_guard lock(rt->state_mutex); + working = rt->state; + preview_epoch_seal = rt->last_epoch_seal; + } + final_ops.clear(); + survivors.clear(); + } + + /// Step 3: validate this item into the current (possibly fresh) chunk and, past all throwing + /// points, publish its effects into `working`/`final_ops`/`survivors`. A failure here fails ONLY + /// this item. `item_ops` was built in step 1 against the pre-boundary state; the carve + /// deduplicates ref names within a batch, so the overflowing item operates on a ref distinct + /// from the just-committed chunk's and re-validating it against the reseeded `working` is + /// consistent. + RefTableState item_scratch = working; + try + { + /// Whole-item shape validation (prerequisite to `dropNamespace`): the + /// per-op loop below previews each op as its OWN single-op trial transaction, so a + /// whole-transaction-shape rule like "remove_namespace must be the FINAL op" trivially + /// passes on every singleton slice regardless of this item's REAL combined shape -- a + /// malformed item (e.g. remove_namespace not last) would otherwise only be caught by + /// `commitRefChunk`'s candidate apply -- which fails the whole chunk, taking every innocent + /// co-batched item with it, and (before the candidate moved ahead of the `PUT`) did so only + /// after the object was already durable. Validate the item's COMPLETE + /// ops array as ONE combined transaction, against a throwaway copy of the pre-item state, + /// before doing any other per-op work -- exactly what the real persisted transaction will + /// contain, using only the public two-phase `applyRefLogTxn` entry point (no need to reach + /// into the state machine's private per-op helpers). + if (!item_ops.empty()) + { + RefTableState shape_check = working; + const RefTxnId shape_id = shape_check.nextTxnId(live_epoch_fn()); + applyRefLogTxn(shape_check, RefLogTxn{ns.string(), shape_id, item_ops, + chainLinkFor(shape_id, preview_epoch_seal)}); + } + if (removal_class && !it->terminal_removal_authorized) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "ref mutation on namespace '{}' attempted `RemoveNamespace` through the generic append " + "surface; only exact catalog removal ownership may append the terminal transaction", + ns.string()); + + for (const RefOp & op : item_ops) + { + /// Admission budget: only STATE-GROWING ops need the check -- + /// an `owner_transition` installing a binding (add or promote) and `set_published_at`. + /// `namespace_birth` is exempt (it grows nothing, and a never-born state's preview has + /// no meaningful "current snapshot" to encode); `remove_namespace` and a pure + /// owner_transition removal shrink state and can never violate the budget. + const bool state_growing = (op.kind == RefOpKind::OwnerTransition && op.new_binding.has_value()) + || op.kind == RefOpKind::SetPublishedAt; + if (state_growing && !admits(item_scratch, op, rt->snapshot_budget, rt->removal_budget)) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "ref mutation on namespace '{}' would exceed the table's admission budget " + "(snapshot_budget={} removal_budget={}) — refusing before any object is created", + ns.string(), rt->snapshot_budget, rt->removal_budget); + /// Apply THIS op to item_scratch now (a single-op trial transaction) so a LATER op of + /// the SAME item (e.g. namespace_birth immediately followed by its first + /// owner_transition) is validated -- both here and by admits's own preview -- against + /// a state that already reflects it, exactly as the real combined transaction will. + const RefTxnId trial_id = item_scratch.nextTxnId(live_epoch_fn()); + applyRefLogTxn(item_scratch, RefLogTxn{ns.string(), trial_id, {op}, + chainLinkFor(trial_id, preview_epoch_seal)}); + } + /// Reserve the growth of BOTH accumulators BEFORE this item's effects are published. These + /// reservations are the ONLY remaining throwing steps; once they succeed the publish below is + /// no-throw -- `working`'s move-assignment is `noexcept`, and the `RefOp` moves and the + /// `shared_ptr` copy land in pre-reserved capacity. Before the fix, `working` was moved and + /// `final_ops` appended before these allocations, so a failure here left a failed item applied + /// to `working` (corrupting later items' validation) and -- when the throw fell between the + /// two accumulator writes -- its ops already in the durably-committed transaction while its + /// own caller was told the append failed. + final_ops.reserve(final_ops.size() + item_ops.size()); + survivors.reserve(survivors.size() + 1); + if (carve_hook_for_test) + carve_hook_for_test(CarvePhaseForTest::ValidateFinalOps); + working = std::move(item_scratch); + for (RefOp & op : item_ops) + final_ops.push_back(std::move(op)); + survivors.push_back(it); + } + catch (...) + { + complete_error({it}, std::current_exception()); + } + } + if (final_ops.empty()) + { + /// Either every item failed validation (already completed via complete_error above, nothing + /// left to do), or every survivor of the LAST chunk contributed ZERO ops (an idempotent no-op, + /// e.g. precommitAdd/promote re-targeting a manifest already exactly committed). Survivors of the + /// latter kind still need marking done -- with no new object created, `committed_id` is the + /// table's current high-water mark. After an earlier committed chunk, `working` was reseeded from + /// the live state, so that mark is the REAL id the earlier chunk persisted, never a discarded + /// trial id. + if (!survivors.empty()) + { + std::lock_guard g(ref_queue_mutex); + for (const auto & it : survivors) + { + it->committed_id = working.getGreatestApplied(); + it->done = true; + } + rt->cv.notify_all(); + } + return; + } + + /// Commit the FINAL chunk of this tenure (spec §3): the remaining accumulated ops form the last -- + /// possibly only -- ref-log transaction. Release the scratch `working` first so `commitRefChunk`'s + /// post-commit overlay fold is in place (the E5 fast path), then run the full committed arm. Its + /// survivors are completed (success or failure) inside it, so nothing is owed here on any outcome, + /// and it no longer throws past its durable `PUT` at all (spec §A1). + working = RefTableState{}; + commitRefChunk(ns, rt, final_ops, survivors); +} + +CasRefLedger::PreparedRefChunk CasRefLedger::prepareRefChunk( + const Layout & layout, const NamespaceLifeId & life, RefTableState state, const RefTxnId & id, + const std::optional & chain_link, std::span ops, uint64_t admitted_generation) +{ + PreparedRefChunk prepared{ + .candidate = std::move(state), + .candidate_base_id = {}, + .chunk_txn = RefLogTxn{life.ns.string(), id, std::vector(ops.begin(), ops.end()), chain_link}, + .prepared_attempt = {}, + .birth_contribution = std::nullopt, + .commit_contribution = RefCkpt{ + .life_epoch = std::nullopt, + .committed_through = id, + .checkpoint_snapshot_id = std::nullopt, + .last_epoch_seal = chain_link}, + }; + prepared.candidate_base_id = prepared.candidate.getGreatestApplied(); + + /// A throw here is a clean PRE-durability failure -- the same class as an ordinary validation + /// reject: no object exists yet, the cache is untouched, and the id is simply never used (the next + /// attempt re-derives it from this same unchanged state). + applyRefLogTxn(prepared.candidate, prepared.chunk_txn); + + /// The COMPLETE attempt (spec §A1 site 3), so the request can read its key and body straight out of + /// it and the `Unresolved` arm only has to MOVE it into the runtime. Building it here rather than + /// after the `PUT` is what keeps an allocation failure from leaving a possibly-DURABLE object with + /// NEITHER the transaction nor the attempt recorded -- strictly worse than a wedge, because the next + /// append would then mint a fresh id against a state missing a landed transaction, which is the + /// divergence the candidate exists to prevent. + /// + /// A rename, not an extra copy: the seal writes its result directly into the attempt's body and the + /// key is computed directly into the attempt's key, so nothing is copied twice on the committed path + /// either. + /// + /// FAULT CLASS, stated because the extraction NARROWED it, and this is the whole of the change. + /// `RefLogTxn`'s construction above and the key computation below used to sit outside any local + /// `try` in `commitRefChunk`, so an allocation failure in either escaped `commitRefChunk` and + /// `flushRefBatch` entirely and was caught by `appendRefOps`' tenure catch, which completes every + /// still-INCOMPLETE owned item with that exception and releases leadership, ending the tenure (an + /// item already made durable by an earlier committed chunk keeps its success -- the guard leaves it + /// `done` with no error, deliberately). Inside this function both are covered by the caller's single + /// `catch`, so the same failure now completes `chunk_survivors` with that exception and returns + /// false, and the tenure continues. That is exactly what an apply failure at this same pre-durability + /// stage already did, which is the point: an allocation failure here is no longer the one fault whose + /// blast radius differs from its immediate neighbours'. + /// + /// Nothing is stranded either way, and there is no universal remainder handler to appeal to -- the + /// two call sites dispose of their own. On a chunk boundary `flushRefBatch` fails this item plus the + /// entire not-yet-attempted remainder retry-later and returns at once. On the FINAL chunk the tail + /// call ignores the return value because no remainder is left: every batch item is by then either + /// already completed (it failed its own validation, or it belonged to an earlier chunk that + /// `commitRefChunk` completed) or is in THIS chunk's `chunk_survivors`, which this path completes. + /// `chunk_survivors` is emphatically NOT the whole batch on that path -- `survivors.clear()` at each + /// chunk boundary leaves it holding only the last chunk's items. + /// + /// One reported error class does change on the boundary path: the not-yet-attempted remainder now + /// gets a retry-later refusal instead of the raw escaping exception. Those items were provably never + /// attempted, so retry-later is the truthful class, and it is the class they already got whenever the + /// preceding chunk failed for any other reason. + prepared.prepared_attempt.txn_id = id; + prepared.prepared_attempt.key = layout.refLogKey(life, id); + prepared.prepared_attempt.admitted_fence_generation = admitted_generation; + prepared.prepared_attempt.bytes = sealObject(FormatId::RefLog, encodeRefLogTxn(prepared.chunk_txn)); + + /// INV-4's `life_epoch`, which ONLY this transaction knows: the writer epoch of the `NamespaceBirth` + /// being appended. Prepared as a VALUE; `commitRefChunk` publishes it, because for a birth chunk that + /// publish is the FIRST durable effect and preparation is everything strictly before it. + if (std::any_of(ops.begin(), ops.end(), + [](const RefOp & op) { return op.kind == RefOpKind::NamespaceBirth; })) + prepared.birth_contribution = RefCkpt{.life_epoch = std::optional{id.writer_epoch}, + .committed_through = std::nullopt, + .checkpoint_snapshot_id = std::nullopt, + .last_epoch_seal = std::nullopt}; + + if (refLogTxnIsEpochSeal(prepared.chunk_txn)) + prepared.commit_contribution.last_epoch_seal = id; + + return prepared; +} + +bool CasRefLedger::commitRefChunk(const RootNamespace & ns, const std::shared_ptr & rt, + const std::vector & chunk_ops, + const std::vector> & chunk_survivors) +{ + /// Reconstructed locally so this arm has the SAME completion + fence semantics as when it lived + /// inline in `flushRefBatch`: `complete_error` wakes a chunk's waiters under `ref_queue_mutex`, and + /// `fence_ok` folds `superseded_by_remount` into the append fence so a self-remount landing between a + /// leader's pre-allocate re-check and its `PUT` reports Unresolved rather than committing against a + /// stale cache. + auto complete_error = [&](const std::vector> & items, std::exception_ptr e) + { + std::lock_guard g(ref_queue_mutex); + for (const auto & it : items) + { + it->error = e; + it->done = true; + } + rt->cv.notify_all(); + }; + const auto fence_ok = [this, &rt] + { + return fence_ok_fn() + && !rt->catalog_life_invalidated.load(std::memory_order_acquire) + && !rt->superseded_by_remount.load(std::memory_order_acquire); + }; + + const bool positive_append = std::any_of(chunk_ops.begin(), chunk_ops.end(), [](const RefOp & op) + { + return (op.kind == RefOpKind::OwnerTransition && op.new_binding.has_value()) + || op.kind == RefOpKind::SetPublishedAt; + }); + const bool removal_append = refLogTxnIsRemovalClass(chunk_ops); + + /// The local capability prevents callers from reaching this point through the generic API; the + /// durable row is the independent final authority. Re-read it immediately before id allocation so + /// a stale exact runtime cannot append a terminal after the catalog life changed, and require the + /// positive lane to still be closed under the same queue lock that guards admission. + if (removal_append) + { + try + { + if (!std::all_of(chunk_survivors.begin(), chunk_survivors.end(), + [](const auto & item) { return item->terminal_removal_authorized; })) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS namespace removal '{}': terminal chunk lacks exact removal ownership", + ns.string()); + + { + std::lock_guard queue_lock(ref_queue_mutex); + if (!rt->removal_admission_closed) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS namespace removal '{}': terminal append reached an open positive lane", + ns.string()); + } + + const uint64_t admitted_generation = rt->admitted_fence_generation; + const CasRefCatalog::Snapshot catalog = CasRefCatalog::read(backend, layout); + check_fence_or_throw(admitted_generation); + catalog.life_index.throwIfAmbiguous("CAS terminal removal append"); + const auto entry_it = std::find_if( + catalog.catalog.entries.begin(), catalog.catalog.entries.end(), + [&](const CatalogEntry & entry) + { + return entry.ns == ns && entry.incarnation == rt->life.incarnation; + }); + if (entry_it == catalog.catalog.entries.end() || entry_it->state != NsState::Removing) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS namespace removal '{}': terminal append requires its exact catalog life to be Removing", + ns.string()); + } + catch (...) + { + complete_error(chunk_survivors, std::current_exception()); + return false; + } + } + + /// The pre-carve seam is above this call, so this is the final catalog admission observation before + /// id allocation. It closes the cached-runtime window in which another actor publishes `Removing` + /// after this writer's ordinary entry gates. A non-`Live` exact row permanently closes this local + /// positive lane; the terminal owner-removal transaction remains the one deliberate exception. + if (positive_append) + { + try + { + const uint64_t admitted_generation = rt->admitted_fence_generation; + const CasRefCatalog::Snapshot catalog = CasRefCatalog::read(backend, layout); + check_fence_or_throw(admitted_generation); + const NamespaceLifeId & life = rt->life; + const auto entry_it = std::find_if(catalog.catalog.entries.begin(), catalog.catalog.entries.end(), + [&](const CatalogEntry & entry) + { + return entry.ns == ns && entry.incarnation == life.incarnation; + }); + if (entry_it == catalog.catalog.entries.end() || entry_it->state != NsState::Live) + { + { + std::lock_guard queue_lock(ref_queue_mutex); + rt->removal_admission_closed = true; + } + throwCasWriteRetryLater(fmt::format( + "CAS ref-log append for namespace '{}': exact catalog life is no longer Live", + ns.string())); + } + } + catch (...) + { + complete_error(chunk_survivors, std::current_exception()); + return false; + } + } + + /// Self-remount re-check BEFORE allocating an id: the top-of-flush gate + /// is passed once, but a leader can stall between it and here -- in `build_ops`' caller I/O -- across + /// the whole fence-loss + remount window, then resume after `armMountFence`. Allocating {new_epoch, + /// seq} now and PUTting it (its live `fence_ok` would pass) would persist a transaction validated + /// against this orphaned runtime's STALE cache -- the C1 data-loss class. `superseded_by_remount` is + /// published before the fence re-arm, so failing closed here (no id, no PUT, no wedge, cache + /// unchanged) keeps the durable log free of any stale-view transaction. The append `fence_ok` + /// (which also checks the flag) is the airtight backstop for the narrow window past this point. + if (rt->superseded_by_remount.load(std::memory_order_acquire)) + { + complete_error(chunk_survivors, makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for server_root '{}': this cached table was superseded by a self-remount " + "before id allocation — retry against the fresh mount incarnation", + config.server_root_id))); + return false; + } + + /// `Ready` is the sole new-id admission state. The resolver either restores it or returns the whole + /// batch closed, so any other state here is an internal lifecycle violation. It can be reached only + /// by a bug or a test injection after the top-of-flush resolver gate; make that contradiction an + /// explicit `Faulted` state and route it through the same anomaly policy as foreign interference. + { + RefLaneState lane_state = RefLaneState::Faulted; + std::optional attempt_key; + { + std::lock_guard lock(rt->state_mutex); + lane_state = rt->lane_state; + if (rt->append_attempt) + attempt_key = rt->append_attempt->key; + } + if (lane_state != RefLaneState::Ready) + { + { + std::lock_guard lock(rt->state_mutex); + rt->append_attempt.reset(); + rt->lane_state = RefLaneState::Faulted; + } + on_impossible_interference(attempt_key.value_or(""), fmt::format( + "ref-log append reached new-id allocation while the lane was in state {} instead of Ready", + static_cast(lane_state)), ns.string()); + /// `Faulted` is a TERMINAL lane state (same as the two other `Faulted` arms further down this + /// function, which both report `CORRUPTED_DATA`) -- reporting it via + /// `makeCasWriteRetryLaterExceptionPtr` would tell the caller a state the lane can never leave + /// on its own is transient. The arm is self-limiting today (the next flush's + /// `resolveWedgeOnce` takes the `invalid_lane_state` `Reason` arm and re-reports it the same + /// way), but a terminal state must never be reported as retryable from ANY arm -- + /// `gtest_cas_ref_writer.cpp`'s `CasAnomalyPolicy.NonReadyAtNewIdAllocationFaultsAndFailsClosed` + /// pins this arm specifically (it already drives this exact seam; only its expected error + /// class needed to change). + complete_error(chunk_survivors, std::make_exception_ptr(Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref-log append for namespace '{}': refusing to allocate a new ref-log id while the " + "lane is not Ready (state {}, attempt key '{}'); the lane is faulted until remount", + ns.string(), static_cast(lane_state), attempt_key.value_or("")))); + return false; + } + } + + /// ONE atomic reading of everything this transaction is derived from that the RUNTIME owns: the + /// state it will be applied to, the id, the mount incarnation that admitted it, and the seal that + /// qualifies the id's sequence number. `prepareRefChunk` below is a pure function of its arguments, + /// and every argument is either one of those four readings, a value DERIVED from one of them outside + /// the lock (`chain_link`, from `id` and the seal -- see `chainLinkFor` below), or an immutable input the + /// runtime does not own at all (the layout, the namespace, this chunk's ops). + /// + /// The candidate is built from this snapshot BEFORE the PUT (spec §A1), so that the region between + /// "this chunk's object is durable" and "the runtime records it" is allocation-free and therefore + /// cannot throw. It used to be the other way round -- PUT, then `applyRefLogTxn(rt->state, ...)` -- and + /// that apply CAN throw on an allocation failure (the COW containers allocate their overlays), which + /// left the transaction durable but invisible to the writer: the apply check of the day admitted any + /// strictly greater id, so a later transaction sailed over the hole and a snapshot published + /// afterwards was labelled with THAT id -- recovery then skips the stranded transaction forever + /// while GC, which folds the ref LOGS, still applies it. That divergence is a data-loss class, not a + /// stale cache. INV-1 now refuses the same hole from the read side too, so the accident this + /// ordering prevents would also have to survive the density check to do any damage. + /// + /// A throw during PREPARATION, by contrast, is a clean PRE-durability failure -- the same class as an + /// ordinary validation reject: no object exists yet, the cache is untouched, and the id is simply + /// never used (the next attempt re-derives it from this same unchanged state). + /// + /// The snapshot copy taken here is cheap because it SHARES the live state's COW bases; the apply + /// inside preparation allocates only the overlay. It is CONSUMED by preparation -- moved, not copied + /// again -- so this remains the single copy of `rt->state` the commit path makes, exactly as it was. + /// The candidate is deliberately NOT materialized: a state that shares its base cannot fold in place, + /// so folding now would rebuild the whole base (O(table) per chunk). The install below restores unique + /// base ownership before the existing post-install fold, which therefore stays O(overlay). + /// + /// The id is derived in the SAME critical section that snapshots the state (INV-1): it is a function + /// of `greatest_applied`, so reading it at a different instant than the state the transaction is + /// applied to would be deriving this chunk's id from a different stream. Only this leader mutates + /// `rt->state`, so the two reads cannot disagree today -- taking them together is what keeps that + /// from being an invariant a future edit has to rediscover. This is also why the id, the generation + /// and the seal are INPUTS to `prepareRefChunk` rather than things it derives: it has no lock to read + /// them under. + /// + /// The fence GENERATION and `last_epoch_seal` are read in the same hold for the same reason. The + /// generation is this transaction's ADMISSION token: one atomic reading of "which mount incarnation + /// allowed this attempt", which the attempt then carries and every later retry and install presents + /// back. The seal is what qualifies the id's sequence number, so reading it at a different instant + /// than the id would be describing a different transition. + std::optional state_snapshot; + RefTxnId id; + uint64_t admitted_fence_generation = 0; + std::optional last_epoch_seal; + { + std::lock_guard lock(rt->state_mutex); + state_snapshot.emplace(rt->state); + id = allocateRefTxnId(*rt); + admitted_fence_generation = rt->admitted_fence_generation; + last_epoch_seal = rt->last_epoch_seal; + } + /// INV-2's chain link, stamped exactly where the grammar requires it: sequence 1 of an epoch above + /// this namespace's genesis names the seal that closed the previous one, and nothing else carries it. + /// `last_epoch_seal` is `nullopt` precisely at genesis (see `RefTableRuntime::last_epoch_seal`), so + /// a genesis birth at sequence 1 finds nothing to name -- which is what makes "no seal" a fact about + /// the stream rather than a defaulted field. Derived outside the `try` below because `chainLinkFor` + /// neither allocates nor throws, so nothing is gained by covering it. + const std::optional chain_link = chainLinkFor(id, last_epoch_seal); + + /// Preparation is ONE pure call, and it is everything decided before this chunk can have any durable + /// effect: the candidate, the transaction with its chain link, the complete attempt (key + sealed + /// bytes), the post-log committed-frontier contribution, and -- for a birth -- the pre-log + /// `life_epoch` contribution as VALUES. + /// + /// THE PLACEMENT IS THE CORRECTNESS ARGUMENT, and it has to hold for BOTH chunk shapes, because + /// `commitRefChunk` has two different first durable effects. An ordinary chunk's is the ref-log + /// `putIfAbsentControlled` far below; a `NamespaceBirth` chunk's is the `_ckpt` publish, which is + /// EARLIER. This call therefore sits above both, and every statement between here and the lock above + /// is in-memory only. A "pure" preparation that published the `_ckpt` itself would be a lie, and + /// moving that publish later would change fault semantics the directive says to preserve. + /// + /// Everything it can throw is a pre-durability rejection with the identical handler, which is why ONE + /// catch replaces the two that used to sit around those steps separately (see the FAULT CLASS note on + /// `prepareRefChunk` for the two statements whose catcher changed). The reachable ones are a rejected + /// apply and a failed seal; an allocation failure anywhere inside is the same class, and so is the + /// `BAD_ARGUMENTS` that `layout.refLogKey` -> `namespaceStreamPrefix` -> `checkNamespace` would raise + /// for a malformed namespace -- unreachable here, since a mounted table's namespace already passed + /// that check, but the list is not meant to read closed. + std::optional prepared; + try + { + prepared.emplace(prepareRefChunk(layout, rt->life, std::move(*state_snapshot), id, chain_link, + chunk_ops, admitted_fence_generation)); + } + catch (...) + { + complete_error(chunk_survivors, std::current_exception()); + return false; + } + const RefTxnId candidate_base_id = prepared->candidate_base_id; + /// The candidate moves back into its own optional so the post-durable install region below is + /// textually untouched by this extraction: it still swaps `*candidate` in and still `reset()`s it in + /// the same place, for the same reason (releasing bases whose `use_count()` the fold then reads). The + /// move is COW-pointer-only and happens here, while nothing is durable. + std::optional candidate{std::move(prepared->candidate)}; + + /// INV-4's FIRST `_ckpt` writer, and the ONLY writer anywhere that knows this namespace's + /// `life_epoch`: it is the writer epoch of its `namespace_birth`, which is this transaction. No + /// later writer can recover it (a table recovered from a snapshot never replays the birth), so if + /// it is not recorded here it is not recorded at all. Spec §3 orders creation `_ckpt` first, THEN + /// the namespace becoming Live, which is exactly this placement -- before the durable `PUT`, where + /// a failure is an ORDINARY pre-durability rejection: the id is not consumed, nothing landed, and + /// the next attempt re-derives the same id. A `_ckpt` for a birth that then failed is inert debris + /// (it names no checkpoint, so nothing is deletable and recovery has no base to prefer), and the + /// next attempt's merge adopts it unchanged. + /// + /// `FencedOut` is a rejection here rather than a shrug: unlike the publisher's, this contribution + /// carries the one fact nothing else can supply, so proceeding without it would put the namespace + /// Live with its genesis epoch permanently unknown. + /// + /// ORDERING, and it is the one deliberate reorder in this extraction: this publish is a birth chunk's + /// FIRST durable effect, so preparation -- INCLUDING the sealed bytes -- now completes ABOVE it, + /// where it used to run below. That matters more than "an allocation could fail earlier", because + /// sealing is not pure serialization -- it VALIDATES. `encodeRefLogTxn` runs `checkRefTxnIdNonzero`, + /// then `validateEpochSealGrammarStructural`, then `checkBudget` over the encoded text + /// (`CasRefLogFormat.cpp`), and throws `CORRUPTED_DATA` -- all three of `checkBudget`'s refusals use + /// that code too. NOT `LIMIT_EXCEEDED`: that code belongs to the carve-time admission caps in + /// `flushRefBatch`, a different check at a different stage. The candidate apply above runs, of INV-2's + /// grammar, only `validateEpochSealGrammarContextual`, which returns early off sequence 1 and owns + /// nothing but the required-iff rule. The two grammar halves are DISJOINT BY + /// DESIGN -- each function's own comment says the other owns the half it does not -- and the budget + /// check belongs to neither. So a chunk can pass the apply and still be refused by the seal, on + /// grammar, on id shape, or on size; after the reorder that refusal lands BEFORE this `_ckpt` is + /// durable instead of after it, so it leaves no inert `_ckpt` behind where it used to leave one. + /// Caller-visible outcome is identical WHEN ONE of the two steps refuses: both orders reach the same + /// already-possible rejection, complete the same survivors with the same error, return false, and + /// consume no id. A birth chunk that would fail BOTH -- an unsealable transaction AND a moved mount + /// fence -- reports a different CLASS after the reorder, because whichever step now runs first is the + /// one that speaks: the seal's `CORRUPTED_DATA` instead of the `FencedOut` retry-later below. Both are + /// truthful about a chunk that was never sent and consumed no id, and neither order can report both. + /// Strictly less durable debris, nothing new that can throw. + if (prepared->birth_contribution) + { + try + { + if (publishCkptContribution(rt->life, *prepared->birth_contribution, + admitted_fence_generation, check_fence_or_throw) + == CkptPublishOutcome::FencedOut) + { + complete_error(chunk_survivors, makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for namespace '{}': the mount fence moved while creating the " + "namespace's _ckpt, so the birth transaction was not sent", ns.string()))); + return false; + } + } + catch (...) + { + complete_error(chunk_survivors, std::current_exception()); + return false; + } + } + + /// [CKPT-FAILED-BIRTH-DEBRIS] REMOVED (increment review Critical B, BACKLOG `{#ckpt-failed-birth-debris}` + /// reopened, `{#ckpt-neverborn-gc-backstop}` filed): the best-effort cleanup that used to live here + /// deleted `_ckpt` by a FRESH `head()` read at cleanup time, never a token captured from this + /// attempt's own publish -- so it deleted WHATEVER was at the key when it ran, not proven still to + /// be this attempt's bytes. All three of its call sites sit inside the `CORRUPTED_DATA` path, below + /// `getCurrentExceptionCode() != ErrorCodes::CORRUPTED_DATA` -- i.e. every branch that called it had + /// just PROVEN a different object occupies the derived key, directly contradicting the safety + /// argument's premise ("reachable only while the namespace's ref-log has never durably held + /// anything"). A namespace's ref-log with a live occupant at the derived key is not empty by the + /// very fact that triggered the call. Concretely: a successor that legitimately owns the same live + /// incarnation (a remount, INV-2's ordinary epoch-seal handoff) may have already read `_ckpt` for + /// its own recovery before this cleanup ran, and the delete could destroy a genesis record + /// (`life_epoch`, recorded nowhere else, no repair path) that successor's own future recovery still + /// needs -- with no way to tell that case apart from ordinary debris at cleanup time. A captured + /// token does not close this either: a successor that only READ `_ckpt` (never re-wrote it) leaves + /// the token matching, so a token-gated delete would still succeed and destroy what that successor + /// leaned on. Removed rather than patched, per the project's own fail-close principle: never take a + /// destructive action on a fallback path when the safe alternative is to skip it and surface the + /// gap. The trade, named rather than left implicit: debris now SURVIVES (a drained server root + /// carrying it will refuse decommission, `claimOwnerOrThrow` -> `CORRUPTED_DATA`, until a backstop + /// exists) instead of risking an unrecoverable delete of a live successor's genesis record. The + /// backstop -- independently re-verifying emptiness with a real LIST, never inferring it from one + /// attempt's own conflict -- is `{#ckpt-neverborn-gc-backstop}`, not built here. + + /// Install the exact attempt before the first possible send. This is NOT preparation -- it mutates + /// `RefTableRuntime` and takes `state_mutex` -- so it stays here, between preparation and the first + /// send. From this point every exit must make one explicit lane transition; there is no independent + /// marker to update or reconstruct later. + bool attempt_armed = false; + { + std::lock_guard lock(rt->state_mutex); + const bool same_base = rt->state.getGreatestApplied() == candidate_base_id; + if (rt->lane_state == RefLaneState::Ready && !rt->append_attempt && same_base) + { + static_assert(std::is_nothrow_move_constructible_v); + rt->append_attempt = std::move(prepared->prepared_attempt); + rt->lane_state = RefLaneState::Writing; + attempt_armed = true; + } + } + if (!attempt_armed) + { + /// NO cleanup call here (review C1, and the whole class removed by increment review Critical B): + /// this arm makes NO lane transition -- it is reached whenever the lane is not what THIS attempt + /// expected, including `getGreatestApplied() != candidate_base_id`, which means some OTHER + /// append for this table already advanced applied state. That other append can be the birth + /// itself, whose `_ckpt` a cleanup call here would then delete out from under it -- the same harm + /// class the ambiguous `Writing -> Wedged` branch is deliberately excluded to avoid, against an + /// object with no repair path (BACKLOG `{#ckpt-damage-no-repair-path}`). "`putIfAbsentControlled` + /// was never reached" is true of THIS attempt; it says nothing about whether a DIFFERENT attempt + /// for this same namespace already made the birth durable. Now that Critical B removed the other + /// call sites too, `_ckpt` debris from a never-born namespace is reclaimed only by the future + /// GC-level backstop (`{#ckpt-neverborn-gc-backstop}`), never here. + complete_error(chunk_survivors, std::make_exception_ptr(Exception( + ErrorCodes::LOGICAL_ERROR, + "CAS ref-log append for namespace '{}': lane changed before attempt {}-{} could be armed", + ns.string(), id.writer_epoch, id.ref_sequence))); + return false; + } + const RefAppendAttempt & active_attempt = *rt->append_attempt; + + CasWriteOutcome outcome{}; + /// WHY an Unresolved came back. Two jobs (finding #37 defect 3): the wedge message stops claiming an + /// exhausted retry budget when in fact no request was ever sent, and -- see the `Unresolved` arm -- + /// the one reason that PROVES nothing was sent decides whether the lane wedges at all. + CasUnresolvedReason unresolved_reason = CasUnresolvedReason::NotUnresolved; + try + { + outcome = ref_request_controller->putIfAbsentControlled( + active_attempt.key, active_attempt.bytes, fence_ok, /*out_token=*/nullptr, &unresolved_reason); + } + catch (...) + { + const std::exception_ptr write_error = std::current_exception(); + /// `putIfAbsentControlled` throws CORRUPTED_DATA when resolve-before-reissue observes a DIFFERENT + /// object already at this txn's key -- a proven different-object conflict, not an unresolved PUT. + /// Any other exception after the send boundary is ambiguous and therefore transfers ownership + /// from `Writing` to `Wedged`; the exact attempt remains installed. + if (getCurrentExceptionCode() != ErrorCodes::CORRUPTED_DATA) + { + { + std::lock_guard lock(rt->state_mutex); + if (rt->lane_state == RefLaneState::Writing && rt->append_attempt + && rt->append_attempt->txn_id == id) + rt->lane_state = RefLaneState::Wedged; + } + complete_error(chunk_survivors, write_error); + return false; + } + /// THREE-WAY ADJUDICATION, the same one the wedge resolution owes [review HIGH-2]. "A different + /// object at our derived key" is not one situation but two, and they call for opposite + /// reactions. One of them is EXPECTED: a successor that sealed our epoch put its epoch-closing + /// record at exactly the id we keep re-deriving, and INV-2 says we must keep re-deriving it + /// ("a dying lane that observes the seal retries T+1, never mints T+2"). Treating that as + /// foreign interference would fence the mount and raise an anomaly alarm on the designed path. + /// The other is a genuine breach of write-exclusivity and must be exactly as loud as before. + /// + /// The occupant is read once, by exact key, here -- `putIfAbsentControlled` proved the mismatch + /// but does not hand back what it saw. One extra request on a path that is already exceptional + /// and already fatal to this attempt. + Occupant occupant = Occupant::Foreign; + bool classified = false; + try + { + if (const auto got = backend.get(active_attempt.key)) + { + occupant = classifyRefLogOccupant(ns, id, got->bytes, active_attempt.bytes); + classified = true; + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// Left unclassified deliberately -- see below. The original conflict is what the survivors + /// are told about; this read's own failure is not their business. + } + if (!classified) + { + /// We could not learn WHICH of the two this is, so we decide NEITHER. Reporting foreign + /// interference would fence the mount on a guess, and reporting a conclusive rejection would + /// acknowledge a deposition we did not observe. The id is not consumed and nothing is + /// recorded, so the next append re-derives the same id, meets the same conflict, and + /// classifies again -- deferring costs one round trip and decides nothing wrongly. + /// + /// It must be COUNTED, because deferring is the one arm here that is quiet by construction: + /// the loud interference report is only reached once the occupant can be read, so a real + /// breach whose occupant keeps failing to read would otherwise show up as nothing but a + /// throttled log line under load. Sustained growth on this counter is the signal that the + /// loud path is being starved. + ProfileEvents::increment(ProfileEvents::CASRefAppendOccupantUnreadable); + { + std::lock_guard lock(rt->state_mutex); + rt->append_attempt.reset(); + rt->lane_state = RefLaneState::Faulted; + } + complete_error(chunk_survivors, std::make_exception_ptr(Exception( + ErrorCodes::CORRUPTED_DATA, + "CAS ref-log append for namespace '{}': a DIFFERENT object occupies the id {}-{} this table " + "derived, and reading it to tell a successor's epoch seal from foreign interference did not " + "succeed — the lane is faulted until remount recovery adjudicates durable state", + ns.string(), id.writer_epoch, id.ref_sequence))); + return false; + } + if (occupant == Occupant::SuccessorSeal) + { + /// Conclusive rejection, identical in meaning to the wedge site's: our bytes provably never + /// landed and never can, the operation was never acknowledged, and the seal IS this + /// namespace's epoch-closing record. No anomaly, no fence -- this is the protocol working. + /// Counted anyway: "the protocol working" here means THIS writer was deposed, and a lane that + /// keeps landing on this arm is a mount that has lost its lease and does not know it yet. + ProfileEvents::increment(ProfileEvents::CASRefAppendSealRejected); + { + std::lock_guard lock(rt->state_mutex); + rt->last_epoch_seal = id; + rt->append_attempt.reset(); + rt->lane_state = RefLaneState::Closed; + } + complete_error(chunk_survivors, std::make_exception_ptr(Exception(ErrorCodes::INVALID_STATE, + "CAS ref-log append for namespace '{}': writer epoch {} was CLOSED by a successor's epoch " + "seal at {}-{}, which conclusively rejects this transaction (it was never acknowledged). " + "This mount's append lane resumes only under a later epoch", + ns.string(), id.writer_epoch, id.writer_epoch, id.ref_sequence))); + return false; + } + /// A genuine breach. This table's appends are now BLOCKED, and that is the intended contract: + /// under mount-lease exclusivity this key is exclusively ours, so a foreign object at it is + /// corruption or a protocol breach, not a race. The id is not consumed, so the next attempt + /// derives the SAME id and hits the SAME conflict, loudly, until a remount-level recovery (a + /// fresh writer epoch is a fresh key namespace) clears it. Advancing past the occupant, which is + /// what the pool-wide allocator did, would have written this table's stream around a foreign + /// object and hidden the violation -- and produced the hole INV-1 exists to forbid. + /// + /// Route it through the anomaly policy, exactly as the wedge-resolution site does for the + /// identical observation [review I5]. Failing closed is right, but failing closed FOREVER is + /// not: without this the mount stays blocked on this table until somebody notices and remounts + /// by hand. One impossibility, one reaction. The report is deliberately BEFORE the survivors are + /// completed, so the fence is closed by the time any caller wakes and can retry. + const String attempt_key = active_attempt.key; + { + std::lock_guard lock(rt->state_mutex); + rt->append_attempt.reset(); + rt->lane_state = RefLaneState::Faulted; + } + on_impossible_interference(attempt_key, + fmt::format("ref-log append for namespace '{}' txn {}-{} observed a DIFFERENT object already at " + "the id it derived, and it is not an epoch seal of this namespace ({})", + ns.string(), id.writer_epoch, id.ref_sequence, + getCurrentExceptionMessage(/*with_stacktrace*/ false)), + ns.string()); + complete_error(chunk_survivors, write_error); + return false; + } + switch (outcome) + { + case CasWriteOutcome::Committed: + { + /// A durable log object is not yet admitted to logical history. Publish its exact frontier + /// under the SAME admission generation before any local consequence can make a later id + /// observable or wake a waiter. `Published` and `IdenticalSkip` both prove the contribution + /// durable. `FencedOut`, contention exhaustion, decode failure, or any other unresolved + /// publication leaves the log known durable but uninstalled, which is exactly + /// `NeedsRecovery`; recovery owns resolution of that window. + const auto check_commit_admitted = [this, &rt](uint64_t expected_generation) + { + check_fence_or_throw(expected_generation); + if (rt->catalog_life_invalidated.load(std::memory_order_acquire) + || rt->superseded_by_remount.load(std::memory_order_acquire)) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': its captured runtime was retired before committed-frontier publication", + rt->life.ns.string())); + }; + CkptPublishOutcome frontier_outcome = CkptPublishOutcome::FencedOut; + try + { + frontier_outcome = publishCkptContribution( + rt->life, prepared->commit_contribution, admitted_fence_generation, check_commit_admitted); + } + catch (...) + { + const std::exception_ptr frontier_error = std::current_exception(); + { + std::lock_guard lock(rt->state_mutex); + requireRecovery(*rt, ns, "committed-frontier publication"); + } + complete_error(chunk_survivors, frontier_error); + return false; + } + if (frontier_outcome == CkptPublishOutcome::FencedOut) + { + { + std::lock_guard lock(rt->state_mutex); + requireRecovery(*rt, ns, "committed-frontier publication fence"); + } + complete_error(chunk_survivors, makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for namespace '{}': txn {}-{} is durable, but the mount fence " + "moved before its checkpoint frontier was published; the lane needs recovery", + ns.string(), id.writer_epoch, id.ref_sequence))); + return false; + } + + if (carve_hook_for_test) + carve_hook_for_test(CarvePhaseForTest::PostDurableInstall); + bool install_refused = false; + std::exception_ptr install_admission_error; + { + std::lock_guard lock(rt->state_mutex); + /// The checkpoint CAS can succeed and the fence can move before the state lock is + /// reached. Re-present the same admission INSIDE the install hold, immediately before + /// inspecting and swapping the candidate. A stale runtime may leave both log and + /// frontier durable, but it must neither install nor acknowledge them. + try + { + check_commit_admitted(admitted_fence_generation); + } + catch (...) + { + install_admission_error = std::current_exception(); + requireRecovery(*rt, ns, "post-frontier install admission"); + } + /// Only this leader mutates `rt->state`, so the candidate's base snapshot is still the + /// current one: there is one append-lane leader per table at a time (the `leader_active` + /// baton), the wedge-resolution apply ran earlier in this same flush on this same thread, + /// recovery installs a state exactly once per runtime and has already completed for this + /// table, and every other consumer (readers, the snapshot publisher) only COPIES the + /// state under this mutex. Evaluated here, one statement before the install, and + /// asserted inside it: the comparison allocates nothing, and the identifier is short + /// enough that even the failure path's message is inline-buffered rather than heap + /// allocated, so no build can turn the assert itself into an allocation in the region. + const bool state_unchanged + = !install_admission_error + && rt->lane_state == RefLaneState::Writing + && rt->append_attempt + && rt->append_attempt->txn_id == id + && rt->append_attempt->bytes == active_attempt.bytes + && rt->state.getGreatestApplied() == candidate_base_id; + if (!install_admission_error && !state_unchanged) + { + /// RELEASE-mode counterpart of the `chassert` inside the region below, which is a + /// no-op in a release build and therefore no guard at all for a window that spans a + /// full network round trip. Swapping the candidate in anyway would DISCARD whatever + /// advanced the table. The object is durable and this runtime cannot record it, + /// `LOGICAL_ERROR` here, where the wedge site's identical refusal reports the + /// retry-later class, and the asymmetry is deliberate: THIS one is reachable only by + /// a second writer inside one process -- a bug in this build, which a debug build + /// should abort on and shout about. The wedge site's is reachable by an ordinary + /// remount racing a slow resolution, which is a retryable fact about the world, not a + /// bug. Same refusal, different provenance, so different loudness. + requireRecovery(*rt, ns, "commitRefChunk install"); + install_refused = true; + } + else if (!install_admission_error) + { + std::optional completed_attempt; + try + { + DENY_ALLOCATIONS_IN_SCOPE; + if (install_region_probe_for_test) + install_region_probe_for_test(); + chassert(state_unchanged); + rt->state.swap(*candidate); + rt->tail_count_since_snapshot.fetch_add(1, std::memory_order_relaxed); + rt->tail_bytes_since_snapshot.fetch_add(active_attempt.bytes.size(), std::memory_order_relaxed); + rt->append_attempt.swap(completed_attempt); + rt->lane_state = RefLaneState::Ready; + } + catch (...) + { + requireRecovery(*rt, ns, "commitRefChunk install"); + throw; + } + candidate.reset(); + completed_attempt.reset(); + try + { + rt->state.materializeCommitted(); + } + catch (...) + { + tryLogCurrentException(getLogger("CasPool"), fmt::format( + "CAS ref-log append for namespace '{}': committed txn {}-{} was applied durably, but " + "the post-commit overlay fold failed and was retained coherently for the next flush", + ns.string(), id.writer_epoch, id.ref_sequence)); + } + } + } + if (install_admission_error) + { + complete_error(chunk_survivors, install_admission_error); + return false; + } + if (install_refused) + { + complete_error(chunk_survivors, std::make_exception_ptr(Exception( + ErrorCodes::LOGICAL_ERROR, + "CAS ref-log append for namespace '{}': txn {}-{} is durable but this table changed " + "before installation; the lane needs recovery and refuses later writes until replay", + ns.string(), id.writer_epoch, id.ref_sequence))); + return false; + } + if (carve_hook_for_test) + carve_hook_for_test(CarvePhaseForTest::PostInstallPreAck); + ProfileEvents::increment(ProfileEvents::CASRefBatchFlushes); + ProfileEvents::increment(ProfileEvents::CASRefBatchedMutations, chunk_survivors.size()); + { + std::lock_guard g(ref_queue_mutex); + for (const auto & it : chunk_survivors) + { + it->committed_id = id; + it->done = true; + } + rt->cv.notify_all(); + } + /// The threshold trigger -- off the lane, + /// dispatched AFTER waking every waiter above so this commit's own callers are never + /// delayed by it. Per chunk (spec §3): each committed chunk schedules its own publication, + /// and settlement coalesces the triggers so a mid-tenure publisher never suppresses a later + /// chunk (`settleSnapshotPublish`). + maybeScheduleSnapshotPublish(ns, rt); + return true; + } + case CasWriteOutcome::DefiniteFailure: + { + /// Proof that nothing became durable returns the exact attempt to `Ready`. + { + std::lock_guard lock(rt->state_mutex); + if (rt->lane_state == RefLaneState::Writing && rt->append_attempt + && rt->append_attempt->txn_id == id) + { + rt->append_attempt.reset(); + rt->lane_state = RefLaneState::Ready; + } + } + ProfileEvents::increment(ProfileEvents::CASRefAppendDefiniteFailure); + complete_error(chunk_survivors, makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for namespace '{}' definitively failed (non-retryable rejection); " + "cached state is unchanged and txn id {}-{} was never used (a retry re-derives it)", + ns.string(), id.writer_epoch, id.ref_sequence))); + return false; + } + case CasWriteOutcome::Unresolved: + { + /// The ONE `Unresolved` shape that must NOT wedge (finding #37 defect 3). The wedge exists + /// because an `Unresolved` PUT MAY HAVE LANDED: the durable log may or may not contain this + /// transaction, only `resolveByExactGet` on that exact key can settle it, and until it does, + /// minting a later id would build on a state that may be missing a landed transaction. All of + /// that presupposes an attempt was SENT. + /// + /// `unresolvedProvesNothingWasSent` is true only for `NoAttemptSent`, which + /// `putIfAbsentControlled` reports only when a pre-attempt gate -- the mount fence or the + /// operation deadline -- rejected while `attempts_sent == 0`, i.e. strictly before the first + /// `backend->putIfAbsent`. Nothing reached the network, so the key is provably unwritten: + /// there is nothing for a wedge to resolve, and wedging is pointless. + /// + /// It is no longer HARMFUL, and the difference is worth stating because the old comment here + /// rested on it: a wedge over a never-written key used to be unclearable, because resolution + /// was a bare read and a read can only ever report absent. The every-attempt rule replaced + /// that with a conditional CREATE, so such a wedge now clears on the next caller's flush by + /// landing the transaction. What remains is that this lane would be blocked until then for no + /// reason at all -- a transient fence blip in the pre-attempt gate would cost the table its + /// write availability, and buy nothing, since there is provably nothing to resolve. + /// + /// The counterexample this argument deliberately excludes: a fence lost or a deadline reached + /// AFTER at least one attempt is `FenceLostMidWay`/`DeadlineMidWay`, and an attempt that + /// COMMITTED but returned under a dropped fence is `FenceLostPostWrite`. Each of those may + /// have left a durable object, so each keeps wedging -- as does anything a future contributor + /// adds to the enum without classifying it (see the predicate's allow-list construction). + if (unresolvedProvesNothingWasSent(unresolved_reason)) + { + { + std::lock_guard lock(rt->state_mutex); + if (rt->lane_state == RefLaneState::Writing && rt->append_attempt + && rt->append_attempt->txn_id == id) + { + rt->append_attempt.reset(); + rt->lane_state = RefLaneState::Ready; + } + } + /// Count it. Before this arm existed these refusals bumped `CASRefAppendWedged`, so + /// removing the wedge also removed the only signal they were happening at all -- and a + /// soak oracle watching that counter fall could not tell "the fix works" from "nothing + /// happened". A separate event keeps both readings available: the wedge counter now means + /// only genuinely ambiguous appends, and this one means availability preserved. + ProfileEvents::increment(ProfileEvents::CASRefAppendPreAttemptRefused); + /// The id is not consumed (INV-1): it was derived from `greatest_applied`, which this + /// refusal leaves exactly as it was, so the next caller on this table derives the SAME id + /// and the durable stream keeps no trace of the refusal. That is the free half of the + /// every-attempt rule -- an attempt that provably sent nothing owes nothing. + /// The installed attempt is retired below; no request was sent. + /// and is what makes the genuinely ambiguous path below allocation-free. + complete_error(chunk_survivors, makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for namespace '{}' txn {}-{} was refused BEFORE any request was " + "sent ({}) — the append lane is NOT wedged (nothing can be durable, so there is " + "nothing to resolve) and the txn id is not consumed (a retry re-derives it)", + ns.string(), id.writer_epoch, id.ref_sequence, + describeUnresolvedReason(unresolved_reason)))); + return false; + } + { + std::lock_guard lock(rt->state_mutex); + if (rt->lane_state == RefLaneState::Writing && rt->append_attempt + && rt->append_attempt->txn_id == id) + rt->lane_state = RefLaneState::Wedged; + } + ProfileEvents::increment(ProfileEvents::CASRefAppendWedged); + complete_error(chunk_survivors, makeCasWriteRetryLaterExceptionPtr(fmt::format( + "CAS ref-log append for namespace '{}' txn {}-{} is UNCERTAIN ({}) — " + "the append lane is wedged until the SAME key resolves durable or a conclusive rejection " + "is observed; this outcome is unproven, not failure", + ns.string(), id.writer_epoch, id.ref_sequence, + describeUnresolvedReason(unresolved_reason)))); + return false; + } + } + /// Unreachable: the switch above covers every `CasWriteOutcome`. Kept explicit so the function has a + /// defined return on all control-flow paths. + return false; +} + +bool CasRefLedger::hasStateBearingSnapshotCandidateUnderStateLock(const RefTableRuntime & rt) const +{ + /// The newest snapshot must be strictly older than the candidate. A seal advances epoch geometry + /// but carries no table state, so it is never a snapshot candidate. + return rt.state.getLifecycle() == RefLifecycle::Live + && (!rt.newest_snapshot_id || *rt.newest_snapshot_id < rt.state.getGreatestApplied()) + && (!rt.last_epoch_seal || *rt.last_epoch_seal != rt.state.getGreatestApplied()); +} + +bool CasRefLedger::admitSnapshotPublishUnderStateLock(RefTableRuntime & rt) +{ + /// Caller holds `rt.state_mutex` (the `may_mutate` fence check is the caller's responsibility, since + /// it is not held under `state_mutex`). The whole decision -- the threshold trigger, the + /// single-in-flight gate, the backoff deadline -- and the `pending_snapshot_publishes` increment all + /// happen under that ONE hold, so two racing dispatchers can never both admit a publish for this + /// table, and the settlement re-evaluation can decrement-and-re-admit without the count transiently + /// reaching zero. + const uint64_t now = boot_ms_now_fn(); + if (!rt.catalog_life_invalidated.load(std::memory_order_acquire) + && !rt.superseded_by_remount.load(std::memory_order_acquire) + /// Use the execution predicate at admission too, so settlement cannot redispatch a recovered seal. + && hasStateBearingSnapshotCandidateUnderStateLock(rt) + /// Single-in-flight gate: at most one background publish per table. + && rt.pending_snapshot_publishes.load(std::memory_order_relaxed) == 0 + /// Backoff deadline: after a non-Committed publish, a saturated backend is not re-dispatched + /// until the bounded backoff elapses (the read-triggered PUT-storm latch). + && now >= rt.publish_backoff_until_ms) + { + /// The threshold trigger reads the tail counters directly -- no walk, no age filter. + /// `tail_count_since_snapshot`/`tail_bytes_since_snapshot` count ONLY applied txns strictly above + /// `newest_snapshot_id` (maintained incrementally by every commit in `commitRefChunk` and by the + /// wedge-resolution apply in `flushRefBatch`), so `over_threshold` here is never true without a + /// real, immediately-coverable candidate. + const uint64_t publishable_count = rt.tail_count_since_snapshot.load(std::memory_order_relaxed); + const uint64_t publishable_bytes = rt.tail_bytes_since_snapshot.load(std::memory_order_relaxed); + const bool over_threshold = publishable_count > config.snapshot_log_count_threshold + || publishable_bytes > config.snapshot_log_bytes_threshold; + if (over_threshold) + { + rt.pending_snapshot_publishes.fetch_add(1, std::memory_order_relaxed); + return true; + } + } + return false; +} + +void CasRefLedger::dispatchSnapshotPublisher(const RootNamespace & ns, const std::shared_ptr & rt) +{ + /// `admitSnapshotPublishUnderStateLock` already incremented `pending_snapshot_publishes` for THIS + /// dispatch. Off the mutation hot path: `tryPublishSnapshotAndAdvanceCheckpointOnce` never touches + /// the append queue, so dispatching it onto an unrelated global-pool thread can never deadlock a flush leader. + /// `pin_owner()` (the Pool's `shared_from_this`) keeps the Pool -- and hence this ledger member -- + /// alive for the thread's lifetime. + ProfileEvents::increment(ProfileEvents::CASRefSnapshotPublishDispatched); + auto owner = pin_owner(); + try + { + ThreadFromGlobalPool([owner, this, ns, rt] + { + setThreadName(ThreadName::CAS_REF_SNAPSHOT_PUBLISH); + try + { + tryPublishSnapshotAndAdvanceCheckpointOnceOnRuntime(ns, rt); + } + catch (...) + { + tryLogCurrentException(getLogger("CasPool"), "CAS background snapshot publish attempt failed"); + } + settleSnapshotPublish(ns, rt); + }).detach(); + } + catch (...) + { + /// The `ThreadFromGlobalPool` ctor can throw (pool exhaustion) AFTER the count was incremented. + /// Undo the count WITHOUT the settlement re-evaluation (else a persistently-failing dispatch could + /// re-fire itself in a loop) and SWALLOW the failure: dispatching a background publish is a + /// best-effort maintenance trigger and must never fail an otherwise-successful read or mutation. + /// The next trigger reschedules. + { + std::lock_guard lock(rt->state_mutex); + rt->pending_snapshot_publishes.fetch_sub(1, std::memory_order_relaxed); + } + rt->publish_settle_cv.notify_all(); + tryLogCurrentException(getLogger("CasPool"), "CAS background snapshot-publish dispatch failed to launch"); + } +} + +void CasRefLedger::settleSnapshotPublish(const RootNamespace & ns, const std::shared_ptr & rt) +{ + /// Fence re-checked outside `state_mutex` (as in `maybeScheduleSnapshotPublish`): a fence lost + /// between this publish's dispatch and its settlement must suppress a follow-up. + const bool live_mount = may_mutate(); + bool redispatch = false; + { + std::lock_guard lock(rt->state_mutex); + /// Drop THIS publish's in-flight count and, under the SAME hold, re-evaluate the accumulated + /// tail. A chunked tenure (or any concurrent mutation) that raised more log above the newest + /// snapshot while this publish was capturing an earlier prefix had its trigger discarded by the + /// single-flight gate; settlement re-fires it here so chunks 2..N are not suppressed until an + /// unrelated later trigger (spec §3 snapshot coalescing). Re-admitting under the SAME lock as the + /// decrement means `pending_snapshot_publishes` never transiently reaches 0 across the handoff, so + /// `waitForSnapshotPublishSettleForTest` never observes a false "settled". A durable publish + /// already subtracted its captured tail, so this self-terminates once the tail is back at/under + /// threshold; a non-durable one armed the backoff, which `admit...` respects -- no PUT storm. + rt->pending_snapshot_publishes.fetch_sub(1, std::memory_order_relaxed); + if (live_mount) + redispatch = admitSnapshotPublishUnderStateLock(*rt); + } + if (redispatch) + dispatchSnapshotPublisher(ns, rt); + else + rt->publish_settle_cv.notify_all(); +} + +void CasRefLedger::maybeScheduleSnapshotPublish(const RootNamespace & ns, const std::shared_ptr & rt) +{ + /// Never dispatch a publisher while the fence is lost: a publish is a + /// conditional PUT that would fail `fence_ok` and return non-Committed anyway, and dispatching one + /// during the self-remount window is exactly the stale-cache-publish race the remount quiesce closes + /// -- with no dispatch here, `quiesceRefTablesForRemount` only has to drain publishers already in + /// flight before the fence dropped, never a moving target. + if (!may_mutate()) + return; + + bool dispatch = false; + { + std::lock_guard lock(rt->state_mutex); + dispatch = admitSnapshotPublishUnderStateLock(*rt); + } + if (dispatch) + dispatchSnapshotPublisher(ns, rt); +} + + +void CasRefLedger::advancePublishBackoff(RefTableRuntime & rt) +{ + /// Caller holds `rt.state_mutex`. Double the interval from `initial` up to `max` per consecutive + /// non-Committed publish outcome; arm the deadline off the boottime clock (`bootMsNow`), so an + /// injected test clock drives it deterministically and a VM-suspend cannot shorten it. + rt.publish_backoff_ms = rt.publish_backoff_ms == 0 + ? config.snapshot_publish_backoff_initial_ms + : std::min(rt.publish_backoff_ms * 2, config.snapshot_publish_backoff_max_ms); + rt.publish_backoff_until_ms = boot_ms_now_fn() + rt.publish_backoff_ms; + ProfileEvents::increment(ProfileEvents::CASRefSnapshotPublishBackoff); +} + +void CasRefLedger::resetPublishBackoff(RefTableRuntime & rt) +{ + /// Caller holds `rt.state_mutex`. A durable publish clears the cooldown. + rt.publish_backoff_ms = 0; + rt.publish_backoff_until_ms = 0; +} + +void CasRefLedger::waitForSnapshotPublishSettleForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return; + std::unique_lock lock(rt->state_mutex); + rt->publish_settle_cv.wait(lock, [&] { return rt->pending_snapshot_publishes.load(std::memory_order_relaxed) == 0; }); +} + +int CasRefLedger::pendingSnapshotPublishesForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return 0; + std::lock_guard lock(rt->state_mutex); + return rt->pending_snapshot_publishes.load(std::memory_order_relaxed); +} + +std::optional CasRefLedger::newestPublishedSnapshotIdForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return std::nullopt; + std::lock_guard lock(rt->state_mutex); + return rt->newest_snapshot_id; +} + +bool CasRefLedger::refRecoveryCancelRequestedForTest(const RootNamespace & ns) +{ + std::lock_guard qlock(ref_queue_mutex); + const auto it = ref_name_slots.find(ns.string()); + return it != ref_name_slots.end() && it->second.current + && it->second.current->recovery_cancel_requested.load(std::memory_order_acquire); +} + +bool CasRefLedger::refTableRecoveredForTest(const RootNamespace & ns) +{ + /// Like every observational seam, deliberately does not recover: the fail-closed tests ask "did that + /// refused recovery install anything", and an observer that recovered on demand would answer its own + /// question with a yes. + std::lock_guard qlock(ref_queue_mutex); + const auto it = ref_name_slots.find(ns.string()); + if (it == ref_name_slots.end() || !it->second.current) + return false; + std::lock_guard lock(it->second.current->state_mutex); + return it->second.current->recovered; +} + +size_t CasRefLedger::tailSinceSnapshotCountForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return 0; + std::lock_guard lock(rt->state_mutex); + return rt->tail_count_since_snapshot.load(std::memory_order_relaxed); +} + +size_t CasRefLedger::committedOverlayEntriesForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return 0; + std::lock_guard lock(rt->state_mutex); + return rt->state.getCommitted().overlayEntriesForTest(); +} + +std::set> CasRefLedger::livePrecommitsForTest(const RootNamespace & ns) +{ + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return {}; + std::lock_guard lock(rt->state_mutex); + return rt->state.getPrecommits(); +} + +namespace +{ +/// A clamped-to-zero fetch-subtract for the tail counters. +/// `tryPublishSnapshotAndAdvanceCheckpointOnce` is public and NOT serialized against itself (two +/// overlapping attempts can finish out of order), so the monotonic guard +/// below +/// skips a stale (superseded) adoption's subtraction outright, but it cannot see a SMALLER-candidate +/// attempt that lands its adoption BEFORE a larger-candidate one already in flight: that ordering would +/// have the larger attempt's `captured_count`/`captured_bytes` double-count the smaller one's +/// already-subtracted region. A plain `fetch_sub` would then underflow the unsigned counter, wrapping it +/// to near `UINT64_MAX` and permanently re-latching the read-triggered PUT-storm trigger on every +/// subsequent read -- a release-build regression of the exact bug this guard prevents. Clamping to +/// zero instead settles for a +/// benign, self-healing under-count (a delayed next dispatch; the NEXT publish always captures the true +/// live state fresh, so snapshot CONTENT is never affected) over an unsafe wraparound. +void clampedCounterSub(std::atomic & counter, uint64_t amount) +{ + uint64_t old_value = counter.load(std::memory_order_relaxed); + while (!counter.compare_exchange_weak(old_value, old_value > amount ? old_value - amount : 0, + std::memory_order_relaxed)) + { + } +} +} + + +CkptPublishOutcome CasRefLedger::publishCkptContribution(const NamespaceLifeId & life, const RefCkpt & contribution, + uint64_t admitted_generation, + const std::function & check_admission) +{ + /// The retry window is the SAME budget every other CAS operation of this ledger rides, measured on + /// the ledger's own injectable boot clock -- so a test drives the exhaustion arm without sleeping, + /// and a VM suspend cannot shorten it. + const CkptDeadline deadline{boot_ms_now_fn, boot_ms_now_fn() + cas_request_budget.operation_deadline_ms}; + const CkptPublishOutcome outcome = publishCkpt( + backend, layout, life, contribution, admitted_generation, check_admission, deadline); + if (outcome == CkptPublishOutcome::Published) + ProfileEvents::increment(ProfileEvents::CASRefCheckpointPublished); + else if (outcome == CkptPublishOutcome::IdenticalSkip) + ProfileEvents::increment(ProfileEvents::CASRefCheckpointIdenticalSkip); + return outcome; +} + +bool CasRefLedger::tryPublishSnapshotAndAdvanceCheckpointOnce(const RootNamespace & ns) +{ + const auto rt = acquireMutableRefTableRuntime(ns); + return tryPublishSnapshotAndAdvanceCheckpointOnceOnRuntime(ns, rt); +} + + +bool CasRefLedger::tryPublishSnapshotAndAdvanceCheckpointOnceOnRuntime( + const RootNamespace & ns, const std::shared_ptr & rt) +{ + ensureRefTableRecovered(ns, *rt); + + /// This attempt's ADMISSION token, captured once, before any of its I/O: which mount incarnation + /// allowed this publish. It is presented back on every `_ckpt` CAS attempt below (spec §3's + /// recheck discipline -- the same value at every site), so a publish admitted under an incarnation + /// that has since been replaced can advance nothing. + const uint64_t admitted_generation = rt->admitted_fence_generation; + const auto runtime_still_admitted = [this, &rt, admitted_generation] + { + return !rt->catalog_life_invalidated.load(std::memory_order_acquire) + && !rt->superseded_by_remount.load(std::memory_order_acquire) + && fence_ok_fn() + && fence_generation_fn() == admitted_generation; + }; + + /// ONE copy of the live state, at a transaction boundary -- no + /// replay, no per-entry retention. The tail counters are captured in the SAME critical section so + /// adoption below subtracts exactly what this attempt's candidate actually covers. + RefTableState candidate_state; + RefTxnId candidate_x; + uint64_t captured_count = 0; + uint64_t captured_bytes = 0; + std::optional blocked_lane; + { + std::lock_guard lock(rt->state_mutex); + /// Snapshot certification is a `Ready`-only operation. This read and the state copy are one + /// critical section, so no transition to `Writing`, `Wedged`, or `NeedsRecovery` can interleave + /// between certification and capture. + if (rt->lane_state != RefLaneState::Ready) + blocked_lane = rt->lane_state; + else if (!hasStateBearingSnapshotCandidateUnderStateLock(*rt)) + return false; /// shares admission: terminal, covered, and seal candidates are all inert + else + { + candidate_state = rt->state; + candidate_x = rt->state.getGreatestApplied(); + captured_count = rt->tail_count_since_snapshot.load(std::memory_order_relaxed); + captured_bytes = rt->tail_bytes_since_snapshot.load(std::memory_order_relaxed); + } + } + + if (blocked_lane) + { + LOG_WARNING(getLogger("CasPool"), + "CAS ref table '{}': refusing snapshot publication while the append lane is not Ready " + "(state {})", + ns.string(), static_cast(*blocked_lane)); + return false; + } + + if (snapshot_after_capture_hook_for_test) + snapshot_after_capture_hook_for_test(); + + /// The candidate holds shared COW bases, so even this stale-runtime exit must release it under the + /// same mutex that protects materialization. More importantly, this is the last test-only pause + /// before the first durable effect: retirement/remount can invalidate the captured runtime while it + /// is paused, and the old holder then becomes inert instead of resolving the name to a successor. + if (!runtime_still_admitted()) + { + std::lock_guard lock(rt->state_mutex); + candidate_state = RefTableState{}; + return false; + } + + const RefTableSnapshot snap = snapshotOf(candidate_state, ns.string()); + + /// `candidate_state` is a COW copy that SHARES `rt->state`'s committed/owned-manifest bases. It is + /// dead past `snapshotOf`. Destroy it HERE, under `state_mutex` -- not at function return outside any + /// lock. Its destruction is a `shared_ptr` release-DECREMENT of those shared bases; the flush thread's + /// in-place `materializeCommitted()` reads their `use_count()` (relaxed) under this same mutex. Doing + /// the release off-lock would leave that load racing this atomic decrement with no happens-before + /// (TSan-reportable) and could momentarily let a flush observe a `use_count()` of 1 while this + /// decrement is in flight. Under the lock the two are serialized. Every subsequent exit path (encode + /// failure, non-Committed PUT, the monotonic-guard early return, success) then destroys an already + /// empty `candidate_state`, which touches no shared base. See both COW headers' materialize safety + /// argument, which relies on exactly this: every cross-thread copy is created AND destroyed under the + /// state lock. + { + std::lock_guard lock(rt->state_mutex); + candidate_state = RefTableState{}; + } + + String bytes; + try + { + bytes = sealObject(FormatId::RefSnapshot, encodeRefTableSnapshot(snap)); + } + catch (...) + { + /// Failure Handling: "Snapshot create fails: keep all logs; writer recovery remains unchanged." + /// Treat like any other non-Committed outcome: arm the backoff so a persistent encode failure + /// does not re-dispatch on every read. + std::lock_guard lock(rt->state_mutex); + advancePublishBackoff(*rt); + return false; + } + const String key = layout.refSnapshotKey(rt->life, candidate_x); + const CasWriteOutcome outcome + = ref_request_controller->putIfAbsentControlled(key, bytes, runtime_still_admitted); + if (outcome != CasWriteOutcome::Committed) + { + /// DefiniteFailure/Unresolved: DO NOT prune (no durable covering snapshot -- pruning the tail + /// without one is data loss). Arm the bounded per-table backoff so the read path does not + /// re-dispatch this full-snapshot encode+PUT until it elapses -- the read-triggered PUT-storm + /// latch breaker. A later trigger past the deadline retries. + std::lock_guard lock(rt->state_mutex); + advancePublishBackoff(*rt); + return false; + } + ProfileEvents::increment(ProfileEvents::CASRefSnapshotPutBytes, bytes.size()); /// account published bytes + + /// INV-4's SECOND `_ckpt` writer, at exactly the point the spec puts it: the snapshot body is + /// durable, and it becomes CLEANUP-AUTHORITATIVE only once the checkpoint names it. Ordering the + /// two this way is what makes the intervening race harmless -- cleanup planned between the body PUT + /// and this CAS still reads the OLD checkpoint, and the deletion gate is "strictly below" it, so it + /// cannot delete the snapshot just published. + /// + /// The contribution is the checkpoint ALONE. A publisher does not know this namespace's + /// `life_epoch` (it may have recovered the table from a snapshot that never replayed the birth), so + /// it contributes NOTHING for it and the semantic-max merge preserves whatever a writer that did + /// know has already recorded -- in either order. + /// + /// A checkpoint that does NOT advance leaves the attempt unadopted: the backoff is armed and this + /// returns false, so a later trigger re-runs the whole publish. The re-run's body PUT resolves to + /// `Committed` against its own identical bytes, so retrying costs one conditional PUT and not a + /// second snapshot. Adopting instead would mark this snapshot as the newest -- suppressing every + /// later publish for it -- while the checkpoint still pointed below it, leaving recovery replaying + /// from an older base with nothing scheduled to fix it. + bool ckpt_advanced = false; + if (!runtime_still_admitted()) + return false; + const auto check_runtime_admission = [this, &rt](uint64_t generation) + { + if (snapshot_before_ckpt_cas_hook_for_test) + snapshot_before_ckpt_cas_hook_for_test(); + check_fence_or_throw(generation); + if (rt->catalog_life_invalidated.load(std::memory_order_acquire) + || rt->superseded_by_remount.load(std::memory_order_acquire)) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': its captured runtime was retired before checkpoint publication", + rt->life.ns.string())); + }; + try + { + ckpt_advanced = publishCkptContribution(rt->life, RefCkpt{.life_epoch = std::nullopt, + .committed_through = candidate_x, + .checkpoint_snapshot_id = candidate_x, + .last_epoch_seal = std::nullopt}, + admitted_generation, + check_runtime_admission) != CkptPublishOutcome::FencedOut; + } + catch (...) + { + /// Swallowed deliberately, and ONLY here: the snapshot body is already durable, so there is + /// nothing to undo and nothing for a caller to decide -- + /// `tryPublishSnapshotAndAdvanceCheckpointOnce` is one best-effort attempt whose every other + /// failure arm also returns false with a backoff. The + /// counter and the log line are what keep it from being silent. + tryLogCurrentException(getLogger("CasPool"), + "CAS ref table '" + ns.string() + "': the snapshot body is durable but its _ckpt checkpoint " + "could not be advanced; the snapshot is not yet cleanup-authoritative and the publish will " + "be retried"); + } + if (!ckpt_advanced) + { + ProfileEvents::increment(ProfileEvents::CASRefCheckpointNotAdvanced); + std::lock_guard lock(rt->state_mutex); + advancePublishBackoff(*rt); + return false; + } + + { + std::lock_guard lock(rt->state_mutex); + if (!runtime_still_admitted()) + return false; + /// A durable publish clears any backoff: progress was made this attempt (even if the + /// monotonic guard below skips the in-memory adoption because a newer snapshot already won). + resetPublishBackoff(*rt); + /// Monotonic adoption guard (CRITICAL): publishes are NOT serialized, so two + /// overlapping attempts can finish out of order (this OLDER-candidate attempt landing its PUT + /// after a NEWER one already adopted). Adopting the older `candidate_x` here would REGRESS + /// `newest_snapshot_id` below what a newer attempt already advanced it to -- the next published + /// snapshot would then omit committed transactions and recovery would lose refs. Skip the + /// in-memory adoption (and the counter subtraction below) whenever a newer-or-equal snapshot is + /// already adopted; the already-durable `_snap/` object is harmless (readers pick + /// the greatest snapshot, GC reclaims covered ones). + if (rt->newest_snapshot_id && !(*rt->newest_snapshot_id < candidate_x)) + return true; + /// Subtract exactly the counters captured at copy time + /// -- more appends (or even another publish's own commits) may have landed on the LIVE counters + /// since, and only those should remain uncovered. Clamped (see `clampedCounterSub`): an + /// out-of-order adoption ordering the guard above does not catch (a SMALLER candidate that + /// adopts before a LARGER one already in flight) could otherwise subtract an already-subtracted + /// region and underflow the unsigned counter. + clampedCounterSub(rt->tail_count_since_snapshot, captured_count); + clampedCounterSub(rt->tail_bytes_since_snapshot, captured_bytes); + /// logs-per-table-after-snapshot: the tail this publish compacted. + ProfileEvents::increment(ProfileEvents::CASRefSnapshotTailLogs, captured_count); + rt->newest_snapshot_id = candidate_x; + /// The new cache-weight base is exactly the snapshot + /// we just encoded and PUT, so its body size is the fresh base weight -- no re-encode needed. + rt->base_snapshot_bytes.store(bytes.size(), std::memory_order_relaxed); + } + return true; +} + + +void CasRefLedger::sweepStalePrecommitsForRead(const RootNamespace & ns, const std::shared_ptr & rt) +{ + /// A read-only caller (resolveRef/listRefs) must not fail its OWN + /// otherwise-successful read because a piggybacked maintenance action (the stale-precommit sweep) + /// hit an uncertain PUT -- the read asked for none of that; a mutation path (appendRefOps's own + /// top-level hoisted call, which calls `maybeSweepStalePrecommits` directly, uncaught) keeps + /// propagating instead, since it must not proceed past a wedged lane anyway. Swallowing here does + /// Do NOT drop the sweep: the failed + /// attempt already re-armed `needs_stale_precommit_sweep` (with a bounded cooldown) inside + /// `maybeSweepStalePrecommits`, so a later read/mutation trigger on THIS mount retries until a + /// sweep completes verified clean -- the old drop-the-shot behavior left a dead incarnation's + /// precommit bindings (and the manifests they protect from the GC orphan sweep) live forever on a + /// long-lived mount whenever the single attempt burned in the post-restart error window. + try + { + maybeSweepStalePrecommits(ns, rt); + } + catch (...) + { + ProfileEvents::increment(ProfileEvents::CASRefSweepDeferred); + tryLogCurrentException(getLogger("CasPool"), + "CAS stale-precommit sweep deferred for namespace '" + ns.string() + + "' (a read-only caller observed the failure and is proceeding with its own read)"); + } +} + +void CasRefLedger::maybeSweepStalePrecommits(const RootNamespace & ns, const std::shared_ptr & rt) +{ + { + std::lock_guard lock(rt->state_mutex); + if (!rt->needs_stale_precommit_sweep) + return; + /// A failed attempt armed a cooldown; do not re-attempt (and do not touch the flag) + /// until it elapses -- the bounded-backoff storm latch, same shape as `publish_backoff_until_ms`. + /// The boottime clock is injectable (`boot_ms_fn`), so tests drive this deterministically. + if (boot_ms_now_fn() < rt->precommit_sweep_backoff_until_ms) + return; + /// Cleared FIRST: `sweepStalePrecommitsNow`'s own `appendRefOps` calls re-enter this same + /// top-level check (via `appendRefOps`'s hoisted call), and must see it already cleared. This + /// clear is for RE-ENTRANCY only, never consumption: any non-clean outcome re-arms below. + rt->needs_stale_precommit_sweep = false; + } + try + { + sweepStalePrecommitsNow(ns, rt); + } + catch (...) + { + /// A failed or partial sweep + /// failed or partial sweep must NOT consume the shot. Under kill-chaos the single attempt lands + /// exactly inside the post-restart error window (an uncertain PUT, a fence blip), and with no + /// retry the dead incarnation's durable precommit bindings -- and the manifests + /// `activeManifestKeys` protects for them -- leaked forever on a long-lived mount (GC has no + /// backstop). Re-arm with a bounded backoff and rethrow: the + /// read path insulates the caller (`sweepStalePrecommitsForRead`), the mutation path propagates + /// as before. + { + std::lock_guard lock(rt->state_mutex); + rt->needs_stale_precommit_sweep = true; + advancePrecommitSweepBackoff(*rt); + } + throw; + } + /// Verified clean: `sweepStalePrecommitsNow` returns only after a full pass over the live state + /// found zero stale bindings, so the flag stays cleared for the rest of this mount; reset the + /// failure cooldown too. + std::lock_guard lock(rt->state_mutex); + resetPrecommitSweepBackoff(*rt); +} + +void CasRefLedger::advancePrecommitSweepBackoff(RefTableRuntime & rt) +{ + /// Caller holds `rt.state_mutex`. Double the interval + /// from `initial` up to `max` per consecutive failed sweep attempt; arm the deadline off the + /// boottime clock (`bootMsNow`), so an injected test clock drives it deterministically and a + /// VM-suspend cannot shorten it. + rt.precommit_sweep_backoff_ms = rt.precommit_sweep_backoff_ms == 0 + ? config.precommit_sweep_backoff_initial_ms + : std::min(rt.precommit_sweep_backoff_ms * 2, config.precommit_sweep_backoff_max_ms); + rt.precommit_sweep_backoff_until_ms = boot_ms_now_fn() + rt.precommit_sweep_backoff_ms; + ProfileEvents::increment(ProfileEvents::CASRefSweepRearmed); +} + +void CasRefLedger::resetPrecommitSweepBackoff(RefTableRuntime & rt) +{ + /// Caller holds `rt.state_mutex`. A verified-clean sweep clears the cooldown. + rt.precommit_sweep_backoff_ms = 0; + rt.precommit_sweep_backoff_until_ms = 0; +} + + +void CasRefLedger::sweepStalePrecommitsNow(const RootNamespace & ns, const std::shared_ptr & rt) +{ + /// After a fresh mount fence and recovery, this writer + /// knows the exact stale precommit bindings -- their `manifest_ref.writer_epoch` predates this + /// incarnation's live writer_epoch, i.e. they belong to a build from a superseded incarnation that + /// can never be promoted. Removed with ordinary exact `owner_transition(old_binding, none)` + /// operations, chunked to `ref_txn_max_ops` per transaction. Interruption is harmless: each chunk + /// re-reads the LIVE state, so a partial sweep just leaves fewer stale bindings for the next chunk + /// (a later retry on this mount, or the next mount's recovery) to find; nothing here can loop + /// forever since only OLDER-epoch bindings ever qualify, and this writer's own new work always uses + /// `live_epoch_fn()` -- which a self-remount bumps in lockstep with the threshold below, so a + /// remount's fresh precommits survive. + /// + /// A GC-side backstop stays deliberately OUT: the responsibility boundary assigns + /// precommit-binding cleanup to the WRITER -- GC never mutates another writer's ref-table state, so + /// a leader-side reclaim would be a new protocol capability (a question about GC-authored + /// ref-log transactions and their fencing), not a bugfix. The retry-until-clean loop above is the + /// writer-side answer; the follow-up (a GC visibility counter for "live precommit binding below the + /// mount-lease epoch" would require a separate protocol decision. + const uint64_t live_epoch = live_epoch_fn(); + while (true) + { + std::vector> chunk; + { + std::lock_guard lock(rt->state_mutex); + for (const auto & [ref_name, mref] : rt->state.getPrecommits()) + { + if (mref.writer_epoch >= live_epoch) + continue; + chunk.emplace_back(ref_name, mref); + if (chunk.size() >= ref_txn_max_ops) + break; + } + } + if (chunk.empty()) + return; + + appendRefOps(ns, MutationScope::wholeShard(), + [chunk](const RefTableState & state) -> std::vector + { + std::vector ops; + for (const auto & [ref_name, mref] : chunk) + if (state.getPrecommits().contains({ref_name, mref})) + { + RefOp op; + op.kind = RefOpKind::OwnerTransition; + op.old_binding = RefOwnerBinding{RefOwnerKind::Precommit, ref_name, mref}; + ops.push_back(op); + } + return ops; + }, + RootMutationOrigin::Writer, RootMutationKind::ReclaimPrecommit); + + /// Audit each binding this sweep reclaimed, so + /// `system.cas_log` records the reclaim and the "abandoned precommits + /// reclaimed" counter is falsifiable (it had ZERO emit sites before). A binding gathered above + /// that is GONE from the live state after the committed append was reclaimed by this sweep's + /// work -- either this chunk's own ops or this lane's just-resolved wedged predecessor txn (a + /// PRIOR attempt of this same sweep whose ack was lost); one still present was skipped by the + /// builder (raced by another owner transition) and will be gathered again next iteration. + /// Collected under the lock, emitted outside it (the sink forwards to the SystemLog). + std::vector> reclaimed; + { + std::lock_guard lock(rt->state_mutex); + for (const auto & [ref_name, mref] : chunk) + if (!rt->state.getPrecommits().contains({ref_name, mref})) + reclaimed.emplace_back(ref_name, mref); + } + ProfileEvents::increment(ProfileEvents::CASRefStalePrecommitsReclaimed, reclaimed.size()); + for (const auto & [ref_name, mref] : reclaimed) + { + EventEmitter{*this}.emit([&](CasEvent & e) + { + e.type = CasEventType::PrecommitReclaim; + e.namespace_ = ns.string(); + e.ref_name = ref_name; + e.object_kind = CasEventObjectKind::Root; + e.object_hash = manifestRefDebugString(mref); + e.reason = "stale-precommit sweep: dangling precommit of a superseded writer incarnation " + "reclaimed by the successor's fenced sweep"; + e.detail = {{"stale_writer_epoch", std::to_string(mref.writer_epoch)}, + {"live_writer_epoch", std::to_string(live_epoch)}}; + }); + } + } +} + + +void CasRefLedger::dropRef(const RootNamespace & ns, const String & ref_name) +{ + /// One `owner_transition` removal ref-log transaction. The + /// exact committed binding must exist; `build_ops` reads it off the CURRENT batch-validation state, + /// so a concurrently-co-batched publish/drop of a DIFFERENT ref sees a consistent view. + ManifestRef dropped_ref; + const RefTxnId txn_id = appendRefOps(ns, MutationScope::ref(ref_name), + [&](const RefTableState & state) -> std::vector + { + const auto it = state.getCommitted().find(ref_name); + if (it == state.getCommitted().end()) + /// Fail-closed (no silent no-op): this item's own exception, the batch survives. + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "dropRef: no such ref {} in namespace {}", ref_name, ns.string()); + + dropped_ref = it->second.manifest_ref; + RefOp op; + op.kind = RefOpKind::OwnerTransition; + op.old_binding = RefOwnerBinding{RefOwnerKind::Committed, ref_name, dropped_ref}; + return {op}; + }, + RootMutationOrigin::Writer, RootMutationKind::Drop); + + /// The ref was dropped (a removal operation GC folds as a true removal). `object_hash` is the + /// manifest the ref named, so a part's "publish -> drop" life is reconstructable from the rows. + if (hasEventSink()) + { + CasEvent _ev3; + _ev3.type = CasEventType::RefDrop; + _ev3.namespace_ = ns.string(); + _ev3.ref_name = ref_name; + _ev3.object_kind = CasEventObjectKind::Manifest; + _ev3.object_hash = manifestRefDebugString(dropped_ref); + _ev3.at_version = txn_id.ref_sequence; + _ev3.outcome = "ok"; + _ev3.reason = "dropRef: appended an owner_transition removal ref-log transaction"; + emitEvent(std::move(_ev3)); + } +} + + +void CasRefLedger::updateRefPublishedAt(const RootNamespace & ns, const String & ref_name, + std::function mutator) +{ + /// One `set_published_at` ref-log transaction. EVERY change (even timestamp-only) is an explicit + /// logged operation -- the immutable append-only log has no other way to record it. + /// `published_at_ms` is the only metadata this op carries (the mutable-file map is gone; every + /// per-part file is an ordinary manifest tree entry now, republished via `repointRef`, never + /// through this side channel). + appendRefOps(ns, MutationScope::ref(ref_name), + [&](const RefTableState & state) -> std::vector + { + const auto it = state.getCommitted().find(ref_name); + if (it == state.getCommitted().end()) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "updateRefPublishedAt: no such ref {} in namespace {}", ref_name, ns.string()); + + /// The mutator edits only `published_at_ms`; the carrier deliberately carries no + /// `manifest_ref`, so a reachability change is structurally impossible here (it goes through + /// publish/drop/repoint instead). + RefPublishedAtUpdate update; + update.published_at_ms = it->second.published_at_ms; + + mutator(update); + + RefOp op; + op.kind = RefOpKind::SetPublishedAt; + op.ref_name = ref_name; + op.expected_manifest_ref = it->second.manifest_ref; + op.published_at_ms = update.published_at_ms; + return {op}; + }, + RootMutationOrigin::Writer, RootMutationKind::UpdateRefPublishedAt); +} + + +NamespaceLifeId CasRefLedger::namespaceLife(const RootNamespace & ns) +{ + auto rt = lookupRefTableRuntime(ns); + if (rt) + { + check_fence_or_throw(rt->admitted_fence_generation); + bool removal_closed = false; + { + std::lock_guard queue_lock(ref_queue_mutex); + removal_closed = rt->removal_admission_closed; + } + if (removal_closed) + { + /// A lost erase response can leave only the detached predecessor's close bit. Reconcile + /// before refusing so an absent/replaced row frees the logical name without rebinding it. + reconcileCatalogCut(CasRefCatalog::read(backend, layout)); + const auto refreshed = lookupRefTableRuntime(ns); + if (refreshed == rt) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}' is Removing: creation waits for its terminal fold and catalog " + "removal to complete; retry later", ns.string())); + rt = refreshed; + } + if (rt) + { + ensureRefTableRecovered(ns, *rt); + return rt->life; + } + } + + /// A cold mutation observes or births the durable identity before allocating any local state. + const uint64_t admitted_generation = fence_generation_fn(); + check_fence_or_throw(admitted_generation); + const CasRefCatalog::Snapshot catalog = CasRefCatalog::read(backend, layout); + check_fence_or_throw(admitted_generation); + const auto entry_it = std::find_if(catalog.catalog.entries.begin(), catalog.catalog.entries.end(), + [&](const CatalogEntry & entry) { return entry.ns == ns; }); + if (entry_it != catalog.catalog.entries.end() && entry_it->state == NsState::Removing) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}' is Removing: creation waits for its terminal fold and catalog " + "removal to complete; retry later", ns.string())); + + const NamespaceLifeId life + = entry_it != catalog.catalog.entries.end() && entry_it->state == NsState::Live + ? NamespaceLifeId::fromCatalogEntry(entry_it->ns, entry_it->incarnation) + : resolveNamespaceLife(ns, admitted_generation, live_epoch_fn()); + check_fence_or_throw(admitted_generation); + rt = acquireRefTableRuntime(life, admitted_generation); + ensureRefTableRecovered(ns, *rt); + return rt->life; +} + +std::optional CasRefLedger::namespaceFilesLifeIfReadable(const RootNamespace & ns) +{ + const auto rt = acquireReadableRefTableRuntime(ns); + + /// A namespace the catalog does not name has no files to read, and a read or an unlink is the wrong + /// event to bring one into existence on. Fresh resolution admits only a catalog `Live` row; + /// `Creating`, `Removing` and absent all answer absent without recovery or mutation. + if (!rt) + return std::nullopt; + ensureRefTableRecovered(ns, *rt); + + std::lock_guard lock(rt->state_mutex); + /// A stale already-held runtime may have applied the terminal before catalog invalidation reaches + /// it. Preserve the stated stale-or-not-found contract by hiding that terminal view. + if (rt->state.getLifecycle() != RefLifecycle::Live && rt->state.getRemoveTxnId().has_value()) + return std::nullopt; + return rt->life; +} + +bool CasRefLedger::namespaceStillLogicallyPresent(const RootNamespace & ns) +{ + /// O(1) fast path: a resident runtime already proven `Live` under an unbroken fence answers without + /// any catalog fetch -- the common case (`existsDirectory` on a warm, ordinary table). Anything + /// short of that falls through to the exact cold-path observation below. + if (const auto current = lookupRefTableRuntime(ns)) + { + check_fence_or_throw(current->admitted_fence_generation); + bool closed = false; + { + std::lock_guard queue_lock(ref_queue_mutex); + closed = current->removal_admission_closed; + } + if (!closed + && !current->catalog_life_invalidated.load(std::memory_order_acquire) + && !current->superseded_by_remount.load(std::memory_order_acquire)) + { + std::lock_guard state_lock(current->state_mutex); + if (current->recovered && current->state.getLifecycle() == RefLifecycle::Live) + return true; + } + } + + /// Cold path: an exact catalog observation. `Creating` and `Live` both answer present immediately -- + /// `true` is always the safe direction, so no revalidation is needed for either. A missing row is + /// the one answer that must never be manufactured by a race, so it alone is re-confirmed against a + /// second read before being trusted (mirroring `acquireReadableRefTableRuntime`'s own token/value + /// revalidation). + const uint64_t admitted_generation = fence_generation_fn(); + check_fence_or_throw(admitted_generation); + const CasRefCatalog::Snapshot first_catalog = CasRefCatalog::read(backend, layout); + check_fence_or_throw(admitted_generation); + if (namespace_presence_probe_after_first_read_hook_for_test) + namespace_presence_probe_after_first_read_hook_for_test(); + const auto find_entry = [&ns](const CasRefCatalog::Snapshot & snap) -> const CatalogEntry * + { + const auto it = std::find_if(snap.catalog.entries.begin(), snap.catalog.entries.end(), + [&](const CatalogEntry & entry) { return entry.ns == ns; }); + return it == snap.catalog.entries.end() ? nullptr : &*it; + }; + + const CatalogEntry * entry = find_entry(first_catalog); + if (!entry) + { + const CasRefCatalog::Snapshot second_catalog = CasRefCatalog::read(backend, layout); + check_fence_or_throw(admitted_generation); + if (second_catalog.token != first_catalog.token || second_catalog.catalog != first_catalog.catalog) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': its catalog changed while probing table-root cleanup completeness; " + "retry from a fresh observation", ns.string())); + return false; /// no catalog row, twice-confirmed: proven absent + } + + if (entry->state == NsState::Creating || entry->state == NsState::Live) + return true; + + /// `Removing`: only the exact incarnation's own durable ref-log proves the terminal + /// `remove_namespace` transaction actually landed -- the catalog row alone cannot distinguish a + /// completed removal from one whose terminal append is still outstanding after a crash. This is the + /// same load-bearing distinction `dropNamespaceImpl` makes before returning early. Once durably + /// proven for this EXACT incarnation, that incarnation's own terminal can never be un-proven -- but + /// the recovery call above (`acquireRefTableRuntime`/`ensureRefTableRecovered`) is real I/O with no + /// upper bound on wall time, and GC deleting this incarnation's now-terminal catalog row plus a + /// same-name rebirth (an explicitly supported sequence -- see `CASRefWriterNamespaceRemoval`'s own + /// same-name-rebirth tests) can both land inside that window. Proving THIS incarnation terminal is + /// therefore not proof that the CURRENT logical namespace `ns` is absent; a fresh catalog read is + /// required before answering `false` for the name. + const NamespaceLifeId life = NamespaceLifeId::fromCatalogEntry(entry->ns, entry->incarnation); + const auto rt = acquireRefTableRuntime(life, admitted_generation); + ensureRefTableRecovered(ns, *rt); + bool this_incarnation_terminal = false; + { + std::lock_guard lock(rt->state_mutex); + this_incarnation_terminal = rt->state.getLifecycle() != RefLifecycle::Live && rt->state.getRemoveTxnId().has_value(); + } + if (!this_incarnation_terminal) + return true; /// removal admitted but not yet terminal: cleanup work remains + + if (namespace_presence_probe_after_terminal_proven_hook_for_test) + namespace_presence_probe_after_terminal_proven_hook_for_test(); + + check_fence_or_throw(admitted_generation); + const CasRefCatalog::Snapshot post_terminal_catalog = CasRefCatalog::read(backend, layout); + const CatalogEntry * post_terminal_entry = find_entry(post_terminal_catalog); + if (!post_terminal_entry || post_terminal_entry->incarnation == entry->incarnation) + return false; /// terminal durably proven and nothing has since occupied `ns` under a new life + /// A successor incarnation now occupies `ns` -- `Creating`, `Live`, or a fresh `Removing` all mean + /// the name is not the proven-absent predecessor this call observed. `true` is always the safe + /// direction; a caller that acts on it retries against the successor's own (correct) state rather + /// than being told the namespace is gone while something already occupies its name. + return true; +} + + +DropNamespaceStats CasRefLedger::dropNamespace(const RootNamespace & ns) +{ + return dropNamespaceImpl(ns, std::nullopt); +} + +DropNamespaceStats CasRefLedger::dropNamespaceImpl( + const RootNamespace & ns, const std::optional & expected_incarnation) +{ + /// One body transaction naming an exact `owner_transition` + /// removal for every committed ref and precommit, followed by `remove_namespace` -- the removal + /// class shares the bigger complete-table byte budget (encodeRefLogTxn's own `checkBudget`, keyed + /// off the presence of a `RemoveNamespace` op) and is exempt from the ordinary per-op admission + /// check (it only ever shrinks state; see `flushRefBatch`'s `state_growing` filter). + const CasRefCatalog::Snapshot initial_catalog = CasRefCatalog::read(backend, layout); + const auto initial_it = std::find_if(initial_catalog.catalog.entries.begin(), initial_catalog.catalog.entries.end(), + [&](const CatalogEntry & entry) { return entry.ns == ns; }); + if (initial_it == initial_catalog.catalog.entries.end()) + return {}; + if (expected_incarnation && initial_it->incarnation != *expected_incarnation) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': exact removal life {} differs from current catalog life {}", + ns.string(), renderIncarnation(*expected_incarnation), renderIncarnation(initial_it->incarnation))); + + if (initial_it->state == NsState::Creating) + { + const CatalogEntry & observed = *initial_it; + const uint64_t admitted_generation = fence_generation_fn(); + switch (CasRefCatalog::cancelStalledCreating( + backend, layout, observed, + [this](const CreatorFence & creator) + { + return isCreatorFenceTerminal( + backend, layout, creator.server_root_id, creator.writer_epoch); + }, + admitted_generation, check_fence_or_throw)) + { + case CasRefCatalog::StalledCreatingCancelOutcome::Cancelled: + invalidateRemovedCatalogLife(NamespaceLifeId::fromCatalogEntry(observed.ns, observed.incarnation)); + return {}; + case CasRefCatalog::StalledCreatingCancelOutcome::CreatorFenceStillLive: + throwCasWriteRetryLater(fmt::format( + "CAS namespace removal '{}': its catalog entry is still Creating under a creator " + "fence that is not yet provably terminal; retry later", ns.string())); + case CasRefCatalog::StalledCreatingCancelOutcome::EntryChanged: + throwCasWriteRetryLater(fmt::format( + "CAS namespace removal '{}': exact Creating row changed before cancellation; retry later", + ns.string())); + case CasRefCatalog::StalledCreatingCancelOutcome::FencedOut: + throwCasWriteRetryLater(fmt::format( + "CAS namespace removal '{}': mount fence moved before stalled creation cancellation", + ns.string())); + } + } + + const NamespaceLifeId observed_life + = NamespaceLifeId::fromCatalogEntry(initial_it->ns, initial_it->incarnation); + const uint64_t runtime_generation = fence_generation_fn(); + const auto rt = acquireRefTableRuntime(observed_life, runtime_generation); + ensureRefTableRecovered(ns, *rt); + { + /// A real terminal transaction is idempotent. The representation also calls an empty, never-born + /// stream `Removed`, but its absent `remove_txn_id` distinguishes that state: a cataloged life may + /// already own `_ckpt` or `_files`, so it still needs durable birth+terminal evidence before GC + /// may delete its catalog row. + std::lock_guard lock(rt->state_mutex); + if (rt->state.getLifecycle() != RefLifecycle::Live && rt->state.getRemoveTxnId().has_value()) + return {}; + } + + /// Close the local positive lane BEFORE publishing `Removing`. Calls already admitted ahead of + /// this point drain first; later positive callers observe the flag in the same queue critical + /// section as enqueue and receive the typed retry-later refusal. The terminal item below is the one + /// deliberate exception. + { + std::unique_lock queue_lock(ref_queue_mutex); + rt->removal_admission_closed = true; + rt->cv.wait(queue_lock, [&] + { + return !rt->leader_active && rt->pending.empty(); + }); + } + + const uint64_t admitted_generation = fence_generation_fn(); + std::optional observed_live; + if (initial_it->state == NsState::Live) + observed_live = *initial_it; + bool removing_durable = false; + try + { + const CasRefCatalog::Snapshot snapshot = CasRefCatalog::read(backend, layout); + const auto entry_it = std::find_if(snapshot.catalog.entries.begin(), snapshot.catalog.entries.end(), + [&](const CatalogEntry & entry) { return entry.ns == ns; }); + if (entry_it == snapshot.catalog.entries.end()) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': its catalog row disappeared before removal admission closed", + ns.string())); + + const NamespaceLifeId & life = rt->life; + if (entry_it->incarnation != life.incarnation) + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': cached life {} differs from catalog life {} while beginning removal", + ns.string(), renderIncarnation(life.incarnation), renderIncarnation(entry_it->incarnation))); + + if (entry_it->state == NsState::Removing) + { + removing_durable = true; /// retry after an earlier terminal append failure/ambiguous CAS + } + else if (entry_it->state == NsState::Live) + { + observed_live = *entry_it; + uint64_t removal_started_round = 0; + if (const auto got = backend.get(layout.gcStateKey())) + removal_started_round = decodeGcState(got->bytes).round; + + switch (CasRefCatalog::beginRemoving( + backend, layout, *observed_live, removal_started_round, + admitted_generation, check_fence_or_throw)) + { + case CasRefCatalog::BeginRemovingOutcome::Transitioned: + case CasRefCatalog::BeginRemovingOutcome::AlreadyRemoving: + removing_durable = true; + break; + case CasRefCatalog::BeginRemovingOutcome::EntryChanged: + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': its exact catalog row changed while beginning removal", + ns.string())); + case CasRefCatalog::BeginRemovingOutcome::FencedOut: + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': the mount fence moved while beginning removal", + ns.string())); + } + } + else + { + throwCasWriteRetryLater(fmt::format( + "CAS namespace '{}': its catalog row is Creating while removal owns a recovered life", + ns.string())); + } + } + catch (...) + { + /// Resolve an ambiguous transition before deciding whether this lane may reopen. `Removing` + /// under the same life is conclusive success. Reopening is permitted only after a fresh exact + /// observation still proves the original `Live` row and the same mount fence; every unreadable, + /// changed or fenced case remains closed (fail-close) and propagates the original error. + try + { + const CasRefCatalog::Snapshot fresh = CasRefCatalog::read(backend, layout); + const auto fresh_it = std::find_if(fresh.catalog.entries.begin(), fresh.catalog.entries.end(), + [&](const CatalogEntry & entry) { return entry.ns == ns; }); + if (observed_live + && fresh_it != fresh.catalog.entries.end() + && fresh_it->incarnation == observed_live->incarnation + && fresh_it->state == NsState::Removing) + { + removing_durable = true; + } + else if (observed_live && fresh_it != fresh.catalog.entries.end() && *fresh_it == *observed_live) + { + check_fence_or_throw(admitted_generation); + std::lock_guard queue_lock(ref_queue_mutex); + rt->removal_admission_closed = false; + rt->cv.notify_all(); + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// The original failure remains the caller-visible one. Failure to prove an exact fresh + /// `Live` row deliberately leaves admission closed. + } + if (!removing_durable) + throw; + } + + chassert(removing_durable); + + /// This call's own removal + /// transaction named, filled from the SAME `state` the ops below are built from -- a retried + /// `build_ops` (a wedge resolving under a resumed leader) simply overwrites it with the final + /// durable transaction's true counts. + DropNamespaceStats stats; + appendRefOpsOnRuntime(ns, rt, MutationScope::wholeShard(), + [&](const RefTableState & state) -> std::vector + { + if (state.getLifecycle() != RefLifecycle::Live && state.getRemoveTxnId().has_value()) + return {}; /// raced: another caller already removed it since our check above + + std::vector ops; + if (state.getLifecycle() != RefLifecycle::Live) + { + RefOp birth; + birth.kind = RefOpKind::NamespaceBirth; + ops.push_back(std::move(birth)); + } + for (const auto [ref_name, row] : state.getCommitted()) + { + RefOp op; + op.kind = RefOpKind::OwnerTransition; + op.old_binding = RefOwnerBinding{RefOwnerKind::Committed, ref_name, row.manifest_ref}; + ops.push_back(op); + } + for (const auto & [ref_name, mref] : state.getPrecommits()) + { + RefOp op; + op.kind = RefOpKind::OwnerTransition; + op.old_binding = RefOwnerBinding{RefOwnerKind::Precommit, ref_name, mref}; + ops.push_back(op); + } + RefOp remove; + remove.kind = RefOpKind::RemoveNamespace; + ops.push_back(remove); + + stats.committed_refs = state.getCommitted().size(); + stats.precommits = state.getPrecommits().size(); + return ops; + }, + RootMutationOrigin::Writer, RootMutationKind::DropNamespace, + /// The operations above already name (and remove) every current precommit + /// binding regardless of epoch, making the ordinary stale-precommit maintenance sweep redundant + /// for THIS call -- and, left enabled, a race: the hoisted sweep runs first and would reclaim an + /// epoch-stale binding in its OWN transaction, so `state.getPrecommits()` above would already be + /// missing it and undercount `stats.precommits`. See `appendRefOps`'s doc comment. + /*skip_stale_precommit_sweep=*/true, + /*terminal_removal_authorized=*/true); + + /// "After the transaction is durable, it applies the same + /// operations to memory, cancels local builds, and rejects further ordinary mutations." Reaching here + /// means the removal is durable (this call's, or a concurrent caller's whose durable result the append + /// lane observed) -- a FAILED append would have thrown above, so cancellation is only ever reached + /// after durability (a failed append leaves the namespace `Removing`, not `Live`, and propagates; + /// the catalog CAS above already made that transition durable before this append was attempted). + /// Cancel + /// every in-flight build TARGETING this namespace so its next op fails closed (`requireAlive`), + /// preventing it from promoting/precommitting a fresh owner into (or staging more debris in) the + /// just-removed namespace. The append lane is the real linearization authority (an `owner_transition` + /// on a non-Live namespace is rejected by the state machine regardless); this stops wasted work early + /// and surfaces a clear error. Builds in OTHER namespaces self-filter (no-op). The build registry + /// (`inflight_builds`) lives on the owning Pool, so the cancellation runs through the injected + /// `cancel_inflight_builds` callback (which collects the live shared_ptrs under `builds_mutex` and + /// cancels OUTSIDE it -- see `Pool::cancelInflightBuildsForNamespace`). + cancel_inflight_builds(ns); + + /// No background publisher may carry the old runtime across the later catalog deletion and its + /// in-place reset. A removal has already succeeded here, so the cached lifecycle is no longer Live + /// and settlement cannot re-dispatch another publisher. + { + std::unique_lock state_lock(rt->state_mutex); + rt->publish_settle_cv.wait(state_lock, [&] + { + return rt->pending_snapshot_publishes.load(std::memory_order_acquire) == 0; + }); + } + + /// The writer performs no physical deletion of ref-log/snapshot objects or namespace files. Once + /// the catalog row is drained, those objects are dead-life debris for the perpetual janitor. + return stats; +} + +DropNamespaceStats CasRefLedger::dropNamespace(const NamespaceLifeId & life) +{ + return dropNamespaceImpl(life.ns, life.incarnation); +} + + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.h new file mode 100644 index 000000000000..a42ec80efb89 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.h @@ -0,0 +1,1246 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Controls whether `resolveRef` emits its `RefResolve` audit event. `Emit` (the default) preserves +/// today's behavior for every existing caller (`listRefs`, `dropRef`, GC, and ordinary reads). +/// `Deferred` is for a caller that itself decides, after inspecting the resolve outcome, whether the +/// access as a whole did real resolve work worth auditing — see `CachedPartFolderAccess::getView`, +/// which re-emits the identical event on every path except a warm view-cache hit that served without +/// re-validating anything. +enum class ResolveAudit : uint8_t { Emit, Deferred }; + +/// The complete state of one table's append lane. It is guarded by `RefTableRuntime::state_mutex`; +/// there is no independent apply marker or durable-id floor whose combinations form a second, +/// implicit state machine. +/// +/// `Ready` is the only state that admits a new append or certifies a cached row. `Writing` owns the +/// exact attempt before its first possible send. `Wedged` owns that same attempt after an ambiguous +/// result. `NeedsRecovery` means a transaction is known durable but cannot be installed in this cache; +/// it is a hard write and certification fence until replay completes. `Closed` records a successor's +/// epoch seal, and `Faulted` records foreign or internally inconsistent durable state. +enum class RefLaneState : uint8_t +{ + Ready, + Writing, + Wedged, + NeedsRecovery, + Closed, + Faulted, +}; + +/// The answer of the relink confirm's gate 1 (`CasRefLedger::confirmExactRef`). +/// +/// `Yes` is the only answer that AUTHORIZES anything, so it is the only one that must be earned: it is +/// returned exclusively when every rule of the lane snapshot holds. `Unknown` is the catch-all for +/// every ambiguity, and it is the answer this primitive is biased towards: a cold, evicted, recovering, +/// busy or non-`Ready` table answers `Unknown` rather than doing any work to find out. +/// +/// `No` means "this runtime's committed row for that ref is not the manifest you asked about" -- and +/// nothing more. It is NOT a proof of the negative about the durable table, because the mount fence is +/// evaluated LAST (rule 6, deliberately): a mount that has already lost its fence, and whose view may +/// therefore be behind another writer's repoint, still answers `No` rather than `Unknown`. That is +/// sound only because `No` and `Unknown` are the SAME outcome for the caller -- both are +/// `SourceProofFailed` (spec §failure-taxonomy) -- so nothing is authorized by either. Do not build a +/// consumer that treats `No` as knowledge; only `Yes` is gated on the fence. +enum class ConfirmAnswer : uint8_t { Yes, No, Unknown }; + +/// Coordinates the writer-side ref-log and ref-table protocol for all namespaces in one mounted pool. +/// It owns the recovered whole-table cache, the flat-combining append lane and its unresolved-`PUT` +/// wedge, snapshot publication, stale-precommit cleanup, cache-budget eviction, and remount/shutdown +/// draining. `ref_queue_mutex` protects cache membership and queue leadership; each table's +/// `state_mutex` protects its decoded state and per-table lifecycle. Network I/O is deliberately performed +/// without holding `state_mutex`, so readers and other maintenance operations are not blocked by retries. +/// +/// The ledger receives storage, configuration, event delivery, and retry-budget dependencies directly. +/// Mount state remains owned by `Pool` and is exposed through callbacks: the live writer epoch, append +/// fence, clocks, mutation gate, unclean-boundary observation, anomaly reaction, owner lifetime pin, and +/// cancellation of in-flight builds. The detached snapshot publisher uses the lifetime pin; no +/// `Pool &` back-reference is retained. `Pool` forwards its existing public operations to this component. +class CasRefLedger +{ +public: + CasRefLedger( + BackendPtr backend_ptr, + const Layout & layout_, + RefLedgerConfig config_, + const CasEventSink & event_sink_, + CasRequestBudget cas_request_budget_, + /// This mount's own `server_root_id` (spec §3's `CreatorFence`): the ledger mints its OWN + /// creator fence out of this plus the live writer epoch and admission fence generation when it + /// resolves a namespace's catalog life (`resolveNamespaceLife`) -- never injected as a callback, + /// unlike the mount-state functions below, because it is a fixed identity for this ledger's + /// whole lifetime (mirrors `CasMountRuntime`'s own by-value `server_root_id`). + String server_root_id_, + /// Monotonic mount clock used by the retry controller; it may be empty when the controller's + /// default clock is appropriate. + std::function controller_boot_ms_fn, + /// Callbacks into mount and watermark state owned by `Pool`, bound for this ledger's lifetime: + std::function live_epoch_fn_, + std::function fence_ok_fn_, + /// The two fence-GENERATION primitives (`CasMountRuntime::fenceGeneration`/`checkFenceOrThrow`), + /// injected exactly as `CasPlainObjects` takes them. `fence_ok_fn` above answers "may this mount + /// write AT ALL, right now"; these two answer the different question an append lane must ask + /// across an I/O window: "is this still the SAME mount incarnation that admitted the transaction + /// I am about to act on?" A wedge captures the generation at admission and presents it back on + /// every later retry and before every install, so a result that returns after a fence loss or a + /// re-arm is inert for the superseded runtime instead of installing a stale view (spec §3, + /// "the mount-fence generation is captured at admission and required on every slot-occupy and + /// install"). + std::function fence_generation_fn_, + std::function check_fence_or_throw_, + std::function boot_ms_now_fn_, + std::function may_mutate_, + std::function &)> on_impossible_interference_, + std::function()> pin_owner_, + std::function cancel_inflight_builds_); + + /// Recovers `ns` on first access and resolves `ref_name` from the authoritative cached table. + /// The optional staleness argument remains for API compatibility; this mounted writer has no + /// alternate shard cache, so the recovered table is always the view used for the result. + /// `audit` defaults to `Emit` so every existing caller keeps emitting `RefResolve` unchanged; + /// `Deferred` suppresses the emit for a caller that re-emits it conditionally itself. + std::optional resolveRef(const RootNamespace & ns, const String & ref_name, bool allow_stale = false, + ResolveAudit audit = ResolveAudit::Emit); + + /// Recovers `ns` on first access and returns every committed ref in canonical name order. Read-side + /// maintenance may schedule snapshot publication and stale-precommit cleanup, but those actions do + /// not change the returned committed view or make a read fail when maintenance has an uncertain `PUT`. + std::map listRefs(const RootNamespace & ns); + + /// Recovers `ns` on first access and reports whether any committed ref name starts with `prefix`, + /// without materializing the full ref map `listRefs` returns. An empty `prefix` means "any ref at + /// all" and short-circuits on the first entry, so this is O(1) for that (dominant, emptiness-probe) + /// case; a non-empty prefix still stays a no-allocation scan. + bool hasAnyRefWithPrefix(const RootNamespace & ns, std::string_view prefix); + + /// Gate 1 of the relink confirm (spec §confirm-primitive): does `ref_name` in `ns` still name + /// EXACTLY `manifest_ref` in this writer's committed view, read under a lane snapshot that cannot + /// observe a stale cache? + /// + /// Performs ZERO object-store I/O: a cold, evicted or recovering table answers `Unknown` rather + /// than recovering from storage, and no runtime is created as a side effect of asking. That is a + /// contract, not an optimization -- the confirm is a read-only interserver query that a remote + /// receiver drives, so it must never be able to make this writer do work. + /// + /// The rules are evaluated as one snapshot spanning both lane mutexes, in this order: table warm + /// and resident; lane state `Ready`; exact committed-row equality; mount fence live last. Every + /// ambiguity answers `Unknown` -- see `ConfirmAnswer`, and the .cpp for why the order and the + /// two-mutex hold are what make a `Yes` a linearization point rather than a guess. + ConfirmAnswer confirmExactRef(const RootNamespace & ns, const String & ref_name, + const ManifestRef & manifest_ref) const; + + /// Appends the transaction that removes one ref and waits for its durable result. A failed append + /// propagates its exception and does not apply the removal to the in-memory table. + void dropRef(const RootNamespace & ns, const String & ref_name); + + /// Builds and appends a published_at_ms update for one ref. The mutator is invoked while + /// constructing the transaction, and its changes become visible only after the append is durable. + void updateRefPublishedAt(const RootNamespace & ns, const String & ref_name, + std::function mutator); + + /// Durably removes the complete namespace, including its current ref/precommit state, then performs + /// the associated cancellation work. The catalog transition to `Removing` happens first; a failed + /// terminal append after that leaves the namespace `Removing` (not `Live`) and propagates. + DropNamespaceStats dropNamespace(const RootNamespace & ns); + + /// Decommission-only exact-life form. Pins recovery to the immutable catalog cut selected by the + /// admin command, so a same-name replacement can never redirect destructive work. + DropNamespaceStats dropNamespace(const NamespaceLifeId & life); + + /// The catalog life every one of this namespace's objects -- ref-layer AND namespace-file -- is keyed + /// under, resolved ONCE per table-open and read from the cache afterwards. This is the WRITE-side + /// resolution, and the ONLY one that CREATES: recovery's step 0 (`resolveNamespaceLife`) mints a + /// life when the catalog names none, so the first namespace file a table ever writes births the + /// namespace exactly as its first ref op would. A read or a removal must not use this -- see the + /// sibling below for why that is a correctness matter and not a preference. + NamespaceLifeId namespaceLife(const RootNamespace & ns); + + /// The life a READER (or a REMOVER) of this namespace's files must use, or `nullopt` when there are + /// no readable files at all -- which is the same answer for a namespace that never existed, one + /// still being created, and one whose catalog row is `Removing` or absent. + /// + /// IT NEVER CREATES A NAMESPACE, and that is the property the callers depend on rather than a + /// side-effect of how it happens to be written: for an uncataloged namespace it answers from a + /// catalog-only lookup and returns without recovering, so an `existsFile` or an + /// `unlinkFile(..., if_exists = true)` against a never-opened table cannot admit an entry into the + /// single pool-wide catalog object. See the implementation for how the guarantee survives a + /// concurrent removal. + /// + /// ONE call, not a predicate plus a resolution, and that is deliberate: readability and the life are + /// answered from the SAME `state_mutex` hold over the SAME recovered runtime, so a reader can never + /// pair "readable" from one observation with a life from another. Returning `optional` rather than a + /// life plus a bool also makes the unreadable case unusable by construction -- a caller that forgets + /// to check gets no life to read with, instead of a plausible-looking one that names the wrong + /// prefix. Absence is the fail-closed direction: only a KNOWN-readable namespace surfaces files, and + /// every failure mode of the underlying reads throws rather than degrading to `nullopt`, so "no life" + /// is only ever reached for a namespace whose absence is durable knowledge. + std::optional namespaceFilesLifeIfReadable(const RootNamespace & ns); + + /// Table-root cleanup-completeness probe: whether this logical namespace still has foreground + /// removal work outstanding, or has never proven that none remains. `true` for `Creating`, every + /// `Live` row (including zero refs and zero namespace files), and `Removing` before its terminal + /// `remove_namespace` transaction is durable; `false` for no catalog row at all, or for `Removing` + /// whose terminal is durably proven (a non-`Live` `RefLifecycle` WITH a `remove_txn_id` -- the same + /// distinction `dropNamespaceImpl` makes before returning early, because a files-only life that + /// never emitted a ref transaction can otherwise look `Removed` without ever having been removed). + /// + /// NEVER creates or mutates a catalog entry -- a probe is the wrong event to birth a namespace on -- + /// and NEVER answers `false` for an unreadable, ambiguous, or changing observation: every such case + /// throws instead, because the caller (`existsDirectory`) uses `false` as permission to physically + /// remove a directory tree. A resident runtime already proven `Live` is trusted as an O(1) fast + /// path so an ordinary warm table does not pay a `ref_catalog` fetch per probe; every other shape + /// re-reads the exact catalog row. + bool namespaceStillLogicallyPresent(const RootNamespace & ns); + + /// Called after a complete catalog observation proves an exact resident life absent or replaced. + /// It does not + /// destroy the runtime: existing callers that already captured the old physical life may finish + /// with their stale-or-not-found contract. The name slot is detached by exact pointer identity; a + /// later name-based touch may publish a distinct successor runtime. + void invalidateRemovedCatalogLife(const NamespaceLifeId & life); + + /// Reconciles resident removal-closed runtimes against one complete catalog observation. An exact + /// life absent from or replaced in the cut is invalidated and exactly detached; a matching current + /// life stays closed. No runtime is reset or rebound. + void reconcileCatalogCut(const CasRefCatalog::Snapshot & catalog_cut); + + /// Queues a mutation for flat-combining with compatible callers. `build_ops` runs at most once in + /// the flush leader and must return operations without writing storage itself. The leader validates + /// the complete batch, writes one ref-log object behind the append fence, and applies the batch to the + /// cache only after a durable result; an unresolved conditional `PUT` wedges the table and blocks + /// later appends until the same object is resolved or definitely rejected. + RefTxnId appendRefOps(const RootNamespace & ns, MutationScope scope, + std::function(const RefTableState &)> build_ops, + RootMutationOrigin origin, RootMutationKind kind, + bool skip_stale_precommit_sweep = false); + + /// Attempts one snapshot publication from a copy of the live state. The copy is made under + /// `state_mutex`, the conditional `PUT` is performed without that mutex, and counters are adopted + /// only when this attempt successfully publishes the captured snapshot. + bool tryPublishSnapshotAndAdvanceCheckpointOnce(const RootNamespace & ns); + + /// Counts tables with an unresolved append `PUT`; the walk takes each table lock briefly and never + /// waits for the network operation that caused a wedge. + size_t wedgedRefLaneCount(); + + /// Marks cached runtimes obsolete before a self-remount reopens the append fence. Leaders holding an + /// orphaned runtime therefore fail closed instead of mutating state from the previous epoch. + void quiesceRefTablesForRemount(); + + /// The self-remount's CANCEL-OR-JOIN barrier over in-flight recoveries (spec §3: "self-remount + /// cancels or waits out recovery before rearming"). Requests cancellation on every cached table, + /// then WAITS until no recovery attempt is in flight anywhere, then clears the request. Returns only + /// once that is true, so the caller may re-arm the mount fence knowing no recovery straddles it. + /// + /// Why this exists on top of the install recheck: the recheck protects the INSTALL, the barrier + /// protects the WINDOW. A recovery admitted under the outgoing incarnation would otherwise keep + /// issuing writes (its seal CAS-walk WRITES) across the whole re-arm, and each one would have to be + /// caught individually at its own site; here it is stopped once, at the boundary, before the + /// incarnation changes underneath it. + /// + /// The narrow window it does NOT close -- a recovery that starts after this returns and before the + /// fence is re-armed -- is closed by the other two members of the same guard: the re-arm bumps the + /// generation, so that recovery's `_ckpt` CAS and its install both refuse; and + /// `quiesceRefTablesForRemount` (which the caller runs next, BEFORE the re-arm) publishes + /// `superseded_by_remount`, which the walk polls at every I/O boundary. + void cancelRecoveriesAndAwaitQuiescence(); + + /// Closes admission, snapshots the current table set, and waits up to `wait_budget_ms` for queued + /// mutations and leaders to finish. The check and enqueue paths share `ref_queue_mutex`, so no new + /// mutation can appear after shutdown has taken its snapshot. + bool drainRefLanesForShutdown(uint64_t wait_budget_ms); + + /// Performs a staged conditional create through the ledger's retry controller and append-fence + /// predicate. Callers do not access either dependency directly, so every attempt observes the same + /// mount admission rule. + CasWriteOutcome stagingPutIfAbsent(std::string_view key, std::string_view bytes, Token * out_token); + + /// Performs a conditional staged create using `attempt`, applying the same retry controller and + /// append-fence policy as `stagingPutIfAbsent`. + CasCreateResult stagingConditionalCreate(std::string_view key, const std::function & attempt); + + /// Same retry/fence policy as `stagingPutIfAbsent`/`stagingConditionalCreate`, for a MUTABLE + /// If-Match overwrite whose bytes are deterministic (safe for GET-based resolution). + CasOverwriteResult stagingConditionalOverwrite(std::string_view key, std::string_view bytes, const Token & expected); + + /// Same retry/fence policy as `stagingPutIfAbsent`, for a MUTABLE marker where an existing + /// DIFFERENT value at the key is a normal Conflict outcome, not corruption (see + /// `CasRequestController::putIfAbsentControlledMutable`). + CasOverwriteResult stagingPutIfAbsentMutable(std::string_view key, std::string_view bytes); + + /// Hooks required by `EventEmitter`: events are delivered to the injected sink when one is present. + bool hasEventSink() const noexcept { return static_cast(event_sink); } + void emitEvent(CasEvent && e) const { if (event_sink) event_sink(std::move(e)); } + + /// Replaces the retry controller's delay seam for deterministic tests; production callers leave it + /// untouched. + void setCasRetrySleepForTest(std::function sleep_fn); + + /// Test-only observability and fault-injection seams for recovery, wedges, cleanup, and publication. + /// The counters expose recovery and publication progress; wedge methods create and inspect the + /// unresolved-`PUT` state; cleanup methods expose sweep eligibility; publication methods expose + /// settling, snapshot identity, and tail accounting. Every observer below is resident-only: it + /// performs no catalog/backend I/O and never materializes or recovers a runtime. + /// Returns the number of exact-read recovery restarts recorded for `ns`. + uint64_t refRecoveryRestartsForTest(const RootNamespace & ns); + /// Reports whether `ns` currently has an unresolved append `PUT`. + bool refLaneWedgedForTest(const RootNamespace & ns); + /// Returns the object key retained for the unresolved append of `ns`. + String wedgedKeyForTest(const RootNamespace & ns); + /// Returns the fence generation retained with the unresolved append of `ns` (0 when not wedged). + uint64_t wedgedAdmittedGenerationForTest(const RootNamespace & ns); + /// Installs a synthetic unresolved append for `ns` so callers can exercise resolution and blocking. + /// `admitted_generation` defaults to the CURRENT fence generation, which is what a real wedge born + /// now would carry; pass an explicit value to model a wedge admitted under an older incarnation. + void forceWedgeForTest(const RootNamespace & ns, uint64_t writer_epoch, uint64_t ref_sequence, + const String & key, const String & bytes, + std::optional admitted_generation = std::nullopt); + /// Returns the seal that closed `ns`'s previous writer epoch -- the `prev_epoch_seal` its next + /// sequence-1 append will carry (`nullopt` at genesis). See `RefTableRuntime::last_epoch_seal`. + std::optional lastEpochSealForTest(const RootNamespace & ns); + /// Installs `seal` as `ns`'s last epoch seal, standing in for the recovery CAS-walk that produces it + /// in production (Task 6). Lets a writer-side test drive the ordinary post-transition append without + /// a whole recovery. + void setLastEpochSealForTest(const RootNamespace & ns, const std::optional & seal); + /// Returns the append lane state of `ns` without forcing recovery. + RefLaneState laneStateForTest(const RootNamespace & ns); + /// Reports whether recovery or a prior incomplete sweep requires stale-precommit cleanup. + bool needsStalePrecommitSweepForTest(const RootNamespace & ns); + /// Waits until all background snapshot publications for `ns` have completed. + void waitForSnapshotPublishSettleForTest(const RootNamespace & ns); + /// Returns the number of background snapshot publications currently in flight for `ns`. + int pendingSnapshotPublishesForTest(const RootNamespace & ns); + /// Returns the newest snapshot id adopted by the cached runtime, if any. + std::optional newestPublishedSnapshotIdForTest(const RootNamespace & ns); + /// Whether `ns` currently has a RECOVERED cached runtime -- WITHOUT forcing a recovery. That is the + /// whole point: the fail-closed tests assert that a refused recovery + /// installed nothing, and a seam that recovered on demand would answer its own question. + bool refTableRecoveredForTest(const RootNamespace & ns); + /// Whether the self-remount barrier has PUBLISHED its cancellation request for `ns` (also without + /// forcing a recovery). The barrier-blocks test needs this as a handshake: it must not release the + /// parked recovery until the request is actually visible to it, or the recovery races past a flag + /// that was set a moment too late and the test observes a completion instead of a cancellation. + bool refRecoveryCancelRequestedForTest(const RootNamespace & ns); + /// Returns the number of applied transactions newer than the adopted snapshot. + size_t tailSinceSnapshotCountForTest(const RootNamespace & ns); + /// Returns the number of committed entries in the mutable overlay, when the COW representation has one. + size_t committedOverlayEntriesForTest(const RootNamespace & ns); + /// Returns this table's LIVE precommit view: the exact `{ref_name, manifest}` owner bindings that + /// `precommitAdd` creates and that `promote` (move to committed) or `abandon` (exact precommit + /// removal) take away again. A leaked binding here is the same-epoch precommit leak the stale sweep + /// -- prior-epoch-scoped -- can never reclaim, so it is what an abandon-path test must assert on. + std::set> livePrecommitsForTest(const RootNamespace & ns); + /// Installs the test hook invoked immediately before the leader carves a compatible batch. + void setRefPreCarveHookForTest(std::function hook) { ref_pre_carve_hook_for_test = std::move(hook); } + + /// Test-only fault seam for the two-phase carve/validation protocol (same `*ForTest` pattern as + /// `setRefPreCarveHookForTest`). `flushRefBatch` fires the hook at each named point of the carve's + /// plan/publish phases and of the per-item validation loop, so a test can inject `std::bad_alloc` + /// and assert the append queue and the batch-validation `working` state stay intact. `PlanSeenRefs`, + /// `PlanBatchGrow` and `PlanReserveOwned` fire in the non-mutating PLAN phase (nothing has been + /// popped yet); `PublishPop` fires at the start of the no-throw PUBLISH phase; `ValidateFinalOps` + /// fires once per admitted item, at the last throwing point before that item's effects are published + /// into `working`/`final_ops`. `ChunkReseed` fires once at each chunk boundary of a chunked flush -- + /// immediately AFTER the just-full chunk committed durably (its survivors already completed) and + /// BEFORE `working`/the trial-id high-water mark are reseeded from the now-live state; it is the + /// injection point for the tenure-exception-containment contract (a throw here, or from the reseed + /// itself, must leave the committed chunk's callers with their success). `PostDurableInstall` fires + /// inside `commitRefChunk` after that chunk's `PUT` returned `Committed` and BEFORE the prepared + /// candidate is installed into the live state -- the seam a test uses to prove that the region + /// between "durable" and "recorded" can no longer strand a transaction (a throw injected there is + /// the only way left to simulate the OLD post-durable apply failure, since the install itself is now + /// allocation-free and cannot throw). `PostInstallPreAck` fires after the candidate swap and overlay + /// materialization, outside the allocation-denied scope and state lock, but before any waiter is + /// marked done or notified. It is the deterministic acknowledgment-order seam. Null in production. + enum class CarvePhaseForTest + { + PlanSeenRefs, + PlanBatchGrow, + PlanReserveOwned, + PublishPop, + ValidateFinalOps, + ChunkReseed, + PostDurableInstall, + PostInstallPreAck, + }; + void setCarveHookForTest(std::function hook) { carve_hook_for_test = std::move(hook); } + + /// Installs the negative control for the post-durable install region (see + /// `install_region_probe_for_test`). + void setInstallRegionProbeForTest(std::function probe) { install_region_probe_for_test = std::move(probe); } + + /// Installs the pre-tenure fault seam (see `ref_pre_tenure_hook_for_test`). + void setRefPreTenureHookForTest(std::function hook) { ref_pre_tenure_hook_for_test = std::move(hook); } + + /// Pauses an ordinary append after it captured an exact runtime but before recovery or enqueue. + void setAppendAfterRuntimeCaptureHookForTest(std::function hook) + { + append_after_runtime_capture_hook_for_test = std::move(hook); + } + + /// Pauses `resolveRef` after it captured/recovered an exact runtime but before the result state lock. + void setReadBeforeStateLockHookForTest(std::function hook) + { + read_before_state_lock_hook_for_test = std::move(hook); + } + + /// Pauses a cold read after its catalog `GET` returned but before the observed life can be + /// published into the local name slot. + void setReadableCatalogAfterObservationHookForTest(std::function hook) + { + readable_catalog_after_observation_hook_for_test = std::move(hook); + } + + /// Pauses `namespaceStillLogicallyPresent`'s cold path after its FIRST catalog `GET` but before any + /// decision is made from it (including the "no row" revalidation's own second read). + void setNamespacePresenceProbeAfterFirstReadHookForTest(std::function hook) + { + namespace_presence_probe_after_first_read_hook_for_test = std::move(hook); + } + + /// Pauses `namespaceStillLogicallyPresent`'s `Removing` branch after it has proven the observed + /// incarnation's terminal (the exact-life recovery/lock section is done) but before its + /// post-terminal catalog revalidation read. + void setNamespacePresenceProbeAfterTerminalProvenHookForTest(std::function hook) + { + namespace_presence_probe_after_terminal_proven_hook_for_test = std::move(hook); + } + + /// Pauses a wedge retry after it captured the exact predecessor attempt but before the request + /// controller is allowed to send a retry. + void setWedgeBeforeSlotOccupyHookForTest(std::function hook) + { + wedge_before_slot_occupy_hook_for_test = std::move(hook); + } + + /// Counts exact recovery-result publications. + uint64_t recoveryInstallCountForTest() const + { + return recovery_install_count_for_test.load(std::memory_order_relaxed); + } + + /// Pauses a direct snapshot publisher after it captured the runtime state but before its first + /// durable effect. Used only to exercise predecessor deletion/rebirth races deterministically. + void setSnapshotAfterCaptureHookForTest(std::function hook) + { + snapshot_after_capture_hook_for_test = std::move(hook); + } + + /// Pauses a snapshot publisher after its body PUT and at the admission check immediately before + /// each `_ckpt` CAS attempt. This is intentionally inside the retrying checkpoint primitive. + void setSnapshotBeforeCkptCasHookForTest(std::function hook) + { + snapshot_before_ckpt_cas_hook_for_test = std::move(hook); + } + + /// Returns the number of queued mutations for `ns` under the queue mutex. + size_t refQueuePendingForTest(const RootNamespace & ns) + { + std::lock_guard g(ref_queue_mutex); + const auto it = ref_name_slots.find(ns.string()); + return it == ref_name_slots.end() ? 0 : it->second.current->pending.size(); + } + + /// Reports whether `ns` currently has an active append-lane leader (the baton). Under the queue mutex. + bool refLeaderActiveForTest(const RootNamespace & ns) + { + std::lock_guard g(ref_queue_mutex); + const auto it = ref_name_slots.find(ns.string()); + return it != ref_name_slots.end() && it->second.current->leader_active; + } + + /// Returns the number of callers currently waiting for `ns` recovery under its state mutex. + uint64_t refRecoveryWaitersForTest(const RootNamespace & ns) + { + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return 0; + std::lock_guard g(rt->state_mutex); + return rt->recovery_waiters_for_test; + } + + /// Returns the number of namespace runtimes currently retained in the cache. + size_t refTablesCachedCountForTest() + { + std::lock_guard g(ref_queue_mutex); + return std::count_if(ref_name_slots.begin(), ref_name_slots.end(), [](const auto & entry) + { + return static_cast(entry.second.current); + }); + } + /// Reports whether `ns` has a cached runtime whose recovery completed. + bool refTableCachedForTest(const RootNamespace & ns) + { + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return false; + std::lock_guard g(rt->state_mutex); + return rt->recovered; + } + /// Stable identity of the cached runtime object, or zero when no slot exists. This distinguishes + /// explicit life invalidation from an eviction/remount that would make a rebirth test pass by + /// constructing a different cache object. + uint64_t refTableRuntimeIdentityForTest(const RootNamespace & ns) + { + std::lock_guard g(ref_queue_mutex); + const auto it = ref_name_slots.find(ns.string()); + return it == ref_name_slots.end() || !it->second.current ? 0 : it->second.current->runtime_id; + } + uint64_t refTableRuntimeAdmittedFenceGenerationForTest(const RootNamespace & ns) + { + std::lock_guard g(ref_queue_mutex); + const auto it = ref_name_slots.find(ns.string()); + return it == ref_name_slots.end() || !it->second.current ? 0 : it->second.current->admitted_fence_generation; + } + /// The physical life currently pinned in the cached runtime, without resolving or recovering it. + std::optional refTableLifeForTest(const RootNamespace & ns) + { + std::shared_ptr rt; + { + std::lock_guard g(ref_queue_mutex); + const auto it = ref_name_slots.find(ns.string()); + if (it == ref_name_slots.end()) + return std::nullopt; + rt = it->second.current; + if (!rt) + return std::nullopt; + } + return rt->life; + } + /// Recovery-publication inventory accessors: the seeded per-table admission budgets, the recovered + /// base snapshot's encoded body size and the tail-since-snapshot byte sum. Together with + /// `newestPublishedSnapshotIdForTest`, + /// `tailSinceSnapshotCountForTest`, `needsStalePrecommitSweepForTest` and the resolved state, they + /// let a test assert EVERY `RecoveryResult` field the install seeds. + uint64_t refSnapshotBudgetForTest(const RootNamespace & ns) + { + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return 0; + std::lock_guard g(rt->state_mutex); + return rt->snapshot_budget; + } + uint64_t refRemovalBudgetForTest(const RootNamespace & ns) + { + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return 0; + std::lock_guard g(rt->state_mutex); + return rt->removal_budget; + } + uint64_t refBaseSnapshotBytesForTest(const RootNamespace & ns) + { + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return 0; + return rt->base_snapshot_bytes.load(std::memory_order_relaxed); + } + uint64_t refTailBytesSinceSnapshotForTest(const RootNamespace & ns) + { + const auto rt = lookupRefTableRuntime(ns); + if (!rt) + return 0; + return rt->tail_bytes_since_snapshot.load(std::memory_order_relaxed); + } + /// Describes the one conditional `PUT` whose outcome is still uncertain for a table. It is owned by + /// `Writing` or `Wedged` and by no other lane state; it stays installed until the object is confirmed + /// durable and applied to the cache, or definitely rejected. + /// + /// The four fields together are the attempt's IDENTITY. THREE of them are compared before any later + /// result is acted on -- `txn_id`, `bytes` and `admitted_fence_generation` (`resolveWedgeOnce`'s + /// post-I/O recheck, which calls them "all three components of the identity"). The key is not + /// compared because it adds nothing: `Layout::refLogKey` is a function of + /// `(NamespaceLifeId, RefTxnId)`, and the namespace life is fixed for the runtime that holds the + /// attempt, so within one runtime an equal `txn_id` already implies an equal key. (Should a runtime + /// ever span two incarnations of one namespace -- it does not today -- that implication is what + /// would need re-checking, not the comparison list.) + /// + /// Comparing the id alone -- or the admission generation alone -- is the aliasing bug the phase-0 + /// model found: two attempts of the same table can carry the same id under the same generation and + /// describe DIFFERENT bytes, and installing one attempt's candidate because the other's key resolved + /// is precisely the acked-then-lost class this every-attempt rule exists to close. + /// + /// Public because it is a member of `PreparedRefChunk`, which `prepareRefChunk` returns. + struct RefAppendAttempt + { + RefTxnId txn_id; + String key; + String bytes; + /// `CasMountRuntime::fenceGeneration()` as read at this transaction's ADMISSION -- the same + /// critical section that snapshotted the state and derived the id, i.e. one atomic reading of + /// "what this attempt was allowed to do". Every later `slotOccupy` retry is gated on THIS value + /// (never the current one), and every install is preceded by presenting it back through + /// `checkFenceOrThrow`: a retry admitted under a dead incarnation must send nothing, and a + /// result that returns after a fence bump/re-arm must install nothing. + uint64_t admitted_fence_generation = 0; + }; + + /// Everything `commitRefChunk` DECIDES before this chunk can have any durable effect, as one value. + struct PreparedRefChunk + { + RefTableState candidate; /// the snapshot with this chunk applied -- deliberately NOT materialized + RefTxnId candidate_base_id; /// greatest-applied of the state prepared FROM + RefLogTxn chunk_txn; /// includes INV-2's chain link + RefAppendAttempt prepared_attempt; /// COMPLETE: txn id, canonical key, sealed bytes, admitted generation + /// Set iff this chunk births the namespace. The VALUE only: `commitRefChunk` still publishes it, + /// because for a birth chunk that publish IS the first durable effect and preparation is by + /// definition everything strictly before it. + std::optional birth_contribution; + /// Published after the log object commits and before this candidate may be installed or + /// acknowledged. Every transaction advances `committed_through`; an epoch-seal transaction + /// contributes the same id as `last_epoch_seal` in this one atomic checkpoint merge. + RefCkpt commit_contribution; + }; + + /// The pure half of `commitRefChunk`: derive the candidate state, the transaction, the canonical key + /// and sealed bytes, the complete attempt, a namespace birth's pre-log `_ckpt` contribution, and + /// the post-log committed-frontier contribution -- all decided before anything can be durable. + /// + /// `static` on purpose, and it is load-bearing rather than stylistic: with no `this` no MEMBER backend + /// is reachable, and `static` is what removes the injected one -- so "backend-free" is + /// CHECKABLE instead of promised. That is what lets the protocol arithmetic be swept exhaustively + /// with no store at all: `gtest_cas_ref_chunk_preparation.cpp` names no backend and constructs none, + /// and no future edit inside this function can quietly reach for one and still compile there. + /// + /// `state` is CONSUMED: the caller snapshots `rt->state` under `state_mutex` and hands the snapshot + /// over, so the candidate IS that snapshot rather than a second copy of it (the pre-extraction code + /// made exactly one copy, under the lock, and this keeps it at one). The snapshot deliberately shares + /// the live state's COW bases and is NOT materialized here -- folding a base-sharing state would + /// rebuild the whole base, O(table) per chunk, which the install path exists to avoid. + /// + /// `id`, `chain_link` and `admitted_generation` are INPUTS, not derived here, because each traces back + /// to the SAME critical section that snapshots the state (INV-1): `id` and `admitted_generation` are + /// read inside that hold, and `chain_link` is derived just outside it from the `last_epoch_seal` read + /// inside it. Deriving the id from a different instant than the state it is applied to would be + /// deriving it from a different stream. The critical section therefore stays in `commitRefChunk`, and + /// this is a pure function of its arguments -- what that one atomic reading saw, plus `layout`, + /// `life` and `ops`. + /// + /// Every throw out of here is an ordinary PRE-durability rejection: no object exists, the cache is + /// untouched, and the id is simply never used (the next attempt re-derives it from the same unchanged + /// state). The reachable ones are a rejected apply and a failed seal; an allocation failure anywhere + /// inside is the same class, and so is `checkNamespace`'s `BAD_ARGUMENTS` on the key-building path + /// (unreachable for a mounted table, listed so the enumeration does not read closed). See the FAULT + /// CLASS note on the definition for what changed about WHO catches them. + /// + /// `life` (Stage B, Task 4-C) is `RefTableRuntime::life` -- the caller's already-resolved catalog + /// life, read under the SAME critical section as `id`/`admitted_generation` (a caller with no + /// resolved life yet has no business preparing a chunk at all: `ensureRefTableRecovered` resolves + /// it before this table is exposed as recovered). + static PreparedRefChunk prepareRefChunk(const Layout & layout, const NamespaceLifeId & life, + RefTableState state, const RefTxnId & id, + const std::optional & chain_link, + std::span ops, uint64_t admitted_generation); + +private: + /// Injected storage and mount environment. The member order is part of construction/destruction + /// behavior because the callbacks and references are used by the runtime owned below. + Backend & backend; + const Layout & layout; + RefLedgerConfig config; + const CasEventSink & event_sink; + CasRequestBudget cas_request_budget; + /// This mount's `server_root_id`; see the constructor parameter's doc for why it is a plain + /// member rather than an injected callback. + String server_root_id; + std::function live_epoch_fn; + std::function fence_ok_fn; + std::function fence_generation_fn; + std::function check_fence_or_throw; + std::function boot_ms_now_fn; + std::function may_mutate; + std::function &)> on_impossible_interference; + std::function()> pin_owner; + std::function cancel_inflight_builds; + + /// Backoff sleep used by `ensureRefTableRecovered`'s transient-retry loop. Default is an + /// interruptible slice-sleep (bails early if `fence_ok_fn` drops, e.g. on shutdown/lease loss); + /// `setCasRetrySleepForTest` overrides it (a unit test injects a clock-advancing no-op). + std::function recovery_retry_sleep_fn; + + /// One queued append caller. `build_ops` is invoked at most once by the flush leader and returns the + /// caller's operations rather than mutating storage directly. Completion fields are synchronized by + /// `ref_queue_mutex`. + struct RefMutationItem + { + MutationScope scope; + std::function(const RefTableState &)> build_ops; + RootMutationOrigin origin = RootMutationOrigin::Writer; + RootMutationKind kind = RootMutationKind::Publish; + /// Capability carried only by `dropNamespaceImpl` after it closed the exact runtime's positive + /// lane. `RootMutationKind::DropNamespace` is descriptive metadata, not removal authority: the + /// public generic append surface accepts that enum and must not thereby gain terminal rights. + bool terminal_removal_authorized = false; + bool done = false; /// guarded by ref_queue_mutex + std::exception_ptr error; /// guarded by ref_queue_mutex + RefTxnId committed_id{}; /// written by the leader before done = true + }; + + /// One coherent decoded `RefTableState` and append runtime for a namespace. It is recovered lazily + /// and evicted only as a whole. `state_mutex` is separate from + /// `ref_queue_mutex` (which only ever guards `pending`/`leader_active`) so a reader (resolveRef/ + /// listRefs) can observe `state` without contending with the flush leader's network round trip -- + /// the leader only holds `state_mutex` for the brief copy-out-before-validate and the + /// apply-after-commit steps, never for the `putIfAbsentControlled` call itself. + struct RefTableRuntime + { + /// An allocator can reuse an evicted predecessor's address for its successor. This monotone id + /// is therefore the only diagnostic identity used to test exact detach/rebirth. + const uint64_t runtime_id; + + /// Exact catalog identity accepted before publication. It is part of the runtime key and can + /// never be rebound; a same-name successor is a different runtime object. + const NamespaceLifeId life; + + RefTableRuntime(uint64_t runtime_id_, NamespaceLifeId life_, uint64_t admitted_fence_generation_) + : runtime_id(runtime_id_) + , life(std::move(life_)) + , admitted_fence_generation(admitted_fence_generation_) + { + } + + /// A later re-arm cannot retarget this runtime; it creates a distinct object instead. + const uint64_t admitted_fence_generation; + + std::mutex state_mutex; + bool recovered = false; + /// Gates the recovery-seal I/O that runs outside `state_mutex`. A second caller waits on + /// `recovery_cv` and rechecks `recovered` after the first caller finishes; otherwise it could + /// perform a competing LIST/replay/seal and misclassify the losing conditional `PUT` as failure. + /// Both this flag and the condition variable are guarded by `state_mutex`. + bool recovery_in_progress = false; + std::condition_variable recovery_cv; + /// The self-remount cancellation request (spec §3: "self-remount cancels or waits out recovery + /// before rearming"). Set by `cancelRecoveriesAndAwaitQuiescence` from the remount thread and + /// polled by the recovery walk at EVERY I/O boundary; a recovery that observes it abandons its + /// attempt having written nothing and installed nothing. + /// + /// ATOMIC, not `state_mutex`-guarded like its two neighbours, and that is the point: the + /// canceller must be able to publish the request WITHOUT queueing behind the very recovery it is + /// trying to stop. The condition variable is still the acknowledgment channel -- the canceller + /// waits for `recovery_in_progress` to fall under the mutex -- so the request is lock-free and + /// only the JOIN takes the lock. + std::atomic recovery_cancel_requested{false}; + /// Test-only count of callers currently waiting for recovery; guarded by `state_mutex` so tests + /// can observe that a concurrent caller reached the wait without depending on scheduling. + uint64_t recovery_waiters_for_test = 0; + RefTableState state; + /// Exact attempt owned by `Writing` or `Wedged`, and retained by `NeedsRecovery` while an + /// otherwise-admitted writer recovery must still adjudicate the precise durable successor it + /// expected. It is cleared only when recovery installs its result or the attempt reaches a + /// conclusive terminal outcome; losing these bytes would turn a foreign replacement into an + /// indistinguishable ordinary recovery transaction. + std::optional append_attempt; + RefLaneState lane_state = RefLaneState::Ready; + /// The `EpochSeal` transaction that closed this namespace's PREVIOUS writer epoch -- exactly the + /// `prev_epoch_seal` that this table's next sequence-1 transaction must carry (INV-2's grammar: + /// required on sequence 1 of every epoch above the namespace's genesis, forbidden everywhere + /// else). Guarded by `state_mutex`, and read in the SAME hold that derives the id, so the field + /// and the sequence number it qualifies are one reading. + /// + /// `nullopt` means GENESIS, and it means it exactly: the namespace's recovered state contains no + /// seal and its `greatest_applied.writer_epoch` is its own `life_epoch`, so its first + /// transaction opens the stream rather than continuing one across a transition. A namespace born + /// at global epoch 5 therefore appends `{5, 1}` with NO `prev_epoch_seal`; its first transition + /// (5 -> 6) seals `{5, T+1}`, and the `{6, 1}` that follows carries that seal. + /// + /// THREE producers set it, and only one of them is the one `commitRefChunk` normally reads from. + /// Recovery's CAS-walk installs the last seal of the chain it walked (Task 6): that is the + /// production path, because a real epoch change arrives with a self-remount, which DISCARDS every + /// cached runtime (`quiesceRefTablesForRemount`) and hands the fresh one its chain link through + /// recovery. The other two are the conclusive-rejection arms -- `resolveWedgeOnce`'s and + /// `commitRefChunk`'s -- which record a seal this runtime observed with its own eyes at a key it + /// owns. Within THIS runtime that record is mostly introspection (the seal it saw closes the + /// epoch it is still living in, so the stamp guard correctly suppresses it); it is written anyway + /// because it is durable evidence nothing else holds, and because it becomes the right answer the + /// moment the live epoch advances past the seal's. Nothing else writes it: a seal is durable + /// evidence, never a local guess. + std::optional last_epoch_seal; + uint64_t recovery_restarts = 0; /// diagnostic: exact-GET restarts forced by a vanished object + /// Per-table admission budgets: raw configured limits minus the table's `4 + ns.size()` wire + /// overhead and the fixed safety margin, computed once at recovery. + uint64_t snapshot_budget = 0; + uint64_t removal_budget = 0; + + /// Number and encoded-byte sum of applied transactions strictly newer than `newest_snapshot_id`. + /// The live `state` is the next snapshot candidate, so no separate tail replay is retained. + /// These counters are atomic because the cache-budget pass reads them while holding + /// `ref_queue_mutex`, whereas append and publication paths hold `state_mutex`; relaxed ordering + /// is sufficient because the counters do not publish any other state. + std::atomic tail_count_since_snapshot{0}; + std::atomic tail_bytes_since_snapshot{0}; + std::optional newest_snapshot_id; + /// Whole-table cache-weight bookkeeping for `enforceRefTableCacheBudget`. + /// `base_snapshot_bytes` is the encoded body size of the snapshot + /// at `newest_snapshot_id` (0 for a never-published table), captured for free from the + /// recovered/published snapshot body -- refreshed only when that snapshot changes (recovery + + /// each publish), never per mutation. The estimated resident weight is + /// `base_snapshot_bytes + tail_bytes_since_snapshot`. `base_snapshot_bytes` is ATOMIC (relaxed) + /// for the same cross-lock `total`-loop read as `tail_bytes_since_snapshot` above. `last_touch_tick` + /// is the monotonic access stamp (`Pool::ref_table_access_tick`) used to evict least-recently- + /// touched tables first; it is read only in the `use_count()==1`-gated candidate loop (no + /// concurrent writer there), so it stays a plain `uint64_t`. + std::atomic base_snapshot_bytes{0}; + uint64_t last_touch_tick = 0; + /// Set true by recovery; cleared when a sweep attempt is + /// dispatched (so the sweep's own nested `appendRefOps` calls do not recurse) and PERMANENTLY + /// only once an attempt completes VERIFIED CLEAN (a full pass over the live state found zero + /// stale bindings). Any failed or partial attempt re-arms it (with the + /// `precommit_sweep_backoff_*` cooldown), so a later read/mutation trigger retries until clean -- + /// a single attempt burned in the post-restart error window must not leave a dead incarnation's + /// precommit bindings protected from GC forever on a long-lived mount. + bool needs_stale_precommit_sweep = false; + /// Per-table retry cooldown for the stale-precommit sweep (guarded by `state_mutex`), + /// mirroring `publish_backoff_*` below: `until` is the boottime instant before which + /// `maybeSweepStalePrecommits` refuses to re-attempt; `ms` is the current exponential interval + /// (0 = no failure yet, or reset by the last verified-clean sweep). + uint64_t precommit_sweep_backoff_until_ms = 0; + uint64_t precommit_sweep_backoff_ms = 0; + /// Test-observability + graceful settling for the background snapshot-publish dispatch (see + /// `maybeScheduleSnapshotPublish`): the count of in-flight publish attempts for this table, and + /// the condvar (guarded by `state_mutex`) a test waits on via `waitForSnapshotPublishSettleForTest`. + std::atomic pending_snapshot_publishes{0}; + std::condition_variable publish_settle_cv; + /// Per-table snapshot-publish dispatch backoff (guarded by + /// `state_mutex`). `publish_backoff_until_ms` is the boottime instant before which + /// `maybeScheduleSnapshotPublish` refuses to dispatch; `publish_backoff_ms` is the current + /// exponential interval (0 = not backing off), doubled on each consecutive non-Committed publish + /// outcome and reset to 0 on the next durable publish. + uint64_t publish_backoff_until_ms = 0; + uint64_t publish_backoff_ms = 0; + + std::deque> pending; /// guarded by ref_queue_mutex + bool leader_active = false; /// guarded by ref_queue_mutex + /// Set before the exact `Live -> Removing` catalog CAS and retained until that life is deleted. + /// New positive mutations check it in the same queue critical section as admission; the one + /// terminal `DropNamespace` item is the sole exception. + bool removal_admission_closed = false; + std::condition_variable cv; + + /// Published after GC commits the exact catalog deletion. The exact cache pointer is detached; + /// old holders observe the flag and remain a predecessor, never a rebindable name handle. + std::atomic catalog_life_invalidated{false}; + + /// Set true when a self-remount detaches this runtime from the cache (`quiesceRefTablesForRemount`): + /// the fresh incarnation re-recovers each table under the new epoch on next touch, so any leader + /// still holding THIS (now-orphaned) runtime must fail closed instead of allocating an id / applying + /// against its stale cache once the re-armed fence re-opens the gate. Stored with release BEFORE the + /// remount re-arms the fence, so a lane that observes `mayMutate` true also observes this flag + /// (release/acquire through the fence) -- there is no interleaving where a stale runtime both passes + /// the fence and reads this flag false. + std::atomic superseded_by_remount{false}; + }; + + /// Appends against an exact runtime already captured by a lifecycle operation. This is the sole + /// path used by namespace removal after its exact `Live -> Removing` catalog transition: resolving + /// the logical name again there would either refuse the required terminal append or, after a + /// replacement, retarget destructive work to the successor. Ordinary mutations enter through the + /// public name-based `appendRefOps` wrapper and can never select this path. + RefTxnId appendRefOpsOnRuntime( + const RootNamespace & ns, const std::shared_ptr & rt, MutationScope scope, + std::function(const RefTableState &)> build_ops, + RootMutationOrigin origin, RootMutationKind kind, bool skip_stale_precommit_sweep, + bool terminal_removal_authorized); + + /// Publishes only from the exact runtime captured by the caller. Background dispatch carries this + /// pointer across the thread hand-off; it must never resolve the logical name again and accidentally + /// publish a same-name successor while settling the predecessor's in-flight accounting. + bool tryPublishSnapshotAndAdvanceCheckpointOnceOnRuntime( + const RootNamespace & ns, const std::shared_ptr & rt); + + /// The logical-name cache owns no lifecycle identity. It merely points at the runtime currently + /// admitted for that name; exact retirement/remount clears this pointer while external holders may + /// continue using the detached predecessor and fail closed against its immutable identity. + struct RefNameSlot + { + std::shared_ptr current; + }; + static constexpr size_t kMaxRefBatch = 1000; + /// Recovery retries at most this many times when an object selected by LIST vanishes before GET. + /// A failed recovery-seal `PUT` is separate: it leaves `recovered` false and the next touch starts + /// a fresh LIST/replay/seal attempt rather than resuming this bounded vanish-retry loop. + static constexpr size_t kRefRecoveryMaxRestarts = 3; + /// How many times the CAS-walk may lose the SAME dead epoch's seal slot to a straggler before it + /// fails closed. Not an arbitrary round number: INV-1's every-attempt rule permits AT MOST ONE + /// in-flight conditional create per (table, writer), and there is one writer per mount, so an honest + /// run needs ONE retry. The margin covers a dying writer whose lane held several attempts across + /// distinct incarnations; anything beyond it is a store that keeps materializing objects underneath + /// a walk, which is a fact to report, never one to keep looping on. + static constexpr size_t kRefRecoveryMaxSlotAttemptsPerEpoch = 64; + /// Fixed safety margin subtracted (alongside the per-table `4 + ns.size()` overhead) from + /// the raw `ref_snapshot_max_bytes`/`ref_removal_max_bytes` hard limits before calling `admits`. + static constexpr uint64_t kRefAdmissionSafetyMargin = 4096; + + /// `mutable` for `confirmExactRef`, the one CONST member function that needs the lane snapshot: + /// taking a mutex to read consistently does not make the read a mutation. + mutable std::mutex ref_queue_mutex; + std::map ref_name_slots; + /// Monotonic access stamp for whole-table cache LRU eviction, bumped on every table touch and + /// recorded in `RefTableRuntime::last_touch_tick`. + std::atomic ref_table_access_tick{0}; + std::atomic next_ref_runtime_id{0}; + std::atomic recovery_install_count_for_test{0}; + /// Latched by `drainRefLanesForShutdown` before it + /// snapshots `ref_name_slots`/waits on each table's queue -- every ordinary ref mutation (`appendRefOps`) + /// checks this under the SAME `ref_queue_mutex` critical section it uses to enqueue its item, so the + /// check-and-enqueue is atomic with the drain's snapshot-and-wait: a caller either enqueues strictly + /// before the drain observes this table (and the drain then waits for it), or observes this flag + /// already true and never enqueues at all. No caller can land a NEW item after the drain has decided + /// this table is idle. + std::atomic shutting_down{false}; + + /// The id `rt`'s next transaction carries (INV-1): `RefTableState::nextTxnId` of the table's OWN + /// state under the live writer epoch. There is no counter behind this -- the id is a pure function + /// of the state the transaction will be applied to, which buys two properties a pool-wide counter + /// could not: + /// - each namespace's ids are dense `1..T` within one epoch, so a reader holding a table's log + /// ids can tell a COMPLETE stream from a truncated one without consulting anything else; + /// - an attempt that provably sent nothing consumes nothing: the state it derived from is + /// unchanged, so the next caller derives the SAME id and no hole is left behind. + /// + /// A post-durable install failure moves the lane to `NeedsRecovery`, so this function is not called + /// again until replay has installed that durable transaction and advanced `greatest_applied`. + /// + /// The epoch component is the live mount incarnation's writer epoch, not the open-time + /// `process_epoch`: a self-remount allocates a strictly-greater durable writer_epoch, so every ref + /// transaction stamped after the remount sorts strictly ABOVE any (dead-incarnation or twin) log + /// still durable under an older epoch. `RefTxnId` compares epoch first, so the epoch bump alone + /// guarantees that a new log is never inserted at or below an already durable table log id. + /// + /// MUST be called with `rt.state_mutex` held, and the caller must apply the transaction to the SAME + /// state it read here: an id derived from one snapshot of a table and applied to another is not that + /// table's successor, and the apply-side density check would (correctly) reject it. + RefTxnId allocateRefTxnId(const RefTableRuntime & rt) const + { + return rt.state.nextTxnId(live_epoch_fn()); + } + + /// The CAS-owned retry controller this Pool's ref-log writer path uses for every + /// conditional log/snapshot `PUT` and uncertain-result resolution. Also shared (via the PartWriteTxn + /// `PartWriteTxn::stageManifest`'s part-manifest body `PUT` and by `PartWriteTxn::uploadFromSource`'s + /// blob-body create — both the streaming + /// `putIfAbsentStream` PUT and `promoteStaged`'s conditional server-side copy — via + /// `conditionalCreateControlled`. The controller is stateless per call (immutable + /// budget/clock/sleep — the sleep fn mutates only through the test-only seam, before traffic), so + /// concurrent lanes and builds use the one instance safely. + std::unique_ptr ref_request_controller; + + /// Test-only hook called before a compatible append batch is carved; null in production. + std::function ref_pre_carve_hook_for_test; + + /// Test-only fault seam fired on the CALLING thread at the instant a queue caller takes append-lane + /// leadership -- i.e. at the FIRST allocation that builds the leader's responsibility set, the last + /// throwing point before the baton (`leader_active`) is published. A throw here must leave the lane + /// idle (baton un-taken, the caller's item un-enqueued), never a permanently non-idle namespace with + /// no live leader. Null in production. See `appendRefOps`. + std::function ref_pre_tenure_hook_for_test; + + /// Test-only hook fired at each carve/validation phase point (see `CarvePhaseForTest`); null in + /// production. + std::function carve_hook_for_test; + + /// Test-only probe fired inside either post-durable state install, under + /// `DENY_ALLOCATIONS_IN_SCOPE`: the ordinary committed install and wedge-resolution adoption. The + /// exact attempt is installed before sending, so `Unresolved` needs no post-I/O object install. It is the negative + /// control for the guard: a probe that allocates must abort a debug build, which is what proves the + /// region is armed and actually entered -- so a future edit that adds an allocating statement there + /// cannot pass unnoticed. It is also the only way left to reach `NeedsRecovery` from one of these + /// otherwise non-throwing regions. A test that installs a throwing probe must therefore disarm it + /// after the region it targets, or every later install throws too. Null in production. + std::function install_region_probe_for_test; + std::function append_after_runtime_capture_hook_for_test; + std::function read_before_state_lock_hook_for_test; + std::function readable_catalog_after_observation_hook_for_test; + std::function namespace_presence_probe_after_first_read_hook_for_test; + std::function namespace_presence_probe_after_terminal_proven_hook_for_test; + std::function wedge_before_slot_occupy_hook_for_test; + std::function snapshot_after_capture_hook_for_test; + std::function snapshot_before_ckpt_cas_hook_for_test; + + /// Non-materializing diagnostic/cache lookup. It never observes the catalog and never creates a + /// name slot or runtime. + std::shared_ptr lookupRefTableRuntime(const RootNamespace & ns) const; + + /// Publishes or returns one runtime for an exact catalog-observed life and admitted mount-fence + /// generation. A conflicting attached identity is a stale observation and fails closed rather than + /// retargeting either runtime. + std::shared_ptr acquireRefTableRuntime( + const NamespaceLifeId & life, uint64_t admitted_generation); + + /// Lookup-first non-minting read acquisition. A cold name consults the catalog and materializes only + /// an exact `Live` life; absence/`Creating`/`Removing` returns no runtime. + std::shared_ptr acquireReadableRefTableRuntime(const RootNamespace & ns); + + /// Lookup-first mutation acquisition. A cold name resolves or births the catalog life before + /// constructing its runtime. + std::shared_ptr acquireMutableRefTableRuntime(const RootNamespace & ns); + + /// Common removal implementation. `expected_incarnation` is present for the decommission-only + /// exact-life overload and is checked before every lifecycle branch, including stalled creation. + DropNamespaceStats dropNamespaceImpl( + const RootNamespace & ns, const std::optional & expected_incarnation); + + /// Lazily recovers `ns` per spec §4: catalog life resolution (first table-open only) -> `_ckpt` -> + /// exact-key base snapshot -> ARITHMETIC tail -> seal CAS-walk -> `_ckpt` CAS -> install. It does not + /// expose the table as recovered until every dead epoch it discovered is durably closed; concurrent + /// callers serialize across the whole unlocked I/O window through `recovery_in_progress`. + void ensureRefTableRecovered(const RootNamespace & ns, RefTableRuntime & rt); + + /// Stage B (spec INV-3, §3): resolves `ns`'s catalog life -- ONCE per table-open, from inside + /// `ensureRefTableRecovered`'s transient-retry envelope, never from a per-write path (a per-write + /// catalog GET is a protocol-step addition and is vetoed). Three cases, closing over the catalog's + /// own three-state grammar: + /// - no entry at all: this call is the namespace's first-ever opener. Mints one via + /// `CasRefCatalog::createNamespace` under a `CreatorFence` built from `server_root_id`, + /// `live_epoch` and `admitted_generation`; + /// - an entry already `Live`/`Removing`: adopts its incarnation directly + /// (`NamespaceLifeId::fromCatalogEntry`) -- `Removing` is adopted exactly like `Live` because + /// this call only needs A life to key objects with; refusing WRITES to a `Removing` namespace + /// is a different mechanism's job (Task 6's read-side contract), not this resolution's; + /// - an entry `Creating`: if its `creator` fence is THIS mount's own (a previous attempt of this + /// same open landed step 1 but not steps 2/3, e.g. after a transient error), resumes + /// `completeCreation` directly over the observed entry; otherwise reconciles it via + /// `CasRefCatalog::reconcileStaleCreator` + `isCreatorFenceTerminal`, refusing retry-later while + /// the old creator's fence is not yet provably dead. + /// Every `createNamespace`/`completeCreation`/`reconcileStaleCreator` outcome that writes nothing + /// (`FencedOut`, `Superseded`, a reconciled entry, `EntryChanged`) re-reads the catalog and loops; + /// `CreatorFenceStillLive` throws the retry-later class, which this function's caller (the transient + /// retry loop) or a higher one re-drives. Bounded against a pathological duel between two openers; + /// each primitive this loop calls has its OWN bounded retry against the catalog's single object, so + /// this bound is only against THIS loop's re-read cycle. + NamespaceLifeId resolveNamespaceLife( + const RootNamespace & ns, uint64_t admitted_generation, uint64_t live_epoch, + bool * lifecycle_refusal = nullptr); + + /// ONE attempt of the recovery walk, run with NO lock held (the candidate is private; nothing + /// touches `rt` until the install). `nullopt` REQUESTS A RESTART from a fresh listing -- the two + /// innocent explanations, a base that vanished under a checkpoint that moved and a hole a racing + /// cleanup could account for. Everything terminal throws. + /// + /// `admitted_generation` is the ONE fence generation this whole recovery was admitted under: the + /// walk presents it to every `slotOccupy` and to the `_ckpt` CAS, and the caller presents the same + /// value once more immediately before installing. + /// `retained_attempt` is copied under `state_mutex` before the unlocked walk. It is evidence from + /// this runtime's admitted writer, not a second recovery authority: only the exact slot it names + /// is compared byte-for-byte, and a successor seal remains the existing conclusive-loss case. + /// `cancelled` is the CALLER's latch, threaded in rather than kept locally: a cancellation is + /// reported through the retry-later class (the caller should retry, against the FRESH incarnation), + /// so without it the transient loop reads the stop as a blip and re-drives the very work the remount + /// just stopped -- while the barrier blocks waiting for that recovery to finish. + std::optional runRecoveryWalkOnce( + const RootNamespace & ns, RefTableRuntime & rt, uint64_t admitted_generation, uint64_t live_epoch, + const std::optional & retained_attempt, std::optional & hole_detail, + bool & cancelled); + + /// The I/O-boundary poll of the walk: is this recovery still entitled to continue? Two independent + /// facts, each of which alone disqualifies it -- a self-remount asked it to stop, or a self-remount + /// already detached its runtime. Throws; cancellation raises the retry-later class and LATCHES + /// through `cancelled` so the caller's transient loop does not re-drive it. + /// + /// The FENCE is deliberately absent: it gates the three sites that spend it (every `slotOccupy`, the + /// `_ckpt` CAS, the install), not every read. See the definition for why. + void checkRecoveryStillAdmitted(const RootNamespace & ns, RefTableRuntime & rt, bool & cancelled) const; + + /// Publishes a completed streaming recovery into the runtime in one atomic step: copies EVERY field + /// `RecoveryResult` carries into `rt` and sets `recovered` LAST, so a waiter woken after this returns + /// never observes a partially-installed table. Struct-driven so a future added publication field + /// cannot be silently dropped from a scattered assignment list (spec §5). MUST hold `rt.state_mutex`. + void installRecoveryResult(RefTableRuntime & rt, RecoveryResult && result); + + /// Evicts least-recently-touched, idle whole-table runtimes until the configured cache budget is met, + /// retaining `keep_ns` even when it is the next candidate. + void enforceRefTableCacheBudget(const RootNamespace & keep_ns); + + /// Runs the append queue leader for `ns`, completing its own item and any compatible items carved + /// into the same batch. Exceptions are stored for waiters and do not leave the leader flag latched. + /// Every item this leader becomes responsible for -- its own `own` plus each item a flush removes + /// from `pending` to form a batch -- is recorded into `owned_items`, so the caller's leadership guard + /// can complete + de-pend any that a flush leaves unfinished on an exceptional exit. + void runRefQueueLeader(const RootNamespace & ns, const std::shared_ptr & rt, + const std::shared_ptr & own, + std::vector> & owned_items); + + /// Validates, durably appends, and applies one compatible batch while preserving copy-before-commit + /// and apply-after-commit ordering -- the LIVE state is still only ever advanced once the object is + /// durable; `commitRefChunk`'s pre-`PUT` apply targets a private candidate that nothing else can + /// observe. Every item it carves out of `pending` is appended to + /// `owned_items` (the leader's responsibility set) at the moment it is carved. When a batch's total + /// op count exceeds `ref_txn_max_ops`, the validation loop emits SEVERAL ref-log transactions in one + /// tenure via `commitRefChunk` -- each a complete commit boundary. + void flushRefBatch(const RootNamespace & ns, const std::shared_ptr & rt, + std::vector> & owned_items); + + enum class WedgeResolution : uint8_t + { + NoWedge, + Adopted, + Rejected, + StillWedged, + Corrupted, + }; + + struct WedgeResolutionResult + { + WedgeResolution kind = WedgeResolution::NoWedge; + std::exception_ptr survivor_error; + }; + + /// ONE bounded resolution attempt for `rt`'s outstanding wedge (spec INV-1's every-attempt rule): + /// at most one `slotOccupy(wedge.key, wedge.bytes, ...)` per calling flush, gated on the wedge's + /// ORIGINAL `admitted_fence_generation` rather than the current one. There is deliberately NO + /// background retry thread and no deadline-resetting loop: a permanently quiet wedged namespace + /// waits for its next caller or for a remount, which is acceptable precisely because the wedged + /// operation was never acknowledged. + /// + /// The conditional CREATE is what makes the rule "every attempt has its own conclusive rejection" + /// affordable: the ref-log key is write-once, so a create either lands our exact bytes (the + /// transaction is durable -- and it is the SAME transaction, byte for byte) or conflicts with + /// whatever is there, which the follow-up read then names. A read alone could only ever report + /// "absent", which is not a rejection: the earlier ambiguous attempt could still land afterwards. + /// + /// Post-I/O recheck: the outcome is adjudicated on an I/O result, so before ANY action follows from + /// it (adopt, acknowledge, unwedge, fail the survivors) this re-acquires `state_mutex`, presents + /// `admitted_fence_generation` back through `checkFenceOrThrow`, and compares the full wedge + /// identity against what is still installed. A result that returns after a fence bump/re-arm, or + /// after the wedge it belonged to was replaced, is INERT for this runtime. + WedgeResolutionResult resolveWedgeOnce( + const RootNamespace & ns, const std::shared_ptr & rt); + + /// Commits ONE chunk of a `flushRefBatch` tenure as a complete ref-log transaction: allocates the + /// real transaction id, PREPARES the chunk (`prepareRefChunk` -- candidate, transaction, key, sealed + /// bytes, complete attempt, birth contribution), durably `PUT`s the sealed bytes, installs the + /// candidate under `state_mutex` by a no-throw swap, advances the tail counters, records the + /// per-transaction metrics, completes exactly `chunk_survivors` with the real id (waking their + /// waiters), and schedules snapshot publication. Preparation completes BEFORE the first durable + /// effect of EITHER chunk shape -- an ordinary chunk's ref-log `PUT`, and a `NamespaceBirth` chunk's + /// earlier `_ckpt` publish -- so that nothing between "durable" and "recorded" can throw (spec §A1); + /// a preparation failure is an ordinary pre-durability rejection. For the same reason the + /// `RefAppendAttempt` is built COMPLETE before the `PUT` -- the request reads its key and body -- so + /// the `Unresolved` arm only has to move it into the runtime: the OTHER thing that must be recorded + /// once the object may be durable. Arming that attempt into the runtime is NOT preparation (it + /// mutates `RefTableRuntime`) and stays here, between preparation and the first send. + /// Returns true when the chunk committed durably; false on any non-throwing failure (a rejected + /// apply / DefiniteFailure / unresolved wedge / a conclusive PUT rejection / an encode failure), + /// after having already failed `chunk_survivors` with the appropriate error. Past the durable `PUT` + /// it does not throw at all. + /// PRECONDITION: the caller has already released its scratch `working` copy so the post-commit + /// overlay fold is in place (the E5 fast path). + bool commitRefChunk(const RootNamespace & ns, const std::shared_ptr & rt, + const std::vector & chunk_ops, + const std::vector> & chunk_survivors); + + /// Enters the hard recovery fence after a transaction is known durable but cannot be installed. + /// The exact attempt is discarded because replay, not another write, is now the only legal owner. + static void requireRecovery(RefTableRuntime & rt, const RootNamespace & ns, std::string_view region) noexcept; + + /// Leadership-exit guard for `appendRefOps`: under `ref_queue_mutex`, completes every still-unfinished + /// item this leader owned (with `flush_exception` when unwinding, or a fail-closed `LOGICAL_ERROR` + /// otherwise), removes each owned item from `pending` so no future leader can carve it, and releases + /// leadership (`leader_active = false` + `cv.notify_all`). On the normal path every owned item is + /// already `done`, so only the leadership release has effect. This is the single authority that + /// resets `leader_active` on any exit from the leader loop. + void completeOwnedItemsAndReleaseLeadership( + const RootNamespace & ns, const std::shared_ptr & rt, + const std::vector> & owned_items, + std::exception_ptr flush_exception); + + /// Schedules best-effort background publication when tail thresholds and backoff permit. The + /// dispatch is fenced and the detached task retains the owner pin until it finishes. + void maybeScheduleSnapshotPublish(const RootNamespace & ns, const std::shared_ptr & rt); + + /// Merges one contribution into `life`'s `_ckpt` (spec INV-4), presenting `admitted_generation` back + /// through the pool's fence callback on every CAS attempt. The ledger owns no `CasMountRuntime`, so + /// this is only the place that assembles the deadline from the ledger's own injectable boot clock + /// and CAS budget; the algorithm itself is `publishCkpt`, shared verbatim with every other writer. + /// + /// THREE call sites (corrected here from "two": `runRecoveryWalkOnce`'s own sealer contribution + /// below is a third and was missing from this count), and they contribute DISJOINT fields: + /// - `commitRefChunk`'s namespace-birth transaction contributes `life_epoch` -- it is the only + /// writer that knows it (this transaction's own writer epoch), and spec §3 has the `_ckpt` + /// created before the namespace becomes Live. The contribution is no longer built at the call + /// site: `prepareRefChunk` returns it as `PreparedRefChunk::birth_contribution` and + /// `commitRefChunk` passes that prepared value here. The split is deliberate -- DECIDING the + /// contribution is pure preparation, while THIS call is a birth chunk's first durable effect, so + /// the publish had to stay behind when preparation moved out; + /// - `tryPublishSnapshotAndAdvanceCheckpointOnce` contributes `checkpoint_snapshot_id` once the + /// snapshot body is durable, and contributes NOTHING about `life_epoch` (an absence, which the semantic-max + /// merge leaves alone) because the publisher does not know it and must never guess; + /// - `runRecoveryWalkOnce` contributes `last_epoch_seal` once its own CAS-walk minted or adopted + /// one -- it is the only writer that mints seals, so it is the only writer that can record + /// where the chain now ends. + CkptPublishOutcome publishCkptContribution(const NamespaceLifeId & life, const RefCkpt & contribution, + uint64_t admitted_generation, + const std::function & check_admission); + + /// Common candidate predicate for scheduler admission and execution after capture. Caller holds + /// `rt.state_mutex`; an epoch seal is not state-bearing and cannot be snapshotted. + bool hasStateBearingSnapshotCandidateUnderStateLock(const RefTableRuntime & rt) const; + + /// The Live + single-in-flight-gate + backoff + tail-threshold admission decision, factored out so + /// both the trigger (`maybeScheduleSnapshotPublish`) and the settlement re-evaluation share ONE + /// authority. The caller MUST hold `rt.state_mutex`; on admission this increments + /// `pending_snapshot_publishes` and returns true (the caller then dispatches). The fence check + /// (`may_mutate`) is the caller's responsibility (it is not held under `state_mutex`). + bool admitSnapshotPublishUnderStateLock(RefTableRuntime & rt); + + /// Launches one detached publish attempt. Assumes `pending_snapshot_publishes` was already + /// incremented for this dispatch (by `admitSnapshotPublishUnderStateLock`). The task settles through + /// `settleSnapshotPublish`; if the thread cannot even be constructed, the count is undone and the + /// settle waiter notified so a leaked in-flight count never wedges shutdown/settle. + void dispatchSnapshotPublisher(const RootNamespace & ns, const std::shared_ptr & rt); + + /// Runs at the end of one detached publish attempt: drops this attempt's in-flight count and, under + /// the SAME `state_mutex` hold, re-evaluates the accumulated tail so a trigger the single-flight gate + /// discarded during this attempt (e.g. chunks 2..N of a chunked tenure whose chunk-1 publish was + /// in flight) is re-fired instead of lost. Re-admitting under the same lock keeps the in-flight count + /// from transiently reaching zero across the handoff, so a settle waiter never observes a false + /// "settled". Notifies the settle condvar only when no follow-up publish is dispatched. + void settleSnapshotPublish(const RootNamespace & ns, const std::shared_ptr & rt); + + /// Advances the exponential delay after a non-durable snapshot publication outcome. + void advancePublishBackoff(RefTableRuntime & rt); + /// Clears the snapshot-publication delay after durable progress. + void resetPublishBackoff(RefTableRuntime & rt); + + /// Checks whether recovery or a mutation requested stale-precommit cleanup and dispatches it when + /// its per-table cooldown permits. + void maybeSweepStalePrecommits(const RootNamespace & ns, const std::shared_ptr & rt); + + /// Performs one fenced stale-precommit sweep. A partial or failed sweep re-arms the requirement and + /// propagates its exception; only a verified-clean pass clears it. + void sweepStalePrecommitsNow(const RootNamespace & ns, const std::shared_ptr & rt); + + /// Runs the read-triggered sweep without allowing an uncertain maintenance append to fail the read. + void sweepStalePrecommitsForRead(const RootNamespace & ns, const std::shared_ptr & rt); + + /// Advances the stale-precommit sweep's exponential cooldown after failure. + void advancePrecommitSweepBackoff(RefTableRuntime & rt); + /// Clears the stale-precommit sweep cooldown after a verified-clean pass. + void resetPrecommitSweepBackoff(RefTableRuntime & rt); + +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefProtocol.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefProtocol.cpp new file mode 100644 index 000000000000..36f126bad20c --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefProtocol.cpp @@ -0,0 +1,1108 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +CatalogLifeIndex::CatalogLifeIndex(const RefCatalog & catalog) +{ + for (const CatalogEntry & entry : catalog.entries) + { + const NamespaceLifePhysicalId life_id = entry.incarnation; + if (auto ambiguous_it = ambiguous_names.find(life_id); ambiguous_it != ambiguous_names.end()) + { + ambiguous_it->second.push_back(entry.ns.string()); + continue; + } + + const auto [it, inserted] = unique_lives.emplace( + life_id, NamespaceLifeId::fromCatalogEntry(entry.ns, life_id)); + if (inserted) + continue; + + std::vector names; + names.push_back(it->second.ns.string()); + names.push_back(entry.ns.string()); + unique_lives.erase(it); + ambiguous_names.emplace(life_id, std::move(names)); + } +} + +bool CatalogLifeIndex::isAmbiguous(NamespaceLifePhysicalId life_id) const +{ + return ambiguous_names.contains(life_id); +} + +std::optional CatalogLifeIndex::resolve(NamespaceLifePhysicalId life_id) const +{ + if (const auto ambiguous_it = ambiguous_names.find(life_id); ambiguous_it != ambiguous_names.end()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS ref catalog: life_id {} is shared by current namespaces '{}' and '{}' -- both rows are unresolvable", + renderIncarnation(life_id), ambiguous_it->second[0], ambiguous_it->second[1]); + if (const auto it = unique_lives.find(life_id); it != unique_lives.end()) + return it->second; + return std::nullopt; +} + +void CatalogLifeIndex::throwIfAmbiguous(std::string_view consumer) const +{ + if (ambiguous_names.empty()) + return; + const auto & [life_id, names] = *ambiguous_names.begin(); + throw Exception(ErrorCodes::CORRUPTED_DATA, + "{}: catalog life_id {} is shared by current namespaces '{}' and '{}' -- refusing a decision from an ambiguous cut", + consumer, renderIncarnation(life_id), names[0], names[1]); +} + +namespace +{ + +/// Txn-wide structural check, NOT a per-op precondition: if `ops` contains a +/// `RemoveNamespace`, it must be the last element, and every earlier op must be an exact +/// owner-removal `owner_transition` (`old_binding` set, `new_binding` empty). The sole other legal +/// form is `[NamespaceBirth, RemoveNamespace]`: a cataloged life may own `_ckpt`/`_files` without ever +/// having emitted a ref transaction, and its empty birth+terminal must be one durable removal record. +/// `CasRefLogCodec` deliberately does not check this shape -- this is the one place that does. +void checkRemoveNamespaceOrdering(const std::vector & ops) +{ + const bool has_remove = std::any_of(ops.begin(), ops.end(), + [](const RefOp & op) { return op.kind == RefOpKind::RemoveNamespace; }); + if (!has_remove) + return; + + if (ops.empty() || ops.back().kind != RefOpKind::RemoveNamespace) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: remove_namespace must be the final operation of its transaction"); + + if (ops.size() == 2 && ops.front().kind == RefOpKind::NamespaceBirth) + return; + + for (size_t i = 0; i + 1 < ops.size(); ++i) + { + const RefOp & op = ops[i]; + const bool pure_removal = op.kind == RefOpKind::OwnerTransition + && op.old_binding.has_value() && !op.new_binding.has_value(); + if (!pure_removal) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: every operation before remove_namespace must be an exact owner removal"); + } +} + +/// Installed test probe for the streaming-recovery memory invariant (see the header). Guarded by its +/// own mutex so an install/clear from a test thread cannot tear against a concurrent recovery. +std::mutex g_recovery_replay_memory_probe_mutex; +std::function g_recovery_replay_memory_probe; + +/// The four legal `owner_transition` shapes, decided purely from the (old_binding, new_binding) +/// optionals and their `RefOwnerKind`s -- no state read. `RefTableState::applyOwnerTransition` (the +/// writer/replay state machine) and `manifestEdgesOfTxn` (the GC fold's edge extractor) both switch +/// over this single classification instead of each carrying their own shape predicates, so a shape +/// neither consumer recognizes cannot silently acquire divergent meaning in one of them. +enum class OwnerTransitionShape : uint8_t +{ + AddPrecommit, /// no old_binding, new_binding.kind == Precommit + RemovePrecommit, /// old_binding.kind == Precommit, no new_binding + RemoveCommitted, /// old_binding.kind == Committed, no new_binding + Promote, /// old_binding.kind == Precommit, new_binding.kind == Committed, SAME ref_name + /// and manifest_ref +}; + +/// Classify `op`'s (old_binding, new_binding) shape into one of the four legal transitions. Anything +/// else -- neither binding, old+new naming DIFFERENT manifests, a promote whose old/new ref_name +/// disagree, or any other kind combination -- throws `CORRUPTED_DATA` naming the offending combination +/// instead of falling through to a caller that would otherwise assign it accidental meaning. +[[nodiscard]] OwnerTransitionShape classifyOwnerTransitionShape(const RefOp & op) +{ + const bool has_old = op.old_binding.has_value(); + const bool has_new = op.new_binding.has_value(); + + if (!has_old && has_new && op.new_binding->kind == RefOwnerKind::Precommit) + return OwnerTransitionShape::AddPrecommit; + + if (has_old && !has_new && op.old_binding->kind == RefOwnerKind::Precommit) + return OwnerTransitionShape::RemovePrecommit; + + if (has_old && !has_new && op.old_binding->kind == RefOwnerKind::Committed) + return OwnerTransitionShape::RemoveCommitted; + + if (has_old && has_new && op.old_binding->kind == RefOwnerKind::Precommit + && op.new_binding->kind == RefOwnerKind::Committed + && op.old_binding->ref_name == op.new_binding->ref_name + && op.old_binding->manifest_ref == op.new_binding->manifest_ref) + return OwnerTransitionShape::Promote; + + throw Exception(ErrorCodes::CORRUPTED_DATA, + "owner_transition does not match any legal transition shape (has_old={}, old_kind={}, " + "has_new={}, new_kind={})", + has_old, has_old ? std::to_string(static_cast(op.old_binding->kind)) : "n/a", + has_new, has_new ? std::to_string(static_cast(op.new_binding->kind)) : "n/a"); +} + +} + +void setRecoveryReplayMemoryProbeForTest(std::function probe) +{ + std::lock_guard lock(g_recovery_replay_memory_probe_mutex); + g_recovery_replay_memory_probe = std::move(probe); +} + +void reportReplayMemoryDelta(int64_t delta_footprint_bytes) +{ + /// Reads the installed probe under the mutex and calls it OUTSIDE the lock so the probe body may + /// itself do arbitrary work. A no-op in production (no probe installed). + std::function probe; + { + std::lock_guard lock(g_recovery_replay_memory_probe_mutex); + probe = g_recovery_replay_memory_probe; + } + if (probe) + probe(delta_footprint_bytes); +} + +uint64_t decodedRefLogTxnFootprint(const RefLogTxn & txn) +{ + /// A deterministic proxy for the heap a decoded transaction keeps alive: the ns string, the op + /// vector's element storage (ops count x per-op record size), and every owned ref-name string. Uses + /// `size()` (not `capacity()`) so the value depends only on the decoded content, hence is identical + /// across a streaming decode and a materialising control's decode of the same object. + uint64_t bytes = txn.ns.size() + txn.ops.size() * sizeof(RefOp); + for (const RefOp & op : txn.ops) + { + bytes += op.ref_name.size(); + if (op.old_binding) + bytes += op.old_binding->ref_name.size(); + if (op.new_binding) + bytes += op.new_binding->ref_name.size(); + } + return bytes; +} + +/// Member-wise swap; see the header for the install-region contract it exists for. Every member is +/// enumerated here by hand rather than swapped through a generated move, so a member added to the +/// class without a line here is a silent state-corruption bug -- the `static_assert`s below are the +/// type-level half of the guarantee (the macro at the call site proves the code path, these prove the +/// contract of the types), and `debugAssertBodyCounters` cross-checks the counters this swap carries. +void RefTableState::swap(RefTableState & other) noexcept +{ + static_assert(std::is_nothrow_swappable_v); + static_assert(std::is_nothrow_swappable_v>); + static_assert(std::is_nothrow_swappable_v); + static_assert(std::is_nothrow_swappable_v>>); + static_assert(std::is_nothrow_swappable_v); + static_assert(noexcept(std::declval().swap(std::declval()))); + static_assert(noexcept(std::declval().swap(std::declval()))); + + using std::swap; + swap(lifecycle, other.lifecycle); + swap(remove_txn_id, other.remove_txn_id); + swap(greatest_applied, other.greatest_applied); + committed.swap(other.committed); + precommits.swap(other.precommits); + owned_manifests.swap(other.owned_manifests); + swap(snapshot_body_bytes, other.snapshot_body_bytes); + swap(removal_body_bytes, other.removal_body_bytes); +} + +/// True iff `manifest_ref` already names an existing committed row or precommit binding under ANY +/// ref_name (the add-precommit rule: "no conflicting owner may name the same manifest"). O(1) via +/// `owned_manifests`, a COW membership index (Pool/CasRefCowManifestSet.h) that every ownership- +/// changing arm below (and `stateFromSnapshot`) maintains in lock-step with `committed` and +/// `precommits`. The old linear scan lives on, in debug/sanitizer builds only, as +/// `debugAssertBodyCounters`'s cross-check that the index has not drifted from those two containers. +bool RefTableState::manifestAlreadyOwned(const ManifestRef & manifest_ref) const +{ + return owned_manifests.contains(manifest_ref); +} + +/// The `owner_transition` op kind: dispatches on the `(old_binding, +/// new_binding)` shape to one of the four legal transitions (add precommit / remove precommit / +/// remove committed / promote). Any other shape is not a recognized transition. +void RefTableState::applyOwnerTransition(const RefOp & op) +{ + if (lifecycle != RefLifecycle::Live) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableState: owner_transition while namespace is not Live"); + + /// Shape legality is decided once, by the shared classifier; everything below is the per-shape + /// PRECONDITION check and effect, unchanged. + switch (classifyOwnerTransitionShape(op)) + { + /// Add precommit: no old_binding, a fresh Precommit new_binding. + case OwnerTransitionShape::AddPrecommit: + { + const RefOwnerBinding & b = *op.new_binding; + if (precommits.contains({b.ref_name, b.manifest_ref})) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: add precommit '{}' already exists for this exact manifest", b.ref_name); + /// Cross-owner uniqueness runs UNCONDITIONALLY, in every apply strategy (writer append AND + /// trusted replay). Since E2 this is an O(1) `owned_manifests` lookup, so the E1-era elision of + /// it under trusted replay (which downgraded it to a debug-only `chassert`) bought nothing + /// measurable while making a corrupted log/snapshot FAIL OPEN -- a double-owner input would drift + /// the index and let ordinary later writes append invariant-violating durable history. Keeping it + /// here is what makes replay fail CLOSED on a corrupted `manifest_ref` collision. + if (manifestAlreadyOwned(b.manifest_ref)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: manifest already has a conflicting owner under another ref_name"); + precommits.emplace(b.ref_name, b.manifest_ref); + snapshot_body_bytes += precommitRowEncodedSize(RefOwnerBinding{RefOwnerKind::Precommit, b.ref_name, b.manifest_ref}); + removal_body_bytes += removalOpEncodedSize(RefOwnerKind::Precommit, b.ref_name, b.manifest_ref); + owned_manifests.insert(b.manifest_ref); + return; + } + + /// Remove precommit: an exact Precommit old_binding, no new_binding. + case OwnerTransitionShape::RemovePrecommit: + { + const RefOwnerBinding & b = *op.old_binding; + if (precommits.erase({b.ref_name, b.manifest_ref}) == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: exact precommit binding '{}' to remove is absent", b.ref_name); + snapshot_body_bytes -= precommitRowEncodedSize(RefOwnerBinding{RefOwnerKind::Precommit, b.ref_name, b.manifest_ref}); + removal_body_bytes -= removalOpEncodedSize(RefOwnerKind::Precommit, b.ref_name, b.manifest_ref); + owned_manifests.erase(b.manifest_ref); + return; + } + + /// Remove committed ref: an exact Committed old_binding, no new_binding. + case OwnerTransitionShape::RemoveCommitted: + { + const RefOwnerBinding & b = *op.old_binding; + const auto it = committed.find(b.ref_name); + if (it == committed.end() || !(it->second.manifest_ref == b.manifest_ref)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: exact committed binding '{}' to remove is absent", b.ref_name); + const RefCommittedRow removed = it->second; + committed.erase(it); + snapshot_body_bytes -= committedRowEncodedSize(removed); + removal_body_bytes -= removalOpEncodedSize(RefOwnerKind::Committed, removed.ref_name, removed.manifest_ref); + owned_manifests.erase(removed.manifest_ref); + return; + } + + /// Promote: the SAME ref_name and manifest_ref move from Precommit to Committed + /// in one atomic step; the resulting row's `published_at_ms` starts unset (installed by the + /// companion set_published_at op in the same transaction, or a later one). + case OwnerTransitionShape::Promote: + { + const RefOwnerBinding & b = *op.old_binding; + if (precommits.erase({b.ref_name, b.manifest_ref}) == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: exact precommit binding '{}' to promote is absent", b.ref_name); + snapshot_body_bytes -= precommitRowEncodedSize(RefOwnerBinding{RefOwnerKind::Precommit, b.ref_name, b.manifest_ref}); + removal_body_bytes -= removalOpEncodedSize(RefOwnerKind::Precommit, b.ref_name, b.manifest_ref); + /// `owned_manifests` is deliberately left untouched by a promote: `b.manifest_ref` moves from + /// precommit ownership to committed ownership without ever giving it up in between -- the + /// same "there is no moment at which the manifest has no owner" invariant this function's + /// header doc states for promote generally. The index tracks "does ANY owner currently name + /// this manifest", not which kind, so an erase-then-insert pair here would be pure overhead. + /// A DIFFERENT manifest already committed under this exact ref_name must be evicted by its + /// own explicit owner_transition(old=Committed, new=None) first (an earlier op of this same + /// transaction, or an earlier transaction) -- never silently here. `GC`'s manifest-edge delta + /// is read off the transaction's explicit ops, not a + /// before/after state diff; a promote that silently evicted a stale committed row would never + /// emit that manifest's "-1" edge, leaking it as phantom-alive forever. + if (committed.contains(b.ref_name)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: promote '{}' would silently displace a different already-committed " + "manifest -- remove it with an explicit owner_transition first", b.ref_name); + RefCommittedRow row; + row.ref_name = b.ref_name; + row.manifest_ref = b.manifest_ref; + snapshot_body_bytes += committedRowEncodedSize(row); + removal_body_bytes += removalOpEncodedSize(RefOwnerKind::Committed, row.ref_name, row.manifest_ref); + committed.emplace(b.ref_name, std::move(row)); + return; + } + } + /// Reachable only if a future `OwnerTransitionShape` enumerator is added without a matching `case` + /// (mirrors `applyOp`'s exhaustive-switch-then-throw shape below) -- `-Wswitch`/`-Werror` catches that + /// at compile time; this throw is the runtime backstop for builds without it. + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableState: unhandled owner_transition shape"); +} + +/// The `set_published_at` op kind: the committed ref must still name `expected_manifest_ref`; replaces +/// `published_at_ms` without touching the manifest edge. +void RefTableState::applySetPublishedAt(const RefOp & op) +{ + if (lifecycle != RefLifecycle::Live) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableState: set_published_at while namespace is not Live"); + + const auto it = committed.find(op.ref_name); + if (it == committed.end() || !(it->second.manifest_ref == op.expected_manifest_ref)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: set_published_at '{}' no longer names its expected_manifest_ref", op.ref_name); + + /// `RefCowMap`'s iterator is read-only (Pool/CasRefCowMap.h): a write always goes through + /// `insert_or_assign`, never through the found iterator in place. Copy the row, apply the same + /// field mutation the old in-place code did, and write the whole row back -- this IS the COW + /// map's single-row copy-out, not a whole-table one. + RefCommittedRow updated = it->second; + const uint64_t old_row_bytes = committedRowEncodedSize(it->second); + updated.published_at_ms = op.published_at_ms; + snapshot_body_bytes -= old_row_bytes; + snapshot_body_bytes += committedRowEncodedSize(updated); + /// removal_body_bytes unchanged: set_published_at touches neither ref_name nor manifest_ref. + committed.insert_or_assign(op.ref_name, std::move(updated)); +} + +/// One operation's local preconditions and effect, shared by +/// `applyRefLogTxn`'s per-op loop and by `admits`'s single-op preview. `txn_id` is only read by +/// `RemoveNamespace` (it becomes the resulting `remove_txn_id`). Validation is identical regardless of +/// which apply strategy reached here, so this takes no mode. +void RefTableState::applyOp(const RefOp & op, const RefTxnId & txn_id) +{ + switch (op.kind) + { + case RefOpKind::NamespaceBirth: + { + /// Namespace birth is legal from an empty runtime admitted under a fresh catalog life, + /// never from `Live`. A predecessor still `Removing` is refused before recovery. + if (lifecycle == RefLifecycle::Live) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableState: namespace_birth while already Live"); + lifecycle = RefLifecycle::Live; + remove_txn_id.reset(); + return; + } + case RefOpKind::OwnerTransition: + applyOwnerTransition(op); + return; + case RefOpKind::SetPublishedAt: + applySetPublishedAt(op); + return; + case RefOpKind::RemoveNamespace: + { + /// Remove namespace: requires Live and both owner sets already empty -- true only + /// if this transaction's earlier ops (checked by `checkRemoveNamespaceOrdering`) actually + /// named every owner that existed when the transaction started. + if (lifecycle != RefLifecycle::Live) + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableState: remove_namespace while not Live"); + if (!committed.empty() || !precommits.empty()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: remove_namespace with nonempty owner sets"); + /// `committed`/`precommits` empty implies `owned_manifests` empty too -- every entry in + /// the index is put there by an ownership change to one of those two containers. A + /// mismatch here means the index has drifted, not that this transaction is invalid. + chassert(owned_manifests.empty()); + lifecycle = RefLifecycle::Removed; + remove_txn_id = txn_id; + return; + } + case RefOpKind::EpochSeal: + /// INV-2's in-band epoch closure. A seal carries NO table content, and that is its whole + /// design: its effect is that it OCCUPIES `{E, T+1}` -- the exact key a dying predecessor's + /// in-flight PUT would have taken, so the store's own write-once create becomes the fence -- + /// and that it advances `greatest_applied`, which `applyTxnInPlace` does after this switch. + /// There is deliberately nothing to do here: a seal that changed a row would be a seal that + /// can lose one. + /// + /// `Live` is required because a seal closes the epoch of a LIVE stream. A never-born + /// namespace has no stream to close, and a `Removed` one already closed its own with the + /// terminal record -- in both cases a seal is a statement about a stream that does not + /// exist, i.e. an object built against a different table's history. Recovery's CAS-walk + /// enforces the identical gate on the minting side, so this is the read half of ONE rule + /// rather than a second one that can drift from it. + if (lifecycle != RefLifecycle::Live) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: epoch_seal at {}-{} while the namespace is not Live -- a seal closes the " + "epoch of a live stream, and this table has none", + txn_id.writer_epoch, txn_id.ref_sequence); + return; + } + /// Reachable only through a hand-corrupted RefOpKind (mirrors CasRefLogCodec.cpp's + /// exhaustive-switch-then-throw shape); every named enumerator returns above. + throw Exception(ErrorCodes::CORRUPTED_DATA, "RefTableState: unknown op kind {}", static_cast(op.kind)); +} + +RefTableState stateFromSnapshot(const RefTableSnapshot & snapshot) +{ + const String bytes = encodeRefTableSnapshot(snapshot); + const RefTableSnapshot validated = decodeRefTableSnapshot(bytes, snapshot.ns, snapshot.snapshot_id); + + RefTableState state; + /// A persisted snapshot is a materialization of a live stream only. `RefTableState` defaults to + /// `Removed`, so this assignment is deliberately explicit rather than relying on construction + /// defaults that have the opposite meaning. + state.lifecycle = RefLifecycle::Live; + state.greatest_applied = validated.snapshot_id; + /// The codec validated sortedness and no-duplicate ref_name/(ref_name, manifest_ref), but NEVER + /// cross-owner `manifest_ref` uniqueness -- a snapshot naming one manifest under two owners + /// (committed/committed, committed/precommit, or precommit/precommit) is semantically corrupt and + /// this is the one place that rejects it. The check runs before each `owned_manifests.insert` so it + /// reports "corrupt snapshot data" rather than the container's "index drifted = code bug" framing, + /// which would be the wrong diagnosis for malformed persisted data. + for (const RefCommittedRow & row : validated.committed) + { + if (state.owned_manifests.contains(row.manifest_ref)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "stateFromSnapshot: snapshot names one manifest under two owners (committed ref '{}')", row.ref_name); + state.committed.emplace(row.ref_name, row); + state.snapshot_body_bytes += committedRowEncodedSize(row); + state.removal_body_bytes += removalOpEncodedSize(RefOwnerKind::Committed, row.ref_name, row.manifest_ref); + state.owned_manifests.insert(row.manifest_ref); + } + for (const RefOwnerBinding & b : validated.precommits) + { + if (state.owned_manifests.contains(b.manifest_ref)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "stateFromSnapshot: snapshot names one manifest under two owners (precommit ref '{}')", b.ref_name); + state.precommits.emplace(b.ref_name, b.manifest_ref); + state.snapshot_body_bytes += precommitRowEncodedSize(RefOwnerBinding{RefOwnerKind::Precommit, b.ref_name, b.manifest_ref}); + state.removal_body_bytes += removalOpEncodedSize(RefOwnerKind::Precommit, b.ref_name, b.manifest_ref); + state.owned_manifests.insert(b.manifest_ref); + } + return state; +} + +#ifdef DEBUG_OR_SANITIZER_BUILD +/// Debug/sanitizer-only: recompute both body totals from scratch and assert the incrementally +/// maintained values match. This is what makes the incremental counters *provably* byte-exact rather +/// than a drift-prone estimate -- the concern the old non-incremental admits() cited. O(N); compiled +/// only in debug and sanitizer builds (`DEBUG_OR_SANITIZER_BUILD`, the same condition `chassert` fires +/// under), so an ASan/TSan run exercises it too, not just a debug build. +/// +/// Also rebuilds the expected `owned_manifests` membership by scanning `committed` + `precommits` +/// (the same linear walk `manifestAlreadyOwned` used to do directly) and cross-checks it against the +/// COW index: every scanned manifest must be present in the index, and the index's total size must +/// equal the number of rows scanned -- together those two checks catch both a missing entry and a +/// stale/extra one, which a size-only or membership-only check could each miss on their own. +void RefTableState::debugAssertBodyCounters() const +{ + uint64_t snap = 0; + uint64_t rem = 0; + size_t owned_scanned = 0; + for (const auto [name, row] : committed) + { + snap += committedRowEncodedSize(row); + rem += removalOpEncodedSize(RefOwnerKind::Committed, name, row.manifest_ref); + chassert(owned_manifests.contains(row.manifest_ref)); + ++owned_scanned; + } + for (const auto & [name, mref] : precommits) + { + snap += precommitRowEncodedSize(RefOwnerBinding{RefOwnerKind::Precommit, name, mref}); + rem += removalOpEncodedSize(RefOwnerKind::Precommit, name, mref); + chassert(owned_manifests.contains(mref)); + ++owned_scanned; + } + chassert(snapshot_body_bytes == snap); + chassert(removal_body_bytes == rem); + chassert(owned_manifests.size() == owned_scanned); +} +#endif + +void RefTableState::applyTxnInPlace(const RefLogTxn & txn) +{ + /// Txn-wide preconditions first, before any mutation. + if (!(greatest_applied < txn.txn_id)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: txn_id {}-{} is not strictly greater than the greatest applied {}-{}", + txn.txn_id.writer_epoch, txn.txn_id.ref_sequence, + greatest_applied.writer_epoch, greatest_applied.ref_sequence); + + /// INV-1: within `(namespace, epoch)` the DURABLE ids are DENSE, so the only admissible id is the + /// one `nextTxnId` derives -- the same rule, on the same state, that the writer mints with. This is + /// what turns "these are the log ids I can see" into "this is the whole stream": a reader that finds + /// `1..T` knows nothing was lost, which no amount of strict-increase checking could tell it. + /// Enforcing it HERE, on the read side, is deliberate -- a hole cannot become durable even if some + /// future writer path forgets the rule, because every apply (writer candidate, recovery replay, + /// `fsck`'s oracle) runs it. + /// + /// The strict-increase check above is not subsumed by this one: it rejects an id under an OLDER + /// epoch, whose sequence would still look like a legitimate fresh-epoch `1`. + if (const RefTxnId expected = nextTxnId(txn.txn_id.writer_epoch); txn.txn_id != expected) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState: txn_id {}-{} does not continue the ref-log stream — the greatest applied id " + "is {}-{}, so the only contiguous successor is {}-{}", + txn.txn_id.writer_epoch, txn.txn_id.ref_sequence, + greatest_applied.writer_epoch, greatest_applied.ref_sequence, + expected.writer_epoch, expected.ref_sequence); + + /// INV-2's CONTEXTUAL seal grammar, on the read side. `prev_epoch_seal` is required on exactly + /// sequence 1 of a NON-genesis epoch and forbidden everywhere else; the structural half (shape, + /// well-formedness, strictly-earlier epoch) is the codec's, and this is the half that needs to know + /// whether this transaction OPENS a life or CONTINUES one across a transition. + /// + /// THE EQUIVALENCE ARGUMENT, because this is the load-bearing part. The answer is DERIVED from the + /// state, exactly and totally -- no `life_epoch` is plumbed in, and there is no optional to + /// substitute a zero for (which would demand a chain link on every sequence-1 transaction and reject + /// every genesis birth -- the trap task 5's interface note names): + /// + /// - state NOT `Live` (never-born, or `Removed`) <=> this transaction can only be a BIRTH <=> the + /// epoch it lands in IS this life's genesis epoch, so a chain link is FORBIDDEN. A link here + /// would name a seal of a previous life this state has no trace of; + /// - state `Live` <=> a prior life is PROVEN to exist below this epoch -- the namespace was applied + /// at `greatest_applied.writer_epoch`, and the density check immediately above just proved this + /// transaction's epoch is above it -- so the transition is non-genesis and a chain link is + /// REQUIRED. Passing `greatest_applied.writer_epoch` yields the same verdict the true + /// `life_epoch` would for EVERY value the true one can hold, since the true one is at or below it + /// and the rule only compares "strictly above / at or below". + /// + /// The derivation is exact for every reachable state and no default exists anywhere in it. + /// + /// It is also STRICTLY BETTER than a plumbed-in `life_epoch` would be, and not only because it needs + /// no plumbing: `life_epoch` is a property of a LIFE, and a namespace can be removed and recreated. + /// A single global value carried alongside the table would answer for the wrong life after a + /// rebirth -- it would demand a chain link from a recreated namespace's very first transaction, + /// which by definition has none. Reading the lifecycle instead makes per-life semantics fall out for + /// free: a `Removed` state receiving a birth makes THAT epoch the NEW life's genesis, with no + /// catalog and no extra field to keep in step. + /// + /// Both arms go through the ONE shared validator rather than re-deriving its rule, so the writer's + /// encode-side check and this one cannot drift. + if (txn.txn_id.ref_sequence == 1) + validateEpochSealGrammarContextual( + txn, lifecycle == RefLifecycle::Live ? greatest_applied.writer_epoch : txn.txn_id.writer_epoch); + + checkRemoveNamespaceOrdering(txn.ops); + + /// Apply every op, in array order, IN PLACE. A throw leaves `*this` PARTIALLY APPLIED ("poisoned"). + /// This is the poisoning strategy (E3, no scratch copy): sound ONLY on a state the caller discards + /// on any throw. The public `applyRefLogTxn` reaches it only through a scratch copy (turning it into + /// the strong guarantee); `replay` reaches it directly on its own local, discard-on-throw state, + /// which is what eliminates the per-transaction deep-copy of the replay tail's unbounded COW + /// overlays -- a K-transaction replay over an N-row base drops from O(K*N) to O(K + N). + for (const RefOp & op : txn.ops) + applyOp(op, txn.txn_id); + greatest_applied = txn.txn_id; +#ifdef DEBUG_OR_SANITIZER_BUILD + /// Reached only on success, where `*this` is fully applied and its incremental body counters and + /// owned-manifest index are consistent -- the invariant this cross-check defends. + debugAssertBodyCounters(); +#endif +} + +RefTxnId nextRefTxnId(RefTxnId greatest_applied, uint64_t live_epoch) +{ + return greatest_applied.writer_epoch == live_epoch + ? RefTxnId{live_epoch, greatest_applied.ref_sequence + 1} + : RefTxnId{live_epoch, 1}; +} + +void applyRefLogTxn(RefTableState & state, const RefLogTxn & txn) +{ + /// The one public apply entry point, ALWAYS the strong exception guarantee: validate and apply the + /// whole transaction against a scratch copy; replace `state` only once the whole transaction + /// succeeds, so a throw anywhere leaves `state` byte-for-byte unchanged and no intra-transaction + /// intermediate state is ever observable. This copy is cheap on every caller: each applies against a + /// materialized (empty-overlay) live state or a small bounded-overlay batch scratch, never the + /// unbounded replay tail (that is `replay`'s job, and it uses the private in-place strategy directly + /// on its own discard-on-throw local state). + RefTableState scratch = state; + scratch.applyTxnInPlace(txn); + state = std::move(scratch); +} + +RefTableSnapshot snapshotOf(const RefTableState & state, const String & ns) +{ + if (state.getLifecycle() != RefLifecycle::Live) + throw Exception( + ErrorCodes::CORRUPTED_DATA, + "snapshotOf: terminal namespace state is not snapshot-serializable"); + + RefTableSnapshot snapshot; + snapshot.ns = ns; + snapshot.snapshot_id = state.getGreatestApplied(); + + snapshot.committed.reserve(state.getCommitted().size()); + for (const auto [name, row] : state.getCommitted()) + snapshot.committed.push_back(row); /// RefCowMap iterates sorted by ref_name (Pool/CasRefCowMap.h) + + snapshot.precommits.reserve(state.getPrecommits().size()); + for (const auto & [name, manifest_ref] : state.getPrecommits()) + snapshot.precommits.push_back(RefOwnerBinding{RefOwnerKind::Precommit, name, manifest_ref}); + /// std::set> iterates sorted by (ref_name, manifest_ref), + /// matching CasRefSnapshotCodec's required precommit sort order exactly. + + return snapshot; +} + +RefTableState replay(const std::optional & snapshot, std::span tail) +{ + RefTableState state = snapshot ? stateFromSnapshot(*snapshot) : RefTableState{}; + + const String * expected_ns = snapshot ? &snapshot->ns : nullptr; + for (const RefLogTxn & txn : tail) + { + if (expected_ns && txn.ns != *expected_ns) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefTableState::replay: transaction ns '{}' does not match the table's ns '{}'", + txn.ns, *expected_ns); + expected_ns = &txn.ns; + /// The one place the poisoning in-place apply strategy is reached (E3): `state` is `replay`'s own + /// local, returned only after the WHOLE tail succeeds, so a mid-tail throw destroys it during + /// unwinding and no consumer ever observes a poisoned state. `applyTxnInPlace` still runs the + /// FULL validation `applyRefLogTxn` does -- including the cross-owner uniqueness check, which is + /// O(1) and no longer elided (post-consult) -- so a corrupted/collision-bearing tail fails closed + /// here rather than silently drifting the index. + state.applyTxnInPlace(txn); + } + return state; +} + +RefReplayBuilder::RefReplayBuilder(std::optional base, uint64_t base_encoded_bytes) +{ + if (base) + { + result.newest_snapshot_id = base->snapshot_id; + result.base_snapshot_bytes = base_encoded_bytes; + expected_ns = base->ns; + candidate = stateFromSnapshot(*base); /// full snapshot revalidation, exactly as `replay` + } +} + +void RefReplayBuilder::applyOne(RefLogTxn && txn, uint64_t encoded_bytes) +{ + /// The streaming-recovery memory probe is driven by the CALLER's loop (around GET->decode->this + /// call->discard), not here: the alive decoded-transaction set is a property of how the loop holds + /// its transactions, which `applyOne` -- seeing one at a time regardless -- cannot observe. See + /// `reportReplayMemoryDelta` / `decodedRefLogTxnFootprint`. + if (expected_ns && txn.ns != *expected_ns) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "RefReplayBuilder: transaction ns '{}' does not match the table's ns '{}'", + txn.ns, *expected_ns); + expected_ns = txn.ns; + /// The same private in-place poisoning path `replay` uses (E3): `candidate` is this builder's own + /// state, discarded on any throw (the builder is destroyed during unwinding), so a mid-tail + /// corruption fails closed here and no consumer ever observes a poisoned candidate. NOT the public + /// scratch-copying `applyRefLogTxn`, which would deep-copy the growing candidate once per + /// transaction and reintroduce the O(K*N) cost `replay` was written to avoid. + candidate.applyTxnInPlace(txn); + ++result.tail_count; + result.tail_bytes += encoded_bytes; +} + +RecoveryResult RefReplayBuilder::finish() && +{ + /// Matches `replay`: the candidate is returned WITHOUT `materializeCommitted` -- the writer's + /// recovery folds the COW overlays once on the result before installing it; the read-only consumers + /// (orphan sweep and fsck) do not need the fold at all. + result.state = std::move(candidate); + return std::move(result); +} + +uint64_t encodedSnapshotBudgetSize(const RefTableState & state) +{ + /// `snapshotOf` uses `snapshot_id = state.greatest_applied` and an empty namespace here. Snapshot + /// lifecycle is fixed to live on the wire, so the framing depends only on those fields and the row + /// count. + const uint64_t rows = state.getCommitted().size() + state.getPrecommits().size(); + return snapshotFramingSize("", state.getGreatestApplied(), rows) + + state.getSnapshotBodyBytes(); +} + +uint64_t encodedRemovalBudgetSize(const RefTableState & state) +{ + /// The hypothetical whole-namespace removal transaction uses a fixed {1,1} preview id, empty ns, + /// and one removal op per owner (committed + precommit) plus a terminal remove_namespace op -- so + /// op_count = committed + precommits + 1. + static constexpr RefTxnId kPreviewTxnId{1, 1}; + const uint64_t rows = state.getCommitted().size() + state.getPrecommits().size(); + return removalFramingSize("", kPreviewTxnId, rows + 1) + state.getRemovalBodyBytes(); +} + +bool admits(const RefTableState & state, const RefOp & op, uint64_t snapshot_budget, uint64_t removal_budget) +{ + /// A fixed nonzero placeholder id: this previews `op` in isolation and the scratch state is + /// discarded immediately after reading its (incrementally maintained) budget sizes. + static constexpr RefTxnId kPreviewTxnId{1, 1}; + + /// Previews an op that has not yet been validated or durably appended anywhere, so it gets the full + /// append-time check (the same `applyOp` the writer's apply path runs -- validation is strategy- + /// independent). + RefTableState scratch = state; + scratch.applyOp(op, kPreviewTxnId); // throws exactly as before if `op` is not a legal transition +#ifdef DEBUG_OR_SANITIZER_BUILD + scratch.debugAssertBodyCounters(); +#endif + + if (encodedSnapshotBudgetSize(scratch) > snapshot_budget) + return false; + return encodedRemovalBudgetSize(scratch) <= removal_budget; +} + +std::vector manifestEdgesOfTxn(const RefLogTxn & txn) +{ + std::vector edges; + edges.reserve(txn.ops.size()); /// every recognized op contributes at most one edge + const RootNamespace ns{txn.ns}; + + for (uint32_t op_ordinal = 0; op_ordinal < txn.ops.size(); ++op_ordinal) + { + const RefOp & op = txn.ops[op_ordinal]; + if (op.kind != RefOpKind::OwnerTransition) + continue; + + /// Shape legality is decided once, by the shared classifier -- the same one + /// `RefTableState::applyOwnerTransition` dispatches on -- so an unrecognized shape throws here + /// instead of silently acquiring accidental edge meaning (e.g. an old+new pair naming different + /// manifests, which the state machine never admits, used to read as a tolerated "replace"). + switch (classifyOwnerTransitionShape(op)) + { + case OwnerTransitionShape::AddPrecommit: + edges.push_back(RefManifestEdge{ + ManifestId{ns, op.new_binding->manifest_ref}, +1, op.new_binding->kind, op_ordinal, 1}); + continue; + case OwnerTransitionShape::RemovePrecommit: + case OwnerTransitionShape::RemoveCommitted: + edges.push_back(RefManifestEdge{ + ManifestId{ns, op.old_binding->manifest_ref}, -1, op.old_binding->kind, op_ordinal, 0}); + continue; + case OwnerTransitionShape::Promote: + /// Same-manifest owner move: the manifest keeps an owner the whole time, so there is no + /// net edge. + continue; + } + /// Reachable only if a future `OwnerTransitionShape` enumerator is added without a matching + /// `case` -- `-Wswitch`/`-Werror` catches that at compile time; this throw is the runtime + /// backstop for builds without it. + throw Exception(ErrorCodes::CORRUPTED_DATA, "manifestEdgesOfTxn: unhandled owner_transition shape"); + } + + return edges; +} + +std::optional removalTxnId(const RefLogTxn & txn) +{ + for (const RefOp & op : txn.ops) + if (op.kind == RefOpKind::RemoveNamespace) + return txn.txn_id; + return std::nullopt; +} + +std::map groupRefKeys( + const Layout & layout, const std::vector & listed_keys) +{ + const String base = layout.casRefsPrefix(); + std::map out; + + for (const String & key : listed_keys) + { + if (!key.starts_with(base)) + continue; + + const auto parsed = layout.parseRefObjectKey(key); + if (!parsed) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "groupRefKeys: key '{}' under the ref prefix is not a valid ref object -- aborting ref folding", key); + + RefTableListing & table = out[parsed->life_id]; + switch (parsed->kind) + { + case RefObjectKind::Log: + table.logs.push_back(parsed->txn_id); + break; + case RefObjectKind::Snap: + table.snapshots.push_back(parsed->txn_id); + break; + } + } + + for (auto & [life_id, table] : out) + { + std::sort(table.logs.begin(), table.logs.end()); + std::sort(table.snapshots.begin(), table.snapshots.end()); + } + + return out; +} + +RefCleanupPlan planRefCleanup(const RefTableListing & listing, const RefTxnId & durable_cursor, + std::optional checkpoint, + std::optional retained_log_proof) +{ + RefCleanupPlan plan; + + /// A physical `_snap` listed after its PUT but before the `_ckpt` CAS is not a recovery base. + /// The caller supplies `checkpoint` only after `readCheckpointSnapshotBase` has exact-read the + /// same-id non-seal `_log` and `_snap`. With no such validated triple cleanup has no coverage + /// authority and deliberately leaks every listed object. + if (!checkpoint) + return plan; + + for (const RefTxnId & log_id : listing.logs) + { + if (durable_cursor < log_id) /// L > cursor: its edge delta is not yet durable + continue; + if (*checkpoint <= log_id) /// the exact witness and its successors remain + continue; + if (retained_log_proof == log_id) /// a later-epoch base still needs its predecessor seal + continue; + plan.deletable_logs.push_back(log_id); + } + + /// The checkpoint's same-id snapshot is the recovery base, so only strictly older listed snapshots + /// are deletion candidates. + for (const RefTxnId & snapshot_id : listing.snapshots) + if (snapshot_id < *checkpoint) + plan.deletable_snapshots.push_back(snapshot_id); + + return plan; +} + +EpochCrossResult crossEpochFromSeal(Backend & backend, const Layout & layout, const RootNamespace & ns, + const RefTxnId & from_seal, std::optional seal_proven, + const RefTxnId & witness, const NamespaceLifeId & life) +{ + EpochCrossResult result; + if (from_seal == RefTxnId{}) + { + result.outcome = EpochCrossOutcome::NothingConsumed; + return result; + } + if (seal_proven && !*seal_proven) + { + result.outcome = EpochCrossOutcome::NotASeal; + return result; + } + + /// `life`: REQUIRED, not resolved here (review NEW-3) -- an internal fallback resolve was tried + /// once already (review C3, `Gc::fold`) and once more here (fsck's own independent walk defaulted + /// to `nullopt` and re-resolved), and both times a caller that had already committed to one `life` + /// for the rest of its walk could silently diverge from this function's OWN resolution if the + /// namespace is dropped and recreated between the two reads. `CasFsck.cpp`'s stream walk resolves + /// `life` once, at the top of its own function, and must pass that SAME value here rather than let + /// this function re-derive it a second time. + uint64_t target_epoch = witness.writer_epoch; + while (target_epoch > from_seal.writer_epoch) + { + const RefTxnId start{target_epoch, 1}; + result.probed = start; + const auto body = backend.get(layout.refLogKey(life, start)); + if (!body) + { + ++result.absent_probes; + result.outcome = EpochCrossOutcome::StartAbsent; + return result; + } + ++result.body_gets; + RefLogTxn head; + try + { + head = decodeRefLogTxn(openObject(FormatId::RefLog, body->bytes), ns.string(), start); + } + catch (const Exception & e) + { + result.outcome = EpochCrossOutcome::StartInvalid; + result.detail = e.message(); + return result; + } + if (!head.prev_epoch_seal || *head.prev_epoch_seal < from_seal) + { + result.outcome = EpochCrossOutcome::ChainDoesNotReach; + return result; + } + if (*head.prev_epoch_seal == from_seal) + { + result.outcome = EpochCrossOutcome::Proved; + result.start = start; + return result; + } + target_epoch = head.prev_epoch_seal->writer_epoch; + } + result.outcome = EpochCrossOutcome::ChainDoesNotReach; + return result; +} + +std::optional nextRefLogIdWithinCommittedFrontier( + const RefTxnId & current, bool is_epoch_seal, const RefTxnId & committed_through) +{ + if (committed_through < current) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS checkpoint-bounded ref walk: current id {}-{} lies above committed_through {}-{}", + current.writer_epoch, current.ref_sequence, + committed_through.writer_epoch, committed_through.ref_sequence); + if (current == committed_through) + return std::nullopt; + + if (is_epoch_seal) + { + if (committed_through.writer_epoch == current.writer_epoch) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS checkpoint-bounded ref walk: committed_through {}-{} lies after EpochSeal {}-{} in " + "the same numeric epoch", + committed_through.writer_epoch, committed_through.ref_sequence, + current.writer_epoch, current.ref_sequence); + if (current.writer_epoch == std::numeric_limits::max()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS checkpoint-bounded ref walk: EpochSeal {}-{} has no representable successor", + current.writer_epoch, current.ref_sequence); + return RefTxnId{current.writer_epoch + 1, 1}; + } + + if (current.ref_sequence == std::numeric_limits::max()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS checkpoint-bounded ref walk: log id {}-{} has no representable successor", + current.writer_epoch, current.ref_sequence); + return RefTxnId{current.writer_epoch, current.ref_sequence + 1}; +} + +CheckpointSnapshotBase readCheckpointSnapshotBase( + Backend & backend, const Layout & layout, const NamespaceLifeId & life, const RefCkpt & checkpoint) +{ + const RootNamespace & ns = life.ns; + if (!checkpoint.checkpoint_snapshot_id) + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery for namespace '{}': checkpoint has no snapshot base", + ns.string()); + } + if (!checkpoint.life_epoch) + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery for namespace '{}': checkpoint-named snapshot base has no life_epoch context", + ns.string()); + } + const RefTxnId snapshot_id = *checkpoint.checkpoint_snapshot_id; + const auto log = backend.get(layout.refLogKey(life, snapshot_id)); + if (!log) + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery for namespace '{}': checkpoint-named base snapshot {}-{} has no matching log under " + "the supplied immutable lifecycle authority", + ns.string(), snapshot_id.writer_epoch, snapshot_id.ref_sequence); + } + + const RefLogTxn base_txn = decodeRefLogTxn( + openObject(FormatId::RefLog, log->bytes), ns.string(), snapshot_id); + if (refLogTxnIsEpochSeal(base_txn)) + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery for namespace '{}': checkpoint-named base snapshot {}-{} names an EpochSeal, " + "not a snapshot base", + ns.string(), snapshot_id.writer_epoch, snapshot_id.ref_sequence); + } + validateEpochSealGrammarContextual(base_txn, *checkpoint.life_epoch); + if (base_txn.prev_epoch_seal && checkpoint.last_epoch_seal && checkpoint.committed_through + && checkpoint.committed_through->writer_epoch == snapshot_id.writer_epoch + && checkpoint.last_epoch_seal->writer_epoch + 1 == snapshot_id.writer_epoch + && *base_txn.prev_epoch_seal != *checkpoint.last_epoch_seal) + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery for namespace '{}': checkpoint-named base snapshot {}-{} refers to previous " + "epoch seal {}-{}, but checkpoint authority names {}-{}", + ns.string(), snapshot_id.writer_epoch, snapshot_id.ref_sequence, + base_txn.prev_epoch_seal->writer_epoch, base_txn.prev_epoch_seal->ref_sequence, + checkpoint.last_epoch_seal->writer_epoch, checkpoint.last_epoch_seal->ref_sequence); + } + + std::optional predecessor_seal_id; + if (base_txn.prev_epoch_seal) + { + predecessor_seal_id = *base_txn.prev_epoch_seal; + const auto predecessor = backend.get(layout.refLogKey(life, *predecessor_seal_id)); + if (!predecessor) + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery for namespace '{}': checkpoint-named base snapshot {}-{} refers to absent " + "previous epoch seal {}-{}", + ns.string(), snapshot_id.writer_epoch, snapshot_id.ref_sequence, + predecessor_seal_id->writer_epoch, predecessor_seal_id->ref_sequence); + } + const RefLogTxn predecessor_txn = decodeRefLogTxn( + openObject(FormatId::RefLog, predecessor->bytes), ns.string(), *predecessor_seal_id); + if (!refLogTxnIsEpochSeal(predecessor_txn)) + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery for namespace '{}': checkpoint-named base snapshot {}-{} refers to non-seal " + "transaction {}-{}", + ns.string(), snapshot_id.writer_epoch, snapshot_id.ref_sequence, + predecessor_seal_id->writer_epoch, predecessor_seal_id->ref_sequence); + } + } + + const auto snapshot = backend.get(layout.refSnapshotKey(life, snapshot_id)); + if (!snapshot) + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS recovery for namespace '{}': checkpoint-named base snapshot {}-{} is absent under the supplied " + "immutable lifecycle authority", + ns.string(), snapshot_id.writer_epoch, snapshot_id.ref_sequence); + } + return CheckpointSnapshotBase{ + .snapshot = decodeRefTableSnapshot(openObject(FormatId::RefSnapshot, snapshot->bytes), ns.string(), snapshot_id), + .bytes = snapshot->bytes.size(), + .predecessor_seal_id = predecessor_seal_id}; +} + +RecoveredRefTable recoverRefTableDetailedFromAuthority( + Backend & backend, const Layout & layout, const std::optional & catalog_entry, + const std::optional & ckpt) +{ + /// The frozen catalog row and `_ckpt` supplied by the caller determine every recovery boundary; + /// this function must not re-read either mutable object, or enumerate the stream, because that + /// would splice unrelated physical observations into the caller's one authority cut. + RecoveryGrounding grounding = chooseRecoveryGrounding(catalog_entry, ckpt); + /// `chooseRecoveryGrounding` has just established that this is a Live/Removing row. Constructing + /// the life from that SAME value, rather than resolving the name again, preserves the caller's + /// catalog-cut join. + const NamespaceLifeId life = NamespaceLifeId::fromCatalogEntry( + catalog_entry->ns, catalog_entry->incarnation); + const RootNamespace & ns = life.ns; + + std::optional base_id = grounding.base; + std::optional base_snapshot; + uint64_t base_snapshot_bytes = 0; + if (base_id) + { + CheckpointSnapshotBase base = readCheckpointSnapshotBase(backend, layout, life, *ckpt); + base_snapshot = std::move(base.snapshot); + base_snapshot_bytes = base.bytes; + } + + RefReplayBuilder builder(std::move(base_snapshot), base_snapshot_bytes); + if (grounding.walk_from && grounding.committed_through) + { + RefTxnId id = *grounding.walk_from; + while (id <= *grounding.committed_through) + { + const auto got = backend.get(layout.refLogKey(life, id)); + if (!got) + { + /// `NamespaceLifeId` is opaque and unique to one logical life. A later birth has a + /// different stream prefix, so no absent slot at or below this life's exact frontier + /// can be explained as a rebirth; it is always durable-data loss. + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS read-only recovery for namespace '{}': committed log id {}-{} is absent under " + "the supplied immutable checkpoint frontier {}-{}", + ns.string(), id.writer_epoch, id.ref_sequence, + grounding.committed_through->writer_epoch, grounding.committed_through->ref_sequence); + } + + RefLogTxn txn = decodeRefLogTxn(openObject(FormatId::RefLog, got->bytes), ns.string(), id); + const bool is_seal = refLogTxnIsEpochSeal(txn); + const int64_t footprint = static_cast(decodedRefLogTxnFootprint(txn)); + reportReplayMemoryDelta(footprint); + SCOPE_EXIT({ reportReplayMemoryDelta(-footprint); }); + builder.applyOne(std::move(txn), got->bytes.size()); + + if (const std::optional next = nextRefLogIdWithinCommittedFrontier( + id, is_seal, *grounding.committed_through)) + id = *next; + else + break; + } + } + + RecoveryResult result = std::move(builder).finish(); + return RecoveredRefTable{ + .state = std::move(result.state), + .newest_snapshot_id = result.newest_snapshot_id, + .last_epoch_seal = ckpt->last_epoch_seal}; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefProtocol.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefProtocol.h new file mode 100644 index 000000000000..d498a4eff062 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefProtocol.h @@ -0,0 +1,794 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Reverse catalog index built from one decoded catalog cut. Every lifecycle state participates. +/// A duplicated physical id remains represented as ambiguous so reporting tools can continue over +/// unrelated unique ids; no caller can accidentally obtain a first-row-wins resolution. +class CatalogLifeIndex +{ +public: + explicit CatalogLifeIndex(const RefCatalog & catalog); + + /// Unique logical life for `life_id`, absence for an id not present in the cut, and + /// `CORRUPTED_DATA` when multiple current rows share the id. + std::optional resolve(NamespaceLifePhysicalId life_id) const; + bool isAmbiguous(NamespaceLifePhysicalId life_id) const; + bool hasAmbiguity() const { return !ambiguous_names.empty(); } + + /// Destructive and catalog-mutating consumers require the whole cut to be unambiguous. + void throwIfAmbiguous(std::string_view consumer) const; + +private: + std::map unique_lives; + std::map> ambiguous_names; +}; + +// Shared value types for the ref-ledger writer and the pure ref-log protocol helpers below. They live +// in this protocol header so the ledger can depend on the carriers without making the pool and ledger +// headers include one another. +/// Whether a root-shard mutation originates from the writer path (user-visible publish/drop/precommit) +/// or from GC/maintenance. Diagnostic-only (`toString`, event logging): recorded on the mutation item. +enum class RootMutationOrigin : uint8_t +{ + Writer, + Gc, +}; + +/// The write-scope of one `appendRefOps` call (ref-append-lane batching): which part of the table the +/// call touches. The flat-combining batch builder admits at most ONE mutation per ref name into a +/// single flush (per-ref durable histories stay bit-identical to the unbatched protocol) and flushes +/// `WholeShard` calls SOLO (dropNamespace and anything touching multiple refs wholesale). +struct MutationScope +{ + enum class Kind : uint8_t { Ref, WholeShard }; + Kind kind = Kind::WholeShard; + String ref_name; /// set iff kind == Ref + + /// Creates a scope for a mutation that touches exactly one ref name. The name is moved into the + /// scope because scopes are normally assembled as part of an append request. + static MutationScope ref(String name) { return {Kind::Ref, std::move(name)}; } + + /// Creates a scope for a mutation that cannot safely share a batch with per-ref mutations. + static MutationScope wholeShard() { return {Kind::WholeShard, {}}; } +}; + +/// Kind of mutation being applied, used in diagnostic logging and metrics. Does not affect behaviour. +enum class RootMutationKind : uint8_t +{ + Publish, + Drop, + Precommit, + Promote, + Abandon, + UpdateRefPublishedAt, + DropNamespace, + ReclaimPrecommit, +}; + +/// Human-readable name for `RootMutationOrigin` (diagnostic logging). +inline std::string_view toString(RootMutationOrigin origin) +{ + switch (origin) + { + case RootMutationOrigin::Writer: return "Writer"; + case RootMutationOrigin::Gc: return "Gc"; + } + return "Unknown"; +} + +/// Human-readable name for `RootMutationKind` (diagnostic logging). +inline std::string_view toString(RootMutationKind kind) +{ + switch (kind) + { + case RootMutationKind::Publish: return "Publish"; + case RootMutationKind::Drop: return "Drop"; + case RootMutationKind::Precommit: return "Precommit"; + case RootMutationKind::Promote: return "Promote"; + case RootMutationKind::Abandon: return "Abandon"; + case RootMutationKind::UpdateRefPublishedAt: return "UpdateRefPublishedAt"; + case RootMutationKind::DropNamespace: return "DropNamespace"; + case RootMutationKind::ReclaimPrecommit: return "ReclaimPrecommit"; + } + return "Unknown"; +} + +/// The result of resolving a ref: its namespace-qualified manifest identity, the manifest size, and +/// the publication timestamp carried by the ref. A `Resolved` value does not own the manifest body. +struct Resolved +{ + /// The namespace-qualified identity of the part manifest this ref names. The owning RootNamespace + /// + the ref's manifest_ref form the ManifestId (the ref carries no namespace itself — that comes + /// from the owning root context). + ManifestId manifest_id; + uint64_t manifest_size = 0; + uint64_t published_at_ms = 0; /// publish wall-clock (epoch ms); 0 = unset +}; + +/// The non-identity portion of a committed-ref update. The ref's manifest identity is deliberately +/// absent: changing reachability must use an owner transition, while this carrier is only for updating +/// the publication timestamp without changing the manifest edge. In the current all-tree +/// representation, per-part files are ordinary manifest entries rather than a separate mutable-file +/// map, so `published_at_ms` is the remaining metadata that can be restamped in isolation. +struct RefPublishedAtUpdate +{ + uint64_t published_at_ms = 0; /// publish wall-clock (epoch ms); 0 = unset +}; + + +/// Counts the committed refs and precommit bindings named by one namespace-removal transaction. The +/// transaction contains one exact owner-removal operation for each count before its final +/// `remove_namespace` operation; callers interested only in completion may ignore this summary. +struct DropNamespaceStats +{ + uint64_t committed_refs = 0; + uint64_t precommits = 0; +}; + +/// Per-owner configuration passed by value to the ref ledger. It is a projection of the flat pool +/// configuration: `server_root_id` is used in ref-lane diagnostics, while boot time and wait-sleep +/// callbacks remain owned by the pool and are supplied separately because they describe live mount +/// state rather than ref-ledger policy. +struct RefLedgerConfig +{ + String server_root_id; + uint64_t gc_shards = 1; + uint64_t snapshot_log_count_threshold = 256; + uint64_t snapshot_log_bytes_threshold = 1ULL << 20; + uint64_t snapshot_publish_backoff_initial_ms = 200; + uint64_t snapshot_publish_backoff_max_ms = 30000; + uint64_t precommit_sweep_backoff_initial_ms = 200; + uint64_t precommit_sweep_backoff_max_ms = 30000; + uint64_t ref_table_cache_bytes = 256ULL << 20; +}; + +/// The ONE id-derivation rule (INV-1): the id that continues `greatest_applied`'s stream under +/// `live_epoch`. Within an epoch a table's ids are dense -- the successor of the greatest applied one -- +/// and an epoch change restarts the sequence at 1, because density is a property of `(namespace, epoch)` +/// and a fresh incarnation's stream is a fresh stream. There is no counter anywhere: the id is a pure +/// function of the state it will be applied to, which is what makes an attempt that sent nothing +/// consume nothing (the next caller derives the same id from the same unchanged state). +/// +/// This is the rule itself, applied to an ANCHOR. Callers do not choose the anchor: they go through +/// `RefTableState::nextTxnId`, which derives from `greatest_applied` -- the writer to +/// mint an id, every trial preview to stamp its throwaway transaction, and `applyTxnInPlace` to decide +/// whether a transaction's id is admissible. Sharing one rule is deliberate: an allocator and a checker +/// that each spell it out separately can drift, and a drift here is either a durable hole or a table +/// that refuses its own writes. +/// +/// Total by construction (no throw): the one input it cannot serve, an exhausted `ref_sequence` under +/// the live epoch, would need 2^64 transactions in one incarnation of one table. Should a corrupt +/// persisted snapshot ever seed such a state, the wrap produces a `ref_sequence` of 0, which +/// `applyTxnInPlace`'s strict-increase precondition rejects before anything is written. +RefTxnId nextRefTxnId(RefTxnId greatest_applied, uint64_t live_epoch); + +/// The in-memory table state: `TableState = Replay(S_X.state, tail(X))`. This class, `applyRefLogTxn`, `snapshotOf`, and +/// `replay` are the ONE shared implementation of that equation -- used verbatim by the writer, its +/// own recovery path, `fsck`, and snapshot construction, so every consumer agrees on what a +/// transaction sequence means. "Namespace" and "table" name the same entity throughout this file +/// (and its callers): one `RefTableState` per `RootNamespace`. +/// +/// Representation note (never-born vs removed): there is no separate "first birth" flag. Both +/// "never born" and "Removed" default `lifecycle` to `RefLifecycle::Removed`; they are told apart by +/// `remove_txn_id`: absent means the namespace has never completed a `remove_namespace` transaction +/// (either truly never born, or -- from this class's point of view -- indistinguishable from it, +/// which is fine because a `namespace_birth` op is legal from EITHER case and nothing else is legal +/// from either). Present means a real removal happened and recorded its `RefTxnId`. `committed` and +/// `precommits` are always empty while `lifecycle == Removed` (an invariant `applyRefLogTxn` +/// maintains: `remove_namespace` only fires once both are already empty, and no other operation is +/// legal until the next `namespace_birth`). +class RefTableState +{ +public: + RefTableState() = default; + + RefLifecycle getLifecycle() const { return lifecycle; } + const std::optional & getRemoveTxnId() const { return remove_txn_id; } + const RefTxnId & getGreatestApplied() const { return greatest_applied; } + const RefCowMap & getCommitted() const { return committed; } + const std::set> & getPrecommits() const { return precommits; } + uint64_t getSnapshotBodyBytes() const { return snapshot_body_bytes; } + uint64_t getRemovalBodyBytes() const { return removal_body_bytes; } + + /// The id this state's next transaction must carry (INV-1) — the ONE derivation, called by the + /// writer to mint an id, by every trial preview to stamp its throwaway transaction, and by + /// `applyTxnInPlace` to decide whether a transaction's id is admissible. Three callers, one rule: + /// an allocator and a checker that each spell it out separately can drift, and a drift here is + /// either a durable hole or a table that refuses its own writes. + /// + RefTxnId nextTxnId(uint64_t live_epoch) const + { + return nextRefTxnId(greatest_applied, live_epoch); + } + + /// State-install point only (once per ref-log flush, never per batch item): folds the committed + /// map's and the owned-manifest index's COW overlays into their bases -- in place when the base is + /// uniquely owned (the production flush case), else into a fresh base (see each container's + /// `materialize`). + void materializeCommitted() { committed.materialize(); owned_manifests.materialize(); } + + /// Member-wise swap, guaranteed non-throwing and allocation-free: every member's own swap is both + /// (`shared_ptr::swap`, `std::map::swap`, `std::set::swap`, `std::optional::swap` over a trivially + /// swappable payload, plus PODs). This is the ONLY sanctioned way to install a prepared candidate + /// state after its transaction is already durable -- see `CasRefLedger::commitRefChunk`'s + /// post-durable install region, which runs under `DENY_ALLOCATIONS_IN_SCOPE`. Move-assignment would + /// ALSO be `noexcept` today, but it would destroy the displaced state (freeing every `precommits` + /// node) INSIDE that region; a swap hands the old state back to the caller, which destroys it + /// outside. That destruction is not merely a tidiness matter: the old state still shares the COW + /// bases, and `materializeCommitted` folds in place only while they are uniquely owned. + void swap(RefTableState & other) noexcept; + +private: + RefLifecycle lifecycle = RefLifecycle::Removed; /// see representation note above + std::optional remove_txn_id; + RefTxnId greatest_applied{}; /// {0, 0} = no transaction applied yet + + RefCowMap committed; /// keyed by ref_name + std::set> precommits; /// (ref_name, manifest_ref) + + /// COW membership index of every `ManifestRef` with a current owner (a `committed` row or a + /// `precommits` binding), maintained O(1) per applied op by every arm of `applyOwnerTransition` + /// and `stateFromSnapshot` that changes ownership. Gives `manifestAlreadyOwned` O(1) instead of + /// a linear scan over `committed` + `precommits`. See Pool/CasRefCowManifestSet.h. + RefCowManifestSet owned_manifests; + + /// Running byte totals of the two admission-budget encodings' *bodies* (row/op lines only, no + /// header/meta/trailer framing), maintained O(1) per applied op by `applyOp` and seeded by + /// `stateFromSnapshot`. A pure function of `(committed, precommits)`: `admits` reads + /// `framing + total` instead of re-encoding the whole table. See `admits`'s doc for why this is + /// byte-exact rather than a drift-prone estimate. + uint64_t snapshot_body_bytes = 0; /// Σ committedRowEncodedSize + Σ precommitRowEncodedSize + uint64_t removal_body_bytes = 0; /// Σ removalOpEncodedSize(one per committed + one per precommit) + + /// One operation's local preconditions and effect, shared by `applyRefLogTxn`'s per-op loop and by + /// `admits`'s single-op preview. `txn_id` is only read by `RemoveNamespace` (it becomes the + /// resulting `remove_txn_id`). Validation is identical no matter which apply strategy reaches here + /// (see `applyTxnInPlace`), so this takes no mode. Was free `applyOpInPlace`. + void applyOp(const RefOp & op, const RefTxnId & txn_id); + + /// The `owner_transition` op kind: dispatches on the `(old_binding, new_binding)` shape to one of + /// the four legal transitions (add precommit / remove precommit / remove committed / promote). Any + /// other shape is not a recognized transition. The add-precommit arm's cross-owner uniqueness check + /// runs unconditionally (it is O(1) via `owned_manifests`). Was free. + void applyOwnerTransition(const RefOp & op); + + /// The `set_published_at` op kind: the committed ref must still name `expected_manifest_ref`; + /// replaces `published_at_ms` without touching the manifest edge. Was free. + void applySetPublishedAt(const RefOp & op); + + /// Applies the COMPLETE transaction to `*this` IN PLACE (the two txn-wide preconditions first, then + /// every op in array order), or throws `CORRUPTED_DATA` -- leaving `*this` PARTIALLY APPLIED + /// ("poisoned") on any throw. This is the poisoning apply strategy: it is sound ONLY on a state the + /// caller discards on any throw. It is deliberately private and reachable from OUTSIDE this + /// translation unit at exactly ONE place -- `replay`, its `friend`, which builds its `RefTableState` + /// locally and returns it only after the WHOLE tail succeeds (any throw destroys that local state + /// during unwinding, so no consumer ever observes a poisoned state). The public + /// `applyRefLogTxn` reaches it too, but only through a scratch copy that turns it into the strong + /// guarantee "throw => the caller's `state` is byte-for-byte unchanged". No caller can express the + /// dangerous combination -- poison a live state that must survive a throw -- because the poisoning + /// path is structurally unreachable except via `replay`. + void applyTxnInPlace(const RefLogTxn & txn); + + /// True iff `manifest_ref` already names an existing committed row or precommit binding under ANY + /// ref_name (the add-precommit rule: "no conflicting owner may name the same manifest"). Was free. + bool manifestAlreadyOwned(const ManifestRef & manifest_ref) const; + +#ifdef DEBUG_OR_SANITIZER_BUILD + /// Debug/sanitizer-only: recompute both body totals from scratch and assert the incrementally + /// maintained values match. Was free. + void debugAssertBodyCounters() const; +#endif + + friend void applyRefLogTxn(RefTableState & state, const RefLogTxn & txn); + friend RefTableState stateFromSnapshot(const RefTableSnapshot & snapshot); + friend RefTableState replay(const std::optional & snapshot, std::span tail); + friend bool admits(const RefTableState & state, const RefOp & op, + uint64_t snapshot_budget, uint64_t removal_budget); + /// The streaming generalisation of `replay`: reaches the same private in-place poisoning path + /// (`applyTxnInPlace`) on its own discard-on-throw candidate, one decoded transaction at a time. + friend class RefReplayBuilder; +}; + +/// The inverse of `snapshotOf`: state from a snapshot's rows. `replay` may receive a hand-built +/// `RefTableSnapshot` that never passed through `decodeRefTableSnapshot`, so this round-trips it +/// through the codec's own `encodeRefTableSnapshot`/`decodeRefTableSnapshot` rather than +/// re-implementing a second, independently-maintained copy of its validation (sortedness, no +/// duplicates, canonical names, nonzero ids, and `manifest_ref` field validity) that could silently +/// miss a case. A decoded snapshot always constructs a `Live` runtime state; terminal lifecycle and +/// its removal evidence exist only in replayed log state. Concretely: a hand-built snapshot with two committed +/// rows sharing one `ref_name` would otherwise DROP the second row via `RefCowMap::emplace` below +/// (same no-overwrite-on-existing-key semantics as `std::map::emplace`) -- the same phantom-alive +/// class of bug as a promote's silent displacement (see `applyOwnerTransition` above), just reached +/// through snapshot loading instead of a transaction. +/// +/// One check this does that the codec does NOT: cross-owner manifest uniqueness. `CasRefSnapshotCodec` +/// only enforces sortedness and no-duplicate `ref_name` (committed) / `(ref_name, manifest_ref)` +/// (precommits); it never checks that a `ManifestRef` has at most one owner across committed rows and +/// precommits. A snapshot naming one manifest under two owners is semantically corrupt (it would +/// double-count GC's `+1/-1` edges and violate the add-precommit uniqueness invariant `applyRefLogTxn` +/// enforces), so as each row is loaded this throws `CORRUPTED_DATA` if the manifest already has an +/// owner. This is the one place that enforces it; `owned_manifests.insert` would also throw, but the +/// explicit check here reports "corrupt snapshot data" rather than the container's "index drifted = +/// code bug" framing, which is the accurate diagnosis for a malformed persisted snapshot. +/// +/// Promoted from `CasRefProtocol.cpp`'s anonymous namespace to the public protocol API: the ONE +/// validated way to construct a state from rows -- tests and benchmarks use it instead of poking +/// fields. +RefTableState stateFromSnapshot(const RefTableSnapshot & snapshot); + +/// Applies the COMPLETE transaction to `state`, or throws `CORRUPTED_DATA` (each transition shape +/// below has exactly one precondition enforced here) -- with the STRONG exception guarantee: a throw +/// anywhere leaves `state` byte-for-byte unchanged. The txn-wide preconditions (a `txn_id` that is the +/// contiguous successor, `remove_namespace` ordering) are checked before any mutation, and the +/// whole apply runs two-phase against a scratch copy that replaces `state` only once the WHOLE +/// transaction has succeeded, so no intra-transaction intermediate state (e.g. a manifest with its +/// precommit already gone but its committed binding not yet installed) is ever observable to a caller +/// -- matching the promote rule: "There is no moment at which the manifest has no owner." This is the +/// only public apply entry point, and it is always the strong guarantee: the writer's append-time +/// contract and every trial/shape-check preview use it as-is. +/// +/// The poisoning in-place apply strategy (E3 -- no scratch copy, `state` partially applied on throw) +/// is NOT reachable here: it is `RefTableState::applyTxnInPlace`, private, used only by `replay` (which +/// discards its local state on any throw). There is no mode argument and no way for an external caller +/// to select the poisoning path. +/// +/// Enforced preconditions: +/// - `txn.txn_id` must be strictly greater than `state.greatest_applied` AND must be exactly the +/// contiguous successor `RefTableState::nextTxnId` derives (INV-1): the next sequence number within +/// the same writer epoch, or 1 under a greater one. A hole in a table's DURABLE stream is corruption, +/// not a tolerated allocation artefact. +/// - `remove_namespace`, if present, must be the transaction's FINAL operation, and every earlier +/// operation must be an exact owner-removal `owner_transition` (`old_binding` set, `new_binding` +/// empty). The codec does not check this shape; this is the one place +/// that does. +/// - `namespace_birth`: legal only while `lifecycle != Live`. Catalog admission guarantees this is +/// either a never-born runtime or a fresh physical life after predecessor deletion. +/// - `owner_transition` add (no `old_binding`, `new_binding.kind == Precommit`): namespace must be +/// `Live`; the exact `(ref_name, manifest_ref)` pair must be absent from `precommits`; AND no +/// existing committed row or precommit binding, under ANY ref_name, may already name the same +/// `manifest_ref` ("no conflicting owner may name the same manifest" -- this +/// is what lets `GC`'s `+1/-1` manifest-edge delta treat one manifest as ever having at most one +/// owner). The build-tuple-is-locally-active-build half of that same sentence is the writer's own +/// concern -- `RefTableState` has no notion of "active builds". +/// - `owner_transition` remove (an `old_binding`, no `new_binding`): the exact binding (Precommit or +/// Committed, matching `ref_name` and `manifest_ref`) must exist. +/// - `owner_transition` promote (`old_binding.kind == Precommit`, `new_binding.kind == Committed`, +/// same `ref_name` and `manifest_ref` on both sides): the exact precommit must exist, AND +/// `ref_name` must not already name a DIFFERENT committed manifest -- that stale row must be +/// evicted by its own explicit `owner_transition(old=Committed, new=None)` first (an earlier op of +/// the same transaction, so the two together read as one atomic replace, or an earlier +/// transaction). Promote never displaces an existing committed row implicitly: `GC`'s +/// manifest-edge delta is read off each transaction's explicit +/// ops, not a before/after state diff, so a silent displacement would never emit the evicted +/// manifest's "-1" edge -- it would leak as phantom-alive forever. On success the precommit is +/// replaced by a committed row whose `published_at_ms` starts UNSET (the initial stamp arrives +/// via a separate `set_published_at` op, in the same transaction or a later one). +/// - Any other `old_binding`/`new_binding` combination is not a recognized transition shape. +/// - `set_published_at`: namespace must be `Live`; the committed ref named by `ref_name` must still name +/// `expected_manifest_ref`. +/// - `remove_namespace`: namespace must be `Live` and both `committed` and `precommits` must already +/// be empty at this point in the (in-array-order) replay -- which is only true if the transaction's +/// earlier removal ops actually named every owner. +/// - Any operation other than `namespace_birth` while `lifecycle == Removed` is rejected: +/// "Any operation other than a valid later namespace_birth while state is +/// Removed is corruption." +/// +/// `CORRUPTED_DATA` throughout: every rejection above uses the same "is corruption" framing, +/// extended uniformly to every precondition in this section, matching how +/// `CasRefLogCodec`/`CasRefSnapshotCodec` already use `CORRUPTED_DATA` for "this data does not +/// correspond to a valid state" one layer down (wire shape rather than transition legality). Recovery +/// and `fsck` -- the primary callers replaying persisted logs -- want exactly that fail-closed +/// framing; a writer that wants a friendlier user-facing rejection for an ordinary attempted mutation +/// (e.g. "ref already exists") checks its own business state before ever building the op. +void applyRefLogTxn(RefTableState & state, const RefLogTxn & txn); + +/// The canonical snapshot of `state` under `ns`: `committed` sorted by +/// bytewise `ref_name` (guaranteed by `RefCowMap`'s sorted merge-iteration order, +/// `Pool/CasRefCowMap.h` -- the same ordering `std::map` gave before it, by design) +/// and `precommits` sorted by `(ref_name, manifest_ref)` (guaranteed by +/// `std::set>`'s iteration order, since `ManifestRef::operator<` +/// matches the tuple order `CasRefSnapshotCodec` itself sorts by). `snapshot_id` is +/// `state.greatest_applied`. A non-`Live` state is terminal replay evidence rather than snapshot +/// state and is rejected with `CORRUPTED_DATA`. This does not otherwise enforce that the result is +/// encodable (a never-born state's `snapshot_id` is `{0, 0}`, which `encodeRefTableSnapshot` +/// already rejects) -- that check already lives in the codec and need not be duplicated here. +RefTableSnapshot snapshotOf(const RefTableState & state, const String & ns); + +/// `TableState = Replay(S_X.state, tail(X))` in one call: starts from `snapshot` +/// (or the empty/never-born state when absent) and applies every transaction in `tail`, in order, via +/// `applyRefLogTxn`. A given `snapshot` is revalidated in full -- sortedness, no duplicates, canonical +/// names, nonzero ids, and `manifest_ref` field validity, i.e. everything +/// `CasRefSnapshotCodec` already enforces -- because `replay` may be handed +/// a hand-built `RefTableSnapshot` that never passed through `decodeRefTableSnapshot` (`fsck`, most +/// notably). Every entry of `tail` must also share one `ns` -- with `snapshot`'s `ns` when a snapshot +/// is given, otherwise with each other. A mismatch (of either kind) throws `CORRUPTED_DATA`: silently +/// accepting a malformed snapshot or replaying transactions from the wrong table would produce a +/// wrong-but-plausible-looking state, exactly the class of bug this equation exists to make +/// impossible. +RefTableState replay(const std::optional & snapshot, std::span tail); + +/// Everything a successful recovery of one ref table seeds. Produced by streaming replay +/// (`RefReplayBuilder::finish`) rather than assigned field-by-field into the runtime, so the whole +/// publication is one value installed atomically -- a prose field list would drift, but a struct that +/// the install copies wholesale cannot silently lose a field (Codex review round 4, spec §5). +/// +/// `finish` populates the fields that are a pure function of `(base snapshot, replayed tail)`: `state`, +/// `newest_snapshot_id`, `tail_count`, `tail_bytes`, and `base_snapshot_bytes`. The +/// remaining fields are recovery-context the streaming builder cannot know -- the writer's own recovery +/// (`CasRefLedger::ensureRefTableRecovered`) fills the admission budgets, +/// `needs_stale_precommit_sweep` and `last_epoch_seal`, before installing the whole struct under +/// `state_mutex` with `recovered` set last. The read-only consumers +/// (`recoverRefTableDetailedFromAuthority` for the orphan sweep and fsck) read only +/// `state` (plus `newest_snapshot_id` for the sweep) and leave the rest at default. +struct RecoveryResult +{ + RefTableState state; + /// Identity of the base snapshot this recovery replayed from: `nullopt` for a never-born table. + std::optional newest_snapshot_id; + /// Applied transactions strictly newer than `newest_snapshot_id`, and the sum of their stored + /// (sealed) object byte sizes -- the tail-since-snapshot accounting the runtime tracks. + uint64_t tail_count = 0; + uint64_t tail_bytes = 0; + /// Encoded (sealed) body size of the base snapshot; 0 for a never-born base. + uint64_t base_snapshot_bytes = 0; + + /// Recovery-context fields (filled by `ensureRefTableRecovered`, default for other consumers): + uint64_t snapshot_budget = 0; + uint64_t removal_budget = 0; + bool needs_stale_precommit_sweep = false; + /// The `EpochSeal` that closed the last dead epoch the recovery CAS-walk crossed -- minted by it, + /// adopted from a concurrent recoverer, or read out of the durable tail. It is the `prev_epoch_seal` + /// this table's next sequence-1 append must carry, and `nullopt` means GENESIS exactly (see + /// `CasRefLedger::RefTableRuntime::last_epoch_seal`). Recovery-context rather than replay-derived + /// because only the walk knows which epochs were dead. + std::optional last_epoch_seal; +}; + +/// The streaming generalisation of `replay` (spec §5): owns a PRIVATE candidate `RefTableState` and +/// applies decoded transactions into it ONE AT A TIME, in place, discarding the candidate on any throw. +/// It is the memory fix for a long post-snapshot tail: `replay` takes the whole `tail` materialised in a +/// vector (every decoded transaction resident at once, each up to the 20 MiB normal-class cap), whereas +/// a caller that GETs+decodes+`applyOne`+discards one object at a time holds at most a single decoded +/// transaction. `applyOne` reaches `RefTableState::applyTxnInPlace` directly -- the same private +/// poisoning path `replay` uses -- NOT the public scratch-copying `applyRefLogTxn`, which would deep-copy +/// the growing candidate once per transaction and reintroduce the O(K*N) cost `replay` was written to +/// avoid. The candidate never touches any live runtime state; a throw destroys it during unwinding, so no +/// consumer ever observes a poisoned candidate. All three full-tail materialisers stream through this: +/// the writer's recovery, `recoverRefTableDetailedFromAuthority` (orphan sweep and fsck). +class RefReplayBuilder +{ +public: + /// Seeds the candidate from `base` (or the empty/never-born state when absent), revalidating the + /// snapshot in full exactly as `replay` does (`stateFromSnapshot`). `base_encoded_bytes` is the + /// stored (sealed) size of that snapshot object, carried through to `RecoveryResult::base_snapshot_bytes` + /// (0 when the caller does not track it -- the read-only consumers do not). + explicit RefReplayBuilder(std::optional base, uint64_t base_encoded_bytes = 0); + + /// Applies one decoded transaction to the candidate in place. `encoded_bytes` is the stored (sealed) + /// object size of `txn`, accumulated into the tail-byte total. A decode/apply corruption throws + /// `CORRUPTED_DATA` (the non-transient class recovery fails fast on), discarding the candidate. + void applyOne(RefLogTxn && txn, uint64_t encoded_bytes); + + /// The candidate's lifecycle AS OF the transactions applied so far. Recovery's CAS-walk needs it at + /// each epoch boundary it reaches, and it must be the LIVE reading rather than one taken from the + /// base snapshot: a removal transaction in the replayed tail is exactly the case where the two + /// differ, and it is the case that decides whether the epoch below gets a seal. + RefLifecycle lifecycle() const { return candidate.getLifecycle(); } + + /// Materialises nothing extra (matches `replay`: the writer's recovery folds the COW overlays via + /// `materializeCommitted` on the result; the read-only consumers do not) and returns the replay-derived + /// `RecoveryResult` fields, moving the candidate out. The builder must not be used afterwards. + RecoveryResult finish() &&; + +private: + RefTableState candidate; + std::optional expected_ns; + RecoveryResult result; +}; + +/// Resident footprint, in bytes, of a decoded ref-log transaction: the heap it keeps alive while it is +/// held in memory -- its op vector's element storage plus every owned string (the transaction `ns` and +/// each op's ref-name strings). A deterministic function of the decoded CONTENT (unlike the compressed +/// stored size, which understates a highly-compressible transaction), so a memory-bound test built on it +/// is stable under ASan. This is what a whole-tail materialiser accumulates N-fold, while the streaming +/// recovery loops hold exactly one decoded transaction resident at a time. +uint64_t decodedRefLogTxnFootprint(const RefLogTxn & txn); + +/// Report a decoded-transaction memory delta to the installed streaming-recovery memory probe, if any +/// (a no-op in production -- no probe is installed). Each recovery loop calls `+footprint` when a +/// decoded transaction becomes resident and `-footprint` when it is discarded, so the probe observes the +/// loop's real resident set. Exposed (rather than confined to one translation unit) because the three +/// recovery loops live in three files and a memory-bound test's materialising control drives the +/// identical seam. +void reportReplayMemoryDelta(int64_t delta_footprint_bytes); + +/// Test-only observability for the streaming-recovery memory invariant (spec §5): while a probe is +/// installed, each recovery loop reports the resident footprint of every decoded transaction it holds, +/// for exactly the span it holds it (`reportReplayMemoryDelta` + `decodedRefLogTxnFootprint`). A +/// memory-bound test tracks the peak of the summed reported footprint and asserts it stays within a +/// single transaction, where the retired whole-tail materialiser -- and the test-local materialising +/// control that stands in for it -- held the entire tail resident at once. Because the report spans the +/// decoded transaction's whole GET->decode->apply->discard lifetime at the LOOP, not one apply in +/// isolation, a regression that materialises the whole tail before applying it is caught. No probe +/// installed => no accounting. Guarded by an internal mutex; install before driving recovery and clear +/// afterwards. +void setRecoveryReplayMemoryProbeForTest(std::function probe); + +/// The exact encoded size of `state`'s canonical snapshot (`encodeRefTableSnapshot(snapshotOf(state, +/// "")).size()`), computed in O(1) from the running body counter plus O(1) framing instead of a full +/// re-encode. Used by `admits` and directly property-tested against the real encoder. +uint64_t encodedSnapshotBudgetSize(const RefTableState & state); + +/// The exact encoded size of `state`'s hypothetical whole-namespace removal transaction, computed in +/// O(1) from the running body counter plus O(1) framing. Used by `admits`. +uint64_t encodedRemovalBudgetSize(const RefTableState & state); + +/// Admission budget: true iff applying `op` to a COPY of `state` (via the same +/// per-operation validator `applyRefLogTxn` uses -- an `op` that is not itself a legal transition +/// throws exactly as `applyRefLogTxn` would, since `admits` answers "would this legal op still fit +/// the budget", not "is this op legal") keeps BOTH the resulting table snapshot and the resulting +/// hypothetical complete-removal transaction within their respective byte budgets. +/// +/// `RefTableState` carries no `ns` (it is per-table but not one of this class's fields), so both +/// hypothetical encodings are measured with an empty `ns`. `ns` is constant for one table for its +/// entire lifetime, so a caller computes its own table's `ns.size()` overhead once (the wire layout's +/// `u32` length prefix itself is present in BOTH the empty-`ns` measurement here and the real encoding, +/// so it cancels -- only the `ns` bytes themselves are the delta; repeated exactly once in a snapshot +/// body and once in a removal-transaction body, see `CasRefSnapshotCodec` / `CasRefLogCodec`'s wire +/// layout) and pre-subtracts it, together with its own safety margin, from the raw +/// `ref_snapshot_max_bytes` / `ref_removal_max_bytes` hard limits before calling `admits`. +/// +/// Implementation: sizes are computed incrementally. `RefTableState` carries running body-byte totals +/// (`snapshot_body_bytes` / `removal_body_bytes`) maintained O(1) per applied op by `RefTableState::applyOp`; +/// `admits` applies `op` to a scratch copy and reads `framing + total` via `encodedSnapshotBudgetSize` +/// / `encodedRemovalBudgetSize`, making the whole check O(touched rows) instead of O(table size). This +/// is byte-exact rather than a drift-prone estimate: both budget encodings are pure per-row sums, the +/// per-row contributions come from the same codec primitives the full encoders use, and a +/// debug/sanitizer-only recompute-and-compare `chassert` (`RefTableState::debugAssertBodyCounters`) +/// reasserts equality on every applied transaction and every `admits` preview. +bool admits(const RefTableState & state, const RefOp & op, uint64_t snapshot_budget, uint64_t removal_budget); + +/// Pure ref-log intake primitives for a GC round. None of these read a +/// manifest body, a snapshot body, or `gc/state`: they turn a global `LIST cas/ns/stream/` result and the +/// decoded bodies of new transactions into (a) the per-table log/snapshot/marker listing, (b) the +/// deterministic manifest-edge delta, and (c) the exact ref-object cleanup plan. The GC round +/// (`CasGc.cpp`) drives the manifest-body reads (`foldManifestEdges`), the fold barrier, the durable +/// cursor, and the batch deletions around these functions. Keeping the delta and cleanup logic pure +/// makes it directly unit-testable (`gtest_cas_ref_intake.cpp`) without a full round. + +/// One `+1`/`-1` manifest edge emitted by one ref-log operation. +/// `manifest_id` is namespace-qualified (equal `ManifestRef` tuples under two tables stay distinct). +/// The ordinals locate the exact operation and edge inside the transaction, giving the spec's +/// `event_id = {namespace, RefTxnId, operation_ordinal, edge_ordinal}` its determinism: replaying the +/// same logs yields byte-identical edges, so retry and competing GC attempts produce the same delta. +struct RefManifestEdge +{ + ManifestId manifest_id; + int change = 0; /// +1 activation | -1 removal + RefOwnerKind owner_kind = RefOwnerKind::Committed; /// kind of the binding that produced this edge: + /// the `new_binding` kind for a `+1`, the `old_binding` kind for a `-1`. + /// The GC fold needs it to classify a missing manifest body -- a removed + /// precommit that never activated is skipped, every other missing body clamps. + uint32_t op_ordinal = 0; /// index of the op within its transaction + uint32_t edge_ordinal = 0; /// 0 = the removal edge, 1 = the activation edge of one op + + bool operator==(const RefManifestEdge &) const = default; +}; + +/// The manifest edges of ONE decoded transaction, in operation order, reading NO manifest body. +/// `owner_transition` recognizes EXACTLY the four shapes `classifyOwnerTransitionShape` +/// (Pool/CasRefProtocol.cpp) also uses to drive `RefTableState::applyOwnerTransition` -- the SAME +/// classification, not a second copy of the shape knowledge: +/// - add precommit (no old_binding, new_binding.kind == Precommit) => `+1` for `new.manifest_ref` +/// - remove precommit / remove committed (old_binding set, no new_binding) +/// => `-1` for `old.manifest_ref` +/// - promote (old_binding.kind == Precommit, new_binding.kind == Committed, SAME ref_name and +/// manifest_ref) => no edge (net zero: the +/// manifest keeps an owner the whole time) +/// - `namespace_birth` / `set_published_at` / `remove_namespace` => no edge +/// Any other `owner_transition` shape -- neither binding, old+new naming DIFFERENT manifests, a +/// promote whose old/new ref_name disagree, or any other kind combination -- throws `CORRUPTED_DATA`. +/// These are exactly the shapes `applyRefLogTxn`/`replay` already reject at the state-machine layer, so +/// a hand-corrupted or adversarial log body is the only way this branch is reached; a legitimately +/// written log never produces one. The GC fold (`CasGc.cpp`) extracts edges inside the same try-block +/// as `decodeRefLogTxn`, so the throw gets the identical "ref log body invalid: ref folding aborted +/// this round" treatment as an undecodable body -- no cursor advance, no deletions, an anomaly +/// recorded. The orphan sweep (`CasOrphanManifestSweep.cpp`) catches it around the whole +/// `activeManifestKeys` construction and skips (or marks errored) that namespace's deletions rather +/// than trusting an incomplete protection view. +/// The `remove_namespace` operation changes lifecycle only; the exact owner removals that must precede +/// it in the same transaction already emit their own `-1` edges. +std::vector manifestEdgesOfTxn(const RefLogTxn & txn); + +/// The GC fold consumes a table's new transactions ONE log at a time, in ascending id order, emitting +/// each log's `manifestEdgesOfTxn` into `foldManifestEdges` and advancing the durable cursor per fully +/// folded log (mirroring the legacy per-event journal fold, including its clamp-on-missing-body barrier). +/// In-batch add+remove cancellation is therefore NOT done as a pre-fold net pass: the idempotent +/// `(blob, source_id)` in-degree set-merge already cancels a `+edge` and matching `-edge` folded into one +/// generation, and a pre-fold net pass would be unsafe -- a mid-batch clamp could split a cancelled pair +/// across the advanced cursor, folding a spurious `-1` in a later round. So there is deliberately no +/// `netManifestDelta` here. + +/// The `remove_txn_id` of a transaction that ends its namespace's life (contains a `remove_namespace` +/// operation), or `nullopt`. The value equals `txn.txn_id`. The round routes it into the durable +/// cleanup evidence of that life's fold-state row. +std::optional removalTxnId(const RefLogTxn & txn); + +/// One table's surviving ref-object keys from this round's global `LIST`, split by kind and sorted +/// ascending. +struct RefTableListing +{ + std::vector logs; + std::vector snapshots; + bool operator==(const RefTableListing &) const = default; +}; + +/// Parse and group a global `LIST` of keys under `layout.casRefsPrefix` by physical life id. Every +/// key is expected to be one of the three immutable stream-object kinds; checkpoints and namespace +/// files live in the separate state tree and are never offered by this hot enumeration. An +/// unrecognized stream key throws `CORRUPTED_DATA`, so the round cannot derive a partial delta or +/// authorize destructive work from an incomplete classification. +/// A key outside `casRefsPrefix` is ignored: the caller lists only the stream prefix, and a foreign key +/// is not this format's concern. +std::map groupRefKeys( + const Layout & layout, const std::vector & listed_keys); + +/// The exact ref objects one round may delete for one namespace life. +/// Pure; acts only on keys THIS round's scan returned, but the scan is never cleanup authority. The +/// caller may supply `checkpoint` only after exact validation of the `_ckpt`-named recovery triple: +/// `_ckpt` plus its same-id non-seal `_log` and `_snap`. A later-epoch base also returns the exact +/// predecessor seal that proved its transition; `retained_log_proof` keeps that log outside the delete +/// plan while the checkpoint remains authoritative. Without a validated base the plan is empty. +/// With it, a log `L` is deletable only when `L < checkpoint` and `L <= durable_cursor`; a listed +/// snapshot is deletable only when its id is `< checkpoint`. The base's same-id `_log` and `_snap`, +/// and every newer stream object, are retained. +struct RefCleanupPlan +{ + std::vector deletable_logs; + std::vector deletable_snapshots; + + bool operator==(const RefCleanupPlan &) const = default; +}; +RefCleanupPlan planRefCleanup(const RefTableListing & listing, const RefTxnId & durable_cursor, + std::optional checkpoint = std::nullopt, + std::optional retained_log_proof = std::nullopt); + +/// Why an epoch crossing failed, or that it was PROVED. See `crossEpochFromSeal`. +enum class EpochCrossOutcome : uint8_t +{ + Proved, /// `start` is sequence 1 of the epoch that chains from `from_seal` + NothingConsumed, /// `from_seal` is `{0, 0}`: nothing has been consumed, so there is no seal to cross from + NotASeal, /// the record at `from_seal` is KNOWN not to be an `EpochSeal` + StartAbsent, /// an epoch-start record the back-chain needs is not there + StartInvalid, /// an epoch-start record is undecodable (`detail` carries the codec's message) + ChainDoesNotReach, /// the chain names no seal at `from_seal` -- a genesis record, or one that skips it +}; + +/// One epoch crossing's outcome plus what it cost, so a caller can attribute the reads it performed. +struct EpochCrossResult +{ + EpochCrossOutcome outcome = EpochCrossOutcome::ChainDoesNotReach; + RefTxnId start{}; /// meaningful only on `Proved` + RefTxnId probed{}; /// the epoch-start id the walk last read -- names the object in a diagnostic + String detail; /// the decode error, on `StartInvalid` + uint64_t body_gets = 0; /// epoch-start bodies actually fetched + uint64_t absent_probes = 0; /// epoch-start reads that came back absent + + bool proved() const { return outcome == EpochCrossOutcome::Proved; } +}; + +/// Cross into the epoch that follows the one `from_seal` closed, and PROVE it rather than guess it. +/// +/// A listing only NOMINATES a candidate epoch (`witness`). The proof is the back-chain: the target +/// epoch's sequence-1 record names the seal that must have been consumed before it (INV-2). If it names +/// a seal ABOVE `from_seal`, an epoch sits in between that the nomination omitted -- the chain is +/// followed back one epoch and retried, which is what makes a crossing independent of any enumeration. +/// Anything else -- no sequence 1, an undecodable one, a genesis record (no `prev_epoch_seal`) above a +/// consumed seal, or a chain that skips the position -- is an unproven crossing and is reported as such. +/// +/// WHAT THE CHAIN PROVES, EXACTLY: the IDENTITY of the position the next epoch chains from, not its +/// KIND. Those come apart when a writer names an ordinary record as `prev_epoch_seal`, and the damage is +/// the one the seal exists to prevent -- epoch `E` declared closed while its writer may still append, so +/// a later `{E, k}` lands permanently below the cursor. So the kind is checked wherever it is knowable: +/// `seal_proven` carries `refLogTxnIsEpochSeal` of the record the CALLER applied at `from_seal` (free -- +/// it decoded the body to apply it). Pass `nullopt` only when the caller applied nothing and `from_seal` +/// is an inherited cursor whose record may since have been cleaned, so its kind is unknowable by any +/// amount of reading here and the crossing rests on chain-trust. +/// +/// Terminates: `validateEpochSealGrammarStructural` guarantees `prev_epoch_seal->writer_epoch + 1 == +/// txn_id.writer_epoch`, so the target epoch decreases one numeric epoch per link and is bounded below +/// by `from_seal`'s own epoch. The record it proves is read once more by the caller's own walk: one +/// redundant `GET` per epoch crossed, and crossings happen once per writer-epoch change. +/// +/// READ-ONLY, and shared deliberately: the GC fold's intake and fsck's audit must not be able to +/// disagree about when an epoch boundary has been proved -- a rule that says which records a cut +/// contains cannot have two implementations. +/// +/// `life`: the namespace's life, REQUIRED (review NEW-3 -- a `nullopt`-resolves-internally default was +/// tried once and reintroduced the exact divergence review C3 removed from `Gc::fold`, just relocated +/// into `CasFsck.cpp`'s independent walk, which had its OWN already-resolved `life` in scope one call +/// site above and simply did not pass it). Every caller must resolve `life` itself, ONCE, and pass the +/// SAME value here that it uses for every other read in its own walk -- this function no longer +/// resolves anything on its own, so there is no second resolution left to disagree with the first. +EpochCrossResult crossEpochFromSeal(Backend & backend, const Layout & layout, const RootNamespace & ns, + const RefTxnId & from_seal, std::optional seal_proven, + const RefTxnId & witness, const NamespaceLifeId & life); + +/// Return the one exact successor an immutable `_ckpt.committed_through` range permits after the +/// decoded record at `current`, or `nullopt` when `current` is the inclusive frontier itself. An +/// `EpochSeal` ends its numeric epoch, so a strictly-later frontier in that SAME epoch is corrupt: +/// advancing to `{E+1,1}` and letting ordinary ordering terminate would silently discard the invalid +/// part of the claimed range. Shared by every checkpoint-bounded reader so recovery, fsck, and the +/// orphan protection walk cannot disagree about that malformed authority. +std::optional nextRefLogIdWithinCommittedFrontier( + const RefTxnId & current, bool is_epoch_seal, const RefTxnId & committed_through); + +/// The result of `recoverRefTableDetailedFromAuthority`: the replayed table state plus the identity of +/// the snapshot recovery actually selected as its base (`nullopt` when it found no snapshot at all). +struct RecoveredRefTable +{ + RefTableState state; + std::optional newest_snapshot_id; + /// The exact lifecycle authority's last sealed epoch. The authoritative read-only entry point + /// copies this from its immutable `_ckpt` input. + std::optional last_epoch_seal; +}; + +/// Exact-read and decode the checkpoint-named recovery base. The anchor is the bounded triple +/// `_ckpt` + same-id non-seal `_log` + same-id `_snap`: read and decode the log first, reject an +/// `EpochSeal`, validate its contextual epoch backlink against `_ckpt.life_epoch` and the exact +/// `_ckpt.last_epoch_seal` when that field describes the base's preceding epoch, exact-read and require +/// the named predecessor to be an `EpochSeal`, then read the snapshot. This order prevents a forged +/// snapshot at any historical seal or contextually invalid epoch start from becoming state. Cleanup +/// retains both the matching log and returned predecessor proof while the checkpoint names this base. +struct CheckpointSnapshotBase +{ + RefTableSnapshot snapshot; + uint64_t bytes = 0; + /// The exact `EpochSeal` named by a non-genesis sequence-1 base. Recovery needs that object as + /// durable transition proof, so cleanup must retain it together with the checkpoint base. + std::optional predecessor_seal_id; +}; + +CheckpointSnapshotBase readCheckpointSnapshotBase( + Backend & backend, const Layout & layout, const NamespaceLifeId & life, const RefCkpt & checkpoint); + +/// Recover a ref table from ONE immutable lifecycle authority cut supplied by the caller. `catalog_entry` +/// is either the exact row from that caller's frozen catalog cut or absence from that same cut; `ckpt` is +/// the exact decoded `_ckpt` that caller read for the row's `NamespaceLifeId`. This function does NOT +/// GET the catalog or `_ckpt` itself: accepting a later, competing cut would make its result disagree +/// with the caller's other decisions. `chooseRecoveryGrounding` makes absent/Creating names non-recoverable +/// and requires a readable `_ckpt` with `life_epoch` for Live/Removing. +/// +/// Recovery performs no stream `LIST`: its replay is exact point GETs from the checkpoint-named base +/// through inclusive `committed_through`; a missing checkpoint-named base or committed log is corruption +/// under this immutable authority. In particular, this read-only API never probes or adopts `F+1`. There +/// is deliberately no self-resolving compatibility overload: every consumer must pass the row from its +/// frozen catalog cut explicitly. +RecoveredRefTable recoverRefTableDetailedFromAuthority( + Backend & backend, const Layout & layout, const std::optional & catalog_entry, + const std::optional & ckpt); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.cpp new file mode 100644 index 000000000000..58ca6559bd83 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.cpp @@ -0,0 +1,1512 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event CASMountLeaseLost; + extern const Event CASMountReleaseSkippedForeignOccupant; + extern const Event CASMountExclusivityViolation; +} + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ABORTED; + extern const int CORRUPTED_DATA; + extern const int FILE_DOESNT_EXIST; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +/// The owner, epoch, and mount-lease wire codecs are implemented in +/// `Formats/CasServerRootFormats`; this file contains the mount-safety protocol logic that uses +/// those codecs. + +namespace +{ +/// TRUE iff a `list(prefix, "", 1)` over `prefix` returns at least one key. +bool prefixHasAnyKey(Backend & b, const String & prefix) +{ + return !b.list(prefix, /*cursor*/ "", /*limit*/ 1).keys.empty(); +} + +uint64_t defaultBootMs() +{ + struct timespec ts{}; + clock_gettime(CLOCK_BOOTTIME, &ts); + return static_cast(ts.tv_sec) * 1000 + static_cast(ts.tv_nsec) / 1000000; +} + +/// Forward declaration: defined below (same TU-unique anonymous namespace) — `allocateWriterEpoch` +/// names the current mount holder in its DecommissionRecovery live-refusal message. +String describeMountHolder(const MountLease & m); + +std::optional readOwnerObject(Backend & b, const Layout & l, const String & server_root_id) +{ + const auto got = b.get(l.ownerKey(server_root_id)); + if (!got) + return std::nullopt; + return decodeOwner(got->bytes); +} + +void throwIfOwnerRetired(const OwnerObject & owner, const String & srid) +{ + if (!owner.retired_at_ms) + return; + + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' was explicitly decommissioned by an operator (tombstoned at {} ms) " + "and is refusing to silently resume — if you genuinely intend to bring this server-root " + "back, manually clear the owner object's tombstone field and restart " + "(same manual-recovery pattern as an owner anchor lost over existing data)", + srid, *owner.retired_at_ms); +} +} + +bool serverRootSubtreeEmpty( + Backend & b, const Layout & l, const String & srid, const RefCatalog & catalog_observation) +{ + const String owned_prefix = srid + "/"; + for (const CatalogEntry & entry : catalog_observation.entries) + if (entry.ns.string() == srid || entry.ns.string().starts_with(owned_prefix)) + return false; + + /// Manifests and loose roots retain logical path identity. Opaque namespace stream/state debris + /// alone is not evidence that this server root owns live work. + if (prefixHasAnyKey(b, l.casManifestsServerPrefix(srid))) + return false; + if (prefixHasAnyKey(b, l.serverRootDataPrefix(srid))) + return false; + return true; +} + +std::optional readOwnerUuid(Backend & b, const Layout & l, const String & server_root_id) +{ + const std::optional owner = readOwnerObject(b, l, server_root_id); + if (!owner) + return std::nullopt; + return owner->server_uuid; +} + +void claimOwnerOrThrow( + Backend & b, const Layout & l, const String & srid, UInt128 our_uuid, + const ObserveRefCatalog & observe_catalog) +{ + if (!observe_catalog) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS server-root '{}': catalog observer is required", srid); + const String key = l.ownerKey(srid); + + /// Owner present → it is identity: equal UUID is ok, a different UUID fails closed regardless + /// of any lease/clock state. + if (const std::optional owner = readOwnerObject(b, l, srid)) + { + if (owner->server_uuid == our_uuid) + { + throwIfOwnerRetired(*owner, srid); + return; + } + /// Mirror mountDoubleStartMessage's operator guidance: the by-far most common cause is a + /// REGENERATED local ClickHouse uuid file (wiped /var/lib/clickhouse, a pod rescheduled + /// without a persistent volume) while the pool kept the old identity — name it and the + /// recovery options instead of a bare refusal. + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' is owned by a different server (owner server_uuid={}, ours={}) — refusing to claim. " + "This usually means THIS server's local uuid file was regenerated (e.g. /var/lib/clickhouse was wiped, " + "or the container/pod was recreated without a persistent volume) while the pool kept the old identity. " + "Recover by restoring the old local uuid file; or configure a fresh for this disk; " + "or — only after verifying that NO server uses this root — manually delete the owner object '{}' and restart.", + srid, u128ToHex(owner->server_uuid), u128ToHex(our_uuid), key); + } + + /// Owner absent. Claiming is allowed ONLY over a provably-empty subtree; an absent owner over + /// existing data means the identity was lost and must never be silently re-claimed. + if (!serverRootSubtreeEmpty(b, l, srid, observe_catalog())) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' has no owner anchor but its data subtree is non-empty " + "(identity lost over existing data) — refusing to re-claim", + srid); + + const PutResult put = b.putIfAbsent(key, encodeOwner(OwnerObject{ + .server_uuid = our_uuid, + .retired_at_ms = std::nullopt, + })); + if (put.outcome == PutOutcome::Done) + return; + + /// The conditional create conflicted. Recompute the whole catalog + manifest + roots bundle; + /// no stale emptiness result is carried across the conflict. + if (!serverRootSubtreeEmpty(b, l, srid, observe_catalog())) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' owner claim conflicted and newly visible owned work blocks recreation", srid); + + /// Race: another process claimed between our get and our putIfAbsent. Re-read and compare. + const std::optional reread = readOwnerObject(b, l, srid); + if (!reread) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' owner anchor vanished during claim", srid); + if (reread->server_uuid == our_uuid) + { + throwIfOwnerRetired(*reread, srid); + return; + } + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' was claimed by a different server during our claim (foreign owner) " + "— refusing to proceed", + srid); +} + +uint64_t allocateWriterEpoch( + Backend & b, const Layout & l, const String & srid, EpochMintPolicy policy, uint64_t now_ms, + const ObserveRefCatalog & observe_catalog) +{ + if (!observe_catalog) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS server-root '{}': catalog observer is required", srid); + const String key = l.epochKey(srid); + + static constexpr int max_attempts = 100; + for (int attempt = 0; attempt < max_attempts; ++attempt) + { + const auto got = b.get(key); + + ServerEpoch current; + std::optional expected; + if (got) + { + current = decodeServerEpoch(got->bytes); + expected = got->token; + } + else + { + /// A missing `epoch` over a non-empty subtree is a reset hazard (durable monotone + /// counter cannot be reconstructed) — fail closed. + if (!serverRootSubtreeEmpty(b, l, srid, observe_catalog())) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' has no durable epoch object but its data subtree is " + "non-empty (writer_epoch reset hazard) — refusing to proceed", + srid); + + /// Same hazard through the CONTROL objects (spec rev.4 Phase C): an absent epoch while + /// a mount object exists means epoch state was lost under a live/recent mount — + /// re-minting epoch 1 there is how a same-(uuid, epoch) twin is born. This is a + /// lifecycle decision, so it uses the authoritative probe, never get-absence. + const SentinelProbeResult mount_probe = b.probeSentinelRaw(l.mountKey(srid)); + switch (mount_probe.outcome) + { + case ProbeOutcome::KeyAbsent: + break; /// authoritative absence — fresh-root bootstrap proceeds below + case ProbeOutcome::Present: + { + if (policy == EpochMintPolicy::DecommissionRecovery) + { + chassert(now_ms != 0); /// the decommission caller must pass its clock + const MountLease surviving = decodeMountLease(*mount_probe.body); + /// Deliberately weaker than claimMount's reclaim gate (this file, ~:370-380), + /// which never trusts a bare wall-clock comparison alone (only gc_fenced / + /// the clean-farewell min_active==UINT64_MAX marker / a caller-proven-dead + /// token justify a reclaim there, because clock skew can misjudge liveness). + /// This is still safe: (a) the mint below is DISTINCT from the survivor's + /// epoch by construction, so no same-(uuid, epoch) pair is ever representable + /// even if this liveness read is wrong; (b) claimMount right after this still + /// applies its own STRONG liveness gate and refuses a genuinely live member + /// regardless of what happens here. So a clock-skewed "terminal" misread can + /// only burn one epoch number on a doomed decommission attempt that aborts at + /// claimMount — it can never admit a claim over a live member. + const bool live = !surviving.gc_fenced && surviving.expires_at_ms > now_ms; + if (live) + throw Exception(ErrorCodes::ABORTED, + "CAS decommission '{}': epoch object missing but a LIVE mount lease " + "exists ({}) — refusing to re-mint an epoch under a live member " + "(stop the server or wait for its lease to lapse)", + srid, describeMountHolder(surviving)); + /// Terminal mount: proceed, but mint an epoch DISTINCT from the survivor's + /// by construction — the same-pair state is unrepresentable on this path. + current.next_writer_epoch = std::max(1, surviving.writer_epoch + 1); + break; + } + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' has no durable epoch object but a mount lease exists — " + "durable epoch state was lost while a mount is live or recently live; " + "refusing to re-mint epoch 1. If no server is live on this root, " + "decommission it or manually remove the stale mount object '{}'.", + srid, l.mountKey(srid)); + } + case ProbeOutcome::ContainerAbsent: + case ProbeOutcome::AccessDenied: + case ProbeOutcome::Indeterminate: + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}': cannot verify mount-lease absence before re-minting " + "the writer epoch (probe outcome: {}) — absence was never proven; failing closed", + srid, magic_enum::enum_name(mount_probe.outcome)); + } + + if (current.next_writer_epoch == 0) + current.next_writer_epoch = 1; + } + + const uint64_t next = current.next_writer_epoch; + ServerEpoch new_state; + new_state.next_writer_epoch = next + 1; + + const CasResult res = b.casPut(key, encodeServerEpoch(new_state), expected); + if (res.outcome == CasOutcome::Committed) + return next; + if (!got) + { + /// The absent-epoch create conflicted. A winner may have installed an epoch while owned + /// work became visible, so recompute the complete catalog + manifest + roots bundle + /// before the next iteration is allowed to accept either a present or absent epoch. + if (!serverRootSubtreeEmpty(b, l, srid, observe_catalog())) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' writer_epoch allocation conflicted and newly visible owned " + "work blocks recreation", + srid); + } + /// Conflict: someone else allocated concurrently — retry against fresh state only after the + /// absent-epoch safety bundle above has been recomputed when required. + } + + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' writer_epoch allocation did not converge after {} attempts", + srid, max_attempts); +} + +namespace +{ +/// Build a fresh mount-lease body for (uuid, epoch) with the given seq, stamped from `now_ms`. +MountLease makeMountBody(UInt128 uuid, uint64_t epoch, uint64_t seq, uint64_t now_ms, uint64_t ttl_ms) +{ + return MountLease{ + .server_uuid = uuid, + .writer_epoch = epoch, + .hostname = getFQDNOrHostName(), + .pid = static_cast(::getpid()), + .started_at_ms = now_ms, + .seq = seq, + .expires_at_ms = now_ms + ttl_ms, + }; +} + +/// Mirrors `mountDoubleStartMessage`'s identity fields. The mount-audit sink is not yet installed +/// during `Pool::open`, so at first-open these refusal messages are the only holder-identity +/// carrier in err.log — name the toucher inline rather than just the key. +String describeMountHolder(const MountLease & m) +{ + return fmt::format("server_uuid={} hostname={} pid={} writer_epoch={} seq={} expires_at_ms={}", + u128ToHex(m.server_uuid), m.hostname, m.pid, m.writer_epoch, m.seq, m.expires_at_ms); +} + +/// The mount-slot "foreign writer" audit instrument: every mount-slot WRITE +/// (`MountClaim`/`MountRelease`) and every OBSERVED foreign/conflicting body (`MountConflict`) +/// becomes one `system.cas_log` row. `observed` is the CURRENT decoded body at the +/// point of decision — for a conflict it carries the identity that made us refuse (holder_uuid/ +/// hostname/pid/epoch/seq/expires); null when no body was observed (e.g. a bare CAS race). +/// No-op when `sink` is unset, so a disabled log does no per-call work. +void emitMountEvent(const CasEventSink & sink, CasEventType type, const String & srid, + const String & branch, const MountLease * observed, const String & reason) +{ + if (!sink) + return; + CasEvent e; + e.type = type; + e.object_kind = CasEventObjectKind::None; + e.outcome = branch; + e.reason = reason; + e.detail["server_root_id"] = srid; + e.detail["branch"] = branch; + if (observed) + { + e.detail["holder_uuid"] = u128ToHex(observed->server_uuid); + e.detail["holder_hostname"] = observed->hostname; + e.detail["holder_pid"] = std::to_string(observed->pid); + e.detail["holder_epoch"] = std::to_string(observed->writer_epoch); + e.detail["holder_seq"] = std::to_string(observed->seq); + e.detail["holder_expires_at_ms"] = std::to_string(observed->expires_at_ms); + } + sink(std::move(e)); +} +} + +MountClaimResult claimMount( + Backend & b, const Layout & l, const String & srid, UInt128 our_uuid, uint64_t our_epoch, + uint64_t now_ms, uint64_t ttl_ms, const std::optional & proven_dead_token, + const CasEventSink & sink) +{ + const String key = l.mountKey(srid); + const auto got = b.get(key); + + /// Absent → fresh claim. + if (!got) + { + const MountLease body = makeMountBody(our_uuid, our_epoch, /*seq=*/ 1, now_ms, ttl_ms); + const PutResult put = b.putIfAbsent(key, encodeMountLease(body)); + if (put.outcome != PutOutcome::Done) + /// Raced with a concurrent writer between get and putIfAbsent. Treat as a live double + /// start — fail closed; never overwrite a slot that appeared under us. No re-read was + /// done, so no conflicting identity is known to attach to an event. + return {.kind = MountClaimResult::LiveDoubleStart, .body = body, .token = std::nullopt}; + emitMountEvent(sink, CasEventType::MountClaim, srid, "mint", nullptr, "fresh mount slot minted"); + return {.kind = MountClaimResult::Claimed, .body = body, .token = std::nullopt}; + } + + const MountLease existing = decodeMountLease(got->bytes); + + /// Foreign owner → fail closed regardless of expiry. (This runs after the owner gate, so a foreign + /// mount should not normally exist, but the lease must never be taken across UUIDs.) + if (existing.server_uuid != our_uuid) + { + emitMountEvent(sink, CasEventType::MountConflict, srid, "foreign_owner", &existing, + "mount slot is held by a foreign server_uuid — refusing to take over across identities"); + return {.kind = MountClaimResult::ForeignOwner, .body = existing, .token = std::nullopt}; + } + + /// Same uuid + same epoch: it is OUR OWN claim — but a FENCED body is terminal for this + /// (uuid, epoch): the GC dropped its ack from the floor when it fenced. Refreshing it in place + /// would resurrect a fenced incarnation — a fence permanently consumes this `(server_uuid, + /// writer_epoch)` pair, so the caller must re-open with a fresh `writer_epoch`. + if (existing.writer_epoch == our_epoch) + { + if (existing.gc_fenced) + { + emitMountEvent(sink, CasEventType::MountConflict, srid, "fenced_by_gc", &existing, + "own (uuid, epoch) mount slot is GC-fenced — terminal for this incarnation; " + "recover with a fresh writer_epoch"); + return {.kind = MountClaimResult::FencedSelf, .body = existing, .token = std::nullopt}; + } + const MountLease body = makeMountBody(our_uuid, our_epoch, existing.seq + 1, now_ms, ttl_ms); + const PutResult put = b.putOverwrite(key, encodeMountLease(body), got->token); + if (put.outcome != PutOutcome::Done) + /// The mount changed under us between get and putOverwrite: `got->token` is now KNOWN + /// STALE (that mismatch is exactly why the put failed), not merely unknown -- leaving + /// `.token` unset (rather than handing back a token the caller would wrongly treat as + /// current) is deliberate, matching the identical race below. + return {.kind = MountClaimResult::LiveDoubleStart, .body = body, .token = std::nullopt}; + emitMountEvent(sink, CasEventType::MountClaim, srid, "refresh", &existing, + "own claim replayed — refreshed seq + expiry"); + return {.kind = MountClaimResult::Claimed, .body = body, .token = std::nullopt}; + } + + /// Same uuid, DIFFERENT epoch: reclaim ONLY on a certificate of death that needs no fresh + /// wall-clock trust — never by comparing `expires_at_ms` against `now_ms`: + /// - `gc_fenced` → the fence-out is terminal for that incarnation by construction (its keeper's + /// every renewal fails the token guard forever, so it can never write again) — there is no + /// liveness left to wait for. This is what makes self-remount (and a fast restart after a + /// fence-out) instant instead of an observation wait. + /// - the clean marker (`min_active == UINT64_MAX`) → the predecessor's OWN graceful farewell + /// (`MountLeaseKeeper::terminate`) — no observation needed either. + /// - `proven_dead_token` matches the token we just read → the CALLER (`claimMountAwaitingExpiry`) + /// already watched this exact token hold stable for the full observation threshold on its own + /// clock; re-deriving that here from a bare wall-clock comparison would be exactly the + /// cross-node trust would make a clock-skewed or delayed observer unsafe. + /// Anything else → `LiveDoubleStart` (do NOT write): a same-uuid, different-epoch, not fenced, not + /// clean-marked, not (yet) proven-dead lease may simply be a live twin, and `expires_at_ms` alone + /// can never distinguish that from a dead predecessor across two different clocks. + const bool clean_marker = existing.min_active == std::numeric_limits::max(); + const bool proven_dead = proven_dead_token && *proven_dead_token == got->token; + if (existing.gc_fenced || clean_marker || proven_dead) + { + const MountLease body = makeMountBody(our_uuid, our_epoch, existing.seq + 1, now_ms, ttl_ms); + const PutResult put = b.putOverwrite(key, encodeMountLease(body), got->token); + if (put.outcome != PutOutcome::Done) + /// The mount changed under us between get and putOverwrite — someone else is racing the + /// reclaim. Fail closed. `got->token` is now KNOWN STALE (that mismatch is exactly why the + /// put failed) -- leaving `.token` unset is deliberate, not an oversight. + return {.kind = MountClaimResult::LiveDoubleStart, .body = body, .token = std::nullopt}; + const MountPriorState prior = existing.gc_fenced ? MountPriorState::Fenced + : clean_marker ? MountPriorState::Clean + : MountPriorState::UncleanObserved; + emitMountEvent(sink, CasEventType::MountClaim, srid, "reclaim", &existing, + existing.gc_fenced ? "same server_uuid, different writer_epoch, GC-fenced — reclaimed" + : clean_marker ? "same server_uuid, different writer_epoch, clean farewell — reclaimed" + : "same server_uuid, different writer_epoch, observed dead by " + "token-stability — reclaimed"); + return {.kind = MountClaimResult::Claimed, .body = body, .prior = prior, .token = std::nullopt}; + } + + emitMountEvent(sink, CasEventType::MountConflict, srid, "live_double_start", &existing, + "same server_uuid, different writer_epoch, not fenced/clean/proven-dead — no wall-clock trust; " + "the caller must run the token-stability observation wait before reclaiming"); + /// No write was attempted on this path -- `got->token` is exactly the CURRENT body's + /// token (what we just read is what's still there), so it is safe to hand back for the caller's + /// observation loop to compare across polls without a redundant re-GET. + return {.kind = MountClaimResult::LiveDoubleStart, .body = existing, .token = got->token}; +} + +String mountDoubleStartMessage(const String & srid, const MountLease & existing) +{ + return fmt::format( + "Content-addressed disk cannot start: server_root_id '{}' is actively mounted by another LIVE server.\n" + " Existing mount: server_uuid={} hostname={} pid={} last_seq={} expires_at_ms={}\n" + "This server already waited for the mount lease to lapse, but it kept being renewed — a second\n" + "server is holding the same CAS namespace. This prevents two ClickHouse servers from writing it.\n" + " - If the other server is running intentionally, configure a unique for this disk.\n" + " - If the other server is a stale/zombie process, stop it; this server will then reclaim the mount on restart.\n" + " - CLOCK SKEW CAVEAT: liveness is judged by comparing the lease's wall-clock expires_at_ms against\n" + " THIS server's clock, so a large clock skew between the two servers can misjudge it (a healthy holder\n" + " may look mounted here, or a dead one may look live). Verify both servers' clocks are in sync (NTP).\n" + " - If the local ClickHouse uuid file was regenerated, restore the old uuid file, or remove the stale\n" + " owner object gc/server-roots/{}/owner only after verifying no server uses this root.\n" + " - As a LAST RESORT, after verifying that NO server is writing this root, manually delete the mount\n" + " object gc/server-roots/{}/mount and restart; this server will then re-claim it.", + srid, u128ToHex(existing.server_uuid), existing.hostname, existing.pid, + existing.seq, existing.expires_at_ms, srid, srid); +} + +namespace +{ +/// Bounded number of observation restarts before giving up on a same-uuid slot whose write-token keeps +/// changing: each restart means the token changed DURING our observation window — i.e. something is +/// actively renewing it. A genuinely dead predecessor's token never changes again after its last +/// renewal, so it is observed stable well within one window; only a truly LIVE writer (a real second +/// incarnation, or the predecessor's own background renewer racing our first few polls) keeps resetting +/// the clock. Bounding this converts "wait forever for a live twin" into the same bounded-then-report +/// shape the old wall-clock wait had, without ever trusting a wall-clock deadline to get there. +constexpr size_t kMaxObservationRestarts = 3; +} + +uint64_t mountObservationThresholdMs(uint64_t ttl_ms, uint64_t cadence_ms) +{ + return ttl_ms + ttl_ms / 20 + cadence_ms; +} + +MountClaimResult claimMountAwaitingExpiry( + Backend & b, const Layout & l, const String & srid, UInt128 our_uuid, uint64_t our_epoch, + const std::function & now_ms_fn, + const std::function & mono_ms_fn, + uint64_t ttl_ms, uint64_t poll_interval_ms, + const std::function & sleep_ms_fn, + const std::function & on_wait_start, + const CasEventSink & sink) +{ + /// A zero poll interval would spin; a single-ms floor keeps the loop a real (bounded) wait. + const uint64_t poll = poll_interval_ms == 0 ? 1 : poll_interval_ms; + + /// Rate-bound observation threshold: the full lease TTL, plus a 5% allowance for clock-rate + /// mismatch between the holder's and our own local clock, plus one poll interval for observation + /// discreteness. It is measured only with OUR OWN clock (`mono_ms_fn`); no cross-node wall-clock + /// comparison participates in this loop. The shared helper keeps the startup and GC thresholds + /// identical. + const uint64_t threshold_ms = mountObservationThresholdMs(ttl_ms, poll); + + std::optional observed; + uint64_t observed_since = 0; + size_t restarts = 0; + + while (true) + { + const bool threshold_met = observed && mono_ms_fn() - observed_since >= threshold_ms; + MountClaimResult r = claimMount(b, l, srid, our_uuid, our_epoch, now_ms_fn(), ttl_ms, + threshold_met ? observed : std::nullopt, sink); + if (r.kind != MountClaimResult::LiveDoubleStart) + return r; + + /// `claimMount` already read the current body. Reuse `r.token` whenever `claimMount` + /// set it (the common case: no write was attempted, so what it read is still current) instead of + /// re-GETting the SAME key here. The rare stale-race branches deliberately leave `.token` unset + /// (see their own comments), so this still falls back to a fresh read exactly there. + std::optional current_token = r.token; + if (!current_token) + { + const auto got = b.get(l.mountKey(srid)); + if (!got) + { + /// The slot vanished between claimMount's own GET and ours — normally self-resolving + /// within one more `claimMount` call (which re-mints fresh on an absent slot), but under + /// slot churn (something else concurrently removing/re-minting it) that resolution could + /// keep losing the same race. Pace this like every other iteration and + /// count it toward the SAME bounded restart budget the token-churn case below uses, + /// instead of spinning `get`/`claimMount`/`put` at backend RTT with no sleep and no bound + /// — a persistently vanishing slot is exactly as "alive and contended" as a persistently + /// renewing token. + if (++restarts > kMaxObservationRestarts) + return r; + sleep_ms_fn(poll); + continue; + } + current_token = got->token; + } + + if (!observed || *observed != *current_token) + { + if (observed && ++restarts > kMaxObservationRestarts) + /// The token kept changing across bounded restarts — the holder is genuinely alive + /// (actively renewing), not a dead predecessor. Report it rather than waiting forever. + return r; + observed = *current_token; + observed_since = mono_ms_fn(); + if (on_wait_start) + on_wait_start(r.body, threshold_ms); + LOG_INFO(getLogger("CasMountLease"), + "Attempting to mount content-addressed server root {} after node change or hard " + "restart; waiting ~{} ms (token-stability observation) to confirm the previous " + "incarnation's operations are all finalized", srid, threshold_ms); + } + + sleep_ms_fn(poll); + } +} + +HeartbeatFloor computeHeartbeatFloor(Backend & b, const Layout & l, uint64_t now_ms, + uint64_t mono_now_ms, uint64_t stable_threshold_ms, + MountObservationMap & obs) +{ + HeartbeatFloor floor; + + /// `obs` is keyed by every srid this leader has EVER observed, but a + /// srid removed from the LIST entirely (its `/mount` key gone -- e.g. `SYSTEM CAS + /// DROP POOL MEMBER`) is never visited by the loop below again, so its entry would otherwise linger + /// forever (~150-250 B/srid, worse on a long-lived leader across many decommissions). Track every + /// srid actually seen THIS pass and prune anything else out of `obs` at the end -- disjoint from the + /// mid-loop `obs.erase(srid)` calls below (those fire for a srid seen but now terminal/fenced/gone + /// this pass; this is for a srid not seen AT ALL). + std::set seen_srids; + + const String prefix = l.serverRootsPrefix(); + String cursor; + while (true) + { + const ListPage page = b.list(prefix, cursor, /*limit*/ 1000); + for (const auto & listed : page.keys) + { + /// `/owner` and `/epoch` objects share the subtree — only mount bodies gate the floor. + static constexpr std::string_view mount_suffix = "/mount"; + if (!listed.key.ends_with(mount_suffix)) + continue; + + const String & key = listed.key; + + /// The srid is the path segment between `serverRootsPrefix()` and the `/mount` suffix + /// (`/gc/server-roots//mount`). Used both for observability (fenced) and as + /// the key into `obs`. + const String srid = key.substr(prefix.size(), + key.size() - prefix.size() - mount_suffix.size()); + seen_srids.insert(srid); + + /// Fence-out on PreconditionFailed re-GETs and reclassifies from the top; bound the retries + /// so a pathologically contended holder cannot spin forever. On exhaustion the entry is + /// counted as live (conservative — never excluded without a landed fence-out). + constexpr int max_reclassify = 4; + for (int attempt = 0; ; ++attempt) + { + const auto got = b.get(key); + if (!got) + { + obs.erase(srid); + break; /// Raced away (deleted) — nothing to classify. + } + + const MountLease m = decodeMountLease(got->bytes); + + if (m.gc_fenced) + { + ++floor.already_fenced; + obs.erase(srid); /// terminal — no further observation needed + break; + } + if (m.min_active == std::numeric_limits::max()) + { + ++floor.terminated; + obs.erase(srid); /// terminal — no further observation needed + break; + } + + /// Observation-based liveness: stable ONLY if the + /// SAME token was already being watched and has now held for the full threshold on our + /// OWN monotonic clock. Anything else — no prior observation, or a changed token (a + /// live renewal, including one raced against our own fence-out attempt below) — + /// (re)starts the observation window and counts as `live` this call. + const auto it = obs.find(srid); + const bool stable = it != obs.end() && it->second.token == got->token + && mono_now_ms - it->second.first_seen_mono_ms >= stable_threshold_ms; + + if (!stable) + { + if (it == obs.end() || it->second.token != got->token) + obs[srid] = MountTokenObservation{got->token, mono_now_ms}; + ++floor.live; + break; + } + + const bool exhausted = attempt >= max_reclassify; + if (exhausted) + { + ++floor.live; /// conservative — never exclude without a landed fence-out + break; + } + + /// Stable past the threshold, not yet fenced → token-guarded fence-out preserving the + /// whole body (gc_fenced = true, seq + 1). + MountLease fenced = m; + fenced.gc_fenced = true; + fenced.seq = m.seq + 1; + const PutResult res = b.putOverwrite(key, encodeMountLease(fenced), got->token); + if (res.outcome == PutOutcome::Done) + { + ++floor.fenced_now; + floor.fenced_srids.push_back(srid); + obs.erase(srid); + LOG_INFO(getLogger("CasHeartbeatFloor"), + "CAS GC fenced out mount lease for content-addressed server root {} at " + "wall-clock ms {}: its write token held unchanged for >= {} ms on the GC " + "leader's own monotonic clock (token-stability observation)", + srid, now_ms, stable_threshold_ms); + break; + } + /// PreconditionFailed: the holder renewed between our GET and PUT — re-GET and + /// reclassify (the observation check above will see the new token and restart it). + } + } + + if (page.next_cursor.empty()) + break; + cursor = page.next_cursor; + } + + /// Prune every `obs` entry for a srid this pass's LIST never saw at all. + for (auto it = obs.begin(); it != obs.end(); ) + it = seen_srids.contains(it->first) ? std::next(it) : obs.erase(it); + + return floor; +} + +std::vector probeNonTerminalMountSlots(Backend & b, const Layout & l) +{ + std::vector slots; + + /// Same enumeration as `computeHeartbeatFloor`'s gate -- LIST the server-roots subtree, keep the + /// `/mount` bodies -- but read-only and without any observation state: this answers "is anyone + /// still entitled to write here", not "may I fence them out". + const String prefix = l.serverRootsPrefix(); + String cursor; + while (true) + { + const ListPage page = b.list(prefix, cursor, /*limit*/ 1000); + for (const auto & listed : page.keys) + { + static constexpr std::string_view mount_suffix = "/mount"; + if (!listed.key.ends_with(mount_suffix)) + continue; /// `/owner` and `/epoch` share the subtree; only the lease says "live". + + const String srid = listed.key.substr(prefix.size(), + listed.key.size() - prefix.size() - mount_suffix.size()); + + const auto got = b.get(listed.key); + if (!got) + continue; /// raced away between LIST and GET -- there is no slot to be held. + + MountLease m; + try + { + m = decodeMountLease(got->bytes); + } + catch (...) + { + /// An undecodable lease is the WORST case for a recreation, not an ignorable one: it is + /// what a slot written by a format this build does not understand looks like, and the + /// holder of that slot is exactly the writer we must not run over. + slots.push_back(NonTerminalMountSlot{srid, fmt::format( + "mount lease could not be decoded by this build ({})", + getCurrentExceptionMessage(/*with_stacktrace=*/false))}); + continue; + } + + if (m.gc_fenced || m.min_active == std::numeric_limits::max()) + continue; /// terminal: fenced out by GC, or the holder's own graceful farewell. + + slots.push_back(NonTerminalMountSlot{srid, fmt::format( + "held by server uuid {} (writer_epoch {}, host '{}', pid {}, lease seq {}, stamped " + "expiry {} ms) with neither a graceful farewell nor a GC fence-out", + u128ToHex(m.server_uuid), m.writer_epoch, m.hostname, m.pid, m.seq, m.expires_at_ms)}); + } + + if (page.next_cursor.empty()) + break; + cursor = page.next_cursor; + } + + return slots; +} + +std::vector listMounts(Backend & backend, const Layout & layout, uint64_t now_ms, uint64_t skew_margin_ms) +{ + std::vector out; + const String prefix = layout.serverRootsPrefix(); + String cursor; + while (true) + { + const ListPage page = backend.list(prefix, cursor, 1000); + for (const auto & k : page.keys) + { + static constexpr std::string_view suffix = "/mount"; + if (!k.key.ends_with(suffix)) + continue; + const auto got = backend.get(k.key); + if (!got) + continue; /// raced a delete — read-only view, skip the row + MountInfo info; + /// The srid is the path segment between `serverRootsPrefix()` and the `/mount` suffix — + /// may itself contain `/` (e.g. `shard-01/replica-a`), so slice by prefix length rather + /// than `rfind('/')`, matching `computeHeartbeatFloor`'s extraction. + info.srid = k.key.substr(prefix.size(), k.key.size() - prefix.size() - suffix.size()); + try + { + info.lease = decodeMountLease(got->bytes); + } + catch (...) + { + info.state = "corrupt"; + out.push_back(std::move(info)); + continue; + } + if (info.lease.gc_fenced) + info.state = "fenced"; + else if (info.lease.min_active == std::numeric_limits::max()) + info.state = "terminated"; + else if (now_ms <= info.lease.expires_at_ms + skew_margin_ms) + info.state = "live"; + else + info.state = "expired"; + out.push_back(std::move(info)); + } + if (page.next_cursor.empty()) + break; + cursor = page.next_cursor; + } + return out; +} + +namespace +{ + +/// The three clock-free certificates `isCreatorFenceTerminal` recognises, plus `None` for a live body +/// that carries none of them -- see the function's header doc for what each one proves and why +/// `fence_generation` is not among them. +enum class FenceCertificate : uint8_t +{ + None, + GcFenced, + CleanFarewell, + SupersededEpoch, +}; + +FenceCertificate classifyFenceCertificate(const MountLease & lease, uint64_t fence_writer_epoch) +{ + if (lease.gc_fenced) + return FenceCertificate::GcFenced; + if (lease.min_active == std::numeric_limits::max()) + return FenceCertificate::CleanFarewell; + if (lease.writer_epoch != fence_writer_epoch) + return FenceCertificate::SupersededEpoch; + return FenceCertificate::None; +} + +} + +bool isCreatorFenceTerminal(Backend & backend, const Layout & layout, const String & server_root_id, + uint64_t writer_epoch) +{ + const auto got = backend.get(layout.mountKey(server_root_id)); + if (!got) + return false; /// absence proves nothing about liveness -- see the header doc + + MountLease lease; + try + { + lease = decodeMountLease(got->bytes); + } + catch (...) + { + return false; /// undecodable -- fail closed, never wave through + } + + /// EXHAUSTIVE switch, not a positive allowlist -- mirrors `CasPool.cpp`'s own exhaustive switch over + /// `MountPriorState` (`claimMount`, in this file, only PRODUCES that classification; the switch + /// consuming it lives in the caller) deliberately: a future `FenceCertificate` enumerator with no + /// verdict assigned here must fail the BUILD (a missing `-Wswitch` case), never silently read as + /// terminal (which would let a reconciler steal a namespace out from under a writer that might + /// still be alive) or as live (which would block a reconciliation the certificate already proves + /// is safe). + /// The initializer is dead: the exhaustive switch below assigns every enumerator, and a future + /// enumerator left unassigned fails the build via `-Wswitch`, not this value. + bool terminal = false; + switch (classifyFenceCertificate(lease, writer_epoch)) + { + case FenceCertificate::None: + terminal = false; + break; + case FenceCertificate::GcFenced: + case FenceCertificate::CleanFarewell: + case FenceCertificate::SupersededEpoch: + terminal = true; + break; + } + return terminal; +} + +MountLeaseKeeper::MountLeaseKeeper( + BackendPtr backend_, const Layout & layout_, const String & srid_, UInt128 server_uuid_, + uint64_t writer_epoch_, std::chrono::milliseconds ttl_, std::function now_ms_fn_, + std::function min_active_fn_, + CasEventSink event_sink_, + std::chrono::milliseconds lease_safety_margin_, + std::function boot_ms_fn_) + : SingleWriterSlot(std::move(backend_), layout_.mountKey(srid_), "mount-lease", "release", "CasMountLeaseKeeper") + , srid(srid_) + , server_uuid(server_uuid_) + , writer_epoch(writer_epoch_) + , ttl(ttl_) + , now_ms_fn(std::move(now_ms_fn_)) + , min_active_fn(std::move(min_active_fn_)) + , event_sink(std::move(event_sink_)) + , lease_safety_margin(lease_safety_margin_) + , boot_ms_fn(boot_ms_fn_ ? std::move(boot_ms_fn_) : defaultBootMs) +{ +} + +void MountLeaseKeeper::refreshConfirmedDeadline(uint64_t anchor_wall_ms) +{ + confirmed_deadline_ms = anchor_wall_ms + static_cast(ttl.count()); +} + +bool MountLeaseKeeper::shouldFenceOnTransientRenewFailure() +{ + /// Defensive: should never observe 0 here (see the member's doc comment) -- fail closed if it ever did. + if (confirmed_deadline_ms == 0) + return true; + const uint64_t now = now_ms_fn(); + const uint64_t margin = static_cast(lease_safety_margin.count()); + return now + margin >= confirmed_deadline_ms; +} + +SingleWriterSlot::RenewPayload MountLeaseKeeper::prepareRenew() const +{ + /// Carry the two dynamic fields (both read OFF the state lock — the merged floor callback reaches + /// into the Pool's own lock): `value` = wall-clock `now_ms` (so `encodeBody` stamps a fresh + /// `expires_at_ms = now_ms + ttl`), `value2` = `min_active` (the build-watermark floor). + /// Pre-I/O anchors (spec rev.4 Phase B): both fence deadlines anchor at this instant — the + /// wall stamp doubles as the payload's now_ms, so anchor <= the durable stamp trivially. + last_attempt_wall_ms = now_ms_fn(); + last_attempt_boot_ms = boot_ms_fn(); + return {.value = last_attempt_wall_ms, .value2 = min_active_fn()}; +} + +String MountLeaseKeeper::encodeBody(uint64_t seq_, const RenewPayload & payload) const +{ + const uint64_t now_ms = payload.value; + const uint64_t ttl_ms = static_cast(ttl.count()); + return encodeMountLease(MountLease{ + .server_uuid = server_uuid, + .writer_epoch = writer_epoch, + .hostname = getFQDNOrHostName(), + .pid = static_cast(::getpid()), + .started_at_ms = now_ms, + .seq = seq_, + .expires_at_ms = now_ms + ttl_ms, + .min_active = payload.value2, + }); +} + +SingleWriterSlot::Token MountLeaseKeeper::claim(const String & body) +{ + /// ADOPT-aware claim. The normal flow is `claimMount` wrote the live mount under + /// (server_uuid, writer_epoch); `start` then adopts that very slot. We must NOT self-trip the + /// live-double-start guard on our own (uuid, epoch). + const HeadResult head = backend->head(key); + if (!head.exists) + { + /// Absent → put it ourselves (a fresh start that ran without a prior claimMount, or a slot + /// that lapsed and was swept). putIfAbsent fails closed if it appears under us; that race has + /// no re-read (no observed body), so there is nothing to attach to a conflict event. + const PutResult res = backend->putIfAbsent(key, body); + if (res.outcome != PutOutcome::Done) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS mount-lease: key '{}' appeared between head and putIfAbsent — concurrent writer on our mount slot", key); + emitMountEvent(event_sink, CasEventType::MountClaim, srid, "mint", nullptr, + "mount slot absent — keeper minted it directly (no prior claimMount)"); + refreshConfirmedDeadline(last_attempt_wall_ms); + return res.token; + } + + /// Read the observed slot to decide adopt vs fail-closed by the (uuid, epoch) discriminator. + const auto got = backend->get(key); + if (!got) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS mount-lease: key '{}' vanished between head and get while claiming", key); + const MountLease observed = decodeMountLease(got->bytes); + + /// Foreign uuid → fail closed (no cross-UUID takeover, ever). The audit payload: the CURRENT decoded + /// body's identity is exactly WHO touched the slot. + if (observed.server_uuid != server_uuid) + { + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "adopt", &observed, + "mount slot is held by a foreign server — failing closed, never taking over"); + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS mount-lease: key '{}' is held by a foreign server ({}) — failing closed, never taking over", + key, describeMountHolder(observed)); + } + + /// Same uuid but a DIFFERENT epoch → a newer incarnation superseded us (or a concurrent + /// double-start). Fail closed. + if (observed.writer_epoch != writer_epoch) + { + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "adopt", &observed, + fmt::format("mount slot is held by a different writer_epoch ({} != ours {}) — superseded, failing closed", + observed.writer_epoch, writer_epoch)); + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS mount-lease: key '{}' is held by a different writer_epoch ({} != ours {}) — superseded, failing closed ({})", + key, observed.writer_epoch, writer_epoch, describeMountHolder(observed)); + } + + /// Same (uuid, epoch) but FENCED: the GC fenced our fresh lease before we adopted it (the + /// lease expired mid-open — e.g. a slow first beat). Terminal for THIS epoch; the open path + /// recovers by allocating a fresh `writer_epoch` and re-claiming. + if (observed.gc_fenced) + { + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "fenced_by_gc", &observed, + "own mount slot fenced by GC after lease expiry — recoverable with a fresh writer_epoch"); + throw MountFencedException(fmt::format( + "CAS mount-lease: key '{}' was fenced by GC after lease expiry ({}) — " + "recoverable: re-open with a fresh writer_epoch", key, describeMountHolder(observed))); + } + + /// Same uuid AND same epoch → it is OUR OWN claim → ADOPT: overwrite against the observed token + /// to refresh seq/expiry. (`body` is encoded for seq=1 by the base `doStart`; that is fine — + /// renewals advance from there.) + const PutResult res = backend->putOverwrite(key, body, got->token); + if (res.outcome != PutOutcome::Done) + { + /// The slot moved between our GET and PUT. Diagnose by the CURRENT body, not the token + /// The current body is the useful diagnostic: a GC fence is the only same-(uuid, epoch)-preserving + /// touch that can normally occur during adoption. + const auto reread = backend->get(key); + if (reread) + { + const MountLease current = decodeMountLease(reread->bytes); + if (current.server_uuid == server_uuid && current.gc_fenced) + { + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "fenced_by_gc", ¤t, + "GC fenced our mount between the adopt's read and write — recoverable with a " + "fresh writer_epoch"); + throw MountFencedException(fmt::format( + "CAS mount-lease: key '{}' was fenced by GC inside the adopt window ({}) — " + "recoverable: re-open with a fresh writer_epoch", key, describeMountHolder(current))); + } + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "adopt", ¤t, + "mount slot was touched while adopting our own mount slot — failing closed"); + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS mount-lease: key '{}' was touched while adopting our own mount slot ({}) — failing closed", + key, describeMountHolder(current)); + } + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS mount-lease: key '{}' vanished while adopting our own mount slot — failing closed", key); + } + emitMountEvent(event_sink, CasEventType::MountClaim, srid, "adopt", &observed, + "adopted our own already-live (uuid, epoch) mount slot"); + refreshConfirmedDeadline(last_attempt_wall_ms); + return res.token; +} + +void MountLeaseKeeper::onRenewCommitted() +{ + /// Anchor at the attempt start (stashed by prepareRenew), never at this ack instant — a slow + /// ack must not extend either fence past what the durable body it acknowledges authorizes + /// (spec rev.4 Phase B). Runs for EVERY successful `renewOnce` — background-driven or a direct + /// caller (e.g. the startup-arm redo in `CasPool.cpp`'s `mountWritable`) — so the wall deadline + /// never goes stale just because a renewal happened to be invoked outside the background loop. + refreshConfirmedDeadline(last_attempt_wall_ms); +} + +void MountLeaseKeeper::onRenewSucceeded() +{ + /// `confirmed_deadline_ms` was already refreshed by `onRenewCommitted` above, which `renewOnce` + /// (base) calls right after recording the successful write — before the background loop reaches + /// this hook. This hook fires only the boot-domain write-fence callback. + if (on_renew_ok) + on_renew_ok(last_attempt_boot_ms); +} + +void MountLeaseKeeper::onRenewFailed() +{ + /// This is THE point at which this runtime stops believing it owns the mount, so it is where the + /// release path's arm-A/arm-B split is decided (see `terminate`). Set BEFORE the fence callback, so + /// a teardown racing this observation reads the conservative value. + deposition_observed.store(true, std::memory_order_release); + /// Background renewal failed: `renewOnce` threw and the loop is stopping. Latch the local write + /// fence to lost so no further mutation proceeds — fail closed. The mismatch itself was already + /// classified and emitted by `onRenewMismatch` (fenced_by_gc / same_epoch_state_uncertain / + /// superseded / foreign_writer / vanished) just before this throw propagated here — this event is + /// only the fence-latch timeline marker. + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "renew_failed", nullptr, + "renew mismatch — see the preceding classified mount_conflict event; write fence latched to lost"); + if (on_lost) + on_lost(); +} + +void MountLeaseKeeper::onRenewMismatch(const String & mismatched_key) +{ + /// The base contract's PreconditionFailed just means "our token didn't match" — re-read the + /// CURRENT body and classify. All four body-present cases and the absent case are covered + /// below, each fail-closed and NONE constructing a `LOGICAL_ERROR`, which aborts debug/ASan + /// builds at exception construction — on this KEEPER THREAD, taking the whole process with it + /// (STID 3982-3b48; parts 1a/1b covered vanished/absent-at-release, this covers the rest). + const auto got = backend->get(mismatched_key); + if (got) + { + const MountLease current = decodeMountLease(got->bytes); + + if (current.server_uuid == server_uuid && current.gc_fenced) + { + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "fenced_by_gc", ¤t, + "own mount slot fenced by GC after lease expiry (late renewal) — recoverable with a " + "fresh writer_epoch"); + throw MountFencedException(fmt::format( + "CAS mount-lease: key '{}' was fenced by GC after lease expiry (late renewal) ({}) — " + "recoverable: re-open with a fresh writer_epoch", mismatched_key, describeMountHolder(current))); + } + + if (current.server_uuid == server_uuid && current.writer_epoch == writer_epoch && !current.gc_fenced) + { + /// The slot advanced past our held token under our OWN (uuid, epoch), unfenced. This is + /// state UNCERTAINTY, not proof of anything (spec rev.4): the common cause is our own + /// earlier renewal PUT that landed while its ack was lost to a client-side timeout; the + /// pathological one is a same-pair twin after durable epoch-state loss (narrowed by the + /// allocateWriterEpoch re-mint guard). Both recover identically and fail closed: stop + /// renewing, latch the write fence, self-remount under a fresh writer_epoch. Never a + /// LOGICAL_ERROR — this shape is reachable by an ordinary network timeout. + ProfileEvents::increment(ProfileEvents::CASMountLeaseLost); + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "same_epoch_state_uncertain", ¤t, + "own mount slot advanced past our held token under our own (uuid, epoch) — state " + "uncertain (ambiguous prior renewal or epoch-state loss); fencing and self-remounting"); + throw Exception(ErrorCodes::ABORTED, + "CAS mount-lease: key '{}' advanced past our held token under our own (uuid, epoch) — " + "state uncertain; fencing and recovering via self-remount (observed {} vs our seq={})", + mismatched_key, describeMountHolder(current), seq); + } + + if (current.server_uuid == server_uuid && current.writer_epoch != writer_epoch) + { + ProfileEvents::increment(ProfileEvents::CASMountLeaseLost); + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "superseded", ¤t, + "own mount slot is held by a different writer_epoch — superseded by a newer incarnation"); + /// A normal fencing outcome (the model's localLost), not a programming assertion: + /// a suspended predecessor legitimately resumes into this after a successor reclaimed. + throw Exception(ErrorCodes::ABORTED, + "CAS mount-lease: key '{}' was superseded by a newer incarnation ({}) — fencing " + "(this incarnation is deposed; recovery is a fresh-epoch self-remount)", + mismatched_key, describeMountHolder(current)); + } + + /// `current.server_uuid != server_uuid` — a DIFFERENT server holds the slot we thought was ours. + /// The owner anchor refuses foreign claims at open and decommission impersonates the victim uuid + /// rather than manufacturing a foreign one, so this is not something a healthy protocol run + /// produces. It is still ENVIRONMENT-REACHABLE, and this comment used to claim otherwise: clear + /// the pool prefix and recreate under a different server id — an operator `rm -rf`, or a + /// recreation over a reused prefix — and the surviving writer's very next renewal lands exactly + /// here. `CasRefContiguousAlloc.SurvivingWriterIsFencedByTheRecreatedPoolsMount` drives it + /// deliberately, which is the plainest possible refutation of "unreachable". + /// + /// So it must not be a `LOGICAL_ERROR`. That class aborts debug/ASan builds at CONSTRUCTION, and + /// this runs on the keeper's background thread, so a condition the environment can create took + /// the whole process down. `ABORTED` instead — the same class the two sibling fencing arms above + /// use, and one the storage layer already treats as a retry-safe mount-lost signal. The OUTCOME + /// is unchanged and is the whole point: renewal stops, `onRenewFailed` latches the write fence, + /// and this incarnation never takes over the foreign holder's slot. + /// + /// NOT yet done at the RELEASE path (`terminate` below): its foreign-incarnation arm has the + /// same defect and is reached by the same test at teardown, but three `EXPECT_DEATH` tests + /// (`CasGcRound.OrphanManifestCursorSweepDeletesAndPersistsCursor`, + /// `CasMountStartup.StaleSelfMountReclaimedAfterWait`, + /// `CasPoolRemount.ForeignOwnerIsNeverTakenOver`) deliberately pin that abort, so changing it is + /// a ruled decision rather than a local fix. + ProfileEvents::increment(ProfileEvents::CASMountLeaseLost); + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "foreign_writer", ¤t, + "mount slot is held by a foreign server — failing closed, never taking over"); + throw Exception(ErrorCodes::ABORTED, + "CAS mount-lease: key '{}' is held by a foreign server ({}) — failing closed, never taking over", + mismatched_key, describeMountHolder(current)); + } + + /// The mount slot object VANISHED (backing store deleted under a live mount -- e.g. an + /// operator or test rm -rf'd the pool dir). This is an ENVIRONMENTAL condition, not a logic + /// error: there is no foreign writer to fail closed against. Stop renewing (fail-closed: the + /// write fence latches to lost, we never re-mint) WITHOUT aborting the server -- + /// LOGICAL_ERROR here aborts debug/ASan builds at exception construction. + ProfileEvents::increment(ProfileEvents::CASMountLeaseLost); + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "vanished", nullptr, + "mount slot object vanished (backing store deleted under a live mount) — stopping renewal, fail-closed"); + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "CAS mount-lease: key '{}' vanished (backing store deleted under a live mount) — " + "stopping renewal, fail-closed (never re-minting)", mismatched_key); + + /// NOTE: the pre-rev.4 trailing `SingleWriterSlot::onRenewMismatch(mismatched_key)` call is + /// GONE — the five cases above are exhaustive for this keeper (body-present × {fenced, + /// same-pair-unfenced, superseded, foreign} + absent), so the base class's generic + /// LOGICAL_ERROR is unreachable here. The base implementation stays for other slot subclasses. +} + +void MountLeaseKeeper::terminate() +{ + /// Terminal op: retire the lease by stamping it already-expired (expires_at_ms = started_at_ms), + /// seq+1, against the token we hold. This makes a same-uuid reopen immediately reclaimable. The + /// merged watermark farewell folds in HERE: `min_active = UINT64_MAX` is the retired sentinel the + /// GC floor treats as "every build_seq of this server is retired" — one release retires both the + /// mount lease and the build watermark. + const uint64_t now_ms = now_ms_fn(); + const String body = encodeMountLease(MountLease{ + .server_uuid = server_uuid, + .writer_epoch = writer_epoch, + .hostname = getFQDNOrHostName(), + .pid = static_cast(::getpid()), + .started_at_ms = now_ms, + .seq = seq + 1, + .expires_at_ms = now_ms, + .min_active = std::numeric_limits::max(), + }); + const PutResult res = backend->putOverwrite(key, body, last_token); + if (res.outcome != PutOutcome::Done) + { + /// A foreign incarnation on OUR release path has one clean cause: GC fenced this mount out + /// after its lease expired (the `gc_fenced` stamp). The slot is already released-by-fence and + /// there is nothing left to retire. + if (const auto got = backend->get(key)) + { + const MountLease current = decodeMountLease(got->bytes); + if (current.gc_fenced) + { + LOG_INFO(getLogger("CasMountLeaseKeeper"), + "CAS mount-lease: '{}' was fenced out by GC (expired lease); release is a no-op", key); + return; + } + + /// Everything else used to be one arm raising `LOGICAL_ERROR` — "the world is broken". It + /// is TWO situations, they mean opposite things, and neither may abort: this runs from + /// `~Pool` via `finishTeardown`, whose `catch` a `LOGICAL_ERROR` defeats by aborting at + /// CONSTRUCTION, so an ordinary failover took the process down. + /// + /// ARM A — this runtime has already stopped believing it owns the mount (renewal failed + /// and the write fence latched). A foreign occupant is then the EXPECTED end state of + /// failover: our successor owns the slot, and the farewell we were about to write would + /// stamp OUR identity over THEIRS. Skip it, leave the slot byte-for-byte untouched, and let + /// teardown finish quietly. Reached whenever a deposed writer shuts down. + /// + /// Nothing here is ABORT-CAPABLE, which is the property that matters on a destructor path — + /// not "nothing throws". `finishTeardown` wraps this call in a `catch` and logs, so a throw + /// is contained; what a `LOGICAL_ERROR` did instead was abort at CONSTRUCTION, before that + /// catch could ever run. This arm happens not to throw at all, but it is the exception CLASS + /// discipline, not the absence of a `throw`, that keeps teardown alive. + if (deposition_observed.load(std::memory_order_acquire)) + { + ProfileEvents::increment(ProfileEvents::CASMountReleaseSkippedForeignOccupant); + emitMountEvent(event_sink, CasEventType::MountRelease, srid, "deposed_foreign_occupant", ¤t, + "mount slot is held by our successor and this incarnation was already deposed — " + "skipping the farewell, slot left untouched"); + LOG_WARNING(getLogger("CasMountLeaseKeeper"), + "CAS mount-lease: '{}' is held by {} and this incarnation was already deposed — skipping " + "the farewell rather than stamping our identity over the successor's; release is a no-op", + key, describeMountHolder(current)); + return; + } + + /// ARM B — we never observed a deposition, so this runtime still believed it owned the mount + /// and a DIFFERENT one is in the slot. That is the single-writer guarantee broken, and it + /// stays maximally loud: named identities on both sides, its own counter, the write fence + /// latched so the runtime stops trusting itself, and NO write (the occupant is left exactly + /// as found — we do not retire someone else's lease). + /// + /// Loud, but still not abort-capable. Logical errors are exceptions here, not crashes, and + /// this verdict rests on a READ of the slot: a stale or adversarial backend can fabricate it + /// from the environment, which is precisely the input class that must never be able to kill + /// the server. There is deliberately no `chassert` either — it would abort exactly the + /// debug/ASan runs of the tests that now have to prove teardown SURVIVES this. + ProfileEvents::increment(ProfileEvents::CASMountExclusivityViolation); + emitMountEvent(event_sink, CasEventType::MountConflict, srid, "exclusivity_violation", ¤t, + "mount slot is held by a foreign incarnation although this runtime never observed a " + "deposition — single-writer exclusivity is broken; refusing the release and fencing"); + LOG_ERROR(getLogger("CasMountLeaseKeeper"), + "CAS mount-lease: release of key '{}' found a FOREIGN incarnation ({}) while this runtime " + "(server_uuid={} writer_epoch={} seq={}) still believed it owned the mount — single-writer " + "exclusivity is broken. Refusing to retire another incarnation's lease; the slot is left " + "untouched and this runtime's write fence is latched.", + key, describeMountHolder(current), u128ToHex(server_uuid), writer_epoch, seq); + if (on_lost) + on_lost(); + throw Exception(ErrorCodes::ABORTED, + "CAS mount-lease: release of key '{}' found a foreign incarnation ({}) while this runtime " + "still believed it owned the mount — single-writer exclusivity is broken; the slot is left " + "untouched and this runtime is fenced", key, describeMountHolder(current)); + } + /// The lease object is ABSENT: the backing store was deleted under us (rm -rf of the pool + /// dir -- the same environmental condition the renewal path classifies as "vanished"). + /// The desired end state of a release is "no live lease object", which is already true, so + /// this is a clean no-op release, never a LOGICAL_ERROR (which aborts debug/ASan builds). + ProfileEvents::increment(ProfileEvents::CASMountLeaseLost); + emitMountEvent(event_sink, CasEventType::MountRelease, srid, "vanished", nullptr, + "mount slot object already gone at release (backing store deleted) — no-op release"); + LOG_INFO(getLogger("CasMountLeaseKeeper"), + "CAS mount-lease: '{}' is already gone at release (backing store deleted); release is a no-op", key); + return; + } + emitMountEvent(event_sink, CasEventType::MountRelease, srid, "farewell", nullptr, + "graceful release — lease stamped already-expired, watermark farewell folded in"); + recordWrite(seq + 1, res.token); +} + +SingleWriterSlot::SingleWriterSlot( + BackendPtr backend_, String key_, std::string_view slot_name_, std::string_view terminal_verb_, + std::string_view logger_name_) + : backend(std::move(backend_)) + , key(std::move(key_)) + , slot_name(slot_name_) + , terminal_verb(terminal_verb_) + , log(getLogger(String(logger_name_))) +{ +} + +SingleWriterSlot::~SingleWriterSlot() +{ + /// Stop the renewal thread only — deliberately NO terminal op. Destruction without a terminal op + /// leaves the slot object persisted with a frozen seq, which full GC observes as stale state. + stopBackground(); +} + +void SingleWriterSlot::recordWrite(uint64_t new_seq, const Token & token) +{ + seq = new_seq; + last_token = token; +} + +void SingleWriterSlot::doStart() +{ + /// Compute the per-call payload BEFORE taking state_mutex: a subclass callback (the watermark's + /// min_active hook) may reach into the Pool's own lock, so we never hold state_mutex across it. + const RenewPayload payload = prepareRenew(); + + std::lock_guard lock(state_mutex); + if (dead) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS {}: start after {} on key '{}'", slot_name, terminal_verb, key); + if (seq != 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS {}: already started on key '{}'", slot_name, key); + + const String body = encodeBody(/*seq=*/1, payload); + const Token token = claim(body); + recordWrite(/*new_seq=*/1, token); +} + +void SingleWriterSlot::renewOnce() +{ + /// Compute the per-call payload BEFORE taking state_mutex (see doStart): never hold state_mutex + /// across the subclass callback. + const RenewPayload payload = prepareRenew(); + + /// INVARIANT: holding `state_mutex` across the PUT below is safe ONLY because (a) doTerminate + /// joins the renewal thread before taking this mutex and (b) renewOnce has a single driver. + /// Do NOT add new `state_mutex`-guarded accessors without revisiting this (they would stall for + /// a full network round trip); the prepareRenew pattern above shows the lock-free alternative. + std::lock_guard lock(state_mutex); + /// Reset BEFORE the guards below: a `dead`/`seq==0` throw (a programming-bug guard, not a backend + /// outcome) must not be misread as a CONFIRMED mismatch by `backgroundLoop` -- it falls into the + /// TRANSIENT bucket by leaving this false, exactly like a `putOverwrite` exception below. + last_renew_failure_was_confirmed_mismatch = false; + if (dead) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS {}: renew after {} on key '{}'", slot_name, terminal_verb, key); + if (seq == 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS {}: renew before start on key '{}'", slot_name, key); + + const String body = encodeBody(seq + 1, payload); + const PutResult res = backend->putOverwrite(key, body, last_token); + if (res.outcome != PutOutcome::Done) + { + /// The PUT completed and observed a foreign token -- a CONFIRMED mismatch (proven + /// supersession), not a transient failure. Mark it BEFORE calling the hook, which always throws. + last_renew_failure_was_confirmed_mismatch = true; + onRenewMismatch(key); + } + + recordWrite(seq + 1, res.token); + /// Reached only on success (the branch above always throws on a mismatch). Notify the subclass + /// EVERY successful renewal is committed — background-driven (`backgroundLoop`'s own call) or a + /// direct caller (a redo site invoking `renewOnce` outright) alike; the mount-lease keeper + /// refreshes its confirmed-lease wall deadline here regardless of who called us. + onRenewCommitted(); +} + +void SingleWriterSlot::onRenewMismatch(const String & mismatched_key) +{ + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS {}: key '{}' was touched by a foreign writer — failing closed, never re-minting", slot_name, mismatched_key); +} + +void SingleWriterSlot::doTerminate() +{ + /// Join the renewal thread before taking the state lock, so no renewal races the terminal op. + stopBackground(); + + std::lock_guard lock(state_mutex); + if (seq == 0) + /// Never started (e.g. `Pool::open` failed before/inside `doStart`) — nothing was claimed, + /// so there is nothing to release. A never-started slot is inert: BOTH/ALL terminate calls on + /// it are quiet no-ops, and — unlike the genuinely-started path below — we do NOT set `dead`, + /// so a second no-op call takes this same early-return rather than tripping the "double + /// terminate" throw below. Throwing here only turned an already-failing teardown into extra + /// `LOGICAL_ERROR` noise during teardown. + return; + if (dead) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS {}: double {} on key '{}'", slot_name, terminal_verb, key); + + /// Dead regardless of what terminate does below: we attempted the terminal op, the keeper must + /// never renew this key again. + dead = true; + terminate(); +} + +void SingleWriterSlot::startBackground(std::chrono::milliseconds period) +{ + /// After a thread-side renewal failure the loop returns (see backgroundLoop) but the thread + /// handle stays joinable, so a subsequent startBackground throws "already running" until + /// stopBackground is called. Intentional fail-closed: we never silently re-arm renewal after it + /// has failed. + std::lock_guard lock(background_mutex); + if (thread.joinable()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS {}: background renewal is already running for key '{}'", slot_name, key); + stop_requested = false; + thread = ThreadFromGlobalPool([this, period] { backgroundLoop(period); }); +} + +void SingleWriterSlot::stopBackground() +{ + ThreadFromGlobalPool to_join; + { + std::lock_guard lock(background_mutex); + if (!thread.joinable()) + return; + stop_requested = true; + wakeup.notify_all(); + to_join = std::move(thread); + } + to_join.join(); +} + +void SingleWriterSlot::backgroundLoop(std::chrono::milliseconds period) +{ + setThreadName(ThreadName::CAS_LEASE_KEEPER); + /// A CONFIRMED mismatch, or a TRANSIENT failure once `shouldFenceOnTransientRenewFailure` says the + /// lease deadline has neared, stops the loop for good: the slot's seq stops + /// advancing and GC observes the frozen seq. No retry, no re-mint. A TRANSIENT failure while the + /// deadline is still safely away keeps the loop alive -- the mount-lease protocol guarantees no + /// other writer can claim the slot before that deadline, so retrying is safe. + std::unique_lock lock(background_mutex); + while (!stop_requested) + { + if (wakeup.wait_for(lock, period, [this] { return stop_requested; })) + break; + + lock.unlock(); + try + { + renewOnce(); + } + catch (...) + { + /// `renewOnce` and this loop run on the SAME background thread, sequentially -- no + /// synchronization needed to read the flag it just set. + const bool confirmed = last_renew_failure_was_confirmed_mismatch; + if (!confirmed && !shouldFenceOnTransientRenewFailure()) + { + tryLogCurrentException(log, fmt::format( + "CAS {}: background renewal failed transiently, retrying while the lease is still valid", + slot_name)); + lock.lock(); + continue; + } + + tryLogCurrentException( + log, fmt::format("CAS {}: background renewal failed, the {} stops advancing", slot_name, slot_name)); + /// Notify the subclass that renewal failed and the loop is stopping (off `state_mutex`). + /// The mount-lease keeper latches its local write fence to lost here. Never let the hook's + /// own throw escape the loop — we are already stopping. + try + { + onRenewFailed(); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// The renewal loop is already stopping; a hook exception must not escape it. + } + return; + } + /// Successful renewal: notify the subclass (off `state_mutex`) before sleeping again. The + /// wall deadline was already refreshed inside `renewOnce` (`onRenewCommitted`); the + /// mount-lease keeper fires the boot-domain write-fence callback here. + try + { + onRenewSucceeded(); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// A notification hook cannot be allowed to stop the already-renewed lease loop. + } + lock.lock(); + } +} + +void sweepOwnMountStaging(IObjectStorage & object_storage, const String & mount_staging_prefix) noexcept +{ + try + { + /// max_keys=0 asks `listObjects` for the FULL listing under the prefix (it paginates until + /// exhausted rather than capping at some default page size) — see `IObjectStorage::listObjects`. + /// A mount's own staging debris is bounded (one mount's in-flight + leaked uploads), so a single + /// unbounded LIST at startup is acceptable; unlike GC's per-round budgets, this runs once per + /// mount, not on a recurring schedule. + RelativePathsWithMetadata children; + object_storage.listObjects(mount_staging_prefix, children, /*max_keys=*/0); + + size_t removed = 0; + for (const auto & child : children) + { + try + { + object_storage.removeObjectIfExists(StoredObject(child->relative_path)); + ++removed; + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// Best-effort: one stubborn key must not abort the sweep of the rest — it is retried + /// by a later mount's sweep. + } + } + + if (removed) + LOG_INFO(getLogger("CasStagingSweeper"), + "Reclaimed {} leaked S3 staging object(s) under '{}' at mount start", + removed, mount_staging_prefix); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// Best-effort: a LIST failure (a transient backend hiccup) at mount time must never fail the + /// mount — any leaked staging objects are bounded debris, reclaimed by a later mount's sweep. + } +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.h new file mode 100644 index 000000000000..051f6469cde3 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.h @@ -0,0 +1,774 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int ABORTED; +} +} + +namespace DB::Cas +{ + +/// Durable single-writer-slot state machine used by the per-server merged heartbeat +/// (`CasServerRoot.h`, `MountLeaseKeeper`): anchors a key that has EXACTLY ONE writer, renews it +/// asynchronously off the write path, and ends with a terminal op; any precondition failure during +/// renewal means a foreign touch — fail closed with an exception, never re-mint. +/// +/// This base owns the common machinery (the `seq`/`last_token`/`dead` state, the renewal thread and +/// its loop, `renewOnce`/`startBackground`/`stopBackground`, and the start/terminal +/// bookkeeping). The noun ("watermark") and verb ("farewell") used in every fail-closed message are +/// passed to the base constructor. Subclasses differ ONLY in EXPLICIT policy hooks: +/// - `prepareRenew` — runs OFF the state lock before each body encode, returning a per-call +/// payload (the watermark reads `min_active` from a Pool callback here). +/// Keeping it off `state_mutex` is load-bearing: the watermark callback +/// reaches into the Pool's own lock. +/// - `encodeBody` — builds the slot's exact JSON bytes for a given `seq` and payload; +/// - `claim` — the slot-specific anchor sequence in `start` (the watermark's +/// head→putIfAbsent/putOverwrite dance), returning the token we now hold; +/// - `terminate` — the terminal op (the watermark's retiring putOverwrite), run under the +/// state lock after `dead` is set, owning its own fail-closed throws and +/// final bookkeeping. +/// +/// Every observable op (the JSON body bytes, the anchor put sequence, the renew cadence, the +/// foreign-touch fail-close conditions and message wording, the stop semantics) is reproduced +/// exactly by the subclass hooks, with no conditionals in the base that would blur the +/// single-writer/fail-closed contract. +class SingleWriterSlot +{ +public: + /// `slot_name_` is the noun in every message ("watermark"); `terminal_verb_` is the + /// verb used in the start/renew/terminal guards ("farewell"). + SingleWriterSlot( + BackendPtr backend_, String key_, std::string_view slot_name_, std::string_view terminal_verb_, + std::string_view logger_name_); + + /// Stops the background thread only (no terminal op). Destruction without a terminal op leaves + /// the slot persisted, with its seq no longer advancing, so full GC observes frozen state. + virtual ~SingleWriterSlot(); + + /// seq++ via putOverwrite against the last token we wrote; LOGICAL_ERROR on a foreign touch + /// (single-writer fail-closed contract). + void renewOnce(); + + /// Starts periodic renewal. The period controls only the wake-up cadence; a failed renewal stops + /// the loop and is not silently re-armed by a later call. + void startBackground(std::chrono::milliseconds period); + + /// Requests renewal-thread termination and joins it. Safe to call repeatedly, including after a + /// renewal failure or from destruction; it does not perform the slot's terminal operation. + void stopBackground(); + +protected: + /// Per-call payload prepared OFF the state lock and handed to `encodeBody`. Subclasses needing + /// dynamic values (the mount lease's `now_ms` + merged `min_active`) carry them through this opaque + /// token. `value2` is a second scalar for slots that renew more than one dynamic field per beat (the + /// merged heartbeat: `value` = now_ms, `value2` = min_active). + struct RenewPayload + { + uint64_t value = 0; + uint64_t value2 = 0; + }; + + using Token = ::DB::Cas::Token; + + /// === policy hooks (see class comment) === + virtual RenewPayload prepareRenew() const = 0; + virtual String encodeBody(uint64_t seq, const RenewPayload & payload) const = 0; + virtual Token claim(const String & body) = 0; + + /// === optional background-renewal observation hooks (default no-op) === + /// Called from `renewOnce` itself, right after a SUCCESSFUL write is recorded (under + /// `state_mutex`, on EVERY caller of `renewOnce` — the background loop AND any direct caller, + /// such as the startup-arm redo in `CasPool.cpp`'s `mountWritable`). The mount-lease keeper + /// refreshes its confirmed-lease wall deadline here, so a direct `renewOnce` (which never goes + /// through `onRenewSucceeded` below) still keeps that deadline current. The watermark keeper does + /// not override it (no-op), so its behavior is unchanged. + virtual void onRenewCommitted() {} + /// Called from the background loop after a SUCCESSFUL `renewOnce` (off `state_mutex`, AFTER + /// `onRenewCommitted` has already run); the mount-lease keeper fires the boot-domain + /// write-fence callback (`on_renew_ok`) here — this hook is background-loop-only, unlike + /// `onRenewCommitted` above. The watermark keeper does not override it (no-op), so its behavior + /// is unchanged. + virtual void onRenewSucceeded() {} + /// Called from the background loop when `renewOnce` THREW (the loop is about to stop, off + /// `state_mutex`); the mount-lease keeper latches the write fence to lost here. Default no-op. + virtual void onRenewFailed() {} + + /// Called from the background loop when `renewOnce` threw and the + /// throw was NOT a confirmed mismatch (`onRenewMismatch` -- an observed PUT outcome proving + /// supersession; see the flag this checks, `last_renew_failure_was_confirmed_mismatch`, in the + /// private section below). Returning `true` means "treat as terminal now" -- fence immediately, the + /// legacy behavior and the correct default for a subclass with no lease-deadline concept. + /// `MountLeaseKeeper` overrides this to ride out a TRANSIENT exception (a `putOverwrite` that threw + /// before any outcome was observed -- a timeout, 5xx, or connection reset) while its last CONFIRMED + /// lease has not yet reached its safety-margin boundary: the mount-lease protocol guarantees no + /// other writer can claim the slot before that deadline, so continuing to retry (not fencing) is + /// safe. Called OFF `state_mutex`, same as `onRenewFailed`. + virtual bool shouldFenceOnTransientRenewFailure() { return true; } + + /// Called when the token-guarded renew PUT hits PreconditionFailed. The base contract stays + /// fail-closed and LOUD; subclasses may re-read and throw a more precisely classified + /// exception (the mount keeper distinguishes a GC fence of our own expired lease from a + /// genuine foreign writer). MUST throw — a renew mismatch never continues. + virtual void onRenewMismatch(const String & mismatched_key); + + /// Runs the slot's terminal op against the held `last_token`. Called under `state_mutex` with + /// `dead` already set (so renewal can never race it). Owns its own fail-closed throws and final + /// bookkeeping (the watermark bumps seq/last_token on its retiring putOverwrite). + /// May throw — `dead` stays set regardless. + virtual void terminate() = 0; + + /// Anchors the slot for seq=1 — durable when `doStart` returns. Subclasses expose this under their + /// own public name (`start`). Computes the payload off the lock, then runs the policy `claim`. + void doStart(); + + /// Stops the background thread, takes the state lock, runs the dead/seq guards (e.g. + /// "double farewell" / "discard before start"), sets `dead`, and delegates the op to `terminate`. + /// Subclasses expose this under their own public name (`farewell`/`discard`). + void doTerminate(); + + /// Bookkeeping after a successful write of the given seq/token: records seq, the token we now + /// hold, and the local-clock renew time. Must be called under `state_mutex`. + void recordWrite(uint64_t new_seq, const Token & token); + + BackendPtr backend; + String key; + + mutable std::mutex state_mutex; + uint64_t seq = 0; /// 0 = not started + Token last_token; /// the incarnation WE wrote — the only one we ever renew + bool dead = false; /// set by the terminal op + +private: + void backgroundLoop(std::chrono::milliseconds period); + + std::string_view slot_name; + std::string_view terminal_verb; + + /// Set immediately before `renewOnce` invokes `onRenewMismatch` (which always + /// throws) so `backgroundLoop`'s catch block can tell a CONFIRMED mismatch (the PUT completed and + /// observed a foreign token -- proven supersession) apart from a TRANSIENT exception + /// (`putOverwrite` itself threw before any outcome was observed, or a defensive `dead`/`seq==0` + /// guard fired). Reset to `false` at the top of every `renewOnce` call. `renewOnce` and + /// `backgroundLoop` run on the SAME background thread, sequentially, so no synchronization is + /// needed for this flag. + bool last_renew_failure_was_confirmed_mismatch = false; + + std::mutex background_mutex; + std::condition_variable wakeup; + bool stop_requested = false; + ThreadFromGlobalPool thread; + + LoggerPtr log; +}; + +/// Validate a `server_root_id` — the explicit, configured identity of the content-addressed layout +/// subtree a server owns. It is a clean relative path: it composes into the +/// object-key tree (`gc/server-roots//...`, `roots//...`), so the same hygiene the layout +/// applies to a namespace applies here (mirrors `CasLayout.h::checkNamespace`): +/// - non-empty; +/// - no leading/trailing '/', no empty segment ("//"); +/// - no '.' or '..' segment; +/// - total length <= 255; +/// - no segment equal to the reserved "_files" / "_manifests". +/// Throws `ErrorCodes::BAD_ARGUMENTS` on any violation. Fail closed — there is no sanitizing fallback. +inline void validateServerRootId(const String & id) +{ + if (id.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "server_root_id must be non-empty"); + + if (id.size() > 255) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "server_root_id '{}' is too long ({} > 255 bytes)", id, id.size()); + + size_t start = 0; + while (true) + { + size_t end = id.find('/', start); + const String segment = id.substr(start, end == String::npos ? String::npos : end - start); + if (segment.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "server_root_id '{}' has an empty segment (leading/trailing or doubled '/')", id); + if (segment == "." || segment == "..") + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "server_root_id '{}' uses a relative segment ('.' or '..')", id); + if (segment == "_files") + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "server_root_id '{}' uses the reserved segment '_files'", id); + if (segment == "_manifests") + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "server_root_id '{}' uses the reserved segment '_manifests'", id); + if (end == String::npos) + break; + start = end + 1; + } +} + +/// The per-server-root control objects (owner, epoch, and mount lease) and their text codecs are kept +/// in `Formats/CasServerRootFormats`. This header includes those definitions so the protocol logic +/// below can use `OwnerObject`, `ServerEpoch`, `MountLease`, and their `encode`/`decode` functions +/// without duplicating the wire-format implementation. + +class Backend; +class Layout; + +/// Mount-safety claim logic. These are the identity and epoch-allocation steps a server +/// runs at startup over its `server_root_id` subtree, BEFORE any ordinary data write. They fail +/// closed (`ErrorCodes::CORRUPTED_DATA`); there is no re-mint or silent-recreate fallback. + +/// True iff the successfully decoded catalog names no current life owned by `server_root_id`, and +/// exact-component probes find neither `cas/manifests//` nor `roots//` work. Opaque +/// `cas/ns/` debris alone does not identify a logical owner. +bool serverRootSubtreeEmpty( + Backend & b, const Layout & l, const String & srid, const RefCatalog & catalog_observation); + +/// Supplied by the pool layer so the low-level server-root protocol always observes the mandatory +/// catalog. Every absent-control retry obtains a fresh, successfully decoded observation. +using ObserveRefCatalog = std::function; + +/// Read the owner anchor (`gc/server-roots//owner`) WITHOUT claiming or validating identity — +/// a plain GET+decode. nullopt = anchor absent. Pool-member decommission uses this to read the +/// victim UUID before mounting writable; `claimOwnerOrThrow` below is the identity-claiming +/// counterpart used by normal opens and reuses this GET+decode path. +std::optional readOwnerUuid(Backend & b, const Layout & l, const String & server_root_id); + +/// Claim (or validate) the sticky owner anchor that binds `srid` to a server UUID (identity). +/// - owner present, equal `our_uuid`, and not tombstoned → ok (return); +/// - owner present and tombstoned → throw `CORRUPTED_DATA` (explicitly retired — fail closed); +/// - owner present, different → throw `CORRUPTED_DATA` (foreign owner — fail closed); +/// - owner absent AND the subtree is provably empty → `putIfAbsent` the owner (claim); +/// - owner absent BUT the subtree is non-empty → throw `CORRUPTED_DATA` (identity lost over +/// existing data — never silently re-claim). +/// The owner object is never deleted and never reassigned to a different UUID. +void claimOwnerOrThrow( + Backend & b, const Layout & l, const String & srid, UInt128 our_uuid, + const ObserveRefCatalog & observe_catalog); + +/// Which mint policy governs `allocateWriterEpoch`'s absent-epoch branch (see below). +enum class EpochMintPolicy : uint8_t +{ + NormalMount, /// absent-epoch re-mint requires authoritative mount absence + DecommissionRecovery, /// absent-epoch re-mint requires a TERMINAL mount; mints a distinct epoch +}; + +/// Allocate the next durable-monotone `writer_epoch` by CAS-bumping the sticky `epoch` object +/// (`ServerEpoch{next_writer_epoch}`), returning the value the caller adopts as its writer_epoch. +/// - `epoch` absent AND the subtree is non-empty → throw `CORRUPTED_DATA` (missing epoch over +/// data is a reset hazard); +/// - `epoch` absent AND the subtree is empty → the absent-epoch branch is a LIFECYCLE decision, +/// so it uses `probeSentinelRaw`'s authoritative outcomes, never plain `get`-absence (which +/// flattens transport faults into "not found"), to decide whether the mount object is really +/// gone: +/// - `KeyAbsent` → authoritative absence, mint epoch 1 (fresh-root bootstrap); +/// - `Present`, `policy == NormalMount` → throw `CORRUPTED_DATA` (durable epoch state was +/// lost while a mount is live or recently live — refusing to re-mint epoch 1 there is how a +/// same-(uuid, epoch) twin is avoided); +/// - `Present`, `policy == DecommissionRecovery` → the surviving mount must be TERMINAL (not +/// live); a live member throws `ABORTED`, otherwise mint `surviving.writer_epoch + 1` — +/// distinct from the survivor's by construction (`now_ms` is required, nonzero, here); +/// - anything else (`ContainerAbsent`/`AccessDenied`/`Indeterminate`) → throw +/// `CORRUPTED_DATA` (absence was never proven; fail closed); +/// - otherwise read `next = current.next_writer_epoch`, `casPut` `{next + 1}` against the +/// observed token, retry on `Conflict` (bounded), and return `next`. +uint64_t allocateWriterEpoch(Backend & b, const Layout & l, const String & srid, + EpochMintPolicy policy, uint64_t now_ms, + const ObserveRefCatalog & observe_catalog); + +/// Which certificate of death justified a same-uuid, different-epoch mount reclaim. `None` when no +/// reclaim of that kind happened (a fresh claim, a +/// same-epoch refresh, `LiveDoubleStart`, `ForeignOwner`, `FencedSelf`). +enum class MountPriorState +{ + None, + Clean, /// the predecessor's own graceful farewell (`min_active == UINT64_MAX`) + Fenced, /// the GC leader's own (already threshold-gated) fence-out (`gc_fenced`) + UncleanObserved, /// OUR observation watched the write-token hold stable for the full threshold +}; + +/// Startup decision for the mount lease (`gc/server-roots//mount`), run AFTER the owner gate +/// (so `our_uuid` is the established owner). The lease is LIVENESS, not identity — the owner object +/// already settled who may write; the lease settles whether a live incarnation currently holds the +/// slot. Decision over `get(mountKey)`: +/// - absent → write our body via `putIfAbsent` → `Claimed`; +/// - same `server_uuid` AND same `writer_epoch` as (our_uuid, our_epoch) → it is OUR OWN claim +/// (a replay / the keeper adopting it): +/// - `gc_fenced` → terminal for THIS (uuid, epoch) — a fence costs an epoch, so refreshing it +/// in place would resurrect a fenced incarnation → `FencedSelf` (no write); +/// - otherwise → refresh (`putOverwrite` to bump seq + fresh `expires_at_ms`) → `Claimed`; +/// - same `server_uuid`, DIFFERENT `writer_epoch` → reclaimed ONLY on a certificate of death that +/// needs no fresh wall-clock trust (see +/// `claimMountAwaitingExpiry` below for how a plain "looks expired" reading is turned into one): +/// - `gc_fenced` (the GC leader already, itself, threshold-gated this incarnation dead; a fence +/// costs an epoch, so its keeper can never renew again) → reclaim, `prior = Fenced`; +/// - the clean marker (`min_active == UINT64_MAX`, the predecessor's own graceful farewell) → +/// reclaim, `prior = Clean`; +/// - `proven_dead_token` matches the CURRENTLY OBSERVED token (the caller itself watched this +/// exact token hold stable for the full observation threshold) → reclaim, `prior = +/// UncleanObserved`; +/// - none of the above → `LiveDoubleStart` (do NOT write). In particular `expires_at_ms <= +/// now_ms` ALONE is never sufficient — comparing a predecessor's stamp against OUR wall clock +/// is unsafe because a clock-skewed or merely late-observing +/// caller must never conclude death from a bare timestamp read); +/// - different `server_uuid` → `ForeignOwner` (do NOT write, regardless of expiry or prior state). +struct MountClaimResult +{ + /// Plain (unscoped) enum: callers compare with `MountClaimResult::Claimed` directly. + enum Kind + { + Claimed, + LiveDoubleStart, + ForeignOwner, + /// Same (uuid, epoch) as ours, but the body is `gc_fenced`: terminal for THIS epoch — a fence + /// costs an epoch. The caller must re-open with a fresh `writer_epoch`; refreshing or adopting + /// a fenced body in place is never correct. + FencedSelf, + }; + Kind kind = ForeignOwner; + MountLease body; + /// Which certificate of death justified a same-uuid, different-epoch `Claimed` reclaim (`None` for + /// every other `Kind`, and for the absent-slot / same-epoch-refresh `Claimed` cases). + MountPriorState prior = MountPriorState::None; + /// The backend token of the body this result observed, for + /// `LiveDoubleStart` only (a fresh `Claimed`/`FencedSelf`/`ForeignOwner` write/observe has no + /// separate "prior body's token to remember" use). `claimMountAwaitingExpiry`'s observation loop + /// used to re-GET the mount key itself just to recover this token that `claimMount` had already + /// read one line earlier and thrown away -- one wasted GET per iteration. Empty for every other + /// `Kind` (nothing to compare against). + std::optional token; +}; + +/// Thrown when a mount operation observes that OUR OWN (uuid, epoch) slot was `gc_fenced` by the GC +/// after our lease expired — a RECOVERABLE state ("a fence costs an epoch"): the caller re-opens with +/// a fresh `writer_epoch`. A CAS-local typed exception rather than a new `ErrorCodes` number: a fork +/// carries these edits indefinitely and the numbered `ErrorCodes` list conflicts with upstream on +/// every rebase. Catch sites match BY TYPE (`catch (const MountFencedException &)`), never by code; +/// the base code is `ABORTED` so an uncaught one still surfaces as a clean startup abort. +class MountFencedException : public DB::Exception +{ +public: + explicit MountFencedException(const String & msg) + : DB::Exception(msg, DB::ErrorCodes::ABORTED) {} +}; + +/// `proven_dead_token`: the write-token of a same-uuid, different-epoch lease that the CALLER already +/// proved dead by observation (see `claimMountAwaitingExpiry`) — matching it against the CURRENTLY +/// observed token is the ONLY way (besides `gc_fenced` / the clean marker) a same-uuid different-epoch +/// lease is ever reclaimed. Absent (`{}`, the default) for a bare claim attempt with no such proof. +MountClaimResult claimMount( + Backend & b, const Layout & l, const String & srid, UInt128 our_uuid, uint64_t our_epoch, + uint64_t now_ms, uint64_t ttl_ms, const std::optional & proven_dead_token = {}, + const CasEventSink & sink = {}); + +/// Format the operator-actionable startup error shown when the mount lease is held by a genuinely +/// live second server (the same `server_root_id` is mounted twice). Produced only AFTER this server +/// has already waited for the lease to lapse (see `claimMountAwaitingExpiry`) and it did not — so the +/// remediation is about a live twin, not about waiting. +String mountDoubleStartMessage(const String & srid, const MountLease & existing); + +/// Observation-based mount claim for restart recovery. +/// Wraps `claimMount` in a loop: +/// - first attempt decided immediately for `Claimed` (fresh / refreshed / reclaimed via `Fenced` or +/// `Clean`), `ForeignOwner`, or `FencedSelf`; +/// - a `LiveDoubleStart` from OUR OWN uuid (a stale lease from a prior incarnation of this server, +/// OR a genuinely live twin — the two are indistinguishable from a bare read) is resolved by +/// WATCHING the lease's write-token on OUR OWN clock (`mono_ms_fn`), NEVER by comparing the +/// lease's stamped `expires_at_ms` against any clock: once the observed token has held stable for +/// the full rate-bound threshold (`ttl_ms + ttl_ms / 20 + poll_interval_ms` — the lease TTL, a 5% +/// clock-drift allowance, and one poll interval of discreteness. This rate bound ensures that a +/// holder which last renewed before the observation began can no longer be within its lease. +/// that token is handed to `claimMount` as `proven_dead_token`, which then reclaims token-guarded +/// (`prior = UncleanObserved`). If the token changes DURING the wait (the holder renewed, or a +/// genuine twin is alive) the observation RESTARTS from the new token; bounded to a handful of +/// restarts before giving up and returning the last `LiveDoubleStart` (a holder whose token keeps +/// changing across that many restarts is alive, not dead). +/// `now_ms_fn` is WALL clock, used only for stamping the body we (may) write / diagnostics — it never +/// participates in the reclaim decision. `mono_ms_fn` is the OBSERVATION clock: monotonic on this +/// process, never compared against any other node's clock, and the ONLY clock the threshold is +/// measured against. `sleep_ms_fn` paces the poll. All three, plus `on_wait_start`, are injected so +/// tests drive fake clocks with no real sleeping. `on_wait_start` (default no-op) fires once per +/// observation-window start (including restarts), with the currently-observed lease and the +/// threshold, for an operator-visible startup log. +/// All callers use this shared formula so a future adjustment cannot silently leave the startup +/// observation path and either GC heartbeat path with different thresholds. `cadence_ms` is the +/// caller's own poll or heartbeat interval; the additional interval accounts for observation +/// discreteness, while `ttl_ms / 20` allows for a five-percent clock-rate difference. +uint64_t mountObservationThresholdMs(uint64_t ttl_ms, uint64_t cadence_ms); + +MountClaimResult claimMountAwaitingExpiry( + Backend & b, const Layout & l, const String & srid, UInt128 our_uuid, uint64_t our_epoch, + const std::function & now_ms_fn, + const std::function & mono_ms_fn, + uint64_t ttl_ms, uint64_t poll_interval_ms, + const std::function & sleep_ms_fn, + const std::function & on_wait_start = {}, + const CasEventSink & sink = {}); + +/// One `server_root_id`'s cross-round token-stability observation, +/// owned by the GC leader instance (`Cas::Gc::mount_obs`) and threaded through consecutive +/// `computeHeartbeatFloor` calls — one GC round is one observation tick. Mirrors +/// `claimMountAwaitingExpiry`'s observation loop, but at heartbeat-gate granularity rather than a +/// tight poll loop. +struct MountTokenObservation +{ + Token token; + uint64_t first_seen_mono_ms = 0; +}; + +/// Keyed by `server_root_id`. In-memory only: a fresh leader (after a steal, or a process restart) +/// starts with an empty map, which only delays fencing an already-dead mount by one extra round while +/// it (re)establishes the observation — safe (never fences early), never unsafe. +using MountObservationMap = std::map; + +/// GC heartbeat gate (GC round protocol step 1). Run by the GC leader at the top of a round: LIST +/// `gc/server-roots/` (O(servers), single-digit counts), GET each mount body, and classify + fence out +/// dead mounts (liveness only — graduation itself paces on GC rounds, not on heartbeat acks). +/// Classification per body: +/// - `gc_fenced` already set → excluded (`already_fenced`); a fenced mount is terminal, no PUT; +/// - terminated (`min_active == UINT64_MAX`, the farewell sentinel stamped by +/// `MountLeaseKeeper::terminate`) → excluded (`terminated`). `expires_at_ms` alone cannot +/// distinguish a graceful farewell from an unclean stop, so the sentinel — not the timestamps — is the +/// terminated marker; +/// - otherwise, observation-based liveness (the same +/// principle `claimMountAwaitingExpiry` uses for a mount's OWN reopen, applied here to the GC's +/// fence-out): `obs` remembers, per srid, the write-token last seen and the leader's OWN +/// monotonic clock reading (`mono_now_ms`) at the moment it first saw that token. A body whose +/// CURRENT token differs from (or is absent from) `obs` is (re)started fresh — counted `live`, +/// never fenced this call, regardless of what its stamped `expires_at_ms` claims (a bare +/// wall-clock stamp is never trusted — see `claimMount`'s "certificate of death" doc). Only once +/// the SAME token has held for `>= stable_threshold_ms` OF THE LEADER'S OWN CLOCK does the body +/// become FENCE-eligible; +/// - FENCE-eligible → one token-guarded `putOverwrite` preserving the WHOLE body, setting +/// `gc_fenced = true` and `seq + 1`. On `Done` → excluded (`fenced_now`); on `PreconditionFailed` +/// (the holder renewed concurrently — a live token change) → re-GET and reclassify from the top +/// (bounded retries; the reclassify sees the new token and restarts the observation, counting it +/// `live` — conservative, never exclude a heartbeat without a landed fence-out). +/// +/// `now_ms` is WALL clock, used only for the audit/diagnostic log line — it never participates in the +/// fence decision (mirrors `claimMountAwaitingExpiry`'s `now_ms_fn` vs `mono_ms_fn` split). +/// `mono_now_ms` is the OBSERVATION clock: the caller's OWN monotonic reading, never compared against +/// any other node's clock. `obs` is owned by the caller and threaded across consecutive calls (one GC +/// leader instance, `Cas::Gc::mount_obs`) — a fresh leader starts with an empty map (safe: delays +/// fencing one round, never fences early). +/// +/// The fence-out is BOTH safety and liveness. Safety: a sleeper's later renewal permanently fails +/// (its `putOverwrite` now mismatches the fenced token → `tripMountLost`), so it can never re-arm +/// without a fresh `open`. Liveness: a dead server's stale mount slot must not linger forever. +/// Preserving the body keeps restart recovery intact: a same-uuid reopen reads the current body and +/// reclaims through the normal expired-our-uuid branch. +struct HeartbeatFloor +{ + size_t live = 0; + size_t terminated = 0; + size_t fenced_now = 0; + size_t already_fenced = 0; + /// The srids of every mount fenced-out THIS call (one GcFenceOut audit event each). + std::vector fenced_srids; +}; + +HeartbeatFloor computeHeartbeatFloor(Backend & b, const Layout & l, uint64_t now_ms, + uint64_t mono_now_ms, uint64_t stable_threshold_ms, + MountObservationMap & obs); + +/// One `gc/server-roots//mount` slot whose holder is not provably finished with the prefix. +struct NonTerminalMountSlot +{ + String server_root_id; + /// What was read: the lease's identifying fields, or why the body could not be interpreted. + String detail; +}; + +/// Read-only scan of every mount slot under the pool prefix, answering ONE question: is some writer +/// still entitled to this prefix? A slot counts as terminal on exactly the two clock-free certificates +/// the mount protocol already recognises (`computeHeartbeatFloor`'s own classification): `gc_fenced` +/// (the GC leader fenced that incarnation out, and a fence costs an epoch, so its keeper can never +/// renew again) and `min_active == UINT64_MAX` (the holder's own graceful farewell). Everything else is +/// reported, INCLUDING a body this build cannot decode -- an unreadable lease of some other format +/// generation is precisely the case that must block, not the one to wave through. +/// +/// Deliberately no wall-clock judgement: `expires_at_ms` alone never proves death (comparing another +/// node's stamp against our clock is exactly what `claimMount` refuses to do), and this is not the +/// place to run an observation window either -- the answer to "someone may still be writing here" is +/// for the operator to stop that writer, not for us to wait it out. +/// +/// The caller is pool RECREATION (`Pool::open`'s bootstrap over a prefix with no authoritative +/// `_pool_meta`): minting a fresh pool identity while a live writer still holds a slot would leave that +/// writer appending its old-format transactions into the new pool. Writes nothing. +std::vector probeNonTerminalMountSlots(Backend & b, const Layout & l); + +/// A read-only snapshot of one server's mount slot, for introspection (`system.cas_mounts`). +/// state: `live` (lease within TTL+skew), `expired` (lease ran out; the next GC round's heartbeat floor +/// will fence it), `terminated` (clean farewell: `min_active == UINT64_MAX`), `fenced` (`gc_fenced`), +/// `corrupt` (body failed to decode — surfaced as a row, never an exception). +struct MountInfo +{ + String srid; + MountLease lease; + String state; +}; + +/// Enumerate every mount slot under `gc/server-roots/`, decoded and classified — the read-only sibling +/// of `computeHeartbeatFloor`: ZERO writes (no fence-out), per-row fail-open. One LIST + one GET per slot. +std::vector listMounts(Backend & backend, const Layout & layout, uint64_t now_ms, uint64_t skew_margin_ms); + +/// Whether the mounted writer identified by `(server_root_id, writer_epoch)` (the two fields of a +/// `CatalogEntry::creator` / `CreatorFence`, `ref_catalog`'s spec INV-3 §3, that this predicate actually +/// needs) is PROVABLY unable to complete anything more — the gate +/// `CasRefCatalog::reconcileStaleCreator` requires before a stalled `Creating` entry may be stolen by a +/// NEW actor. +/// +/// Takes the two scalars rather than a whole `CreatorFence` (review C4): that type lives on the +/// ref-catalog side (`Formats/CasRefCatalogFormat.h`), and this file is already widely included +/// through `CasPool.h` (mount/server-root plumbing reaches nearly every CAS translation unit), so +/// naming that type here would make the mount layer depend on the ref-catalog format instead of the +/// other way round, for a struct this function reads only two fields of. A caller holding a +/// `CreatorFence` passes `fence.server_root_id, fence.writer_epoch` directly. +/// +/// `fence_generation` is deliberately NOT one of the two scalars this function takes, and not because +/// it is unavailable -- the catalog persists it (`cfg` in `CasRefCatalogFormat.cpp`) -- but because it +/// is not the property this predicate needs. It mirrors `CasMountRuntime::fence_generation`, an +/// in-process atomic that every mount bumps from its OWN zero on every open, so a DIFFERENT actor's +/// counter (or the SAME actor's after a restart) starts over at the same small values and cannot +/// answer "is the incarnation that minted this entry still alive" -- comparing it across actors +/// compares two unrelated counts that happen to share a name. This reads the durable, cross-process +/// proof instead: `server_root_id`'s CURRENT mount slot (`Layout::mountKey`), classified by the SAME +/// two clock-free certificates `probeNonTerminalMountSlots`/`computeHeartbeatFloor` already use for the +/// identical question at pool-prefix and GC-heartbeat granularity — +/// - `gc_fenced` (the GC leader already fenced this incarnation; a fence costs an epoch, so its +/// keeper can never renew again), +/// - the clean-farewell sentinel `min_active == UINT64_MAX`, +/// PLUS one more certificate available here that neither of those needs: a DIFFERENT `writer_epoch` +/// currently live at that slot proves `writer_epoch`'s specific incarnation is superseded regardless of +/// its OWN certificate — `allocateWriterEpoch`/`claimMount` are why an epoch, once superseded, is never +/// reclaimed by its former holder. The classification is an EXHAUSTIVE switch over these three +/// certificates, not a positive allowlist -- mirrors `CasPool.cpp`'s own exhaustive switch over +/// `MountPriorState` (the classification `claimMount`, in THIS file, only PRODUCES; the switch +/// consuming it lives in the caller) deliberately: a future certificate with no verdict assigned here +/// must fail the BUILD (a missing `-Wswitch` case), never silently read as terminal. +/// +/// Deliberately conservative on the two cases that are NOT proof of death: an ABSENT mount slot +/// (`Backend::get` returning `nullopt` answers nothing about liveness — it is not proof either way) +/// and an UNDECODABLE body (an unreadable lease of some other format generation is precisely the case +/// that must block, not the one to wave through, mirroring `probeNonTerminalMountSlots`'s own stated +/// discipline for that case) both return `false` — refuse reconciliation rather than guess. +/// A merely `expired` lease (a wall-clock reading past `expires_at_ms`) is likewise NEVER +/// treated as a certificate, for the same reason `claimMount` itself refuses to trust one: comparing +/// another node's stamp against a clock is exactly the unsafe comparison the mount protocol exists to +/// avoid, and there is not even a caller-supplied clock offered here to make that comparison with. +/// +/// WHAT THIS DOES NOT PROVE, stated rather than left implicit: `true` means the CURRENT mount-slot body +/// carries a certificate against `writer_epoch` specifically -- it is not a claim about the server +/// root's OTHER activity, about whether `server_root_id` will ever mount again, or about +/// anything beyond this one slot's current body at the instant of this GET. A caller that needs a +/// stronger, race-free guarantee (e.g. "and it will never come back") must build that from a WIDER +/// observation, the way `claimMountAwaitingExpiry`'s token-stability window does for its own decision -- +/// this function performs no such window and answers from one point-in-time read alone. Answering +/// "unknown" (`false`, refuse) is the fail-closed choice on every path already listed above; there is +/// no path where this function answers `true` on evidence weaker than one of the three certificates. +bool isCreatorFenceTerminal(Backend & backend, const Layout & layout, const String & server_root_id, + uint64_t writer_epoch); + +/// Per-server MERGED heartbeat: one `SingleWriterSlot` over the per-server-root mount object carries +/// the mount lease (liveness) AND the build-watermark floor (`min_active`). One renewal PUT stamps the +/// clock and the build-watermark floor together. Anchors the slot synchronously on `start`, renews it +/// async off the write path, and fails closed on any foreign touch (`renewOnce` throws on a +/// precondition miss). `graceful stop` folds the watermark farewell (`min_active = UINT64_MAX`) into +/// the terminal already-expired mount body. +/// +/// ADOPT RULE (critical): the steady-state flow is `claimMount(...)` writes the live mount under +/// (our_uuid, our_epoch), THEN `keeper.start()`. So `start`'s `claim` hook must ADOPT a live mount +/// that is ALREADY ours — same `server_uuid` AND same `writer_epoch` — instead of self-tripping the +/// live-double-start guard. The discriminator is the (uuid, epoch) pair: +/// - same uuid + same epoch → our own just-written claim (or a replay) → adopt: `putOverwrite` +/// against the observed token to refresh seq/expiry (no fail); +/// - same uuid + DIFFERENT live epoch → a newer incarnation superseded us → fail closed; +/// - foreign uuid → fail closed; +/// - absent → `putIfAbsent`; expired-our-uuid (any epoch) → `putOverwrite` reclaim. +/// After `start`, `renewOnce` (base) keeps the slot alive and already fails closed on a foreign touch. +class MountLeaseKeeper : public SingleWriterSlot +{ +public: + /// `min_active_fn_` is read OFF the state lock on each beat (via `prepareRenew`) and stamped into + /// the mount body — the merged watermark floor. It reaches into the Pool's own lock, so it must + /// never run under `state_mutex`. + MountLeaseKeeper( + BackendPtr backend_, const Layout & layout_, const String & srid_, UInt128 server_uuid_, + uint64_t writer_epoch_, std::chrono::milliseconds ttl_, std::function now_ms_fn_, + std::function min_active_fn_, + CasEventSink event_sink_ = {}, + std::chrono::milliseconds lease_safety_margin_ = std::chrono::milliseconds(2000), + /// boot-domain clock for the on_renew_ok anchor; empty = real CLOCK_BOOTTIME. Injectable for + /// tests and wired by CasMountRuntime::installKeeper. + std::function boot_ms_fn_ = {}); + + /// Claims (adopts) the mount slot for (server_uuid, writer_epoch) with seq following the observed + /// one — durable when `start` returns. + void start() { doStart(); } + + /// Releases the mount: the terminal op. Stops the background thread first. + void stop() { doTerminate(); } + + /// Join the renewal thread BEFORE this object's own `std::function` members (`on_renew_ok` / + /// `on_lost`, which reach back into the Pool) are destroyed. The base `~SingleWriterSlot` also + /// calls `stopBackground`, but it runs AFTER the derived members are gone — a renewal firing + /// `on_lost` in that window would call a destroyed `std::function`. Stopping here closes that window. + ~MountLeaseKeeper() override { stopBackground(); } + + /// Local write-fence coupling (set once by `Pool::open` before `startBackground`): the keeper is + /// the ONLY thing that touches S3 for the lease, so the fence must reflect the live lease without a + /// per-write S3 read. On each SUCCESSFUL background renew the keeper calls `on_renew_ok` (the Pool + /// refreshes its monotonic fence deadline); when background renewal FAILS (a foreign/superseded + /// touch makes `renewOnce` throw and the loop stop) the keeper calls `on_lost` (the Pool latches + /// its fence to lost). Both default to no-op so a keeper used without a Pool is inert. + /// `on_renew_ok` receives the ATTEMPT-START boot-domain instant of the renewal it acknowledges — + /// the fence deadline must anchor there, never at response time (spec rev.4 Phase B). + void setFenceCallbacks(std::function on_renew_ok_, std::function on_lost_) + { + on_renew_ok = std::move(on_renew_ok_); + on_lost = std::move(on_lost_); + } + +protected: + /// Reads the current wall-clock stamp and watermark floor without holding the base state lock; + /// the callbacks can acquire the Pool lock, so reversing this order would deadlock renewal. + RenewPayload prepareRenew() const override; + + /// Encodes one complete mount body, combining the sequence assigned by the base slot with the + /// dynamic values returned by `prepareRenew`. + String encodeBody(uint64_t seq_, const RenewPayload & payload) const override; + + /// Adopts or claims the mount object for this `(server_uuid, writer_epoch)`, returning the token + /// from the durable write that the base slot must retain for its next token-guarded renewal. + Token claim(const String & body) override; + + /// Stamps the terminal, already-expired mount body and folds the watermark farewell into it; + /// the base state lock and `dead` flag prevent another renewal from racing this write. + void terminate() override; + + /// Refreshes `confirmed_deadline_ms` after EVERY successful renewal (background OR a direct + /// caller) — see `onRenewCommitted`'s base doc comment. + void onRenewCommitted() override; + + /// Fires the boot-domain write-fence callback (`on_renew_ok`) after a confirmed BACKGROUND + /// lease renewal. `confirmed_deadline_ms` itself is refreshed by `onRenewCommitted` above (which + /// already ran, for this same successful renewal, before the background loop calls this). + void onRenewSucceeded() override; + + /// Latches the Pool's local write fence when renewal can no longer prove that this incarnation + /// owns the mount slot. + void onRenewFailed() override; + + /// Re-reads a failed renewal and classifies it five ways -- fenced_by_gc, same_epoch_state_uncertain + /// (this incarnation's own token was advanced past under our own (uuid, epoch): ambiguous, not + /// proof of anything, and NOT fatal), superseded (a newer epoch), foreign_writer, and vanished + /// (the backing object disappeared) -- every branch stays terminal and fail closed, and NONE + /// constructs a `LOGICAL_ERROR`: they throw non-aborting codes (`ABORTED`/`FILE_DOESNT_EXIST`), + /// because each is reachable by an ordinary network timeout or environmental condition rather than + /// a programming bug -- and because this runs on the keeper's background thread, where an abort at + /// exception construction takes the whole process with it. + void onRenewMismatch(const String & mismatched_key) override; + /// Fence immediately only once our last CONFIRMED lease (the last successful + /// `claim`/renew) has reached its safety-margin boundary -- `confirmed_deadline_ms - now <= + /// lease_safety_margin`. Until then the mount-lease protocol still guarantees exclusivity. + bool shouldFenceOnTransientRenewFailure() override; + +private: + /// Refreshes `confirmed_deadline_ms` from `anchor_wall_ms` + `ttl` — anchor = the pre-I/O wall + /// instant of the confirming attempt. Called on every point this keeper KNOWS it holds a live + /// lease: both success paths of `claim` (mint and adopt), and every successful `renewOnce` + /// (`onRenewCommitted`) — background-driven or a direct caller alike. + void refreshConfirmedDeadline(uint64_t anchor_wall_ms); + + String srid; + UInt128 server_uuid; + uint64_t writer_epoch; + std::chrono::milliseconds ttl; + std::function now_ms_fn; + std::function min_active_fn; + std::function on_renew_ok; + std::function on_lost; + CasEventSink event_sink; + std::chrono::milliseconds lease_safety_margin; + /// boot-domain clock for the on_renew_ok anchor; empty = real CLOCK_BOOTTIME. Injectable for + /// tests and wired by CasMountRuntime::installKeeper. + std::function boot_ms_fn; + /// BOOTTIME-ms deadline (same clock as `now_ms_fn`/`MountFence`, and for the SAME suspend-safety + /// reason -- see `MountFence`'s doc comment in `CasMountRuntime.h`) of the last CONFIRMED lease. + /// 0 = none yet (`claim` always sets this before `startBackground` can run, so + /// `shouldFenceOnTransientRenewFailure` observing 0 is defensive, not an expected steady state). + uint64_t confirmed_deadline_ms = 0; + /// Whether this runtime has STOPPED BELIEVING IT OWNS THE MOUNT. Set at `onRenewFailed`, which is + /// that one point and is deliberately broader than "a mismatch was classified": the background loop + /// also reaches it when a TRANSIENT renewal failure persists past the confirmed lease's + /// safety-margin boundary (`shouldFenceOnTransientRenewFailure`), where nothing was classified at + /// all and the write fence latches anyway. Both are the same fact for the reader below, and the + /// broader one is the SAFE one: what the release path needs to know is whether this runtime still + /// claims the slot, not why it stopped. + /// + /// The release path reads it to tell two opposite situations apart — a deposed writer meeting its + /// successor in the slot (the expected end of a failover) from a writer that still believed it owned + /// the mount meeting a stranger there (single-writer exclusivity broken). Atomic because the + /// keeper's background thread sets it and teardown reads it. + std::atomic deposition_observed{false}; + /// Pre-I/O anchors of the CURRENT attempt, stashed by prepareRenew (which runs at the start of + /// every doStart/renewOnce attempt, off the state lock) and consumed by the success hooks. + /// `mutable` + no synchronization is safe: prepareRenew, claim, and the hooks all run on the + /// single renewal driver thread (see renewOnce's single-driver invariant). + mutable uint64_t last_attempt_wall_ms = 0; + mutable uint64_t last_attempt_boot_ms = 0; +}; + +/// Mount-lease-scoped staging sweeper for objects left behind by S3-native staging. +/// +/// A leaked S3 staging object happens two ways: (1) an exception between `promoteStaged` succeeding and +/// `cleanupPendingTempFiles` deleting the staging key (`ContentAddressedTransaction.cpp`), or (2) an +/// aborted/cancelled transaction whose pending blobs were staged but never promoted — by design, +/// `cleanupPendingTempFiles` deliberately leaves an S3 staging object in place on the abort path (never a +/// bare `fs::remove` on a remote key), so this sweeper is its ONLY reclaimer. Debris from either case is +/// bounded to `staging//` — the ONE mount that could ever have written under that prefix, since +/// every staging key this mount ever mints comes from `ContentAddressedMetadataStorage::stagingKeyPrefix()` +/// (`physicalKey(pool_prefix + "/staging/" + server_root_id)`), keyed by THIS mount's own `server_root_id`. +/// +/// LEASE-FENCE (fail-closed, never fail-open): `sweepOwnMountStaging` removes ONLY objects whose key +/// starts with the given `mount_staging_prefix` — pass your OWN mount's prefix, never another mount's. +/// The caller (`ContentAddressedMetadataStorage::startup()`) invokes this exactly once, at mount start, +/// with `stagingKeyPrefix() + "/"` — the SAME prefix construction the writer uses to mint staging keys, so +/// this sweep can never reach a different mount's `staging//` subtree: no other writer +/// ever stages a key under THIS mount's own `server_root_id` prefix, and this function never lists or +/// touches anything outside the prefix it is given. +/// +/// Best-effort and NEVER THROWS: one stubborn key (or a LIST failure) must never abort the sweep of the +/// rest, and must never fail the mount (mirrors `feedback_ca_gc_never_throw_on_404` — a throw here would +/// only wedge startup, not GC, but the same fail-open-on-error discipline applies to any best-effort +/// reclaim of debris). +/// +/// GC excludes `staging/` entirely: GC blob discovery LISTs `Layout::blobsPrefix()` +/// (`/blobs/`) — a distinct top-level prefix from `staging/`, `cas/ns/`, and +/// `cas/manifests/` (see `CasLayout.h`) — so a `staging/` object is never listed, HEAD'd, or condemned by +/// GC's fold. This sweeper is the ONLY reclaimer of `staging/` debris. +void sweepOwnMountStaging(IObjectStorage & object_storage, const String & mount_staging_prefix) noexcept; + +} From 9d08bc07f12ee0b6dec343f6257f0242f5c0996a Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:34 +0200 Subject: [PATCH 17/30] CAS subsystem: Parts layer Part-path parsing and the part-folder access facade over manifests. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../Parts/PartFolderAccess.cpp | 762 ++++++++++++++++++ .../ContentAddressed/Parts/PartFolderAccess.h | 411 ++++++++++ .../ContentAddressed/Parts/PartPathParser.cpp | 400 +++++++++ .../ContentAddressed/Parts/PartPathParser.h | 146 ++++ 4 files changed, 1719 insertions(+) create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartFolderAccess.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartFolderAccess.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartPathParser.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartPathParser.h diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartFolderAccess.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartFolderAccess.cpp new file mode 100644 index 000000000000..bce2e9423704 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartFolderAccess.cpp @@ -0,0 +1,762 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int FILE_DOESNT_EXIST; + extern const int ABORTED; + extern const int LOGICAL_ERROR; +} +} + +namespace ProfileEvents +{ + extern const Event CASPartFolderViewHits; + extern const Event CASPartFolderViewValidationMismatches; + extern const Event CASPartFolderViewMisses; + extern const Event CASPartFolderViewOversizedBypasses; + extern const Event CASPartFolderViewInvalidations; + extern const Event CASRefRollbackBestEffortDropFailed; + extern const Event CASPartFolderValidateSkipped; + extern const Event CASRefRepoint; +} + +namespace CurrentMetrics +{ + extern const Metric CASPartFolderCacheBytes; + extern const Metric CASPartFolderCacheEntries; +} + +namespace DB::Cas +{ + +PartFolderView::PartFolderView(PartRefKey key_, Cas::ManifestId manifest_id_, uint64_t manifest_size_, + std::shared_ptr manifest_, uint64_t validated_at_ms_) + : key(std::move(key_)) + , manifest_id(std::move(manifest_id_)) + , manifest_size(manifest_size_) + , manifest_body(std::move(manifest_)) + , validated_at_ms(validated_at_ms_) +{ + chassert(manifest_body); + /// The binary-search contract: entries must be strictly ascending by `path` (sorted and unique) — + /// `decodePartManifest` enforces exactly this for every decoded body, and `findEntry`'s binary + /// search assumes uniqueness. `adjacent_find` with `!(a.path < b.path)` flags any adjacent pair + /// that is out-of-order OR duplicate (stronger than `is_sorted`, which permits duplicates); a + /// hand-constructed manifest (tests) must honor it too. + chassert(std::adjacent_find(manifest_body->entries.begin(), manifest_body->entries.end(), + [](const Cas::ManifestEntry & a, const Cas::ManifestEntry & b) { return !(a.path < b.path); }) + == manifest_body->entries.end()); +} + +std::shared_ptr PartFolderView::make( + PartRefKey key, const Cas::Resolved & resolved, std::shared_ptr manifest, + uint64_t validated_at_ms) +{ + return std::make_shared( + std::move(key), resolved.manifest_id, resolved.manifest_size, + std::move(manifest), validated_at_ms); +} + +std::optional PartFolderView::projectionDirPrefix(const std::string & file) +{ + if (file.empty()) + return std::nullopt; + const auto last_slash = file.find_last_of('/'); + const std::string_view last_component + = last_slash == std::string::npos ? std::string_view(file) : std::string_view(file).substr(last_slash + 1); + if (last_component.ends_with(".proj") || last_component.ends_with(".tmp_proj")) + return file + "/"; + return std::nullopt; +} + +const Cas::ManifestEntry * PartFolderView::findFile(const String & path) const +{ + return Cas::findEntry(manifest_body->entries, path); +} + +bool PartFolderView::hasFile(const String & path) const +{ + return findFile(path) != nullptr; +} + +std::optional PartFolderView::fileSize(const String & path) const +{ + if (const auto * e = findFile(path)) + return e->size(); + return std::nullopt; +} + +std::optional PartFolderView::inlineBytes(const String & path) const +{ + const auto * e = findFile(path); + if (e && e->placement == Cas::EntryPlacement::Inline) + return e->inline_bytes; + return std::nullopt; +} + +std::vector PartFolderView::listChildren(const String & dir_prefix) const +{ + /// Collapse each entry to its first child component. Projection folders are structurally flat in + /// `MergeTree`, so this produces the same names as the old projection-specific path handling while + /// keeping one directory-listing rule for all manifest folders. + std::unordered_set names; + auto add = [&](const String & full) + { + if (!full.starts_with(dir_prefix) || full.size() <= dir_prefix.size()) + return; + const std::string_view rest = std::string_view(full).substr(dir_prefix.size()); + const auto slash = rest.find('/'); + names.emplace(slash == std::string_view::npos ? rest : rest.substr(0, slash)); + }; + const auto [first, last] = Cas::entryRange(manifest_body->entries, dir_prefix); + for (const auto * e = first; e != last; ++e) + add(e->path); + return {std::make_move_iterator(names.begin()), std::make_move_iterator(names.end())}; +} + +bool PartFolderView::hasDirectory(const String & dir_prefix) const +{ + const auto [first, last] = Cas::entryRange(manifest_body->entries, dir_prefix); + return first != last; +} + +size_t PartFolderView::estimatedBytes() const +{ + /// Conservative cache weight: fixed overhead plus `manifest_size`. This deliberately over-counts + /// the shared decode, which is safe because eviction should happen before the budget is exceeded. + return 256 + manifest_size; +} + +CachedPartFolderAccess::CachedPartFolderAccess(Cas::PoolPtr store_) + : CachedPartFolderAccess(std::move(store_), CacheParams{}) +{ +} + +CachedPartFolderAccess::CachedPartFolderAccess(Cas::PoolPtr store_, CacheParams params_, std::function now_ms_fn_) + : store(std::move(store_)), params(params_), now_ms_fn(std::move(now_ms_fn_)) +{ + if (!now_ms_fn) + now_ms_fn = []() -> uint64_t { return timeInMilliseconds(std::chrono::system_clock::now()); }; + if (params.cache_bytes > 0) + view_cache = std::make_unique( + "LRU", CurrentMetrics::CASPartFolderCacheBytes, CurrentMetrics::CASPartFolderCacheEntries, + params.cache_bytes, params.max_entries, ViewCache::DEFAULT_SIZE_RATIO); +} + +std::shared_ptr +CachedPartFolderAccess::getView(const PartRefKey & key, Freshness freshness) const +{ + /// Resolve first on every access. Absence is never retained, and the same ref-resolution result + /// supplies the manifest ID used to validate a retained view. The emit is deferred: a warm + /// `CachedForLoad` hit below serves the call without doing any real resolve work worth auditing, so + /// this call site decides itself, per path, whether to re-emit the identical `RefResolve` event. + auto resolved = resolve(key, freshness, Cas::ResolveAudit::Deferred); + if (!resolved) + return nullptr; + + /// Reuse one canonical key for the retained view and the optional diagnostic journal. + const String cache_key = key.cacheKey(); + + /// Retained views serve `CachedForLoad` directly only after their manifest ID matches the fresh + /// resolve. `ForceFresh` must re-prove the manifest body unless the configured validation policy + /// explicitly permits a recent retained view; a fresh ref resolve proves ref currency, not body existence. + if (freshness == Freshness::CachedForLoad && view_cache) + { + if (auto cached = view_cache->get(cache_key)) + { + if (cached->manifestId() == resolved->manifest_id) + { + /// The warm hit: the retained view already reflects this exact manifest, so this access + /// did no real resolve work beyond a cache lookup — no `RefResolve` audit row for it. + ProfileEvents::increment(ProfileEvents::CASPartFolderViewHits); + recordDecision(cache_key, LastDecision::Hit, cached.get(), /*retained=*/true); + return cached; + } + ProfileEvents::increment(ProfileEvents::CASPartFolderViewValidationMismatches); + /// Rebuild below; the stale entry is superseded by the new view when retention is enabled. + } + } + + /// With a non-`Always` validation policy, `ForceFresh` may serve a retained view without another + /// body HEAD when its manifest ID still matches and its validation timestamp is within the age + /// policy. `StrictValidate` bypasses retention. A manifest-ID mismatch always rebuilds, because all + /// part content is represented by the manifest. + if (freshness == Freshness::ForceFresh && view_cache && params.validate.mode != PartFolderValidate::Mode::Always) + { + if (auto cached = view_cache->get(cache_key); + cached && cached->manifestId() == resolved->manifest_id) + { + const bool fresh_enough = params.validate.mode == PartFolderValidate::Mode::Never + || (now_ms_fn() - cached->validatedAtMs()) < params.validate.age_seconds * 1000ULL; + if (fresh_enough) + { + ProfileEvents::increment(ProfileEvents::CASPartFolderViewHits); + ProfileEvents::increment(ProfileEvents::CASPartFolderValidateSkipped); + recordDecision(cache_key, LastDecision::Hit, cached.get(), /*retained=*/true); + emitResolveEvent(key, *resolved); + return cached; + } + } + } + + auto view = buildView(key, *resolved, freshness); + + /// Retain eligible views. `StrictValidate` never populates the cache, and oversized views are + /// served but not retained. + /// `oversized` is tracked separately from `retained`: with retention disabled (`view_cache == + /// nullptr`), `retained` is also false, but that is an ordinary disabled-mode miss rather than + /// an oversized bypass. + bool retained = false; + bool oversized = false; + if (freshness != Freshness::StrictValidate && view_cache) + { + if (view->estimatedBytes() <= params.max_entry_bytes) + { + /// CacheBase stores mutable pointers; views are logically const (never mutated). + view_cache->set(cache_key, std::const_pointer_cast(view)); + retained = true; + } + else + { + oversized = true; + ProfileEvents::increment(ProfileEvents::CASPartFolderViewOversizedBypasses); + } + } + ProfileEvents::increment(ProfileEvents::CASPartFolderViewMisses); + recordDecision(cache_key, + freshness == Freshness::CachedForLoad ? (oversized ? LastDecision::OversizedBypass : LastDecision::Miss) + : freshness == Freshness::ForceFresh ? LastDecision::ForceFreshRead + : LastDecision::StrictBypass, + view.get(), retained); + emitResolveEvent(key, *resolved); + return view; +} + +void CachedPartFolderAccess::emitResolveEvent(const PartRefKey & key, const Cas::Resolved & resolved) const +{ + /// Mirrors `CasRefLedger::resolveRef`'s own (deferred-here) emit exactly: same event type and + /// fields, built from the `Resolved` this call already holds. + Cas::EventEmitter{*store}.emit([&](Cas::CasEvent & e) + { + e.type = Cas::CasEventType::RefResolve; + e.namespace_ = key.ns.string(); + e.ref_name = key.ref; + e.object_kind = Cas::CasEventObjectKind::Manifest; + e.object_hash = Cas::manifestRefDebugString(resolved.manifest_id.ref); + e.outcome = "resolved"; + e.reason = "read-side resolve of a ref to its part manifest"; + }); +} + +std::shared_ptr CachedPartFolderAccess::buildView( + const PartRefKey & key, const Cas::Resolved & resolved, Freshness freshness) const +{ + /// Fresh modes do not coalesce: each `ForceFresh`/`StrictValidate` call owns its mandatory HEAD. + /// Only cold `CachedForLoad` builds use single-flight. + if (freshness != Freshness::CachedForLoad) + return PartFolderView::make(key, resolved, store->readManifestShared(resolved.manifest_id), now_ms_fn()); + + std::promise> promise; + std::shared_future> future; + bool leader = false; + { + std::lock_guard lock(inflight_mutex); + if (auto it = inflight.find(key.cacheKey()); it != inflight.end()) + future = it->second; /// Follower: share the leader's build. + else + { + leader = true; + future = promise.get_future().share(); + inflight.emplace(key.cacheKey(), future); + } + } + if (!leader) + return future.get(); /// Rethrows the leader's exception, if any. + + SCOPE_EXIT({ + std::lock_guard lock(inflight_mutex); + inflight.erase(key.cacheKey()); + }); + try + { + auto view = PartFolderView::make(key, resolved, store->readManifestShared(resolved.manifest_id), now_ms_fn()); + promise.set_value(view); + return view; + } + catch (...) + { + promise.set_exception(std::current_exception()); /// Followers see the leader's exception. + throw; + } +} + +void CachedPartFolderAccess::eraseView(const PartRefKey & key) +{ + const String cache_key = key.cacheKey(); + if (view_cache) + view_cache->remove(cache_key); + ProfileEvents::increment(ProfileEvents::CASPartFolderViewInvalidations); + recordDecision(cache_key, LastDecision::Invalidated, nullptr, /*retained=*/false); +} + +std::optional +CachedPartFolderAccess::resolve(const PartRefKey & key, Freshness freshness, Cas::ResolveAudit audit) const +{ + return store->resolveRef(key.ns, key.ref, /*allow_stale=*/freshness == Freshness::CachedForLoad, audit); +} + +bool CachedPartFolderAccess::existsRef(const PartRefKey & key, Freshness freshness) const +{ + return resolve(key, freshness).has_value(); +} + +Cas::CommitOutcome CachedPartFolderAccess::promoteBuild(Cas::PartWriteTxn & build, const PartRefKey & key, UInt128 build_id, + const Cas::ManifestId & manifest_id, bool allow_repoint, + bool * commit_recorded) +{ + /// `build.promote` derives `created` INSIDE its own `appendRefOps` builder (the same in-closure + /// pattern as that builder's `repoint_old`) and returns it the instant the append confirms. + /// + /// The outcome's STRINGS are copied BEFORE the append, so the only thing the post-durable region + /// has left to do is store a bool -- which makes that region allocation-free, hence non-throwing. + /// This is Part A's rule applied one layer up: nothing after a durable commit may throw before the + /// caller has recorded it. Assembling the outcome afterwards (as this did) put two `String` copies + /// and a cache invalidation between "the ref is committed" and "the handle knows", so a + /// `MEMORY_LIMIT_EXCEEDED` there landed in the caller's failed-promote handler -- which abandons + /// the build and, one layer up again, reports a byte fallback for a relink that already published. + Cas::CommitOutcome outcome{key.ns, key.ref, manifest_id.ref, /*created=*/false}; + + const bool created = build.promote(key.ns, key.ref, build_id, manifest_id, allow_repoint); + + /// POST-DURABLE REGION. Two plain stores, no allocation, no call that can fail. + { + DENY_ALLOCATIONS_IN_SCOPE; + outcome.created = created; + if (commit_recorded) + *commit_recorded = true; + } + + /// Test-only (see `setPostCommitProbeForTest`): models an allocation failure in the throwable + /// post-commit work below. It fires AFTER the region above, because the property under test is that + /// the commit is already recorded by the time anything here can throw. + if (post_commit_probe_for_test) + post_commit_probe_for_test(); + + eraseView(key); + return outcome; +} + +namespace +{ + +/// A failed publish must not leak a live-epoch precommit binding: only `abandon` removes it (the build +/// destructor merely retires the build seq; the stale-precommit sweep is prior-epoch-scoped and GC +/// never touches a live precommit). `abandon` may itself fail on the same broken backend -- log and let +/// whatever error the caller is already carrying stay primary. Returns whether the abandon completed, +/// so a caller tracking a terminal state can tell a finished transaction from one still owing cleanup +/// (`PartWriteTxn::abandon` stays retryable after an append failure). +bool abandonBuildBestEffort(Cas::PartWriteTxn & build, const char * context) noexcept +{ + try + { + build.abandon(); + return true; + } + catch (...) + { + tryLogCurrentException(getLogger("CachedPartFolderAccess"), context); + return false; + } +} + +} + +PreparedPartWrite::PreparedPartWrite(CachedPartFolderAccess & owner_, PartWriteTxnPtr build_, + PartRefKey key_, ManifestId id_) + : owner(&owner_), build(std::move(build_)), key(std::move(key_)), id(std::move(id_)) +{ +} + +PreparedPartWrite::PreparedPartWrite(PreparedPartWrite && other) noexcept + : owner(other.owner), build(std::move(other.build)), key(std::move(other.key)), id(std::move(other.id)) + , terminal(other.terminal) +{ + /// The move takes over the owed terminal operation in full: the source must never run it again, + /// so it is left terminal (and holding no transaction) rather than merely emptied. + other.owner = nullptr; + other.terminal = true; +} + +PreparedPartWrite::~PreparedPartWrite() +{ + if (terminal || !build) + return; + /// Last-resort guard. Reaching here means no terminal operation COMPLETED -- it was forgotten, an + /// exception skipped it, or one was attempted and its append failed -- so the precommit binding is + /// still live and its removal is appended here rather than left to leak. Best-effort and noexcept: + /// a destructor that propagated the append failure would terminate the process, and the durable + /// backstop for a removal that cannot land at all is the same one every other abandon path uses. + LOG_ERROR(getLogger("CachedPartFolderAccess"), + "PreparedPartWrite for '{}/{}' was destroyed while still owing its terminal operation; " + "aborting it now", key.ns.string(), key.ref); + abandonBuildBestEffort(*build, "aborting a prepared part write destroyed without a terminal operation"); +} + +bool PreparedPartWrite::commitIsUnresolved() const +{ + return build && build->commitState() == Cas::PartWriteTxn::CommitState::Uncertain; +} + +Cas::CommitOutcome PreparedPartWrite::promote(bool allow_repoint) +{ + if (terminal) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "PreparedPartWrite for '{}/{}' has already been promoted or aborted; it owes exactly one " + "terminal operation", + key.ns.string(), key.ref); + try + { + /// `terminal` is set by `promoteBuild` itself, inside the allocation-free region immediately + /// after the durable append -- NOT after this call returns. The difference is the whole point: + /// the outcome assembly and the cache invalidation that follow the append can throw, and a + /// handle that had not yet recorded the commit would then take the catch below and abandon a + /// build whose ref is already published. + return owner->promoteBuild(*build, key, build->buildId(), id, allow_repoint, &terminal); + } + catch (...) + { + /// The commit is durable and this handle knows it, so the duty is discharged and there is + /// nothing to abandon -- the error is post-commit work failing, which the caller still hears + /// about, unchanged. + if (terminal) + throw; + /// The catch-abandon-rethrow discipline the atomic `publishEntries` has always owned. It stays + /// correct even when the append itself was ambiguous: `abandon` appends a PRECOMMIT removal, + /// and a promote that did land moved the binding to committed, so the removal is rejected by + /// the state machine rather than undoing a published ref. The terminal flag flips only if the + /// abandon actually landed: a failed abandon leaves the handle owing cleanup, which the + /// destructor then retries. + terminal = abandonBuildBestEffort(*build, "abandoning the build of a failed PreparedPartWrite::promote"); + throw; + } +} + +void PreparedPartWrite::abort() +{ + if (terminal) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "PreparedPartWrite for '{}/{}' has already been promoted or aborted; it owes exactly one " + "terminal operation", + key.ns.string(), key.ref); + /// `abandon` appends the EXACT precommit removal through the reliable append lane -- that append, + /// not the destruction of the transaction object, is what releases the manifest's `+1`. A failure + /// propagates: the caller may retry the same handle, and the destructor is the backstop. + build->abandon(); + terminal = true; +} + +PreparedPartWrite CachedPartFolderAccess::prepareEntries(const PartRefKey & dst, + const std::vector & entries, Cas::ProvenanceOp op) +{ + auto build = store->beginPartWrite(Cas::PartWriteInfo{.intended_ref = dst.ns.string() + "/" + dst.ref, + .intended_namespace = dst.ns, .op = op}); + try + { + /// Record write evidence for each non-inline entry. No pool HEAD/GET is performed before + /// precommit; the promote path re-proves each dependency fail-closed. Inline entries need no evidence. + for (const auto & entry : entries) + build->adoptEvidence(entry); + /// Stage a fresh manifest over the same entries. Blobs are content-addressed, but each part owns + /// its manifest ID, so `dst` receives a distinct manifest before ownership moves to it. + const Cas::ManifestId id = build->stageManifest(entries); + build->precommitAdd(dst.ns, dst.ref, id); + return PreparedPartWrite(*this, std::move(build), dst, id); + } + catch (...) + { + /// No handle is returned on this path, so the cleanup cannot be deferred to one: abandon here. + abandonBuildBestEffort(*build, "abandoning the build of a failed prepareEntries"); + throw; + } +} + +Cas::CommitOutcome CachedPartFolderAccess::publishEntries(const PartRefKey & dst, + const std::vector & entries, Cas::ProvenanceOp op, bool allow_repoint) +{ + /// The atomic form: prepare and promote back to back, with no window in between. `promote` carries + /// the catch-abandon-rethrow discipline, so a failure here behaves exactly as it did when this was + /// one function. + PreparedPartWrite prepared = prepareEntries(dst, entries, op); + return prepared.promote(allow_repoint); +} + +bool CachedPartFolderAccess::republishRef(const PartRefKey & src, const PartRefKey & dst) +{ + /// Content addressing has no rename, so move a committed ref by reading the source body freshly, + /// publishing equivalent entries at the destination, and then dropping the source. The source + /// body is re-proved and is never taken from a retained view. + auto resolved = store->resolveRef(src.ns, src.ref); + if (!resolved) + return false; + const auto src_manifest = store->readManifestShared(resolved->manifest_id); + + /// If `dst` is already committed, a previous attempt may have completed its promote before the + /// source drop. Compare content rather than the whole manifest, whose ref/namespace/digest + /// legitimately differ: equal content completes the move by dropping `src`; different content + /// is a conflict and leaves the source intact. + if (auto dst_resolved = store->resolveRef(dst.ns, dst.ref)) + { + const auto dst_manifest = store->readManifestShared(dst_resolved->manifest_id); + if (dst_manifest->entries != src_manifest->entries) + throw Exception(ErrorCodes::ABORTED, + "republishRef: destination '{}' is already committed with different content — refusing " + "(rename/attach conflict)", dst.ns.string() + "/" + dst.ref); + dropRef(src); + return true; + } + + publishEntries(dst, src_manifest->entries, Cas::ProvenanceOp::Other); + dropRef(src); + return true; +} + +Cas::CommitOutcome CachedPartFolderAccess::repointRef(const PartRefKey & key, std::vector entries, Cas::ProvenanceOp op) +{ + /// Compare the candidate `entries` against the currently committed manifest's + /// decoded entries. This must NOT stage a candidate manifest first: `stageManifest` mints a + /// non-content-derived `ManifestRef` (epoch/build_seq/ordinal) AND durably PUTs the encoded body + /// on every call (CasPartWriteTxn.cpp), so staging-then-comparing IDs would itself be a pool mutation on + /// the byte-equal path — violating the "ZERO pool mutations" contract this primitive exists to + /// provide. + /// + /// The comparison must be symmetric. `committed_manifest->entries` already went + /// through one `decodePartManifest` round-trip, which does not carry `blob_size` for Inline + /// entries on the wire (it is redundant with `inline_bytes.size()`, use `ManifestEntry::size()` for + /// the logical size instead, and it is excluded from both the canonical encoding and the payload + /// digest — see `CasPartManifestFormat.cpp`'s `writeEntryRecord`/`decodePartManifest`). The + /// freshly constructed `entries` may not have the same incidental fields (the inline write path no longer sets + /// `blob_size`), but a straight struct compare against them is still not guaranteed byte-identical + /// (canonical path ordering, etc.), so route the candidate through the identical encode/decode + /// round-trip before comparing regardless — this is the same content comparison used by `republishRef`, + /// which is symmetric for the same reason (both sides there are already decoded). + auto resolved = resolve(key, Freshness::ForceFresh); + if (resolved) + { + const auto committed_manifest = store->readManifestShared(resolved->manifest_id); + Cas::PartManifest probe; + probe.ref = committed_manifest->ref; + probe.root_namespace_id = committed_manifest->root_namespace_id; + probe.entries = entries; + probe.payload_digest = Cas::computePayloadDigest(probe); + const Cas::PartManifest canonical_candidate = Cas::decodePartManifest(Cas::encodePartManifest(probe)); + if (committed_manifest->entries == canonical_candidate.entries) + /// ZERO pool mutations: the outcome describes the manifest ALREADY committed, unchanged. + return Cas::CommitOutcome{key.ns, key.ref, resolved->manifest_id.ref, /*created=*/false}; + } + /// `publishEntries` takes `entries` by const reference, so the caller-owned vector remains valid. + /// Capture the exact outcome IMMEDIATELY -- before the ProfileEvent/logging below -- so it is + /// published ahead of any further (even if non-throwing in practice) post-commit work. + const Cas::CommitOutcome oc = publishEntries(key, entries, op, /*allow_repoint=*/true); + ProfileEvents::increment(ProfileEvents::CASRefRepoint); + if (resolved) + { + /// Repoint is the normal mechanism for effective standalone writes/removes on committed parts, + /// so this routine event is logged at debug level while the counter remains an operator-facing signal. + LOG_DEBUG(getLogger("CachedPartFolderAccess"), + "Repointed committed ref {}/{} ({} entries) — standalone write/remove on a committed part", + key.ns.string(), key.ref, entries.size()); + } + else + { + /// A repoint normally targets an existing ref. Keep an unexpected create-shaped call visible + /// at warning level rather than silently treating it as ordinary publication. + LOG_WARNING(getLogger("CachedPartFolderAccess"), + "repointRef published {}/{} ({} entries) with no prior committed ref to repoint — " + "unexpected call shape (repointRef requires an existing committed ref)", + key.ns.string(), key.ref, entries.size()); + } + return oc; +} + +void CachedPartFolderAccess::dropRef(const PartRefKey & key) +{ + store->dropRef(key.ns, key.ref); + eraseView(key); +} + +void CachedPartFolderAccess::dropRefIfPresent(const PartRefKey & key) +{ + /// resolveRef gates the common case (a temporary ref that was never committed is a no-op, not an + /// error); dropRef re-reads the shard inside its own CAS loop, so a concurrent drop can land in + /// the window between our resolve and that re-read — surfacing as FILE_DOESNT_EXIST. Removal is + /// replay-safe, so an already-gone ref is success; any other exception still propagates. The view + /// is also erased on the early-return absent path. + if (!store->resolveRef(key.ns, key.ref, /*allow_stale=*/true)) + { + eraseView(key); + return; + } + try + { + store->dropRef(key.ns, key.ref); + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::FILE_DOESNT_EXIST) + throw; + eraseView(key); + return; /// Raced away between the gate and dropRef; nothing was actually dropped here. + } + eraseView(key); +} + +void CachedPartFolderAccess::dropRefBestEffort(const PartRefKey & key) noexcept +{ + try + { + store->dropRef(key.ns, key.ref); + } + catch (...) + { + /// Best-effort destructor/rollback cleanup: debris is GC-reclaimed, but swallowing the + /// exception without a diagnostic could leave a live phantom ref after a backend outage. + ProfileEvents::increment(ProfileEvents::CASRefRollbackBestEffortDropFailed); + tryLogCurrentException(getLogger("CachedPartFolderAccess"), + fmt::format("CA best-effort rollback dropRef failed (ns={} ref={}); the ref may remain live", + key.ns.string(), key.ref)); + } + /// In destructor/rollback context the ref's durable state is unknown, so invalidate the view even + /// after a swallowed cleanup exception. + eraseView(key); +} + +bool CachedPartFolderAccess::dropRefIfMatches(const PartRefKey & key, const Cas::ManifestRef & expected) noexcept +{ + /// One `appendRefOps` builder does both the read and the conditional removal, mirroring + /// `CasRefLedger::dropRef`'s own protocol shape (read the committed binding, emit ONE + /// `OwnerTransition` removal op) but with the removal guarded on `expected` inside the SAME + /// closure -- the leader-thread read of `state.getCommitted()` is the authoritative committed + /// binding at append time, so this is race-free the same way `PartWriteTxn::promote`'s + /// `repoint_old`/idempotent-guard reads are: `build_ops` runs at most once, on the flush leader, + /// against the batch-validated state. A mismatch (repointed since `expected` was observed, or + /// already absent) returns an empty op list -- a legitimate no-op, not an error, exactly like + /// `promote`'s own idempotent-redrive branch. + bool removed = false; + try + { + const RefTxnId txn_id = store->appendRefOps(key.ns, MutationScope::ref(key.ref), + [&](const RefTableState & state) -> std::vector + { + const auto it = state.getCommitted().find(key.ref); + if (it == state.getCommitted().end() || !(it->second.manifest_ref == expected)) + return {}; /// absent, or repointed away from `expected` -- leave it alone + + removed = true; + RefOp op; + op.kind = RefOpKind::OwnerTransition; + op.old_binding = RefOwnerBinding{RefOwnerKind::Committed, key.ref, expected}; + return {op}; + }, + RootMutationOrigin::Writer, RootMutationKind::Drop); + + /// Audit a successful conditional removal exactly like `CasRefLedger::dropRef` audits its own + /// unconditional one -- otherwise a rollback drop (this method's only caller, as of Task 3) + /// would be invisible in `system.cas_log`. Byte-neutral to the ref-log: this is + /// an audit event only, emitted after the removal is already durable. + if (removed && store->hasEventSink()) + { + Cas::CasEvent ev; + ev.type = Cas::CasEventType::RefDrop; + ev.namespace_ = key.ns.string(); + ev.ref_name = key.ref; + ev.object_kind = Cas::CasEventObjectKind::Manifest; + ev.object_hash = Cas::manifestRefDebugString(expected); + ev.at_version = txn_id.ref_sequence; + ev.outcome = "ok"; + ev.reason = "dropRefIfMatches: conditional rollback removed the exact manifest this caller committed"; + store->emitEvent(std::move(ev)); + } + } + catch (...) + { + /// Best-effort rollback cleanup, like dropRefBestEffort: debris is GC-reclaimed, but swallowing + /// without a diagnostic could leave a live phantom ref after a backend outage. + removed = false; + ProfileEvents::increment(ProfileEvents::CASRefRollbackBestEffortDropFailed); + tryLogCurrentException(getLogger("CachedPartFolderAccess"), + fmt::format("CA conditional rollback dropRefIfMatches failed (ns={} ref={} expected={}); " + "the ref may remain live", key.ns.string(), key.ref, Cas::manifestRefDebugString(expected))); + } + /// The read above is authoritative fresh state regardless of outcome, so any locally retained view + /// is invalidated unconditionally -- cheap and conservative, matching dropRefBestEffort. + eraseView(key); + return removed; +} + +void CachedPartFolderAccess::dropNamespace(const Cas::RootNamespace & ns) +{ + store->dropNamespace(ns); + if (view_cache) + { + const String prefix = ns.string() + '\0'; + view_cache->remove([&](const String & k, const auto &) { return k.starts_with(prefix); }); + } + ProfileEvents::increment(ProfileEvents::CASPartFolderViewInvalidations); +} + +void CachedPartFolderAccess::recordDecision(const String & cache_key, LastDecision decision, + const PartFolderView * view, bool retained) const +{ + if (!params.explain_enabled) + return; /// Disabled diagnostics keep the read path free of journal locking and allocation. + std::lock_guard lock(explain_mutex); + if (explain_map.size() >= EXPLAIN_MAX_ENTRIES) + explain_map.clear(); + auto & e = explain_map[cache_key]; + e.last_decision = decision; + e.retained = retained; + if (view) + { + e.manifest_ref = Cas::manifestRefDebugString(view->manifestId().ref); + e.estimated_bytes = view->estimatedBytes(); + } +} + +size_t CachedPartFolderAccess::explainJournalSizeForTest() const +{ + std::lock_guard lock(explain_mutex); + return explain_map.size(); +} + +CachedPartFolderAccess::ExplainResult CachedPartFolderAccess::explain(const PartRefKey & key) const +{ + ExplainResult result; + { + std::lock_guard lock(explain_mutex); + const auto it = explain_map.find(key.cacheKey()); + if (it != explain_map.end()) + result = it->second; + } + /// `retained` is reported live against the cache, not from the decision snapshot: `dropNamespace` + /// erases every key of a namespace via one `CacheBase::remove` predicate sweep without a per-key + /// `recordDecision` call, so a snapshot value would go stale for every key it touches except the + /// one last read. A live membership check is authoritative for every eraser (write-through or + /// namespace-wide) and costs one more `CacheBase` lookup on this test/log-only path. + result.retained = view_cache && view_cache->get(key.cacheKey()) != nullptr; + return result; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartFolderAccess.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartFolderAccess.h new file mode 100644 index 000000000000..cfa81f83662a --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartFolderAccess.h @@ -0,0 +1,411 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Stable identity of a committed part or projection folder: its owning root namespace and +/// committed-ref name (for example, a part name or `detached/`). The key is used for both +/// storage operations and retained-view indexing, so its equality and cache-key representations +/// must describe exactly the same ref. +struct PartRefKey +{ + Cas::RootNamespace ns{""}; + String ref; + + bool operator==(const PartRefKey & o) const { return ns.string() == o.ns.string() && ref == o.ref; } + + /// Canonical map key. '\0' cannot occur in namespace strings or ref names (both derive from + /// disk paths), so the join is unambiguous even though refs may contain '/'. + String cacheKey() const { return ns.string() + '\0' + ref; } +}; + +/// Exact record of what one commit-shaped primitive (`promoteBuild`/`repointRef`) durably committed, +/// derived INSIDE the primitive's own `appendRefOps` builder rather than read back with a separate +/// call afterward -- a post-read would race against a concurrent repoint of the SAME ref and could +/// observe someone else's commit instead of this caller's own. `created` distinguishes a first-time +/// bind (no prior committed row for `ref`) from a repoint of an already-committed one; `manifest_ref` +/// is the manifest now bound to `ref` (unchanged from before the call on a `repointRef` byte-equal +/// no-op). A caller that needs to roll back its own commit later can drop `ref` conditionally on this +/// exact `manifest_ref` (`dropRefIfMatches`) instead of unconditionally (`dropRef`), which would +/// remove whatever manifest currently occupies the name -- unsafe once a concurrent writer may have +/// repointed it since. +struct CommitOutcome +{ + RootNamespace ns; + String ref; + ManifestRef manifest_ref; + bool created = false; +}; + +/// Read-freshness policy at the part-folder access boundary. The +/// mutable-read-vs-write-evidence distinction is carried by the METHOD, not a fourth value: +/// mutable per-part reads call `resolve` (no manifest involved); write-path source reads call +/// `getView`, which under ForceFresh always re-proves the manifest body (mandatory HEAD in +/// `readManifestShared` — a fresh ref resolve alone proves ref currency, NOT body existence). +enum class Freshness +{ + CachedForLoad, /// repeated load-window reads; stale-tolerant resolve (allow_stale=true) + ForceFresh, /// mutable per-part reads and write-path source reads; resolve fresh + StrictValidate, /// fsck/debug: bypass retained views entirely; fresh resolve + validated read +}; + + +/// Immutable snapshot of one resolved committed part/projection folder. Index-free: the decoder guarantees strictly +/// ascending canonical path order, so file lookup is a binary search and directory listing is a +/// contiguous range scan over the SHARED decode (`manifest` is the same object the Pool's +/// manifest cache holds). No I/O; never mutated after construction. All answers are pure functions +/// of the members. Every per-part file is an ordinary manifest tree entry, so a content change is +/// a manifest change through `repointRef`; comparing manifest IDs is therefore sufficient to detect +/// a stale retained view. The view never performs I/O or mutates the shared manifest. +class PartFolderView +{ +public: + /// Creates a view from a resolved ref and its validated shared manifest decode. The manifest + /// must be non-null and its entries must be strictly ascending by canonical path; this is the + /// ordering required by the binary-search and range-scan helpers. + PartFolderView(PartRefKey key_, Cas::ManifestId manifest_id_, uint64_t manifest_size_, + std::shared_ptr manifest_, uint64_t validated_at_ms_); + + /// Joins a fresh `Resolved` with its validated shared decode. `validated_at_ms` is supplied by + /// the caller after `readManifestShared` has proven the manifest body with a HEAD. Keeping the + /// timestamp outside this helper lets `CachedPartFolderAccess` use one injectable clock for both + /// the stamp and its age-window comparison. + static std::shared_ptr make( + PartRefKey key, const Cas::Resolved & resolved, + std::shared_ptr manifest, uint64_t validated_at_ms); + + /// Recognizes a projection directory by its last path component, `.proj` or `.tmp_proj`, and + /// returns the corresponding in-tree prefix. The input is the routed file path; unrelated paths + /// return nullopt. + static std::optional projectionDirPrefix(const std::string & file); + + const PartRefKey & refKey() const { return key; } + const Cas::ManifestId & manifestId() const { return manifest_id; } + const std::shared_ptr & manifest() const { return manifest_body; } + /// The wall-clock ms at which this view's manifest body was last proven live by a HEAD. A + /// refresh that changes only ref metadata carries the original stamp forward because it did not + /// re-prove the body. + uint64_t validatedAtMs() const { return validated_at_ms; } + + /// Finds an entry by canonical path using the manifest's sorted-entry invariant. + const Cas::ManifestEntry * findFile(const String & path) const; + /// Returns whether the manifest contains an entry at `path`. + bool hasFile(const String & path) const; + /// Returns the logical size of an inline or blob entry, or nullopt when absent. + std::optional fileSize(const String & path) const; + /// Returns inline bytes for an inline entry, or nullopt for absent and blob entries. + std::optional inlineBytes(const String & path) const; + /// Lists immediate child names below `dir_prefix`; the result is not required to be sorted. + std::vector listChildren(const String & dir_prefix) const; + /// Returns whether any manifest entry lies below `dir_prefix`. + bool hasDirectory(const String & dir_prefix) const; + /// Estimates the retained-cache weight, conservatively including the encoded manifest size. + size_t estimatedBytes() const; + +private: + PartRefKey key; + Cas::ManifestId manifest_id; + uint64_t manifest_size = 0; + std::shared_ptr manifest_body; + uint64_t validated_at_ms = 0; +}; + +} + +namespace DB::Cas { class PartWriteTxn; } + +namespace DB::Cas +{ + +/// Controls whether `ForceFresh` must re-prove the manifest body on every access. `Always` (the default) +/// preserves the fail-closed body check; `Age` and `Never` may serve a retained view after a fresh ref +/// resolve when its manifest ID matches. A ref resolve proves ref currency, but not that the manifest +/// body still exists, so these modes trade that additional check for a bounded performance optimization. +struct PartFolderValidate +{ + enum class Mode : uint8_t { Always, Age, Never }; + Mode mode = Mode::Always; + uint64_t age_seconds = 0; /// only meaningful for Mode::Age +}; + +class CachedPartFolderAccess; + +/// A part write that has been staged and PRECOMMITTED but not yet promoted -- the durable-but- +/// unpromoted state made into an owned object rather than an interval inside one call +/// (spec §relink-handle). It exists because the relink confirm has to interpose between the receiver's +/// `+1` becoming durable and the promote, so that the source can be asked whether it still holds the +/// manifest before the receiver commits to it. +/// +/// The handle OWNS an open `PartWriteTxn`, and DESTRUCTION IS NOT CLEANUP at the transaction level: +/// `~PartWriteTxn` only retires the build sequence, so a dropped precommit would keep a live-epoch +/// binding -- one the stale-precommit sweep (prior-epoch-scoped) never reclaims and GC never touches -- +/// permanently retaining the manifest's blobs. Exactly one terminal operation is therefore owed: +/// `promote` (commit) or `abort` (append the exact precommit removal). +/// +/// Getting that wrong is made impossible rather than merely discouraged: +/// - move-CONSTRUCT-only, and a move leaves the source terminal, so the duty is never held twice. +/// Move ASSIGNMENT is deleted: overwriting a handle that still owes a terminal has no correct +/// implementation. Discharging the duty first can FAIL (`abandon` appends through the ref lane, and +/// the lane can be wedged or fenced), and the assignment cannot report that -- so it would either +/// drop a cleanup owner permanently or refuse to complete an operation the language says cannot +/// fail. Nothing needs it: the one handle that travels (the interserver relink's) is move +/// CONSTRUCTED into place, and a contract that cannot be relied on is worse than no contract; +/// - an explicit terminal flag, set only once the underlying operation has actually completed, so a +/// second `promote`/`abort` is rejected with `LOGICAL_ERROR` instead of re-driving a dead +/// transaction -- while a terminal that FAILED (an append the caller may legitimately retry) leaves +/// the handle non-terminal; +/// - the destructor is the last-resort guard: a handle that reaches it non-terminal aborts +/// best-effort and logs, so a forgotten or exception-skipped terminal still appends the removal. +class PreparedPartWrite +{ +public: + PreparedPartWrite(const PreparedPartWrite &) = delete; + PreparedPartWrite & operator=(const PreparedPartWrite &) = delete; + PreparedPartWrite(PreparedPartWrite && other) noexcept; + PreparedPartWrite & operator=(PreparedPartWrite && other) = delete; + /// Best-effort abort of a handle that never reached a terminal state; never throws. + ~PreparedPartWrite(); + + /// Completes the write: the atomic precommit-to-committed owner move, plus the facade's cache + /// invalidation. The handle records the commit INSIDE the allocation-free region that immediately + /// follows the durable append, so nothing between "the ref is committed" and "this handle knows it" + /// can throw. On a failure that is PROVEN to have committed nothing the build is abandoned (the + /// catch-abandon-rethrow discipline the atomic `publishEntries` has always applied); the original + /// error propagates in either case. + CommitOutcome promote(bool allow_repoint = false); + /// Durably abandons the write: appends the EXACT precommit removal so the manifest's `+1` is + /// released. Propagates an append failure -- the caller may retry, and the destructor is the + /// backstop if it does not. + void abort(); + + /// Whether the owed terminal operation has already completed. A moved-from handle is terminal. + bool isTerminal() const { return terminal; } + /// Whether a `promote` attempt reached the ref lane's append and did not come back with a verdict, + /// i.e. the commit MAY be durable. Only meaningful after `promote` threw; false everywhere else, + /// including on a promote that failed its pre-append validation (proof of the negative) and on a + /// moved-from handle. A caller that treats a failed promote as "nothing was published" -- the + /// interserver relink's byte fallback -- must consult this first: doing that after a commit that + /// actually landed publishes the same part twice. + bool commitIsUnresolved() const; + /// The ref this write will commit to, and the manifest staged for it. + const PartRefKey & refKey() const { return key; } + const ManifestId & manifestId() const { return id; } + +private: + friend class CachedPartFolderAccess; + PreparedPartWrite(CachedPartFolderAccess & owner_, PartWriteTxnPtr build_, PartRefKey key_, ManifestId id_); + + CachedPartFolderAccess * owner = nullptr; + PartWriteTxnPtr build; + PartRefKey key; + ManifestId id; + bool terminal = false; +}; + +/// Single facade for committed content-addressed part-folder access. Reads build immutable +/// `PartFolderView`s; committed-ref mutations are facade methods so cache invalidation is write-through +/// rather than a caller responsibility. A bounded retained-view map is consulted for `CachedForLoad` +/// and checked against every fresh ref resolve. `cache_bytes == 0` disables retention while preserving +/// the uncached read path. The facade is thread-safe and shared by all readers and transactions of one disk. +class CachedPartFolderAccess +{ +public: + /// Retention knobs. `cache_bytes == 0` (the unit-test + /// default) disables retention entirely — the disk factory default is 64 MiB. + struct CacheParams + { + uint64_t cache_bytes = 0; /// 0 = retention disabled (unit-test default; + /// the DISK default is 64 MiB, set in the factory) + uint64_t max_entries = 10000; + uint64_t max_entry_bytes = 16ULL << 20; + /// The explain decision journal is test/log-only and its `recordDecision` + /// path takes a per-disk global mutex and allocates on EVERY read. Off by default so the read + /// hit path never pays for it; the disk factory / tests turn it on when they consult `explain`. + bool explain_enabled = false; + /// The `ForceFresh` manifest-body re-proof policy. `Always` is the fail-closed default. + PartFolderValidate validate; + }; + + /// `CacheParams params_ = {}` cannot be a default argument here — Clang's complete-class- + /// context rule requires the enclosing class (`CachedPartFolderAccess`) to be complete before a + /// nested class's (`CacheParams`) default member initializers can be evaluated, and a default + /// argument written inside the class body is evaluated too early. Two overloads sidestep it; the + /// single-arg form default-constructs `CacheParams` (retention disabled) out-of-line. + explicit CachedPartFolderAccess(Cas::PoolPtr store_); + /// `now_ms_fn_`: wall-clock ms, injected (tests) for the age-window comparison AND the + /// retained view's `validated_at_ms` stamp -- the SAME function drives both, so a test controls + /// each side of the comparison exactly. Defaults to `std::chrono::system_clock` (mirrors + /// `Cas::Gc`'s `now_ms_fn` convention) when empty. + CachedPartFolderAccess(Cas::PoolPtr store_, CacheParams params_, std::function now_ms_fn_ = {}); + + /// Resolves the ref and, when present, reads and validates its manifest into an immutable view. + /// `nullptr` means the ref is absent. Strict validation and the default `ForceFresh` policy reach + /// `readManifestShared`'s mandatory HEAD because a fresh ref resolve alone does not prove that the + /// manifest body still exists. + std::shared_ptr getView(const PartRefKey & key, Freshness freshness) const; + + /// Ref-only resolution (per-part reads, part-dir existence, publish stamps): no + /// manifest is read. `CachedForLoad` = stale-tolerant; other modes force-fresh. `audit` defaults to + /// `Emit` so every caller other than `getView` keeps emitting `RefResolve` unchanged; `getView` + /// passes `Deferred` and re-emits the event itself once it knows whether a warm view-cache hit + /// served the call without doing any real resolve work. + std::optional resolve(const PartRefKey & key, Freshness freshness, + Cas::ResolveAudit audit = Cas::ResolveAudit::Emit) const; + bool existsRef(const PartRefKey & key, Freshness freshness) const; + + /// ==== committed part-ref writes ==== + /// Each primitive performs the protocol operation and owns the cache side effect: + /// erase the affected view on success; on exception cache state is untouched — except + /// dropRefBestEffort, which erases even on a swallowed failure: in its destructor/rollback + /// context the ref's durable state is unknown, so dropping the view is the conservative + /// direction). Committed-ref mutations anywhere else in wiring are style-check failures. + + /// Completes a staged transaction with the atomic owner move and invalidates the affected view. + /// `allow_repoint` permits replacing a committed ref that names a different manifest. The returned + /// `CommitOutcome` is exact: `created` is derived INSIDE `PartWriteTxn::promote`'s `appendRefOps` + /// builder (the same in-closure-output pattern as that builder's own `repoint_old`), not read back + /// afterward. + /// + /// `commit_recorded`, when supplied, is set to `true` inside the allocation-free region that + /// immediately follows the durable append -- before the `CommitOutcome` is finished and before the + /// cache invalidation, both of which allocate and may therefore throw. It exists so a caller + /// holding a terminal duty (`PreparedPartWrite`) can record "committed" with nothing throwable in + /// between; without it an allocation failure in the post-commit work lands in the caller's + /// failed-promote handler with the ref already published. + CommitOutcome promoteBuild(Cas::PartWriteTxn & build, const PartRefKey & key, UInt128 build_id, + const Cas::ManifestId & manifest_id, bool allow_repoint = false, + bool * commit_recorded = nullptr); + /// The first half of the committed-publish sequence: adopt evidence over `entries`, stage a fresh + /// manifest, and precommit it -- then STOP. The receiver's `+1` is durable, the promote is deferred + /// until the caller has proven the source still holds the manifest (spec §relink-handle). The + /// returned handle OWNS the open transaction and must be either `promote`d or `abort`ed -- + /// destruction alone is not cleanup at the transaction level (`~PartWriteTxn` only retires the + /// build sequence), which is why the handle's own destructor aborts as a last resort. A failure + /// inside `prepareEntries` itself abandons the build before propagating, so no handle is returned + /// and no precommit is leaked. + PreparedPartWrite prepareEntries(const PartRefKey & dst, const std::vector & entries, + Cas::ProvenanceOp op); + /// Performs the shared committed-publish sequence: adopt evidence over + /// `entries`, stage a fresh manifest, precommit it, and promote it. The new manifest is fully + /// prepared before the committed ref is moved. Returns `promoteBuild`'s exact `CommitOutcome`. + /// Implemented as `prepareEntries` immediately followed by `promote`, so the atomic callers and the + /// confirm-interposed relink path share one protocol sequence. + CommitOutcome publishEntries(const PartRefKey & dst, const std::vector & entries, + Cas::ProvenanceOp op, bool allow_repoint = false); + /// Moves a committed ref by publishing the source entries at `dst` and then dropping `src`. + /// Returns false when the source is absent; a pre-existing destination with different content + /// is rejected rather than silently discarding the source. + bool republishRef(const PartRefKey & src, const PartRefKey & dst); + /// Republishes an already committed part with `entries` (for a standalone write or removal): + /// republishes `key`'s manifest with `entries`. Byte-equal candidate (same decoded entries as the + /// currently committed manifest) is a ZERO-pool-mutation no-op, returns false. Otherwise republishes + /// a byte-equal candidate returns false without pool mutation; an effective repoint publishes + /// through `publishEntries(allow_repoint=true)`, emits the repoint audit signals, invalidates the + /// cached view, and returns the exact `CommitOutcome` (`created` is false on both the byte-equal + /// no-op path -- `manifest_ref` names the manifest ALREADY committed, unchanged -- and on an + /// effective repoint of an existing ref). `key` must already resolve. + CommitOutcome repointRef(const PartRefKey & key, std::vector entries, Cas::ProvenanceOp op); + /// Drops a committed ref and invalidates its retained view after the drop succeeds. + void dropRef(const PartRefKey & key); + /// Idempotent removal: absent ref is success; a drop racing between resolve and the shard + /// re-read (FILE_DOESNT_EXIST) is success too — the removal unit is replay-safe. + void dropRefIfPresent(const PartRefKey & key); + /// Best-effort destructor/rollback cleanup: never throws, logs failure, and relies on GC to reclaim + /// lingering debris. The view is invalidated even when the durable ref state becomes unknown. + void dropRefBestEffort(const PartRefKey & key) noexcept; + /// Conditional rollback drop: removes `key`'s committed ref ONLY if its CURRENT committed manifest + /// binding equals `expected` (typically the `manifest_ref` from a `CommitOutcome` this caller + /// itself just produced). A ref repointed by someone else since then -- or already absent -- is + /// left untouched; the caller's stale rollback attempt must never clobber a newer commit. Reads + /// the committed binding and emits the removal op inside ONE `appendRefOps` builder (mirrors + /// `dropRef`'s underlying protocol call, guarded by the equality check inside the same closure). + /// `noexcept`: best-effort rollback context, like `dropRefBestEffort` -- swallows and logs a + /// failure rather than propagating it. Returns whether it actually removed the ref. + bool dropRefIfMatches(const PartRefKey & key, const ManifestRef & expected) noexcept; + /// Drops all refs in a namespace and removes every retained view belonging to that namespace. + void dropNamespace(const Cas::RootNamespace & ns); + + /// ==== diagnostics ==== + enum class LastDecision : uint8_t + { Hit, Miss, OversizedBypass, StrictBypass, ForceFreshRead, Invalidated }; + struct ExplainResult + { + bool retained = false; /// whether the last-served view is currently retained + LastDecision last_decision = LastDecision::Miss; + String manifest_ref; /// manifestRefDebugString of the last-served view + size_t estimated_bytes = 0; + }; + /// Returns the test/log-only decision record for `key`; an absent key yields the default result. + ExplainResult explain(const PartRefKey & key) const; + + /// Test-only seam (nothing installs it in production): fires in `promoteBuild` immediately after + /// the commit has been recorded and before the throwable post-commit work (`CommitOutcome` + /// assembly's copies, `eraseView`). A test uses it to model an allocation failure in exactly that + /// window and assert that the caller's terminal duty is already discharged. + void setPostCommitProbeForTest(std::function fn) { post_commit_probe_for_test = std::move(fn); } + /// Test-only: number of entries in the decision journal (0 whenever explain is disabled). + size_t explainJournalSizeForTest() const; + +private: + Cas::PoolPtr store; + CacheParams params; + /// Wall-clock milliseconds; see the constructor comment. `std::function::operator` is const, so this is + /// callable from const methods (`getView`, `buildView`) without a `mutable` qualifier. + std::function now_ms_fn; + + /// Supplies the conservative encoded-manifest weight used by `CacheBase` for eviction decisions. + struct ViewWeight + { + size_t operator()(const PartFolderView & v) const { return v.estimatedBytes(); } + }; + using ViewCache = CacheBase, ViewWeight>; + + /// nullptr <=> retention disabled (cache_bytes == 0): same call graph, no retained map. + std::unique_ptr view_cache; + + /// Single-flight per PartRefKey for the build path: concurrent cold builders of the same key + /// share ONE readManifestShared. NEVER held across I/O — the map only hands out futures. + mutable std::mutex inflight_mutex; + mutable std::unordered_map>> inflight; + + /// Reads a manifest and constructs a view. Cold `CachedForLoad` builds are single-flight per key; + /// fresh modes perform their own read so each call retains its validation guarantee. + std::shared_ptr buildView( + const PartRefKey & key, const Cas::Resolved & resolved, Freshness freshness) const; + /// Removes a retained view and records the invalidation for diagnostics. + void eraseView(const PartRefKey & key); + /// Emits the same `RefResolve` `CasEvent` `CasRefLedger::resolveRef` would have emitted for + /// `resolved`, a no-op when no sink is installed. Used by `getView`, which defers the emit from its + /// own `resolve(..., ResolveAudit::Deferred)` call so it can skip it on a warm view-cache hit that + /// served without any real resolve work. + void emitResolveEvent(const PartRefKey & key, const Cas::Resolved & resolved) const; + + /// Decision journal for `explain` (test/log-only). Bounded by wholesale + /// clear — debug state, never consulted by the read/write paths. + static constexpr size_t EXPLAIN_MAX_ENTRIES = 10000; + mutable std::mutex explain_mutex; + mutable std::unordered_map explain_map; + /// Records the latest diagnostic decision without affecting read or write behavior. + void recordDecision(const String & cache_key, LastDecision decision, + const PartFolderView * view, bool retained) const; + + /// See `setPostCommitProbeForTest`. Empty in production. + std::function post_commit_probe_for_test; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartPathParser.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartPathParser.cpp new file mode 100644 index 000000000000..fc27132b211c --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartPathParser.cpp @@ -0,0 +1,400 @@ +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Split a path into non-empty components, treating repeated or leading/trailing '/' characters as +/// separators. The parser deliberately does not normalize or otherwise interpret components: path +/// classification must be based on the names supplied by the disk layer. +static std::vector splitNonEmpty(const std::string & path) +{ + std::vector parts; + std::string cur; + for (char c : path) + { + if (c == '/') + { + if (!cur.empty()) + parts.push_back(std::move(cur)); + cur.clear(); + } + else + { + cur.push_back(c); + } + } + if (!cur.empty()) + parts.push_back(std::move(cur)); + return parts; +} + +namespace +{ + +/// The split of a disk-relative path into non-empty components is the dominant allocation of every +/// path classifier, and the CA metadata read path runs SEVERAL of them on the SAME raw path per +/// logical file-open (each of `existsFile` / `getFileSize` / `getStorageObjects` first calls +/// `isPartFilePath`, then `parsePartFilePath`). The split is a PURE function of the path, so a small +/// thread-local FIFO ring cache keyed on the raw path is always correct, disk-agnostic and +/// lock-free. It is a fixed-capacity round-robin ring, NOT an LRU/MRU: a hit does not move or +/// promote its slot, so under sustained eviction pressure a path can be re-split on a later call +/// even if it was seen recently — always still CORRECT (re-splitting just re-derives the same +/// result), only less effective as a cache. The returned reference stays valid until the next +/// `splitCached` call on the SAME thread; every classifier consumes its split before splitting +/// again (none splits while holding another's split). +struct SplitCache +{ + static constexpr size_t kCapacity = 8; + std::array>, kCapacity> slots; + size_t count = 0; /// populated slots (<= kCapacity) + size_t next = 0; /// round-robin insertion cursor + size_t misses = 0; /// underlying `splitNonEmpty` invocations (observability / test oracle) + + /// Return the cached split for `path`, or replace the next round-robin slot with a newly split + /// value. A hit does not promote its slot, so this is a fixed-capacity FIFO-style ring rather + /// than an LRU. The returned reference remains valid until the next `get` call on this thread. + const std::vector & get(const std::string & path) + { + for (size_t i = 0; i < count; ++i) + if (slots[i].first == path) + return slots[i].second; + ++misses; + auto & slot = slots[next]; + slot.first = path; + slot.second = splitNonEmpty(path); + next = (next + 1) % kCapacity; + if (count < kCapacity) + ++count; + return slot.second; + } +}; + +thread_local SplitCache tls_split_cache; + +const std::vector & splitCached(const std::string & path) +{ + // Every caller consumes the returned components before asking for another split on this + // thread, so the cache may safely reuse its ring slots between classifier invocations. + return tls_split_cache.get(path); +} + +} + +/// Whether every character of `s` is a lowercase hex digit. +static bool isLowerHex(std::string_view s) +{ + return std::all_of(s.begin(), s.end(), [](char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); }); +} + +/// Whether `s` has the exact shape of a canonical UUID string: 36 characters, dashes at positions +/// 8/13/18/23, lowercase hex everywhere else. +static bool looksLikeUuidDirName(std::string_view s) +{ + if (s.size() != 36) + return false; + for (size_t i = 0; i < 36; ++i) + { + const bool dash_pos = (i == 8 || i == 13 || i == 18 || i == 23); + if (dash_pos != (s[i] == '-')) + return false; + if (!dash_pos && !((s[i] >= '0' && s[i] <= '9') || (s[i] >= 'a' && s[i] <= 'f'))) + return false; + } + return true; +} + +/// Locate the `/` anchor inside a split Atomic path. The leading prefix is normally +/// `store`, but may be absent from a disk-relative path, so the parser identifies the pair by its +/// shape instead of requiring a particular prefix. Return the index of the UUID component; the +/// component immediately after it is the part directory when one exists. +static std::optional findTableUuidComponent(const std::vector & p) +{ + for (size_t i = 1; i < p.size(); ++i) + { + const auto & prefix = p[i - 1]; + const auto & uuid = p[i]; + /// Shape-based on purpose (robust to a missing `store/`), but the shape is now the REAL + /// Atomic one: 3 lowercase-hex chars followed by a well-formed UUID sharing that prefix — + /// a 3-char database named like its table (`data/abc/abcxyz/...`) no longer false-anchors. + if (prefix.size() == 3 && isLowerHex(prefix) && looksLikeUuidDirName(uuid) + && uuid.compare(0, 3, prefix) == 0) + return i; + } + return std::nullopt; +} + +/// Return whether a component has the MergeTree part-directory grammar. Non-Atomic layouts do not +/// have an Atomic UUID anchor, so their part boundary is found from the final three non-empty +/// decimal underscore-separated groups: `_min_max_level`, optionally preceded by a mutation number. +/// The grammar also covers temporary and operation prefixes such as `tmp_insert_all_1_1_0` and +/// `delete_tmp_all_1_1_0`; this helper has no Storage dependency and is used only as the non-Atomic +/// fallback. +static bool looksLikePartDir(const std::string & name) +{ + std::vector groups; + std::string cur; + for (char c : name) + { + if (c == '_') + { + groups.push_back(cur); + cur.clear(); + } + else + cur.push_back(c); + } + groups.push_back(cur); + + // Need at least ___: a partition group plus 3 trailing numeric groups. + if (groups.size() < 4) + return false; + + auto is_number = [](const std::string & s) + { + if (s.empty()) + return false; + for (char c : s) + if (c < '0' || c > '9') + return false; + return true; + }; + + const size_t n = groups.size(); + return is_number(groups[n - 1]) && is_number(groups[n - 2]) && is_number(groups[n - 3]); +} + +/// Describes the boundary found by `findPartDirComponent`: components in [table_start, part_idx) +/// form the table identifier, `part_idx` names the part or reserved part container, and all later +/// components form the in-part file path. Atomic identifiers contain one UUID component; non-Atomic +/// identifiers contain the complete `data//` prefix. +struct PartDirAnchor +{ + size_t table_start; + size_t part_idx; +}; + +/// Locate the part-directory component. Prefer the UUID anchor for Atomic paths. Without it, treat +/// the first reserved `detached` or `moving` component after the table root as the boundary, then +/// fall back to the rightmost component matching the part-directory grammar. The reserved-directory +/// scan must precede the right-to-left grammar scan: otherwise the inner part name would be mistaken +/// for the boundary and the reserved directory would become part of a spurious table namespace that +/// table cleanup does not own. Return nullopt when the path has no part component. +static std::optional findPartDirComponent(const std::vector & p) +{ + if (auto uuid_idx = findTableUuidComponent(p)) + { + const size_t part_idx = *uuid_idx + 1; + if (part_idx < p.size()) + { + // The reserved deduplication-log directory is a table-level subdir, not a part dir + // (see kDeduplicationLogsDirName). + if (p[part_idx] == kDeduplicationLogsDirName) + return std::nullopt; + return PartDirAnchor{*uuid_idx, part_idx}; // table id = the single component + } + return std::nullopt; // table dir, no part component after the uuid + } + + // No uuid anchor: a non-Atomic table path. `detached` (data///detached//...) + // and `moving` (data//
/moving//...) are both reserved table-level subdirs, + // exactly like the Atomic layout where the uuid anchor makes them the part_name for free. + // Anchor on either FIRST (leftmost, index >= 1): the right-to-left part-dir scan below would + // otherwise anchor on the INNER -shaped component and fold the reserved dir into a + // spurious table id (data//
/detached or .../moving) that DROP TABLE never cleans, + // orphaning a permanently-live ref. Mirrors + // route()'s part_name == kDetachedDirName / kMovingDirName folding. + for (size_t i = 1; i < p.size(); ++i) + if (p[i] == kDetachedDirName || p[i] == kMovingDirName) + return PartDirAnchor{0, i}; // table id = the whole path before the reserved dir + + // A non-Atomic database or table literally named `detached` is necessarily interpreted as the + // reserved directory. The two shapes are indistinguishable from a path string alone; resolving + // the ambiguity requires caller-supplied knowledge of existing databases and tables, which this + // pure string parser intentionally does not have. The reserved interpretation is retained so + // ordinary detached-part paths continue to map into the real table namespace. The test + // `CasPartPathParser.DetachedNamedTableIsKnownAmbiguityFoldedAsReservedDir` pins this behavior so + // any future change here is a conscious one. + + // The table identifier must be at least one component (a real table dir, never the bare disk + // root), so the part dir is at index >= 1. Scan right to left so a part-dir-shaped + // table/partition name earlier in the path cannot steal the anchor. + for (size_t i = p.size(); i-- > 1;) + if (looksLikePartDir(p[i])) + return PartDirAnchor{0, i}; // table id = the whole path before the part dir + return std::nullopt; +} + +/// Join components [start, end) with '/' into the stable table identifier used by the routing layer: +/// one UUID component for Atomic paths or the complete `data//` path for non-Atomic paths. +static std::string joinTableId(const std::vector & p, size_t start, size_t end) +{ + std::string id; + for (size_t i = start; i < end; ++i) + { + if (!id.empty()) + id += "/"; + id += p[i]; + } + return id; +} + +/// Return the number of underlying `splitNonEmpty` invocations on the current thread for cache +/// observability and tests. +size_t splitCacheMissesForTest() +{ + return tls_split_cache.misses; +} + +void resetSplitCacheForTest() +{ + tls_split_cache = SplitCache{}; +} + +std::optional parsePartFilePath(const std::string & path) +{ + const auto & p = splitCached(path); + auto anchor = findPartDirComponent(p); + if (!anchor) + return std::nullopt; + + PartFilePath r; + r.table_uuid = joinTableId(p, anchor->table_start, anchor->part_idx); + r.part_name = p[anchor->part_idx]; + if (anchor->part_idx + 1 < p.size()) + { + std::string file = p[anchor->part_idx + 1]; + for (size_t i = anchor->part_idx + 2; i < p.size(); ++i) + file += "/" + p[i]; + r.file = file; + } + // FREEZE target: shadow//.../. Capture both the backup name (the component + // right after the reserved "shadow" root) and the literal shadow table dir — the joined + // components before the part dir — for the commit / read / remove routing. The inner uuid + // anchor above is unaffected by the prefix. + if (p.size() >= 2 && p[0] == kShadowDirName) + { + r.backup_name = p[1]; + r.shadow_table_dir = joinTableId(p, 0, anchor->part_idx); + } + return r; +} + +std::optional parseTableUuid(const std::string & path) +{ + const auto & p = splitCached(path); + + // Atomic layout: exactly the table dir //[/] — nothing after the uuid. + if (auto uuid_idx = findTableUuidComponent(p); uuid_idx && *uuid_idx + 1 == p.size()) + return p[*uuid_idx]; + + // Non-Atomic layout: a directory path with no part-dir component is the table dir + // data//
. Require at least two components so the bare disk root (or a single generic + // dir) is never taken as a table dir. + if (findTableUuidComponent(p)) + return std::nullopt; // had a uuid anchor but something followed it: not a table dir + if (p.size() >= 2 && !findPartDirComponent(p)) + return joinTableId(p, 0, p.size()); + return std::nullopt; +} + +bool isAtomicShardDir(const std::string & path) +{ + // The Atomic on-disk layout shards table dirs as `store//@cas@`, so `store/` is a + // pure intermediate shard directory: the literal `store` root followed by exactly one 3-char + // uuid-prefix component, with nothing after it. This is ambiguous with the non-Atomic + // data/ fallback (both are two non-part components with no uuid anchor), so the router uses + // this strict predicate to disambiguate before parseTableUuid. + const auto & p = splitCached(path); + return p.size() == 2 && p[0] == "store" && p[1].size() == 3; +} + +bool endsWithTableUuidPair(const std::string & path) +{ + const auto & p = splitCached(path); + auto uuid_idx = findTableUuidComponent(p); + return uuid_idx && *uuid_idx + 1 == p.size(); +} + +bool isPartFilePath(const std::string & path) +{ + // A file inside a part dir: // => at least one component after the + // part dir, for both the Atomic and non-Atomic layouts. + const auto & p = splitCached(path); + auto anchor = findPartDirComponent(p); + return anchor && anchor->part_idx + 1 < p.size(); +} + +std::optional parseTableFilePath(const std::string & path) +{ + const auto & p = splitCached(path); + + // Atomic layout: a table-level file lives under the table dir, i.e. at least one component + // after the uuid. The tail is EVERYTHING after the uuid joined by '/', so a table-level file in + // a subdirectory (deduplication_logs/deduplication_log_1.txt) keeps its full sub-path. A part + // file is excluded earlier by isPartFilePath; this function is only reached for non-part paths. + if (auto uuid_idx = findTableUuidComponent(p)) + { + if (*uuid_idx + 1 >= p.size()) + return std::nullopt; // the bare table dir, no file tail + TableFilePath r; + r.table_uuid = p[*uuid_idx]; + r.tail = joinTableId(p, *uuid_idx + 1, p.size()); + return r; + } + + // Non-Atomic layout: a path with no part-dir component whose last component is the table-level + // file, and the components before it are the table dir data//
. Require the table id + // to be at least one component (a real table, never the bare disk root). + if (p.size() < 2 || findPartDirComponent(p)) + return std::nullopt; + + // A reserved table-level subdirectory (deduplication_logs/) splits the path explicitly: the + // table id is everything before it, the tail is the reserved dir and everything under it. + // Without this the generic "last component is the file" rule would fold the subdir into the + // table id and mis-scope the log objects. Index >= 1 so the table id is never the bare root. + for (size_t i = 1; i + 1 < p.size(); ++i) + { + if (p[i] == kDeduplicationLogsDirName) + { + TableFilePath r; + r.table_uuid = joinTableId(p, 0, i); + r.tail = joinTableId(p, i, p.size()); + return r; + } + } + + TableFilePath r; + r.table_uuid = joinTableId(p, 0, p.size() - 1); + r.tail = p.back(); + return r; +} + +std::string mirroredArchiveNamespace(const std::string & table_uuid) +{ + if (table_uuid.find('/') == std::string::npos) + { + /// Atomic: a bare uuid; mirror ClickHouse's store// fanout. + const std::string u3 = table_uuid.substr(0, 3); + return "store/" + u3 + "/" + table_uuid + std::string(kCasArchiveSuffix); + } + /// Non-Atomic: a full data// path already; append the suffix to the last segment. + return table_uuid + std::string(kCasArchiveSuffix); +} + +bool isShadowPath(const std::string & path) +{ + size_t i = 0; + while (i < path.size() && path[i] == '/') + ++i; + const auto first_end = path.find('/', i); + const std::string_view first = first_end == std::string::npos + ? std::string_view(path).substr(i) + : std::string_view(path).substr(i, first_end - i); + return first == kShadowDirName; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartPathParser.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartPathParser.h new file mode 100644 index 000000000000..ab509e837452 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Parts/PartPathParser.h @@ -0,0 +1,146 @@ +#pragma once +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Classifies disk-relative ClickHouse paths for the content-addressed metadata-storage wiring. +/// It handles Atomic and non-Atomic table layouts, FREEZE shadow trees, detached and moving part +/// directories, table-level deduplication logs, projections, and temporary or operation part names. +/// The functions are pure and side-effect-free: they identify the table, part, and file portions, +/// while content-addressed key construction remains in `Cas::Layout`. The `Cas` core therefore +/// never needs to understand ClickHouse path conventions. + +/// The literal first path component reserved for FREEZE snapshots +/// (shadow//store////...). +inline constexpr std::string_view kShadowDirName = "shadow"; + +/// The MergeTree detached-parts directory. parsePartFilePath reports a detached path with +/// part_name == kDetachedDirName and the real detached part dir as the FIRST component of `file`. +/// The transaction and read routing re-split that `file` value to recover the detached part's +/// actual name and in-part path. +inline constexpr std::string_view kDetachedDirName = "detached"; + +/// The MergeTree part-mover staging directory (`MergeTreeData::MOVING_DIR_NAME`, +/// `MergeTreeData.h`). A part being relocated to another disk (explicit `ALTER … MOVE +/// PART|PARTITION`, or a background TTL/policy move) is cloned under TABLE/moving// +/// before the atomic rename into its final place. `parsePartFilePath` reports such a path with +/// part_name == kMovingDirName and the real part dir as the FIRST component of `file` -- the +/// exact same shape `kDetachedDirName` already produces, for free, on the Atomic layout (no parser +/// change is needed there: "moving" already lands on `part_idx` +/// because it is the component right after the table , same as "detached"). Mirroring +/// detached, `route()` folds this onto a `moving/`-PREFIXED ref (kMovingRefPrefix) -- NOT the +/// part's final ref directly. Publishing the clone under the final ref would break move +/// crash-atomicity: a crash between the clone commit and the mover's rename would leave a +/// committed live ref before the swap ever happened, and `moving/`'s own startup cleanup +/// couldn't tell that premature ref apart from a real live part. The staging ref keeps the +/// pre-swap clone un-live; the mover's rename does a real ref repoint moving/ -> . +inline constexpr std::string_view kMovingDirName = "moving"; + +/// Detached parts live inside the table's own archive namespace as refs keyed by this prefix — +/// `detached/PART` versus a live `PART`. One namespace per table; the live-vs-detached +/// name collision is impossible because the ref names differ. The routing prepends this to the +/// detached part name to form the ref, and the `TABLE/detached` container dir surfaces the +/// table's refs filtered to this prefix (stripped for display). No parallel detached namespace +/// exists anymore (the old `detachedNamespace` is gone). +inline constexpr std::string_view kDetachedRefPrefix = "detached/"; + +/// MOVE-to-CA fix: mirrors kDetachedRefPrefix exactly, but for the mover's `moving/` staging +/// dir instead of `detached/`. Keeps a moved-but-not-yet-swapped part's ref distinct from its +/// eventual live ref ``, so the destination CA transaction (`clonePart`'s CA branch) can +/// publish it WITHOUT prematurely making it live or colliding with an existing live part of the +/// same name. +inline constexpr std::string_view kMovingRefPrefix = "moving/"; + +/// The content-addressing boundary marker: a SUFFIX on a table-dir segment (`…/@cas@`), not a +/// path segment. It marks where the mirrored ClickHouse path ends and the content-addressed archive +/// begins — like a `.zip` extension (`foo.zip/inner/file`). `@` is S3-safe and never occurs in +/// ClickHouse uuids, part names, detached prefixes, projection names, or column files, so it cannot +/// collide with real path data. Namespace discovery comes from the catalog, not path classification. +inline constexpr std::string_view kCasArchiveSuffix = "@cas@"; + +/// Compose the mirrored content-addressed archive path for a table identifier as the parser reports +/// it. Atomic tables report the bare `` → reconstruct `store//@cas@` (u3 = first 3 +/// chars, matching ClickHouse's store fanout). Non-Atomic tables report the full joined +/// `data//` path → append `@cas@` to it verbatim. The `@cas@` suffix lands on the +/// table-dir (last) segment in both cases. Pure; no ClickHouse dependency. +std::string mirroredArchiveNamespace(const std::string & table_uuid); + +/// Reserved table-level subdirectory: TABLE_DIR/deduplication_logs/FILE is structurally +/// indistinguishable from a part file in the Atomic layout, so the name is reserved — never a part +/// dir; its contents are table-level verbatim files. ClickHouse part names never take this form. +inline constexpr std::string_view kDeduplicationLogsDirName = "deduplication_logs"; + +/// The result of splitting a part path. `table_uuid` is a bare UUID for an Atomic table and the +/// joined `data//` path for a non-Atomic table. For detached or moving paths, +/// `part_name` is the reserved directory name and `file` begins with the actual part directory; +/// this preserves the on-disk shape expected by the routing layer. FREEZE paths additionally +/// retain both the backup name and the literal shadow table directory for shadow-specific routing. +struct PartFilePath +{ + std::string table_uuid; + std::string part_name; + std::string file; /// empty when the path is a part directory + /// Set to the backup name when the path is a FREEZE target shadow//.../[/]. + /// Empty for a normal live-part path. + std::string backup_name; + /// Set (alongside backup_name) for a FREEZE target: the LITERAL shadow table dir under the disk + /// root excluding the part and file (shadow//store//). Empty otherwise. + std::string shadow_table_dir; +}; + +/// Parse a disk-relative ClickHouse path to its (table, part, in-part file) split. Anchors on the +/// Atomic / pair anywhere in the path (robust to a leading store/ or shadow/ +/// prefix); falls back to the RIGHTMOST part-dir-shaped component for non-Atomic layouts +/// (data/db/table/part/...). Returns nullopt for the table dir or shallower. +std::optional parsePartFilePath(const std::string & path); + +/// Returns the table identifier iff path is exactly a table dir: the bare for the Atomic +/// layout, the full joined data/db/table path for non-Atomic. +std::optional parseTableUuid(const std::string & path); + +/// True iff the path is an Atomic-layout INTERMEDIATE shard directory `store/`, where is a +/// 3-character uuid prefix (the only child it has on disk is a uuid-anchored `/` table +/// dir). This shape is ambiguous with the non-Atomic `data/` fallback of parseTableUuid, so the +/// metadata router must consult it FIRST and treat `store/` as a generic intermediate dir to be +/// enumerated by a mirrored LIST — never as a non-Atomic table id. +bool isAtomicShardDir(const std::string & path); + +/// Strict "this dir IS a uuid-anchored table dir" predicate: the path's LAST two components form +/// an Atomic / pair. Unlike parseTableUuid it rejects the non-Atomic fallback — +/// the shadow router uses it to tell a shadow TABLE dir from a shadow INTERMEDIATE dir. +bool endsWithTableUuidPair(const std::string & path); + +/// True iff the path addresses a file INSIDE a part dir (content-addressed). Table-level files +/// (format_version.txt, deduplication_logs/...) and generic disk files are excluded. +bool isPartFilePath(const std::string & path); + +/// The result of splitting a table-level file path. `table_uuid` uses the same Atomic versus +/// non-Atomic representation as `PartFilePath`, while `tail` preserves the complete path below +/// that table directory, including a nested `deduplication_logs/` prefix when present. +struct TableFilePath +{ + std::string table_uuid; + std::string tail; /// path beyond the table dir, full sub-path preserved +}; + +/// Parse a non-part table-level file path. Returns nullopt for the bare table dir, shallower +/// paths, part files, and generic disk-root files (e.g. clickhouse_access_check_*). +std::optional parseTableFilePath(const std::string & path); + +/// True iff the path's FIRST component is the reserved FREEZE shadow root. Routed BEFORE the +/// live-table branches (a shadow table dir also satisfies parseTableUuid). +bool isShadowPath(const std::string & path); + +/// Return the number of underlying `splitNonEmpty` invocations on the current thread. This is an +/// observability seam for verifying that repeated classifiers reuse the thread-local split cache. +size_t splitCacheMissesForTest(); + +/// Clear the current thread's split cache and reset its miss counter. This is intended for tests; +/// production callers must not rely on cache state surviving between operations. +void resetSplitCacheForTest(); + +} From a840c03b425e61a570a69431af97cfca8eb11d04 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:35 +0200 Subject: [PATCH 18/30] CAS subsystem: GC layer The garbage-collection round: fold, in-degree settlement, lease and heartbeat, prune, baseline rebuild, ack-floor fencing. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../ContentAddressed/Gc/CasBlobInDegree.cpp | 723 +++ .../ContentAddressed/Gc/CasBlobInDegree.h | 413 ++ .../ContentAddressed/Gc/CasGc.cpp | 4661 +++++++++++++++++ .../ContentAddressed/Gc/CasGc.h | 1012 ++++ .../Gc/CasGcMaintenanceState.cpp | 40 + .../Gc/CasGcMaintenanceState.h | 29 + .../ContentAddressed/Gc/CasGcPhaseTimer.h | 86 + .../ContentAddressed/Gc/CasGcScheduler.cpp | 419 ++ .../ContentAddressed/Gc/CasGcScheduler.h | 227 + .../ContentAddressed/Gc/CasGcShardPlan.cpp | 65 + .../ContentAddressed/Gc/CasGcShardPlan.h | 138 + .../Gc/CasNamespaceJanitor.cpp | 143 + .../ContentAddressed/Gc/CasNamespaceJanitor.h | 34 + .../Gc/CasOrphanManifestSweep.cpp | 915 ++++ .../Gc/CasOrphanManifestSweep.h | 206 + .../Gc/CatalogLifecycleReconciler.cpp | 121 + .../Gc/CatalogLifecycleReconciler.h | 64 + 17 files changed, 9296 insertions(+) create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasBlobInDegree.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasBlobInDegree.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMaintenanceState.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMaintenanceState.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcPhaseTimer.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcScheduler.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcScheduler.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcShardPlan.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcShardPlan.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CatalogLifecycleReconciler.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CatalogLifecycleReconciler.h diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasBlobInDegree.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasBlobInDegree.cpp new file mode 100644 index 000000000000..1702682a0003 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasBlobInDegree.cpp @@ -0,0 +1,723 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event CASGCRetiredSparedByReref; + extern const Event CASGCUnmatchedRemoveDeltas; +} +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::Cas +{ + +namespace +{ + +const UInt128 kZeroSourceId{0}; + +/// Streams a shard's prior source-edge run at O(one block) resident memory: chains the run SEGMENTS the +/// caller resolved from the parent seal (`blob_target_runs` filtered to one shard) and exposes a one-row +/// lookahead for the fold merge. The prior run carries +/// BOTH surviving edges (`kEdgeActive`) AND the retired `kCondemned` sentinel rows at the zero source id, +/// so the cursor stops at edges AND at condemned rows (exposing the type via `rowType`), while zero-marker +/// sentinels are dropped on carry (per-generation, never carried forward). Row/key invariants are enforced +/// while streaming: `kEdgeActive` never at `source_id = 0`; sentinel rows (`kZeroMarker` / +/// `kCondemned`) ONLY at `source_id = 0`; at most one sentinel per blob; an unknown value byte or an empty +/// payload is `CORRUPTED_DATA`. Resolution uses the exact object references supplied by the caller, so a run +/// sealed for generation G that physically lives under an older generation's key is reached +/// without key construction. An empty `segments` is the fresh-pool / empty baseline. The row stream is +/// globally sorted by (blob_hash, source_id), so `key()` values are non-decreasing. +class PriorEdgeCursor +{ +public: + /// The key codec is stateless and self-describing, so a run may freely mix supported hash algorithms. + PriorEdgeCursor(Backend & backend_, const std::vector & segments_) + : backend(backend_), segments(segments_) + { + advance(); + } + + bool valid() const { return has_current; } + const String & key() const { return current_key; } + /// The value byte of the current row: `kEdgeActive` (a surviving edge) or `kCondemned` (a retired + /// sentinel row). Zero markers are never surfaced (dropped on carry). + char rowType() const { return current_type; } + /// The decoded retired sentinel for the current row (only valid when `rowType() == kCondemned`). + const CondemnedRow & condemnedRow() const { return current_condemned; } + + /// Advance to the next surviving edge OR retired sentinel, dropping zero markers, enforcing the + /// row/key invariants, and crossing segment boundaries. + void advance() + { + while (true) + { + /// Pull rows from the open segment until a surviving edge, retired sentinel, or the segment ends. + if (reader) + { + String k; + String p; + while (reader->next(k, p)) + { + BlobRef bh; + UInt128 sid; + /// `parse` throws CORRUPTED_DATA on a malformed size / NOT_IMPLEMENTED on an + /// unknown algo byte (fail-closed). + SourceEdgeKeyCodec::parse(k, bh, sid); + if (p.empty()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS source-edge run: empty row payload"); + const char v = p[0]; + const bool sentinel_key = (sid == kZeroSourceId); + + if (sentinel_key) + { + /// A sentinel key carries exactly one row per blob and never an edge. + if (v == kEdgeActive) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS source-edge run: active edge at the reserved sentinel source_id 0"); + if (v != kZeroMarker && v != kCondemned) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS source-edge run: unknown sentinel row type 0x{:02x}", static_cast(v)); + if (have_sentinel_blob && sentinel_blob == bh) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS source-edge run: duplicate sentinel row for one blob"); + have_sentinel_blob = true; + sentinel_blob = bh; + if (v == kZeroMarker) + continue; // A zero marker is per-generation and is dropped on carry. + /// A retired sentinel: decode and surface it (settled at close-out, not an edge). + current_condemned = decodeCondemnedRow(p); + current_key = k; + current_type = kCondemned; + has_current = true; + return; + } + + /// A non-sentinel key must carry a surviving edge and nothing else. + if (v != kEdgeActive) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS source-edge run: sentinel row type 0x{:02x} at a non-sentinel key", + static_cast(v)); + current_key = k; + current_type = kEdgeActive; + has_current = true; + return; + } + /// Segment fully drained: verify its whole-file checksum against + /// the seal's RunRef.checksum BEFORE the fold acts on any of its rows. This is the fold — + /// the checksum's most important consumer (it decides deletions). Fail-closed on mismatch. + reader->verifyAgainst(segments[seg_idx].checksum); + reader.reset(); + ++seg_idx; + } + + /// Open the next resolved segment; the segment list is exhausted => the chain is done. + if (seg_idx >= segments.size()) + { + has_current = false; + return; + } + /// Typed open validates the NDJSON header before any row is consumed. Each row carries its + /// own algorithm byte, so no separate width gate is needed. + reader = openSourceEdgeRun(backend, segments[seg_idx].key); + } + } + +private: + Backend & backend; + const std::vector & segments; + + size_t seg_idx = 0; + std::optional reader; + String current_key; + char current_type = kEdgeActive; + CondemnedRow current_condemned; + bool has_current = false; + + /// Duplicate-sentinel guard: the last blob for which a sentinel row was seen (across skipped zero + /// markers too). Rows are globally sorted, so two sentinels for one blob are adjacent. + bool have_sentinel_blob = false; + BlobRef sentinel_blob{}; +}; + +} + +UInt128 sourceEdgeId(const ManifestId & id, const String & path) +{ + String canon; + canon += id.root_namespace.string(); + canon += '\0'; + auto beU64 = [&](uint64_t v) { for (int i = 7; i >= 0; --i) canon += static_cast((v >> (8 * i)) & 0xFF); }; + auto beU32 = [&](uint32_t v) { for (int i = 3; i >= 0; --i) canon += static_cast((v >> (8 * i)) & 0xFF); }; + beU64(id.ref.writer_epoch); beU64(id.ref.build_sequence); beU32(id.ref.manifest_ordinal); + canon += '\0'; + canon += path; + const auto h = CityHash_v1_0_2::CityHash128(canon.data(), canon.size()); + const UInt128 result = (static_cast(h.high64) << 64) | static_cast(h.low64); + if (result == UInt128{0}) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS source edge: hash collided with the reserved sentinel id 0"); + return result; +} + +void assertValidSourceEdgeId(const UInt128 & source_id) +{ + if (source_id == UInt128{0}) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS source edge: source_id 0 is the reserved sentinel key"); +} + +String encodeCondemnedRow(const CondemnedRow & row) +{ + String out; + out.push_back(kCondemned); + out.push_back(static_cast((row.delete_pending ? 1 : 0) | (row.marker_confirmed ? 2 : 0))); + out.push_back(static_cast(row.token.type)); + auto beU64 = [&](uint64_t v) { for (int i = 7; i >= 0; --i) out += static_cast((v >> (8 * i)) & 0xFF); }; + beU64(row.condemn_round); + beU64(row.size); + if (row.token.value.size() > 0xFFFF) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS condemned row: token too long ({})", row.token.value.size()); + out += static_cast((row.token.value.size() >> 8) & 0xFF); + out += static_cast(row.token.value.size() & 0xFF); + out += row.token.value; + return out; +} + +CondemnedRow decodeCondemnedRow(std::string_view p) +{ + /// [0]=0x02 [1]=flags [2]=token_type [3..10]=round [11..18]=size [19..20]=len [21..]=value + constexpr size_t kFixed = 21; + if (p.size() < kFixed || p[0] != kCondemned) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS condemned row: malformed header"); + CondemnedRow row; + const uint8_t flags = static_cast(p[1]); + if (flags > 3) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS condemned row: unknown flags 0x{:02x}", flags); + row.delete_pending = flags & 1; + row.marker_confirmed = flags & 2; + const uint8_t type = static_cast(p[2]); + if (type < 1 || type > 3) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS condemned row: unknown token_type {}", type); + row.token.type = static_cast(type); + auto beU64 = [&](size_t off) { uint64_t v = 0; for (int i = 0; i < 8; ++i) v = (v << 8) | static_cast(p[off + i]); return v; }; + row.condemn_round = beU64(3); + row.size = beU64(11); + const size_t len = (static_cast(p[19]) << 8) | static_cast(p[20]); + if (p.size() != kFixed + len) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS condemned row: declared token_len {} vs payload {}", len, p.size() - kFixed); + row.token.value = String(p.substr(kFixed, len)); + return row; +} + +SourceEdgeRunView::SourceEdgeRunView(std::unique_ptr stream_) + : stream(std::move(stream_)) + , reader(std::make_unique(*stream)) /// the reader borrows *stream; both are members +{ +} + +bool SourceEdgeRunView::next(String & key, String & payload) +{ + SourceEdgeRecord rec; + if (!reader->next(rec)) + return false; + /// Reconstruct the packed SourceEdgeKeyCodec key + the ORIGINAL payload bytes (a single marker byte, + /// or the encoded condemned row) so the fold / zeroInDegree / previewDeletes / fsck consumers keep + /// their exact parse/compare logic against the NDJSON codec. + key = SourceEdgeKeyCodec::key(rec.ref, rec.source_id); + switch (rec.marker) + { + case kEdgeActive: + case kZeroMarker: + payload = String(1, rec.marker); + break; + case kCondemned: + payload = encodeCondemnedRow(CondemnedRow{.delete_pending = rec.delete_pending, + .token = rec.token, .size = rec.size, + .condemn_round = rec.condemn_round, + .marker_confirmed = rec.marker_confirmed}); + break; + default: + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS source-edge run: unknown row marker 0x{:02x}", static_cast(rec.marker)); + } + return true; +} + +void SourceEdgeRunView::verifyAgainst(const UInt128 & expected) +{ + reader->verifyAgainst(expected); +} + +UInt128 SourceEdgeRunView::accumulatedChecksum() +{ + return reader->accumulatedChecksum(); +} + +SourceEdgeRunView openSourceEdgeRun(std::string_view bytes) +{ + return SourceEdgeRunView(std::make_unique(bytes.data(), bytes.size())); +} + +SourceEdgeRunView openSourceEdgeRun(Backend & backend, const String & key) +{ + /// Streaming: `getStream` is a forward-only read of the write-once run — nothing is + /// materialized whole (cas_run is object_cap = 0). Absent object => fail-closed. + auto sr = backend.getStream(key); + if (!sr) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS source-edge run: object {} is absent", key); + return SourceEdgeRunView(std::move(sr->stream)); +} + +namespace +{ +/// len-drift guard (mirrors DigestCodec::checkZeroTail, CasBlobDigest.h): a caller passing a digest +/// wider than the algo's own width is a programming bug, not corrupted on-disk data — chassert, not +/// throw. +void checkZeroTailForAlgo(const BlobDigest & d, uint8_t digest_len, [[maybe_unused]] const char * what) +{ + for (size_t i = digest_len; i < d.bytes.size(); ++i) + chassert(d.bytes[i] == 0, fmt::format("SourceEdgeKeyCodec::{}: non-zero byte at tail position {} (digest_len={})", what, i, digest_len)); +} +} + +String SourceEdgeKeyCodec::key(const BlobRef & ref, const UInt128 & source_id) +{ + const uint8_t digest_len = static_cast(blobHashLenFor(ref.algo)); + checkZeroTailForAlgo(ref.digest, digest_len, "key"); + String out; + out.push_back(static_cast(static_cast(ref.algo))); + out += String(reinterpret_cast(ref.digest.bytes.data()), digest_len); + out += u128ToBytesBE(source_id); + return out; +} + +void SourceEdgeKeyCodec::parse(std::string_view key, BlobRef & ref, UInt128 & source_id) +{ + if (key.empty()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS source-edge run: empty key"); + const uint8_t algo_byte = static_cast(key[0]); + BlobHashAlgo algo{}; + switch (algo_byte) + { + case static_cast(BlobHashAlgo::CityHash128): algo = BlobHashAlgo::CityHash128; break; + case static_cast(BlobHashAlgo::XXH3_128): algo = BlobHashAlgo::XXH3_128; break; + case static_cast(BlobHashAlgo::Sha256): algo = BlobHashAlgo::Sha256; break; + default: + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "CAS source-edge run: unknown algo byte {} in key", algo_byte); + } + const uint8_t digest_len = static_cast(blobHashLenFor(algo)); + if (key.size() != 1 + static_cast(digest_len) + 16) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS source-edge run: malformed key ({} bytes, expected {})", key.size(), 1 + digest_len + 16); + ref = BlobRef{}; + ref.algo = algo; + memcpy(ref.digest.bytes.data(), key.data() + 1, digest_len); + source_id = u128FromBytesBE(String(key.substr(1 + digest_len, 16)), "src-edge run key source_id"); +} + +void putDeterministicArtifact(Backend & backend, const String & key, const String & bytes) +{ + if (backend.putIfAbsent(key, bytes).outcome == PutOutcome::PreconditionFailed) + { + const auto existing = backend.get(key); + if (!existing || existing->bytes != bytes) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS gc: deterministic artifact at {} occupied by divergent bytes (impossible under " + "correct operation; refusing to proceed)", key); + /// byte-equal => our own deterministic replay; adopt (no-op). + } +} + +void foldDeltasIntoGeneration(Backend & backend, const Layout & layout, + const std::vector & prior_runs, + uint64_t new_generation, uint64_t attempt, + uint64_t shard, + std::vector scattered, std::vector & out_runs, + uint64_t current_round, uint64_t condemn_round, + const std::function(const BlobRef &)> & head_blob, + const std::function(const BlobRef &)> & peek_head, + const std::function & confirm_condemned_marker, + RetiredMergeResult * out_retired, + bool suppress_destructive, + std::vector * out_applied_by_txn_ordinal, + std::vector source_retirements, + GcRoundWorkBudget * work_budget) +{ + RetiredMergeResult sink; + RetiredMergeResult & rmr = out_retired ? *out_retired : sink; + + // Deterministic input ordering produces a byte-reproducible run for safe retry and adoption. + // MUST be stable: for the same (ref, source_id) the journal ordering is + // activation-before-removal; "last wins" then correctly resolves to removal (edge absent). + // An unstable sort can put removal before activation => last=activation => false positive. + // The comparator is exactly (ref.algo, ref.digest, source_id) == BlobRef::operator< then + // source_id — the same order the raw keys sort in (`SourceEdgeKeyCodec::key`'s algo + // byte decides before any digest byte can), so the merge below stays a plain key comparison. + std::stable_sort(scattered.begin(), scattered.end(), + [](const BlobDelta & a, const BlobDelta & b) + { + if (a.ref != b.ref) return a.ref < b.ref; + return a.source_id < b.source_id; + }); + std::sort(source_retirements.begin(), source_retirements.end(), + [](const BlobSourceRetirement & a, const BlobSourceRetirement & b) + { + if (a.ref != b.ref) return a.ref < b.ref; + return a.source_id < b.source_id; + }); + + PriorEdgeCursor cursor(backend, prior_runs); + + DB::WriteBufferFromOwnString out; + SourceEdgeRunWriter writer(out); // sorted NDJSON; byte-deterministic for write-once adoption + + // Streaming two-cursor merge over the prior run (surviving edges AND retired kCondemned + // sentinel rows at the zero source id) and this round's edge deltas (by (blob_hash, source_id)). All + // rows for one blob are adjacent in both inputs; the sentinel key (source_id 0) sorts first. We resolve + // final presence per edge locally (idempotent: prior present + activate => present; any remove => + // absent), settle each blob's carried retired row against its post-merge in-degree at close-out, and + // re-emit the surviving retired rows / zero-transition markers. O(block) IO + O(1) per current blob. + size_t di = 0; + size_t ri = 0; + BlobRef cur_blob{}; + bool have_blob = false; + uint64_t cur_edges = 0; // surviving edges of cur_blob so far + bool cur_touched = false; // cur_blob had prior edges or deltas this generation + std::optional cur_condemned; // the retired sentinel carried on the prior run for cur_blob + + auto toRetiredEntry = [](const BlobRef & ref, const CondemnedRow & r) -> RetiredEntry + { + RetiredEntry e; + e.kind = ObjectKind::Blob; + e.ref = ref; + e.token = r.token; + e.size = r.size; + e.condemn_round = r.condemn_round; + e.delete_pending = r.delete_pending; + e.marker_confirmed = r.marker_confirmed; + return e; + }; + /// THE DELETE-SITE IN-DEGREE RE-READ IS NORMATIVE (spec §5, third arm). It is not an optimization + /// and not defense-in-depth: it is the last of the three things that keep a delete from racing a + /// live edge, and the only one that acts on THIS pass's freshly merged view. + /// + /// 1. the round-paced floor: a blob condemned in round R cannot graduate before R+1, so a `+1` + /// that lands in the same round as the condemnation is always folded before any delete; + /// 2. the exact-token delete: a writer that resurrected the blob replaced its incarnation, so a + /// stale token's delete finds a TokenMismatch and removes nothing; + /// 3. THIS: the entry is settled against `indeg` recomputed by the merge that just ran, so an edge + /// folded after the condemnation but before the delete pass spares the blob outright -- + /// `indeg > 0` wins over `delete_pending`, unconditionally and past the floor. + /// + /// Arms 1 and 2 bound WHEN and WHAT a delete may remove; only this one asks whether the blob is + /// still referenced at the moment the pass decides. Removing it -- or reordering the branches so + /// that `delete_pending` is checked first -- silently deletes re-referenced blobs on exactly the + /// interleaving the other two arms do not cover. Any change here needs a test that fails without it. + auto settleEntry = [&](const RetiredEntry & e, uint64_t indeg) + { + chassert(e.kind == ObjectKind::Blob); /// the in-degree merge settles Blob entries only + if (indeg > 0) + { + /// A delete_pending entry recovering in-degree is the expected shape of a dedup-adopt vs + /// condemn race, not an ack-floor violation: a graduated blob carries NO surviving prior + /// edges (see the comment on the sentinel emission below), so any edge that resurrects its + /// in-degree is necessarily a FRESH this-generation edge -- a writer's `observeAndAdmit` + /// point-read of the per-hash meta raced GC's `Condemned` write and adopted the (about to + /// be deleted) token instead of resurrecting from source. Spare it LOUDLY (never a + /// fail-closed abort, never a delete of a re-referenced blob), but at Debug: this is a + /// routine, safely-handled race, not something to page on. + if (e.delete_pending) + { + /// No LoggerPtr is threaded this deep (foldDeltasIntoGeneration is a free function + /// shared by the non-sharded fold and CasGcShardPlan's per-shard reduce); scope the + /// message with the pool's own key prefix instead so a multi-disk process's logs can + /// still be attributed. + LOG_DEBUG(getLogger("CasGcFold"), + "CAS gc fold ({}): delete_pending blob {} (condemned at round {}, observed at round {}) " + "recovered in-degree {} -- a fresh dedup-adopt raced the condemn; sparing (never a " + "fail-closed delete)", layout.poolPrefix(), blobIdOf(e.ref), e.condemn_round, current_round, indeg); + ProfileEvents::increment(ProfileEvents::CASGCRetiredSparedByReref); + } + rmr.spared.push_back(e); /// recovery wins, even past the floor + } + else if (e.delete_pending) + { + /// Excess past the round's redelete budget is carried unchanged (still `delete_pending`) — + /// exactly the suppressed-pass shape below — rather than skipped ahead in `scattered`, so a + /// budget-exhausted round retries the same entry next round instead of losing it. + if (suppress_destructive || (work_budget && !work_budget->redeleteAvailable())) + rmr.still_retired.push_back(e); /// clamp-suppressed or budget-exhausted: carry UNCHANGED + else + { + if (work_budget) + ++work_budget->redeletes_used; + rmr.redelete.push_back(e); /// published pending by a PRIOR pass — execute + drop + } + } + else if (!suppress_destructive && e.condemn_round < current_round) + { + /// Graduation gate (triage 2026-07-17 §3.4): publishing delete_pending is the one edge that + /// authorizes an irreversible delete, and it requires CONFIRMED durable Condemned evidence + /// for this exact (hash, token) — the marker is the writer's adopt gate, so an entry whose + /// marker write was swallowed could be same-token adopted invisibly to this fold's cut. + /// Unconfirmed => carry unchanged (fail-safe delay; the gate callback retries the marker so a + /// later pass can confirm). This gates a DELETE on missing evidence; it never throws. + if (e.marker_confirmed || !confirm_condemned_marker || confirm_condemned_marker(e)) + { + /// Excess past the round's graduation budget carries the floor-passed entry unchanged + /// (still condemned, not yet delete_pending) — it re-evaluates the floor next round and + /// graduates then; nothing is lost, only delayed. + if (work_budget && !work_budget->graduationAvailable()) + rmr.still_retired.push_back(e); + else + { + if (work_budget) + ++work_budget->graduations_used; + RetiredEntry pending = e; /// newly floor-passed: publish pending; delete NEXT pass + pending.delete_pending = true; + pending.marker_confirmed = true; + rmr.graduated.push_back(pending); + rmr.still_retired.push_back(std::move(pending)); + } + } + else + rmr.still_retired.push_back(e); /// no durable condemn-marker evidence yet — carried + } + else + rmr.still_retired.push_back(e); /// carried unchanged until the floor passes it + }; + + auto closeBlob = [&]() + { + if (!have_blob) + return; + const size_t retired_before = rmr.still_retired.size(); + + /// Settle the retired row carried on the prior run for the blob being closed, against its + /// post-merge in-degree... + if (cur_condemned) + { + /// The retired row already identifies the blob with the native `BlobRef` used by the run. + const RetiredEntry stale = toRetiredEntry(cur_blob, *cur_condemned); + /// On a re-reference cycle (touched this window, net in-degree 0), + /// re-observe the CURRENT token. If it differs from the retired row's token, a resurrect + /// replaced the incarnation at this key — supersede the stale entry with a fresh condemn of the + /// current token so the replacement enters the pipeline (the stale token's exact-token delete + /// would only find the new token and no-op). Keyed on (hash, current token), matching GRetire. + /// `peek_head` is deliberately side-effect-free and is not `head_blob` — + /// `head_blob` is the fresh-condemn hook (emits `BlobRetire` + increments + /// `CASGCRetiredCondemned`); calling it here would double-emit `blob_retire` alongside the + /// `blob_retire_replaced` this supersede already produces below, and double-count the + /// condemned counter for one physical condemnation. + bool superseded = false; + if (cur_edges == 0 && cur_touched && peek_head) + { + if (const auto hr = peek_head(cur_blob); + hr && hr->exists && hr->token != stale.token) + { + RetiredEntry fresh; + fresh.kind = ObjectKind::Blob; + fresh.ref = cur_blob; + fresh.token = hr->token; + fresh.size = hr->size; + fresh.condemn_round = condemn_round; + ReplacedEntry re; + re.old_token = stale.token; /// the stale token this supersede replaces + re.fresh = fresh; + rmr.replaced.push_back(std::move(re)); /// caller emits blob_retire_replaced + rmr.still_retired.push_back(std::move(fresh)); + superseded = true; + } + } + if (!superseded) + settleEntry(stale, cur_edges); + } + /// ...or condemn a fresh transition-to-zero (no carried row). `head_blob` captures the exact + /// incarnation token for the later exact-token delete; an absent object needs no entry. + else if (cur_edges == 0 && cur_touched && head_blob) + { + if (const auto hr = head_blob(cur_blob); hr && hr->exists) + { + RetiredEntry fresh; + fresh.kind = ObjectKind::Blob; + fresh.ref = cur_blob; + fresh.token = hr->token; + fresh.size = hr->size; + fresh.condemn_round = condemn_round; + rmr.still_retired.push_back(std::move(fresh)); + } + } + + /// Emit at most one sentinel row per blob: the `kCondemned` row when the + /// blob is condemned/carried/graduated this pass (still_retired grew for it), else a per-generation + /// `kZeroMarker` when it transitioned to zero this pass but was not condemned (redelete-dropped or + /// absent-at-condemn). A blob with surviving edges (cur_edges > 0) emits neither — its edge rows + /// were appended inline, and a condemned/zeroed blob has NO surviving edges, so appending the + /// sentinel now (its key sorts first for the blob, and no edge rows precede it) keeps the run + /// sorted. `still_retired` therefore mirrors exactly the emitted `kCondemned` rows, in order. + if (rmr.still_retired.size() > retired_before) + { + const RetiredEntry & e = rmr.still_retired.back(); + writer.append(SourceEdgeRecord{.ref = cur_blob, .source_id = kZeroSourceId, .marker = kCondemned, + .delete_pending = e.delete_pending, .token = e.token, + .size = e.size, .condemn_round = e.condemn_round, + .marker_confirmed = e.marker_confirmed}); + } + else if (cur_edges == 0 && cur_touched) + writer.append(SourceEdgeRecord{.ref = cur_blob, .source_id = kZeroSourceId, .marker = kZeroMarker}); + }; + auto openBlobIfNeeded = [&](const BlobRef & b) + { + if (!have_blob || b != cur_blob) + { + closeBlob(); + cur_blob = b; have_blob = true; cur_edges = 0; cur_touched = false; cur_condemned.reset(); + } + }; + + while (cursor.valid() || di < scattered.size() || ri < source_retirements.size()) + { + // Pick the smallest row key across the prior-run cursor and this round's deltas. + String key; + bool from_prior = false; + if (cursor.valid()) { key = cursor.key(); from_prior = true; } + if (di < scattered.size()) + { + const String dk = SourceEdgeKeyCodec::key(scattered[di].ref, scattered[di].source_id); + if (!from_prior || dk < key) { key = dk; from_prior = false; } + } + if (ri < source_retirements.size()) + { + const String rk = SourceEdgeKeyCodec::key(source_retirements[ri].ref, source_retirements[ri].source_id); + if ((!from_prior && key.empty()) || rk < key) { key = rk; from_prior = false; } + } + + BlobRef blob_ref; + UInt128 source_id; + SourceEdgeKeyCodec::parse(key, blob_ref, source_id); // throws CORRUPTED_DATA on a malformed key (fail-closed) + openBlobIfNeeded(blob_ref); + + /// A retired sentinel row from the prior run: stash it for close-out settlement. It is not an edge + /// and NEVER a touch — a carried kCondemned row must not force a zero-marker or a peek_head HEAD + /// and never a touch. Deltas never key the zero source id, so no delta merges at this key. + if (from_prior && cursor.rowType() == kCondemned) + { + cur_condemned = cursor.condemnedRow(); + cursor.advance(); + continue; + } + + bool present = false; + if (from_prior && cursor.key() == key) { present = true; cursor.advance(); cur_touched = true; } + while (di < scattered.size() + && scattered[di].ref == blob_ref && scattered[di].source_id == source_id) + { + /// An unmatched remove: `present` was false immediately before this remove delta is + /// applied, meaning neither the prior run nor an earlier delta in this same scattered run + /// for this key had activated it. The set semantics make this a harmless per-key no-op + /// (never a false deletion), but a persistent nonzero rate is a correctness signal — count + /// it and hand ONE example back to the caller, who logs once per round (never from this + /// hot inner loop; it runs over potentially millions of rows). + if (scattered[di].remove && !present) + { + ++rmr.unmatched_removes; + ProfileEvents::increment(ProfileEvents::CASGCUnmatchedRemoveDeltas); + if (!rmr.unmatched_remove_example) + rmr.unmatched_remove_example = UnmatchedRemoveExample{blob_ref, source_id}; + } + /// PROBE B2: this delta reached a reducer and is being CONSUMED. Marked here rather than + /// at run flush because the in-degree model is a SET — an unmatched `-1` and a duplicate + /// `+1` legitimately vanish inside the merge, so a flush-side mark would fire on healthy + /// rounds. See `Cas::TxnApplyLedger`. + if (out_applied_by_txn_ordinal) + (*out_applied_by_txn_ordinal)[scattered[di].txn_ordinal] = 1; + present = scattered[di].remove ? false : true; // apply in order; last wins + cur_touched = true; + ++di; + } + + /// Orphan nomination retires this exact manifest-source identity after ordinary ref deltas at + /// the same key. It is accounting-neutral: absence is an idempotent no-op, not an unmatched + /// ref removal, and there is no transaction ordinal to mark in B2's apply ledger. + while (ri < source_retirements.size() + && source_retirements[ri].ref == blob_ref && source_retirements[ri].source_id == source_id) + { + present = false; + cur_touched = true; + ++ri; + } + + if (present) + { + writer.append(SourceEdgeRecord{.ref = blob_ref, .source_id = source_id, .marker = kEdgeActive}); + ++cur_edges; + } + } + closeBlob(); + + writer.finish(); + out.finalize(); + const String run_bytes = out.str(); + /// Whole-object streaming checksum: the same chained CityHash128 the reader + /// accumulates on the read path, replacing the retired one-shot cityHash128. Carried by the fold + /// seal's RunRef.checksum and verified before any consumer acts on the run. + const UInt128 run_checksum = sourceEdgeRunChecksum(run_bytes); + const String run_key = layout.blobTargetRunKey(new_generation, attempt, shard, 0); + putDeterministicArtifact(backend, run_key, run_bytes); + out_runs.push_back(RunRef{.key = run_key, .checksum = run_checksum, + .shard = shard, .generation = new_generation}); +} + +std::vector zeroInDegree(Backend & backend, const std::vector & runs) +{ + std::vector result; + for (const RunRef & run : runs) + { + /// The caller passes the exact object key, so a run sealed + /// for a later generation but physically living under an older key is reached directly. The run is + /// streamed at O(one block) resident memory, never materialized whole. `openSourceEdgeRun` enforces + /// the run kind + key schema; `kCondemned` sentinel rows are skipped (only `kZeroMarker` counts). + SourceEdgeRunView r = openSourceEdgeRun(backend, run.key); + String k; + String p; + while (r.next(k, p)) + if (!p.empty() && p[0] == kZeroMarker) + { + BlobRef bh; + UInt128 sid; + /// `parse` throws `CORRUPTED_DATA` on a malformed key; malformed rows must not be silently + /// treated as absent candidates. + SourceEdgeKeyCodec::parse(k, bh, sid); + result.push_back(BlobCandidate{.ref = bh}); + } + /// Whole-file checksum: verify the drained run against the seal's + /// RunRef.checksum BEFORE its candidates feed a GC delete decision. Fail-closed on mismatch. + r.verifyAgainst(run.checksum); + } + return result; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasBlobInDegree.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasBlobInDegree.h new file mode 100644 index 000000000000..7067b71d8b9c --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasBlobInDegree.h @@ -0,0 +1,413 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// In-memory description of a blob incarnation condemned by the in-degree merge. The exact token and +/// size are captured from the blob HEAD so the GC caller can issue an exact-token deletion later; +/// `condemn_round` controls round-paced graduation. Entries are decoded from `kCondemned` rows and are +/// returned through `RetiredMergeResult`; the type itself has no serialized representation. +struct RetiredEntry +{ + ObjectKind kind = ObjectKind::Blob; + BlobRef ref{}; + Token token; /// the exact incarnation token GC observed (exact-token delete) + uint64_t size = 0; + uint64_t condemn_round = 0; /// the GC round that condemned this incarnation (round-paced + /// graduation: an entry graduates only once condemn_round < the + /// current round). Consulted by GC only; the writer never reads it. + bool delete_pending = false; /// Two-phase graduation: floor-passed and + /// published for deletion; the NEXT pass executes the exact-token + /// delete (pre-CAS, safe at any leader staleness) and drops the entry. + /// Terminal: a pending entry is never un-pended (writers keep seeing + /// it condemned and recreate). + bool marker_confirmed = false; /// Durable `Condemned` meta CONFIRMED for this entry — the + /// graduation gate (triage 2026-07-17 §3.4): the per-hash condemn + /// marker is the writer's adopt gate, so graduation to + /// delete_pending requires confirmed durable evidence; an + /// unconfirmed entry is CARRIED, never fail-open deleted. Set at + /// graduation (delete_pending rows always carry it). +}; + +/// Backend-independent codec for source-edge keys. A key is `algo` (u8), the digest at that algorithm's +/// native width, and `source_id` (16 bytes, big-endian). The packed byte order is exactly +/// `(BlobRef, source_id)` order, which lets the fold merge compare keys directly. The leading algorithm +/// byte makes the digest width self-describing; supported algorithms may therefore be mixed in one run. +class SourceEdgeKeyCodec +{ +public: + SourceEdgeKeyCodec() = delete; + + /// key = algo(u8) ++ digest[blobHashLenFor(algo)] ++ source_id(16 BE); 33 or 49 bytes. + static String key(const BlobRef & ref, const UInt128 & source_id); + /// Parse a key. Throws `NOT_IMPLEMENTED` on an unknown algo byte, `CORRUPTED_DATA` on a wrong + /// total length for a known algo. Zero-tails the digest (beyond the algo's own width). + static void parse(std::string_view key, BlobRef & ref, UInt128 & source_id); +}; + +/// Deterministic 16-byte id of a source edge (ManifestId, path). Distinctness only — not reconstructable. +UInt128 sourceEdgeId(const ManifestId & id, const String & path); + +/// The zero source_id is the reserved sentinel key (used internally for the zero-marker row) — +/// producers of real source edges must fail closed on a hash collision with it. +void assertValidSourceEdgeId(const UInt128 & source_id); + +/// Serialized payload of a condemned source-edge sentinel. The payload retains the full incarnation +/// token, including its type, because deletion must remain exact-token guarded. Its fixed prefix is +/// `[0x02][flags][token_type][round BE64][size BE64][token_len BE16]`, followed by token bytes. +/// `flags` bit 0 is `delete_pending`, bit 1 is `marker_confirmed`. +struct CondemnedRow +{ + bool delete_pending = false; + Token token; // {value, type} — the full token required by exact-token deletion + uint64_t size = 0; + uint64_t condemn_round = 0; + bool marker_confirmed = false; // durable Condemned meta confirmed (graduation gate) + bool operator==(const CondemnedRow &) const = default; +}; + +/// Encode a condemned-row payload. Throws `CORRUPTED_DATA` if the token cannot fit in its u16 length. +String encodeCondemnedRow(const CondemnedRow & row); + +/// Decode and validate a condemned-row payload. Unknown flags, token types, or inconsistent lengths +/// throw `CORRUPTED_DATA`. +CondemnedRow decodeCondemnedRow(std::string_view payload); + +/// Bridges the backend-free `Formats/CasRecordStreamFormat` NDJSON reader to the +/// `(key, payload)` BYTE interface the fold / `zeroInDegree` / `previewDeletes` / `fsck` consumers use: +/// `next` reconstructs the packed `SourceEdgeKeyCodec` key and the original payload bytes (a single +/// marker byte for an edge / zero row, or the `encodeCondemnedRow` blob for a condemned row) from the +/// decoded NDJSON record. So the codec stays backend-free while the consumers keep their exact parse / +/// compare logic. The whole-object chained CityHash128 is accumulated as the run streams; `verifyAgainst` +/// checks it against the fold seal's `RunRef.checksum` AFTER the run is fully drained and BEFORE the +/// caller acts on it (a deletion decision). +class SourceEdgeRunView +{ +public: + /// false once the run's `{"n"}` trailer is consumed (the trailer count is verified there). `key` is + /// the reconstructed `SourceEdgeKeyCodec::key(ref, source_id)`; `payload` is the original marker byte + /// or `encodeCondemnedRow` bytes. + bool next(String & key, String & payload); + /// Verify the accumulated whole-file checksum against the seal's `RunRef.checksum`; CORRUPTED_DATA on + /// mismatch. Call after draining the run and before acting on its records. + void verifyAgainst(const UInt128 & expected); + /// The accumulated whole-file checksum after draining the run — non-throwing, for a read-only auditor + /// (fsck) that catalogues a mismatch and continues instead of failing closed. Call after draining. + UInt128 accumulatedChecksum(); + +private: + friend SourceEdgeRunView openSourceEdgeRun(std::string_view bytes); + friend SourceEdgeRunView openSourceEdgeRun(Backend & backend, const String & key); + /// Keep the underlying stream alive for the reader, which borrows it rather than owning it. + explicit SourceEdgeRunView(std::unique_ptr stream_); + + std::unique_ptr stream; /// owns the backend stream / memory buffer the reader borrows + std::unique_ptr reader; /// over *stream (non-movable => held by pointer, destroyed before stream) +}; + +/// Open a typed source-edge run. The NDJSON header must identify a `cas_run` of kind `source_edge`; +/// otherwise opening fails closed. The memory overload borrows caller-owned bytes. The backend overload +/// streams the write-once object through `getStream`, retaining only one record-sized buffer. +SourceEdgeRunView openSourceEdgeRun(std::string_view bytes); +SourceEdgeRunView openSourceEdgeRun(Backend & backend, const String & key); + +/// Store a deterministic write-once artifact (same inputs => byte-identical bytes): the blob in-degree +/// runs and fold seals. `putIfAbsent`; on a `PreconditionFailed` the key is already +/// occupied — `get` it and compare bytes: byte-equal means our own deterministic replay (adopt, no-op), +/// divergent bytes are impossible under correct operation and we fail closed with `CORRUPTED_DATA` +/// rather than let a divergent artifact disagree with the adopted snapshot. Deterministic artifacts are +/// therefore byte-equal-or-`CORRUPTED_DATA`. It is +/// NOT for observation-bearing artifacts (outcome logs) — those carry HEAD-observed +/// tokens that two observers may legitimately differ on and keep first-durable-write-wins semantics. +void putDeterministicArtifact(Backend & backend, const String & key, const String & bytes); + +/// One source-edge update before merging: the edge `(ref, source_id)`, and whether it is an activation +/// (+edge) or a removal (−edge). Idempotent under re-fold at the merge (set membership, not a counter). +struct BlobDelta +{ + BlobRef ref{}; + UInt128 source_id{}; /// `sourceEdgeId(ManifestId, path)` — an edge identity, not a content hash + bool remove = false; + /// Round-local index of the ref transaction that emitted this delta, into `Cas::TxnApplyLedger`. + /// NEVER persisted and never part of any comparator: it exists only so the reducer can prove it + /// consumed at least one delta from every transaction the round declared covered (probe B2). It + /// lands in the struct's existing tail padding — see `kBlobDeltaSize` below. + /// + /// DECLARED LAST, and that is load-bearing: several tests brace-initialise a `BlobDelta` + /// POSITIONALLY as `{ref, source_id, remove}`. Inserting a field ahead of `remove` silently rebinds + /// that third initialiser to the ordinal and turns every removal delta into an activation — a + /// compiling, type-correct, wrong-answer change. Any future field goes after this one, or every + /// aggregate initialiser gets converted to designated form first. + uint32_t txn_ordinal = 0; +}; + +/// The fold's per-edge row runs over potentially millions of records per round, so its size is a +/// deliberate property, not an accident. Pinned here (rather than left to a perf run to discover) +/// because probe B2's `txn_ordinal` was added into the existing tail padding: the row did NOT grow. +/// A future field that breaks this assertion is a hot-path change and must be argued as one. +static constexpr size_t kBlobDeltaSize = 64; +static_assert(sizeof(BlobDelta) == kBlobDeltaSize, + "BlobDelta is the fold's hot per-edge row; growing it is a hot-path change"); + +/// One orphan-manifest source edge to retire without pretending it came from a ref transaction. +/// The reducer applies this as an idempotent exact-key removal, but it deliberately consumes no B2 +/// transaction ordinal and an already-absent edge is not an unmatched-remove correctness signal. +struct BlobSourceRetirement +{ + BlobRef ref{}; + UInt128 source_id{}; +}; + +/// A blob whose active source-edge set became empty this generation — a retire candidate. +struct BlobCandidate +{ + BlobRef ref{}; +}; + +/// Merge the prior generation's blob source-edge run for `shard` with `scattered` deltas, producing the +/// new generation's write-once run under blobTargetRunKey(new_generation, attempt, shard, 0). Streaming: +/// prior run + scattered deltas are sorted by (blob_hash, source_id) and merged via two-cursor scan; the +/// source-edge set is idempotent under re-fold (identical (blob_hash, source_id) pairs deduplicate). A blob +/// whose active edge set transitions to exactly empty this generation is written as an explicit zero-marker +/// row so `zeroInDegree` can stream it; prior-generation zero markers are dropped. Appends the produced +/// run's `RunRef` (key + footer checksum + `shard` + `new_generation`) to `out_runs` for the fold seal. +/// +/// `prior_runs` are the parent generation's run segments for this shard, resolved by the caller from the +/// parent fold seal's `blob_target_runs` (filtered to `shard`). An empty vector is the fresh-pool / empty +/// baseline. A run sealed for one generation may physically live under an older generation's key, so the +/// seal's exact reference is authoritative and key construction is not used. `new_generation`, `attempt`, +/// and `shard` name only the output run's key namespace. +/// The fresh entry that re-condemns the current +/// token, paired with the STALE entry's token it superseded. Kept as its own struct (rather than a +/// field bolted onto `RetiredEntry`) so the common merge element stays slim — only replaced entries +/// carry the extra superseded token. +struct ReplacedEntry +{ + RetiredEntry fresh; /// the freshly condemned CURRENT token (also pushed into still_retired byte-identically) + Token old_token; /// the superseded (stale) entry's token — what the resurrect replaced +}; + +/// One example of an unmatched-remove delta, kept for the caller's single once-per-round WARNING +/// (see `RetiredMergeResult::unmatched_removes` below) — naming the blob and source id is enough to +/// start an investigation without logging every occurrence from the hot inner loop. +struct UnmatchedRemoveExample +{ + BlobRef ref{}; + UInt128 source_id{}; +}; + +/// Outcome of the retired merge: the same +/// streaming pass that folds edges settles every prior retired entry and detects new candidates. +struct RetiredMergeResult +{ + std::vector still_retired; /// carried + newly-condemned + newly-PENDING entries (the next list) + std::vector graduated; /// newly floor-passed this pass — published pending, deleted NEXT pass + std::vector spared; /// in-degree recovered — entry dropped + std::vector redelete; /// pending in the PRIOR list — execute deleteExact pre-CAS, drop + std::vector replaced; /// re-condemned CURRENT tokens that superseded a stale entry (resurrect-replaced); caller emits blob_retire_replaced + + /// Count of `remove == true` deltas that matched no presence for their `(BlobRef, source_id)` key — + /// neither the prior run nor an earlier delta in the same `scattered` batch had activated it. The + /// in-degree model is a SET, so an unmatched remove is a per-key no-op BY DESIGN (never a false + /// deletion), but a persistent nonzero rate means removal deltas are reaching the reducer without + /// their matching activation, which is a correctness signal worth paging on. See + /// `ProfileEvents::CASGCUnmatchedRemoveDeltas`. + uint64_t unmatched_removes = 0; + /// The first unmatched remove observed this merge, for the caller's WARNING (one example is enough + /// to start an investigation; logging every occurrence would flood a hot per-edge inner loop). + std::optional unmatched_remove_example; +}; + +/// Cumulative per-round cap over EVERY destructive-or-observability-write work family a round touches — +/// blob graduation and redelete (this file), the orphan-manifest planner's namespace fan-out and recovery +/// walk (`CasOrphanManifestSweep.cpp`), ref-object/generation-prefix cleanup, the post-CAS hand-off +/// reclaim, the post-CAS manifest-body cleanup, and the `GcOutcomes` audit log (`CasGc.cpp`). One instance +/// is owned by `Gc::runRegularRound` per round and passed by reference/pointer into each family's call +/// in turn, so a cap is cumulative across the WHOLE round, not reset per shard or per namespace. `0` in +/// any bound is unbounded — the same opt-out convention every other CAS round budget uses, so a +/// default-constructed instance reproduces pre-budget behavior everywhere. Exhausting a bound never drops +/// WORK: each destructive family's caller carries the excess in its own already-durable pipeline (still +/// `delete_pending`/condemned rows, a retained sweep candidate, a ref object left for next round) and +/// retries next round with a fresh budget, or -- where nothing durable is left to carry (manifest cleanup, +/// the outcome log) -- exhaustion drops only the redundant record of a decision the round already made +/// safely elsewhere (see each field's own comment). Reusable as-is by a future bounded-parallel-walk: give +/// each worker an atomic increment (or a partitioned share of a bound) instead of inventing a new +/// accounting shape. +struct GcRoundWorkBudget +{ + uint64_t max_graduations = 0; + uint64_t max_redeletes = 0; + uint64_t graduations_used = 0; + uint64_t redeletes_used = 0; + + bool graduationAvailable() const { return max_graduations == 0 || graduations_used < max_graduations; } + bool redeleteAvailable() const { return max_redeletes == 0 || redeletes_used < max_redeletes; } + + /// Orphan-manifest planner (`planManifestCursorPage` / `activeManifestKeys`): how many DISTINCT + /// namespaces the sweep may build a fresh protection view for in total this round (cumulative + /// across pages, not per page), and how many ref-log GET/decode + /// operations the committed-tail recovery walk may spend in total across every namespace this + /// round. Both caps exist because building a namespace's protection view (a catalog-authoritative + /// table recovery plus a committed-tail walk) is the expensive, potentially-unbounded step the + /// nomination/list budgets never covered. + uint64_t max_sweep_namespaces = 0; + uint64_t max_sweep_recovery_ops = 0; + uint64_t sweep_namespaces_used = 0; + uint64_t sweep_recovery_ops_used = 0; + + bool sweepNamespaceAvailable() const { return max_sweep_namespaces == 0 || sweep_namespaces_used < max_sweep_namespaces; } + bool sweepRecoveryOpAvailable() const { return max_sweep_recovery_ops == 0 || sweep_recovery_ops_used < max_sweep_recovery_ops; } + + /// Cleanup families (`Gc::cleanupRefObjects`, `deletePrefixWholesale`'s caller in + /// `Gc::pruneSupersededGenerations`). `deletePrefixWholesale` already takes a `bounded_remaining` + /// count; `prefixWholesaleRemaining` turns the shared round budget into that same count (its ONE + /// unbounded sentinel, `0 == max_prefix_wholesale_objects`, maps to the function's own "no cap" + /// value, `UINT64_MAX`) so every caller can pass "the round's remainder" instead of `UINT64_MAX` + /// outright. + uint64_t max_ref_cleanup_objects = 0; + uint64_t ref_cleanup_objects_used = 0; + uint64_t max_prefix_wholesale_objects = 0; + uint64_t prefix_wholesale_objects_used = 0; + + bool refCleanupAvailable() const { return max_ref_cleanup_objects == 0 || ref_cleanup_objects_used < max_ref_cleanup_objects; } + uint64_t prefixWholesaleRemaining() const + { + if (max_prefix_wholesale_objects == 0) + return std::numeric_limits::max(); + return prefix_wholesale_objects_used < max_prefix_wholesale_objects + ? max_prefix_wholesale_objects - prefix_wholesale_objects_used : 0; + } + + /// The post-CAS hand-off reclaim (`Gc::runRegularRound`) draws from its OWN reserve, never from + /// `max_prefix_wholesale_objects` above. The prune is safe to under-serve in any one round -- its + /// cursor never regresses, so a partially-drained generation is simply finished next round -- but the + /// hand-off is a ONE-SHOT event with no reclaimer behind it besides `fsck`: a generation it cannot + /// fully reclaim this round is never revisited (the parent-seal difference that triggers it does not + /// recur once the ref has moved). Sharing one pool would let a prune-heavy round starve the hand-off + /// to zero every time; a separate reserve makes that impossible regardless of how much the prune + /// consumes. + uint64_t max_handoff_prefix_wholesale_objects = 0; + uint64_t handoff_prefix_wholesale_objects_used = 0; + + uint64_t handoffPrefixWholesaleRemaining() const + { + if (max_handoff_prefix_wholesale_objects == 0) + return std::numeric_limits::max(); + return handoff_prefix_wholesale_objects_used < max_handoff_prefix_wholesale_objects + ? max_handoff_prefix_wholesale_objects - handoff_prefix_wholesale_objects_used : 0; + } + + /// `GcOutcomes` per-shard body (`Gc::runRegularRound`'s `redelete`/`spared` loops): a pure + /// observability record of a settlement decision that has ALREADY happened -- the merge unconditionally + /// decides `spared` for any entry whose in-degree recovered (INV_NO_LOSS: a fresh dedup-adopt must + /// never be treated as still condemned, past any budget), so this cap governs only whether the decision + /// gets an audit-log row, never the decision itself. Nothing is retained or retried on exhaustion -- + /// there is nothing left to retry, the entry already left the retired pipeline correctly. + uint64_t max_outcome_entries = 0; + uint64_t outcome_entries_used = 0; + + bool outcomeEntryAvailable() const { return max_outcome_entries == 0 || outcome_entries_used < max_outcome_entries; } +}; + +/// Merge the prior generation's source-edge run with new deltas. The prior run's `kCondemned` rows RIDE +/// the source-edge run itself at the zero-sentinel key (`source_id = 0`), so there is no separate +/// `prior_retired` cursor — the prior run IS the retired input. `PriorEdgeCursor` decodes each sentinel +/// `kCondemned` row and hands it to the per-blob close-out (in ascending hash order, exactly the order +/// the old sorted vector had). Settlement rules, in order, per condemned row for blob `h` with post-merge +/// in-degree `d`: +/// delete_pending (prior pass) -> redelete if d = 0 (the caller executes the exact-token +/// delete pre-CAS and the entry drops); d > 0 for a pending +/// entry is structurally impossible but reachable under +/// races — spared + a loud log, never a +/// fail-closed abort; +/// d > 0 -> spared (recovery wins even past the floor); +/// d = 0 and condemn_round < current_round -> graduated: REPUBLISHED as delete_pending (two-phase +/// graduation) — deleted the NEXT pass. GATED on a +/// confirmed durable condemn marker (see +/// `confirm_condemned_marker` below): an unconfirmed +/// entry is carried unchanged instead; +/// d = 0 otherwise -> still_retired, carried byte-unchanged. +/// A carried `kCondemned` row is SETTLEMENT-ONLY: it never sets the blob's `cur_touched` bit, so a +/// generation that only carries the row emits no zero-marker and pays no `peek_head` HEAD. The surviving +/// `still_retired` entries are re-emitted as `kCondemned` sentinel rows into the OUTPUT run (one sentinel +/// per blob, emitted before the blob's edges since the sentinel key sorts first), so the next generation +/// reads them back — `still_retired` mirrors exactly those rows, in the same order. +/// When the pass is clamped on any shard, landed-before-cut events may remain unfolded behind the clamp, +/// so graduating or executing pending +/// deletes over this pass's in-degrees can delete a blob whose +1 is pending behind the clamp (the +/// model's SabotageSkipChangedShard counterexample, realized). With `suppress_destructive` the merge +/// neither graduates nor redeletes: pending entries carry UNCHANGED (still delete_pending) and +/// floor-passed entries stay condemned-only. Condemnation and sparing remain (both non-destructive). +/// A blob that transitions to zero THIS pass with no prior entry is condemned: `head_blob` captures +/// the exact incarnation token/size (absent object or empty head_blob -> nothing to delete, skipped) +/// and the entry is minted at `condemn_round`. Entries for blobs the merged stream never visits +/// (no edges, no deltas) settle at in-degree 0 by definition. The retired cursor never changes the +/// snapshot run bytes. Defaults preserve the empty-retired behavior +/// current_round 0 => nothing graduates, no head_blob => nothing condemned). +/// +/// `peek_head` is a side-effect-free HEAD used only by the resurrect-supersede +/// branch (a `prior_retired` entry whose blob re-touched this pass at net in-degree 0, current token +/// differs from the stale entry's token). `head_blob` is the FRESH-CONDEMN observation hook — it emits +/// the `IndegZero`/`GcRetireObserve`/`BlobRetire` trail and increments `CASGCRetiredCondemned`, which is +/// wrong for a supersede (the supersede's own event is `blob_retire_replaced`, emitted once by the +/// caller from `RetiredMergeResult::replaced`). Calling `head_blob` from the supersede branch used to +/// double-emit `blob_retire` alongside `blob_retire_replaced` and double-count the condemned counter; +/// `peek_head` is a plain HEAD with no events and no counters. No supersede detection happens if +/// `peek_head` is unset (default `{}`), independent of whether `head_blob` is set. +/// +/// `confirm_condemned_marker` is the GRADUATION GATE (triage 2026-07-17 §3.4): graduation to +/// `delete_pending` is the one edge that authorizes an irreversible delete, and the per-hash condemn +/// marker (`writeCondemnedMeta`) is the writer's adopt gate — an entry whose marker write was silently +/// swallowed can be same-token adopted by a writer invisible to this fold's cut, so it must NOT +/// graduate. The callback returns whether durable `Condemned` evidence is confirmed for the entry's +/// exact (hash, token); on false the entry is carried unchanged (fail-safe delay — the caller is +/// expected to retry the marker so a later pass can confirm). An entry whose `marker_confirmed` bit is +/// already set skips the callback. Unset (default `{}`) means UNGATED — the pre-gate merge semantics, +/// for merge-mechanics unit tests only; the real GC round always passes the gate. +/// The merge comparator is exactly `(ref.algo, ref.digest, source_id)` (that is, `BlobRef::operator<` +/// followed by `source_id`), which is also the raw key order produced by `SourceEdgeKeyCodec`. +void foldDeltasIntoGeneration(Backend & backend, const Layout & layout, + const std::vector & prior_runs, + uint64_t new_generation, uint64_t attempt, + uint64_t shard, + std::vector scattered, std::vector & out_runs, + uint64_t current_round = 0, uint64_t condemn_round = 0, + const std::function(const BlobRef &)> & head_blob = {}, + const std::function(const BlobRef &)> & peek_head = {}, + const std::function & confirm_condemned_marker = {}, + RetiredMergeResult * out_retired = nullptr, + bool suppress_destructive = false, + /// PROBE B2 (see `Cas::TxnApplyLedger`): when set, one byte is stored per + /// CONSUMED delta at `(*out_applied_by_txn_ordinal)[d.txn_ordinal]`. A raw + /// vector rather than a callback: this runs once per delta over a stream + /// that can reach millions of rows, and a `std::function` call there is + /// not free. Never read by the merge; write-only. The caller must size it + /// to cover every `txn_ordinal` present in `scattered`. + std::vector * out_applied_by_txn_ordinal = nullptr, + std::vector source_retirements = {}, + /// Shared by reference across every shard's call within one round; + /// `nullptr` (default) is unbounded, matching every existing caller. + GcRoundWorkBudget * work_budget = nullptr); + +/// Stream the sealed in-degree runs named by `runs` (the current seal's `blob_target_runs` filtered to one +/// shard) and return every blob written at in-degree 0 (the candidates that transitioned to zero). An +/// empty `runs` is an empty baseline. Each `RunRef` supplies the exact object key, so resolution never +/// reconstructs a key from generation metadata. +std::vector zeroInDegree(Backend & backend, const std::vector & runs); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp new file mode 100644 index 000000000000..10f75d953dbc --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -0,0 +1,4661 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event CASGCClampSuppressedPasses; + extern const Event CASGCDeadPrecommitSkipped; + extern const Event CASGCRetiredCondemned; + extern const Event CASGCRetiredSpared; + extern const Event CASGCRetiredGraduated; + extern const Event CASGCRetiredRedeleted; + extern const Event CASGCRetireReplaced; + extern const Event CASGCCondemnMarkerUnconfirmedCarry; + extern const Event CASGCHeartbeatFenceOuts; + extern const Event CASGCMetaWriteAnomaly; + extern const Event CASGCMetaOps; + extern const Event CASGCEnumerationPages; + extern const Event CASGCRefWalkPlansBuilt; + extern const Event CASGCUnmatchedAdoptedParentLives; + extern const Event CASGCStuckRemovals; + extern const Event CASGCNamespaceCleanupLeaks; + extern const Event CASGCRebuildVirginByEnumeration; + extern const Event CASGCUnappliedFoldedTransactions; + extern const Event CASRefGlobalListPages; + extern const Event CASRefLogBodyGets; + extern const Event CASRefManifestBodyFoldGets; + extern const Event CASRefEmittedEdges; + extern const Event CASRefCleanupObjectsDeleted; +} + +namespace CurrentMetrics +{ + extern const Metric LocalThread; + extern const Metric LocalThreadActive; + extern const Metric LocalThreadScheduled; +} + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ABORTED; + extern const int BAD_ARGUMENTS; + extern const int CORRUPTED_DATA; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +namespace +{ + +/// The `on_page_fetched` hook GC passes to every `forEachListedKey`/`recoverRefTable` +/// call it owns (never passed by fsck/offline-repair callers of those shared helpers) -- one increment +/// per physical LIST page, never per listed key. +void onGcEnumerationPage() +{ + ProfileEvents::increment(ProfileEvents::CASGCEnumerationPages); +} + +/// Defined below; forward-declared so the post-CAS hand-off delete in `runRegularRound` can +/// reach the same wholesale LIST-delete helper the retention prune uses. +uint64_t deletePrefixWholesale(Backend & backend, const String & prefix, uint64_t bounded_remaining, + bool * out_fully_drained = nullptr); + +/// The per-hash freshness-meta operations GC schedules on the bounded pool are +/// best-effort/idempotent by design. The meta is only a point-read freshness marker for the writer/ +/// promote gate; the ledger retired-set + the exact-token body delete remain the actual safety +/// core. A lost CAS here is never a correctness problem, only a (rare, +/// self-healing) staleness window for the NEXT point-reader — with ONE exception (triage 2026-07-17 +/// §3.4): the CONDEMN marker is load-bearing for the delete edge. The exact-token delete argument +/// below assumes the marker was durably written before the delete fires; a swallowed condemn-marker +/// write lets a writer observe absent/Clean meta and adopt the SAME token the graduated entry later +/// deletes (a dangling manifest). Graduation to `delete_pending` is therefore GATED on confirmed +/// durable Condemned evidence for the exact (hash, token) — recorded in-process when the scheduled +/// `writeCondemnedMeta` reports success, or re-established by a synchronous `loadMeta` re-check at +/// graduation time. An unconfirmed entry is CARRIED (fail-safe delay) and its marker write retried; +/// the delete itself and every other meta op stay async/advisory. +/// +/// GC freshness meta is ADD-ONLY: GC may publish `Condemned`, and may REMOVE the meta once the exact body +/// token is confirmed deleted/absent (`deleteConfirmedMeta`), but it NEVER transitions `Condemned -> +/// Clean` on a spare. The SOLE `-> Clean` transition is a WRITER that has already displaced the body with +/// a fresh incarnation token (`PartWriteTxn::uploadFromSource` + `writeResurrectMetaClean`). Rationale: a deposed leader that cleared a +/// spare's meta then lost its round CAS would leave a durable stray-`Clean` over a still-condemned body; +/// a writer reading `Clean` would reuse the exact condemned token, which a stale pre-CAS exact-token +/// redelete then deletes -- live-blob data loss (INV_NO_LOSS). Removing the clear restores the exact-token +/// delete argument in full: once a hash is `Condemned`, observing `Clean` means EITHER the condemned body +/// is absent OR a writer already changed its incarnation token, so every stale `deleteExact(t1)` finds the +/// body absent or `TokenMismatch`. + +/// Write the per-hash meta to Condemned: a blob newly entering the retired set this round (either the +/// fresh zero-in-degree condemn, or a resurrect-supersede re-condemn of the current token). Absent meta +/// is created fresh; an already-Condemned meta (a racing condemn, or a replay of this same round) is left +/// alone rather than clobbering a possibly-newer condemn_round. +/// +/// Returns whether durable Condemned evidence exists after the call: the conditional write committed, +/// or an already-Condemned meta was observed. A lost CAS reports false and writes nothing further (the +/// loser re-reads next time); a thrown backend error propagates (the scheduling wrapper swallows it) — +/// either way the entry stays UNCONFIRMED and the graduation gate carries it (triage §3.4). +bool writeCondemnedMeta(Pool & pool, const BlobRef & ref, uint64_t condemn_round, uint64_t size) +{ + const auto lm = loadMeta(pool.backend(), pool.layout(), ref); + const BlobMeta desired{.state = MetaState::Condemned, .condemn_round = condemn_round, .size = size}; + if (!lm) + return putMetaIfAbsent(pool, ref, desired).outcome == CasOverwriteOutcome::Committed; + if (lm->meta.state != MetaState::Condemned) + return casMeta(pool, ref, lm->etag, desired).outcome == CasOverwriteOutcome::Committed; + return true; +} + +/// Drop the meta after its body was physically deleted (or already found absent) by the round's +/// exact-token delete. NO tombstone -- an absent meta reads exactly like a Clean one (absent +/// means not condemned"). Idempotent: an already-absent meta, or one a racing writer/GC pass already +/// moved, is a silent no-op. +void deleteConfirmedMeta(Backend & backend, const Layout & layout, const BlobRef & ref) +{ + const auto lm = loadMeta(backend, layout, ref); + if (!lm) + return; + deleteMetaExact(backend, layout, ref, lm->etag); +} + +} + +std::set RefPlan::lifeIds() const +{ + std::set out; + for (const auto & [life_id, row] : rows) + out.insert(life_id); + return out; +} + +std::vector RefPlan::lives() const +{ + std::vector out; + out.reserve(rows.size()); + for (const auto & [life_id, row] : rows) + out.push_back(row.life); + return out; +} + +std::map RefPlan::parentFoldStates() const +{ + std::map out; + for (const auto & [life_id, row] : rows) + if (row.has_parent_fold_state) + out.emplace(life_id, row.fold_state); + return out; +} + +std::map RefPlan::successorFoldStates() const +{ + std::map out; + for (const auto & [life_id, row] : rows) + out.emplace(life_id, row.fold_state); + return out; +} + +size_t RefPlan::changedRows() const +{ + return std::count_if(rows.begin(), rows.end(), [](const auto & item) + { + const RefWalkPlanRow & row = item.second; + return row.tail_observation + && row.fold_state.coverage.last_folded_ref_id < *row.tail_observation; + }); +} + +std::optional stuckRemovalWarning( + const RefWalkPlanRow & row, uint64_t current_round, uint64_t threshold_rounds, + const Layout & layout) +{ + if (!row.removal_started_round || row.fold_state.cleanup_evidence) + return std::nullopt; + const uint64_t started = *row.removal_started_round; + if (current_round < started || current_round - started < threshold_rounds) + return std::nullopt; + + const uint64_t age = current_round - started; + const std::optional & hold = row.fold_state.coverage.hold; + if (hold && hold->reason == HoldReason::BodyUndecodable) + return fmt::format( + "CAS GC namespace removal is stuck: namespace='{}', life_id={}, removal_started_round={}, " + "current_round={}, age_rounds={}; cleanup evidence is absent because ref-log body '{}' is unreadable; " + "restore the exact object or recreate the pool", + row.life.ns.string(), u128ToHex(row.life.incarnation), started, current_round, age, + layout.refLogKey(row.life, hold->offending_position)); + + return fmt::format( + "CAS GC namespace removal is stuck: namespace='{}', life_id={}, removal_started_round={}, " + "current_round={}, age_rounds={}; cleanup evidence is absent because terminal has not folded", + row.life.ns.string(), u128ToHex(row.life.incarnation), started, current_round, age); +} + +RefPlan buildRefWalkPlan(RoundInput && round_input) +{ + ProfileEvents::increment(ProfileEvents::CASGCRefWalkPlansBuilt); + RefPlan plan{std::move(round_input.ref_scan), std::move(round_input.catalog_cut)}; + const CasRefCatalog::Snapshot & catalog_cut = plan.catalog_cut; + const RefScanSummary & ref_scan = plan.ref_scan; + catalog_cut.life_index.throwIfAmbiguous("CAS ref walk plan"); + + /// The sole admission loop. Everything below can only find one of these rows. + for (const CatalogEntry & entry : catalog_cut.catalog.entries) + { + if (entry.state == NsState::Creating) + continue; + plan.rows.emplace(entry.incarnation, RefWalkPlanRow{ + .life = NamespaceLifeId::fromCatalogEntry(entry.ns, entry.incarnation), + .fold_state = {}, + .removal_started_round = entry.removal_started_round, + .has_parent_fold_state = false, + .listed_hint = false, + .checkpoint_observation = std::nullopt, + .tail_observation = std::nullopt}); + } + + for (const auto & [life_id, state] : ref_scan.parent_ref_lives) + { + const auto it = plan.rows.find(life_id); + if (it == plan.rows.end()) + { + /// The ordinary end of a namespace's life: its removal completed, GC deleted the catalog + /// row, and the parent seal still carries the fold state of a life the current cut no + /// longer names. There is nothing to attach it to and nothing for anyone to do, so this + /// is counted, not narrated -- a per-drop log line would be pure noise, and it could not + /// discriminate the expected case from an illegitimately vanished row anyway: this site + /// sees only the absence, never the evidence. The signal lives in + /// `CASGCUnmatchedAdoptedParentLives` and in the round's own + /// `walk_plan_dropped_parent_rows` phase metric; proving that a row disappeared WITHOUT a + /// completed removal belongs to fsck, which can compare against the removal evidence. + ProfileEvents::increment(ProfileEvents::CASGCUnmatchedAdoptedParentLives); + ++plan.dropped_parent_rows; + continue; + } + it->second.fold_state = state; + it->second.has_parent_fold_state = true; + } + for (const UInt128 & life_id : ref_scan.listed_lives) + { + const auto it = plan.rows.find(life_id); + if (it == plan.rows.end()) + { + ++plan.dropped_listed_lives; + continue; + } + it->second.listed_hint = true; + } + for (const auto & [life_id, hold] : ref_scan.holds) + { + const auto it = plan.rows.find(life_id); + if (it == plan.rows.end()) + { + ++plan.dropped_holds; + continue; + } + it->second.fold_state.coverage.classification = 4; + it->second.fold_state.coverage.hold = hold; + } + for (const auto & [life_id, checkpoint] : ref_scan.checkpoint_observations) + { + const auto it = plan.rows.find(life_id); + if (it == plan.rows.end()) + { + ++plan.dropped_checkpoints; + continue; + } + it->second.checkpoint_observation = checkpoint; + } + for (const auto & [life_id, tail] : ref_scan.max_log_by_life) + { + const auto it = plan.rows.find(life_id); + if (it == plan.rows.end()) + { + ++plan.dropped_tails; + continue; + } + it->second.tail_observation = tail; + } + return plan; +} + +namespace tests +{ + +RefPlan buildRefWalkPlanForTest(RefScanSummary ref_scan, CasRefCatalog::Snapshot catalog_cut) +{ + return buildRefWalkPlan(RoundInput{std::move(ref_scan), std::move(catalog_cut)}); +} + +} + +uint64_t retiredLogicalSize(ObjectKind kind, uint64_t object_size, uint64_t blob_header_len) +{ + if (kind != ObjectKind::Blob) + return object_size; + if (object_size < blob_header_len) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS gc retire: blob object of {} bytes is smaller than the pool's fixed blob header ({} bytes)", + object_size, blob_header_len); + return object_size - blob_header_len; +} + +bool shouldDeferRound(size_t changed_shards, bool graduation_due, uint64_t rounds_since_last_fold, + uint64_t fold_threshold, uint64_t fold_max_defer_rounds) +{ + if (graduation_due) + return false; + if (changed_shards >= fold_threshold) + return false; + if (rounds_since_last_fold >= fold_max_defer_rounds) + return false; + return true; +} + +Gc::Gc(PoolPtr store_, UInt128 gc_id_, std::function now_ms_fn_, + std::function mono_ms_fn_, LoggerPtr log_) + : store(std::move(store_)) + , gc_id(gc_id_) + , logger(log_ ? std::move(log_) : getLogger("CasGc")) + , now_ms_fn(std::move(now_ms_fn_)) + , mono_ms_fn(std::move(mono_ms_fn_)) +{ + if (!store) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cas::Gc: store must not be null"); + if (gc_id == UInt128(0)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cas::Gc: gc_id must not be 0 (reserved for 'lease never held')"); + if (store->poolConfig().gc_stuck_removal_rounds == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cas::Gc: gc_stuck_removal_rounds must be nonzero"); + if (!now_ms_fn) + now_ms_fn = []() -> uint64_t + { + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + }; + /// Use `store->bootMsNow()`, not the raw static `Pool::bootMs()` + /// -- the latter bypasses the Pool's own injectable `config.boot_ms_fn`, so a time-controlled test + /// that fakes the mount's clock via `boot_ms_fn` (but constructs a `Gc` without an explicit + /// `mono_ms_fn`) would silently run its GC-side threshold math against the REAL wall clock while the + /// mount side runs against the fake one -- two desynced clocks passing for the same test. + /// `bootMsNow()` already falls back to `bootMs()` itself when no `boot_ms_fn` is injected, so + /// production (no test seam in play) is unaffected. + if (!mono_ms_fn) + mono_ms_fn = [s = store]() -> uint64_t { return s->bootMsNow(); }; + /// Build the bounded pool for this round's per-hash freshness-meta writes here (ctor body), + /// not a member-initializer, so it can safely read `store->poolConfig()` AFTER the null check above. + const uint64_t configured_pool_size = store->poolConfig().gc_meta_pool_size; + const size_t pool_size = static_cast(std::max(1, configured_pool_size)); + meta_pool = std::make_unique(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, + CurrentMetrics::LocalThreadScheduled, pool_size); +} + +void Gc::scheduleMetaJob(std::function job) +{ + /// Wrap once: `run` is safe to invoke either on the pool or inline (the scheduling-failure fallback + /// below), and NEVER lets an exception escape: a per-hash meta + /// op is advisory; the ledger + exact-token body delete are the actual safety core). + /// Capture the logger by value under a distinct name (the pool job may outlive nothing here, but it + /// must not depend on `this`; the copy also keeps the capture from shadowing the `logger` member). + /// `completed` is the `meta_pool_wait` phase's only visible signal: that phase's work runs HERE, on + /// a pool thread, so it contributes nothing to the round thread's ProfileEvents delta. Captured as a + /// raw pointer to the atomic rather than `this`, keeping `run`'s existing "must not depend on `this`" + /// property -- the pool is a member of the same `Gc` and is joined by `~Gc` before the atomic dies. + auto run = [job, job_logger = this->logger, completed = &meta_jobs_completed_]() + { + /// Count one per-hash freshness-meta op EXECUTED (attempt, not success) on this + /// bounded pool. `run` is invoked on the pool thread (the common path below) or inline on the + /// round's own thread (the scheduling-failure fallback below) -- either way this is pool-scoped + /// work, so the counter is GLOBAL-only by design. + ProfileEvents::increment(ProfileEvents::CASGCMetaOps); + try + { + job(); + } + catch (...) + { + ProfileEvents::increment(ProfileEvents::CASGCMetaWriteAnomaly); + tryLogCurrentException(job_logger, + "CAS gc: a per-hash freshness-meta op failed on the bounded pool (advisory-only; " + "never wedges the round)"); + } + /// After the catch: a job that threw still FINISHED, and `meta_pool_wait` reports drain + /// progress, not success (the anomaly counter above is what reports failure). + completed->fetch_add(1, std::memory_order_relaxed); + }; + meta_jobs_scheduled_.fetch_add(1, std::memory_order_relaxed); + try + { + meta_pool->scheduleOrThrowOnError(run); + } + catch (...) + { + /// Scheduling itself failed (e.g. resource exhaustion under a mass-DROP burst) -- run inline + /// rather than silently lose the meta write. `run` still never throws. + ProfileEvents::increment(ProfileEvents::CASGCMetaWriteAnomaly); + tryLogCurrentException(logger, + "CAS gc: meta pool scheduling failed; running the op inline on the round's own thread"); + run(); + } +} + +void Gc::scheduleCondemnMarkerWrite(const BlobRef & ref, const Token & token, + uint64_t condemn_round, uint64_t size) +{ + scheduleMetaJob([this, ref, token, condemn_round, size]() + { + if (writeCondemnedMeta(*store, ref, condemn_round, size)) + noteCondemnMarkerDurable(ref, token); + /// A lost CAS / thrown error leaves the (ref, token) UNCONFIRMED: the graduation gate then + /// carries the entry and retries this write on a later round (fail-safe delay, triage §3.4). + }); +} + +void Gc::noteCondemnMarkerDurable(const BlobRef & ref, const Token & token) +{ + std::lock_guard lock(condemn_marker_mutex); + condemn_markers_confirmed.emplace(ref, token.value); +} + +bool Gc::condemnMarkerConfirmedInProcess(const BlobRef & ref, const Token & token) +{ + std::lock_guard lock(condemn_marker_mutex); + return condemn_markers_confirmed.contains({ref, token.value}); +} + +void Gc::forgetCondemnMarker(const BlobRef & ref, const Token & token) +{ + std::lock_guard lock(condemn_marker_mutex); + condemn_markers_confirmed.erase({ref, token.value}); +} + +void Gc::runNamespaceJanitorPage( + const GcState & leased_state, bool suppress_destructive, uint64_t cleanup_evidence_rows) +{ + GcPhaseTimer t(phase_sink, "namespace_cleanup"); + t.metric("evidence_rows", cleanup_evidence_rows); + NamespaceJanitorResult janitor_result; + try + { + Backend & backend = store->backend(); + const Layout & layout = store->layout(); + NamespaceJanitor janitor(backend, layout, 1000); + const uint64_t admitted_generation = leased_state.lease.seq; + janitor_result = janitor.runOnePage(suppress_destructive, [&] + { + const auto got = backend.get(layout.gcStateKey()); + if (!got) + return false; + const GcState current = decodeGcState(got->bytes); + return current.lease.owner == gc_id && current.lease.seq == admitted_generation; + }); + for (const String & anomaly : janitor_result.anomalies) + LOG_WARNING(logger, "CAS namespace janitor: {}", anomaly); + if (janitor_result.leaked) + ProfileEvents::increment(ProfileEvents::CASGCNamespaceCleanupLeaks, janitor_result.leaked); + } + catch (const std::exception & e) + { + LOG_WARNING(logger, "CAS namespace janitor skipped this round: {}", e.what()); + } + t.metric("janitor_pages", janitor_result.pages); + t.metric("janitor_keys", janitor_result.keys); + t.metric("janitor_deleted", janitor_result.deleted); + t.metric("leaked", janitor_result.leaked); +} + +RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool allow_steal, UniversePolicy policy) +{ + RoundReport report; + GcState state; + Token state_token; + /// PHASE 1/18 `lease`. Also the ONLY phase a `NotALeader` round emits, which is why the phase rows + /// are correlated by `round_id` and not by the round number a follower never learns. + { + GcPhaseTimer t(phase_sink, "lease"); + report.acquired_lease = acquireOrRenewLease(state, state_token, allow_steal); + t.metric("acquired", report.acquired_lease ? 1 : 0); + t.metric("steal_allowed", allow_steal ? 1 : 0); + } + if (!report.acquired_lease) + return report; + + /// Baseline for the `meta_pool_wait` phase's job counts (the pool is per-`Gc`, the counters + /// cumulative), taken before anything in this round can schedule a job. + const uint64_t meta_jobs_scheduled_at_round_start = meta_jobs_scheduled_.load(std::memory_order_relaxed); + const uint64_t meta_jobs_completed_at_round_start = meta_jobs_completed_.load(std::memory_order_relaxed); + + /// Fire the acquire-time hook BEFORE the long fold below, not after the round + /// returns - a new leader's first round could otherwise run for the whole fold with no + /// heartbeat cover, letting a follower steal deterministically once it freezes (owner, seq) + /// across two of its own ticks. + if (on_lease_acquired) + on_lease_acquired(); + + /// ONE-PASS round. There is no crash-resume step anymore: the round commits everything in the + /// SINGLE gc/state CAS at the end, so a crashed pass leaves only attempt-scoped debris that is + /// never adopted (retention prunes it), and every destructive PRE-CAS action below is justified by + /// PREVIOUSLY PUBLISHED durable state only (delete_pending entries), so replay under a fresh + /// attempt is idempotent. + + const Layout & layout = store->layout(); + Backend & backend = store->backend(); + const uint64_t new_round = state.round + 1; + + /// ONE budget instance for the WHOLE round, threaded into every destructive-or-observability-write + /// family below: `fold` (blob graduation/redelete, the `GcOutcomes` audit rows, the orphan-manifest + /// planner), `pruneSupersededGenerations`, the post-CAS hand-off reclaim (its OWN reserve, never + /// `pruneSupersededGenerations`' shared remainder), the post-CAS manifest-body cleanup, and + /// `cleanupRefObjects`. A cap is therefore cumulative over the round, never reset between families or + /// shards. See `GcRoundWorkBudget`'s own comment for the fail-closed contract each family applies on + /// exhaustion. + GcRoundWorkBudget round_work_budget; + round_work_budget.max_graduations = store->poolConfig().gc_round_graduation_budget; + round_work_budget.max_redeletes = store->poolConfig().gc_round_redelete_budget; + round_work_budget.max_sweep_namespaces = store->poolConfig().gc_round_sweep_namespace_budget; + round_work_budget.max_sweep_recovery_ops = store->poolConfig().gc_round_sweep_recovery_op_budget; + round_work_budget.max_ref_cleanup_objects = store->poolConfig().gc_round_ref_cleanup_budget; + round_work_budget.max_prefix_wholesale_objects = store->poolConfig().gc_round_prefix_wholesale_budget; + round_work_budget.max_handoff_prefix_wholesale_objects = store->poolConfig().gc_round_handoff_prefix_wholesale_budget; + round_work_budget.max_outcome_entries = store->poolConfig().gc_round_outcome_entry_budget; + + /// The helping barrier precedes heartbeat work, DEFER, the hot stream LIST, and every successor + /// artifact. A deferred invocation therefore cannot leave a row the adopted parent already proved + /// complete, and a folding invocation takes its catalog cut only after the deletion settles. + { + GcPhaseTimer t(phase_sink, "pre_fold_ref_drain"); + const CatalogLifecycleReconcileResult drain_result = drainCompletedRemoving(state); + for (const NamespaceLifeId & retired_life : drain_result.retired_lives) + store->invalidateRemovedCatalogLife(retired_life); + if (drain_result.authority_status != AuthorityStatus::Authoritative + || drain_result.catalog_resolution != CatalogResolution::DrainComplete) + throwCasWriteRetryLater("CAS GC pre-fold drain lost authority before the catalog settled"); + t.metric("deleted", drain_result.deleted); + } + + /// Token-guarded fence-out of dead mounts (liveness only — graduation itself paces on GC + /// rounds via `new_round`, not on heartbeat acks). Fencing no longer trusts a predecessor's stamped + /// `expires_at_ms` against our wall clock — it + /// fences ONLY once `mount_obs` has watched the mount's write-token hold unchanged for the full + /// threshold on THIS leader's own monotonic clock (mirrors `claimMountAwaitingExpiry`'s identical + /// `TTL + Drift` threshold for a mount's own reopen). + const uint64_t ttl_ms = static_cast(store->poolConfig().mount_lease_ttl_ms.count()); + /// The formula is shared with `claimMountAwaitingExpiry` via + /// `mountObservationThresholdMs` -- see its doc comment (CasServerRoot.h). + const uint64_t stable_threshold_ms = mountObservationThresholdMs( + ttl_ms, static_cast(store->poolConfig().mount_renew_period.count())); + + /// PHASE 3/18 `heartbeat_floor`: one LIST of `gc/server-roots/`, one GET per mount slot, and a fence + /// PUT per newly-fenced mount. + { + GcPhaseTimer t(phase_sink, "heartbeat_floor"); + const HeartbeatFloor floor = computeHeartbeatFloor(backend, layout, now_ms_fn(), mono_ms_fn(), + stable_threshold_ms, mount_obs); + report.fence_outs = floor.fenced_now; + if (floor.fenced_now > 0) + ProfileEvents::increment(ProfileEvents::CASGCHeartbeatFenceOuts, floor.fenced_now); + + /// GcFenceOut audit row per expired mount fenced-out this round: the round latched a fence-out to + /// re-arm a sleeper's write fence (its held token is now invalid). One row per srid so the log + /// reconstructs which mount was reclaimed. + for (const String & srid : floor.fenced_srids) + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::GcFenceOut; + e.object_kind = CasEventObjectKind::Snap; + e.round = new_round; + e.gen = state.snap_generation; + e.outcome = "fenced"; + e.reason = "expired mount lease past skew margin; token-guarded fence-out re-arms the write " + "fence (prevents a resumed sleeper from mutating)"; + e.detail = {{"server_root_id", srid}}; + }); + + /// Emit the round's heartbeat classification (what mounts are live/terminated/fenced this round). + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::GcFence; + e.object_kind = CasEventObjectKind::Snap; + e.round = new_round; + e.gen = state.snap_generation; + e.outcome = "floor"; + e.reason = "R1: heartbeat classification (live/terminated/fenced mounts)"; + e.detail = {{"live", std::to_string(floor.live)}, + {"terminated", std::to_string(floor.terminated)}, + {"fenced_now", std::to_string(floor.fenced_now)}, + {"already_fenced", std::to_string(floor.already_fenced)}}; + }); + + t.metric("live", floor.live); + t.metric("terminated", floor.terminated); + t.metric("fenced_now", floor.fenced_now); + t.metric("already_fenced", floor.already_fenced); + } + + /// Decide DEFER vs FOLD from cheap pre-fold signals. + /// A DEFER round re-adopts the sealed generation — no fold, no delete, no gc/state write — so a + /// slow idle/small-delta round no longer rebuilds the whole in-degree snapshot. Safety: a due + /// graduation forces a FOLD (graduationDue), so no destructive decision runs on a stale snapshot. + /// + /// `listRefPrefix` is the round's one full enumeration of `cas/ns/stream/`. Its result is retained + /// (rather than discarded once the defer decision is taken) because `fold` regroups the very same + /// keys instead of listing the prefix again. A deferred round simply drops it. + /// + /// PHASE 4/18 `defer_decision`. `ref_scan` OUTLIVES the timer because the fold consumes it, and + /// `report.deferred` is set INSIDE the scope so the row already reflects the verdict when the + /// timer's destructor fires on the deferred round's early return. This phase also performs TWO of + /// the round's reads of the adopted fold seal (`graduationDue` and `listRefPrefix` each read the + /// same key) -- see `fold_seal_reads` below. + std::optional walk_plan; + bool defer_round = false; + { + GcPhaseTimer t(phase_sink, "defer_decision"); + const bool graduation_due = graduationDue(state, new_round); + walk_plan.emplace(buildRefWalkPlan(listRefPrefix(state))); + reportStuckRemovals(*walk_plan, state.round); + const RefScanSummary & ref_scan = walk_plan->refScan(); + const size_t changed = walk_plan->changedRows(); + defer_round = shouldDeferRound(changed, graduation_due, rounds_since_last_fold_, + store->poolConfig().gc_fold_threshold, + store->poolConfig().gc_fold_max_defer_rounds); + uint64_t ref_log_keys = 0; + for (const auto & [scanned_life, ids] : ref_scan.logs_by_life) + ref_log_keys += ids.size(); + t.metric("changed_shards", changed); + t.metric("namespaces_seen", ref_scan.max_log_by_life.size()); + t.metric("ref_log_keys_listed", ref_log_keys); + t.metric("ref_keys_listed", ref_scan.keys.size()); + t.metric("graduation_due", graduation_due ? 1 : 0); + t.metric("dead_life_debris", ref_scan.dead_life_debris); + t.metric("walk_plan_builds", 1); + t.metric("walk_plan_rows", walk_plan->size()); + t.metric("walk_plan_dropped_parent_rows", walk_plan->droppedParentRows()); + t.metric("walk_plan_dropped_listed_lives", walk_plan->droppedListedLives()); + t.metric("walk_plan_dropped_tails", walk_plan->droppedTails()); + t.metric("deferred", defer_round ? 1 : 0); + /// The number of consecutive rounds already deferred BEFORE this one (this round's own verdict is + /// `deferred` above), so the pair reads unambiguously against `gc_fold_max_defer_rounds`. + t.metric("rounds_deferred_before", rounds_since_last_fold_); + /// `graduationDue` and `listRefPrefix` each GET the adopted fold seal at the SAME + /// (generation, attempt). Recorded, not fixed -- see the `fold_seal_read` phase, which records + /// the other duplicate pair; the round GETs that one key FIVE times on a folding round. + t.metric("fold_seal_reads", 2); + + if (defer_round) + { + ++rounds_since_last_fold_; + report.deferred = true; + /// A DEFER round mints no new round -- unlike the fold path below (CasGc.cpp:642), which sets + /// `report.round = state.round` only AFTER the round's single `gc/state` CAS has committed + /// `next.round = new_round` and `state` was reassigned to that committed `next` (so on that + /// path `state.round` reads the FRESH round number). Here the round CAS never runs, so `state` + /// is still the round that was already adopted BEFORE this round started: `state.round` is the + /// honest, already-durable round number, while `new_round` (`state.round + 1`) would report a + /// round that never actually happened. Use `state.round` so `RoundReport::round` and the + /// `system.cas_gc_log` row it feeds never print a fabricated + /// round number on a deferred round. + report.round = state.round; + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::GcFence; /// reuse the Snap round-event channel; outcome = "deferred" + e.object_kind = CasEventObjectKind::Snap; + e.round = state.round; + e.gen = state.snap_generation; + e.outcome = "deferred"; + e.reason = "skip-unchanged: no changed shard reached the fold threshold and no graduation " + "is due; re-adopting the sealed generation (snapshot rebuild elided)"; + e.detail = {{"changed_shards", std::to_string(changed)}, + {"rounds_since_last_fold", std::to_string(rounds_since_last_fold_)}}; + }); + /// Return after the timer scope so the independently timed janitor phase is not nested + /// inside `defer_decision`. + } + else + rounds_since_last_fold_ = 0; /// this round folds + } + + if (defer_round) + { + /// DEFER has no `FoldResult`, hence no complete global destructive verdict. The janitor still + /// takes its bounded page and catalog cut, but suppression keeps both deletes and valid-page + /// cursor progress at the same position for the bounded forced fold to retry. + runNamespaceJanitorPage(state, /*suppress_destructive=*/true, /*cleanup_evidence_rows=*/0); + return report; /// no fold, no pre-CAS deletes, no gc/state CAS — sealed generation stays pinned + } + + /// Emit that the round's single pass begins. + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::GcFoldBegin; + e.object_kind = CasEventObjectKind::Snap; + e.round = state.round; + e.gen = state.snap_generation; + e.reason = "R2: one-pass fold (edges x deltas x retired) into a new durable generation"; + }); + + /// Capture the PARENT seal's run refs BEFORE fold mutates + /// `state.snap_generation`/`snap_attempt` in-memory (CasGc.cpp:838). We compare these against the + /// NEW seal's refs post-CAS to detect a ref that moved OFF an already-pruned generation (the + /// wholesale prune skipped it while it was still referenced and its cursor advanced past it), and + /// hand-off delete that generation's now-unreferenced leftover. Absent parent seal => empty. + /// + /// PHASE 5/18 `parent_seal_read`: the round's THIRD GET of the adopted fold seal (`graduationDue` + /// and `listRefPrefix` already read it in `defer_decision`, and `fold` reads it twice more). One + /// small GET, given its own row rather than left untimed, because "the same key, five times a round" + /// is only actionable if each read is attributable to a phase. + std::vector parent_seal_runs; + { + GcPhaseTimer t(phase_sink, "parent_seal_read"); + if (const auto parent_seal = readFoldSeal(state.snap_generation, state.snap_attempt)) + parent_seal_runs = parent_seal->blob_target_runs; + t.metric("parent_runs", parent_seal_runs.size()); + } + + /// The pass performs discovery, windowing, and the three-cursor merge (spare / graduate / condemn). + /// It emits phases 5..10 of its own. + FoldResult folded = fold(state, state_token, report, new_round, *walk_plan, policy, round_work_budget); + + /// THE ROUND'S DESTRUCTIVE GATE, read once, here, and consulted at EVERY destructive site below. + /// It is available this early because `fold` computes it (see `FoldResult::suppress_destructive`), + /// and it has to be: the first destructive site of the post-CAS tail is the hand-off reclaim, which + /// used to run before this value was ever read. + const bool suppress_destructive = folded.suppress_destructive; + + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::GcFoldEnd; + e.object_kind = CasEventObjectKind::Snap; + e.round = state.round; + e.gen = state.snap_generation; + e.outcome = "ok"; + e.reason = "R2 complete"; + e.detail = {{"shards", std::to_string(folded.root_shards.size())}, + {"anomalies", std::to_string(report.anomalies.size())}}; + }); + + const uint64_t generation = state.snap_generation; /// set in-memory by fold; committed below + const uint64_t attempt = state.snap_attempt; + + /// PRE-CAS deletes affect ONLY entries the PREVIOUS pass published as delete_pending (justified by + /// durable state and safe at any leader staleness), plus outcome bookkeeping for + /// every settled entry. THE SINGLE CONTENT-DELETE SITE. + /// + /// PHASE 11/18 `pending_deletes`, covering both the exact-token delete loop and the outcome-log + /// writes it feeds. Held in an `optional` rather than a `{ }` scope purely so this long, delicate + /// block is not reindented wholesale; `reset()` below is what emits the row, and an exception + /// escaping before it still emits from the destructor. + std::optional pending_deletes_timer; + pending_deletes_timer.emplace(phase_sink, "pending_deletes"); + const uint64_t redeleted_before = report.redeleted; + const uint64_t graduated_before = report.graduated; + std::map outcomes; + for (uint64_t shard = 0; shard < folded.retired_merge.size(); ++shard) + { + RetiredMergeResult & merge = folded.retired_merge[shard]; + /// The gate, stated at the site, and scoped to the DELETES alone: the spare / graduate / replace + /// bookkeeping below is not destructive and must still run on a suppressed round (a suppressed + /// round still condemns, spares and carries -- only irreversible work stops). A suppressed pass + /// produces an EMPTY `redelete` by construction (`settleEntry` carries every pending entry + /// unchanged instead of promoting it), so this loop would already do nothing -- but "would + /// already do nothing" is a property of another file, and the content-delete site does not + /// delegate its own gate. If the two ever disagree, the round deletes nothing rather than + /// deleting on a frontier it cannot prove. + /// + /// WHAT THIS GUARD DOES AND DOES NOT COVER, measured rather than assumed: it stops the delete + /// I/O, and nothing else. `settleEntry`'s gate is the primary one because promoting an entry to + /// `redelete` also DROPS it from `still_retired` -- so a build with only this guard performs no + /// delete and still loses the entry from the pipeline, leaking the blob instead of reclaiming + /// it. Do not read this as a licence to relax the merge-side gate. + static const std::vector kNothingToDelete; + const std::vector & redelete_now = + suppress_destructive ? kNothingToDelete : merge.redelete; + for (const RetiredEntry & entry : redelete_now) + { + DeleteOutcome del = backend.deleteExact(layout.blobKey(entry.ref), entry.token); + if (del.created_delete_marker) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS gc: delete of blob {} created a delete marker — versioning is enabled " + "on the pool (mis-provisioned; the capability probe must reject this)", blobIdOf(entry.ref)); + + /// A RustFS quirk: a conditional delete (`If-Match`) against an ABSENT + /// object can answer HTTP 412 (precondition failed) instead of 404 — we map that 412 to + /// TokenMismatch. Backend-agnostically disambiguate here: a genuine TokenMismatch means the + /// object exists under a different (fresh) token; if a follow-up HEAD shows the object is + /// gone, the "mismatch" was actually the object being absent — treat it as Absent (NotFound) + /// end-to-end so the `.meta` cleanup below still runs. + bool absent_on_mismatch_quirk = false; + if (del.kind == DeleteOutcome::Kind::TokenMismatch) + { + const HeadResult head = backend.head(layout.blobKey(entry.ref)); + if (!head.exists) + { + del.kind = DeleteOutcome::Kind::NotFound; + absent_on_mismatch_quirk = true; + } + } + + const DeleteClass del_class = classifyDeleteOutcome(del); + const OutcomeKind outcome_kind = del_class == DeleteClass::Deleted ? OutcomeKind::Deleted + : del_class == DeleteClass::Absent ? OutcomeKind::Absent + : OutcomeKind::Replaced; + OutcomeEntry outcome{.kind = entry.kind, .ref = entry.ref, .token = entry.token, .outcome = outcome_kind}; + const String del_outcome{deleteClassName(del_class)}; + /// The single content-delete site is attributable per row. TokenMismatch (a writer + /// recreated the incarnation) is terminal-OK: the fresh incarnation is a live object. + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::BlobDelete; + e.object_kind = CasEventObjectKind::Blob; + e.object_hash = blobIdOf(entry.ref); + e.token = entry.token.value; + e.round = new_round; + e.gen = generation; + e.outcome = del_outcome; + e.reason = absent_on_mismatch_quirk + ? "delete_pending published by a prior pass; exact-token delete (pre-CAS) " + "(delete returned token-mismatch but the object is absent — backend 412-on-absent quirk)" + : "delete_pending published by a prior pass; exact-token delete (pre-CAS)"; + e.detail = {{"condemn_round", std::to_string(entry.condemn_round)}, + {"key", layout.blobKey(entry.ref)}}; + }); + /// The audit row is observability only -- the delete above already executed regardless of + /// this cap. Skipping it here bounds the per-shard `GcOutcomes` body without skipping or + /// deferring any destructive work. + if (round_work_budget.outcomeEntryAvailable()) + { + outcomes[shard].entries.push_back(std::move(outcome)); + ++round_work_budget.outcome_entries_used; + } + ++report.redeleted; + ProfileEvents::increment(ProfileEvents::CASGCRetiredRedeleted); + /// Drop the per-hash meta only on Deleted/NotFound — a Replaced (TokenMismatch) outcome + /// means a writer already resurrected a fresh incarnation at this hash (INV-1), and that + /// writer's own resurrect path already flipped the meta back to Clean; blindly deleting here + /// would race that legitimate Clean write for no reason (the meta is advisory, but there is no + /// reason to touch it on that path at all). + if (del_class == DeleteClass::Deleted || del_class == DeleteClass::Absent) + { + const BlobRef ref = entry.ref; + scheduleMetaJob([this, ref]() { deleteConfirmedMeta(store->backend(), store->layout(), ref); }); + } + /// The entry left the pipeline — drop its in-process condemn-marker confirmation. + forgetCondemnMarker(entry.ref, entry.token); + } + for (const RetiredEntry & entry : merge.spared) + { + /// A fresh dedup-adopt raced the condemn (see the matching CasGcFold Debug log emitted + /// during the merge, which increments CASGCRetiredSparedByReref) -- not an ack-floor + /// violation, so this is Debug, not a page-worthy Warning. + if (entry.delete_pending) + LOG_DEBUG(logger, + "CAS gc: delete_pending blob {} (condemned at round {}, this round {}) recovered " + "in-degree -- a fresh dedup-adopt raced the condemn; spared (never a fail-closed delete)", + blobIdOf(entry.ref), entry.condemn_round, new_round); + /// Emit the spare verdict — a publish re-pinned the candidate before graduation. + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::GcRecheckVerdict; + e.object_kind = CasEventObjectKind::Blob; + e.object_hash = blobIdOf(entry.ref); + e.token = entry.token.value; + e.round = new_round; + e.gen = generation; + e.outcome = "spared"; + e.reason = "in-degree recovered in the pass merge; entry dropped"; + }); + /// The audit row is observability only -- `settleEntry` already unconditionally spared this + /// entry (INV_NO_LOSS: recovery wins past any budget), so this cap can never re-condemn it; + /// it only bounds whether the decision gets a `GcOutcomes` row. + if (round_work_budget.outcomeEntryAvailable()) + { + outcomes[shard].entries.push_back(OutcomeEntry{.kind = entry.kind, .ref = entry.ref, + .token = entry.token, .outcome = OutcomeKind::Spared}); + ++round_work_budget.outcome_entries_used; + } + ProfileEvents::increment(ProfileEvents::CASGCRetiredSpared); + /// A spare does NOT touch the + /// meta. GC freshness meta is add-only — GC never publishes `Clean`. The in-degree recovered, + /// but the meta stays `Condemned` (conservative marker) until a WRITER displaces the body with + /// a fresh incarnation token (`uploadFromSource` + `writeResurrectMetaClean`) — the SOLE + /// `Condemned -> Clean` transition. Clearing here on a + /// deposed leader that then lost its round CAS would strand a stray-`Clean` over a still-live + /// condemned token and lose the reuse to a stale exact-token redelete (INV_NO_LOSS); see + /// The next `putBlob` self-heals + /// the marker: `observeAndAdmit` refuses same-token adoption on `Condemned` and resurrects. + /// The entry left the pipeline — drop its in-process condemn-marker confirmation. + forgetCondemnMarker(entry.ref, entry.token); + } + for (const RetiredEntry & entry : merge.graduated) + { + ++report.graduated; + ProfileEvents::increment(ProfileEvents::CASGCRetiredGraduated); + /// Floor-passed — republished pending; the NEXT pass executes the delete. + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::GcRecheckVerdict; + e.object_kind = CasEventObjectKind::Blob; + e.object_hash = blobIdOf(entry.ref); + e.token = entry.token.value; + e.round = new_round; + e.gen = generation; + e.outcome = "pending"; + e.reason = "condemn_round < current_round; published delete_pending (two-phase graduation)"; + e.detail = {{"condemn_round", std::to_string(entry.condemn_round)}}; + }); + } + for (const ReplacedEntry & replaced : merge.replaced) + { + const RetiredEntry & entry = replaced.fresh; + ProfileEvents::increment(ProfileEvents::CASGCRetireReplaced); + /// RESURRECT-REUPLOAD-ORPHAN: the current object token differed from a stale retired entry; + /// the fold superseded that entry and re-condemned the current token in the same window. + /// `detail["superseded_token"]` carries the STALE token the supersede + /// dropped — `entry.token` above is only the fresh CURRENT token, and without the old one + /// this event cannot tell an operator WHICH incarnation was replaced. + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::BlobRetireReplaced; + e.object_kind = CasEventObjectKind::Blob; + e.object_hash = blobIdOf(entry.ref); + e.token = entry.token.value; + e.round = new_round; + e.gen = generation; + e.outcome = "replaced"; + e.reason = "current object token differs from the retired entry — resurrect replaced the " + "incarnation; superseded the stale entry and re-condemned the current token"; + e.detail = {{"superseded_token", replaced.old_token.value}}; + }); + /// The supersede is ALSO a blob entering the retired set fresh (a re-condemn of the + /// CURRENT token) — write the meta Condemned exactly like a fresh `head_blob` condemn would, + /// so a NEXT writer's point-read gate sees it (and the graduation gate gets its (hash, token) + /// confirmation on success). `peek_head` itself stays side-effect-free (it runs once per + /// closed candidate, not just on a real supersede — see its own comment). The SUPERSEDED + /// (stale) token's in-process confirmation is dropped — that entry left the pipeline. + scheduleCondemnMarkerWrite(entry.ref, entry.token, entry.condemn_round, entry.size); + forgetCondemnMarker(entry.ref, replaced.old_token); + } + } + + /// Outcome logs: write-once + byte-adopt (observation-bearing HEAD tokens — never the + /// deterministic-artifact path). Tally the report from the FINAL durable logs. + for (auto & [shard, log] : outcomes) + { + const String key = layout.outcomesKey(generation, attempt, new_round, shard); + const String body = sealObject(FormatId::GcOutcomes, encodeOutcomeLog(log)); + if (backend.putIfAbsent(key, body).outcome == PutOutcome::PreconditionFailed) + { + const auto existing = backend.get(key); + if (!existing) + throw Exception(ErrorCodes::ABORTED, + "CAS gc: outcome log at {} vanished between putIfAbsent and read", key); + if (existing->bytes != body) + { + try { log = decodeOutcomeLog(openObject(FormatId::GcOutcomes, existing->bytes)); } + catch (const Exception & e) + { + throw Exception(ErrorCodes::ABORTED, + "CAS gc: undecodable outcome log at {} cannot be adopted: {}", key, e.message()); + } + } + } + for (const OutcomeEntry & o : log.entries) + { + switch (o.outcome) + { + case OutcomeKind::Deleted: ++report.deleted; break; + case OutcomeKind::Absent: ++report.absent; break; + case OutcomeKind::Replaced: ++report.replaced; break; + case OutcomeKind::Spared: ++report.spared; break; + } + } + } + pending_deletes_timer->metric("redeleted", report.redeleted - redeleted_before); + pending_deletes_timer->metric("graduated", report.graduated - graduated_before); + pending_deletes_timer->metric("deleted", report.deleted); + pending_deletes_timer->metric("absent", report.absent); + pending_deletes_timer->metric("replaced", report.replaced); + pending_deletes_timer->metric("spared", report.spared); + pending_deletes_timer->metric("outcome_logs_written", outcomes.size()); + pending_deletes_timer.reset(); /// emits the `pending_deletes` row + + /// Wait for the round's whole batch of per-hash freshness-meta writes (condemned during the + /// fold above, spared/redeleted-confirmed during R3 above) BEFORE the round's retired-list publish and + /// its single gc/state CAS below — the writer's meta point-read gate must see this round's condemns + /// durable no later than the ledger it is paired with. `wait()` never throws here: every scheduled job + /// already caught its own exception (see `scheduleMetaJob`). + /// + /// PHASE 12/18 `meta_pool_wait`, AND THE ONE HONEST GAP IN THIS INSTRUMENTATION: the work being + /// waited on runs on `meta_pool` threads, so none of it appears in this thread's `ProfileEvents` + /// delta and the row's `ProfileEvents` map is EMPTY BY CONSTRUCTION. That is not a phase with no + /// cost -- it is a phase whose cost this mechanism cannot see, so it carries explicit job counts + /// instead: read `jobs_scheduled` / `jobs_completed` next to the duration to tell "the queue was + /// deep" from "the endpoint was slow". `jobs_completed` is sampled BEFORE the wait deliberately -- + /// after it, it would always equal `jobs_scheduled` and say nothing. + { + GcPhaseTimer t(phase_sink, "meta_pool_wait"); + const uint64_t scheduled = meta_jobs_scheduled_.load(std::memory_order_relaxed) + - meta_jobs_scheduled_at_round_start; + const uint64_t completed_on_entry = meta_jobs_completed_.load(std::memory_order_relaxed) + - meta_jobs_completed_at_round_start; + meta_pool->wait(); + t.metric("jobs_scheduled", scheduled); + t.metric("jobs_completed_on_entry", completed_on_entry); + t.metric("jobs_completed", meta_jobs_completed_.load(std::memory_order_relaxed) + - meta_jobs_completed_at_round_start); + } + + /// Retired-in-snapshot — there is NO separate retired-list object to publish anymore. The + /// round's surviving condemned entries were already sealed as `kCondemned` rows inside the fold's + /// `blob_target_runs` (durable before this CAS, via `putDeterministicArtifact`), and the per-shard + /// `condemned_summary` the seal carries makes the next round's graduation/carry decisions zero-I/O. + + /// The SINGLE round CAS publishes the round, adopted (generation, attempt), and retention cursor. + /// + /// PHASE 13/18 `round_commit`. It deliberately covers BOTH the retention prune (heavy: LISTs and + /// wholesale deletes, bounded at 64 generations a round) and the single `gc/state` CAS (trivial), + /// because the prune's writes are only safe as a pre-CAS action and splitting the two would suggest + /// they are independently retryable. `generations_visited` vs the prune's actual deletes is what + /// separates the two costs inside the row. + std::optional round_commit_timer; + round_commit_timer.emplace(phase_sink, "round_commit"); + const String manifest_sweep_cursor_before = state.manifest_sweep_cursor; + GcState next = state; + next.round = new_round; + if (!suppress_destructive && store->poolConfig().manifest_sweep_list_budget_keys > 0) + next.manifest_sweep_cursor = folded.orphan_sweep.next_cursor; + /// The generations the adopted seal's runs physically live in (reference-parent carry can point a + /// current shard's run back at an older generation's key). Retention must never reclaim these. + std::set referenced_generations; + for (const RunRef & r : folded.fold_seal.blob_target_runs) + referenced_generations.insert(r.generation); + /// ALSO protect every generation the PARENT (currently-adopted, pre-fold) seal references + /// (`parent_seal_runs`, captured above): this prune runs BEFORE the round's own gc/state CAS below, so + /// a losing leader must not destroy what the winning leader's already-adopted seal still points at — + /// pre-CAS destructive actions may only rely on PREVIOUSLY PUBLISHED state (triage #5). + for (const RunRef & r : parent_seal_runs) + referenced_generations.insert(r.generation); + /// Retention floor uses THIS round's (post-fold) `generation`, so `gc_snapshot_generations_to_keep` + /// keeps exactly that many generations back from the current one. If this round's `gc/state` CAS + /// then LOSES, the prune reclaimed one generation deeper than the durably-adopted generation would + /// imply -- an accepted forensics-window slack, not a data-loss risk: every still-reachable + /// run/blob is independently protected via `referenced_generations` (captured pre-fold above). + const uint64_t pruned_through_before = state.snap_pruned_through; + pruneSupersededGenerations(generation, attempt, next, referenced_generations, suppress_destructive, + round_work_budget); + round_commit_timer->metric("generations_visited", next.snap_pruned_through - pruned_through_before); + round_commit_timer->metric("pruned_through", next.snap_pruned_through); + round_commit_timer->metric("generations_referenced", referenced_generations.size()); + const CasResult res = backend.casPut(layout.gcStateKey(), encodeGcState(next), state_token); + if (res.outcome != CasOutcome::Committed) + throw Exception(ErrorCodes::ABORTED, + "CAS gc round: gc/state moved during the round (another leader advanced it); retry next round"); + state = std::move(next); + state_token = res.token; + report.round = state.round; + round_commit_timer->metric("round", report.round); + round_commit_timer->metric("generation", generation); + round_commit_timer.reset(); /// emits the `round_commit` row + + /// Task 7: the retire pipeline's REMAINING sizes, read from the seal this round's CAS just + /// committed (`folded.fold_seal.condemned_summary` is TOTAL over every gc-shard -- see its own doc + /// comment in `CasFoldSealFormat.h`). Zero-shard pools (never folded) leave these at 0. + for (const auto & [shard, summary] : folded.fold_seal.condemned_summary) + { + report.pending_retired += summary.pending_total; + report.pending_candidates += summary.condemned_total - summary.pending_total; + report.pending_condemned += summary.condemned_total; + } + + /// Post-CAS reference-parent HAND-OFF DELETE. `pruneSupersededGenerations` SKIPS a + /// generation the live seal still references AND advances `snap_pruned_through` PAST it + /// (CasGc.cpp:1066 computes the cursor as `g - 1` after the loop increments `g` over every skipped + /// generation). So once a skipped generation is behind the cursor, the wholesale prune NEVER revisits + /// it — a ref that later moves off it would strand that generation's WHOLE prefix (fold seal, retired/ + /// outcomes sets, all shards' runs), not just the single carried run object. Reclaim it HERE, now that + /// the ref has moved: for every parent ref whose generation is already pruned-through and whose + /// generation NO new live ref still references, wholesale-delete that generation's prefix — the exact + /// reclaimer the normal prune would have used, deferred until the ref finally moved off. Best-effort: + /// a crash between the CAS and here leaks the prefix to fsck (single-crash window, no permanent leak — + /// but note the cursor already advanced, so a plain retry will NOT re-attempt it; fsck is the backstop). + /// + /// PHASE 14/18 `handoff_reclaim`. + { + GcPhaseTimer t(phase_sink, "handoff_reclaim"); + uint64_t objects_reclaimed = 0; + std::set new_referenced_generations; + for (const RunRef & r : folded.fold_seal.blob_target_runs) + new_referenced_generations.insert(r.generation); + + std::set handed_off; /// dedupe: multiple parent refs can share one generation + /// GATED like every other destructive site, and it is also the FIRST destructive site of the + /// post-CAS tail -- which is why the gate is read before the tail begins rather than partway + /// down it. + /// + /// UNLIKE EVERY OTHER GATED SITE, SUPPRESSION HERE LOSES THE WORK RATHER THAN POSTPONING IT. + /// The hand-off is a one-shot DIFFERENCE between the parent seal's runs and the new seal's, and + /// a suppressed round still FOLDS -- only the irreversible half stops -- so the ref moves off + /// the old generation on this very round and the next round's parent seal no longer names it. + /// Nothing revisits it: `snap_pruned_through` is already past that generation and the wholesale + /// prune only walks forward. The prefix is left to `fsck`, which is the same outcome this site + /// already documents for a crash in this window (see the PHASE 14/18 comment above). Bounded -- + /// one small run per shard per occurrence of a suppressed round that also folded a delta -- and + /// not a correctness problem, but it is the one place where the gate costs something permanent, + /// so it is asserted rather than left to be discovered. + static const std::vector kNoRuns; + const std::vector & handoff_candidates = + suppress_destructive ? kNoRuns : parent_seal_runs; + for (const RunRef & old_ref : handoff_candidates) + { + /// Only generations the wholesale prune already passed AND that no live ref still pins. + if (old_ref.generation > state.snap_pruned_through) + continue; /// not yet pruned-through: the normal prune will reclaim it when it ages out + if (new_referenced_generations.contains(old_ref.generation)) + continue; /// still referenced by a (possibly different-shard) live ref: keep it + if (!handed_off.insert(old_ref.generation).second) + continue; /// already reclaimed this round via another shard's ref + /// `bounded_remaining` draws from the hand-off's OWN reserve, never `UINT64_MAX` and never + /// `pruneSupersededGenerations`' shared remainder: this hand-off is a ONE-SHOT event (see the + /// PHASE 14/18 comment above) -- a generation this call reclaims only PARTIALLY is left + /// exactly like a crash in this window already is -- to `fsck`, never revisited by a later + /// round's hand-off (the parent-seal difference that triggers it does not recur once the ref + /// has moved). The prune, by contrast, safely retries an under-served generation next round + /// via its cursor, so sharing one pool would let a prune-heavy round strand this one-shot + /// reclaim at zero every time; the separate reserve makes that impossible. + const uint64_t remaining = round_work_budget.handoffPrefixWholesaleRemaining(); + if (remaining == 0) + break; + const uint64_t reclaimed = deletePrefixWholesale( + backend, layout.gcGenPrefix(old_ref.generation), remaining); + round_work_budget.handoff_prefix_wholesale_objects_used += reclaimed; + objects_reclaimed += reclaimed; + LOG_TRACE(logger, + "CAS GC hand-off: generation {} moved out of the live seal below the retention cursor " + "({} objects) — post-CAS wholesale reclaim (the prune had skipped it while referenced)", + old_ref.generation, reclaimed); + } + t.metric("generations_reclaimed", handed_off.size()); + t.metric("objects_reclaimed", objects_reclaimed); + t.metric("suppressed", suppress_destructive ? 1 : 0); + } + + /// Post-CAS: owner-removed manifest bodies — deleted ONLY now, after their decrements were + /// ADOPTED by the round CAS (delete-after-sealed-decrements). NOT durable across rounds: the + /// ref-log intake cursor that discovered each `-1` edge is committed by THIS round's CAS above, + /// so a log already folded is never re-visited and never re-populates `mf_cleanup`. This phase is + /// deliberately unbudgeted: a cap would leave a declined entry unreachable from any live ref AND + /// never re-derived by this pipeline, converting a bounded burst into a permanent leak. It drains + /// the whole of `folded.mf_cleanup` every round it runs; only a crash (or the destructive-suppression + /// gate below) leaves an entry for the orphan-manifest sweep to reclaim later. + /// + /// PHASE 15/18 `manifest_deletes`. + { + GcPhaseTimer t(phase_sink, "manifest_deletes"); + const uint64_t manifests_deleted_before = report.manifests_deleted; + /// GATED. A manifest body is content the ref graph still describes until its decrements are both + /// sealed AND taken on a round that could prove its frontier -- an unprovable round's `-1` may + /// itself be the observation that is missing an owner elsewhere, so deleting the body on it is + /// exactly the irreversible step the gate exists to withhold. + static const std::map kNoManifestCleanup; + const std::map & mf_cleanup_now = + suppress_destructive ? kNoManifestCleanup : folded.mf_cleanup; + uint64_t attempted = 0; + for (const auto & [id, token] : mf_cleanup_now) + { + ++attempted; + const DeleteOutcome mdel = backend.deleteExact(layout.manifestKey(id), token); /// NotFound/TokenMismatch tolerated + const DeleteClass mdel_class = classifyDeleteOutcome(mdel); + if (mdel_class == DeleteClass::Deleted) + ++report.manifests_deleted; + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::ManifestDelete; + e.namespace_ = id.root_namespace.string(); + e.object_kind = CasEventObjectKind::Manifest; + e.object_hash = manifestRefDebugString(id.ref); + e.token = token.value; + e.round = new_round; + e.gen = generation; + e.outcome = String{deleteClassName(mdel_class)}; + e.reason = "owner-removed manifest body; exact-token delete after decrements adopted"; + }); + } + t.metric("attempted", attempted); + t.metric("deleted", report.manifests_deleted - manifests_deleted_before); + t.metric("suppressed", suppress_destructive ? 1 : 0); + } + + /// Removal completion has no physical pass. The terminal fold placed positive evidence in the + /// life row; a later invocation's catalog-only pre-fold drain owns lifecycle deletion, while the + /// perpetual janitor owns dead-life bytes. + uint64_t cleanup_evidence_rows = 0; + for (const auto & [life_id, ref_life_state] : folded.fold_seal.ref_lives) + cleanup_evidence_rows += ref_life_state.cleanup_evidence ? 1 : 0; + runNamespaceJanitorPage(state, suppress_destructive, cleanup_evidence_rows); + /// PHASE 17/18 `ref_object_cleanup`. Emitted even when the whole pass is skipped (`trim_enabled` is + /// a test seam, `suppressed` gates the deletes), because "this phase did nothing and why" is exactly + /// what a reader of a round that reclaimed nothing needs to see. + { + GcPhaseTimer t(phase_sink, "ref_object_cleanup"); + if (trim_enabled) + cleanupRefObjects(folded, state.lease, suppress_destructive, round_work_budget); + t.metric("suppressed", suppress_destructive ? 1 : 0); + t.metric("trim_enabled", trim_enabled ? 1 : 0); + t.metric("namespaces_planned", folded.ref_tables.size()); + } + + /// Bounded orphan-manifest backstop. The fold already exact-read each candidate, retired its exact + /// source edges into the adopted runs and placed cursor progress in the SAME `gc/state` CAS above. + /// Only this post-CAS tail may delete candidate bodies. + /// PHASE 18/18 `orphan_sweep`. + { + GcPhaseTimer t(phase_sink, "orphan_sweep"); + ManifestSweepResult & sweep = folded.orphan_sweep; + for (const ManifestSweepResult::Nomination & nomination : sweep.nominations) + { + const DeleteOutcome outcome = backend.deleteExact(nomination.key, nomination.token); + const DeleteClass outcome_class = classifyDeleteOutcome(outcome); + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::ManifestDelete; + e.namespace_ = nomination.id.root_namespace.string(); + e.object_kind = CasEventObjectKind::Manifest; + e.object_hash = nomination.key; + e.token = nomination.token.value; + e.round = new_round; + e.gen = generation; + e.outcome = String{deleteClassName(outcome_class)}; + e.reason = "orphan-manifest sweep: source edges retired and adopted before exact-token delete"; + }); + if (outcome.kind == DeleteOutcome::Kind::TokenMismatch) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS orphan sweep: manifest key {} changed token after exact GET; immutable manifest " + "identity suffered illegal ABA, retained replacement", nomination.key); + if (outcome_class == DeleteClass::Deleted) + ++sweep.deleted; + else + ++sweep.skipped; + } + reportSweepRetention(sweep); + t.metric("cursor_advanced", state.manifest_sweep_cursor != manifest_sweep_cursor_before ? 1 : 0); + t.metric("list_budget_keys", store->poolConfig().manifest_sweep_list_budget_keys); + t.metric("suppressed", suppress_destructive ? 1 : 0); + t.metric("listed", sweep.listed); + t.metric("deleted", sweep.deleted); + t.metric("skipped", sweep.skipped); + /// THE §6 PREMISE'S SHARE OF `skipped`, BY REASON CLASS. Rule (1) is satisfiable only for a + /// closed-and-folded epoch, so a pass in which everything examined was RETAINED is an ordinary + /// outcome, and without these numbers it is indistinguishable on the row from a pass that found + /// nothing to do. + /// A row where `deleted` is 0 and all four are 0 means the sweep genuinely had no candidates. + t.metric("retained_no_coverage", sweep.retained_no_coverage); + t.metric("retained_hold", sweep.retained_hold); + t.metric("retained_unconsumed_seal", sweep.retained_unconsumed_seal); + t.metric("retained_tail_removal", sweep.retained_tail_removal); + } + + return report; +} + +void Gc::reportStuckRemovals(const RefPlan & plan, uint64_t current_round) +{ + for (const UInt128 & life_id : plan.lifeIds()) + { + const auto warning = stuckRemovalWarning( + plan.row(life_id), current_round, store->poolConfig().gc_stuck_removal_rounds, + store->layout()); + if (!warning) + continue; + ProfileEvents::increment(ProfileEvents::CASGCStuckRemovals); + LOG_WARNING(logger, "{}", *warning); + } +} + +bool Gc::foldManifestEdges(const ManifestId & id, int sign, std::vector & deltas, + std::map & mf_cleanup, uint32_t txn_ordinal) +{ + Backend & backend = store->backend(); + const Layout & layout = store->layout(); + + const String key = layout.manifestKey(id); + /// ONE ROUND TRIP PER EDGE. The GET alone carries the absence signal a HEAD would have carried, so + /// the HEAD that used to precede it bought nothing and cost a second serial round trip on the + /// hottest read path of the round (one per manifest edge, on every folded log). `!got` is the SAME + /// absent outcome the missing HEAD used to produce -- record-and-continue, and the caller decides + /// what an absent body means for that edge (a missing-body precommit is a barrier; a committed one + /// fails closed). Never a throw: a 404 during the fold is an observation, not an error. + const auto got = backend.get(key); + if (!got) + return false; /// absent body: caller decides (missing-body precommit OK; committed => fail closed) + ProfileEvents::increment(ProfileEvents::CASRefManifestBodyFoldGets); /// one body GET per manifest fold + + const PartManifest body = decodePartManifest(openObject(FormatId::PartManifest, got->bytes)); + if (!refMatchesBody(id.ref, body)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS gc fold: manifest body ref mismatch at {} (refMatchesBody fail-closed)", key); + if (!manifestNamespaceMatches(id.root_namespace, body)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS gc fold: manifest body namespace mismatch at {} (manifestNamespaceMatches fail-closed)", key); + /// The manifest-wide `blob_hash_len` foreign-width gate is GONE (the field + /// itself was deleted — entries now carry their own per-entry algo/width). `decodePartManifest` + /// already fail-closes on an algo byte this BUILD does not know (`blobHashAlgoName` throws + /// CORRUPTED_DATA there) -- but a known algo may still not be ADMITTED to THIS pool yet (a stale + /// in-memory `admitted_algos` cache reading a manifest another node already admitted a new algo + /// for). Per-entry admission validation refreshes on miss BEFORE + /// failing closed, so a genuinely fresh admission is never mistaken for corruption. + for (const ManifestEntry & entry : body.entries) + if (entry.placement == EntryPlacement::Blob && !store->isAlgoAdmitted(entry.ref.algo)) + { + const std::vector refreshed = store->refreshAdmittedAlgos(); + if (!store->isAlgoAdmitted(entry.ref.algo)) + { + String names; + for (size_t i = 0; i < refreshed.size(); ++i) + { + if (i != 0) + names += ", "; + names += blobHashAlgoName(static_cast(refreshed[i])); + } + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS gc fold: manifest entry algo {} not admitted to this pool (algos_used {{{}}})", + blobHashAlgoName(entry.ref.algo), names); + } + } + + for (const ManifestEntry & entry : body.entries) + if (entry.placement == EntryPlacement::Blob) + { + /// The fold settles the FULL `BlobRef` pair natively -- no bare-digest bridge remains. + deltas.push_back(BlobDelta{ + .ref = entry.ref, + .source_id = sourceEdgeId(id, entry.path), + .remove = (sign < 0), + .txn_ordinal = txn_ordinal}); + /// A folded owner edge over this blob (the manifest-model analog of the old + /// `RootAdd`). +1 = the manifest's owner activated this blob's reference; -1 = + /// the owner was removed, dropping the reference. Reconstructs WHY a blob's in-degree moved. + EventEmitter{*store}.emit([&](CasEvent & ev) + { + ev.type = sign > 0 ? CasEventType::RootAdd : CasEventType::RootRemove; + ev.namespace_ = id.root_namespace.string(); + ev.object_kind = CasEventObjectKind::Blob; + ev.object_hash = blobIdOf(entry.ref); + ev.outcome = sign > 0 ? "edge_added" : "edge_removed"; + ev.reason = sign > 0 + ? "fold: manifest owner activated; +1 blob edge" + : "fold: manifest owner removed; -1 blob edge"; + ev.detail = {{"manifest_ref_instance", manifestRefDebugString(id.ref)}, + {"path", entry.path}}; + }); + } + + if (sign < 0) + mf_cleanup.emplace(id, got->token); /// owner removed: defer exact-token body delete to recheck + return true; +} + +Gc::CheckpointWitnesses Gc::readCheckpointWitnesses(const std::map & ref_tables, + const CasRefCatalog::Snapshot & catalog_cut) +{ + /// Read the checkpoint of every namespace in the round's catalog cut, every namespace `ref_tables` + /// names, PLUS every namespace a HELD row in `parent_cursors` names. A catalog-only namespace is the + /// one whose second witness matters MOST: a genuinely empty listing cannot distinguish a namespace + /// that has no records from one whose records the same enumeration missed. + /// + /// EXACT KEY, ALWAYS -- never `RefTableListing::has_ckpt`. Skipping the read because the listing did + /// not show a `_ckpt` would make the second witness a function of the first, which is precisely the + /// dependency it exists to break: the listing is a SNAPSHOT, and a `_ckpt` that became durable after + /// the enumeration is exactly the one whose namespace has records the same enumeration also missed. + Backend & backend = store->backend(); + const Layout & layout = store->layout(); + + std::set witness_namespaces; + for (const CatalogEntry & entry : catalog_cut.catalog.entries) + if (entry.state == NsState::Live || entry.state == NsState::Removing) + witness_namespaces.insert(entry.ns.string()); + for (const auto & [ns_str, listing] : ref_tables) + witness_namespaces.insert(ns_str); + + CheckpointWitnesses out; + for (const String & ns_str : witness_namespaces) + { + const RootNamespace ns{ns_str}; + /// Review C3: use the SAME complete catalog cut the round's walk resolved, never an independent + /// catalog re-read. A namespace absent from the cut, or present only as a non-walkable + /// `Creating` row, has no admitted witness key to read this round. + const auto entry_it = std::lower_bound( + catalog_cut.catalog.entries.begin(), catalog_cut.catalog.entries.end(), ns, + [](const CatalogEntry & entry, const RootNamespace & needle) { return entry.ns < needle; }); + if (entry_it == catalog_cut.catalog.entries.end() || entry_it->ns != ns + || (entry_it->state != NsState::Live && entry_it->state != NsState::Removing)) + continue; + const String ckpt_key = layout.refCkptKey(NamespaceLifeId::fromCatalogEntry(entry_it->ns, entry_it->incarnation)); + /// THE GET AND THE DECODE ARE SPLIT HERE, rather than taken together through `readCkpt`, so the + /// catch below can scope to the DECODE ALONE. Wrapping the read too would turn a transport + /// failure -- which says nothing about this object and everything about the round's ability to + /// read anything -- into a per-namespace hold, silently narrowing a pool-wide outage to one + /// namespace. A backend throw still propagates and fails the round, exactly as it always did. + const std::optional got = backend.get(ckpt_key); + /// ABSENT IS NORMAL AND IS NOT A WITNESS: a namespace has no `_ckpt` until its first snapshot + /// publication commits, and one that 404s mid-round is a namespace being reclaimed. Neither says + /// anything about which ids exist, so neither may hold the walk -- and neither may throw + /// (a GC fold never fails a round on a 404). + if (!got) + continue; + + RefCkpt ckpt; + try + { + /// Materialized read, then decode (`readCkpt`'s rule): the object is MUTABLE, so the body + /// must be fixed before it is parsed. + ckpt = decodeRefCkpt(got->bytes); + } + catch (const Exception & e) + { + /// PER-NAMESPACE, NEVER ROUND-WIDE (spec §5: every per-namespace failure is a clamp or a + /// hold). This object belongs to exactly one namespace, so it can never be grounds for + /// refusing to fold another one -- and refusing the whole round is what a single unreadable + /// 4 KiB object used to do, stopping every namespace's cursor, seal and cleanup for as long + /// as it stayed unrepaired. It is recorded, named, and left to the walk to hold. + /// + /// NAME THE OBJECT. The decode's own message says what is wrong with the bytes and nothing + /// about WHICH bytes, so without the key an operator cannot find the object to repair. + out.undecodable.emplace(ns_str, ckpt_key + ": " + e.message()); + continue; + } + /// A checkpoint without `checkpoint_snapshot_id` is silent rather than empty: the object exists + /// because some OTHER field (`life_epoch`, `last_epoch_seal`) was published into it first. + if (ckpt.checkpoint_snapshot_id) + out.witnesses.emplace(ns_str, *ckpt.checkpoint_snapshot_id); + if (ckpt.life_epoch) + out.life_epochs.emplace(ns_str, *ckpt.life_epoch); + out.recovery_checkpoints.emplace(ns_str, std::move(ckpt)); + } + return out; +} + +std::optional> Gc::newestFoldSealRef() +{ + Backend & backend = store->backend(); + const Layout & layout = store->layout(); + const String gen_prefix = layout.gcGenPrefix(0); + const String top = gen_prefix.substr(0, gen_prefix.size() - 2); /// ".../gc/gen/" + + /// THE WIDE ENUMERATION IS A HINT HERE TOO, exactly as it is in the ref walk. Trusting it for + /// NEWEST-ness would reopen the same hole one layer up: an enumeration that omits the true newest + /// seal hands back an older one, and every hold detected since that older seal is silently lost -- + /// an under-carry, which is the failure this whole path exists to prevent. + std::set listed_generations; + bool listed_anything = false; + std::optional> newest; + forEachListedKey(backend, top, [&](const ListedKey & k) + { + listed_anything = true; + const size_t from = top.size(); + const size_t gen_end = k.key.find('/', from); + if (gen_end == String::npos) + return; + uint64_t generation = 0; + try + { + generation = std::stoull(k.key.substr(from, gen_end - from)); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + return; /// foreign key shape under `gc/gen` is debris, not a generation number + } + listed_generations.insert(generation); + }, 1000, onGcEnumerationPage); + const uint64_t listed_max_generation = listed_generations.empty() ? 0 : *listed_generations.rbegin(); + + /// STEP DOWN THROUGH THE GENERATIONS THE LISTING ITSELF REPORTED until one carries a seal. The + /// newest generation routinely exists WITHOUT one: a round writes its runs during the reduce phase + /// and its fold seal only at phase 10/18, so an ordinary crash in between leaves exactly that + /// shape. Stopping at the maximum would then refuse a pool whose holds are sitting readable one + /// generation down -- turning a plain crash into "recreate the pool". + /// + /// Stepping down costs no trust that has not already been spent: these are the generations the wide + /// listing reported, and its maximum -- which the probes above are checking -- is one of them. What + /// is NOT weakened is the refusal above: a seal found ABOVE the maximum stays terminal, because + /// that is the listing being caught in a lie rather than merely being incomplete about seals. + /// + /// "Never step PAST an unreadable seal" falls out of returning the FIRST generation that carries + /// one: the caller decodes it and refuses if it cannot, so an undecodable seal ends the search + /// instead of being skipped over in favour of an older, readable one. + for (auto it = listed_generations.rbegin(); it != listed_generations.rend(); ++it) + { + if (const auto probe = probeGenerationForSeal(*it); probe.seal_attempt) + { + newest = std::make_pair(*it, *probe.seal_attempt); + break; + } + } + + /// DETECTION, NOT PROOF -- and labelled as such deliberately, in the same spirit as probe A. Two + /// NARROW single-generation probes above the wide listing's maximum ask whether that maximum was a + /// lie. The generation half of the question is arithmetic (generations are dense in minting: a fold + /// takes `snap_generation + 1`, a rebuild `max_gen + 1`), but the attempt half is not and cannot be + /// made so -- `attempt` is `lease.seq`, a global counter that advances on EVERY round including + /// deferred ones, so consecutive generations carry attempts separated by unbounded gaps and there is + /// no `attempt + 1` to point-read. So the step is an enumeration WITHIN ONE DIRECTORY: strictly + /// narrower than the pool-wide listing whose maximum it is checking, and honestly not the exact + /// read the ref walk gets. + /// + /// A seal found above the maximum means the wide listing lied about the very thing this path is + /// deciding, so the answer is REFUSAL, not adoption of the newer seal: a store that misreports its + /// own enumeration DURING DISASTER RECOVERY does not get a second guess, and silently adopting + /// whatever the second query returned would just move the trust one query along. + static constexpr uint64_t kProbeGenerationsAbove = 2; + for (uint64_t above = 1; above <= kProbeGenerationsAbove; ++above) + { + const uint64_t generation = listed_max_generation + above; + const GenerationSealProbe probe = probeGenerationForSeal(generation); + if (!probe.seal_attempt) + continue; + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC rebuild: a fold seal exists at generation {} (attempt {}) while the pool-wide " + "enumeration of {} reported nothing above generation {}. The enumeration this rebuild " + "would have taken its baseline from is demonstrably incomplete, so the holds it carries " + "cannot be trusted to be all of them. GC refuses to rebuild; this pool must be recreated.", + generation, *probe.seal_attempt, top, listed_max_generation); + } + + if (newest) + return newest; + + /// THE VIRGIN VERDICT, and everything it rests on. `gc/state` is already known absent or + /// unreadable (the caller's precondition); the wide listing found nothing; and one narrow probe of + /// generation 1 -- the first generation any pool would ever mint -- also finds nothing. That is + /// three pieces of ENUMERATION evidence and no point read, because the seal key's attempt component + /// has no arithmetic successor to probe. + /// + /// NAMED RESIDUAL, and the generation-1 probe NARROWS it rather than closing it. On a pool that has + /// been pruned, generation 1 LEGITIMATELY does not exist -- `pruneSupersededGenerations` deletes + /// whole old generation prefixes once they age past `gc_snapshot_generations_to_keep` -- so an empty + /// generation-1 probe proves nothing there. A total enumeration blackout on a lived-in, pruned pool + /// therefore still reads virgin here, and grants it a clean slate with no holds. What the probe + /// does buy is the un-pruned case: a young pool whose seals the wide listing hid is caught. + /// + /// No closure exists in the current key shapes, because a fold seal cannot be point-read from its + /// generation alone. The fix is a derivable per-generation marker that CAN be point-read, and it + /// would survive the blackout precisely because it needs no enumeration (see the report and + /// `docs/superpowers/cas/BACKLOG.md`). Anything the listing DOES show above its own maximum is + /// caught by the refusal above instead. + const GenerationSealProbe genesis = probeGenerationForSeal(1); + if (listed_anything || genesis.generation_exists) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC rebuild: the pool-wide enumeration of {} yielded no fold seal, yet {} -- so the " + "pool is NOT provably new and its holds cannot be enumerated. GC refuses to rebuild; this " + "pool must be recreated.", + top, + listed_anything ? "that enumeration did return objects under it" + : "a narrow probe of generation 1 found objects the wide listing omitted"); + + ProfileEvents::increment(ProfileEvents::CASGCRebuildVirginByEnumeration); + LOG_WARNING(logger, + "CAS GC rebuild PROCEEDING AS NEVER-SEALED: no fold seal was found by the broad listing of {} " + "or by the generation-1 probe, and gc/state is absent or unreadable, so NO durable hold is " + "carried forward. IMPLICATION: if this pool HAS sealed and then pruned, generation 1 is gone " + "legitimately and this verdict rests on a total enumeration blackout -- holds may be lost and " + "GC may reclaim blobs a held namespace still protects. The verdict rests on ENUMERATION ALONE: " + "no point read can prove it, because a fold seal key needs an attempt component that is a " + "lease sequence number. Verify with the store operator that the object listing is complete " + "before trusting this rebuild.", + top); + return std::nullopt; +} + +Gc::GenerationSealProbe Gc::probeGenerationForSeal(uint64_t generation) +{ + Backend & backend = store->backend(); + const Layout & layout = store->layout(); + + GenerationSealProbe probe; + forEachListedKey(backend, layout.gcGenPrefix(generation), [&](const ListedKey & k) + { + probe.generation_exists = true; /// ANY object proves this generation was minted + /// Parse a candidate attempt out of the path and then PROVE it by rebuilding the key: only a + /// string `foldSealKey` itself would have produced is a fold seal. Everything else under a + /// generation -- run objects, outcome sets, debris of a lost era -- must not get to decide + /// which baseline this pool's holds are read from. + static constexpr std::string_view kAttempt = "/attempt/"; + const size_t a_begin = k.key.find(kAttempt); + if (a_begin == String::npos) + return; + const size_t a_from = a_begin + kAttempt.size(); + const size_t a_end = k.key.find('/', a_from); + if (a_end == String::npos) + return; + uint64_t attempt = 0; + try + { + attempt = std::stoull(k.key.substr(a_from, a_end - a_from)); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + return; /// foreign key shape is debris, not an attempt + } + if (layout.foldSealKey(generation, attempt) != k.key) + return; + if (!probe.seal_attempt || *probe.seal_attempt < attempt) + probe.seal_attempt = attempt; + }, 1000, onGcEnumerationPage); + return probe; +} + +uint64_t Gc::FoldResult::FrontierDeficit::total() const +{ + return checkpoint_unusable + checkpoint_frontier_empty + committed_below_cursor + + held + probe_budget + fold_aborted + unattributed; +} + +String Gc::FoldResult::FrontierDeficit::describe() const +{ + String out; + const auto add = [&](const char * name, uint64_t count) + { + if (count == 0) + return; + if (!out.empty()) + out += ", "; + out += fmt::format("{}={}", name, count); + }; + add("checkpoint_unusable", checkpoint_unusable); + add("checkpoint_frontier_empty", checkpoint_frontier_empty); + add("committed_below_cursor", committed_below_cursor); + add("held", held); + add("probe_budget", probe_budget); + add("fold_aborted", fold_aborted); + add("unattributed", unattributed); + return out; +} + +void Gc::FoldResult::FrontierDeficit::count(FrontierUnproven reason) +{ + switch (reason) + { + case FrontierUnproven::Proven: return; + case FrontierUnproven::CheckpointUnusable: ++checkpoint_unusable; return; + case FrontierUnproven::CheckpointFrontierEmpty: ++checkpoint_frontier_empty; return; + case FrontierUnproven::CommittedBelowCursor: ++committed_below_cursor; return; + case FrontierUnproven::Held: ++held; return; + case FrontierUnproven::Unattributed: ++unattributed; return; + } +} + +Gc::FoldResult Gc::fold(GcState & state, Token & /*state_token*/, RoundReport & report, + uint64_t current_round, const RefPlan & walk_plan, UniversePolicy policy, + GcRoundWorkBudget & work_budget) +{ + Backend & backend = store->backend(); + const Layout & layout = store->layout(); + FoldResult result; + + /// 1. Group the round's one enumeration of `cas/ns/stream/` (taken before the defer decision) into + /// per-table immutable-object listings. That single enumeration serves the defer signal, the + /// ref-log intake, and ref-object cleanup planning alike. + /// + /// THE ROUND LISTS THIS PREFIX ONCE. The intake does not need a second opinion about the listing + /// because it does not consult the listing for completeness at all -- it walks by exact key from + /// the cursor. + /// + /// PHASE 6/18 `fold_ref_group`: the strict regrouping -- what this round will fold, decided before it + /// reads a single body. No I/O: the keys are already in hand. + std::optional ref_list_timer; + ref_list_timer.emplace(phase_sink, "fold_ref_group"); + const RefScanSummary & ref_scan = walk_plan.refScan(); + const std::vector & ref_object_keys = ref_scan.keys; + + /// Stage B (spec INV-3): the round's ONE catalog `GET`. `live_incarnation` names, for every + /// namespace the catalog admits as `Live`/`Removing`, the ONE incarnation the fold may act on -- + /// `Creating` is excluded, matching `discoverUniverse` (no publication can exist yet). This is what + /// makes the fold catalog-authoritative rather than LIST-authoritative: the pool-wide ref LIST + /// remains the round's intra-namespace hint (what a namespace the catalog already named has + /// listed), never the source of WHICH namespaces exist. Reused below for the catalog-only walk + /// targets, so the round pays this GET once. + const CasRefCatalog::Snapshot & catalog_snapshot = walk_plan.catalogCut(); + std::map live_incarnation; + /// Final review F1: a `Creating` life IS named by the catalog -- `live_incarnation` excludes it + /// only because it is not yet WALKABLE (spec §3, no publication can exist), not because the + /// namespace is unaccounted. `completeCreation` publishes `_ckpt` (step 2) strictly BEFORE the + /// `Creating -> Live` CAS (step 3), so every ordinary namespace creation has a real window -- + /// crash-stalled or merely mid-flight -- where a `Creating` entry's own `_ckpt` is durable and + /// listed while the entry itself is absent from `live_incarnation`. R10's un-cataloged anomaly + /// below must tell that apart from genuine "nothing in the catalog names this at all" debris, or + /// an ordinary or stalled creation suppresses the whole round's reclamation until someone + /// recreates the exact name and drives `reconcileStaleCreator` -- unbounded in the stalled case. + for (const NamespaceLifeId & life : walk_plan.lives()) + live_incarnation.emplace(life.ns.string(), life.incarnation); + /// Carry the complete cut on the result (review C3) so every later consumer this round -- + /// `cleanupRefObjects` and terminal-evidence attribution -- retains both the chosen incarnation and + /// the lifecycle/absence distinction instead of re-reading or reducing the catalog independently. + result.catalog_cut = catalog_snapshot; + /// THE POSITIVE EMPTY-UNIVERSE PROOF (see the destructive gate below). `token` is guaranteed by + /// `CasRefCatalog::read` on every operational path -- absence there is `CORRUPTED_DATA`, never an + /// empty snapshot -- but the check stays here so this fails closed if a bootstrap/test snapshot + /// ever reaches this line. `entries` (not `live_incarnation`, which drops `Creating`) is the right + /// source: a catalog holding only `Creating` rows must NOT read as an empty universe, and `entries` + /// is the one view that still carries those rows. + result.catalog_cut_proved_empty = catalog_snapshot.token.has_value() && catalog_snapshot.catalog.entries.empty(); + + /// A malformed ref-object key or namespace aborts ref folding for the whole round: the + /// round produces no ref delta, advances no cursor, and authorizes no destructive work -- recorded as + /// an anomaly (which drives `suppress_destructive`), never a throw that wedges the round. + std::map ref_tables; + bool ref_folding_aborted = false; + try + { + const auto physical_tables = groupRefKeys(layout, ref_object_keys); + for (const auto & [life_id, listing] : physical_tables) + { + const auto life = catalog_snapshot.life_index.resolve(life_id); + if (!life) + continue; /// absent from the post-LIST cut: inert dead-life debris + const auto live_it = live_incarnation.find(life->ns.string()); + if (live_it != live_incarnation.end() && live_it->second == life_id) + ref_tables.emplace(life->ns.string(), listing); + } + } + catch (const Exception & e) + { + ref_folding_aborted = true; + report.recordAnomaly(RootNamespace{}, 0, ManifestId{}, + "malformed ref-object key: ref folding aborted this round"); + LOG_WARNING(logger, + "CAS GC ref intake: {} -- aborting ref folding for the round", e.message()); + } + for (const auto & [ns_str, listing] : ref_tables) + result.root_shards.emplace_back(RootNamespace{ns_str}, 0); + + ref_list_timer->metric("ref_keys_listed", ref_object_keys.size()); + ref_list_timer->metric("namespaces_seen", ref_tables.size()); + /// The round's only remaining whole-round ref abort: a key attributable to no namespace. Reported on + /// every round, healthy or not — a column that is always 0 is what makes the one round where it is + /// not stand out. + ref_list_timer->metric("ref_folding_aborted", ref_folding_aborted ? 1 : 0); + ref_list_timer.reset(); /// emits the `fold_ref_group` row + + /// Parent cursors — the per-(ns,shard) cursors a prior round sealed. `listRefPrefix` read them from + /// the fold seal at the adopted (snap_generation, snap_attempt) before it built this round's one + /// walk plan (the fold seal IS the coverage record). + /// Absent => fresh pool (cursor 0). A folded event must never be re-folded from 0 (that double-counts + /// blob in-degree => silent over-pin/leak). + /// A live `gc/state` whose adopted + /// fold seal OBJECT is MISSING is corrupt bookkeeping, never an empty baseline — treating it as + /// empty would re-fold only journal tails and mass-condemn everything the lost snapshot + /// protected. NOTE the distinction from a PRESENT seal with empty `ref_lives` (a legitimate + /// empty-universe generation) — the audit keys on object absence, not coverage emptiness. + /// + /// PHASE 7/18 `fold_seal_read`. The scope reaches down to `discover_ref_seal` below, because that is + /// a SECOND GET of the SAME key at the SAME (generation, attempt) -- the two reads belong on one row + /// or the redundancy is invisible. Everything between them is I/O-free (a `resize`, three lambda + /// DEFINITIONS, and plain assignments), so the duration is honestly the two GETs and their decodes. + /// Instrumented, NOT fixed: removing the second read is a behaviour change the follow-up study + /// decides, and the `redundant_reads` metric is the evidence it will need. + std::optional seal_read_timer; + seal_read_timer.emplace(phase_sink, "fold_seal_read"); + const std::optional adopted_seal = readFoldSeal(state.snap_generation, state.snap_attempt); + if (!adopted_seal && state.snap_generation > 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC: the adopted fold seal (generation {}, attempt {}) is missing under a live " + "gc/state — GC bookkeeping is corrupt. GC refuses to run; recover with " + "SYSTEM CAS GC REBUILD.", + state.snap_generation, state.snap_attempt); + /// Every fold decision below starts from the immutable plan, never from the raw scan that supplied + /// it. The successor seal owns its mutable copy; this separate const snapshot remains the prior + /// coverage/hold view while the successor earns changes later in this fold. + const std::map parent_ref_lives = walk_plan.parentFoldStates(); + const uint64_t dropped_parent_ref_lives = walk_plan.droppedParentRows(); + result.fold_seal.ref_lives = walk_plan.successorFoldStates(); + + /// Retired-in-snapshot: the prior generation's condemned entries RIDE the source-edge run as + /// `kCondemned` sentinel rows, so the round no longer reads any separate retired-list object — + /// the parent seal's `blob_target_runs` ARE the retired input. The per-gc-shard `condemned_summary` + /// the seal carries below is distilled from the `still_retired` rows each shard re-emits, making the + /// next round's `graduationDue` / pure-carry decisions zero-I/O. + const uint64_t condemn_round = state.round + 1; + result.retired_merge.resize(state.gc_shards); + + /// Condemn-time observation: ONE HEAD per new zero-transition captures the exact incarnation token + /// the eventual delete carries (absent => a prior landed delete => nothing to condemn). Emits the + /// Candidate trail (IndegZero / GcRetireObserve / BlobRetire) exactly where the decision is made. + const auto head_blob = [&](const BlobRef & ref) -> std::optional + { + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::IndegZero; + e.object_kind = CasEventObjectKind::Blob; + e.object_hash = blobIdOf(ref); + e.round = condemn_round; + e.gen = state.snap_generation + 1; + e.reason = "last folded owner edge dropped; in-degree reached 0"; + }); + const HeadResult observed = backend.head(layout.blobKey(ref)); + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::GcRetireObserve; + e.object_kind = CasEventObjectKind::Blob; + e.object_hash = blobIdOf(ref); + e.token = observed.exists ? observed.token.value : ""; + e.round = condemn_round; + e.gen = state.snap_generation + 1; + e.outcome = observed.exists ? "present" : "absent"; + e.reason = "zero-in-degree candidate; HEAD-observe the current token"; + }); + if (!observed.exists) + return std::nullopt; + ++report.candidates; + ++report.condemned; + ProfileEvents::increment(ProfileEvents::CASGCRetiredCondemned); + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::BlobRetire; + e.object_kind = CasEventObjectKind::Blob; + e.object_hash = blobIdOf(ref); + e.token = observed.token.value; + e.round = condemn_round; + e.gen = state.snap_generation + 1; + e.outcome = "retired"; + e.reason = "condemned zero-in-degree candidate; entering the current retired list"; + }); + HeadResult adjusted = observed; + adjusted.size = retiredLogicalSize(ObjectKind::Blob, observed.size, store->poolMeta().blob_header_len); + /// This candidate unconditionally becomes a fresh `RetiredEntry` in `closeBlob` (the ONLY + /// caller of `head_blob`) whenever this lambda returns a value — so this is exactly the round's + /// side-effecting condemn site. Write the meta Condemned so the writer's point-read gate + /// sees it; a successful write records the in-process (hash, token) confirmation the graduation + /// gate consumes (`scheduleCondemnMarkerWrite` captures everything BY VALUE — never by reference + /// to `cur_blob`, which the fold's tight streaming loop mutates while the job is queued). + scheduleCondemnMarkerWrite(ref, observed.token, condemn_round, adjusted.size); + return adjusted; + }; + + /// Side-effect-free peek: the fold's resurrect-supersede branch (inside + /// `foldDeltasIntoGeneration`) needs the CURRENT token to detect that a resurrect replaced a stale + /// retired entry, but must NOT emit the fresh-condemn trail or bump `CASGCRetiredCondemned` — that + /// hook is `head_blob` above, reserved for a genuinely NEW zero-in-degree candidate. A supersede's + /// own event is `blob_retire_replaced`, emitted once below from `merge.replaced`. Plain HEAD, no + /// events, no counters. + const auto peek_head = [&](const BlobRef & ref) -> std::optional + { + HeadResult hr = backend.head(layout.blobKey(ref)); + if (!hr.exists) + return std::nullopt; + hr.size = retiredLogicalSize(ObjectKind::Blob, hr.size, store->poolMeta().blob_header_len); + return hr; + }; + + /// Graduation gate (triage 2026-07-17 §3.4): the merge consults this before publishing an entry + /// delete_pending. Confirmation sources, in order: the in-process (hash, token) record left by a + /// successful `writeCondemnedMeta` completion, then ONE synchronous `loadMeta` re-check — a durable + /// `Condemned` meta observed NOW is sufficient evidence, because a writer that same-token adopted + /// must have observed a non-Condemned meta EARLIER, its edge (EDGE-BEFORE-OBSERVE) landed before the + /// meta turned Condemned, and the redelete only fires from a LATER fold whose cut postdates this + /// round — that fold sees the edge and spares. (`BlobMeta` carries no token, so the re-check is + /// per-hash by design; the two-phase pipeline + the exact-token delete carry the rest.) No durable + /// evidence => count the carry, RETRY the marker write (liveness: a swallowed write would otherwise + /// carry forever), and refuse — never throw (an unreadable meta is missing evidence, not a wedge). + const auto confirm_condemned_marker = [&](const RetiredEntry & entry) -> bool + { + if (condemnMarkerConfirmedInProcess(entry.ref, entry.token)) + return true; + try + { + if (const auto lm = loadMeta(backend, layout, entry.ref); lm && lm->meta.state == MetaState::Condemned) + { + noteCondemnMarkerDurable(entry.ref, entry.token); /// memoize for a round-CAS-abort replay + return true; + } + } + catch (...) + { + ProfileEvents::increment(ProfileEvents::CASGCMetaWriteAnomaly); + tryLogCurrentException(logger, + "CAS gc: condemn-marker re-check failed to read the meta (treated as missing evidence; " + "the entry is carried, never wedges the round)"); + } + ProfileEvents::increment(ProfileEvents::CASGCCondemnMarkerUnconfirmedCarry); + /// Accepted race: this retry writes `Condemned` per-hash, with no token check. If a writer + /// resurrected this exact hash under a FRESH token between the original swallowed write and this + /// retry, the retry stamps `Condemned` over that writer's live, uncondemned incarnation. This is + /// never destructive -- the eventual exact-token delete is a no-op against the fresh token + /// (`DeleteOutcome::TokenMismatch`/`NotFound`) -- worst case the resurrecting writer's later + /// same-token adopter sees a stale `Condemned` meta and re-uploads once (a spurious resurrect). + scheduleCondemnMarkerWrite(entry.ref, entry.token, entry.condemn_round, entry.size); + return false; + }; + + const uint64_t new_generation = state.snap_generation + 1; + /// The fold mints THIS round's attempt id from `lease.seq` (the renew/steal paths bump it every + /// round, so it is a fresh monotonic per-round id). EVERY fold-artifact WRITE below lands under this + /// attempt; the PARENT-generation READS keep using `state.snap_attempt` (the attempt the prior round + /// adopted). The fold-adopt CAS #1 then commits `(new_generation, attempt)` together — a deposed + /// leader's fold lands under its own unadopted attempt and is invisible to every reader. + const uint64_t attempt = state.lease.seq; + result.fold_seal.generation = new_generation; + result.fold_seal.parent_generation = state.snap_generation; + + std::vector deltas; + /// PROBE B2's round-local ledger (see `TxnApplyLedger`). Grows one entry per ref log the intake + /// opens; the reducers mark `applied` through `&ledger.applied` at the point they consume a delta. + TxnApplyLedger ledger; + bool folded_any = false; + + /// PROBE B1 -- intake-layer identity. `logs_applied` counts, at the SINGLE cursor-advance site + /// below, every log whose whole body folded. At seal time it is compared against a recomputation + /// from the sealed coverage and the listing. The two are derived differently (a running counter vs + /// a recomputation), so a control-flow bug that advances a cursor without folding breaks the + /// equality. + /// + /// REACH, stated so this is not over-trusted: the recomputation reads the SAME listing the intake + /// read, so B1 is BLIND to a record missing from that listing. It is a control-flow assertion, not + /// a detector for the skipped-transaction defect -- probe A covers the listing, probe B2 covers + /// everything below the intake. + uint64_t logs_applied = 0; + + /// The adopted parent seal, read once at the ADOPTED (snap_generation, snap_attempt): it carries the + /// parent generation's `blob_target_runs` (resolved below into per-gc-shard prior runs) and the parent + /// `condemned_summary` (the pure-carry decision). A completed round leaves its fold seal there; a fresh + /// pool has none (empty seal). Under the snapshot+log ref model there is no per-shard token-diff Skip: + /// the "did this table change" signal is simply whether the global LIST returned any log id above the + /// table's durable cursor, which the per-table loop below tests directly. + /// SECOND read of the key `adopted_seal` already holds: same generation, same attempt, same bytes + /// (nothing between the two touches `state.snap_generation` / `snap_attempt` or writes that key). + /// One redundant GET per folding round, and the round's FIFTH GET of this one key overall -- + /// `graduationDue` and `listRefPrefix` make two in `defer_decision`, `parent_seal_read` a third, + /// `adopted_seal` above a fourth. Recorded on this row, not fixed here. + CasFoldSeal discover_ref_seal; + if (const auto fold_seal = readFoldSeal(state.snap_generation, state.snap_attempt)) + discover_ref_seal = *fold_seal; + seal_read_timer->metric("seal_reads", 2); + seal_read_timer->metric("redundant_reads", 1); + seal_read_timer->metric("parent_ref_lives", parent_ref_lives.size()); + seal_read_timer->metric("dropped_parent_ref_lives", dropped_parent_ref_lives); + seal_read_timer->metric("parent_runs", discover_ref_seal.blob_target_runs.size()); + seal_read_timer->metric("parent_cleanup_evidence", std::count_if( + discover_ref_seal.ref_lives.begin(), discover_ref_seal.ref_lives.end(), + [](const auto & row) { return row.second.cleanup_evidence.has_value(); })); + seal_read_timer.reset(); /// emits the `fold_seal_read` row + + /// Each fully-folded `remove_namespace` transaction earns terminal cleanup evidence. + std::vector> new_removals; + + /// 2-3. Ref-log intake, by ARITHMETIC (spec §5). For each table the walk steps + /// `expected = cursor + 1` WITHIN the cursor's epoch and reads that exact key: under INV-1 the ids of + /// one `(namespace, writer_epoch)` are dense `1..T`, so the next record's id is computable and the + /// round never asks the listing what to read. Each body is decoded+validated and every explicit + /// owner-change folds into `foldManifestEdges` (which reads the manifest body and appends per-blob + /// `BlobDelta`s to `deltas`). The durable cursor advances per FULLY folded log; a missing manifest body + /// clamps this table below the log (barrier, re-read next round) while other tables keep folding. + /// + /// THE LISTING IS A HINT, and demoting it is the point of this loop. It used to be the source of + /// truth for which records exist, so a store that omitted a durable key from an enumeration -- the + /// observed `0x1430c`/`0x1430d` shape -- made the round skip those records' owner edges and then seal + /// a cursor ABOVE them, which is unrecoverable (a record below the cursor is never re-read). Under + /// arithmetic intake such an omission is a NON-EVENT: the exact GET finds the record anyway. The hint + /// keeps exactly two jobs here: + /// (a) the genesis start of a never-folded namespace (`cursor == {0,0}` has no arithmetic + /// predecessor; Stage B's `_ckpt.life_epoch` is what finally supplies it -- see below), and + /// (b) the WITNESS set that makes an absent expected-next decidable: + /// absent, no listed id above it => this namespace's frontier this round (normal end); + /// absent, a listed id above it => impossible under contiguity, so the store is lying or a + /// durable record was lost: HOLD the namespace at + /// classification 4 with its cursor unmoved. + /// + /// Epochs are crossed only over a consumed `EpochSeal` (INV-2): the seal folds as an applied table + /// no-op (probe B2 `produced=false`) and the next epoch's start is `{E', 1}`, reached through the + /// `prev_epoch_seal` back-chain rather than guessed -- so an epoch the hint omits entirely is still + /// walked, and a crossing with no consumed seal behind it is an impossible shape that holds. Read + /// `crossFromSeal` for what that is proved FROM: within a round the seal's kind is checked outright, + /// across rounds it rests on the chain until Task 8 carries the kind in the durable cursor. + /// + /// PER-NAMESPACE FAILURES ARE PER-NAMESPACE (spec §5): an unreadable or undecodable body belongs to + /// exactly one namespace and clamps only it. The whole-round abort survives ONLY for a key that cannot + /// be attributed to any namespace at all (`groupRefKeys` above), which is why nothing in this loop + /// sets `ref_folding_aborted` anymore. + /// + /// PHASE 8/18 `fold_ref_intake`: one GET per record (always owed -- the round read every body anyway), + /// plus one exact 404 probe per namespace to prove its frontier, one extra GET per epoch crossed, and + /// one GET per manifest edge, which on a busy pool is where the round's object-read budget goes. + /// It also carries probe B1's two numbers -- reported on EVERY healthy round, so + /// "logs_accounted always equals logs_applied" becomes an observable property of the table rather than + /// a claim in a comment. + std::optional intake_timer; + intake_timer.emplace(phase_sink, "fold_ref_intake"); + uint64_t intake_tables_changed = 0; + uint64_t intake_tables_clamped = 0; + uint64_t intake_tables_held = 0; + uint64_t intake_dead_precommits_skipped = 0; + uint64_t intake_absent_probes = 0; + uint64_t intake_epoch_crossings = 0; + + /// Probe B1's raw material: per namespace, the CONTIGUOUS runs of ids this round walked, one per epoch + /// entered, recorded as `[first, last]`. The recomputation below turns them back into a count and + /// compares it with the counter the advance site incremented -- so a cursor that moved over a position + /// nothing applied inflates the first number and fails the round closed. + std::map>> walked_segments; + + /// The round's SECOND witness source, independent of the listing -- see `readCheckpointWitnesses` + /// for what it decides and why a listing alone cannot decide it. Its `undecodable` half names the + /// namespaces whose `_ckpt` is present and unreadable; each of those is HELD below, and only those. + const CheckpointWitnesses checkpoints = readCheckpointWitnesses(ref_tables, catalog_snapshot); + const std::map & checkpoint_witness = checkpoints.witnesses; + + /// WHICH NAMESPACES THIS ROUND WALKS -- i.e. THE ROUND'S UNIVERSE, the set the destructive gate owes + /// a frontier proof for (spec §5; `UniversePolicy` for why only the catalog can bound it): EVERY + /// `Live`/`Removing` row of this round's frozen catalog cut, and nothing else. + /// + /// The listing is a HINT and cannot change that set in either direction. It cannot SHRINK it -- a + /// store that goes quiet about a namespace, or stops listing one to clear its hold, leaves the + /// catalog row and therefore the obligation untouched -- and it cannot GROW it either, since a + /// physical id the catalog does not name is inert debris rather than a namespace to prove. + /// + /// A namespace with no hint entry walks against an EMPTY listing: it has no hint witnesses, but it + /// still reads its expected-next by exact key -- ONE `GET`, whose absence IS the frontier proof and + /// whose PRESENCE means the namespace was wrongly quiet and gets walked properly this round. A + /// carried hold additionally supplies the witness that keeps an absent below its position from + /// reading as a frontier. + /// + /// COST: held namespaces are always walked (their retry is a liveness obligation, not a budgeted + /// nicety); the merely-QUIET ones are bounded by `gc_frontier_probe_budget`, and the ones the budget + /// does not reach are simply unproven, which suppresses the round's destruction. In a healthy pool + /// that budget is rarely touched; dead physical lives are excluded by the catalog-built plan. + + /// THE ROUND'S WORK-SET IS FROZEN AT ROUND START, AND THAT IS WHAT MAKES A ROUND END. + /// + /// Arithmetic intake reads the next id by exact key, so on its own it walks WHILE RECORDS EXIST -- + /// and a namespace whose writer appends concurrently therefore has no last record to reach. Round + /// time stopped being `backlog / walker_rate` and became `backlog / (walker_rate - writer_rate)`, + /// which diverges the moment a writer keeps up with the walker. Measured: zero completed rounds in + /// 42 minutes on a hot pool. Everything the round paces on rounds then stops too -- the fold seal and + /// its cursors, the sampled store-quality detector, ref-object cleanup -- so the backlog the round + /// was falling behind on grows without bound. + /// + /// The bound is `_ckpt.committed_through`, snapshotted once per namespace before the walk (see + /// `readCheckpointWitnesses`) and never re-read within the round: the ceiling test at the top of the + /// walk refuses every position above it, so the work is fixed at round start whatever the writer + /// does meanwhile. It is also the AUTHORITY ceiling -- a record above it is durable but is not + /// logical history yet -- so ONE comparison serves both purposes and there is no second bound that + /// could drift out of agreement with it. NO NEW PERSISTED STATE: the number is read from an object + /// the fold reads anyway. + /// + /// THE BOUND IS ON FOLDING, NOT ON READING, and that distinction is the whole design. + /// + /// The walk still reads its expected-next exactly as before -- one exact `GET` at `cursor + 1` -- + /// because that read IS the frontier proof, and a namespace that stops being read stops being + /// provable. An unprovable namespace leaves `frontier_complete` false forever, which suppresses every + /// destructive decision, which stops ref-object cleanup, which means its listing never drains and it + /// never becomes provable by any other route either. So "skip reading a quiet namespace" is not a + /// cheaper version of this design; it is a GC that permanently reclaims nothing. The saving it + /// appears to offer is exactly one `GET` per namespace per round, and that `GET` is the proof. + /// + /// The listed tail bounds nothing, but it is still CLASSIFIED for the phase row, in three classes: + /// * `tail > cursor` -- the round has records to fold; + /// * `tail == cursor` -- the listing shows nothing new, so the walk folds nothing and its single + /// read is the frontier probe. On a wide pool this is most namespaces, most rounds; + /// * `tail < cursor` -- the listing's greatest id is BELOW a cursor we folded through. Cleanup + /// deletes logs from the bottom up, so a listed log below the cursor with none at or above it is + /// a stale or lying listing, not a shape cleanup produces. The cursor is the truth (the sampled + /// store-quality detector is this class's observer). + struct WalkTarget + { + String ns; + UInt128 life_id{}; + const RefTableListing * listing; + }; + + static const RefTableListing kNoListing; + std::vector walk_targets; + /// The three buckets classify the hinted, UNHELD namespaces -- the ones whose tail comparison + /// decided how much they could fold. A held namespace folds up to a bound its HOLD also determines + /// and is reported by `tables_held`, so counting it here would make `tails_unchanged` stop meaning + /// "namespaces with no work this round", which is the number the row exists to publish. + uint64_t intake_tails_advanced = 0; + uint64_t intake_tails_unchanged = 0; + uint64_t intake_tails_below_cursor = 0; + uint64_t intake_unhinted_held = 0; + uint64_t intake_unhinted_quiet = 0; + uint64_t frontier_probe_budget = store->poolConfig().gc_frontier_probe_budget; + uint64_t intake_unprobed_budget = 0; + uint64_t intake_catalog_only = 0; + for (const auto & [catalog_ns_str, catalog_incarnation] : live_incarnation) + { + const auto listing_it = ref_tables.find(catalog_ns_str); + const RefTableListing * listing = listing_it != ref_tables.end() ? &listing_it->second : &kNoListing; + const auto parent_it = parent_ref_lives.find(catalog_incarnation); + const RefCoverage * parent_cov = parent_it != parent_ref_lives.end() ? &parent_it->second.coverage : nullptr; + + if (listing_it == ref_tables.end() && parent_cov) + { + if (parent_cov->hold) + ++intake_unhinted_held; + else if (checkpoints.recovery_checkpoints.contains(catalog_ns_str)) + ++intake_unhinted_quiet; + else if (frontier_probe_budget == 0) + { + ++intake_unprobed_budget; + continue; + } + else + { + --frontier_probe_budget; + ++intake_unhinted_quiet; + } + } + else if (listing_it == ref_tables.end()) + ++intake_catalog_only; + + walk_targets.push_back({catalog_ns_str, catalog_incarnation, listing}); + if (listing->logs.empty() || (parent_cov && parent_cov->hold)) + continue; + + const RefTxnId tail = listing->logs.back(); + const RefTxnId cursor = parent_cov ? parent_cov->last_folded_ref_id : RefTxnId{}; + if (cursor < tail) + ++intake_tails_advanced; + else if (cursor == tail) + ++intake_tails_unchanged; + else + ++intake_tails_below_cursor; + } + + for (const WalkTarget & target : walk_targets) + { + const String & ns_str = target.ns; + const RefTableListing & listing = *target.listing; + if (ref_folding_aborted) + break; + const RootNamespace ns{ns_str}; + /// Every walk target came out of this round's own catalog read, so its incarnation is the REAL + /// one and the life below is minted from a catalog entry rather than guessed from a key. + const NamespaceLifeId life = NamespaceLifeId::fromCatalogEntry(ns, target.life_id); + + /// Parent cursor = the durable last_folded_ref_id this table folded to (absent => {0,0}). + /// + /// Id ordering does NOT subsume remove+recreate: under INV-1 (`nextRefTxnId`) ids are derived per + /// namespace from that table's own state, so a namespace removed and recreated within one writer + /// epoch restarts at `{E, 1}`, at or below a cursor sealed for the PREVIOUS life -- and the walk + /// starts at `cursor + 1`, so those edges would never fold and the recreated refs' manifests + /// would look unreferenced. That is closed STRUCTURALLY, not by comparing ids: this map and the + /// walk targets are both keyed by CATALOG INCARNATION, so a rebirth is a different key and + /// inherits no cursor, and a life the catalog no longer names contributes no walk target and + /// therefore re-carries no cursor into the new seal. Do not "fix" it by comparing ids. + const auto cursor_it = parent_ref_lives.find(target.life_id); + const RefTxnId cursor = cursor_it != parent_ref_lives.end() + ? cursor_it->second.coverage.last_folded_ref_id : RefTxnId{}; + + /// Baseline guard: a table with NO sealed cursor whose logs at/below its + /// newest snapshot have all been cleaned means a prior fold advanced+cleaned them and then gc/state + /// was lost -- folding from {0,0} would miss those edges and mass-condemn their blobs. Fail closed; + /// recover with the explicit rebuild. A fresh table (writer already snapshotted, logs still present) + /// passes because its logs at or below the snapshot survive. + if (cursor_it == parent_ref_lives.end() && !listing.snapshots.empty() + && !checkpoints.recovery_checkpoints.contains(ns_str)) + { + const RefTxnId newest_snapshot = listing.snapshots.back(); + const bool logs_below_snapshot_gone = listing.logs.empty() || newest_snapshot < listing.logs.front(); + if (logs_below_snapshot_gone) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC baseline guard: table {} has snapshot {} but no surviving log at or below it and " + "no sealed fold cursor -- gc/state was lost after cleaning covered logs. GC refuses to " + "run; recover with SYSTEM CAS GC REBUILD.", + ns_str, renderRefTxnId(newest_snapshot)); + } + + /// The hold the PARENT seal left on this namespace, if any. It is three things at once: the + /// position this round must retry by exact key, a durable witness (see `witnessAbove`), and the + /// hold that rides forward unless this round resolves that position. + const std::optional carried_hold = + cursor_it != parent_ref_lives.end() ? cursor_it->second.coverage.hold : std::nullopt; + + RefCoverage cov; + cov.classification = 0; + bool table_changed = false; + /// THE FRONTIER PROOF for this namespace, and there is exactly one thing that establishes it: + /// the walk read the expected-next position by exact key, found it ABSENT, and no witness put + /// anything above it. That is the honest end of the record stream. Every other way out of the + /// loop leaves the namespace unproven -- a hold (this round's or a carried one), and the walk + /// that never started because a never-folded namespace's hint offered no genesis position AND + /// no `_ckpt.life_epoch` was on record either (the `expected` initialization below reads + /// `checkpoints.life_epochs` for exactly this case; a namespace with neither has a genuinely + /// unknown genesis, so no probe is taken, nothing is proved, and fail-closed says unproven). + bool frontier_proven = false; + /// Which exit left this namespace unproven, for the round's deficit tally. It starts at + /// `Unattributed` so that an exit which forgets to name itself is reported as such instead of + /// being silently absorbed into some other bucket. + FoldResult::FrontierUnproven unproven_reason = FoldResult::FrontierUnproven::Unattributed; + + /// Use the same frozen catalog row + decoded checkpoint authority as read-only recovery. The + /// ref LIST is only a bounded-work hint; it may neither choose this life's genesis nor extend + /// its committed frontier. + const auto catalog_entry_it = std::lower_bound( + catalog_snapshot.catalog.entries.begin(), catalog_snapshot.catalog.entries.end(), ns, + [](const CatalogEntry & entry, const RootNamespace & needle) { return entry.ns < needle; }); + chassert(catalog_entry_it != catalog_snapshot.catalog.entries.end()); + chassert(catalog_entry_it->ns == ns); + chassert(catalog_entry_it->incarnation == target.life_id); + std::optional checkpoint; + if (const auto checkpoint_it = checkpoints.recovery_checkpoints.find(ns_str); + checkpoint_it != checkpoints.recovery_checkpoints.end()) + checkpoint = checkpoint_it->second; + + std::optional grounding; + String checkpoint_failure; + if (const auto bad_checkpoint = checkpoints.undecodable.find(ns_str); + bad_checkpoint != checkpoints.undecodable.end()) + { + checkpoint_failure = bad_checkpoint->second; + } + else + { + try + { + grounding = chooseRecoveryGrounding(std::optional{*catalog_entry_it}, checkpoint); + } + catch (const Exception & e) + { + checkpoint_failure = e.message(); + } + } + + /// The hold THIS round detected, at the position it stopped. It IS the clamp signal: there is + /// no separate boolean that could disagree with it about whether the namespace stopped. + std::optional fired; + RefTxnId resolved_through = cursor; /// advances per fully-folded log; a clamp keeps it below the log + + /// The witness role: the smallest id strictly above `id` that SOMETHING says exists. An absent + /// expected-next means "frontier" without a witness and "impossible shape" with one, so the + /// witness set decides whether the walk stops quietly or holds. THREE independent sources, none + /// of them authoritative alone: + /// * the hint, which may omit durable records (that is the whole reason intake is arithmetic); + /// * `_ckpt.checkpoint`, the namespace's own durable tail -- a listing is a SNAPSHOT, so a + /// record durable after the enumeration is invisible to that round's probes, and this one + /// is not (phase-0 model `_fix_ckptwitness`); + /// * the CARRIED HOLD's offending position, which is durable proof that a previous round + /// reached that position -- with ONE exception, `CheckpointUndecodable`, the only hold + /// minted before the walk reads anything: its position is the walk's OWN next position, so + /// the strict comparison below never lets it witness against a position this walk goes on + /// to read. It weakens nothing, because a hold that proves nothing also claims nothing. + /// Under contiguity everything at or below a REACHED position must exist, so an absent + /// below it is a gap. This is what makes "retry the exact offending position" work + /// for a hold that sits above an epoch boundary: the crossing needs a witness to aim at, + /// and without this the walk would stop one position short and never re-read it. + /// The SMALLEST wins, because the nearest witness is the one that decides same-epoch gap versus + /// epoch crossing. + const auto witnessAbove = [&](const RefTxnId & id) -> std::optional + { + std::optional nearest; + const auto consider = [&](const RefTxnId & w) + { + if (id < w && (!nearest || w < *nearest)) + nearest = w; + }; + const auto it = std::upper_bound(listing.logs.begin(), listing.logs.end(), id); + if (it != listing.logs.end()) + consider(*it); + if (const auto ck = checkpoint_witness.find(ns_str); ck != checkpoint_witness.end()) + consider(ck->second); + if (grounding && grounding->committed_through) + consider(*grounding->committed_through); + if (carried_hold) + consider(carried_hold->offending_position); + return nearest; + }; + + /// Record the position an impossible shape was detected at, hold the namespace, and stop walking + /// it. The reason is a BOUNDED enum because it is persisted in the seal: an operator reading a + /// held row learns what stopped the namespace and exactly where, without correlating logs. + const auto hold = [&](const RefTxnId & at, HoldReason reason, const char * message) + { + report.recordAnomaly(ns, 0, ManifestId{ns, {}}, message); + EventEmitter{*store}.emit([&](CasEvent & ev) + { + ev.type = CasEventType::GcFoldClamp; + ev.namespace_ = ns_str; + ev.object_kind = CasEventObjectKind::Root; + ev.outcome = "held"; + ev.reason = message; + ev.detail = {{"expected", renderRefTxnId(at)}, + {"resolved_through", + resolved_through == RefTxnId{} ? "none" : renderRefTxnId(resolved_through)}}; + }); + LOG_ERROR(logger, + "CAS GC ref intake: namespace {} HELD at {} -- {}. The cursor stays at {} and this " + "namespace folds nothing further this round.", + ns_str, renderRefTxnId(at), message, + resolved_through == RefTxnId{} ? "none" : renderRefTxnId(resolved_through)); + fired = RefHold{.reason = reason, .offending_position = at, + .retry_count = 0, .next_retry_round = 0}; /// retry fields filled in at the seal below + }; + + /// Cross into the epoch that follows the one `from_seal` closed. The rule itself is + /// `crossEpochFromSeal` (Pool/CasRefProtocol.cpp) -- read its doc comment for what the back-chain + /// proves and why the hint may only NOMINATE the target epoch. It lives there rather than here + /// because fsck's audit walks the same streams: if the round and the audit could disagree about + /// when an epoch boundary is proved, they would disagree about which records a cut contains. + /// + /// This wrapper adds only the round's own accounting -- the reads the crossing performed land on + /// the row that spent them, and an undecodable epoch-start record is logged with the key. + const auto crossFromSeal = [&](const RefTxnId & from_seal, const std::optional & seal_proven, + const RefTxnId & witness) -> std::optional + { + const EpochCrossResult crossing = + crossEpochFromSeal(backend, layout, ns, from_seal, seal_proven, witness, life); + intake_absent_probes += crossing.absent_probes; /// a failed crossing pays its reads too + ProfileEvents::increment(ProfileEvents::CASRefLogBodyGets, crossing.body_gets); + if (crossing.outcome == EpochCrossOutcome::StartInvalid) + LOG_WARNING(logger, "CAS GC ref intake: epoch-start log {} invalid: {}", + layout.refLogKey(life, crossing.probed), crossing.detail); + return crossing.proved() ? std::optional(crossing.start) : std::nullopt; + }; + + /// The first position is arithmetic from the sealed cursor, or from the exact checkpoint's life + /// epoch on the first fold. A listing never chooses a logical history start. An unusable + /// checkpoint still has a canonical retry position once a cursor exists, so its durable hold + /// remains observable rather than degrading into an anonymous anomaly. + std::optional expected; + if (cursor != RefTxnId{}) + expected = RefTxnId{cursor.writer_epoch, cursor.ref_sequence + 1}; + else if (grounding) + { + expected = RefTxnId{*checkpoint->life_epoch, 1}; + } + + /// An unavailable or invalid `_ckpt` quarantines its own namespace and nothing else (spec §5). The object + /// belongs to exactly one namespace and both of its consumers are keyed by that namespace -- + /// `witnessAbove` above, and `cleanupRefObjects`' delete boundaries via `result.checkpoints` -- + /// so the damage is confinable, and confining it is the difference between one namespace waiting + /// for a repair and the whole pool's cursors, seals and cleanup stopping on one 4 KiB object. + /// + /// FOLD NOTHING FOR IT, rather than fold and merely refuse the frontier proof. The cursor + /// advance is the one irreversible thing the walk does (nothing ever re-reads below it), and + /// this namespace has just proved that a piece of its own durable state cannot be read. Spending + /// that irreversible step against a namespace in that condition buys reclamation latency and + /// costs recovery options; the hold costs only the latency. + /// + /// The hold sits at the position the walk WOULD have read, which is canonical by construction. + /// A namespace with no such position is a never-folded one whose listing shows no log -- the + /// `_ckpt`-only "phantom table" `parseRefCkptKey` deliberately admits is exactly this shape -- + /// and there is no walk to stop; the anomaly alone carries it, because the other consumer is + /// `cleanupRefObjects`, whose boundaries an ABSENT checkpoint WIDENS, and only the round's + /// destructive gate keeps the snapshot this unreadable object names from being deleted. + if (!grounding) + { + LOG_WARNING(logger, + "CAS GC ref intake: namespace {} has no usable checkpoint -- {}. This namespace " + "folds nothing and reclaims nothing until that object is repaired; every other " + "namespace folds normally.", + ns_str, checkpoint_failure); + if (expected) + hold(*expected, HoldReason::CheckpointUndecodable, + "ref intake: the namespace's `_ckpt` is unavailable or invalid -- its durable " + "checkpoint authority cannot be read, so nothing above the cursor is accountable"); + else + report.recordAnomaly(ns, 0, ManifestId{ns, {}}, + "ref intake: the catalog life has no usable `_ckpt` (no authoritative walk position)"); + unproven_reason = FoldResult::FrontierUnproven::CheckpointUnusable; + expected.reset(); /// the cursor rides verbatim into this round's seal + } + + if (grounding && !grounding->committed_through) + { + if (cursor == RefTxnId{}) + frontier_proven = true; + else + { + report.recordAnomaly(ns, 0, ManifestId{ns, {}}, + "ref intake: an empty checkpoint frontier cannot explain a nonzero sealed cursor"); + unproven_reason = FoldResult::FrontierUnproven::CheckpointFrontierEmpty; + } + expected.reset(); + } + + /// Whether the record this round last applied (the one `resolved_through` names) was an + /// `EpochSeal`. `nullopt` = this round has applied nothing yet, so `resolved_through` is still + /// the inherited cursor and its kind is not knowable here -- see `crossFromSeal`. + std::optional last_applied_is_seal; + + /// Probe B1's per-epoch contiguous run: opened at the first position walked in an epoch, closed + /// when the walk leaves that epoch or stops. + std::optional segment_first; + std::vector> segments; + const auto closeSegment = [&]() + { + if (segment_first) + segments.emplace_back(*segment_first, resolved_through); + segment_first.reset(); + }; + + while (expected) + { + /// `_ckpt.committed_through` is the inclusive authority ceiling. Stop before reading a + /// durable but uncommitted `F+1`, so neither it nor a later 404 can advance this cursor or + /// establish the destructive frontier. + if (*grounding->committed_through < *expected) + { + if (resolved_through == *grounding->committed_through) + frontier_proven = true; + else + { + report.recordAnomaly(ns, 0, ManifestId{ns, {}}, + "ref intake: checkpoint committed_through precedes the sealed cursor"); + unproven_reason = FoldResult::FrontierUnproven::CommittedBelowCursor; + } + break; + } + + /// GET + decode the expected record. Absence is the decision point of the whole walk, and + /// an invalid body is a per-namespace hold: the key belongs to exactly one namespace, so it + /// can never be grounds for discarding another namespace's fold. + const auto got = backend.get(layout.refLogKey(life, *expected)); + if (!got) + { + ++intake_absent_probes; + const auto witness = witnessAbove(*expected); + if (!witness) + { + /// The ceiling test above already refused everything past `committed_through`, so + /// this absent position is committed and its record owes us an answer. + hold(*expected, HoldReason::GapBelowWitness, + "ref intake: a checkpoint-committed ref log is absent -- its authoritative " + "frontier cannot be complete"); + break; + } + + if (witness->writer_epoch == expected->writer_epoch) + { + hold(*expected, HoldReason::GapBelowWitness, + "ref intake: expected next id absent below a same-epoch witness -- " + "contiguity says this cannot happen, so a durable record is missing"); + break; + } + + const auto crossed = crossFromSeal(resolved_through, last_applied_is_seal, *witness); + if (!crossed) + { + hold(*expected, HoldReason::UnconsumedSealCrossing, + "ref intake: a later epoch's records are reachable but this epoch's " + "closing seal was never consumed (or the position they chain from is " + "not one) -- the crossing has no proof"); + break; + } + /// Every iteration must move `expected` strictly forward. A crossing that lands back on + /// the position just read as absent makes no progress and would spin: the epoch-start + /// record answered a GET inside `crossFromSeal` and then not here, so it is vanishing + /// under us. Treat it as the impossible shape it is. + if (!(*expected < *crossed)) + { + /// The record at `*crossed` answered the crossing's GET and then stopped answering + /// the walk's: an ABOVE-CURSOR object that vanished under us. Nothing may + /// legitimately remove one, so this is corruption, and it is the one hold shape no + /// amount of waiting clears -- which is exactly why it must be named durably. A + /// later round that read the namespace as merely quiet would otherwise grant it a + /// frontier proof and license destruction against a cut that is missing records. + hold(*expected, HoldReason::WitnessDisappeared, + "ref intake: the epoch crossing resolved back to the position that " + "just read absent -- the epoch-start record is not stably readable"); + break; + } + ++intake_epoch_crossings; + closeSegment(); + expected = *crossed; + continue; + } + const RefTxnId log_id = *expected; + + /// Probe B2: open this transaction's ledger entry. A clamped log is opened but never + /// committed, so it is correctly not reported unapplied. + const uint32_t txn_ordinal = ledger.open(ns, log_id); + + ProfileEvents::increment(ProfileEvents::CASRefLogBodyGets); /// one body GET per new log + RefLogTxn txn; + std::vector edges; + try + { + txn = decodeRefLogTxn(openObject(FormatId::RefLog, got->bytes), ns_str, log_id); + /// Extraction shares the decode try-block: an unrecognized owner_transition shape + /// (`manifestEdgesOfTxn` -> `classifyOwnerTransitionShape`, Pool/CasRefProtocol.cpp) is + /// exactly as untrustworthy as an undecodable body -- both mean this log cannot be + /// folded -- so both get the SAME per-namespace hold below. Do + /// NOT widen this catch over `foldManifestEdges` -- only intake, not the fold itself, + /// shares this discipline. + edges = manifestEdgesOfTxn(txn); + } + catch (const Exception & e) + { + LOG_WARNING(logger, "CAS GC ref intake: log {} invalid: {}", + layout.refLogKey(life, log_id), e.message()); + hold(log_id, HoldReason::BodyUndecodable, "ref log body invalid: namespace held below it"); + break; + } + + /// Fold every explicit manifest edge of the log. A transaction applies + /// ATOMICALLY ("either the complete transaction applies or none of it applies"): stage this + /// log's blob deltas and owner-removed manifest cleanup in PER-LOG buffers and merge them into + /// the round buffers only once the WHOLE log folds. A mid-log clamp DISCARDS the staged buffers + /// so the cursor stays coherent -- merging a partially folded log's `-1` cleanup would let the + /// post-CAS body delete remove a body whose edge is still unfolded behind the clamp, and the + /// re-fold would then clamp on that missing body forever. A missing manifest body is a per-table + /// CLAMP (barrier), never a round abort: keep the cursor below THIS log and re-read it next + /// round. A removed precommit whose body never existed emitted no edge -- skip, no clamp. + std::vector log_deltas; + std::map log_mf_cleanup; + for (const RefManifestEdge & edge : edges) + { + ProfileEvents::increment(ProfileEvents::CASRefEmittedEdges); /// one manifest-edge event + if (foldManifestEdges(edge.manifest_id, edge.change, log_deltas, log_mf_cleanup, + txn_ordinal)) + continue; + + if (edge.change < 0 && edge.owner_kind == RefOwnerKind::Precommit) + continue; /// removed precommit that never activated: no edge to mirror, no clamp + + /// A `+1` precommit whose body is absent normally holds the fold barrier (the writer may + /// still be uploading it). But a precommit naming a build PROVABLY DEAD by the durable + /// watermark floor -- the SAME fact the orphan sweep uses to reclaim the body -- can never + /// activate, and its body will never return. Skip it (non-activating, advance the log) so + /// the barrier is not held forever on a body no writer will complete; otherwise this table + /// clamps every round with no terminal resolution. Fail-closed: no durable watermark (the + /// build cannot be proven dead) keeps the barrier. + if (edge.change > 0 && edge.owner_kind == RefOwnerKind::Precommit + && prefixEligible(*store, ns, + BuildPrefix{.writer_epoch = edge.manifest_id.ref.writer_epoch, + .build_sequence = edge.manifest_id.ref.build_sequence})) + { + ProfileEvents::increment(ProfileEvents::CASGCDeadPrecommitSkipped); + ++intake_dead_precommits_skipped; + EventEmitter{*store}.emit([&](CasEvent & ev) + { + ev.type = CasEventType::GcFoldClamp; /// reuse the fold-decision channel; outcome distinguishes + ev.namespace_ = ns_str; + ev.object_kind = CasEventObjectKind::Root; + ev.object_hash = manifestRefDebugString(edge.manifest_id.ref); + ev.outcome = "dead_precommit_skipped"; + ev.reason = "live precommit body absent AND its build is below the watermark floor " + "(provably dead); skip the non-activating edge instead of clamping forever"; + ev.detail = {{"log", renderRefTxnId(log_id)}}; + }); + continue; + } + + const char * reason = edge.change > 0 + ? (edge.owner_kind == RefOwnerKind::Precommit + ? "fold barrier: live precommit body not yet present (non-activating)" + : "committed/promoted ref names a missing manifest body") + : "owner-removal: edge-bearing committed body missing at removal-fold"; + report.recordAnomaly(ns, 0, edge.manifest_id, reason); + EventEmitter{*store}.emit([&](CasEvent & ev) + { + ev.type = CasEventType::GcFoldClamp; + ev.namespace_ = ns_str; + ev.object_kind = CasEventObjectKind::Root; + ev.object_hash = manifestRefDebugString(edge.manifest_id.ref); + ev.outcome = "clamped"; + ev.reason = reason; + ev.detail = {{"log", renderRefTxnId(log_id)}, + {"resolved_through", + resolved_through == RefTxnId{} ? "none" : renderRefTxnId(resolved_through)}}; + }); + /// The barrier is a HOLD like any other -- same durable shape, same clearing rule (fold + /// through `log_id`), and the same suppression consequence. Its reason is the one whose + /// ordinary cause is benign (a writer that appended its record before finishing the + /// manifest upload), and naming it durably is what lets an operator tell that apart + /// from a namespace that has been stuck for hours. The anomaly and event were already + /// emitted above with the manifest identity, which `hold` cannot carry, so this site + /// sets the hold directly instead of calling it. + fired = RefHold{.reason = HoldReason::ManifestBodyMissing, .offending_position = log_id, + .retry_count = 0, .next_retry_round = 0}; + break; /// stop folding this log's edges; the cursor stays at resolved_through (< log_id) + } + + if (fired) + break; /// discard the staged log_deltas / log_mf_cleanup (never merged) and stop this table + + /// The whole log folded: merge its staged transaction into the round buffers. + if (!log_deltas.empty()) + ledger.markProduced(txn_ordinal); + ledger.markCommitted(txn_ordinal); + for (BlobDelta & d : log_deltas) + deltas.push_back(std::move(d)); + for (const auto & [mid, tok] : log_mf_cleanup) + result.mf_cleanup.emplace(mid, tok); + + /// A fully-folded `remove_namespace` transaction hands its `{ns, remove_txn_id}` to the + /// life row's cleanup evidence; its owner-removal edges were folded above. + if (const auto removal = removalTxnId(txn)) + new_removals.emplace_back(ns, *removal); + if (!segment_first) + segment_first = log_id; /// this epoch's contiguous run opens at the first applied id + resolved_through = log_id; /// this log fully folded + /// The KIND of the record the cursor now sits on, remembered for the crossing below: the + /// chain check proves the identity of the position an epoch chains from, never that that + /// position is a seal, and this is the one place the answer is free (the body is decoded and + /// in hand). See the crossing's own comment for the half of this that cannot be re-checked. + last_applied_is_seal = refLogTxnIsEpochSeal(txn); + ++logs_applied; /// probe B1: the SINGLE cursor-advance site + table_changed = true; + + if (const std::optional next = nextRefLogIdWithinCommittedFrontier( + log_id, *last_applied_is_seal, *grounding->committed_through)) + { + if (*last_applied_is_seal && next->writer_epoch != log_id.writer_epoch) + { + const auto crossed = crossFromSeal(log_id, last_applied_is_seal, *grounding->committed_through); + if (!crossed) + { + hold(*next, HoldReason::UnconsumedSealCrossing, + "ref intake: checkpoint frontier cannot prove the successor of the epoch seal " + "just consumed"); + break; + } + ++intake_epoch_crossings; + closeSegment(); + expected = *crossed; + } + else + expected = *next; + } + else + { + frontier_proven = true; + expected.reset(); + } + } + closeSegment(); + if (!segments.empty()) + walked_segments[ns_str] = std::move(segments); + + cov.last_folded_ref_id = resolved_through; + + /// THE CLEARING RULE (spec §5), and the only place it is decided. A hold clears by exactly one + /// event: this walk RESOLVING the offending position -- folding through it and sealing a cursor + /// at or above it. It never clears by observing another absent, because an absent is precisely + /// what a lying store produces and precisely what made the hold necessary; and it never clears + /// by the hint going quiet, because a quiet hint is not evidence about anything. + /// + /// Three cases, total and in order: + /// * this round detected a hold -- adopt it. It sits at or above the cursor, so any carried + /// hold at a HIGHER position is simply not reached yet and will be re-detected once this + /// one clears; a carried hold at a LOWER position was folded through, which is its + /// clearance. + /// * no new hold, but the walk stopped BELOW a carried hold's position -- it did not resolve + /// it, so the hold rides VERBATIM (same reason, same position). + /// * otherwise -- either there was no hold, or the walk folded through it. Cleared. + std::optional effective; + if (fired) + effective = fired; + else if (carried_hold && resolved_through < carried_hold->offending_position) + { + effective = carried_hold; + /// The round produced no anomaly of its own for this namespace (the walk ended quietly), + /// yet the namespace IS held: record one so this round's destructive work is suppressed on + /// today's `anomalies`-based rule as well as on the hold set itself. Without it a hold + /// carried through a quiet round would suppress nothing, which is the hole the carry exists + /// to close. + report.recordAnomaly(ns, 0, ManifestId{ns, {}}, + "ref intake: namespace still held below an unresolved position"); + EventEmitter{*store}.emit([&](CasEvent & ev) + { + ev.type = CasEventType::GcFoldClamp; + ev.namespace_ = ns_str; + ev.object_kind = CasEventObjectKind::Root; + ev.outcome = "held"; + ev.reason = "carried hold: the offending position did not resolve this round"; + ev.detail = {{"expected", renderRefTxnId(carried_hold->offending_position)}, + {"resolved_through", + resolved_through == RefTxnId{} ? "none" : renderRefTxnId(resolved_through)}, + {"retry_count", std::to_string(carried_hold->retry_count)}}; + }); + } + + if (effective) + { + /// Retry bookkeeping. The count belongs to a POSITION, so it continues only while the hold + /// stays at the same one; a hold that moved is a different stop and starts over. The count + /// saturates rather than wrapping -- a wrapped counter would report a namespace stuck for + /// four billion rounds as freshly held. + const bool same_position = carried_hold + && carried_hold->offending_position == effective->offending_position; + effective->retry_count = same_position && carried_hold->retry_count < UINT32_MAX + ? carried_hold->retry_count + 1 + : (same_position ? UINT32_MAX : 0); + effective->next_retry_round = current_round + 1; + cov.hold = effective; + cov.classification = 4; + ++intake_tables_held; + /// A held namespace is unproven BY DEFINITION -- the hold names a position the walk could + /// not resolve, so everything at or above it is unaccounted. Stated here rather than left to + /// the loop's control flow: a carried hold rides forward on a round whose own walk ended + /// quietly, and that quiet end must not be mistaken for a proof. + frontier_proven = false; + unproven_reason = FoldResult::FrontierUnproven::Held; + } + else + cov.classification = table_changed ? 2 : 1; + + result.fold_seal.ref_lives.at(target.life_id).coverage = cov; + ++result.frontier_namespaces; + if (frontier_proven) + ++result.frontier_proven; + else + result.frontier_deficit.count(unproven_reason); + if (table_changed) + { + folded_any = true; + ++intake_tables_changed; + } + if (fired) + ++intake_tables_clamped; + } + + /// Namespaces the round KNOWS about but never probed, because the frontier-probe budget ran out + /// before reaching them. Their cursors ride VERBATIM -- dropping a cursor because a round ran out of + /// budget would hand the next round a namespace to re-fold from `{0,0}`, which is a far worse + /// outcome than the unproven frontier this already is. They count toward the universe and NOT toward + /// the proofs, which is what suppresses the round. + /// + /// THE DENOMINATOR IS THE SEALED SET, BY CONSTRUCTION. `frontier_namespaces` is incremented by the + /// number of rows this loop actually ADDED to the seal, never by the count the skip loop predicted. + /// The two are the same set under the same filters, so they agree -- but agreeing "because both + /// filters were written the same way" is exactly the kind of coincidence that decays under editing, + /// and `frontier_namespaces` is the denominator an operator (and the integration test) reads as THE + /// universe. Deriving it from the seal removes the possibility of publishing a number that + /// describes a different universe than the round sealed. The `chassert` states the equality so a + /// future divergence surfaces in debug rather than silently; it is metric integrity, not a safety + /// gate, because both paths suppress the round either way. + if (intake_unprobed_budget > 0) + { + result.frontier_namespaces += intake_unprobed_budget; + result.frontier_deficit.probe_budget += intake_unprobed_budget; + LOG_WARNING(logger, + "CAS GC ref intake: the frontier-probe budget ({}) ran out with {} known namespace(s) " + "unprobed; their cursors ride unchanged and ALL destructive work is suppressed this round", + store->poolConfig().gc_frontier_probe_budget, intake_unprobed_budget); + } + + /// All-or-nothing: a malformed key/body anywhere aborts the round's ref folding. Discard + /// every ref delta and cursor advance already accumulated, carry each table's parent cursor verbatim, + /// and let the recorded anomaly suppress destructive work. + if (ref_folding_aborted) + { + deltas.clear(); + ledger = TxnApplyLedger{}; /// the deltas are gone, so nothing can be unapplied + result.mf_cleanup.clear(); + folded_any = false; + /// An abort discards this round's walk, so it discards its proofs with it: nothing this round + /// observed may be offered as a frontier proof to the destructive gate. + result.frontier_proven = 0; + result.frontier_deficit = FoldResult::FrontierDeficit{}; + result.frontier_deficit.fold_aborted = result.frontier_namespaces; + for (const WalkTarget & target : walk_targets) + { + RefCoverage cov; + cov.classification = 1; + if (const auto pit = parent_ref_lives.find(target.life_id); pit != parent_ref_lives.end()) + { + cov.last_folded_ref_id = pit->second.coverage.last_folded_ref_id; + /// An abort discards this round's work; it does not resolve anything, so a hold it + /// found in the parent seal rides forward untouched -- not even the retry count moves, + /// because nothing was retried. + if (pit->second.coverage.hold) + { + cov.hold = pit->second.coverage.hold; + cov.classification = 4; + } + } + RefLifeFoldState & ref_life_state = result.fold_seal.ref_lives.at(target.life_id); + ref_life_state.coverage = cov; + } + } + + /// PROBE B1's recomputation, taken HERE rather than at the seal write: every input it reads + /// (`walked_segments`, the sealed ref-life coverage) is final as of this line, and nothing + /// between here and the seal write touches any of them. Computing it inside the intake phase is what + /// lets the `fold_ref_intake` row carry both numbers; the comparison and its fail-closed throw stay + /// where they were, just before the seal write. Both stay 0 on a ref-folding abort -- that path + /// discards every cursor advance and carries the parent cursors, so the identity does not apply. + /// + /// It counts the CUT ARITHMETICALLY, not by listed ids. Under arithmetic intake a listed-id count is + /// not even the right question: a hint hole means a round legitimately applies records the listing + /// never mentioned, so the old recomputation would report fewer logs than folded and fail every + /// healthy round on a lying store -- it would have made this task's own fix unshippable. + /// + /// BE HONEST ABOUT WHAT IS LEFT. The old formula could disagree with reality because it was derived + /// from a different source (the listing) than the counter. This one is derived from the runs the + /// single advance site produced, so for the current code shape it is close to tautological, and B1's + /// discriminating power went DOWN with this change rather than up. What it still asserts is worth + /// keeping and is not free: THE SEALED CURSOR IS THE WALK'S CURSOR. The last run of each namespace is + /// measured against the DURABLE ref-life coverage the next round will trust, not against the + /// walk's own end, so a cursor sealed from anywhere other than this walk -- a stale carry, a mutated + /// coverage row, a future edit that advances the cursor away from the advance site -- either fails + /// the epoch/order check below or lands as a count that no longer matches `logs_applied`. + logs_accounted_this_round = 0; + logs_applied_this_round = 0; + if (!ref_folding_aborted) + { + uint64_t logs_accounted = 0; + for (const auto & [ns_str, segments] : walked_segments) + { + const UInt128 life_id = live_incarnation.at(ns_str); + RefTxnId sealed{}; + if (const auto sit = result.fold_seal.ref_lives.find(life_id); + sit != result.fold_seal.ref_lives.end()) + sealed = sit->second.coverage.last_folded_ref_id; + + for (size_t i = 0; i < segments.size(); ++i) + { + const auto & [first, last] = segments[i]; + /// The final run must end exactly where the seal says this namespace stopped, in the same + /// epoch. A disagreement is a sealed cursor that did not come from this walk, so it fails + /// closed here rather than travelling into the next round's baseline. + const RefTxnId end = i + 1 == segments.size() ? sealed : last; + if (end.writer_epoch != first.writer_epoch || end < first) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC fold: namespace {} sealed a cursor at {} that does not close the run it " + "walked (opened at {}). GC refuses to commit the round; recover with " + "SYSTEM CAS GC REBUILD.", + ns_str, end == RefTxnId{} ? "none" : renderRefTxnId(end), renderRefTxnId(first)); + logs_accounted += end.ref_sequence - first.ref_sequence + 1; + } + } + logs_accounted_this_round = logs_accounted; + logs_applied_this_round = logs_applied; + } + + intake_timer->metric("logs_accounted", logs_accounted_this_round); + intake_timer->metric("logs_applied", logs_applied_this_round); + intake_timer->metric("deltas_emitted", deltas.size()); + intake_timer->metric("transactions_opened", ledger.txns.size()); + intake_timer->metric("tables_scanned", ref_tables.size()); + intake_timer->metric("tables_changed", intake_tables_changed); + intake_timer->metric("tables_clamped", intake_tables_clamped); + /// `tables_held` counts the coverage rows this round SEALED held -- the ones it detected plus the + /// ones it carried. It is what `suppress_destructive` keys on, so a round that reclaims nothing has + /// this column to explain itself. `unhinted_held_walked` is the subset the hint never mentioned: + /// nonzero means the store stopped listing a namespace that is still held, which is the shape that + /// used to clear a hold silently. + intake_timer->metric("tables_held", intake_tables_held); + intake_timer->metric("unhinted_held_walked", intake_unhinted_held); + /// THE FROZEN WORK-SET, ON THE ROW THAT PAID FOR IT. `tails_advanced` is how many hinted namespaces + /// had records to fold this round because their listed tail sat above their cursor -- the round's + /// real work -- and `tails_unchanged` is how many folded NOTHING and paid only their single frontier + /// probe, which on a wide pool with a few hot namespaces is most of them and is the number that + /// explains a short round. `tails_below_cursor` is the anomaly of the three: cleanup deletes logs + /// from the bottom up, so a listing whose greatest id is below a cursor we folded through is stale or + /// lying, and a column that is normally 0 is what makes the pool where it is not stand out. + intake_timer->metric("tails_advanced", intake_tails_advanced); + intake_timer->metric("tails_unchanged", intake_tails_unchanged); + intake_timer->metric("tails_below_cursor", intake_tails_below_cursor); + /// THE FRONTIER OBLIGATION, on the row that explains a round which reclaimed nothing: + /// `frontier_namespaces` is the round's universe (hint ∪ sealed cursors ∪ catalog `Live`/`Removing` + /// entries), `frontier_proven` the part of it that reached an honest end-of-stream, and the + /// remaining columns say where the rest went -- walked because the hint had gone quiet about them, + /// not walked at all because the probe budget ran out, or walked ONLY because the catalog named a + /// namespace neither the hint nor a carried cursor did (`catalog_only_walked` -- always 0 until a + /// namespace is admitted with no listed objects and no sealed cursor yet). + intake_timer->metric("frontier_namespaces", result.frontier_namespaces); + intake_timer->metric("frontier_proven", result.frontier_proven); + /// `catalog_entries` is the hot-scan catalog cut's own row count (every lifecycle state, `Creating` + /// included), and `catalog_proved_empty` is the derived verdict the destructive gate's non-vacuity + /// term consults. Together they let an operator tell a proved-empty `0/0` (success) apart from a + /// `Creating`-only or otherwise unprovable `0/0` (still suppressed) without re-deriving either fact. + intake_timer->metric("catalog_entries", catalog_snapshot.catalog.entries.size()); + intake_timer->metric("catalog_proved_empty", result.catalog_cut_proved_empty ? 1 : 0); + intake_timer->metric("unhinted_quiet_walked", intake_unhinted_quiet); + intake_timer->metric("frontier_unprobed_budget", intake_unprobed_budget); + intake_timer->metric("catalog_only_walked", intake_catalog_only); + intake_timer->metric("dead_precommits_skipped", intake_dead_precommits_skipped); + /// The exact reads arithmetic intake pays that the listing-driven loop did not. `absent_probes` + /// counts EVERY read that came back absent: the expected-next of each namespace walked (at least one + /// -- the absent expected-next IS the frontier proof) and the epoch-start read of a crossing that + /// failed, which a namespace that holds every round pays every round. `epoch_crossings` counts the + /// successful ones. Both are on the row so the cost shows up where it is spent. + intake_timer->metric("absent_probes", intake_absent_probes); + intake_timer->metric("epoch_crossings", intake_epoch_crossings); + intake_timer->metric("namespace_removals", new_removals.size()); + intake_timer->metric("ref_folding_aborted", ref_folding_aborted ? 1 : 0); + intake_timer.reset(); /// emits the `fold_ref_intake` row + + result.frontier_unprobed_budget = intake_unprobed_budget; + + /// Reuse the round's single ref LIST for post-CAS ref-object cleanup: one LIST serves + /// intake AND cleanup planning). + result.ref_tables = ref_tables; + /// Same reuse for the checkpoints: the intake walk already paid for them as its second witness, and + /// the cleanup ranges below are the other consumer of the same fact. + /// + /// An UNDECODABLE `_ckpt` contributes no entry here and therefore grants no cleanup authority. The + /// walk above also held (or recorded an anomaly for) every such namespace, so the round's destructive + /// gate is shut and `cleanupRefObjects` deletes nothing at all this round. Any future change that + /// narrows the gate from round-wide to per-namespace must carry this set with it. + result.checkpoints = checkpoints.recovery_checkpoints; + + /// Folding the terminal record earns positive cleanup evidence directly on the catalog-admitted + /// life row. It does not claim that any physical debris was removed: `_ckpt`, stream and `_files` + /// residue belongs to the perpetual janitor, while orphan manifests belong to the manifest sweep. + /// Consequently namespace removal performs no physical LIST and has no Pending/Completed handshake. + if (!ref_folding_aborted) + for (const auto & [rns, remove_txn_id] : new_removals) + { + const auto life_it = live_incarnation.find(rns.string()); + if (life_it == live_incarnation.end()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC fold: terminal removal for namespace '{}' has no row in the round catalog cut", + rns.string()); + auto row_it = result.fold_seal.ref_lives.find(life_it->second); + if (row_it == result.fold_seal.ref_lives.end()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC fold: terminal removal for namespace '{}' has no admitted ref-life row", + rns.string()); + row_it->second.cleanup_evidence = RefCleanupEvidence{.remove_txn_id = remove_txn_id}; + } + + /// The parent generation's per-shard run segments, resolved from + /// the parent fold seal's `blob_target_runs` and grouped by the ref's explicit `shard`. The same seal + /// `discover_ref_seal` the token-diff already read is the run source — consumers resolve runs THROUGH + /// refs (a run sealed for the parent generation may physically live under an older generation's key), + /// never by `blobTargetRunKey` construction. + std::map> parent_runs_by_shard; + for (const RunRef & r : discover_ref_seal.blob_target_runs) + parent_runs_by_shard[r.shard].push_back(r); + + /// PURE REF-CARRY: a gc-shard with an EMPTY delta bucket + /// AND an EMPTY retired input list neither reads nor writes its run — the new fold_seal copies the + /// parent's `RunRef`s VERBATIM (key/checksum/shard/generation) so the next round resolves them. This + /// is deterministic (same refs for the same inputs), so seal determinism / crash-replay adoption hold. + /// An empty delta with a NON-EMPTY retired list still runs the merge: settlement must happen every + /// pass (carried/graduated/redeleted entries), and that pass reads the run to recompute in-degrees. + /// Distill one shard's `condemned_summary` entry from the `kCondemned` rows it re-emitted this pass + /// (`still_retired` mirrors those rows exactly). Folding shards call this; it makes the next + /// round's `graduationDue` and pure-carry decisions read only the seal, never a run. + auto summarize = [](const std::vector & still) -> CondemnedSummary + { + CondemnedSummary s; + s.condemned_total = still.size(); + for (const RetiredEntry & e : still) + { + if (e.delete_pending) + ++s.pending_total; + else + s.oldest_nonpending_condemn_round = + std::min(s.oldest_nonpending_condemn_round, e.condemn_round); + } + return s; + }; + /// The parent seal's summary for a shard, used ONLY for the pure-carry DECISION (condemned_total==0). + /// Missing => zero: a fresh pool has no parent entry (correct baseline), and a snap_generation>0 seal + /// that is missing an entry we would pure-carry fails closed inside `carryParentRefs` when it copies. + auto summaryOfParent = [&](uint64_t shard) -> CondemnedSummary + { + const auto it = discover_ref_seal.condemned_summary.find(shard); + return it != discover_ref_seal.condemned_summary.end() ? it->second : CondemnedSummary{}; + }; + auto carryParentRefs = [&](uint64_t shard) + { + const auto it = parent_runs_by_shard.find(shard); + if (it != parent_runs_by_shard.end()) + for (const RunRef & r : it->second) + result.fold_seal.blob_target_runs.push_back(r); /// verbatim: parent key/checksum/gen + /// Totality: a pure-carry shard settled nothing, so its `condemned_summary` entry is the parent's + /// VERBATIM. On a fresh pool (no adopted parent seal) it is the explicit zero baseline; otherwise + /// the parent seal MUST carry the entry (a live seal is total over gc_shards) — a missing entry is + /// corrupt bookkeeping, never silently treated as zero. + if (state.snap_generation == 0) + result.fold_seal.condemned_summary[shard] = CondemnedSummary{}; + else + { + const auto sit = discover_ref_seal.condemned_summary.find(shard); + if (sit == discover_ref_seal.condemned_summary.end()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS gc fold: parent fold seal (generation {}, attempt {}) lacks a condemned_summary " + "entry for gc-shard {} — the seal is not total over gc_shards; GC bookkeeping is corrupt", + state.snap_generation, state.snap_attempt, shard); + result.fold_seal.condemned_summary[shard] = sit->second; + } + }; + auto priorRunsFor = [&](uint64_t shard) -> const std::vector & + { + static const std::vector empty; + const auto it = parent_runs_by_shard.find(shard); + return it != parent_runs_by_shard.end() ? it->second : empty; + }; + + + /// CLAMP SUPPRESSION: any clamp + /// this pass means landed-before-cut events may be UNFOLDED behind a clamped cursor — the + /// floor's "landed before cut => folded before graduation" lemma does not hold, so this pass + /// must not graduate NOR execute pending deletes (the merge carries everything; condemnation + /// and sparing continue). Deletes resume on the first clamp-free pass. This is the honest-mode + /// counterpart of the model's SabotageSkipChangedShard counterexample. + /// + /// PHASE 9/18 `fold_reduce`: the prior-run streaming GETs, one HEAD per zero-transition candidate, + /// and the run PUTs -- the heaviest phase of a folding round on a pool with churn. It also carries + /// probe B2's verdict (`transactions_unapplied`), which is 0 on every committed round because a nonzero + /// value throws a few lines below; the row is therefore the forensic record of a round that failed. + std::optional reduce_timer; + reduce_timer.emplace(phase_sink, "fold_reduce"); + const uint64_t deltas_in = deltas.size(); + const uint64_t condemned_before = report.condemned; + uint64_t shards_pure_carry = 0; + /// ============================ THE DESTRUCTIVE GATE ============================ + /// + /// Computed ONCE, here, from three independent terms, and consulted at every destructive site of + /// the round (see `FoldResult::suppress_destructive`). It sits at this point in the fold because + /// every input is final: the coverage rows the seal will carry are written, the anomalies are + /// recorded, and the frontier tally is closed. + /// + /// Term 2 is STRUCTURAL. Today every hold also records an anomaly, so term 1 happens to imply it -- + /// but that is a property of the current code, not the invariant, and a gate that relies on a + /// coincidence opens the day the coincidence stops holding. The hold SET is the invariant, so the + /// gate reads the seal it is about to make durable. + const std::vector> carried_holds = result.carriedHolds(); + + /// Term 3, the universe seam. `Authoritative` means the round's universe is the catalog's own + /// `Live`/`Removing` set, so the per-namespace proofs decide on their own; `StageA_Suppressed` + /// refuses outright, which is the posture a test asserting inertness selects. See `UniversePolicy`. + const bool universe_authoritative = policy == UniversePolicy::Authoritative; + /// `frontier_proven == frontier_namespaces` is `0 == 0` -- TRUE -- + /// on an empty universe, which is not a proof of anything BY ITSELF: a fresh pool, a damaged + /// catalog, or a read that legitimately returns nothing all produce zero entries. `frontier_namespaces + /// > 0` closes that degenerate case for the ordinary, nonempty pool -- a nonzero count still needs + /// every namespace PROVEN, which the equality above still checks. + /// + /// That floor alone is unsound for a pool whose LAST namespace was removed: the catalog then reads + /// genuinely empty forever, `frontier_namespaces` can never again exceed 0, and the equality's + /// vacuous truth can never be licensed -- an emptied pool would stop reclaiming permanently. The + /// floor's job was never "reject `frontier_namespaces == 0`", it was "reject the UNSUPPORTED case of + /// it". `catalog_cut_proved_empty` is the supported case: the round's own hot-scan catalog cut, + /// read once and reused (never a second `GET`), decoded successfully, token-bearing, and holding + /// zero rows of every lifecycle state including `Creating`. Under this pool's protocol every live or + /// live-precommit edge requires an exact `Live` catalog row (INV-3), so that cut is a positive proof + /// that no namespace anywhere holds one -- not merely an absence of proof. A catalog holding only + /// `Creating` rows produces the SAME `frontier_namespaces == 0` but is a birth in progress, not an + /// empty universe, and `catalog_cut_proved_empty` is false for it (see `entries.empty()` above). + result.frontier_complete = universe_authoritative + && result.frontier_proven == result.frontier_namespaces + && (result.frontier_namespaces > 0 || result.catalog_cut_proved_empty); + const bool frontier_incomplete = !result.frontier_complete; + + result.suppress_destructive = + !report.anomalies.empty() || !carried_holds.empty() || frontier_incomplete; + const bool suppress_destructive = result.suppress_destructive; + if (suppress_destructive) + { + ProfileEvents::increment(ProfileEvents::CASGCClampSuppressedPasses); + /// LEVEL SPLIT, deliberately. A pass suppressed by an anomaly, a hold, an unproven namespace or + /// an empty universe has a per-round cause an operator can chase, and that is a WARNING. A pass + /// suppressed because the CALLER refused to supply a universe carries no such cause -- nothing on + /// the pool explains it -- so it is reported at Info with the same numbers rather than raising an + /// alarm nobody can act on. + /// + /// An AUTHORITATIVE-but-EMPTY universe that the catalog cut did NOT prove empty + /// (`frontier_namespaces == 0 && !catalog_cut_proved_empty` -- e.g. a `Creating`-only catalog) + /// is a per-round cause and is named explicitly here, because the bare equality + /// `frontier_proven != frontier_namespaces` reads `0 != 0` (false) and would otherwise miss it. + /// A catalog the cut DID prove empty is not a suppression cause at all: it satisfies + /// `frontier_complete`, so a round reaching this block with one is suppressed by an anomaly or + /// a hold, never by the frontier term. + const bool per_round_cause = !report.anomalies.empty() || !carried_holds.empty() + || result.frontier_proven != result.frontier_namespaces + || (universe_authoritative && result.frontier_namespaces == 0 && !result.catalog_cut_proved_empty); + const char * const universe_note = + universe_authoritative ? "" : "; the caller supplied no universe"; + /// Names the real reason a `Creating`-only (or otherwise unprovable) catalog is not an empty + /// universe, so the operator does not read a suppressed round with rows on file as "empty". + const String catalog_empty_note = + (universe_authoritative && result.frontier_namespaces == 0 && !result.catalog_cut_proved_empty) + ? fmt::format("; catalog holds {} row(s), none walkable/provable", catalog_snapshot.catalog.entries.size()) + : String{}; + /// The per-cause breakdown of the unproven namespaces. Without it "N of M proven" names a + /// deficit but not its cause, and the causes want opposite operator responses. + const String deficit_note = result.frontier_deficit.total() == 0 + ? String{} + : fmt::format("; unproven: {}", result.frontier_deficit.describe()); + if (per_round_cause) + LOG_WARNING(logger, + "CAS GC fold: destructive work SUPPRESSED this pass — {} anomaly(ies), {} held " + "namespace(s), frontier {} ({} of {} namespace(s) proven{}{}{}). Graduations and pending " + "deletes are carried; nothing irreversible runs until a pass that clears all three.", + report.anomalies.size(), carried_holds.size(), + result.frontier_complete ? "complete" : "INCOMPLETE", + result.frontier_proven, result.frontier_namespaces, universe_note, catalog_empty_note, + deficit_note); + else + LOG_INFO(logger, + "CAS GC fold: destructive work SUPPRESSED this pass — {} anomaly(ies), {} held " + "namespace(s), frontier {} ({} of {} namespace(s) proven{}{}{}). Graduations and pending " + "deletes are carried; nothing irreversible runs until a pass that clears all three.", + report.anomalies.size(), carried_holds.size(), + result.frontier_complete ? "complete" : "INCOMPLETE", + result.frontier_proven, result.frontier_namespaces, universe_note, catalog_empty_note, + deficit_note); + } + + std::vector orphan_source_retirements; + if (!suppress_destructive && store->poolConfig().manifest_sweep_list_budget_keys > 0) + { + result.orphan_sweep = planManifestCursorPage( + *store, + state.manifest_sweep_cursor, + store->poolConfig().manifest_sweep_list_budget_keys, + store->poolConfig().manifest_sweep_delete_budget_keys, + /// The sweep may recover a catalog-named namespace's debris only from the same frozen catalog + /// cut and `_ckpt` frontier the round's own universe came from -- which is exactly what an + /// authoritative universe means, and is why this is the gate's term and not a separate one. + universe_authoritative, + &work_budget); + for (const ManifestSweepResult::Nomination & nomination : result.orphan_sweep.nominations) + orphan_source_retirements.insert( + orphan_source_retirements.end(), + nomination.source_retirements.begin(), + nomination.source_retirements.end()); + } + + if (state.gc_shards == 1) + { + /// SINGLE-SHARD PATH (gc_shards == 1). Every blob routes to shard 0, so the entire delta stream + /// folds into one `blobTargetRunKey(new_generation, 0, 0)` run. + if (!folded_any && orphan_source_retirements.empty() && summaryOfParent(0).condemned_total == 0) + { + /// Pure ref-carry: nothing changed and no condemned entries to settle => zero run I/O. Carry the + /// parent shard-0 refs + summary into the seal so coverage/resume/graduation stay durable. + carryParentRefs(0); + ++shards_pure_carry; + } + else + { + /// Either a real delta or a non-empty retired input: run the merge (empty deltas still settle + /// the kCondemned rows riding the parent run). The prior runs are the parent seal's shard-0 refs. + foldDeltasIntoGeneration(backend, layout, priorRunsFor(0), + new_generation, attempt, /*shard*/0, + std::move(deltas), result.fold_seal.blob_target_runs, + current_round, condemn_round, head_blob, peek_head, + confirm_condemned_marker, + result.retired_merge.data(), suppress_destructive, + &ledger.applied, std::move(orphan_source_retirements), + &work_budget); + result.fold_seal.condemned_summary[0] = summarize(result.retired_merge[0].still_retired); + } + } + else + { + /// SHARDED PATH (gc_shards > 1) — target-sharded reducers. Each blob's + /// `BlobDelta` carries its full signed edge stream; `blobShard(blob_hash, gc_shards)` partitions + /// the stream into `gc_shards` disjoint buckets. Each bucket folds via its own `ShardReducer` + /// into `blobTargetRunKey(new_generation, shard, 0)`. The `RootOwnerEvent`'s paired old/new + /// bindings produced the `-1`/`+1` deltas above, so a promote that displaces a blob's owner + /// emits BOTH the `-1` (old binding) and the `+1` (new binding) at the SAME source event. This + /// is why cross-shard displacement needs no special handling: each delta routes independently and + /// deterministically to whichever target shard owns its blob; the old/new pair is solved at the + /// source, not by a cross-shard fixup. + std::vector> buckets(state.gc_shards); + for (BlobDelta & d : deltas) + buckets[blobShard(d.ref, state.gc_shards)].push_back(std::move(d)); + std::vector> retirement_buckets(state.gc_shards); + for (BlobSourceRetirement & retirement : orphan_source_retirements) + retirement_buckets[blobShard(retirement.ref, state.gc_shards)].push_back(std::move(retirement)); + + for (uint64_t shard = 0; shard < state.gc_shards; ++shard) + { + if (buckets[shard].empty() && retirement_buckets[shard].empty() + && summaryOfParent(shard).condemned_total == 0) + { + /// Pure ref-carry for this shard: empty delta + no condemned entries => zero run I/O. + carryParentRefs(shard); + ++shards_pure_carry; + continue; + } + /// A reducer owns exactly one disjoint shard. Two replicas may run reducers for DIFFERENT + /// shards concurrently (CasGcScheduler ownership); their run-key namespaces never collide. + std::vector shard_runs; + foldDeltasIntoGeneration( + backend, layout, priorRunsFor(shard), new_generation, attempt, shard, + std::move(buckets[shard]), shard_runs, + current_round, condemn_round, head_blob, peek_head, + confirm_condemned_marker, + &result.retired_merge[shard], suppress_destructive, + &ledger.applied, std::move(retirement_buckets[shard]), + &work_budget); + for (RunRef & r : shard_runs) + result.fold_seal.blob_target_runs.push_back(std::move(r)); + result.fold_seal.condemned_summary[shard] = summarize(result.retired_merge[shard].still_retired); + } + } + + /// Aggregate the unmatched-remove signal across every gc-shard this pass touched and log ONCE per + /// round with the total plus one example, rather than from the hot per-edge inner loop that detects + /// them (see `foldDeltasIntoGeneration`'s comment) — that loop runs over potentially millions of + /// rows, so it only counts (`ProfileEvents::CASGCUnmatchedRemoveDeltas`, incremented per occurrence) + /// and hands back one example; this is the bounded, once-per-round operator-visible trail. + { + uint64_t total_unmatched_removes = 0; + std::optional example; + for (const RetiredMergeResult & merge : result.retired_merge) + { + total_unmatched_removes += merge.unmatched_removes; + if (!example && merge.unmatched_remove_example) + example = merge.unmatched_remove_example; + } + if (total_unmatched_removes > 0 && example) + LOG_WARNING(logger, + "CAS GC fold: {} unmatched removal delta(s) this pass (matched no existing source edge; " + "a harmless per-key no-op by design, since the in-degree model is a set, not a counter — " + "but a persistent nonzero rate means removal deltas are reaching the reducer without " + "their matching activation, which is a correctness signal) — example: blob {} source {}", + total_unmatched_removes, blobIdOf(example->ref), u128ToHex(example->source_id)); + reduce_timer->metric("unmatched_removes", total_unmatched_removes); + } + + /// PROBE B2's ledger verdict, computed inside the reduce phase so the row carries it; the + /// fail-closed throw it drives stays below, at its original site before the seal write. + const std::vector unapplied_txns = ledger.unapplied(); + transactions_unapplied_this_round = unapplied_txns.size(); + { + uint64_t graduated = 0; + uint64_t spared = 0; + uint64_t redelete_pending = 0; + for (const RetiredMergeResult & merge : result.retired_merge) + { + graduated += merge.graduated.size(); + spared += merge.spared.size(); + redelete_pending += merge.redelete.size(); + } + reduce_timer->metric("shards_total", state.gc_shards); + reduce_timer->metric("shards_pure_carry", shards_pure_carry); + reduce_timer->metric("shards_reduced", state.gc_shards - shards_pure_carry); + reduce_timer->metric("deltas_in", deltas_in); + reduce_timer->metric("runs_written", result.fold_seal.blob_target_runs.size()); + reduce_timer->metric("condemned", report.condemned - condemned_before); + reduce_timer->metric("graduated", graduated); + reduce_timer->metric("spared", spared); + reduce_timer->metric("redelete_pending", redelete_pending); + reduce_timer->metric("suppress_destructive", suppress_destructive ? 1 : 0); + /// Published separately from `suppress_destructive` so a reader can tell the frontier term apart + /// from the anomaly and hold terms without re-deriving the formula from the tally. + reduce_timer->metric("frontier_complete", result.frontier_complete ? 1 : 0); + reduce_timer->metric("transactions_unapplied", transactions_unapplied_this_round); + } + reduce_timer.reset(); /// emits the `fold_reduce` row + + /// The part-manifest cleanup RUN + its fold-seal record are removed: the run + /// object had no reader — the manifest cleanups execute inline from `result.mf_cleanup` (below / + /// the recheck path), so the durable bundle was pure dead weight. `result.mf_cleanup` is unchanged. + + /// Write-once CasFoldSeal: its existence marks fold complete. The fold seal is DETERMINISTIC (same + /// fold inputs => byte-identical seal), so it goes through `putDeterministicArtifact`: a byte-equal + /// occupant is our own crash/deterministic replay (adopt, no-op); divergent bytes are impossible + /// under correct operation and fail closed with `CORRUPTED_DATA`. A deposed leader writes under its + /// own unadopted attempt so it never collides with the adopted seal — the occupant here is only ever + /// our own prior attempt-scoped write. + /// PROBE B2's verdict. A committed transaction that produced deltas but whose deltas never + /// reached a reducer means this round LOST a durable record it had already read and decoded. + /// Unlike a 404 during a fold (missing evidence, which must never wedge the round), this is proof + /// of loss, so the round fails CLOSED: nothing is adopted, GC reclaims and deletes nothing, and an + /// operator has to intervene. Thrown before the seal write, and therefore long before the single + /// `gc/state` CAS, so the whole round evaporates. + /// `unapplied_txns` was computed in the reduce phase above (so its row could report it); the verdict + /// itself is unchanged and still fires here, before the seal write. + if (!unapplied_txns.empty()) + { + String detail; + for (size_t i = 0; i < unapplied_txns.size() && i < 8; ++i) + { + if (i != 0) + detail += ", "; + detail += ledger.namespaces[unapplied_txns[i]] + "@" + + renderRefTxnId(ledger.txns[unapplied_txns[i]]); + } + ProfileEvents::increment(ProfileEvents::CASGCUnappliedFoldedTransactions, unapplied_txns.size()); + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC fold: {} ref transaction(s) folded and merged into the round buffers but NONE of " + "their blob deltas reached a shard reducer ({}{}). The round would have advanced its " + "cursor past a transaction it never applied. GC refuses to commit the round; recover with " + "SYSTEM CAS GC REBUILD.", + unapplied_txns.size(), detail, unapplied_txns.size() > 8 ? ", ..." : ""); + } + + /// PROBE B1's comparison. Both terms were derived at the end of the ref intake (see the + /// recomputation there, which is also what the `fold_ref_intake` row reports) and are 0 on a + /// ref-folding abort, where the identity does not apply -- so the inequality below can only fire on + /// a round that actually folded. + if (logs_accounted_this_round != logs_applied_this_round) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC fold: the round sealed coverage over {} ref log(s) but only {} fully folded -- " + "a cursor advanced past a log this round never applied. GC refuses to commit the round; " + "recover with SYSTEM CAS GC REBUILD.", + logs_accounted_this_round, logs_applied_this_round); + + /// PHASE 10/18 `fold_seal_write`: one PUT (or, on a deterministic replay, a byte-compare GET). + { + GcPhaseTimer t(phase_sink, "fold_seal_write"); + validateFoldSealForWrite(result.fold_seal, layout, store->poolConfig().gc_shards); + const String seal_body = encodeFoldSeal(result.fold_seal); + t.metric("seal_bytes", seal_body.size()); + t.metric("seal_runs", result.fold_seal.blob_target_runs.size()); + t.metric("seal_ref_lives", result.fold_seal.ref_lives.size()); + t.metric("seal_cleanup_evidence", std::count_if( + result.fold_seal.ref_lives.begin(), result.fold_seal.ref_lives.end(), + [](const auto & item) { return item.second.cleanup_evidence.has_value(); })); + putDeterministicArtifact(backend, layout.foldSealKey(new_generation, attempt), seal_body); + } + + /// One-pass round: the fold NO LONGER CASes gc/state. (new_generation, attempt) are adopted + /// in-memory here and committed — together with the round, the retired refs, and the retention + /// cursor — by the SINGLE round CAS in runRegularRound. A deposed leader's whole pass therefore + /// evaporates at that one CAS; its attempt-scoped artifacts are never adopted. + state.snap_generation = new_generation; + state.snap_attempt = attempt; + return result; +} + +void Gc::reportSweepRetention(const ManifestSweepResult & result) +{ + const auto top = result.topRetainReason(); + if (top.second == 0) + { + /// Nothing retained by the premise. Re-arm, so that the next retention -- however far off -- + /// is reported as the change it is rather than swallowed by a repeat counter. + last_retain_rollup.reset(); + retain_rollup_passes_since_report = 0; + return; + } + + const bool changed = !last_retain_rollup || *last_retain_rollup != top; + if (!changed && ++retain_rollup_passes_since_report < kRetainRollupRepeatPasses) + return; + + /// INFO, not WARNING: retention is the CORRECT outcome whenever rule (1) is unsatisfiable (it is + /// satisfiable only for a closed-and-folded epoch), so warning here would alarm on healthy rounds. It is + /// still the operator's answer to "why is manifest debris not shrinking?", which is why it is not + /// left at DEBUG with the per-object sentences. + /// The "X of Y" denominator is the RETAINED total, not `skipped`. `skipped` is a strictly larger + /// population -- it also counts malformed keys, ineligible prefixes, protected owners and + /// budget-deferred candidates -- so measuring the top class against it would understate the class's + /// share of the very number this sentence just reported. + const uint64_t retained = result.retained_no_coverage + result.retained_hold + + result.retained_unconsumed_seal + result.retained_tail_removal; + LOG_INFO(logger, + "CAS gc orphan sweep: retained {} manifest body(ies) this pass, most of them ({} of {}) for " + "'{}' -- see the fold seal's coverage for that namespace; deleted {}, listed {}, skipped {}", + retained, top.second, retained, sweepRetainClassName(top.first), + result.deleted, result.listed, result.skipped); + + last_retain_rollup = top; + retain_rollup_passes_since_report = 0; +} + +void Gc::cleanupRefObjects( + const FoldResult & folded, const GcLease & adopted_lease, bool suppress_destructive, + GcRoundWorkBudget & work_budget) +{ + /// A clamp / ref-folding abort this round may leave landed-before-cut edges unfolded behind the clamp, + /// so a covered-log cleanup could delete a log whose delta is not yet durable -- defer to a clean pass. + if (suppress_destructive) + return; + + Backend & backend = store->backend(); + const Layout & layout = store->layout(); + if (!folded.catalog_cut) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS GC ref cleanup: fold result carries no catalog cut"); + + for (const auto & [ns_str, listing] : folded.ref_tables) + { + const RootNamespace ns{ns_str}; + /// Review C3: look up the SAME complete cut the round's walk resolved, never re-resolve + /// independently. A fresh read here could see a namespace dropped and recreated since the + /// walk and delete the successor's objects using predecessor bounds. An absent or `Creating` + /// row is skipped rather than mapped to a fabricated key. + const auto entry_it = std::lower_bound( + folded.catalog_cut->catalog.entries.begin(), folded.catalog_cut->catalog.entries.end(), ns, + [](const CatalogEntry & entry, const RootNamespace & needle) { return entry.ns < needle; }); + if (entry_it == folded.catalog_cut->catalog.entries.end() || entry_it->ns != ns + || (entry_it->state != NsState::Live && entry_it->state != NsState::Removing)) + continue; + const CatalogEntry & observed_entry = *entry_it; + const NamespaceLifeId life = NamespaceLifeId::fromCatalogEntry(entry_it->ns, entry_it->incarnation); + + /// Current-life ref cleanup is not the dead-life janitor: every irreversible key delete must + /// still be licensed by the SAME complete catalog observation and GC lease that adopted the + /// fold. Re-read both after the target HEAD and immediately before `deleteExact`. A moved token, + /// changed row/life, missing or unreadable authority object, or changed owner/sequence stops the + /// whole cleanup pass. Continuing with another row/key would turn a refusal into a fallback. + const auto deleteRefObject = [&](const String & key) + { + const HeadResult h = backend.head(key); + if (!h.exists) + return true; + + try + { + const CasRefCatalog::Snapshot current_catalog = CasRefCatalog::read(backend, layout); + current_catalog.life_index.throwIfAmbiguous("CAS GC ref cleanup revalidation"); + const auto current_entry_it = std::lower_bound( + current_catalog.catalog.entries.begin(), current_catalog.catalog.entries.end(), ns, + [](const CatalogEntry & entry, const RootNamespace & needle) { return entry.ns < needle; }); + const std::optional current_life + = current_catalog.life_index.resolve(life.incarnation); + if (current_catalog.token != folded.catalog_cut->token + || current_entry_it == current_catalog.catalog.entries.end() + || current_entry_it->ns != ns || *current_entry_it != observed_entry + || !current_life || *current_life != life) + { + LOG_DEBUG(logger, + "CAS GC ref cleanup stopped before deleting '{}': catalog observation/life moved", + key); + return false; + } + + const auto current_state_object = backend.get(layout.gcStateKey()); + if (!current_state_object) + { + LOG_WARNING(logger, + "CAS GC ref cleanup stopped before deleting '{}': mandatory gc/state is absent", + key); + return false; + } + const GcState current_state = decodeGcState(current_state_object->bytes); + if (current_state.lease.owner != adopted_lease.owner + || current_state.lease.seq != adopted_lease.seq) + { + LOG_DEBUG(logger, + "CAS GC ref cleanup stopped before deleting '{}': GC fence moved", + key); + return false; + } + } + catch (const std::exception & e) + { + LOG_WARNING(logger, + "CAS GC ref cleanup stopped before deleting '{}': authority revalidation failed: {}", + key, e.what()); + return false; + } + + backend.deleteExact(key, h.token); + ProfileEvents::increment(ProfileEvents::CASRefCleanupObjectsDeleted); /// cleanup object deletion + return true; + }; + + const auto row_it = folded.fold_seal.ref_lives.find(entry_it->incarnation); + const RefTxnId durable_cursor = row_it != folded.fold_seal.ref_lives.end() + ? row_it->second.coverage.last_folded_ref_id + : RefTxnId{}; + + /// Only a checkpoint-named RECOVERY TRIPLE licenses deletion. A listed snapshot may have landed + /// before its publisher's checkpoint CAS, so it must never be promoted into cleanup authority. + /// Validate the exact same-id non-seal `_log` and `_snap` through recovery's one shared helper; + /// failure is confined to this namespace and leaks its listed objects for a later round. + std::optional checkpoint; + if (const auto ckit = folded.checkpoints.find(ns_str); ckit != folded.checkpoints.end()) + checkpoint = ckit->second; + if (!checkpoint || !checkpoint->checkpoint_snapshot_id) + continue; + const RefTxnId checkpoint_snapshot_id = *checkpoint->checkpoint_snapshot_id; + + std::optional retained_log_proof; + try + { + retained_log_proof = readCheckpointSnapshotBase(backend, layout, life, *checkpoint).predecessor_seal_id; + } + catch (const Exception & e) + { + LOG_WARNING(logger, + "CAS GC ref cleanup retained namespace '{}': checkpoint base {} is not a valid recovery triple: {}", + ns_str, renderRefTxnId(checkpoint_snapshot_id), e.message()); + continue; + } + + const RefCleanupPlan plan = planRefCleanup( + listing, durable_cursor, checkpoint_snapshot_id, retained_log_proof); + for (const RefTxnId & log_id : plan.deletable_logs) + { + /// Cumulative per-round cap, never amortized against the per-key fail-close + /// validation `deleteRefObject` performs (HEAD + catalog re-read + gc/state re-read before + /// every exact delete stays exactly as expensive per key as before). Exhaustion simply stops + /// the round's cleanup pass here; `planRefCleanup` recomputes the SAME remaining candidates + /// from durable state next round, so nothing here needs its own cursor. + if (!work_budget.refCleanupAvailable()) + return; + if (!deleteRefObject(layout.refLogKey(life, log_id))) + return; + ++work_budget.ref_cleanup_objects_used; + } + for (const RefTxnId & snap_id : plan.deletable_snapshots) + { + /// Task 5's rule, asserted where it is acted on rather than only where it is computed: the + /// snapshot the checkpoint names is the one a recovering reader will sample, so it must + /// survive every cleanup that the same checkpoint authorized. + chassert(snap_id < checkpoint_snapshot_id); + if (!work_budget.refCleanupAvailable()) + return; + if (!deleteRefObject(layout.refSnapshotKey(life, snap_id))) + return; + ++work_budget.ref_cleanup_objects_used; + } + } +} + +namespace +{ +/// GC-metadata wholesale delete of every object under `prefix`. Returns the number of objects deleted. +/// `bounded_remaining` caps how many objects this call may delete (0 => stop immediately, deleting none). +/// +/// Token source: the in-memory and S3 backends surface a per-key token through `list` +/// (`supportsListTokens()`), so `deleteExact` straight from the listed token; otherwise HEAD first. +/// +/// 404 / NotFound is FAIL-OPEN: an object that vanished between LIST and delete (a concurrent crashed +/// attempt, or a racing prune) is already reclaimed — never throw on a benign missing GC-internal object +/// during a prune (it would only wedge GC). A genuine TokenMismatch is +/// likewise tolerated here: the object was rewritten under us (another attempt is live at this key) — the +/// safe direction during a best-effort prune is to leave it for a later round, never to force-delete. +/// `out_fully_drained`, when set, reports whether the WHOLE prefix was exhausted (every listed key +/// visited) rather than the call stopping early because `bounded_remaining` ran out. A +/// caller advancing a monotone cursor past this prefix must consult this: a `false` here means objects +/// remain, and the cursor must stay put so a later round's fresh budget can finish the same prefix +/// instead of stranding the remainder permanently. `bounded_remaining == 0` conservatively reports +/// `false` (nothing was even examined, so completeness cannot be claimed). +uint64_t deletePrefixWholesale(Backend & backend, const String & prefix, uint64_t bounded_remaining, + bool * out_fully_drained) +{ + if (out_fully_drained) + *out_fully_drained = false; + static constexpr size_t kListPageLimit = 1000; + uint64_t deleted = 0; + String cursor; + while (deleted < bounded_remaining) + { + ListPage page = backend.list(prefix, cursor, kListPageLimit); + /// One page fetched, not one increment per listed key below. + ProfileEvents::increment(ProfileEvents::CASGCEnumerationPages); + for (const auto & listed : page.keys) + { + if (deleted >= bounded_remaining) + return deleted; + if (listed.token.has_value()) + { + /// deleteExact tolerates NotFound (returns Kind::NotFound) and TokenMismatch — both are + /// benign here (already gone / rewritten by a live attempt); do not throw. + backend.deleteExact(listed.key, *listed.token); + } + else if (const auto head = backend.head(listed.key); head.exists) + { + backend.deleteExact(listed.key, head.token); + } + ++deleted; + } + if (page.next_cursor.empty()) + { + if (out_fully_drained) + *out_fully_drained = true; + break; + } + cursor = page.next_cursor; + } + return deleted; +} +} + +void Gc::pruneSupersededGenerations(uint64_t adopted_generation, uint64_t attempt, GcState & next, + const std::set & referenced_generations, + bool suppress_destructive, GcRoundWorkBudget & work_budget) +{ + /// GATED, and `snap_pruned_through` stays where it is. The cursor is a monotone high-water mark the + /// wholesale prune never revisits, so advancing it over a generation this round declined to delete + /// would strand that generation's whole prefix with no reclaimer left (the hand-off only covers + /// generations a LIVE ref moved off, not ones skipped for suppression). + if (suppress_destructive) + return; + + const uint64_t keep = store->poolConfig().gc_snapshot_generations_to_keep; + if (keep == 0) + return; /// keep ALL (debug/forensics — replay GC's in-degree view as-of a past round) + + Backend & backend = store->backend(); + const Layout & layout = store->layout(); + + static constexpr uint64_t kMaxPrunePerRound = 64; /// bound the per-round prune burst + + /// (1) WHOLESALE generation-retention (correctness). A single generation may hold artifacts under + /// MULTIPLE attempts: every round mints a fresh `lease.seq` (= attempt), and a deposed leader writes + /// its fold_seal/runs/cleanup AND its attempt-scoped retired/outcomes sets under its OWN unadopted + /// attempt before its CAS fails. The old per-key single-attempt prune (keyed on the final + /// snap_attempt) therefore leaked every non-adopted attempt's debris. Instead, LIST the whole + /// `gc/gen//` prefix and delete every listed object — reclaiming ALL attempts of `g`, including + /// the retired/ and outcomes/ sets that now live under `gc/gen//attempt//`. Bounded per round + /// by generation count (`kMaxPrunePerRound`) AND by the round's shared object-count work budget; + /// fail-open on 404. `snap_pruned_through` advances over every generation the loop + /// FULLY processes this round — ref-retained (skipped) generations count as fully processed (there + /// is nothing left for THIS loop to do to them), but a generation whose delete the work budget cut + /// short does NOT: the loop stops there, so the cursor never strands a partially-drained prefix + /// behind it. It is a monotone high-water cursor, NOT a proof that everything below it is gone. + if (adopted_generation > keep) + { + const uint64_t prune_floor = adopted_generation - keep; /// prune generations <= prune_floor + uint64_t g = next.snap_pruned_through + 1; + uint64_t pruned = 0; + for (; g <= prune_floor && pruned < kMaxPrunePerRound; ++g, ++pruned) + { + /// A generation whose run the LIVE adopted seal still + /// references (reference-parent carry: an idle shard's current run physically lives at an + /// older generation's key) must NOT be reclaimed — deleting it would strand the live seal's + /// ref. Skip its prefix delete; the run stays alive as long as it is referenced. NOTE the + /// cursor still advances past this skipped generation (see the `g - 1` cursor note above), so + /// the wholesale prune NEVER revisits it once it is behind the cursor. LEAK-FREEDOM therefore + /// rests on the post-CAS hand-off in `runRegularRound`: the round that finally moves the + /// ref OFF this generation (a later delta writes a fresh run) wholesale-deletes this whole + /// prefix right after its CAS. So every formerly-referenced generation is eventually FULLY + /// reclaimed — either here (if the ref moved off before the cursor reached it, WholesalePrune* + /// test) or by the hand-off (if the cursor passed it while still referenced, HandOffDeletes* + /// test). Until the ref moves it persists safely (bounded: one small run per shard). + if (referenced_generations.contains(g)) + { + LOG_TRACE(logger, + "CAS GC prune: retaining generation {} — still referenced by the live adopted seal", + g); + continue; + } + /// `bounded_remaining` is the round's remainder shared across every PRUNE + /// `deletePrefixWholesale` call this round (never `UINT64_MAX`; the post-CAS hand-off draws + /// from its own separate reserve, never this one). A generation whose prefix + /// this call cannot FULLY drain within the remaining budget must not let the cursor advance + /// past it -- `snap_pruned_through` is a monotone high-water mark this loop never revisits, + /// so stranding a partially-drained generation behind it would leak the remainder forever. + /// Stop the loop here; `g - 1` (the previous, fully-processed generation) is what gets + /// persisted below. + const uint64_t remaining = work_budget.prefixWholesaleRemaining(); + if (remaining == 0) + break; + bool fully_drained = false; + const uint64_t reclaimed = deletePrefixWholesale( + backend, layout.gcGenPrefix(g), remaining, &fully_drained); + work_budget.prefix_wholesale_objects_used += reclaimed; + if (!fully_drained) + break; + } + next.snap_pruned_through = g - 1; /// highest generation FULLY processed this round + } + + /// (2) NO per-round current-generation attempt-sweep (KISS). A previous revision LISTed the FOLD + /// generation's `gc/gen//` prefix EVERY completed round to delete non-adopted attempts with + /// `a < snap_attempt` — debris a deposed leader of the just-completed round left under its own + /// (unadopted) `lease.seq`. That per-round LIST was steady-state S3 budget spent for the RARE case + /// of a concurrent-leader collision (the GC-DISCOVERY-LIST-QUADRATIC concern), so it is removed. + /// + /// The wholesale generation-retention prune in (1) is now the SOLE reclaimer of ALL attempt debris, + /// including a deposed leader's: every artifact of generation `g` — across every attempt — lives + /// under `gc/gen//`, and the prefix-delete in (1) reclaims the whole subtree once `g` ages past + /// `keep`. Deposed-leader current-generation debris is therefore BOUNDED space (one collision leaves + /// at most a handful of small objects per generation) that waits at most `keep` completion-advances + /// to be reclaimed. This trades ~`keep` rounds of reclaim latency on (rare) concurrent-leader + /// collisions for eliminating a per-round LIST on the common (single-leader) path. When `keep == 0` + /// (keep-all / forensics mode) nothing is reclaimed by design — same as before. + (void)attempt; +} + +std::optional Gc::readFoldSeal(uint64_t generation, uint64_t attempt) +{ + if (const auto got = store->backend().get(store->layout().foldSealKey(generation, attempt))) + return decodeFoldSeal( + got->bytes, store->layout(), store->poolConfig().gc_shards, generation); + return std::nullopt; +} + +namespace +{ + +/// `Layout::parseRefObjectKey` for a key coming from a global `cas/ns/stream/` enumeration, with the one +/// refusal it can raise absorbed into the ordinary "unrecognized" answer. +/// +/// A ref object naming no life (the un-incarnated shape) is the single malformed key the parser +/// REFUSES by name instead of classifying as debris, and both global enumerations run OUTSIDE the +/// fold's catch. Letting the refusal escape one of them would not merely lose a round: GC is the only +/// thing that could ever delete the key, so every future round would die on it too, with nothing able +/// to clear it. Absorbed here, the key stays in the enumeration's raw key list, unindexed, exactly +/// like every other malformed shape, and `groupRefKeys` raises it once inside the fold's catch -- +/// louder than before (an anomaly plus `suppress_destructive`), and without the wedge. +/// +/// Only `CORRUPTED_DATA` is absorbed: any other exception is a real failure of the enumeration itself. +std::optional parseRefObjectKeyForEnumeration(const Layout & layout, const String & key) +{ + try + { + return layout.parseRefObjectKey(key); + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::CORRUPTED_DATA) + throw; + return std::nullopt; + } +} + +} + +std::vector Gc::discoverUniverse() +{ + /// One catalog GET replaces the pool-wide `LIST(cas/ns/stream/)` this used to run to + /// discover WHICH namespaces exist. `Creating` entries are excluded -- spec §3, "no publication can + /// exist" while a namespace is still being created, so there is nothing here for a discovery path to + /// walk; `Live` and `Removing` entries are both returned. `fromCatalogEntry` mints each life directly + /// from the row that is its own authority for both fields, never from a listed key (which could name + /// a DEAD incarnation of the same namespace name). + /// + /// The filter itself lives in `CasRefCatalog::liveUniverse` (review Important C) -- fsck's own + /// reachability walk needed the identical catalog-authoritative set and is not this class, so the + /// filter moved to where both can share it rather than grow a second copy that could disagree. + return CasRefCatalog::liveUniverse(store->backend(), store->layout()); +} + +bool Gc::graduationDue(const GcState & state, uint64_t current_round) +{ + /// Retired-in-snapshot: the graduation signal is read from the adopted fold seal's per-shard + /// `condemned_summary` — ZERO backend I/O beyond the single seal read. A summary distilled from this + /// generation's `kCondemned` rows says, per shard, how many entries are `delete_pending` (a graduation + /// is already published) and the oldest non-pending condemn round (one crosses the floor once + /// `condemn_round < current_round`). + if (state.snap_generation == 0) + return false; /// fresh pool: nothing condemned yet, nothing to graduate. + + /// FAIL-CLOSED: a missing / undecodable seal, or a summary that is not TOTAL over gc_shards, is + /// corrupt GC bookkeeping — force a FOLD so the round's own fail-closed path surfaces it, never a + /// silent defer (matching the fold's throw-on-missing-adopted-seal treatment). + std::optional seal; + try + { + seal = readFoldSeal(state.snap_generation, state.snap_attempt); + } + catch (...) + { + return true; /// undecodable seal => fail-closed force-fold + } + if (!seal) + return true; + for (uint64_t shard = 0; shard < state.gc_shards; ++shard) + { + const auto it = seal->condemned_summary.find(shard); + if (it == seal->condemned_summary.end()) + return true; /// summary not total over gc_shards => fail-closed force-fold + if (it->second.pending_total > 0 || it->second.oldest_nonpending_condemn_round < current_round) + return true; + } + return false; +} + +RefScanSummary Gc::enumerateRefPrefix() +{ + /// One full enumeration of `cas/ns/stream/`: the raw keys, plus a lenient per-life index of the + /// Log-kind ids among them. Lenient is deliberate — a malformed key is kept in `keys` and left + /// unindexed, so the STRICT validation (and the round-abort it can raise) happens exactly once, in + /// the fold's `groupRefKeys`. That holds for EVERY malformed shape: the one the parser refuses by + /// name is absorbed per key by `parseRefObjectKeyForEnumeration`, which is what keeps this + /// enumeration -- which runs before the fold, outside its catch -- unable to wedge the round. + const Layout & layout = store->layout(); + Backend & backend = store->backend(); + + RefScanSummary scan; + static constexpr size_t kListPageLimit = 1000; + size_t count_in_page = 0; + forEachListedKey(backend, layout.casRefsPrefix(), [&](const ListedKey & lk) + { + scan.keys.push_back(lk.key); + const auto parsed = parseRefObjectKeyForEnumeration(layout, lk.key); + if (parsed) + { + scan.listed_lives.insert(parsed->life_id); + if (parsed->kind == RefObjectKind::Log) + { + scan.logs_by_life[parsed->life_id].insert(parsed->txn_id); + RefTxnId & g = scan.max_log_by_life[parsed->life_id]; + if (g < parsed->txn_id) + g = parsed->txn_id; + } + } + if (++count_in_page == kListPageLimit) + { + count_in_page = 0; + ProfileEvents::increment(ProfileEvents::CASRefGlobalListPages); + } + }, kListPageLimit, onGcEnumerationPage); + /// The walk's `backend.list` lands at least once even for an empty/undersized final page -- + /// count it (one increment per physical LIST call). + if (count_in_page > 0 || scan.keys.empty()) + ProfileEvents::increment(ProfileEvents::CASRefGlobalListPages); + return scan; +} + +RoundInput Gc::listRefPrefix(const GcState & state) +{ + /// The round's ONE hint enumeration, followed by the parent coverage and authoritative catalog cut + /// needed to build the DEFER/FOLD walk plan. The caller computes the DEFER signal from that frozen + /// plan. A listed id absent from the later cut is dead, inert debris: it contributes no work and + /// cannot force DEFER. + RefScanSummary scan = enumerateRefPrefix(); + const CasRefCatalog::Snapshot catalog_cut = CasRefCatalog::read(store->backend(), store->layout()); + /// TEST SEAM: see `setPostHotScanCatalogReadHookForTest`. Moved into a local before invoking (the + /// same reason `create_namespace_step1_pre_read_hook_for_test` is swapped rather than called + /// directly): a hook that reassigns the member from inside its own body would otherwise reassign + /// the very `std::function` executing it. + if (post_hot_scan_catalog_read_hook_for_test) + { + std::function hook_to_run; + std::swap(hook_to_run, post_hot_scan_catalog_read_hook_for_test); + hook_to_run(); + } + catalog_cut.life_index.throwIfAmbiguous("CAS GC hot scan"); + store->reconcileRefCatalogCut(catalog_cut); + + const std::optional seal = readFoldSeal(state.snap_generation, state.snap_attempt); + if (!seal && state.snap_generation > 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC hot scan: adopted fold seal (generation {}, attempt {}) is missing", + state.snap_generation, state.snap_attempt); + if (seal) + scan.parent_ref_lives = seal->ref_lives; + + for (const NamespaceLifePhysicalId life_id : scan.listed_lives) + if (!catalog_cut.life_index.resolve(life_id)) + ++scan.dead_life_debris; + + return RoundInput{std::move(scan), catalog_cut}; +} + +RebuildReport Gc::rebuildBaseline(bool force) +{ + /// The `gc/state` disaster-recovery command. + /// DRY: the engine is the round's own bricks — one catalog cut for the universe, + /// foldManifestEdges(+1) for edge emission, foldDeltasIntoGeneration with EMPTY priors + /// (attempt-iterated for O(budget) memory), computeHeartbeatFloor for the round mint. + /// Writes ONLY the GC plane; namespace streams/state, manifests, and blobs are read-only inputs; + /// the rebuild never deletes them. + RebuildReport rep; + Backend & backend = store->backend(); + const Layout & layout = store->layout(); + + /// Read bookkeeping health before the lease (the lease acquire on an absent state CREATES a + /// bootstrap body, which must not make scenario (а) look healthy). A generation-0 ref-baseline + /// check is deliberately postponed to the sole post-LIST work cut below. + /// The prior seal, when `gc/state` claims one. It is the ONLY place holds live, so the rebuild + /// either reads it and carries every hold forward, or refuses (see the refusal below). + std::optional prior_seal; + bool healthy = false; + bool validate_generation_zero_ref_baseline = false; + { + const auto got = backend.get(layout.gcStateKey()); + /// The state's own decode stays inside its own try: an undecodable `gc/state` IS scenario (а), + /// the disaster this command exists for. The prior-seal refusal below must NOT be swallowed by + /// that catch, so the seal is read outside it. + std::optional decoded; + if (got) + { + try + { + decoded = decodeGcState(got->bytes); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// undecodable state = scenario (а) + } + } + if (decoded) + { + const GcState & st = *decoded; + healthy = true; + if (st.snap_generation == 0) + { + /// This check needs the rebuild universe. Delay it until after lease acquisition, the + /// zero-mutation generation-0 drain, the completed hot LIST and the sole fresh work cut. + /// No pre-lease catalog snapshot may become authority for successor construction. + validate_generation_zero_ref_baseline = true; + } + if (st.snap_generation > 0) + { + /// THE REFUSAL (spec r9-1). The prior seal is where every hold lives, and a rebuild + /// rewrites coverage from owner state -- so with no readable seal it would hand back a + /// baseline that LOOKS proven while silently discarding holds it cannot even enumerate. + /// The alternative, a "pool-wide hold" on the rebuilt baseline, is not representable: + /// every hold names a position the fold must fold THROUGH, and a pool-wide one would + /// need an invented position nothing could ever resolve. So the honest branch is the + /// already-safe one -- refuse, and name pool recreation. FORCE does not buy past this: + /// force means "rebuild deliberately", never "drop the holds". It THROWS rather than + /// returning `rep.refusal` because, unlike every other refusal here, no flag and no + /// retry makes it succeed. + std::optional seal; + try + { + seal = readFoldSeal(st.snap_generation, st.snap_attempt); + } + catch (const Exception & e) + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC rebuild: the prior fold seal (generation {}, attempt {}) is UNDECODABLE " + "({}), so the holds it carries cannot be read. A rebuild that dropped them would " + "bless a baseline whose frontier is unproven. GC refuses to rebuild; this pool " + "must be recreated.", + st.snap_generation, st.snap_attempt, e.message()); + } + if (!seal) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC rebuild: the prior fold seal (generation {}, attempt {}) is MISSING under " + "a gc/state that claims it, so the holds it carries cannot be read. A rebuild " + "that dropped them would bless a baseline whose frontier is unproven. GC refuses " + "to rebuild; this pool must be recreated.", + st.snap_generation, st.snap_attempt); + for (const RunRef & r : seal->blob_target_runs) + if (!backend.head(r.key).exists) + healthy = false; + prior_seal = std::move(seal); + rep.adopted_seal_generation = st.snap_generation; + } + } + + /// NO ADOPTED BASELINE IS NAMED: `gc/state` is absent, undecodable, or sits at generation 0. + /// The holds live in the SEAL, not in the pointer to it, so stopping here would make losing the + /// pointer -- the LESSER corruption -- produce a hold-free baseline, while an unreadable seal + /// refuses. That asymmetry is inverted, and it matters because holds are not re-derivable: + /// `WitnessDisappeared` names a record that is gone, so the next walk reads a clean frontier + /// and hands the namespace exactly the proof the hold exists to deny. + /// + /// So find the newest fold seal OBJECT by enumeration and carry ITS holds. This keeps the + /// pool's disaster recovery intact -- losing `gc/state` on a lived-in pool is the scenario this + /// command exists for -- while making a hold-free baseline over a pool that had holds + /// unreachable. + /// + /// An unadopted deposed-leader attempt can be the newest object, and carrying its holds + /// over-holds RATHER THAN under-holds -- but that claim needs its qualifier, because it is not + /// universal. It holds outside the lying-store corner. The exception is a deposed attempt that + /// FOLDED THROUGH a position on a read that was transient: its seal records the hold as + /// cleared, the adopted leader's seal still holds it, and adopting the deposed one loses that + /// hold. Bounded by the refusal above (a seal above the listing's maximum refuses outright) and + /// by the fact that both attempts walked the same cursor state. + if (!decoded || decoded->snap_generation == 0) + { + const auto newest = newestFoldSealRef(); + /// Absent here is the VIRGIN verdict, and it is the one path left on which a durable hold + /// can still be dropped without anything failing. It is REPORTED on the command's own row, + /// not merely logged: this command is run by hand during a disaster, and its operator is + /// exactly the person who needs to know the clean slate was inferred rather than proved. + rep.virgin_by_enumeration = !newest.has_value(); + if (newest) + { + std::optional seal; + try + { + seal = readFoldSeal(newest->first, newest->second); + } + catch (const Exception & e) + { + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC rebuild: gc/state names no adopted baseline and the newest fold seal " + "(generation {}, attempt {}) is UNDECODABLE ({}), so the holds this pool carries " + "cannot be read. A rebuild that dropped them would bless a baseline whose " + "frontier is unproven. GC refuses to rebuild; this pool must be recreated.", + newest->first, newest->second, e.message()); + } + if (!seal) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC rebuild: gc/state names no adopted baseline and the newest fold seal " + "(generation {}, attempt {}) vanished between the enumeration and the read, so " + "the holds this pool carries cannot be read. GC refuses to rebuild; this pool " + "must be recreated.", + newest->first, newest->second); + prior_seal = std::move(seal); + rep.adopted_seal_generation = newest->first; + } + /// else: no fold seal object anywhere. That is the ONE proof that dropping nothing is safe + /// -- a pool that never sealed a baseline has no hold to lose. + } + } + if (healthy && !force && !validate_generation_zero_ref_baseline) + { + rep.refusal = "gc/state and every referenced artifact are healthy — a rebuild would discard " + "live bookkeeping; re-run with FORCE to rebuild deliberately"; + return rep; + } + + /// Lease: single leader vs regular rounds and other rebuilds. On an absent state this CREATES + /// a lease-bearing bootstrap body whose token anchors our final CAS. allow_steal=false: this is a + /// manual disaster-recovery command, same reasoning as the manual GC round (runRegularRound's doc + /// comment) — though it is structurally moot here too (a fresh one-shot `this` with + /// has_observation==false always takes the non-steal branch on its one and only call), pass it + /// explicitly rather than rely on that invariant. + GcState state; + Token state_token; + if (!acquireOrRenewLease(state, state_token, /*allow_steal=*/false)) + { + rep.refusal = "another GC leader holds the lease"; + return rep; + } + + /// Healthy `FORCE REBUILD` shares the same parent-authorized barrier as an ordinary round. A + /// damaged-state rebuild has no adopted parent (`snap_generation == 0`) and the barrier performs + /// zero catalog mutations. Only after that distinction is resolved do we complete the hot LIST and + /// take the sole fresh work cut. + const CatalogLifecycleReconcileResult drain_result = drainCompletedRemoving(state); + for (const NamespaceLifeId & retired_life : drain_result.retired_lives) + store->invalidateRemovedCatalogLife(retired_life); + if (drain_result.authority_status != AuthorityStatus::Authoritative + || drain_result.catalog_resolution != CatalogResolution::DrainComplete) + throwCasWriteRetryLater("CAS GC rebuild lost authority before the catalog settled"); + const RefScanSummary rebuild_ref_scan = enumerateRefPrefix(); + const CasRefCatalog::Snapshot rebuild_work_catalog_cut = CasRefCatalog::read(backend, layout); + + RefScanSummary rebuild_round_scan = rebuild_ref_scan; + if (prior_seal) + rebuild_round_scan.parent_ref_lives = prior_seal->ref_lives; + const RefPlan rebuild_walk_plan = buildRefWalkPlan( + RoundInput{std::move(rebuild_round_scan), rebuild_work_catalog_cut}); + const std::vector rebuild_walk_universe = rebuild_walk_plan.lives(); + /// The exact checkpoint sample is paired with the same frozen catalog cut that chose the rebuild + /// universe. `recoverRefTableDetailedFromAuthority` deliberately has no internal catalog or + /// checkpoint read: a later cut could admit a different life or frontier than the one every other + /// part of this rebuild is using. + const CheckpointWitnesses rebuild_checkpoints = readCheckpointWitnesses({}, rebuild_walk_plan.catalogCut()); + + if (validate_generation_zero_ref_baseline) + { + /// A generation-0 state is healthy only when no table proves that a now-lost cursor cleaned + /// covered logs. The check consumes the same catalog-built universe as reconstruction; it does + /// not own an earlier catalog cut or a second admission rule. + for (const NamespaceLifeId & life : rebuild_walk_universe) + { + std::vector table_keys; + forEachListedKey(backend, layout.namespaceStreamPrefix(life), + [&](const ListedKey & lk) { table_keys.push_back(lk.key); }, 1000, onGcEnumerationPage); + std::map grouped; + try + { + grouped = groupRefKeys(layout, table_keys); + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::CORRUPTED_DATA) + throw; + healthy = false; + break; + } + const auto grouped_it = grouped.find(life.incarnation); + if (grouped_it != grouped.end() && !grouped_it->second.snapshots.empty() + && (grouped_it->second.logs.empty() + || grouped_it->second.snapshots.back() < grouped_it->second.logs.front())) + { + healthy = false; + break; + } + } + if (healthy && !force) + { + rep.refusal = "gc/state and every referenced artifact are healthy — a rebuild would discard " + "live bookkeeping; re-run with FORCE to rebuild deliberately"; + return rep; + } + } + + /// Numbering, part 1: generation above ANY surviving gc/gen prefix (putDeterministicArtifact + /// must never collide with debris of the lost era). + uint64_t max_gen = state.snap_generation; + { + const String gen_prefix = layout.gcGenPrefix(0); + const String top = gen_prefix.substr(0, gen_prefix.size() - 2); /// ".../gc/gen/" + forEachListedKey(backend, top, [&](const ListedKey & k) + { + const size_t from = top.size(); + const size_t slash = k.key.find('/', from); + if (slash == String::npos) + return; + try + { + max_gen = std::max(max_gen, static_cast(std::stoull(k.key.substr(from, slash - from)))); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// Foreign key shape under `gc/gen` is debris, not a numbering input. + } + }, 1000, onGcEnumerationPage); + } + const uint64_t generation = max_gen + 1; + const uint64_t budget = rebuild_edge_budget_override ? rebuild_edge_budget_override + : store->poolConfig().rebuild_edge_budget; + + /// Per-gc-shard attempt-iterated fold state: batch k folds with attempt k and the previous + /// attempt's runs as priors; the FINAL attempt's runs go into the seal. + const uint64_t gc_shards = state.gc_shards ? state.gc_shards : store->poolConfig().gc_shards; + std::vector> buckets(gc_shards); + std::vector> prior_runs(gc_shards); + std::vector attempt_of(gc_shards, 0); + /// The fold is EDGE-ONLY here: a rebuild condemns nothing (spec §7, and the deletion below), so no + /// condemn round is stamped and no head source is supplied. `current_round` 0 graduates nothing and + /// `condemn_round` 0 with an empty `head_blob` mints no `kCondemned` row -- this call is + /// `foldDeltasIntoGeneration`'s pure edge form. + auto flush_shard = [&](uint64_t shard) + { + if (buckets[shard].empty()) + return; + std::vector out; + foldDeltasIntoGeneration(backend, layout, prior_runs[shard], generation, ++attempt_of[shard], + shard, std::move(buckets[shard]), out, + /*current_round*/0, /*condemn_round*/0, /*head_blob*/{}, + /*peek_head*/{}, /*confirm_condemned_marker*/{}, + /*out_retired*/nullptr, /*suppress_destructive*/false, + /// Probe B2 does not apply to the rebuild: it derives edges from raw + /// owner STATE, not from a stream of ref transactions, so there is no + /// transaction whose deltas could go unapplied and no fold cursor to + /// advance past one. Every delta it emits carries ordinal 0. + /*out_applied_by_txn_ordinal*/nullptr); + buckets[shard].clear(); + prior_runs[shard] = std::move(out); + }; + auto route_deltas = [&](std::vector & deltas) + { + rep.edges += deltas.size(); + for (BlobDelta & d : deltas) + { + const uint64_t shard = blobShard(d.ref, gc_shards); + buckets[shard].push_back(std::move(d)); + if (buckets[shard].size() >= budget) + flush_shard(shard); + } + deltas.clear(); + }; + + /// `rebuild_walk_plan` was frozen immediately after the completed hot LIST and sole fresh catalog + /// cut. Listed ids absent from that later cut are inert dead-life debris and cannot mint work or + /// refuse reconstruction. + std::set seen_ns; + std::set owned_manifest_keys; + CasFoldSeal seal; + seal.generation = generation; + seal.parent_generation = state.snap_generation; + seal.ref_lives = rebuild_walk_plan.successorFoldStates(); + /// Life ids whose hold this rebuild MINTED (see `minted_here`); they are stamped with the retry + /// round once it is known, and nothing else is touched. + std::set minted_hold_lives; + uint64_t max_fence_round = 0; + std::map mf_cleanup_unused; + + for (const NamespaceLifeId & life : rebuild_walk_universe) + { + const RootNamespace & ns = life.ns; + seen_ns.insert(ns.string()); + ++rep.shards; + + /// Recover from the exact catalog row and exact checkpoint paired with this plan. A visible + /// log above `committed_through` is not logical history yet, and a Live/Removing row without a + /// readable checkpoint has no bounded recovery frontier; both cases must refuse rather than + /// letting a stream LIST decide what this baseline protects. + const auto entry_it = std::lower_bound( + rebuild_walk_plan.catalogCut().catalog.entries.begin(), rebuild_walk_plan.catalogCut().catalog.entries.end(), ns, + [](const CatalogEntry & entry, const RootNamespace & needle) { return entry.ns < needle; }); + chassert(entry_it != rebuild_walk_plan.catalogCut().catalog.entries.end()); + chassert(entry_it->ns == ns); + chassert(entry_it->incarnation == life.incarnation); + if (const auto bad_checkpoint = rebuild_checkpoints.undecodable.find(ns.string()); + bad_checkpoint != rebuild_checkpoints.undecodable.end()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC rebuild: catalog life {} has an undecodable checkpoint: {}", ns.string(), bad_checkpoint->second); + std::optional checkpoint; + if (const auto checkpoint_it = rebuild_checkpoints.recovery_checkpoints.find(ns.string()); + checkpoint_it != rebuild_checkpoints.recovery_checkpoints.end()) + checkpoint = checkpoint_it->second; + const RecoveredRefTable recovered = recoverRefTableDetailedFromAuthority( + backend, layout, *entry_it, checkpoint); + const RefTableState & st = recovered.state; + + RefCoverage cov; + cov.classification = 2; /// Folded (full coverage) unless a bodiless precommit clamps + cov.last_folded_ref_id = st.getGreatestApplied(); + /// Whether the hold on this row was minted BY THIS REBUILD (and so still owes a retry round) + /// rather than carried from the prior seal. Tracked explicitly instead of by looking for a + /// `next_retry_round` of 0: 0 is a perfectly good wire value, and a carried hold that happened + /// to hold it would have its backoff silently rewritten by the stamping pass. + bool minted_here = false; + + std::vector deltas; + + /// Committed owners: a missing/invalid body under a committed ref is DATA LOSS the rebuild must + /// not bless (INV_NO_DANGLE) -- refuse. + for (const auto [ref_name, row] : st.getCommitted()) + { + const ManifestId id{ns, row.manifest_ref}; + owned_manifest_keys.insert(layout.manifestKey(id)); + if (!foldManifestEdges(id, +1, deltas, mf_cleanup_unused, /*txn_ordinal=*/0)) + { + rep.refusal = "committed ref '" + ns.string() + "/" + ref_name + + "' names a missing or invalid part manifest — that is DATA LOSS the rebuild " + "must not bless; run fsck forensics first"; + return rep; + } + ++rep.committed_refs; + } + + /// Live precommits: a present body contributes edges; a bodiless one is non-activating and clamps + /// (the fold barrier -- the first regular round folds it once the body lands). + for (const auto & [ref_name, manifest_ref] : st.getPrecommits()) + { + const ManifestId id{ns, manifest_ref}; + owned_manifest_keys.insert(layout.manifestKey(id)); + if (foldManifestEdges(id, +1, deltas, mf_cleanup_unused, /*txn_ordinal=*/0)) + ++rep.live_precommits; + else + { + /// A bodiless live precommit leaves the rebuilt baseline INCOMPLETE for this namespace: + /// nothing can enumerate the blobs it pins, so nothing can protect them. That is a + /// durable hold, not a report field -- and it is the one hold the rebuild has to invent + /// a position for, because it derives edges from owner STATE and cannot name the log + /// that introduced the precommit (that log is below the rebuilt cursor and no round + /// re-reads it). It holds at the FIRST POSITION THE NEXT ROUND WILL READ, which makes a + /// quiet namespace stay held indefinitely -- the fail-close answer while the body is + /// still missing -- and clears once the namespace makes durable progress. + /// RESIDUAL, named rather than hidden: progress unrelated to this precommit also clears + /// it, and the precommit's edges stay missing until another rebuild. + cov.classification = 4; /// Clamped + cov.hold = RefHold{.reason = HoldReason::ManifestBodyMissing, + .offending_position = RefTxnId{cov.last_folded_ref_id.writer_epoch, + cov.last_folded_ref_id.ref_sequence + 1}, + .retry_count = 0, + .next_retry_round = 0}; /// stamped below, once `round` is minted + minted_here = true; + ++rep.clamped_shards; + } + } + route_deltas(deltas); + + /// HOLDS RIDE THROUGH A REBUILD VERBATIM (spec §5/§7). The rebuild derives coverage from owner + /// state, which knows nothing about a ref-log position that would not resolve -- so without + /// this every held row would be overwritten by a clean one and the rebuilt baseline would claim + /// a frontier proof it does not have. `retry_count` and `next_retry_round` ride unchanged too: + /// the rebuild retried nothing, so it must not reset the count that says how long the namespace + /// has been stuck. The ordinary clearing rule then applies from the next round on. + if (prior_seal) + { + const auto pit = prior_seal->ref_lives.find(life.incarnation); + if (pit != prior_seal->ref_lives.end() && pit->second.coverage.hold) + { + cov.classification = 4; + cov.hold = pit->second.coverage.hold; + minted_here = false; /// a carried hold rides VERBATIM; its retry fields are not ours + } + } + if (minted_here) + minted_hold_lives.insert(life.incarnation); + + RefLifeFoldState & row = seal.ref_lives.at(life.incarnation); + row.coverage = cov; + } + rep.namespaces = seen_ns.size(); + + /// Trimmed-but-live precommits: a build alive across trim has NO journal + /// evidence; its manifests look unowned. Include edges of every manifest that is unowned AND + /// not provably build-dead (the watermark fact) — over-protect. An unowned manifest that later + /// dies without journal evidence leaks its edges until a future rebuild (documented, bounded, + /// fsck-visible); provably-dead ones stay excluded (the orphan sweep owns their bodies). + for (const String & ns_str : seen_ns) + { + const RootNamespace ns{ns_str}; + std::vector deltas; + forEachListedKey(backend, layout.manifestNamespacePrefix(ns), [&](const ListedKey & k) + { + if (owned_manifest_keys.contains(k.key)) + return; + /// The one shared manifest-path parser for the canonical hexadecimal manifest identifier, + /// also used by fsck's parseBuildPrefix and the orphan sweep's parseListedManifestObject. + const auto parsed = layout.parseManifestKey(k.key); + if (!parsed) + return; /// foreign key shape — debris + const ManifestRef & mref = parsed->ref; + if (prefixEligible(*store, ns, BuildPrefix{mref.writer_epoch, mref.build_sequence})) + return; /// provably dead — the orphan sweep's territory, never an edge + const ManifestId id{ns, mref}; + if (foldManifestEdges(id, +1, deltas, mf_cleanup_unused, /*txn_ordinal=*/0)) + { + ++rep.unowned_alive_manifests; + route_deltas(deltas); + } + /// A missing/invalid UNOWNED body is debris (no owner claims it) — skip, never refuse. + }, 1000, onGcEnumerationPage); + } + + /// A REBUILD CONDEMNS NOTHING (spec §7). + /// + /// It used to end here with a LIST of `blobs/`, condemning every listed body its traversal had not + /// reached ("pipeline blindness repair": the fold discovers candidates by TRANSITIONS to zero, so a + /// blob whose edges were already gone by rebuild time would have no row and never be reclaimed). + /// The premise was that a full traversal knows every live blob. It does not. BOTH legs of the + /// traversal above are listing-driven -- the owner replay reads the ref prefix, the trimmed-but-live + /// pass reads the manifest prefix -- so a store that omits a durable key from one enumeration hides + /// a LIVE owner, and this pass would then condemn the very blob that owner pins. That is + /// r5-finding-4: one lying enumeration, and acked data is scheduled for deletion. Hiding is not + /// hypothetical here; it is the observed `0x1430c` shape that made every ref walk arithmetic. + /// + /// THE NAMED RESIDUAL this leaves (Stage-A staging contract, register R4): a blob whose manifest no + /// longer exists anywhere is unreclaimable -- nothing can enumerate it safely -- until the + /// build/upload registry can say which uploads are in flight. It is retention, not loss, and it is + /// bounded by that registry landing. NO substitute reclamation is added in its place: any cheaper + /// rule that reclaims from an enumeration is the same vector wearing a different hat, and a + /// fallback that deletes on incomplete evidence is precisely what "fail closed" forbids. + /// + /// Numbering, part 2: the round above every surviving fence/state/generation number. + const uint64_t round = std::max({max_fence_round, state.round, max_gen}) + 1; + + /// Stamp the retry round on the holds THIS rebuild minted (the bodiless-precommit ones, left unset + /// because the round was not minted yet). Carried holds are named by no key here and are not + /// touched: their `next_retry_round` and `retry_count` ride verbatim, since a rebuild retries + /// nothing. + for (const UInt128 life_id : minted_hold_lives) + if (const auto it = seal.ref_lives.find(life_id); + it != seal.ref_lives.end() && it->second.coverage.hold) + it->second.coverage.hold->next_retry_round = round + 1; + + for (uint64_t shard = 0; shard < gc_shards; ++shard) + flush_shard(shard); /// real-edge rows only: a rebuild condemns nothing, so it seeds nothing + + for (uint64_t shard = 0; shard < gc_shards; ++shard) + for (const RunRef & r : prior_runs[shard]) + seal.blob_target_runs.push_back(r); + + /// Also fence out any dead mounts as part of the disaster-recovery pass (liveness cleanup; the + /// returned classification counts are not needed for the round mint — graduation paces on rounds). + /// Use the same threshold/`mount_obs` as the regular round. + const uint64_t ttl_ms = static_cast(store->poolConfig().mount_lease_ttl_ms.count()); + /// Share the identical formula with `claimMountAwaitingExpiry` via + /// `mountObservationThresholdMs` -- see its doc comment (CasServerRoot.h). + const uint64_t stable_threshold_ms = mountObservationThresholdMs( + ttl_ms, static_cast(store->poolConfig().mount_renew_period.count())); + computeHeartbeatFloor(backend, layout, now_ms_fn(), mono_ms_fn(), stable_threshold_ms, mount_obs); + + /// Retired-in-snapshot: the rebuilt seal's `condemned_summary` must be TOTAL over gc_shards so a + /// subsequent regular round reads graduation/carry decisions zero-I/O off it (and its `carryParentRefs` + /// totality check does not fail closed). Every entry is EMPTY -- a rebuild condemns nothing -- and + /// the totality is still owed: an ABSENT row and a zero row are different claims, and the round that + /// reads this seal fails closed on the absent one. + for (uint64_t shard = 0; shard < gc_shards; ++shard) + seal.condemned_summary[shard] = CondemnedSummary{}; + + /// Seal (deterministic artifact) + the single state CAS. attempt = the max per-shard attempt + /// (>= 1 so the seal key is stable даже for an empty universe). + uint64_t seal_attempt = 1; + for (uint64_t a : attempt_of) + seal_attempt = std::max(seal_attempt, a); + validateFoldSealForWrite(seal, layout, gc_shards); + putDeterministicArtifact(backend, layout.foldSealKey(generation, seal_attempt), encodeFoldSeal(seal)); + + GcState next = state; + next.round = round; + next.snap_generation = generation; + next.snap_attempt = seal_attempt; + /// No retired set to publish: a rebuild condemns nothing (the deletion above), so there is nothing + /// to retire in the first place. Retired-in-snapshot removed the separate `RetiredSet` object + /// family independently of that, and the two reasons are stated apart on purpose — a future reader + /// must not take this line as evidence that REBUILD still produces condemnations somewhere. + next.manifest_sweep_cursor = ""; + const CasResult res = backend.casPut(layout.gcStateKey(), encodeGcState(next), state_token); + if (res.outcome != CasOutcome::Committed) + { + rep.refusal = "gc/state changed under the rebuild (a competing writer) — re-run"; + return rep; + } + + rep.performed = true; + rep.round = round; + rep.generation = generation; + EventEmitter{*store}.emit([&](CasEvent & e) + { + e.type = CasEventType::GcRebuild; + e.object_kind = CasEventObjectKind::Snap; + e.round = round; + e.gen = generation; + e.outcome = "performed"; + e.reason = "raw baseline rebuild from owner state (gc/state disaster recovery)"; + e.detail = {{"namespaces", std::to_string(rep.namespaces)}, + {"shards", std::to_string(rep.shards)}, + {"committed_refs", std::to_string(rep.committed_refs)}, + {"live_precommits", std::to_string(rep.live_precommits)}, + {"unowned_alive_manifests", std::to_string(rep.unowned_alive_manifests)}, + {"edges", std::to_string(rep.edges)}, + {"clamped_shards", std::to_string(rep.clamped_shards)}, + {"force", force ? "1" : "0"}}; + }); + return rep; +} + +std::vector Gc::previewDeletes() +{ + std::vector out; + + const auto state_bytes = store->backend().get(store->layout().gcStateKey()); + if (!state_bytes) + return out; + const GcState state = decodeGcState(state_bytes->bytes); + + const Layout & layout = store->layout(); + Backend & backend = store->backend(); + + /// Resolve the run objects THROUGH the adopted seal's refs, never by + /// `blobTargetRunKey` construction: with reference-parent carry a shard's current run may physically + /// live under an older generation's key, and the seal ref is the only authority for the real key. + /// Group the adopted `blob_target_runs` by the ref's explicit `shard`. Absent seal => no candidates. + std::map> runs_by_shard; + if (const auto adopted = readFoldSeal(state.snap_generation, state.snap_attempt)) + for (const RunRef & r : adopted->blob_target_runs) + runs_by_shard[r.shard].push_back(r); + + /// Scan every blob-target shard (see `retire`): a preview that only looked at shard 0 would miss the + /// zero-in-degree candidates owned by shards 1..N under `gc_shards > 1`. + for (uint64_t shard = 0; shard < state.gc_shards; ++shard) + { + const auto it = runs_by_shard.find(shard); + static const std::vector kEmptyRuns; + const std::vector & shard_runs = it != runs_by_shard.end() ? it->second : kEmptyRuns; + for (const BlobCandidate & cand : zeroInDegree(backend, shard_runs)) + { + const HeadResult observed = backend.head(layout.blobKey(cand.ref)); + if (!observed.exists) + continue; + PreviewEntry e; + e.kind = ObjectKind::Blob; + e.ref = cand.ref; + e.key = layout.blobKey(cand.ref); + e.size = observed.size; + e.reason = "unreachable"; + out.push_back(std::move(e)); + } + + /// Retired-in-snapshot: stream the SAME adopted seal runs and emit every `kCondemned` + /// sentinel row. The stored token IS the authority — NO HEAD here (a HEAD would defeat the point + /// and cost I/O). `delete_pending` rows are deleted next fold; the rest await graduation. Preview + /// stays WRITE-FREE (`openSourceEdgeRun` is a pure reader). Output is a superset of the above. + for (const RunRef & run : shard_runs) + { + SourceEdgeRunView reader = openSourceEdgeRun(backend, run.key); + String key; + String payload; + while (reader.next(key, payload)) + { + if (payload.empty() || payload[0] != kCondemned) + continue; + BlobRef ref; + UInt128 source_id; + SourceEdgeKeyCodec::parse(key, ref, source_id); // throws CORRUPTED_DATA on a malformed key + const CondemnedRow row = decodeCondemnedRow(payload); + PreviewEntry e; + e.kind = ObjectKind::Blob; + e.ref = ref; + e.key = layout.blobKey(ref); + e.size = row.size; + e.token = row.token; + e.condemn_round = row.condemn_round; + e.reason = row.delete_pending ? "delete_pending" : "awaiting_graduation"; + out.push_back(std::move(e)); + } + /// Whole-file seal-checksum: verify the drained run before its condemned + /// rows are trusted in the preview. Fail-closed on mismatch. + reader.verifyAgainst(run.checksum); + } + } + return out; +} + +void Gc::rememberObservation(const GcLease & lease) +{ + has_observation = true; + last_seen_owner = lease.owner; + last_seen_seq = lease.seq; +} + +void Gc::pulseHeartbeat(Pool & store, UInt128 gc_id) +{ + const String key = store.layout().gcHbKey(); + const auto got = store.backend().get(key); + GcHeartbeat hb; + std::optional expected; + if (got) + { + hb = decodeGcHeartbeat(got->bytes); + expected = got->token; + } + hb.owner = gc_id; + ++hb.hb_seq; + store.backend().casPut(key, encodeGcHeartbeat(hb), expected); +} + +bool Gc::acquireOrRenewLease(GcState & state, Token & state_token, bool allow_steal) +{ + const String key = store->layout().gcStateKey(); + + for (int attempt = 0; attempt < 2; ++attempt) + { + const auto got = store->backend().get(key); + + if (!got) + { + if (has_observation) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS gc/state vanished after being observed (owner {}, seq {})", + u128ToHex(last_seen_owner), last_seen_seq); + + GcState fresh; + fresh.lease = GcLease{gc_id, 1}; + /// Creation-time only: gc_shards is set ONCE on first-ever acquire; subsequent rounds read + /// the authoritative value from the persisted GcState (pool is authoritative on reopen). + /// PoolConfig carries the configured value from the disk XML. + fresh.gc_shards = store->poolConfig().gc_shards; + const CasResult acquire_res = store->backend().casPut(key, encodeGcState(fresh), std::nullopt); + if (acquire_res.outcome == CasOutcome::Committed) + { + rememberObservation(fresh.lease); + state = std::move(fresh); + state_token = acquire_res.token; + return true; + } + continue; + } + + GcState current = decodeGcState(got->bytes); + if (current.gc_shards != store->poolConfig().gc_shards) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS gc/state gc_shards {} disagrees with the pool-authoritative _pool_meta value {}", + current.gc_shards, store->poolConfig().gc_shards); + + if (current.lease.owner == gc_id) + { + GcState next = current; + ++next.lease.seq; + const CasResult renew_res = store->backend().casPut(key, encodeGcState(next), got->token); + if (renew_res.outcome == CasOutcome::Committed) + { + rememberObservation(next.lease); + state = std::move(next); + state_token = renew_res.token; + return true; + } + continue; + } + + GcHeartbeat hb; + if (const auto hb_got = store->backend().get(store->layout().gcHbKey())) + hb = decodeGcHeartbeat(hb_got->bytes); + /// Observation-based heartbeat liveness, symmetric with the frozen-lease-tuple check below: + /// ANY movement of the observed (owner, hb_seq) pair between this contender's two ticks is + /// proof of life, and `hb_seq` values are comparable only under the SAME remembered hb owner. + /// Deliberately NOT compared against `current.lease.owner`: a deposed leader's heartbeat + /// thread keeps pulsing (with `owner = itself`) until its next round resets `i_am_leader`, + /// and its writes can race out the live new leader's pulses — an hb pair that keeps moving + /// under the OLD owner's name must still read as "alive", or a live, pulsing new leader gets + /// its lease stolen. An hb owner change re-arms the window (this tick's pair is remembered + /// below); a steal happens only once the lease tuple AND the hb pair are both frozen across + /// a full window. + const bool hb_alive = has_observation + && (hb.owner != last_seen_hb_owner || hb.hb_seq > last_seen_hb_seq); + + const bool incumbent_renewed = !has_observation + || current.lease.owner != last_seen_owner + || current.lease.seq != last_seen_seq; + if (incumbent_renewed || hb_alive || !allow_steal) + { + /// Only ARM the steal-decision state (last_seen_owner/seq/hb_*) when this call is itself + /// allowed to act on a frozen tuple - i.e. the loop path. A caller with allow_steal=false + /// (manual `SYSTEM ... GC`) reads current state for its own acquire/renew/back-off decision + /// above, but must NOT record this foreign-incumbent observation: doing so would let the + /// loop's own very next tick treat THIS snapshot as one half of ITS two-observation window, + /// without the real wall-time gap (>= H) the window's safety argument requires between the + /// loop's own ticks (a manual command can land microseconds before a scheduled tick). Leaving + /// last_seen_* untouched here restores the pre-A7 invariant exactly: the frozen-tuple + /// comparison that can actually trigger a steal is only ever between two LOOP observations, + /// always >= interval apart. + if (allow_steal) + { + rememberObservation(current.lease); + last_seen_hb_owner = hb.owner; + last_seen_hb_seq = hb.hb_seq; + } + return false; + } + + GcState next = current; + next.lease.owner = gc_id; + ++next.lease.seq; + const CasResult steal_res = store->backend().casPut(key, encodeGcState(next), got->token); + if (steal_res.outcome == CasOutcome::Committed) + { + rememberObservation(next.lease); + state = std::move(next); + state_token = steal_res.token; + return true; + } + + if (const auto reread = store->backend().get(key)) + rememberObservation(decodeGcState(reread->bytes).lease); + return false; + } + + return false; +} + +CatalogLifecycleReconcileResult Gc::drainCompletedRemoving(const GcState & leased_state) +{ + if (leased_state.snap_generation == 0) + return { + .authority_status = AuthorityStatus::Authoritative, + .catalog_resolution = CatalogResolution::DrainComplete, + .retired_lives = {}, + .final_catalog_cut = std::nullopt, + .deleted = 0}; + + const std::optional parent = readFoldSeal( + leased_state.snap_generation, leased_state.snap_attempt); + if (!parent) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC pre-fold drain: adopted parent seal (generation {}, attempt {}) is missing", + leased_state.snap_generation, leased_state.snap_attempt); + + Backend & backend = store->backend(); + const Layout & layout = store->layout(); + const uint64_t admitted_generation = leased_state.lease.seq; + const auto check_fence = [&](uint64_t expected_generation) + { + if (expected_generation != admitted_generation) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS GC pre-fold drain: internal leader generation mismatch (expected {}, admitted {})", + expected_generation, admitted_generation); + const auto got = backend.get(layout.gcStateKey()); + if (!got) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS GC pre-fold drain: gc/state vanished while checking leader generation {}", + admitted_generation); + const GcState current = decodeGcState(got->bytes); + if (current.lease.owner != gc_id || current.lease.seq != admitted_generation) + return CasRefCatalog::LeaderFenceStatus::Moved; + return CasRefCatalog::LeaderFenceStatus::Held; + }; + + return CatalogLifecycleReconciler( + backend, layout, *parent, admitted_generation, check_fence).reconcile(); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h new file mode 100644 index 000000000000..51d640f39665 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -0,0 +1,1012 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// WHOSE UNIVERSE THE DESTRUCTIVE GATE IS ALLOWED TO TRUST. +/// +/// A round may destroy only while holding a FRONTIER PROOF for EVERY namespace that can hold a live +/// edge — because reachability is a property of the whole pool, not of one namespace. The proof per +/// namespace is cheap and exact (one `GET` at the cursor's arithmetic successor, `CasGc.cpp`'s intake +/// walk). What those proofs cannot supply is the SET they must cover, and only the catalog supplies it: +/// a listing may omit a durable namespace, and a sealed fold cursor names only namespaces some round +/// already folded, so neither source bounds the set on its own. +/// +/// The scenario that makes this a correctness question rather than a tidiness one: a hidden acked `+1` +/// lands in a namespace absent from BOTH the hint and `gc/state`, while a visible `-1` elsewhere drives +/// the shared blob's OBSERVABLE in-degree to zero. Every namespace the round knows about is walked to a +/// proven frontier, every probe comes back clean, and the round would delete a blob a durable committed +/// edge still owns. No amount of per-namespace proof detects it: the namespace is not in the set being +/// proved. +enum class UniversePolicy : uint8_t +{ + /// The caller supplies no universe, so `frontier_incomplete` is unconditionally true and every + /// destructive site in the round declines. Passed explicitly by a test whose SUBJECT is the + /// suppression; nothing in production selects it. + StageA_Suppressed = 0, + /// The round's universe is the catalog's own `Live`/`Removing` set, so the per-namespace frontier + /// proofs decide the gate on their own. + Authoritative = 1, + + /// The gate opens only on a COMPLETE, CATALOG-PROVEN frontier: every namespace in the universe + /// reached a proven frontier, no anomaly was recorded, and no hold rides forward. A universe of size + /// zero is unproven by DEFAULT (`frontier_namespaces == 0` alone -- an empty-BY-COUNTER universe -- + /// is never a proof; a fresh pool, a damaged catalog, and a legitimately empty pool all produce it + /// identically). It is proven only when the round's OWN hot-scan catalog cut positively demonstrates + /// it: present, token-bearing, decoded, and holding zero rows of every lifecycle state including + /// `Creating` (`FoldResult::catalog_cut_proved_empty`). Without that positive proof an emptied pool + /// would stop reclaiming permanently, because its universe can never again exceed zero. Any of these + /// failing suppresses every destructive site POOL-WIDE — narrowing the suppression to the offending + /// namespace would be unsound, because blob in-degree is a pool-wide property and no ownership + /// partition of the blob space exists. + kDefault = Authoritative, +}; + +/// The logical (GC-bookkeeping) size of a retired object: a blob subtracts the pool's fixed +/// blob_header_len (a blob OBJECT smaller than the fixed header is corrupt — CORRUPTED_DATA, +/// fail closed, never a wrapped-around size). Sizes feed cost/health accounting only — no protocol +/// decision ever reads them. +uint64_t retiredLogicalSize(ObjectKind kind, uint64_t object_size, uint64_t blob_header_len); + +/// Pure skip-unchanged decision. Returns true iff the current round may be +/// DEFERRED (re-adopt the sealed generation, no fold/delete). A round MUST fold when: enough shards +/// changed (>= fold_threshold), OR a destructive decision is due (graduation_due), OR the defer bound +/// is reached (rounds_since_last_fold >= fold_max_defer_rounds). The graduation_due term is the +/// load-bearing safety guard: no destructive decision ever runs on a not-fully-folded snapshot. +bool shouldDeferRound(size_t changed_shards, bool graduation_due, uint64_t rounds_since_last_fold, + uint64_t fold_threshold, uint64_t fold_max_defer_rounds); + +/// One anomaly the fold surfaced (a clamped cursor — a missing committed/removal body, or the fold +/// barrier on a live missing-body precommit). Surfaced to fsck/logs; recording an anomaly never throws: +/// the round records the problem and continues, while the affected shard remains conservatively clamped. +struct RoundAnomaly +{ + RootNamespace ns; + uint64_t shard = 0; + ManifestId id; + String reason; +}; + +/// Outcome of `Gc::rebuildBaseline`: either a live-conservative baseline was published, with its minted +/// numbering and coverage counters, or the rebuild was refused. Refusal is fail-closed: `refusal` says +/// why, no partial baseline is committed, and `gc/state` remains untouched. +struct RebuildReport +{ + bool performed = false; + String refusal; + uint64_t round = 0; + uint64_t generation = 0; + uint64_t namespaces = 0; + uint64_t shards = 0; + uint64_t committed_refs = 0; + uint64_t live_precommits = 0; + /// Trimmed-but-live manifests kept over-protected because ownership could not be proved. + uint64_t unowned_alive_manifests = 0; + uint64_t edges = 0; + uint64_t clamped_shards = 0; + /// This rebuild concluded FROM ENUMERATION ALONE that the pool had never sealed a baseline, and so + /// carried NO durable hold forward. It is the single route by which a hold can still be dropped + /// silently, so the hand-run disaster command REPORTS it rather than leaving it to a log line: on a + /// pool that has ever completed a GC round, a 1 here means the enumeration lied. + bool virgin_by_enumeration = false; + /// WHICH generation's fold seal this rebuild carried holds from; 0 when it carried none. The two + /// verdicts this command reaches about the pool's history are then both on its own row — whether a + /// baseline was found at all, and which one it was. Grepping that out of the logs is the thing an + /// operator should not have to do in the middle of a disaster, and it is also how a step-down past + /// a crashed newest generation becomes visible rather than invisible. + uint64_t adopted_seal_generation = 0; +}; + +/// Counters and diagnostics returned by one `runRegularRound`. These values describe work attempted or +/// observed by the round; durable `gc/state` and the fold artifacts remain the protocol state. +struct RoundReport +{ + bool acquired_lease = false; /// false => another leader is alive; nothing else was done + /// True iff this round deferred (re-adopted the sealed + /// in-degree generation instead of folding). A deferred round performs no fold, no pre-CAS + /// deletes, and no gc/state CAS -- every other RoundReport field below is meaningless/zero on it, + /// EXCEPT `round` (below), which a deferred round still sets to the honest, already-adopted round + /// number (see the defer branch of runRegularRound, CasGc.cpp) -- it just never advances it. + bool deferred = false; + uint64_t round = 0; + uint64_t candidates = 0; /// retired entries WRITTEN this round (absent candidates are skipped) + uint64_t deleted = 0; + uint64_t absent = 0; + uint64_t replaced = 0; /// 412-saves — a health metric + uint64_t spared = 0; + uint64_t manifests_deleted = 0; /// owner-removed manifest bodies deleted, distinct from blob deletes + /// Retired-cursor pipeline observability: the pipeline's per-round transitions. + size_t condemned = 0; /// entries newly condemned into the retired list this round + size_t graduated = 0; /// entries newly floor-passed (published delete_pending) this round + size_t redeleted = 0; /// pending deletes executed this round (exact-token blob deletes) + size_t fence_outs = 0; /// expired mounts fenced-out by the round's heartbeat floor + std::vector anomalies; /// fold clamps surfaced this round (never wedge the round) + + /// Retire-pipeline REMAINING sizes as of the gc/state this round's single CAS just published -- + /// unlike `condemned`/`graduated`/`redeleted` above (this round's DELTAs), these are the pipeline's + /// current outstanding totals, summed over every gc-shard's sealed `CondemnedSummary` + /// (`Formats/CasFoldSealFormat.h`). `pending_retired` = still delete_pending (graduated, awaiting the + /// exact-token delete next pass); `pending_candidates` = condemned but not yet floor-passed; + /// `pending_condemned` = their total (candidates + retired), the overall pipeline gauge. Left at 0 on + /// a `!acquired_lease`/`deferred` round -- those flags already mark the row non-authoritative. + size_t pending_candidates = 0; + size_t pending_condemned = 0; + size_t pending_retired = 0; + + /// Record a fold/recheck anomaly (a clamped cursor). Surfacing, never throwing. + void recordAnomaly(const RootNamespace & ns_, uint64_t shard_, const ManifestId & id_, const char * reason_) + { + anomalies.push_back(RoundAnomaly{.ns = ns_, .shard = shard_, .id = id_, .reason = reason_}); + } + + /// Return whether this report already contains an anomaly for the specified namespace and shard. + bool hasAnomaly(const RootNamespace & ns_, uint64_t shard_) const + { + for (const RoundAnomaly & a : anomalies) + if (a.ns.string() == ns_.string() && a.shard == shard_) + return true; + return false; + } +}; + +/// One phase of one GC round. Pure data with no Interpreters dependency -- the same discipline +/// `GcRoundLogRecord` follows, so the round engine can be instrumented without linking the system-log +/// machinery. `CasGcScheduler` converts it into one `Phase` row of +/// `system.cas_gc_log`. +/// +/// DELIBERATELY NO VERB COLUMNS. `profile_events` is this phase's delta of the ordinary process +/// counters, so `ProfileEvents['S3ListObjects']` on a `Phase` row answers "which phase burns the LIST +/// budget" with no schema invention; `metrics` carries only the semantic counts a phase computes for +/// itself and no counter can supply. +/// +/// This lives in `CasGc.h` rather than next to `GcRoundLogRecord` in `CasGcScheduler.h` because +/// `Cas::Gc` holds the sink as a member, and `CasGcScheduler.h` already includes this header -- putting +/// it there would make the include cycle. +struct GcPhaseRecord +{ + String phase; + UInt64 duration_us = 0; + std::map metrics; + std::map profile_events; /// this phase's ProfileEvents delta +}; + +/// Where a phase record goes. Installed for the duration of one round by `CasGcScheduler`; empty +/// elsewhere (a unit test driving `Gc` directly emits nothing). +using GcPhaseSink = std::function; + +/// The round's one hint enumeration of `cas/ns/stream/`, taken before the defer decision and consumed by +/// everything downstream: the DEFER signal (`changed_shards`), and — on a folding round — the strict +/// grouping the fold works from (`keys`, regrouped by `groupRefKeys`). +/// +/// It is a HINT, never a census. The fold's intake walks the ref stream ARITHMETICALLY by exact key +/// (`cursor + 1`, epoch crossings proved through the seal chain), so an id this enumeration omits is +/// still folded, and the round listing the prefix a second time would buy the intake nothing. +struct RefScanSummary +{ + size_t changed_shards = 0; /// tables with a log above their sealed cursor + std::vector keys; /// every listed key, verbatim, for the fold's strict grouping + std::set listed_lives; /// every parsed stream kind, classified by the later catalog cut + std::map> logs_by_life; + std::map max_log_by_life; + /// The validated adopted parent's rows read for this scan. They enrich the one walk plan built + /// after the catalog cut; the fold consumes that plan instead of reading them into a second one. + std::map parent_ref_lives; + std::map holds; + std::map checkpoint_observations; + /// Listed ids absent from the `RoundInput` catalog cut are inert dead-life debris. + size_t dead_life_debris = 0; +}; + +class RefPlan; +class RoundInput; +RefPlan buildRefWalkPlan(RoundInput && round_input); + +namespace tests +{ +RefPlan buildRefWalkPlanForTest(RefScanSummary ref_scan, CasRefCatalog::Snapshot catalog_cut); +class GcRoundPlanSignatureAccess; +} + +/// The one owned observation boundary between the reconciled hot LIST and every ref-plan consumer. +/// `Gc` constructs it from the completed hot LIST and its later catalog cut, then the sole builder +/// moves those observations into a `RefPlan`. The scan is never separately pairable with a plan +/// downstream. +class RoundInput +{ +public: + RoundInput(const RoundInput &) = delete; + RoundInput(RoundInput &&) = default; + RoundInput & operator=(const RoundInput &) = delete; + RoundInput & operator=(RoundInput &&) = delete; + +private: + friend class Gc; + friend RefPlan buildRefWalkPlan(RoundInput && round_input); + friend RefPlan tests::buildRefWalkPlanForTest(RefScanSummary ref_scan, CasRefCatalog::Snapshot catalog_cut); + + RoundInput(RefScanSummary ref_scan_, CasRefCatalog::Snapshot catalog_cut_) + : ref_scan(std::move(ref_scan_)), catalog_cut(std::move(catalog_cut_)) + { + } + + RefScanSummary ref_scan; + CasRefCatalog::Snapshot catalog_cut; +}; + +struct RefWalkPlanRow +{ + NamespaceLifeId life; + RefLifeFoldState fold_state; + std::optional removal_started_round; + bool has_parent_fold_state = false; + bool listed_hint = false; + std::optional checkpoint_observation; + std::optional tail_observation; +}; + +/// Diagnostic-only classification of a catalog `Removing` life against the adopted round. Returns +/// no value before the threshold, when terminal cleanup evidence exists, or for a non-removing row. +/// It performs no I/O and changes no round decision. +std::optional stuckRemovalWarning( + const RefWalkPlanRow & row, uint64_t current_round, uint64_t threshold_rounds, + const Layout & layout); + +/// A frozen catalog-built row set. Enrichment is exposed only through exact lookup; there is no +/// `operator[]`, insertion method, or mutable row map through which a hint, parent, hold, or checkpoint +/// can mint a logical life. +class RefPlan +{ +public: + RefPlan(const RefPlan &) = delete; + RefPlan(RefPlan &&) = default; + RefPlan & operator=(const RefPlan &) = delete; + RefPlan & operator=(RefPlan &&) = delete; + + bool contains(const UInt128 & life_id) const { return rows.contains(life_id); } + const RefWalkPlanRow & row(const UInt128 & life_id) const { return rows.at(life_id); } + std::set lifeIds() const; + std::vector lives() const; + const RefScanSummary & refScan() const { return ref_scan; } + const CasRefCatalog::Snapshot & catalogCut() const { return catalog_cut; } + std::map parentFoldStates() const; + std::map successorFoldStates() const; + size_t size() const { return rows.size(); } + size_t changedRows() const; + uint64_t droppedParentRows() const { return dropped_parent_rows; } + uint64_t droppedListedLives() const { return dropped_listed_lives; } + uint64_t droppedHolds() const { return dropped_holds; } + uint64_t droppedCheckpoints() const { return dropped_checkpoints; } + uint64_t droppedTails() const { return dropped_tails; } + +private: + friend RefPlan buildRefWalkPlan(RoundInput && round_input); + + RefPlan(RefScanSummary ref_scan_, CasRefCatalog::Snapshot catalog_cut_) + : ref_scan(std::move(ref_scan_)), catalog_cut(std::move(catalog_cut_)) + { + } + RefScanSummary ref_scan; + CasRefCatalog::Snapshot catalog_cut; + std::map rows; + uint64_t dropped_parent_rows = 0; + uint64_t dropped_listed_lives = 0; + uint64_t dropped_holds = 0; + uint64_t dropped_checkpoints = 0; + uint64_t dropped_tails = 0; +}; + +/// Builds the one authoritative ref-life key set used by ordinary GC and healthy `REBUILD`. +/// Adapters run only after the catalog loop has frozen that set and attach observations by `at`-style +/// lookup; unknown, absent, or `Creating` ids are counted and dropped. +/// PROBE B2 — end-to-end transaction accounting for ONE round. Round-local, never persisted. +/// +/// The naive "intended vs applied" counter pair is VACUOUS: in the intake both counts increment in the +/// same basic block, so they cannot differ. This ledger separates them across the whole pipeline +/// instead — a transaction is `committed` when the intake merges its staged buffers, `produced` when +/// it emitted at least one `BlobDelta`, and `applied` only when a shard reducer actually CONSUMED one +/// of its deltas. A delta lost between the intake and a reducer (routing across gc shards, a skipped +/// bucket, a future filter) leaves a committed+produced transaction unapplied. +/// +/// Marking happens at reducer CONSUMPTION, not at run flush: the in-degree model is a SET, so an +/// unmatched `-1` and a duplicate `+1` legitimately vanish inside the reducer and a flush-side mark +/// would fire on healthy rounds. Loss INSIDE the reducer's own set collapse is a different class, +/// covered by `CASGCUnmatchedRemoveDeltas` and by the mirror safety test in +/// `gtest_cas_holey_list_detector.cpp`; probe B2 does not claim it. +/// +/// SINGLE-THREADED: the shard reducers run sequentially on the fold thread (`Gc::fold`), so `applied` +/// needs no synchronisation. A future parallel reducer must revisit this. +struct TxnApplyLedger +{ + std::vector txns; /// ordinal -> the log id + std::vector namespaces; /// ordinal -> namespace, for the failure message + std::vector produced; /// this transaction emitted >= 1 BlobDelta + std::vector committed; /// this transaction folded fully and merged into the round buffers + /// >= 1 of this transaction's deltas was consumed by a reducer. Public and written through a raw + /// pointer by `foldDeltasIntoGeneration`: that write sits in a loop over potentially millions of + /// rows, where a `std::function` hop is not free. + std::vector applied; + + /// Open a transaction and return its round-local ordinal. + uint32_t open(const RootNamespace & ns, const RefTxnId & id) + { + const uint32_t ordinal = static_cast(txns.size()); + txns.push_back(id); + namespaces.push_back(ns.string()); + produced.push_back(0); + committed.push_back(0); + applied.push_back(0); + return ordinal; + } + void markProduced(uint32_t ordinal) { produced[ordinal] = 1; } + void markCommitted(uint32_t ordinal) { committed[ordinal] = 1; } + void markApplied(uint32_t ordinal) { applied[ordinal] = 1; } + + /// Ordinals that were committed AND produced deltas but whose deltas never reached a reducer. + /// Empty on a healthy round. + std::vector unapplied() const + { + std::vector out; + for (uint32_t i = 0; i < txns.size(); ++i) + if (committed[i] && produced[i] && !applied[i]) + out.push_back(i); + return out; + } +}; + +/// Leader-paced regular GC: one pass per round — heartbeat +/// ack floor -> fold (two-cursor merge) -> pre-CAS deletes of previously-published pending +/// entries -> single `gc/state` CAS -> post-CAS cleanup/trim, over the root-local part-manifest model. The lease is work deduplication only — +/// every step is idempotent and split-brain-safe (monotone `gc/state`, append-by-unique-path retire and +/// outcome logs, exact-token deletes). These properties mean that a stale leader can duplicate work but +/// cannot roll back state or delete a newer object incarnation; the safety argument does not depend on +/// the lease being perfectly exclusive. +/// +/// LEASE / STEAL WINDOW (deterministic — this class NEVER reads a clock). The lease lives inside +/// `gc/state` as {owner, seq} and moves only by CAS on the whole `gc/state` object. The +/// `acquireOrRenewLease` method implements the observation, renewal, and steal protocol. +/// +/// NOT thread-safe: one pacing thread drives a Gc instance. gc_id uniqueness across instances +/// (a random u128) is a CALLER obligation — duplicate ids make two leaders indistinguishable. +class Gc +{ + friend class tests::GcRoundPlanSignatureAccess; + +public: + /// `now_ms_fn` is the WALL clock (injected for tests): audit/diagnostic stamps only (e.g. the + /// heartbeat floor's `now_ms` argument) — it never gates a fence decision. `mono_ms_fn` is the + /// OBSERVATION clock: monotonic on this + /// process, injected for tests, defaults to `Pool::bootMs()`, and the ONLY clock the heartbeat + /// gate's own fence-out threshold is measured against (mirrors `claimMountAwaitingExpiry`'s + /// `mono_ms_fn`, but at heartbeat-gate granularity — one GC round is one observation tick). + /// Everything else in the round stays deterministic/clock-free. + /// + /// `log_` is the logger every round-engine log line is emitted through; pass a disk/srid-scoped + /// logger (e.g. `CasGcScheduler`'s own `log`, built from a `fmt::format("{}::...", storage_path)` + /// name) so a multi-disk process's GC logs are attributable. Defaults to the process-global + /// `getLogger("CasGc")` for callers with no natural scope of their own (tests, one-shot commands). + Gc(PoolPtr store_, UInt128 gc_id_, std::function now_ms_fn_ = {}, + std::function mono_ms_fn_ = {}, LoggerPtr log_ = nullptr); + + /// One full round. Returns acquired_lease=false (nothing else done) if another leader is alive. + /// `on_lease_acquired`, if set, is invoked ONCE, synchronously, immediately after the lease is + /// acquired/renewed and BEFORE the (potentially long) fold begins - the scheduler uses this to + /// mark itself leader and fire the first advisory heartbeat pulse right away: a new + /// leader's first round must not run unprotected for the whole fold before the pacing thread's + /// post-round bookkeeping would otherwise have set it). Never called when the lease is not held; + /// exceptions from the callback propagate like any other round failure (the caller is expected to + /// keep it advisory/non-throwing, matching pulseHeartbeat's own contract). + /// + /// `allow_steal` (default true — the paced background loop's semantics, unchanged): gates the + /// observation-window protocol's steal branch (see acquireOrRenewLease). The window's safety + /// argument requires the two observations that flag an incumbent "frozen" to be spaced by real + /// wall time (>= the heartbeat cadence H) so a live incumbent gets a chance to pulse in between — + /// a guarantee only the loop's own interval-paced ticks provide. A caller with no such guarantee + /// (e.g. a manual `SYSTEM ... GC` command, where two calls can land microseconds apart) must pass + /// `false`: it may still acquire a FREE lease or renew ITS OWN, but never executes the steal CAS — + /// dead-incumbent recovery stays the loop's job. + /// + /// `policy` is the destructive gate's universe seam — see `UniversePolicy`. Production passes + /// nothing; a test whose subject is the suppressed gate passes `StageA_Suppressed` here, which is + /// the only way to reach that posture. + RoundReport runRegularRound(std::function on_lease_acquired = {}, bool allow_steal = true, + UniversePolicy policy = UniversePolicy::kDefault); + + /// Advisory heartbeat: bump /gc/hb to {gc_id, hb_seq+1}. Best-effort (a lost CAS is + /// harmless — the next pulse retries). Touches NO Gc instance state. Static by design. + static void pulseHeartbeat(Pool & store, UInt128 gc_id); + + + /// One deletion that the next regular round would preview, with the reason it is eligible. This is + /// diagnostic data only and does not carry the durable token or authorization for a delete by itself. + struct PreviewEntry + { + ObjectKind kind = ObjectKind::Blob; + BlobRef ref{}; + String key; + uint64_t size = 0; + String reason; /// "unreachable" | "delete_pending" | "awaiting_graduation" + Token token; /// stored condemn-time token (empty for "unreachable") + uint64_t condemn_round = 0; + }; + + /// WRITE-FREE preview of the next round's deletes, derived from the DURABLE sealed in-degree + /// generation + gc/state. Diagnostic / cross-check ONLY — its output must never feed a real delete. + /// It reads the durable generation WITHOUT folding new owner events, so at NON-QUIESCENCE it can + /// OVER-REPORT a blob a since-landed publish re-referenced (the real round folds first and spares + /// it). The {preview} ⊆ {genuinely-unreachable} guarantee holds ONLY at quiescence. No CAS/delete. + std::vector previewDeletes(); + + /// Raw baseline rebuild — the `gc/state` disaster-recovery command. Recomputes + /// the in-degree snapshot from raw owner state (committed refs + live precommits + the unowned + /// not-provably-dead over-protection), mints round/generation above every surviving ack/number, + /// publishes EMPTY retired lists, and CASes gc/state. Live-conservative; fail-closed refusals. + RebuildReport rebuildBaseline(bool force); + + /// Install (or clear) the per-phase sink for the round that is about to run. `CasGcScheduler` + /// installs it around one `runRegularRound` and clears it afterwards, so the sink never outlives the + /// scheduler locals it captures. Not thread-safe, like the rest of `Gc`: one pacing thread drives one + /// instance, and the scheduler holds `gc_round_mutex` across both the install and the round. + void setPhaseSink(GcPhaseSink sink) { phase_sink = std::move(sink); } + + void setRebuildEdgeBudgetForTest(uint64_t n) { rebuild_edge_budget_override = n; } + + /// TEST SEAM: disable the round's journal trim so a folded event stays in the journal + /// across rounds — exactly the lazy-trim / partial-trim-after-crash condition under which the + /// next round's fold MUST recover the exact sealed cursor (else it re-folds the event and double-counts + /// blob in-degree). Production never calls this; trim is always enabled. + void setTrimEnabledForTest(bool enabled) { trim_enabled = enabled; } + + /// Fires once, synchronously, right after `listRefPrefix`'s hot-scan catalog `GET` + /// (`CasRefCatalog::read`) returns -- the exact instant the round's catalog cut is taken, before + /// the round does anything else with it. Lets a test land a real namespace birth (through the + /// production writer path) in the window between that cut and the round's later destructive work, + /// driving the interleaving deterministically instead of relying on real thread scheduling. Empty + /// (no-op) in production, mirroring `CasRefCatalog::setCreateNamespaceStep1PreReadHookForTest`. + void setPostHotScanCatalogReadHookForTest(std::function hook) + { + post_hot_scan_catalog_read_hook_for_test = std::move(hook); + } + + /// Abandoned-precommit cleanup belongs to the writer: it appends exact `owner_transition` removals in + /// ref logs. GC never invents a ref transition, because doing so could make the fold disagree with the + /// writer's durable ownership history. + + /// DELETE-SITE INVARIANT: the round's pre-CAS + /// redelete phase in `runRegularRound` holds the ONLY content (blob reachability) delete in + /// the whole core, restricted to previously-published delete_pending entries. + /// The recheck's manifest-body exact-token delete (after its decrements are sealed), the retired-set + /// drop, the resume path (GC metadata), dropNamespace (verbatim files), and the capability probe + /// (throwaway keys) remove non-content objects they own. Adding a second content-delete site is a + /// protocol defect. + +private: + /// Lease acquire/renew/steal per the documented observation protocol. On success `state` holds the + /// committed gc/state (with our lease) and `state_token` its backend token. `allow_steal=false` + /// suppresses only the steal CAS (see runRegularRound's doc comment) — acquiring a free lease and + /// renewing our own are unaffected. + bool acquireOrRenewLease(GcState & state, Token & state_token, bool allow_steal); + + /// Catalog-only helping barrier run immediately after lease acquisition. It validates the adopted + /// parent and delegates deterministic `Removing`-row settlement to `CatalogLifecycleReconciler`. + /// It performs no physical LIST or delete. + CatalogLifecycleReconcileResult drainCompletedRemoving(const GcState & leased_state); + + /// Run exactly one independently paced physical namespace-maintenance page. The caller supplies + /// the one round-wide destructive verdict when it exists; DEFER passes suppression because it has + /// no folded frontier verdict. This helper owns only janitor I/O and phase metrics, never lifecycle + /// transitions or the hot stream walk plan. + void runNamespaceJanitorPage( + const GcState & leased_state, bool suppress_destructive, uint64_t cleanup_evidence_rows); + + void reportStuckRemovals(const RefPlan & plan, uint64_t current_round); + + /// What one fold produced. The blob deltas are sealed + /// into a write-once generation; `fold_seal` is the durable index of WHAT WAS FOLDED (a CasFoldSeal), + /// `root_shards` the discovered universe, `mf_cleanup` the part-manifest cleanup work keyed by + /// ManifestId (owner-removed bodies whose exact-token delete is deferred until their decrements are + /// sealed), and `retired_merge` the per-gc-shard ack-floor retired-cursor outcome. + struct FoldResult + { + CasFoldSeal fold_seal; + std::vector> root_shards; + std::map mf_cleanup; + /// Bounded orphan candidates exact-read before reduce. Their source retirements ride this + /// fold's runs; their manifest tokens become deletable only after the round CAS adopts them. + ManifestSweepResult orphan_sweep; + /// Ack-floor one-pass round: the retired-cursor outcome per gc-shard (settled entries, new + /// condemnations, floor-passed pendings, and the prior pendings to delete pre-CAS). + std::vector retired_merge; + /// The round's one global ref LIST, grouped per table. Reused post-CAS for + /// ref-object cleanup (covered logs / superseded snapshots) so a second LIST is never issued. + std::map ref_tables; + + /// The round's complete immutable post-LIST catalog cut (review C3). Every later consumer in + /// the SAME round (`cleanupRefObjects`) must look a namespace up here rather than issuing an + /// independent catalog re-read, which can see a DIFFERENT answer if the namespace was dropped + /// and recreated between the fold's walk and the later call -- the exact "delete plan computed + /// under one life, applied under another" shape C3 named. + /// Keeping the full cut also preserves the distinction between an absent namespace and a + /// cataloged but non-walkable `Creating` row; reducing it to a `Live`/`Removing` map loses that + /// lifecycle fact. A namespace absent from this cut has no legitimate destructive key space; + /// never assume the Stage-A sentinel applies. + std::optional catalog_cut; + + /// The round's per-namespace decoded `_ckpt`, read ONCE by the intake walk (its second, + /// hint-independent witness) and reused post-CAS by `cleanupRefObjects` for its delete ranges -- + /// the same DRY reason `ref_tables` is carried here rather than re-listed. A namespace whose + /// `_ckpt` is present but UNDECODABLE has no entry, and the walk either HELD it or -- when it + /// offered no position to walk from -- RECORDED AN ANOMALY for it. Either way an absent entry + /// grants no cleanup authority; the shut destructive gate additionally prevents every other + /// namespace from deleting against the incomplete round. + std::map checkpoints; + /// THE DESTRUCTIVE GATE for this round, computed ONCE in `fold()` and threaded everywhere so no + /// two destructive sites can disagree about it: + /// + /// suppress_destructive = any anomaly this round + /// OR carriedHolds() is non-empty + /// OR the frontier is incomplete + /// + /// The second term is STRUCTURAL, not decorative. Every hold the fold seals also records an + /// anomaly today, so the first term happens to cover the second — but that is a coincidence of + /// the current code, not the invariant. The invariant is the HOLD SET: a hold means some + /// namespace's frontier is unproven, and no proof taken elsewhere can license destruction while + /// one stands. Reading the seal directly is what keeps a future change to anomaly recording from + /// silently opening the gate. + /// + /// STAGE B'S NARROWING OF THIS GATE TO PER-NAMESPACE MUST CARRY TWO THINGS, NOT ONE. The first + /// is the hold-set term just described. The second is the namespace whose `_ckpt` is + /// undecodable AND which offers the walk no position to hold at: it mints no hold on purpose (a + /// fabricated `offending_position` would become a durable false witness), so its fail-close + /// rests on `recordAnomaly` plus `frontier_incomplete` and on nothing else -- the only + /// per-namespace failure in this file whose gate rests entirely on that pair. A per-namespace + /// gate that carried only the hold set would license destruction against a namespace whose own + /// checkpoint could not be read. + bool suppress_destructive = false; + + /// Whether EVERY namespace in this round's universe reached a proven frontier — see + /// `UniversePolicy` for what the universe is and why the catalog is the only source that can + /// bound it. False under `StageA_Suppressed` no matter what the per-namespace probes found. + bool frontier_complete = false; + + /// Whether the round's OWN hot-scan catalog cut (`catalog_cut`, the same `GET` the walk plan + /// already paid for -- never a second one) is itself the positive proof that the universe is + /// empty: a present, token-bearing, successfully decoded catalog with zero rows of ANY + /// lifecycle state, `Creating` included. `frontier_namespaces == 0` alone is NOT this proof -- + /// it is also what a catalog holding only `Creating` rows produces, and a `Creating` row is a + /// birth in progress, not an empty universe. This is the second, POSITIVE way the frontier's + /// non-vacuity term can be satisfied, alongside `frontier_namespaces > 0`. + bool catalog_cut_proved_empty = false; + + /// The universe's size and how much of it this round proved, for the `fold_ref_intake` row: a + /// round that suppressed everything owes the reader the two numbers that explain why. + uint64_t frontier_namespaces = 0; + uint64_t frontier_proven = 0; + /// Namespaces the round KNEW about (a cursor in the adopted seal) but did not probe at all, + /// because the round's frontier-probe budget ran out first. Each one is an unproven frontier. + uint64_t frontier_unprobed_budget = 0; + + /// The reason ONE namespace ended its walk unproven. `Proven` is the absence of a reason; every + /// other value names the exit that produced it. `Unattributed` is what an exit that forgets to + /// name itself is reported as, so it surfaces as a number instead of disappearing into some + /// other bucket. A value is added here only together with the exit that sets it — an enumerator + /// no exit can reach makes the deficit claim a cause the code cannot produce. + enum class FrontierUnproven : uint8_t + { + Proven, + CheckpointUnusable, + CheckpointFrontierEmpty, + CommittedBelowCursor, + Held, + Unattributed, + }; + + /// WHY the round's unproven namespaces are unproven, one bucket each, so the buckets sum to + /// `frontier_namespaces - frontier_proven`. Without it an operator reading "N of M proven" sees + /// that the round suppressed but not which of several unrelated causes did it, and the causes + /// want opposite responses: a hold is something to chase, an exhausted probe budget is something + /// to raise. `unattributed` is expected to be zero on every round; a nonzero value means the + /// reason enumeration has stopped being exhaustive over the walk's exits. + struct FrontierDeficit + { + uint64_t checkpoint_unusable = 0; + uint64_t checkpoint_frontier_empty = 0; + uint64_t committed_below_cursor = 0; + uint64_t held = 0; + uint64_t probe_budget = 0; + uint64_t fold_aborted = 0; + uint64_t unattributed = 0; + + void count(FrontierUnproven reason); + uint64_t total() const; + /// The nonzero buckets, as `name=count` pairs, for the suppression warning. + String describe() const; + }; + + FrontierDeficit frontier_deficit; + + /// Every hold this round SEALED, as `(life id, hold)` — both the ones it detected and the + /// ones it carried from the parent seal because their offending position is still unresolved. + /// It reads the seal that is about to become durable, so it is the round's final answer rather + /// than an intermediate one. + /// + /// This is the input to the destructive-round suppression rule (`suppress_destructive` = + /// current anomalies OR every carried hold): a hold means some namespace's frontier is + /// unproven, and no frontier proof taken this round can license destruction while one stands. + /// Every hold the fold seals also records an anomaly today, so the two agree; the accessor + /// exists because that coincidence is not the invariant — the hold set is. + std::vector> carriedHolds() const + { + std::vector> out; + for (const auto & [life_id, state] : fold_seal.ref_lives) + if (state.coverage.hold) + out.emplace_back(life_id, *state.coverage.hold); + return out; + } + }; + + /// Per changed root shard, stream the one ordered + /// RootOwnerEvent journal in transition_version order and dispatch each event by comparing + /// old_binding.manifest_ref to new_binding.manifest_ref: + /// - EQUAL (an owner move, e.g. a promote Precommit->Committed at the SAME ref) => NO blob delta, + /// NO part-manifest cleanup (the activating PrecommitAdd was folded earlier — see the barrier); + /// - TRUE REMOVAL (old present, the ref not owned afterwards) => read the OLD body, emit -1 per + /// blob entry + queue the body for cleanup (an old precommit never activated emitted no edges); + /// - ACTIVATION (new present) => read the NEW body, emit +1 per blob entry, SUBJECT TO THE FOLD + /// BARRIER: do not advance the durable fold cursor past a `RootOwnerEvent` that + /// leaves a LIVE precommit binding whose manifest body is not present+valid; re-read each round. + /// 404 RULE: a body that is PRESENT-but-invalid (ref/namespace mismatch) is genuine corruption => + /// CORRUPTED_DATA (hard). A MISSING body (404) is handled by where it appears: a precommit + /// activation new missing body => no edges + barrier holds the cursor; a committed/promote new + /// missing body or a true-removal old body missing at removal-fold => fail-closed FOR THAT DECISION + /// (clamp the shard's last_folded_ref_id below it, record the anomaly, stop folding THIS shard) — + /// never guess a delta and never wedge the round on a missing body. + /// On success `state` carries the committed snap_generation and `state_token` the committed gc/state + /// token. The committed pair is THREADED into retire, never re-read (zombie-steal protection). + /// Round-paced graduation: `current_round` (= state.round + 1, the SAME basis condemn_round is + /// stamped at) is the threshold the fold's two-cursor merge graduates/condemns against — an entry + /// graduates once `condemn_round < current_round`, i.e. it survived at least one full round after + /// being condemned. The fold no longer CASes gc/state — it sets (snap_generation, snap_attempt) + /// in-memory; the SINGLE round CAS commits them. + /// `walk_plan` owns the round's one enumeration of `cas/ns/stream/` (see `RefScanSummary`) and + /// its catalog cut; the fold regroups those keys strictly rather than listing the prefix again. + FoldResult fold(GcState & state, Token & state_token, RoundReport & report, uint64_t current_round, + const RefPlan & walk_plan, UniversePolicy policy, + /// One instance for the WHOLE round, owned by `runRegularRound` and threaded through + /// every destructive-work family the round touches — see `GcRoundWorkBudget`. + GcRoundWorkBudget & work_budget); + + /// The round's `_ckpt.checkpoint` witness per namespace — the SECOND, hint-independent witness the + /// walk decides its absents against. ONE call site, in the fold, right where the hint is grouped. + /// + /// The listing cannot be the sole witness source: it is a snapshot, so a record that became durable + /// after the enumeration is invisible to that round's probes, and an absent expected-next then reads + /// as a frontier when it is really a gap. `_ckpt.checkpoint` is the namespace's own durable tail, + /// read by the fold anyway for cleanup ranges, and it decides the same question without asking the + /// listing anything — so it is read by EXACT KEY, never gated on `RefTableListing::has_ckpt`. + /// + /// It takes BOTH of the round's namespace sources because it owes a witness to both: `ref_tables` + /// is the hint, and `parent_cursors` is where a carried hold names a namespace the hint may have + /// stopped mentioning — precisely the namespace whose witness matters most. An absent `_ckpt`, and a + /// present one with no `checkpoint_snapshot_id`, both contribute no entry. + struct CheckpointWitnesses + { + /// namespace -> `_ckpt.checkpoint_snapshot_id`, for the namespaces that published one. + std::map witnesses; + + /// namespace -> `_ckpt.life_epoch`, for the namespaces that published one. Stage B (Task 4-C): + /// the walk's ONLY use is seeding `expected` for a namespace that has never folded a cursor AND + /// whose listing shows no log at all -- a `Live` catalog namespace the round has not yet touched. + /// Without this, such a namespace can never leave `expected == nullopt`, so its `while (expected)` + /// loop never runs and `frontier_proven` stays false forever, even though nothing above `{life_epoch, + /// 1}` can possibly exist yet. A namespace admitted without a `_ckpt` at all (the test-only + /// `casAdmitEntry` bridge, not `completeCreation`'s production path) has no entry here and stays + /// correctly unproven -- fail-closed on a genuinely unknown genesis, never a guessed one. + std::map life_epochs; + + /// The complete decoded checkpoint for each catalog-admitted Live/Removing life. `REBUILD` + /// passes this exact sample together with the same immutable catalog row to read-only recovery; + /// it must not re-read either authority object after its plan has been frozen. + std::map recovery_checkpoints; + + /// namespace -> the decode failure's message, for the namespaces whose `_ckpt` is PRESENT and + /// UNREADABLE. Separate from an absent entry because the two mean opposite things: an absent + /// entry says "this namespace published no checkpoint", which the walk may treat as no witness, + /// while an entry here says "this namespace HAS a checkpoint and we cannot read it", which it + /// may not — an unread witness is not an absent one. Both consumers of `witnesses` read it per + /// namespace, so the damage is confined to the namespace that owns the object: the walk holds it + /// (`HoldReason::CheckpointUndecodable`) and folds every other namespace normally. + std::map undecodable; + }; + CheckpointWitnesses readCheckpointWitnesses(const std::map & ref_tables, + const CasRefCatalog::Snapshot & catalog_cut); + + /// What ONE generation's prefix says about itself: whether the generation exists at all, and the + /// greatest attempt under it whose key is one `foldSealKey` would have produced. + struct GenerationSealProbe + { + bool generation_exists = false; + std::optional seal_attempt; + }; + GenerationSealProbe probeGenerationForSeal(uint64_t generation); + + /// The newest fold-seal OBJECT in the pool by `(generation, attempt)`, or absent when the pool has + /// never sealed a baseline. Used whenever `gc/state` names no adopted baseline: holds live in the + /// SEAL, not in the pointer to it, so losing the POINTER must not silently produce a hold-free + /// baseline while an unreadable SEAL refuses. + /// + /// The pool-wide enumeration is a HINT, never the answer: one that omitted the true newest seal + /// would hand back an older one and lose every hold detected since — the same hole one layer up. + /// Two NARROW single-generation probes above the listing's maximum ask whether it lied, and a seal + /// found there THROWS a refusal rather than being adopted; a store that misreports its own + /// enumeration during disaster recovery does not get a second guess. + /// + /// That is DETECTION, not proof, and the implementation says so at the site: the generation half of + /// the question is arithmetic (generations are dense in minting), while the attempt half is an + /// enumeration within ONE directory, because a seal key's attempt component is a lease sequence + /// number with no arithmetic successor to point-read. + /// + /// THROWS `CORRUPTED_DATA` on that refusal, and on a pool whose enumeration yielded no seal but + /// which is not provably new. Returning `nullopt` is the VIRGIN verdict — logged at WARNING with + /// its evidence enumerated and counted by `CASGCRebuildVirginByEnumeration`, because it rests on + /// enumeration alone and no point read can prove it. + std::optional> newestFoldSealRef(); + + /// Read ONE part manifest named by `id`, validate it, and append sign*(+1) blob deltas for each + /// blob entry to `deltas`. On sign<0 queue (id -> token) into mf_cleanup. Returns whether a body was + /// read+validated: false => ABSENT body (404; the caller decides per the 404 rule). A body that is + /// PRESENT but fails refMatchesBody / manifestNamespaceMatches throws CORRUPTED_DATA. + /// `txn_ordinal` stamps every delta this call pushes with the round-local ordinal of the ref + /// transaction that emitted it (probe B2 — see `TxnApplyLedger`). + bool foldManifestEdges(const ManifestId & id, int sign, std::vector & deltas, + std::map & mf_cleanup, uint32_t txn_ordinal); + + + + + /// Ref-object cleanup: delete each table's ref logs covered by BOTH the durable fold cursor and a + /// checkpoint-named validated recovery triple, and listed snapshots strictly older than that base, + /// in batches of <=1000 exact keys. A folded terminal's cleanup evidence carries its tail-covering + /// snapshot id directly on the same ref-life row. Runs post-CAS and only on a clamp-free round. + /// + /// A listed `_snap` never licenses cleanup on its own: only `folded.checkpoints` whose exact + /// same-id `_log` and `_snap` validate through `readCheckpointSnapshotBase` do. Logs and snapshots + /// are then deletable only strictly BELOW that checkpoint (and logs also at or below the durable + /// cursor). A namespace with no checkpoint base, or an invalid triple, is leak-only this pass. + void cleanupRefObjects( + const FoldResult & folded, const GcLease & adopted_lease, bool suppress_destructive, + GcRoundWorkBudget & work_budget); + + /// Emit the sweep's per-pass retention rollup, throttled (see `last_retain_rollup`). Separate from + /// the pass itself so the phase row and the log line read the SAME counters rather than two + /// independently-derived views of the page. + void reportSweepRetention(const ManifestSweepResult & result); + + /// Stage B (spec INV-3): the round's universe -- every namespace life the destructive gate may one + /// day owe a frontier proof for -- is catalog-authoritative, ONE `cas/ref_catalog` `GET` replacing + /// the pool-wide `LIST(cas/ns/stream/)` this used to run. `Creating` entries are excluded: no publication + /// can exist yet (spec §3), so there is nothing here for a discovery path to walk; `Live` and + /// `Removing` entries are both returned, each minted via `NamespaceLifeId::fromCatalogEntry` directly + /// from the row that is its own authority for both fields -- never from a listed key, which could + /// name a DEAD incarnation of the same namespace name. Fresh pool (no catalog entries yet) => empty + /// result. + /// + /// Used by the fold (as the R11-guarded frontier universe) and by `rebuildBaseline`'s + /// disaster-recovery scan. The pool-wide ref LIST (`enumerateRefPrefix`/`groupRefKeys`) remains the + /// INTRA-namespace hint -- what a namespace this call already named has listed -- never the + /// discovery source for WHICH namespaces exist. + std::vector discoverUniverse(); + + /// The two cheap pre-fold GC round-defer signals — both computed from + /// state already reachable before the fold's snapshot merge (O(retired)/O(shards), no snapshot + /// read), so `runRegularRound` can decide DEFER-vs-FOLD before paying the fold's cost. + /// + /// True iff a graduation is due this round, read ZERO-I/O from the adopted fold seal's per-shard + /// `condemned_summary` (retired-in-snapshot): `∃ shard: pending_total > 0 || + /// oldest_nonpending_condemn_round < current_round`. `snap_generation == 0` (fresh pool) => false. + /// This is the load-bearing safety signal: it forces a fold before any + /// destructive decision. FAIL-CLOSED: a missing / undecodable seal, or a summary not TOTAL over + /// gc_shards, is corrupt GC bookkeeping — returns TRUE (forces a fold so the round's own fail-closed + /// path surfaces it); a round must never silently defer on corrupt bookkeeping. + bool graduationDue(const GcState & state, uint64_t current_round); + + /// One full enumeration of `cas/ns/stream/`: the raw keys plus a lenient per-life index of the + /// Log-kind ids among them. A malformed key lands in `keys` and is not indexed; `groupRefKeys` in + /// the fold does the strict validation and the round-abort. This never throws on a malformed key, + /// including the one shape `parseRefObjectKey` refuses by name -- see + /// `parseRefObjectKeyForEnumeration`. + RefScanSummary enumerateRefPrefix(); + + /// The round's ONE hint enumeration (`enumerateRefPrefix`), followed by its one fresh catalog cut + /// and the validated adopted-parent rows. Its `RoundInput` can only be moved into the authoritative + /// plan builder; the resulting `RefPlan` owns all three and is the only value that reaches consumers. + RoundInput listRefPrefix(const GcState & state); + + /// (`reclaimDroppedShards` was removed with the snapshot+log ref model: there is no mutable + /// per-namespace shard object to tombstone and reclaim; the perpetual janitor owns dead-life bytes.) + + /// Attempt-scoped generation retention. The sole reclaimer — bounded per round and fail-open on a benign + /// 404 (never throw during a prune — it would only wedge GC): + /// WHOLESALE generation-retention: every generation at or below the retention floor + /// (`adopted_generation - gc_snapshot_generations_to_keep`) is reclaimed by LISTing its whole + /// `gc/gen//` prefix and deleting every object — ALL attempts, including the attempt-scoped + /// retired/ and outcomes/ sets AND any deposed-leader debris under a non-adopted attempt. Walks + /// forward from `next.snap_pruned_through`, advancing it over generations fully reclaimed (persisted + /// by the round-commit CAS). + /// There is deliberately NO per-round current-generation attempt-sweep (it cost a per-round LIST for a + /// rare concurrent-leader collision, the GC-DISCOVERY-LIST-QUADRATIC concern). Deposed-leader + /// current-generation debris is bounded space that waits at most `keep` completion-advances for the + /// wholesale prune to reclaim it. keep==0 prunes nothing (keep-all forensics mode). `attempt` is the + /// adopted attempt (`next.snap_attempt`); it is currently unused (retention keys on generation alone). + /// + /// `suppress_destructive` is the round's gate: a suppressed round prunes NOTHING and leaves + /// `snap_pruned_through` where it was. The cursor must not advance either -- it is a monotone + /// high-water mark that the wholesale prune never revisits, so advancing it over a generation this + /// round declined to delete would strand that generation's whole prefix permanently. + void pruneSupersededGenerations(uint64_t adopted_generation, uint64_t attempt, GcState & next, + const std::set & referenced_generations, + bool suppress_destructive, GcRoundWorkBudget & work_budget); + + /// Read the fold seal for (generation, attempt) (nullopt when absent). Used by resume + parent-cursor reads. + std::optional readFoldSeal(uint64_t generation, uint64_t attempt); + + /// Update the remembered observation (steal protocol step 3/4). + void rememberObservation(const GcLease & lease); + + /// Submit one per-hash freshness-meta op (condemn/spare/delete) to the bounded `meta_pool`. + /// NEVER throws: `job` is wrapped in its own try/catch (an exception counts into the + /// `CASGCMetaWriteAnomaly` profile event + a log line); if scheduling itself fails (e.g. resource + /// exhaustion) the op runs inline rather than being silently lost. Callers must capture every value + /// `job` touches BY VALUE (never by reference to a loop-local like the fold's `cur_blob`, which + /// mutates across iterations while this job may still be queued). + void scheduleMetaJob(std::function job); + + /// Schedule the async per-hash condemn-marker write for a (ref, token) entering or being carried in + /// the retired set; when `writeCondemnedMeta` reports durable Condemned evidence, the in-process + /// confirmation for the exact (ref, token) is recorded (the graduation gate's fast path). A swallowed + /// write records nothing — the entry stays unconfirmed and graduation carries it (triage §3.4). + void scheduleCondemnMarkerWrite(const BlobRef & ref, const Token & token, + uint64_t condemn_round, uint64_t size); + + /// The in-process condemn-marker confirmation registry (see `condemn_markers_confirmed`). + void noteCondemnMarkerDurable(const BlobRef & ref, const Token & token); + bool condemnMarkerConfirmedInProcess(const BlobRef & ref, const Token & token); + void forgetCondemnMarker(const BlobRef & ref, const Token & token); + + PoolPtr store; + /// Where `GcPhaseTimer` sends one record per GC phase. Empty unless a `CasGcScheduler` installed one + /// for the current round, in which case every phase of that round emits a row. + GcPhaseSink phase_sink; + UInt128 gc_id{}; /// this leader's identity (random u128, never 0) + /// Every round-engine log line goes through this logger (see the constructor's `log_` doc + /// comment) so multi-disk processes can attribute GC logs to the disk/srid that produced them. + LoggerPtr logger; + uint64_t rebuild_edge_budget_override = 0; /// tests force tiny batches + std::function now_ms_fn; /// wall-clock ms; injected (tests), defaults to system_clock + /// The heartbeat gate's own observation clock — + /// monotonic on this process, injected (tests), defaults to `Pool::bootMs()`. Never compared + /// against another node's clock; see `computeHeartbeatFloor`. + std::function mono_ms_fn; + bool trim_enabled = true; /// TEST SEAM ONLY: production always trims; see setTrimEnabledForTest + /// TEST SEAM ONLY: see setPostHotScanCatalogReadHookForTest. Empty in production. + std::function post_hot_scan_catalog_read_hook_for_test; + /// Leader-local, in-memory count of consecutive deferred + /// rounds since the last FOLD. NOT persisted (a fresh/stolen leader starts at 0 -- conservative: + /// it may fold one round sooner than a long-lived leader would, never later). Reset to 0 whenever + /// a round folds; incremented on every DEFER. Bounds batching via `gc_fold_max_defer_rounds`. + uint64_t rounds_since_last_fold_ = 0; + + /// the contender's observation window (steal protocol) + bool has_observation = false; + UInt128 last_seen_owner{}; + uint64_t last_seen_seq = 0; + /// Heartbeat pair observed alongside the lease (gates the steal): a steal requires the lease + /// tuple AND this (owner, hb_seq) pair to be frozen across a full window. `hb_seq` is compared + /// only when the remembered hb owner matches; an owner change counts as movement (alive). + UInt128 last_seen_hb_owner{}; + uint64_t last_seen_hb_seq = 0; + + /// The heartbeat gate's cross-round, per-srid + /// write-token observation (`computeHeartbeatFloor`'s `obs` argument). In-memory only — a new + /// leader (after a steal, or a process restart) starts empty, which only delays fencing an + /// already-dead mount by one extra round (safe: never fences early). + MountObservationMap mount_obs; + + /// Throttle for the orphan sweep's retention rollup. The premise retains on any pass whose epochs are + /// not closed-and-folded, so an unconditional `LOG_INFO` would repeat the same sentence for as long as + /// that lasts and train operators to filter out the channel carrying the answer they need. It is reported when the + /// verdict CHANGES (a different top reason class, or a different count), and otherwise re-stated + /// every `kRetainRollupRepeatPasses` passes so a newly-arrived operator is never left with silence + /// they would have to read as "nothing to report". Leader-owned and in-memory: a fresh leader + /// re-states once, which is the harmless direction. + static constexpr uint64_t kRetainRollupRepeatPasses = 64; + std::optional> last_retain_rollup; + uint64_t retain_rollup_passes_since_report = 0; + + /// Bounded pool for the round's per-hash freshness-meta writes (condemn/spare/delete); + /// sized from `PoolConfig::gc_meta_pool_size` (constructed in the ctor, after the null/id checks -- + /// never touches a possibly-null `store` at member-init time). A `unique_ptr` (not a plain member) + /// so construction can happen in the ctor body, after validating `store`. + std::unique_ptr meta_pool; + + /// Cumulative counts of the per-hash freshness-meta jobs this leader handed to `meta_pool` and of + /// those that finished. The `meta_pool_wait` phase reports the ROUND's deltas: that phase's work runs + /// on other threads, so its `ProfileEvents` delta is empty BY CONSTRUCTION, and these two numbers are + /// what distinguish "the queue was deep" from "the endpoint was slow" when read next to its duration. + /// Atomic because a pool thread increments `meta_jobs_completed_` while the round thread reads it. + std::atomic meta_jobs_scheduled_{0}; + std::atomic meta_jobs_completed_{0}; + + /// In-process confirmations of durable condemn-marker writes, keyed (blob, exact incarnation-token + /// value): inserted by `scheduleCondemnMarkerWrite`'s completion (and the rebuild's synchronous + /// marker publish) when `writeCondemnedMeta` reports durable Condemned evidence; consulted by the + /// graduation gate (the delete-authorizing edge, triage §3.4); pruned when the entry settles + /// (redelete / spare / supersede). In-memory only — after a restart or leader change the graduation + /// gate falls back to a synchronous `loadMeta` re-check (the marker itself is durable). Guarded by + /// `condemn_marker_mutex`: meta-pool completions insert concurrently with the fold thread's reads. + std::mutex condemn_marker_mutex; + std::set> condemn_markers_confirmed; + + /// Probe B1's two numbers for the round: the ref-log POSITIONS the sealed coverage declares covered + /// (counted arithmetically over each namespace's cut -- not by listed ids, which under arithmetic + /// intake say nothing about what was applied), and the ref logs that actually folded. They are EQUAL + /// on every committed round (the fold throws otherwise) -- carrying them is what turns "they are + /// always equal" from an assumption into an observable property once the per-phase GC row emitter + /// reports them on the `fold_ref_intake` row. + /// Both stay 0 on a ref-folding abort, where the identity does not apply. + uint64_t logs_accounted_this_round = 0; + uint64_t logs_applied_this_round = 0; + + /// Probe B2's verdict for the round: ref transactions the round folded and merged but whose blob + /// deltas never reached a shard reducer. 0 on every COMMITTED round -- a nonzero value is + /// accompanied by the fold's fail-closed throw, so the value only ever exists as the forensic + /// record of a round that then failed. + uint64_t transactions_unapplied_this_round = 0; + +public: + /// TEST SEAM: expose catalog-based universe discovery so unit tests can assert it against a catalog + /// they construct, without driving a full round. + std::vector discoverUniverseForTest() + { + return discoverUniverse(); + } + + /// TEST SEAM: expose the two cheap pre-fold GC round-defer signals so unit tests can + /// assert them directly without driving a full round. + bool graduationDueForTest(const GcState & state, uint64_t current_round) + { + return graduationDue(state, current_round); + } + + /// Test-only access to the round's ref-prefix enumeration (its `changed_shards` is the defer signal). + RefScanSummary listRefPrefixForTest(const GcState & state) + { + const RefPlan plan = buildRefWalkPlan(listRefPrefix(state)); + RefScanSummary scan = plan.refScan(); + scan.changed_shards = plan.changedRows(); + return scan; + } + +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMaintenanceState.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMaintenanceState.cpp new file mode 100644 index 000000000000..19ff16bb956c --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMaintenanceState.cpp @@ -0,0 +1,40 @@ +#include +#include + +namespace DB::ErrorCodes +{ + extern const int CORRUPTED_DATA; +} + +namespace DB::Cas +{ + +GcMaintenanceReadResult readGcMaintenanceState(Backend & backend, const Layout & layout) +{ + const auto got = backend.get(layout.gcMaintenanceStateKey()); + if (!got) + return {.status = GcMaintenanceReadStatus::Absent, .state = std::nullopt, .token = std::nullopt, .diagnostic = {}}; + try + { + return {.status = GcMaintenanceReadStatus::Valid, .state = decodeGcMaintenanceState(got->bytes), + .token = got->token, .diagnostic = {}}; + } + catch (const DB::Exception & e) + { + if (e.code() != ErrorCodes::CORRUPTED_DATA) + throw; + return {.status = GcMaintenanceReadStatus::Corrupt, .state = std::nullopt, + .token = got->token, .diagnostic = e.message()}; + } +} + +GcMaintenanceCasResult casGcMaintenanceState( + Backend & backend, const Layout & layout, const std::optional & expected, const GcMaintenanceState & next) +{ + const CasResult result = backend.casPut(layout.gcMaintenanceStateKey(), encodeGcMaintenanceState(next), expected); + if (result.outcome == CasOutcome::Committed) + return {.outcome = GcMaintenanceCasOutcome::Committed, .token = result.token}; + return {.outcome = GcMaintenanceCasOutcome::Conflict, .token = {}}; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMaintenanceState.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMaintenanceState.h new file mode 100644 index 000000000000..6d941177d19f --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMaintenanceState.h @@ -0,0 +1,29 @@ +#pragma once +#include +#include +#include +#include + +namespace DB::Cas +{ + +enum class GcMaintenanceReadStatus : uint8_t { Absent, Valid, Corrupt }; +struct GcMaintenanceReadResult +{ + GcMaintenanceReadStatus status; + std::optional state; + std::optional token; + String diagnostic; +}; +enum class GcMaintenanceCasOutcome : uint8_t { Committed, Conflict }; +struct GcMaintenanceCasResult +{ + GcMaintenanceCasOutcome outcome = GcMaintenanceCasOutcome::Conflict; + Token token; +}; + +GcMaintenanceReadResult readGcMaintenanceState(Backend & backend, const Layout & layout); +GcMaintenanceCasResult casGcMaintenanceState( + Backend & backend, const Layout & layout, const std::optional & expected, const GcMaintenanceState & next); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcPhaseTimer.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcPhaseTimer.h new file mode 100644 index 000000000000..c0201e5fb651 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcPhaseTimer.h @@ -0,0 +1,86 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Times ONE GC phase and emits a `GcPhaseRecord` through the round's sink on destruction. +/// +/// The `ProfileEvents` delta is a plain snapshot DIFFERENCE of whatever counters container is currently +/// attached to this thread. It deliberately does NOT use a nested `ProfileEventsScope`: that RE-PARENTS +/// the thread's counters, and the round-level scope `CasGcScheduler::runRoundLogged` installs is already +/// holding that slot. A snapshot diff composes with the outer scope instead of fighting it, and degrades +/// to an empty map on a thread with no `ThreadStatus` (a bare gtest thread) -- the same degradation the +/// round-level capture already accepts and documents. +/// +/// WHAT THE DELTA DOES NOT COVER: work this phase hands to another thread. `meta_pool_wait` is the one +/// phase where that is the whole content of the phase, so its row carries explicit `jobs_scheduled` / +/// `jobs_completed` metrics instead; see its instrumentation site in `Gc::runRegularRound`. Anywhere +/// else, a phase that shows a long duration and an empty delta means the time went somewhere this +/// instrumentation cannot see, and that is a finding rather than a blank to be ignored. +/// +/// RAII AND FAILURE: the record is emitted from the destructor, so a phase that THREW still produces its +/// row while the stack unwinds. That is deliberate -- a round that failed is the round a reader most +/// needs, and it is why the correlator is `round_id` (which always exists) and not the round number +/// (which a failed round never obtains). +/// +/// Cost per phase: one `Stopwatch` and two counters snapshots, against phases that each perform network +/// I/O. Always on; no setting, deliberately -- a knob whose default nobody remembers is how +/// instrumentation degrades to silence. +class GcPhaseTimer +{ +public: + /// `sink_` is `Gc::phase_sink`, which outlives every timer of the round (the scheduler clears it only + /// after `runRegularRound` returns). `phase_` must be a string literal. + GcPhaseTimer(const GcPhaseSink & sink_, const char * phase_) + : sink(sink_), phase(phase_), attached(CurrentThread::isInitialized()) + { + if (attached) + before = CurrentThread::getProfileEvents().getPartiallyAtomicSnapshot(); + } + + GcPhaseTimer(const GcPhaseTimer &) = delete; + GcPhaseTimer & operator=(const GcPhaseTimer &) = delete; + + /// Record one phase-specific count. Overwrites a previous value for the same key. + void metric(const String & key, UInt64 value) { metrics[key] = value; } + + ~GcPhaseTimer() + { + if (!sink) + return; + GcPhaseRecord rec; + rec.phase = phase; + rec.duration_us = watch.elapsedMicroseconds(); + rec.metrics = std::move(metrics); + if (attached) + { + const auto after = CurrentThread::getProfileEvents().getPartiallyAtomicSnapshot(); + for (ProfileEvents::Event e = ProfileEvents::Event(0); e < ProfileEvents::Counters::num_counters; ++e) + { + const auto delta = after[e] - before[e]; + if (delta != 0) + rec.profile_events.emplace(String(ProfileEvents::getName(e)), static_cast(delta)); + } + } + /// Best-effort, exactly like the round-row sink: instrumentation must never break GC, and this + /// runs in a destructor that may already be unwinding a round exception. + try { sink(rec); } catch (...) {} // NOLINT(bugprone-empty-catch) + } + +private: + const GcPhaseSink & sink; + const char * phase; + bool attached; + Stopwatch watch{CLOCK_MONOTONIC}; + ProfileEvents::Counters::Snapshot before; + std::map metrics; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcScheduler.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcScheduler.cpp new file mode 100644 index 000000000000..f265444576a8 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcScheduler.cpp @@ -0,0 +1,419 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +namespace +{ + /// Non-zero events of a per-round snapshot, keyed by event name. The snapshot is already a + /// delta when produced by a ProfileEventsScope (its counters start at zero on this thread). + std::map snapshotToMap(const ProfileEvents::Counters::Snapshot & snap) + { + std::map out; + for (ProfileEvents::Event e = ProfileEvents::Event(0); e < ProfileEvents::Counters::num_counters; ++e) + { + const auto value = snap[e]; + if (value != 0) + out.emplace(String(ProfileEvents::getName(e)), static_cast(value)); + } + return out; + } +} + +CasGcScheduler::CasGcScheduler( + Cas::PoolPtr store_, + std::chrono::seconds interval_, + const String & log_name, + String disk_name_, + GcRoundLogger logger_) + : store(std::move(store_)) + , interval(interval_) + /// Keep the advisory pulse comfortably inside the follower's observation window. The lower + /// bound prevents an unusually small configured interval from creating an excessively busy + /// heartbeat worker. + , hb_interval(std::max( + std::chrono::milliseconds(50), + std::chrono::duration_cast(interval_) / 4)) + , log(getLogger(log_name)) + /// gc_id uniqueness across instances is the Gc caller obligation - a random u128 per scheduler. + , gc_id((static_cast(thread_local_rng()) << 64) | thread_local_rng()) + , disk_name(std::move(disk_name_)) + , logger(std::move(logger_)) + /// Thread this scheduler's own disk/srid-scoped logger (built from `log_name` above) into the + /// round engine, so `Gc`'s log lines carry the same scope as the round-outcome log records. + , gc(store, gc_id, {}, {}, log) +{ +} + +CasGcScheduler::~CasGcScheduler() +{ + stop(); +} + +void CasGcScheduler::start() +{ + std::lock_guard lock(mutex); + if (thread.joinable()) + return; + stopping = false; + thread = ThreadFromGlobalPool([this] { loop(); }); + hb_thread = ThreadFromGlobalPool([this] { heartbeatLoop(); }); +} + +void CasGcScheduler::stop() +{ + { + std::lock_guard lock(mutex); + stopping = true; + } + wake.notify_all(); + if (thread.joinable()) + thread.join(); + if (hb_thread.joinable()) + hb_thread.join(); + /// Clear the in-process leadership hint AFTER both workers are joined, so it is final: no round can set + /// it again (a round in flight is joined above and clears `round_in_flight` on the way out; clearing + /// `i_am_leader` before the join would race a completing round that re-sets it). The durable `gc/state` + /// lease stays authoritative and untouched here -- a restarted scheduler re-enters leadership through the + /// next round's normal acquisition, never by inheriting a stale hint. This is what lets `SYSTEM CONTENT + /// ADDRESSED GC STOP` keep the scheduler object alive (restartable) yet report "no longer leading". + i_am_leader.store(false, std::memory_order_relaxed); +} + +void CasGcScheduler::requestRoundSoon() +{ + { + std::lock_guard lock(mutex); + if (stopping || !thread.joinable()) + return; + round_requested = true; + } + wake.notify_all(); +} + +void CasGcScheduler::onLeaseAcquired() +{ + i_am_leader.store(true, std::memory_order_relaxed); + try + { + Cas::Gc::pulseHeartbeat(*store, gc_id); + } + catch (...) + { + /// Advisory, same as heartbeatLoop's own pulse: a lost pulse is harmless, the next + /// one (from heartbeatLoop, hb_interval later) retries. Must never fail the round. + tryLogCurrentException(log, "CA GC acquire-time heartbeat pulse failed (advisory; will retry)"); + } +} + +Cas::RoundReport CasGcScheduler::runRoundLogged(Cas::Gc & round_gc, GcRoundLogRecord::Trigger trigger, + std::function on_lease_acquired, bool allow_steal) +{ + using Rec = GcRoundLogRecord; + + /// Mark a round in flight for the whole body (success AND exception paths). `isQuiescent` reads this; + /// the FORGET / `GC STOP` tests use it to prove the scheduler's workers were joined (no round can be + /// mid-flight once `stop()` returned). See `isQuiescent`. + round_in_flight.store(true, std::memory_order_release); + SCOPE_EXIT({ round_in_flight.store(false, std::memory_order_release); }); + + /// Best-effort: the table row must never break GC. A throwing sink is swallowed. + auto emit = [&](const Rec & r) + { + if (logger) + { + try + { + logger(r); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + } + } + }; + + /// The correlator every row of this round attempt carries -- see `GcRoundLogRecord::round_id` for why + /// it is a fresh id rather than the round number. + const String round_id = Cas::u128ToHex( + (static_cast(thread_local_rng()) << 64) | thread_local_rng()); + + Rec start; + start.event_type = Rec::EventType::Start; + start.trigger = trigger; + start.disk_name = disk_name; + start.srid = store->poolConfig().server_root_id; + start.gc_id = Cas::u128ToHex(gc_id); + start.round_id = round_id; + emit(start); + + /// The phase sink handed to the round engine for the duration of THIS round. Same best-effort + /// discipline as `emit`: a throwing sink must never break GC. Cleared on the way out (on the + /// exception path too), so it never outlives the locals it captures -- this guard is declared after + /// them, so it is destroyed before them. + round_gc.setPhaseSink([&](const Cas::GcPhaseRecord & p) + { + Rec row = start; + row.event_type = Rec::EventType::Phase; + row.phase = p.phase; + row.phase_duration_microseconds = p.duration_us; + row.phase_metrics = p.metrics; + row.profile_events = p.profile_events; + emit(row); + }); + SCOPE_EXIT({ round_gc.setPhaseSink({}); }); + + const auto t0 = std::chrono::steady_clock::now(); + /// ProfileEventsScope requires an attached ThreadStatus (CurrentThread::get throws otherwise). + /// On the server it always is: the Scheduled path runs on a ThreadFromGlobalPool, the Manual + /// path on the query thread. In unit tests calling runOneRoundNow on a bare gtest thread there + /// is none — skip the per-round delta there rather than fail the round. + std::optional profile_scope; + if (CurrentThread::isInitialized()) + profile_scope.emplace(); + + auto collect_profile_events = [&]() -> std::map + { + if (!profile_scope) + return {}; + return snapshotToMap(*profile_scope->getSnapshot()); + }; + + Rec fin = start; + fin.event_type = Rec::EventType::Finish; + try + { + const Cas::RoundReport rep = round_gc.runRegularRound(std::move(on_lease_acquired), allow_steal); + if (rep.acquired_lease) + { + /// Keep health state per scheduler. Process-global gauges cannot distinguish multiple + /// content-addressed disks in one server process. + pending_reclaim.fetch_add( + static_cast(rep.condemned) - static_cast(rep.redeleted), + std::memory_order_relaxed); + last_success_ms.store( + static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count()), + std::memory_order_relaxed); + } + fin.outcome = !rep.acquired_lease ? Rec::Outcome::NotALeader + : rep.deferred ? Rec::Outcome::Deferred + : Rec::Outcome::Success; + fin.round = rep.round; + fin.candidates_marked = rep.candidates; + fin.objects_deleted = rep.deleted; + fin.objects_absent = rep.absent; + fin.objects_replaced = rep.replaced; + fin.objects_spared = rep.spared; + fin.manifests_deleted = rep.manifests_deleted; + fin.entries_condemned = rep.condemned; + fin.entries_graduated = rep.graduated; + fin.entries_redeleted = rep.redeleted; + fin.fence_outs = rep.fence_outs; + fin.anomalies = rep.anomalies.size(); + fin.duration_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0).count(); + fin.profile_events = collect_profile_events(); + emit(fin); + return rep; + } + catch (...) + { + fin.outcome = Rec::Outcome::Failed; + fin.error = getCurrentExceptionMessage(false); + fin.duration_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0).count(); + fin.profile_events = collect_profile_events(); + emit(fin); + throw; + } +} + +Cas::RoundReport CasGcScheduler::runOneRoundNow(GcRoundLogRecord::Trigger trigger) +{ + std::lock_guard round_lock(gc_round_mutex); + /// allow_steal=false: a manual round may acquire a FREE lease or renew ITS OWN, but must never + /// steal a live incumbent — see Cas::Gc::runRegularRound's doc comment. Dead-incumbent recovery + /// stays the loop's job (bounded ~2*interval; loop() below passes the default allow_steal=true). + const Cas::RoundReport report = runRoundLogged(gc, trigger, [this] { onLeaseAcquired(); }, /*allow_steal=*/false); + i_am_leader.store(report.acquired_lease, std::memory_order_relaxed); + return report; +} + +void CasGcScheduler::loop() +{ + setThreadName(ThreadName::CAS_GC_SCHEDULER); + size_t consecutive_backoffs = 0; + while (true) + { + { + std::unique_lock lock(mutex); + wake.wait_for(lock, interval, [this] { return stopping || round_requested; }); + if (stopping) + return; + round_requested = false; + } + /// rev.7 §3 [C1] + rev.8 §9 item 8: self-exit the pacing loop the moment the pool reaches — or is + /// being driven toward — ANY terminal state. A NATURAL terminal transition (`VanishedReplaced` after + /// a foreign pool took the prefix, or `IdentityLost` once the sentinels are gone) never calls + /// `stop()` on this scheduler: only `~Pool`/FORGET join it. Without this check the loop would tick + /// FOREVER — `acquireOrRenewLease` throws `CORRUPTED_DATA` against the vanished `gc/state` every + /// interval (the G2 zombie: an error-log line + a Failed round row each tick), and worse, after + /// `VanishedReplaced` the `allow_steal=true` rounds could STEAL the FOREIGN pool's `gc/state` lease + /// and fold/condemn/delete its objects. We also exit on a published FORGET intent + /// (`vanishedIntentPublished`, still pre-terminal) — earliest-signal discipline — and on `IdentityLost` + /// (rev.8: a fail-loud terminal state; the last G2-zombie case — eternal `CORRUPTED_DATA` retries + /// against a half-erased pool — closes with it). Clearing `i_am_leader` before returning keeps + /// `gcHealth` honest (a terminal, self-exited scheduler reports it no longer leads). The thread exits + /// its OWN loop here — no join from this context (C6-safe); `stop()`/`~CasGcScheduler` still join the + /// finished thread cleanly. + if (store->isVanished() || store->vanishedIntentPublished() + || store->lifecycle() == Cas::PoolLifecycle::IdentityLost) + { + i_am_leader.store(false, std::memory_order_relaxed); + { + std::lock_guard exit_lock(terminal_exit_mutex); + loop_exited_on_terminal_for_test.store(true, std::memory_order_release); + } + terminal_exit_cv.notify_all(); + return; + } + try + { + /// LOW/benign: if stop() flips `stopping` while we're blocked here (a concurrent manual + /// round holds gc_round_mutex), we still run one more Scheduled round once it unblocks, + /// before the next wait_for() observes `stopping` - an accepted extra round, not a + /// correctness issue. + std::lock_guard round_lock(gc_round_mutex); + + /// runRoundLogged emits the Start + Finish table rows (incl. the per-round + /// ProfileEvents delta) and rethrows on a round exception (after an Aborted Finish). + /// on_lease_acquired (onLeaseAcquired, shared with runOneRoundNow) fires the instant the + /// lease is (re)acquired, before the fold runs - a new leader's first round is otherwise + /// unprotected (i_am_leader would only flip below, AFTER the whole round returns), so a + /// follower observing the frozen (owner, seq) across two of its own ticks would steal + /// deterministically once that first round outlasts them. allow_steal defaults to true here + /// (the loop is the ONLY caller allowed to execute the steal CAS). + const Cas::RoundReport report = runRoundLogged(gc, GcRoundLogRecord::Trigger::Scheduled, [this] { onLeaseAcquired(); }); + i_am_leader.store(report.acquired_lease, std::memory_order_relaxed); + if (report.acquired_lease) + { + consecutive_backoffs = 0; + /// A deferred round never folds -- every counter below stays zero by construction + /// (RoundReport's own doc comment), so printing them through the SAME line as a folding + /// round reads as "GC is dead / stuck", indistinguishable from a round that genuinely + /// folded and found nothing. Keep the deferred case on its own, clearly-worded line + /// instead of adding a boolean to the counters line. + if (report.deferred) + LOG_DEBUG(log, "CA GC round {}: deferred (skip-unchanged; no changed shard reached the " + "fold threshold and no graduation was due; sealed generation re-adopted, no fold ran)", + report.round); + else + LOG_DEBUG(log, "CA GC round {}: candidates={} deleted={} absent={} replaced={} spared={} manifests_deleted={}", + report.round, report.candidates, report.deleted, report.absent, + report.replaced, report.spared, report.manifests_deleted); + } + else + { + /// NEVER silent: a follower backing off is the normal multi-mounter state, but a + /// pool where this scheduler never leads must be observable. The lease layer + /// handles safety, while these messages expose a liveness problem such as every + /// round using a new scheduler identity and never accumulating the observations + /// needed for dead-incumbent recovery. + ++consecutive_backoffs; + if (consecutive_backoffs % 10 == 0) + LOG_INFO(log, "CA GC: lease held by another mounter for {} consecutive ticks " + "(normal for a follower; investigate if no mounter is reclaiming)", consecutive_backoffs); + else + LOG_TRACE(log, "CA GC: lease held by another mounter (tick {})", consecutive_backoffs); + } + } + catch (...) + { + /// Idempotent round - the next tick retries; failures must never kill the pacing thread. + /// runRoundLogged already emitted the Aborted Finish row before rethrowing. + i_am_leader.store(false, std::memory_order_relaxed); + tryLogCurrentException(log, "CA GC round failed (will retry next tick)"); + } + } +} + +void CasGcScheduler::heartbeatLoop() +{ + setThreadName(ThreadName::CAS_GC_HEARTBEAT); + /// Advance the advisory heartbeat independently of round progress. A long round updates the + /// durable lease only when it completes, so without these pulses a follower could mistake a + /// live leader for a dead one during the observation window. A missed pulse is harmless because + /// the heartbeat is advisory and the next cadence retries it. + while (true) + { + { + std::unique_lock lock(mutex); + if (wake.wait_for(lock, hb_interval, [this] { return stopping; })) + return; + } + /// rev.7 §3 [C1] + rev.8 §9 item 8: self-exit on ANY terminal (or FORGET-intent) pool, same as + /// `loop()`. A terminal pool's advisory pulses would target a deleted `gc/hb` key (`IdentityLost`) or + /// a FOREIGN pool's key (`VanishedReplaced`) — stop pulsing the moment the pool goes terminal. + if (store->isVanished() || store->vanishedIntentPublished() + || store->lifecycle() == Cas::PoolLifecycle::IdentityLost) + { + { + std::lock_guard exit_lock(terminal_exit_mutex); + hb_exited_on_terminal_for_test.store(true, std::memory_order_release); + } + terminal_exit_cv.notify_all(); + return; + } + if (!i_am_leader.load(std::memory_order_relaxed)) + continue; + try + { + Cas::Gc::pulseHeartbeat(*store, gc_id); + } + catch (...) + { + tryLogCurrentException(log, "CA GC heartbeat pulse failed (advisory; will retry)"); + } + } +} + +CasGcScheduler::GcHealth CasGcScheduler::gcHealth() const +{ + GcHealth h; + h.is_leader = i_am_leader.load(std::memory_order_relaxed); + h.pending_reclaim = pending_reclaim.load(std::memory_order_relaxed); + const UInt64 last_ms = last_success_ms.load(std::memory_order_relaxed); + h.ever_succeeded = last_ms != 0; + if (last_ms != 0) + { + const UInt64 now_ms = static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count()); + h.last_success_age_seconds = now_ms > last_ms ? (now_ms - last_ms) / 1000 : 0; + } + h.wedged_namespace_count = store->wedgedRefLaneCount(); + return h; +} + +bool CasGcScheduler::waitForTerminalSelfExitForTest(std::chrono::milliseconds timeout) +{ + std::unique_lock lock(terminal_exit_mutex); + return terminal_exit_cv.wait_for(lock, timeout, [this] + { + return loop_exited_on_terminal_for_test.load(std::memory_order_acquire) + && hb_exited_on_terminal_for_test.load(std::memory_order_acquire); + }); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcScheduler.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcScheduler.h new file mode 100644 index 000000000000..2cce48399b66 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcScheduler.h @@ -0,0 +1,227 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// The decoupled, pure-data record the Disks-layer GC scheduler emits per round. It carries NO +/// Interpreters dependency: the metadata storage converts it into a +/// ContentAddressedGarbageCollectionLogElement and forwards it to the SystemLog. Keeping it a plain +/// POD lets the scheduler (and its unit tests) stay free of the system-log machinery. +struct GcRoundLogRecord +{ + /// `Phase`: one row per GC phase of the round, carrying that phase's own duration, semantic counts, + /// and `ProfileEvents` delta. Emitted between the round's `Start` and `Finish`, correlated with them + /// by `round_id`. + enum class EventType { Start, Finish, Phase }; + /// `Deferred`: the round acquired the lease and took the skip-unchanged fast path (`RoundReport::deferred`) + /// -- no fold, no pre-CAS deletes, no `gc/state` CAS. Distinct from `Success` so a reader of + /// `system.cas_gc_log` (or this scheduler's own log line) can tell a round + /// that genuinely folded and found nothing apart from one that never folded at all. + enum class Outcome { Unknown, Success, NotALeader, Failed, Deferred }; + enum class Trigger { Scheduled, Manual }; + + EventType event_type = EventType::Start; + Outcome outcome = Outcome::Unknown; /// Unknown until a round finishes + Trigger trigger = Trigger::Scheduled; + String disk_name; + String srid; /// server_root_id of the mount that ran this round (this pool's poolConfig()) + String gc_id; /// hex of the scheduler's gc_id + UInt64 round = 0; + UInt64 candidates_marked = 0; + UInt64 objects_deleted = 0; + UInt64 objects_absent = 0; + UInt64 objects_replaced = 0; + UInt64 objects_spared = 0; + UInt64 manifests_deleted = 0; /// owner-removed manifest bodies deleted, distinct from blob deletes + /// Counts for the three-stage deletion pipeline reported by `Cas::RoundReport`. + UInt64 entries_condemned = 0; /// entries newly condemned into the retired list this round + UInt64 entries_graduated = 0; /// entries newly floor-passed (published delete_pending) this round + UInt64 entries_redeleted = 0; /// pending exact-token blob deletes executed this round + UInt64 fence_outs = 0; /// expired mounts fenced-out by the round's heartbeat floor + UInt64 anomalies = 0; /// fold clamps surfaced (never wedging) this round + UInt64 duration_ms = 0; + String error; + /// On a `Start`/`Finish` row: the whole round's delta. On a `Phase` row: THAT PHASE's delta. + std::map profile_events; + + /// Correlator for every row of ONE round attempt: a fresh random hex id minted per `runRoundLogged` + /// invocation and stamped on the `Start` row, every `Phase` row, and the `Finish` row. + /// + /// DELIBERATELY NOT `round`. `round` is 0 on `Start`, is only known after the round's single + /// `gc/state` CAS on a folding round, and does not exist AT ALL on a `NotALeader` round -- and the + /// rounds a reader most needs to reconstruct are exactly the ones that never got that far. + String round_id; + /// The GC phase this row describes; empty on `Start`/`Finish`. The literals are the ones passed to + /// `Cas::GcPhaseTimer` at each phase's single instrumentation site. + String phase; + /// Wall time of this phase. MICROseconds, not milliseconds: `meta_pool_wait`, `round_commit` and + /// `parent_seal_read` are routinely sub-millisecond and the whole point of the row is seeing when + /// they are not. + UInt64 phase_duration_microseconds = 0; + /// Phase-specific semantic counts (`Phase` rows only). The per-phase verb counts ride + /// `profile_events` above instead, so no verb columns are invented. + std::map phase_metrics; +}; + +using GcRoundLogger = std::function; + +/// Paces regular content-addressed garbage-collection rounds for one pool. The scheduler does not +/// implement the GC protocol: `Cas::Gc` owns lease acquisition, work deduplication, and the +/// split-brain-safe round operations, so schedulers on different mounters may run independently. +/// This class waits for a tick, runs one round, records its result, and retries after exceptions; +/// round operations are idempotent, while `LOGICAL_ERROR` exceptions remain visible in the log. +/// +/// The background and heartbeat workers are `ThreadFromGlobalPool` instances. The background worker +/// therefore has the attached `ThreadStatus` required by `ProfileEventsScope` for per-round +/// `ProfileEvents` deltas. A manual round runs on the caller's thread; if that thread has no +/// `ThreadStatus`, the per-round delta is omitted rather than making the GC round fail. +class CasGcScheduler +{ +public: + CasGcScheduler( + Cas::PoolPtr store_, + std::chrono::seconds interval_, + const String & log_name, + String disk_name_, + GcRoundLogger logger_ = {}); + ~CasGcScheduler(); + + /// Starts the periodic round and heartbeat workers. Calling `start` more than once while the + /// scheduler is running is a no-op; after `stop`, it may be started again on the SAME instance -- + /// the persistent `gc` observer and `gc_id` are preserved, so the lease's observation-window + /// protocol continues across a stop/start (`SYSTEM CAS GC START`). + void start(); + + /// Stops both workers, wakes them if they are waiting, and joins them before returning. It is + /// safe to call `stop` when the scheduler is not running and from the destructor. It also clears + /// the in-process `i_am_leader` hint (after the joins), so a stopped scheduler reports it no longer + /// leads GC; the durable `gc/state` lease is untouched (`SYSTEM CAS GC STOP`). + void stop(); + + /// Wake the periodic worker so it starts the next ordinary round promptly. This does not create a + /// second execution path: the same worker, lease protocol, and `gc_round_mutex` remain the sole + /// scheduled-round authority. Requests coalesce into one boolean while a round is pending. + void requestRoundSoon(); + + /// Test/diagnostics hook: run ONE round synchronously on the caller's thread. Returns the round + /// report so the SYSTEM command / tests can inspect it. Emits a Start + Finish record. + Cas::RoundReport runOneRoundNow(GcRoundLogRecord::Trigger trigger = GcRoundLogRecord::Trigger::Manual); + + /// Returns per-disk GC health for `system.cas_mounts`. The fields describing + /// rounds snapshot this scheduler's state, while `wedged_namespace_count` is read live from the + /// store's ref lanes; keeping the state here avoids process-global gauges colliding across disks. + struct GcHealth + { + bool is_leader = false; + bool ever_succeeded = false; + Int64 pending_reclaim = 0; /// cumulative condemned - executed deletes (this process) + UInt64 last_success_age_seconds = 0; /// seconds since the last led round (0 if never) + UInt64 wedged_namespace_count = 0; + }; + /// Takes a consistent-enough atomic snapshot for diagnostics. The returned counters are local + /// health indicators rather than durable GC state, and the wedged-namespace count is queried + /// directly from the store. + GcHealth gcHealth() const; + + /// Whether GC is quiescent right now: TRUE iff NO round is currently in flight on this scheduler + /// (`round_in_flight` is held for the WHOLE round body via `SCOPE_EXIT` in `runRoundLogged`, on both the + /// success and the exception path). Used by the `SYSTEM CAS FORGET` / `GC STOP` tests to + /// prove the scheduler's worker threads were joined — no round can be mid-flight once `stop()` returned. + bool isQuiescent() const { return !round_in_flight.load(std::memory_order_acquire); } + + /// Test seam: force the in-flight-round flag `isQuiescent` reads, so a test can drive + /// "running round => not quiescent" without spinning up a real round against a live backend. + void setRoundInFlightForTest(bool v) { round_in_flight.store(v, std::memory_order_release); } + + /// Test seam (rev.7 §3 [C1]): block up to `timeout` for BOTH the pacing and heartbeat loops to have + /// SELF-EXITED via the terminal-lifecycle check — a `Vanished` pool or a published FORGET intent — as + /// opposed to exiting through `stop()`'s `stopping` flag. Returns false on timeout. Predicate-based + /// wait (no sleeps); the loops set their flag under `terminal_exit_mutex` before notifying, so there is + /// no lost-wakeup window. Lets a test prove the self-exit path fired without relying on any wall-clock + /// delay. + bool waitForTerminalSelfExitForTest(std::chrono::milliseconds timeout); + +private: + /// Waits for the configured interval, runs scheduled rounds while the scheduler is active, and + /// logs exceptions before continuing with the next tick. The round lock serializes this worker + /// with `runOneRoundNow` because the persistent `gc` object is not thread-safe. + void loop(); + + /// While this scheduler believes it owns the lease, periodically advances the advisory + /// heartbeat independently of round progress. The cadence is shorter than the lease + /// observation window, so a long round does not look like a dead leader to another scheduler. + /// Heartbeat failures are advisory and are retried on the next cadence. + void heartbeatLoop(); + + /// Runs synchronously when `Cas::Gc` acquires or renews the lease, before the round's potentially + /// long fold begins. It marks this scheduler as the heartbeat owner and sends the first pulse + /// immediately; otherwise a new leader's first round could appear inactive until it returned. + /// The same hook is used by scheduled and manual rounds so both acquisition paths are protected. + void onLeaseAcquired(); + + /// Run one round through the full logging path (Start record, ProfileEventsScope, Finish + /// record). Used by BOTH loop() and runOneRoundNow. Logging is best-effort - the logger sink + /// never throws into the round. Rethrows a round exception (after emitting an Aborted Finish). + /// `allow_steal` is forwarded to `Cas::Gc::runRegularRound` verbatim (see its doc comment). + Cas::RoundReport runRoundLogged(Cas::Gc & round_gc, GcRoundLogRecord::Trigger trigger, + std::function on_lease_acquired = {}, bool allow_steal = true); + + const Cas::PoolPtr store; + const std::chrono::seconds interval; + const std::chrono::milliseconds hb_interval; /// advisory heartbeat cadence, interval / 4 with a 50 ms minimum + const LoggerPtr log; + const UInt128 gc_id; + const String disk_name; + const GcRoundLogger logger; + + /// One persistent Gc for BOTH loop() and runOneRoundNow: the lease's observation-window steal + /// protocol REQUIRES a stable observer (it compares the lease across consecutive runRegularRound + /// calls of the same instance). A throwaway per call could never recover a dead-incumbent lease. + Cas::Gc gc; + /// Serializes the manual round against the background round so the two never touch the single + /// (not-thread-safe) `gc` concurrently. Distinct from `mutex`: the loop releases `mutex` before + /// the round so stop()/heartbeatLoop are not blocked, so the round cannot hold `mutex`. + std::mutex gc_round_mutex; + + std::mutex mutex; + std::condition_variable wake; + bool stopping = false; + bool round_requested = false; /// guarded by `mutex`; coalesced external wake request + ThreadFromGlobalPool thread; + /// Set by the round worker and read by the heartbeat worker. It is only an in-process hint: the + /// durable lease remains the authority, and a failed round clears the hint before retrying. + std::atomic i_am_leader{false}; + ThreadFromGlobalPool hb_thread; + + /// Set true for the whole body of one round (`runRoundLogged`, held across the `gc_round_mutex` + /// critical section a scheduled or manual round runs under) and cleared when it returns, on the + /// success AND exception paths. Read by `isQuiescent` (the FORGET / GC-STOP join-completion signal). + std::atomic round_in_flight{false}; + + /// rev.7 §3 [C1] test-observation seam: set (under `terminal_exit_mutex`) by `loop`/`heartbeatLoop` + /// respectively when they SELF-EXIT via the terminal-lifecycle check, NOT when `stop()` flips + /// `stopping`. `waitForTerminalSelfExitForTest` waits on `terminal_exit_cv` for BOTH, so a test proves + /// the self-exit path fired without any sleep. Purely diagnostic; production behavior never reads them. + std::atomic loop_exited_on_terminal_for_test{false}; + std::atomic hb_exited_on_terminal_for_test{false}; + std::mutex terminal_exit_mutex; + std::condition_variable terminal_exit_cv; + + /// Cumulative condemned entries minus exact-token deletes completed by this scheduler while it + /// led. It is an approximate health gauge, not durable GC state. + std::atomic pending_reclaim{0}; + /// Steady-clock timestamp of the last round that acquired the lease; zero means never. + std::atomic last_success_ms{0}; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcShardPlan.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcShardPlan.cpp new file mode 100644 index 000000000000..c3b44569d074 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcShardPlan.cpp @@ -0,0 +1,65 @@ +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} +} + +namespace DB::Cas +{ + +uint64_t manifestCleanupShard(const ManifestId & id, uint64_t gc_shards) +{ + /// gc_shards >= 1 is enforced by GcState decode (CORRUPTED_DATA on 0). + /// Hash the qualified id (namespace plus all three `ManifestRef` components) using the same + /// mixing as `std::hash`. Two namespaces can legally carry the same + /// `ManifestRef` without addressing the same object, so their cleanup work must never be + /// merged. + const size_t h = std::hash{}(id); + return static_cast(h) % gc_shards; +} + +ShardReducer::ShardReducer(uint64_t shard_, uint64_t gc_shards_) + : shard(shard_), gc_shards(gc_shards_) +{ + if (gc_shards_ == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "ShardReducer: gc_shards must be >= 1"); + if (shard_ >= gc_shards_) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "ShardReducer: shard {} is out of range [0, {})", shard_, gc_shards_); +} + +bool ShardReducer::owns(const BlobRef & ref) const +{ + return blobShard(ref, gc_shards) == shard; +} + +std::vector ShardReducer::reduce(Backend & backend, const Layout & layout, + const std::vector & prior_runs, + uint64_t new_generation, uint64_t attempt, + std::vector shard_deltas, + uint64_t current_round, uint64_t condemn_round, + const std::function(const BlobRef &)> & head_blob, + const std::function(const BlobRef &)> & peek_head, + const std::function & confirm_condemned_marker, + RetiredMergeResult * out_retired, + bool suppress_destructive, + std::vector * out_applied_by_txn_ordinal, + GcRoundWorkBudget * work_budget) const +{ + std::vector out_runs; + foldDeltasIntoGeneration(backend, layout, prior_runs, new_generation, attempt, shard, + std::move(shard_deltas), out_runs, + current_round, condemn_round, head_blob, peek_head, + confirm_condemned_marker, out_retired, + suppress_destructive, out_applied_by_txn_ordinal, {}, work_budget); + return out_runs; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcShardPlan.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcShardPlan.h new file mode 100644 index 000000000000..20edc54dfb88 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcShardPlan.h @@ -0,0 +1,138 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Blob-target shard for a blob identity. +/// +/// Deterministic and total over all hashes: the blob-target sharding axis uses the high 64 bits of +/// the 128-bit blob hash modulo `gc_shards`. The high bits are taken rather than the low bits so +/// that blobs whose hashes differ only in the low 64 bits — an adversarial corner case — still +/// spread across shards. `CityHash128` output has high entropy in BOTH halves, so both choices +/// are equivalent for organic workloads; high bits are the documented canonical choice. +/// +/// Properties: +/// - Deterministic/stable: same inputs always yield the same shard. +/// - Total: result is in [0, gc_shards). +/// - Single-shard equivalence: gc_shards == 1 always returns 0. +/// +/// The argument is the complete `BlobRef`, so callers cannot silently discard its algorithm while +/// routing. The shard number deliberately uses only `ref.digest`: distribution comes from the +/// digest bytes, while the algorithm remains part of the identity carried through the GC pipeline. +inline uint64_t blobShard(const BlobRef & ref, uint64_t gc_shards) +{ + /// gc_shards >= 1 is enforced by GcState decode (CORRUPTED_DATA on 0). + /// BE-u64 of bytes[0:8] — an EXPLICIT big-endian read, bit-identical to the old + /// `static_cast(blob_hash >> 64)` for every 128-bit digest (`fromU128` writes the + /// UInt128 big-endian into bytes[0:16], so bytes[0:8] IS the old high 64 bits). MUST stay an + /// explicit big-endian read, never a native-endian memcpy (would silently reshard on an LE host). + uint64_t high64 = 0; + for (int i = 0; i < 8; ++i) + high64 = (high64 << 8) | ref.digest.bytes[static_cast(i)]; + return high64 % gc_shards; +} + +/// Route a part-manifest cleanup bundle to a worker by its namespace-qualified `ManifestId`. Workers +/// own disjoint ranges; routing by `ManifestRef` alone would merge cleanup work from two namespaces +/// that happen to reuse the same reference components. `gc_shards == 1` routes every `ManifestId` +/// to owner shard 0. +/// +/// The hash mixes both the `root_namespace` string and the three `ManifestRef` components — the +/// same mixing used by `std::hash`. Two `ManifestId`s that share the same `ManifestRef` +/// but carry different namespaces produce independent hash values and may route to different shards. +/// +/// Properties: +/// - Deterministic/stable: same inputs always yield the same shard. +/// - Total: result is in [0, gc_shards). +/// - Single-shard equivalence: gc_shards == 1 always returns 0. +uint64_t manifestCleanupShard(const ManifestId & id, uint64_t gc_shards); + +/// Per-shard in-degree reducer for the sharded GC fold. +/// +/// `ShardReducer` owns exactly ONE target shard (`shard` in [0, `gc_shards`)). It accepts the +/// caller's per-shard slice of `BlobDelta`s — produced by `foldManifestEdges` and bucketed by +/// `blobShard` — and merges them into a per-shard `CasBlobInDegree` generation run via +/// `foldDeltasIntoGeneration`. +/// +/// Ownership invariant: a reducer touches ONLY blobs it owns — i.e. `blobShard(h, gc_shards) == shard`. +/// Two reducers for DIFFERENT shards may run concurrently; their key namespaces are disjoint +/// (`blobTargetRunKey(gen, shard0, seq)` vs `blobTargetRunKey(gen, shard1, seq)`). +/// +/// The `reduce` method delegates to `foldDeltasIntoGeneration` (the same path the non-sharded fold +/// uses with `shard == 0`), so `gc_shards == 1` with `shard == 0` reproduces the non-sharded fold +/// byte-for-byte. This keeps the one-shard configuration compatible with the original fold path. +/// +/// NOTE on durable writes: `reduce` writes the per-shard in-degree run directly via `backend` +/// (under `blobTargetRunKey(new_generation, shard, 0)`), exactly as `foldDeltasIntoGeneration` +/// does. Returning the durable write here (rather than an in-memory map) keeps the round driver +/// stateless: it simply constructs a `ShardReducer` per shard, calls `reduce`, and the sealed +/// run is already present for the `zeroInDegree` consumer (and the fold's two-cursor merge). An +/// in-memory return value is unnecessary because the backend is directly queryable; tests read the +/// sealed run back over an `InMemoryBackend`. +class ShardReducer +{ +public: + /// Construct a reducer that owns `shard` (in [0, `gc_shards`)). + ShardReducer(uint64_t shard_, uint64_t gc_shards_); + + /// True iff this reducer owns `ref` — i.e. `blobShard(ref, gc_shards) == shard`. + bool owns(const BlobRef & ref) const; + + /// Merge `shard_deltas` (the caller's per-shard `BlobDelta` slice produced by `foldManifestEdges` + /// and bucketed by `blobShard`) into a new in-degree generation for this shard. Writes the sealed + /// run under `blobTargetRunKey(new_generation, shard, 0)` via `backend`, appends its `RunRef` to + /// `out_runs`, and returns the `RunRef`. The call is idempotent (write-once via `putIfAbsent`). + /// + /// `prior_runs` are the parent generation's run segments for this shard, resolved BY THE CALLER from + /// the parent fold seal's `blob_target_runs` filtered to `shard`. An empty vector is the + /// fresh-pool / empty baseline. + /// + /// PRECONDITION: every `BlobDelta` in `shard_deltas` must be owned by this reducer + /// (`blobShard(d.ref, gc_shards) == shard`). This is a caller contract; there is no + /// underflow throw backstopping it — pass a misbucketed delta and the fold silently misroutes it. + std::vector reduce(Backend & backend, const Layout & layout, + const std::vector & prior_runs, + uint64_t new_generation, uint64_t attempt, + std::vector shard_deltas, + uint64_t current_round = 0, uint64_t condemn_round = 0, + const std::function(const BlobRef &)> & head_blob = {}, + const std::function(const BlobRef &)> & peek_head = {}, + const std::function & confirm_condemned_marker = {}, + RetiredMergeResult * out_retired = nullptr, + bool suppress_destructive = false, + /// PROBE B2: forwarded verbatim to `foldDeltasIntoGeneration` — see its + /// declaration and `Cas::TxnApplyLedger`. + std::vector * out_applied_by_txn_ordinal = nullptr, + /// Forwarded verbatim to `foldDeltasIntoGeneration`; the caller shares + /// one instance across every shard's reduce within a round. + GcRoundWorkBudget * work_budget = nullptr) const; + +private: + uint64_t shard; + uint64_t gc_shards; +}; + +/// Role split of a sharded GC round (`gc_shards > 1`): +/// +/// - COORDINATOR (exactly one per round — the lease holder): owns input-seal, round-visibility, +/// the single GLOBAL fence (over all LIST-discovered shards), and generation-advance. These +/// steps span the whole fence universe and must NOT be sharded: a publish into one root shard +/// can protect a blob assigned to ANY target shard, so an independent per-reducer fence is +/// unsafe. `Gc::fence` therefore stays the single coordinator fence over the entire universe. +/// +/// - REDUCERS / CLEANUP WORKERS (one per disjoint shard): own ONLY their shard's blob-target reduce +/// (`ShardReducer`) or part-manifest cleanup (`manifestCleanupShard`). Their key namespaces are +/// disjoint, so two replicas may reduce DIFFERENT shards concurrently. Reducer work needs NO lease: +/// the lease is work-dedup only (see `CasGcScheduler`), not a coordination primitive. + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp new file mode 100644 index 000000000000..e2eb6fc761e4 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.cpp @@ -0,0 +1,143 @@ +#include +#include +#include +#include + +namespace DB::Cas +{ + +NamespaceJanitorResult NamespaceJanitor::runOnePage( + bool suppress_deletes, const std::function & fence_held) +{ + NamespaceJanitorResult result; + const GcMaintenanceReadResult progress = readGcMaintenanceState(backend, layout); + if (progress.status == GcMaintenanceReadStatus::Corrupt) + { + result.anomalies.push_back(progress.diagnostic); + (void)casGcMaintenanceState(backend, layout, progress.token, GcMaintenanceState{}); + return result; + } + + const String cursor = progress.state ? progress.state->janitor_cursor : String{}; + ListPage page; + try + { + page = backend.list(layout.namespaceRootPrefix(), cursor, page_budget); + } + catch (...) + { + (void)casGcMaintenanceState(backend, layout, progress.token, GcMaintenanceState{}); + throw; + } + result.pages = 1; + result.keys = page.keys.size(); + + const CasRefCatalog::Snapshot catalog_cut = CasRefCatalog::read(backend, layout); + bool ambiguous = false; + try + { + catalog_cut.life_index.throwIfAmbiguous("CAS namespace janitor"); + } + catch (const DB::Exception & e) + { + result.anomalies.push_back(e.message()); + ambiguous = true; + } + /// A valid page is complete only when the round had deletion authority for every dead-life + /// candidate on it. Advancing while the global gate is closed can phase-lock a dead page onto + /// every suppressed round and a different page onto every bounded forced fold. Ambiguous cuts and + /// observed fence loss have the same shape: retain the old cursor so an authoritative round + /// retries the exact page. Malformed keys, absent objects and token mismatches are final per-key + /// outcomes and therefore do not by themselves prevent progress. + bool page_decided = !ambiguous && !suppress_deletes; + + for (const ListedKey & listed : page.keys) + { + std::optional life_id; + try + { + if (listed.key.starts_with(layout.namespaceStreamRootPrefix())) + { + if (const auto parsed = layout.parseRefObjectKey(listed.key)) + life_id = parsed->life_id; + } + else if (listed.key.starts_with(layout.namespaceStateRootPrefix())) + { + if (const auto parsed = layout.parseRefCkptKey(listed.key)) + life_id = *parsed; + else if (const auto file_parsed = layout.parseNamespaceFileKey(listed.key)) + life_id = file_parsed->life_id; + } + } + catch (const DB::Exception & e) + { + result.anomalies.push_back(listed.key + ": " + e.message()); + continue; + } + + if (!life_id) + { + result.anomalies.push_back(listed.key + ": unrecognized namespace object key"); + continue; + } + if (ambiguous || suppress_deletes || catalog_cut.life_index.resolve(*life_id)) + continue; + + std::optional token = listed.token; + if (!token) + { + try + { + const HeadResult current = backend.head(listed.key); + if (!current.exists) + continue; + token = current.token; + } + catch (const std::exception & e) + { + ++result.leaked; + result.anomalies.push_back( + "leaked dead-life object '" + listed.key + "': exact HEAD failed: " + e.what()); + continue; + } + } + if (!fence_held()) + { + page_decided = false; + break; + } + try + { + if (backend.deleteExact(listed.key, *token).kind == DeleteOutcome::Kind::Deleted) + ++result.deleted; + } + catch (const std::exception & e) + { + ++result.leaked; + result.anomalies.push_back( + "leaked dead-life object '" + listed.key + "': exact delete failed: " + e.what()); + } + } + + /// Recheck even when the page had no dead candidate. A tenure that observes fence loss after LIST + /// or after the last exact delete must not publish progress. Loss after this check may still race + /// with the leak-only maintenance CAS; already completed exact deletes remain safe to repeat. + if (page_decided && !fence_held()) + page_decided = false; + + if (page_decided) + { + const GcMaintenanceState next{.janitor_cursor = page.next_cursor}; + try + { + (void)casGcMaintenanceState(backend, layout, progress.token, next); + } + catch (const std::exception & e) + { + result.anomalies.push_back("cursor publication failed: " + String(e.what())); + } + } + return result; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h new file mode 100644 index 000000000000..e60f87e2e6a0 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasNamespaceJanitor.h @@ -0,0 +1,34 @@ +#pragma once +#include +#include +#include +#include + +namespace DB::Cas +{ + +struct NamespaceJanitorResult +{ + uint64_t pages = 0; + uint64_t keys = 0; + uint64_t deleted = 0; + uint64_t leaked = 0; + std::vector anomalies; +}; + +/// Runs one bounded, leak-only page over the physical namespace ownership tree. +class NamespaceJanitor +{ +public: + NamespaceJanitor(Backend & backend_, const Layout & layout_, size_t page_budget_) + : backend(backend_), layout(layout_), page_budget(page_budget_) {} + + NamespaceJanitorResult runOnePage(bool suppress_deletes, const std::function & fence_held); + +private: + Backend & backend; + const Layout & layout; + size_t page_budget; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.cpp new file mode 100644 index 000000000000..b019212e00a6 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.cpp @@ -0,0 +1,915 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event CASGCEnumerationPages; +} + +namespace DB::Cas +{ + +namespace +{ + +/// Hook used by this file's GC-owned enumeration calls to `forEachListedKey` and `recoverRefTable`. +/// It increments once per physical LIST page, never once per listed key. +void onGcEnumerationPage() +{ + ProfileEvents::increment(ProfileEvents::CASGCEnumerationPages); +} + +/// The durable build floor is stored in the per-server mount lease together with the writer epoch. A +/// namespace is rooted by `server_root_id`, but that id is a clean relative path and can contain slashes. +/// Try namespace prefixes from longest to shortest and accept the first durable mount body. Without a +/// mount there is no deletion authority, so the caller must leave the prefix untouched. The mount's +/// `writer_epoch` and `min_active` are the single durable epoch/floor pair used for eligibility, including +/// across process replacement and the retired sentinel. +std::optional floorForNamespace(Pool & store, const RootNamespace & ns) +{ + const String & value = ns.string(); + size_t pos = value.size(); + while (true) + { + pos = value.rfind('/', pos == 0 ? 0 : pos - 1); + if (pos == String::npos) + break; + + const String server_root_id = value.substr(0, pos); + if (!server_root_id.empty()) + { + if (const auto got = store.backend().get(store.layout().mountKey(server_root_id))) + return decodeMountLease(got->bytes); + } + if (pos == 0) + break; + } + return std::nullopt; +} + +struct ListedManifestObject +{ + RootNamespace ns; + BuildPrefix prefix; + ManifestRef ref; + String key; +}; + +/// Delegates to the shared `Layout::parseManifestKey`, which validates the canonical manifest-key +/// encoding. Keeping parsing in `Layout` avoids a second interpretation of the manifest path and ensures +/// the sweep and filesystem checker derive the same namespace and build identity. +std::optional parseListedManifestObject(const Layout & layout, const String & key) +{ + const auto parsed = layout.parseManifestKey(key); + if (!parsed) + return std::nullopt; + + return ListedManifestObject{ + .ns = parsed->root_namespace, + .prefix = BuildPrefix{.writer_epoch = parsed->ref.writer_epoch, .build_sequence = parsed->ref.build_sequence}, + .ref = parsed->ref, + .key = key}; +} + +/// The fold seal `gc/state` currently adopts, or `nullopt` when the pool has no `gc/state` or no seal +/// at `(snap_generation, snap_attempt)` — a pool whose GC has never completed a round. It is read ONCE +/// per sweep pass; every namespace the pass touches takes its coverage row out of the same object. +std::optional readAdoptedFoldSeal(Pool & store) +{ + const Layout & layout = store.layout(); + const auto state_got = store.backend().get(layout.gcStateKey()); + if (!state_got) + return std::nullopt; + const GcState state = decodeGcState(state_got->bytes); + const auto got = store.backend().get(layout.foldSealKey(state.snap_generation, state.snap_attempt)); + if (!got) + return std::nullopt; + return decodeFoldSeal(got->bytes, state.snap_generation); +} + + +/// One catalog-named life row out of an adopted seal. The catalog cut supplies the name-to-id join; +/// neither an absent name nor a `Creating` row may recover coverage from a historical life. +std::optional coverageOf( + const std::optional & seal, + const CasRefCatalog::Snapshot & catalog_cut, + const RootNamespace & ns) +{ + if (!seal) + return std::nullopt; + + const auto catalog_it = std::find_if( + catalog_cut.catalog.entries.begin(), catalog_cut.catalog.entries.end(), + [&](const CatalogEntry & entry) { return entry.ns == ns; }); + if (catalog_it == catalog_cut.catalog.entries.end() || catalog_it->state == NsState::Creating) + return std::nullopt; + + const auto row_it = seal->ref_lives.find(catalog_it->incarnation); + if (row_it == seal->ref_lives.end()) + return std::nullopt; + return row_it->second.coverage; +} + +/// Return this operation's ONE catalog row for `ns`. Callers hold the enclosing `Snapshot` for their +/// entire decision; a later name resolution would splice its life into an earlier coverage decision. +const CatalogEntry * catalogEntryOf(const CasRefCatalog::Snapshot & catalog_cut, const RootNamespace & ns) +{ + const auto it = std::find_if( + catalog_cut.catalog.entries.begin(), catalog_cut.catalog.entries.end(), + [&](const CatalogEntry & entry) { return entry.ns == ns; }); + return it == catalog_cut.catalog.entries.end() ? nullptr : &*it; +} + +/// The durable `last_folded_ref_id` of a coverage row, `{0, 0}` when there is none — as for a fresh +/// pool. A manifest removed by a log above this cursor has not had its `-1` decrement folded, so its +/// body remains load-bearing until the fold catches up. +RefTxnId sealedRefCursor(const std::optional & coverage) +{ + return coverage ? coverage->last_folded_ref_id : RefTxnId{}; +} + +/// One namespace's protection view: the manifest object keys the sweep must never delete, split so the +/// §6 premise can test the removal half on its own. `active` is the pre-existing union the delete sites +/// consult; `tail_removal_targets` is the subset the unconsumed tail above the cursor names as removal +/// targets, which is what rule (2) is stated over. +struct NamespaceProtection +{ + std::set active; + std::set tail_removal_targets; + /// Set when the committed-tail recovery walk stopped early because `work_budget`'s recovery-op cap + /// was spent (see `activeManifestKeys` below), or the walk was never attempted because the cap was + /// already spent by an earlier namespace this round. `active`/`tail_removal_targets` are then a + /// PARTIAL view — the caller must never use them to authorize a deletion decision, only to retain + /// every one of this namespace's candidates on the current page. + bool recovery_incomplete = false; +}; + + +/// The active manifest-object-key set for one namespace, built as the +/// same complete view writer recovery uses: +/// owners in the newest snapshot + owner changes in every later log (== `recoverRefTable`'s committed +/// rows and live precommits) + manifests removed anywhere in the tail above the durable +/// `last_folded_ref_id` (their `-1` is not yet folded, so the GC fold still needs the body). +/// Keys (not ManifestIds) so a listed object key can be tested directly. Throws on a corrupt snapshot / +/// invalid transaction (via the authority-grounded recovery / `decodeRefLogTxn`); the caller SKIPS the +/// namespace's deletions on such a throw rather than substituting an empty owner set. +/// +/// `work_budget`, when set, bounds the committed-tail walk below: each ref-log GET the walk issues +/// consumes one unit of `GcRoundWorkBudget::sweep_recovery_op_budget`, shared with every other +/// namespace this round touches. Exhaustion sets `NamespaceProtection::recovery_incomplete` and stops +/// the walk — it is deliberately NOT plumbed into `recoverRefTableDetailedFromAuthority` itself: that +/// function is a shared recovery primitive also used by `fsck` (which needs a COMPLETE table to audit) +/// and the GC rebuild path (which needs a complete table to reconstruct in-degree from scratch), so +/// capping its own internal cost is out of scope here — this file only bounds the walk it owns. +/// Reaching the budget before even calling `recoverRefTableDetailedFromAuthority` (already spent by an +/// earlier namespace) skips that call entirely and reports incomplete immediately. +NamespaceProtection activeManifestKeys( + Pool & store, const CatalogEntry & catalog_entry, const RefCkpt & ckpt, + const std::optional & coverage, GcRoundWorkBudget * work_budget = nullptr) +{ + NamespaceProtection protection; + if (work_budget && !work_budget->sweepRecoveryOpAvailable()) + { + protection.recovery_incomplete = true; + return protection; + } + std::set & active = protection.active; + const Layout & layout = store.layout(); + Backend & backend = store.backend(); + const RootNamespace & ns = catalog_entry.ns; + const NamespaceLifeId life = NamespaceLifeId::fromCatalogEntry(catalog_entry.ns, catalog_entry.incarnation); + + /// Current owners = snapshot + replayed tail (committed rows + live precommits). + /// The exact row and `_ckpt` come from the caller's frozen catalog cut. Do not resolve `ns` here: + /// a later catalog cut can name a reborn life and turn this old life into an apparent orphan. + const RecoveredRefTable recovered = recoverRefTableDetailedFromAuthority( + backend, layout, catalog_entry, ckpt); + if (work_budget) + ++work_budget->sweep_recovery_ops_used; /// one coarse unit for the snapshot+tail recovery itself + const RefTableState & state = recovered.state; + for (const auto [ref_name, row] : state.getCommitted()) + active.insert(layout.manifestKey(ManifestId{ns, row.manifest_ref})); + for (const auto & [ref_name, manifest_ref] : state.getPrecommits()) + active.insert(layout.manifestKey(ManifestId{ns, manifest_ref})); + + /// Tail-removal protection: every manifest removed by a log ABOVE the durable fold cursor stays active + /// until its `-1` folds. The exact checkpoint frontier is the finite upper bound; LIST cannot decide + /// whether a removal exists. A namespace-removal transaction names every removed owner explicitly, so + /// this also protects a whole removed namespace's bodies until the fold catches up. + const RefTxnId cursor = sealedRefCursor(coverage); + if (!ckpt.committed_through || !(cursor < *ckpt.committed_through)) + return protection; + + /// When cleanup has removed the inherited cursor body, only the exact next global epoch may prove + /// that cursor was a seal. Asking `crossEpochFromSeal` about `{E+1,1}` preserves its backlink/body + /// validation without allowing a later witness to jump over a missing empty-epoch seal. + const auto cross_from_missing_cursor = [&](const RefTxnId & from_cursor) + { + if (from_cursor.writer_epoch == std::numeric_limits::max()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS orphan sweep: folded cursor {} has no representable next epoch", + renderRefTxnId(from_cursor)); + const RefTxnId exact_next_epoch{from_cursor.writer_epoch + 1, 1}; + const EpochCrossResult crossing = crossEpochFromSeal( + backend, layout, ns, from_cursor, std::nullopt, exact_next_epoch, life); + if (!crossing.proved()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS orphan sweep: exact next-epoch record {} does not prove that folded cursor {} was its seal " + "(cross outcome {})", + renderRefTxnId(exact_next_epoch), renderRefTxnId(from_cursor), + static_cast(crossing.outcome)); + return crossing.start; + }; + + RefTxnId id; + std::optional prior; + std::optional prior_is_seal; + if (cursor == RefTxnId{} || (ckpt.life_epoch && cursor.writer_epoch < *ckpt.life_epoch)) + { + id = RefTxnId{*ckpt.life_epoch, 1}; + } + else + { + /// A cursor can name any old epoch's `EpochSeal`, not just `ckpt.last_epoch_seal`. When its + /// body survives, decode it and cross with that direct kind evidence. If compaction removed the + /// cursor, first try its ordinary same-epoch successor; only a 404 there can ask the shared + /// chain proof to establish a cross with kind unknown. That preserves tails after a cleaned + /// cursor without guessing that a missing cursor was a seal. + const auto cursor_got = backend.get(layout.refLogKey(life, cursor)); + if (cursor_got) + { + const RefLogTxn cursor_txn = decodeRefLogTxn( + openObject(FormatId::RefLog, cursor_got->bytes), ns.string(), cursor); + if (refLogTxnIsEpochSeal(cursor_txn)) + { + if (cursor.writer_epoch == std::numeric_limits::max()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS orphan sweep: folded epoch seal {} has no representable exact successor", + renderRefTxnId(cursor)); + id = RefTxnId{cursor.writer_epoch + 1, 1}; + } + else + { + if (cursor.ref_sequence == std::numeric_limits::max()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS orphan sweep: folded cursor {} has no representable exact successor", + renderRefTxnId(cursor)); + id = RefTxnId{cursor.writer_epoch, cursor.ref_sequence + 1}; + prior = cursor; + prior_is_seal = false; + } + } + else + { + if (cursor.ref_sequence == std::numeric_limits::max()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS orphan sweep: folded cursor {} has no representable exact successor", + renderRefTxnId(cursor)); + id = RefTxnId{cursor.writer_epoch, cursor.ref_sequence + 1}; + prior = cursor; + prior_is_seal = std::nullopt; + } + } + + while (id <= *ckpt.committed_through) + { + /// UNCERTAINTY, work-budget arm: the committed-tail walk is a finite but potentially huge range + /// (up to `ckpt.committed_through`), and the round shares one recovery-op budget across every + /// namespace it touches. Stopping HERE — before the next GET — leaves `active`/ + /// `tail_removal_targets` genuinely partial, so the caller must treat the whole namespace as + /// undecided this page (fail-closed retain), never authorize a deletion from what was collected + /// so far. + if (work_budget && !work_budget->sweepRecoveryOpAvailable()) + { + protection.recovery_incomplete = true; + break; + } + const auto got = backend.get(layout.refLogKey(life, id)); + if (work_budget) + ++work_budget->sweep_recovery_ops_used; + if (!got) + { + /// A known ordinary predecessor cannot cross an epoch. Only an inherited cursor whose body + /// was compacted has unknown kind, and even then INV-2 permits exactly `{E+1,1}` rather than + /// an arbitrary later witness. + if (!prior || prior_is_seal.has_value()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS orphan sweep: committed tail log {} is absent under the supplied _ckpt frontier", + renderRefTxnId(id)); + id = cross_from_missing_cursor(*prior); + prior.reset(); + prior_is_seal.reset(); + continue; + } + const RefLogTxn txn = decodeRefLogTxn(openObject(FormatId::RefLog, got->bytes), ns.string(), id); + for (const RefManifestEdge & edge : manifestEdgesOfTxn(txn)) + if (edge.change < 0) + { + const String key = layout.manifestKey(edge.manifest_id); + active.insert(key); + protection.tail_removal_targets.insert(key); + } + + const bool is_seal = refLogTxnIsEpochSeal(txn); + if (const std::optional next = nextRefLogIdWithinCommittedFrontier( + id, is_seal, *ckpt.committed_through)) + { + if (is_seal) + { + prior.reset(); + prior_is_seal.reset(); + } + else + { + prior = id; + prior_is_seal = false; + } + id = *next; + } + else + break; + } + return protection; +} + +} + +NamespaceFoldView namespaceFoldView(Pool & store, const RootNamespace & ns) +{ + NamespaceFoldView view; + const CasRefCatalog::Snapshot catalog_cut = CasRefCatalog::read(store.backend(), store.layout()); + catalog_cut.life_index.throwIfAmbiguous("CAS orphan manifest sweep"); + view.coverage = coverageOf(readAdoptedFoldSeal(store), catalog_cut, ns); + return view; +} + +std::string_view sweepRetainClassName(SweepRetainClass c) +{ + switch (c) + { + case SweepRetainClass::None: return "none"; + case SweepRetainClass::NoCoverage: return "no_coverage"; + case SweepRetainClass::Hold: return "hold"; + case SweepRetainClass::UnconsumedSeal: return "unconsumed_seal"; + case SweepRetainClass::TailRemoval: return "tail_removal"; + case SweepRetainClass::WorkBudgetExhausted: return "work_budget_exhausted"; + } + return "unknown"; +} + +std::pair ManifestSweepResult::topRetainReason() const +{ + /// Enum order, so a tie resolves the same way on every pass and an unchanged pool keeps reporting + /// the same verdict instead of alternating between two equally-large classes. + const std::pair candidates[] = { + {SweepRetainClass::NoCoverage, retained_no_coverage}, + {SweepRetainClass::Hold, retained_hold}, + {SweepRetainClass::UnconsumedSeal, retained_unconsumed_seal}, + {SweepRetainClass::TailRemoval, retained_tail_removal}, + {SweepRetainClass::WorkBudgetExhausted, retained_work_budget}, + }; + std::pair top{SweepRetainClass::None, 0}; + for (const auto & c : candidates) + if (c.second > top.second) + top = c; + return top; +} + +bool manifestDeletionPremise(const NamespaceFoldView & view, const ManifestKey & manifest, + String * retain_reason, SweepRetainClass * retain_class) +{ + const auto retain = [&](SweepRetainClass klass, String why) + { + if (retain_reason) + *retain_reason = std::move(why); + if (retain_class) + *retain_class = klass; + return false; + }; + if (retain_class) + *retain_class = SweepRetainClass::None; + const String build_epoch = std::to_string(manifest.prefix.writer_epoch); + + /// UNCERTAINTY, and it comes first because it is the case where the predicate knows NOTHING. With no + /// sealed coverage row, no round has folded a ref cursor for this namespace, so no epoch's closing + /// seal can be shown consumed. The sweep's own protection view cannot stand in for the missing + /// proof: that view is assembled from the very enumeration arithmetic intake distrusts. + if (!view.coverage) + return retain(SweepRetainClass::NoCoverage, + "no sealed fold coverage for the namespace: no round has folded a ref cursor for " + "it, so epoch " + build_epoch + "'s closing seal cannot be shown consumed"); + + const RefCoverage & cov = *view.coverage; + + /// UNCERTAINTY, hold arm. A hold names the exact position the fold could not resolve, and everything + /// at or above it is unaccounted -- including, for all this predicate can tell, the record that + /// grants or removes this very manifest. `classification == 4` is tested separately from the hold + /// even though the seal's strict grammar pairs them: the thing standing between a clamped namespace + /// and an irreversible delete must not be a codec invariant enforced somewhere else. + if (cov.hold) + return retain(SweepRetainClass::Hold, "namespace held at " + renderRefTxnId(cov.hold->offending_position) + " (" + + String{holdReasonToWord(cov.hold->reason)} + ", retried " + + std::to_string(cov.hold->retry_count) + " round(s)): every record at or above " + "that position is unaccounted for"); + if (cov.classification == 4) + return retain(SweepRetainClass::Hold, + "namespace coverage is classified clamped (4) with no hold recorded: whatever " + "stopped the fold was not carried, so nothing above its cursor is accounted for"); + if (cov.classification == 0) + return retain(SweepRetainClass::NoCoverage, + "namespace coverage is classified absent (0): no round folded it, so its cursor " + "is not the result of any walk"); + + /// RULE 1 (spec §6). Grants do not cross epochs, so every `+1` that could name an epoch-`E` build + /// lives among epoch `E`'s own records; and an epoch is left ONLY over its consumed `EpochSeal` + /// (INV-2), so a sealed cursor in a STRICTLY HIGHER epoch is durable proof that every one of those + /// records has folded -- proof by arithmetic, which is what makes it independent of the listing. + /// + /// A cursor still INSIDE epoch `E` proves nothing of the sort, and that stays true even when it + /// happens to sit on `E`'s own seal: the durable cursor records a POSITION, never the KIND of the + /// record there, so "the cursor is the seal" is not a readable fact here. Retaining that case costs + /// one more round and resolves itself -- the next epoch's first record crosses the seal, and the + /// round after that sweeps the build. + if (!(manifest.prefix.writer_epoch < cov.last_folded_ref_id.writer_epoch)) + return retain(SweepRetainClass::UnconsumedSeal, + "epoch " + build_epoch + "'s closing seal is not consumed: the sealed cursor is at " + + renderRefTxnId(cov.last_folded_ref_id) + ", so a grant naming this build may " + "still be unfolded above it"); + + /// RULE 2 (spec §6). Removals DO cross epochs, so a record in a LATER epoch can name this build as a + /// removal target; deleting the body before that `-1` folds leaves the fold clamping forever on a + /// manifest it must read to emit the decrement (the GC-WEDGE-2026-07-10 shape). + /// + /// The sweep's protection set already spares a tail removal target before this predicate is + /// consulted, so this test is belt over suspenders BY DESIGN: the rule belongs to the premise, so + /// that a future sweep path that assembles its protection set differently cannot lose it. Note what + /// it is NOT: its negative direction is no proof, because the set comes from the same enumeration + /// rule (1) exists to stop trusting. Rule (1) is what makes the tail decidable; this makes the + /// decision explicit. + if (view.tail_removal_targets.contains(manifest.key)) + return retain(SweepRetainClass::TailRemoval, + "an unconsumed tail record above the cursor names this manifest as a removal " + "target: its `-1` has not folded, so the fold still needs the body"); + + if (retain_reason) + retain_reason->clear(); + return true; +} + +bool prefixEligible(Pool & store, const RootNamespace & ns, const BuildPrefix & prefix) +{ + /// Eligibility comes only from the durable mount-lease floor. A missing floor means NOT eligible; + /// do not replace that authority check with a frozen-sequence or judged-dead guess. Compare + /// `writer_epoch` first, then `build_sequence`, so old-epoch + /// debris drains after a process restart even when its build_sequence is above the current min_active. + const auto floor = floorForNamespace(store, ns); + if (!floor) + return false; + + const MountLease & w = *floor; + if (prefix.writer_epoch < w.writer_epoch) + return true; + if (prefix.writer_epoch > w.writer_epoch) + return false; + if (w.min_active == std::numeric_limits::max()) + return true; /// farewell/retired sentinel: every seq is retired + return w.min_active > prefix.build_sequence; +} + +uint64_t sweepNamespace(Pool & store, const RootNamespace & ns, const BuildPrefix & prefix, + std::vector * warnings) +{ + if (!prefixEligible(store, ns, prefix)) + return 0; /// not eligible by the durable watermark fact — delete nothing (controls #8/#9) + + const Layout & layout = store.layout(); + Backend & backend = store.backend(); + + /// Build the protection view. A missing snapshot body, an invalid transaction, or an incomplete + /// ordered view throws, causing the sweep to skip deletion and surface the error; it never substitutes + /// an empty owner set. Skip this namespace's deletions on such a throw rather than deleting against a + /// wrong (empty) view. When a + /// caller opted in (`warnings != nullptr`), this "cannot confirm emptiness" also lands in `*warnings` + /// -- not just the log -- so a decommission run does not silently report a clean drain that never + /// happened; the log-only default stays exactly as before for every other caller, which treats this + /// the same way the periodic sweep always has: skip and retry next round. + /// The §6 premise's durable half, read before the protection view so both share one seal read. + NamespaceFoldView view; + const CasRefCatalog::Snapshot catalog_cut = CasRefCatalog::read(store.backend(), store.layout()); + catalog_cut.life_index.throwIfAmbiguous("CAS orphan manifest sweep"); + view.coverage = coverageOf(readAdoptedFoldSeal(store), catalog_cut, ns); + const CatalogEntry * catalog_entry = catalogEntryOf(catalog_cut, ns); + if (!catalog_entry || catalog_entry->state == NsState::Creating) + return 0; /// absent/Creating names have no recovery authority and therefore no deletion authority + + NamespaceProtection protection; + try + { + const NamespaceLifeId life = NamespaceLifeId::fromCatalogEntry(catalog_entry->ns, catalog_entry->incarnation); + const std::optional ckpt = readCkpt(backend, layout, life); + if (!ckpt) + { + const String warning = "CAS orphan sweep: namespace " + ns.string() + + " is catalog-named but its required _ckpt is absent; skipped, emptiness not confirmed"; + LOG_WARNING(getLogger("CasOrphanManifestSweep"), "{}", warning); + if (warnings) + warnings->push_back(warning); + return 0; + } + protection = activeManifestKeys(store, *catalog_entry, ckpt->ckpt, view.coverage); + view.tail_removal_targets = protection.tail_removal_targets; + + } + catch (const Exception & e) + { + LOG_WARNING(getLogger("CasOrphanManifestSweep"), + "CAS orphan sweep: namespace {} protection view unavailable ({}); skipping its deletions", + ns.string(), e.message()); + if (warnings) + warnings->push_back("CAS orphan sweep: namespace " + ns.string() + " protection view unavailable (" + + e.message() + "); skipped, emptiness not confirmed"); + return 0; + } + + /// Enumerate the ONE build prefix: cas/manifests//-/ in the canonical + /// hexadecimal form -- the same rendering `Layout::manifestKey` uses. + const String prefix_key = layout.manifestNamespacePrefix(ns) + + renderRefTxnId(RefTxnId{prefix.writer_epoch, prefix.build_sequence}) + "/"; + + uint64_t deleted = 0; + forEachListedKey(backend, prefix_key, [&](const ListedKey & listed) + { + if (protection.active.contains(listed.key)) + return; /// owned by a committed or precommit owner — never sweep + + /// THE §6 SAFETY FLOOR, under the watermark eligibility already established above. The + /// watermark says the build is retired; the premise says the ref stream can be shown not to + /// name this body. A refusal is recorded rather than silent (Constraint 10). + String retain_reason; + if (!manifestDeletionPremise(view, ManifestKey{listed.key, prefix}, &retain_reason)) + { + LOG_DEBUG(getLogger("CasOrphanManifestSweep"), + "CAS orphan sweep: retaining {} -- {}", listed.key, retain_reason); + if (warnings) + warnings->push_back("CAS orphan sweep: retained " + listed.key + " -- " + retain_reason); + return; + } + + /// Exact-token delete: HEAD for the current token, then deleteExact. A 404 between HEAD and + /// delete (or a TokenMismatch — a fresh owner reclaimed it) is tolerated (record-and-continue), + /// same as always, regardless of `warnings` -- that is the normal "someone else already reclaimed + /// it" race, not a failure. A THROWN exception (a transient backend hiccup) is the one thing + /// `warnings` changes: opted-in (non-null), it is recorded and the sweep moves to the next key; + /// opted-out (nullptr, every pre-existing caller), it propagates exactly as before (fail-close). + try + { + const HeadResult head = backend.head(listed.key); + if (!head.exists) + return; + const DeleteOutcome outcome = backend.deleteExact(listed.key, head.token); /// NotFound/TokenMismatch spared + if (classifyDeleteOutcome(outcome) == DeleteClass::Deleted) + ++deleted; + } + catch (...) + { + if (!warnings) + throw; + warnings->push_back("CAS orphan sweep: " + listed.key + " delete failed: " + + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + } + }, 1000, onGcEnumerationPage); + return deleted; +} + +ManifestSweepResult planManifestCursorPage( + Pool & store, + const String & cursor, + uint64_t list_budget, + uint64_t nomination_budget, + bool catalog_recovery_authoritative, + GcRoundWorkBudget * work_budget) +{ + ManifestSweepResult result; + result.next_cursor = cursor; + if (list_budget == 0) + return result; + + Backend & backend = store.backend(); + const Layout & layout = store.layout(); + const ListPage page = backend.list(layout.casManifestsPrefix(), cursor, list_budget); + /// This pass fetches exactly one page per round (the cursor advances across rounds, not within this + /// call), so the metric increments once per call, not once per listed key. + ProfileEvents::increment(ProfileEvents::CASGCEnumerationPages); + + /// Freeze every possible destructive candidate BEFORE the later catalog cut. A same-name rebirth can + /// replace this logical manifest key between the observations; classifying the old bytes against the + /// later lifecycle cut is safe only when deletion retains the old exact token, so the replacement + /// loses `deleteExact`. Do not take a fresh GET after the catalog read: that would splice new-life + /// bytes into old candidate selection and authorize their deletion with the new token. + /// + /// Bounded to `nomination_budget` well-formed keys — never the whole `list_budget`-sized + /// page — since `nomination_budget` is the hard ceiling on how many of them this call can ever + /// nominate. A well-formed key beyond this cap has no frozen body; it is retained where its absence + /// is discovered below, in the exact same "budget exhausted, cursor does not step over it" shape the + /// nomination-count exhaustion already uses. + std::map> observed_candidates; + if (nomination_budget > 0) + { + uint64_t frozen = 0; + for (const ListedKey & listed : page.keys) + { + if (frozen >= nomination_budget) + break; + if (parseListedManifestObject(layout, listed.key)) + { + observed_candidates.emplace(listed.key, backend.get(listed.key)); + ++frozen; + } + } + } + + /// One seal and one later catalog cut for the whole page; every namespace joins through those same + /// immutable observations. + const std::optional adopted_seal = readAdoptedFoldSeal(store); + const CasRefCatalog::Snapshot catalog_cut = CasRefCatalog::read(backend, layout); + catalog_cut.life_index.throwIfAmbiguous("CAS orphan manifest sweep"); + + std::map eligible_by_prefix; + std::map view_by_ns; + std::map> active_by_ns; + std::set errored_namespaces; /// protection view unavailable => skip, never delete + + /// The key of the last candidate this page actually DECIDED on. The cursor resumes strictly after + /// it (`ListPage::next_cursor` is the last returned key), so a candidate the page never decided on + /// stays ahead of the cursor and is examined next pass. See the budget rule below. + String decided_through; + bool budget_exhausted = false; + + for (const ListedKey & listed : page.keys) + { + ++result.listed; + + /// UNCERTAINTY, budget arm (§6). Once a NON-ZERO delete budget is used up, the page stops + /// deciding: the candidates behind it are retained AND the cursor does not step over them. + /// Advancing past a candidate nothing examined would turn "retained this round" into + /// "unexamined until the cursor wraps the whole `cas/manifests/` keyspace" -- the same trade + /// the round-level gate refuses when it freezes the cursor along with a suppressed sweep. + /// A budget of ZERO is not exhaustion but a list-only pass: nothing is ever deletable, so + /// freezing the cursor on it would make the sweep spin on one page forever. That pass keeps + /// its pre-existing behaviour and advances. + if (budget_exhausted || (nomination_budget > 0 && result.nominations.size() >= nomination_budget)) + { + budget_exhausted = true; + ++result.skipped; + continue; + } + + const auto parsed = parseListedManifestObject(layout, listed.key); + if (!parsed) + { + ++result.skipped; + decided_through = listed.key; + continue; + } + + if (nomination_budget == 0) + { + /// The list-only pass: examined, not deletable, cursor advances (see the budget rule above). + ++result.skipped; + decided_through = listed.key; + continue; + } + + const CatalogEntry * catalog_entry = catalogEntryOf(catalog_cut, parsed->ns); + if (catalog_entry) + { + const String eligibility_key = parsed->ns.string() + "\n" + + std::to_string(parsed->prefix.writer_epoch) + "\n" + + std::to_string(parsed->prefix.build_sequence); + auto [eligible_it, eligible_inserted] = eligible_by_prefix.emplace(eligibility_key, false); + if (eligible_inserted) + eligible_it->second = prefixEligible(store, parsed->ns, parsed->prefix); + if (!eligible_it->second) + { + ++result.skipped; + decided_through = listed.key; + continue; + } + } + + auto [view_it, view_inserted] = view_by_ns.emplace(parsed->ns.string(), NamespaceFoldView{}); + auto [active_it, inserted] = active_by_ns.emplace(parsed->ns.string(), std::set{}); + if (inserted) + { + /// UNCERTAINTY, work-budget arm: building a fresh namespace's protection view + /// is the expensive step (a catalog-authoritative table recovery plus a committed-tail + /// walk) the LIST/nomination budgets never bounded. Once the round's per-page namespace cap + /// is spent, a NEW namespace gets no view at all -- retained, exactly like every other + /// "cannot confirm emptiness" cause below, never a partial or best-effort one. + if (work_budget && !work_budget->sweepNamespaceAvailable()) + { + LOG_DEBUG(getLogger("CasOrphanManifestSweep"), + "CAS orphan sweep: retaining {} -- round's per-page namespace work budget exhausted", + parsed->key); + errored_namespaces.insert(parsed->ns.string()); + ++result.retained_work_budget; + ++result.skipped; + decided_through = listed.key; + continue; + } + if (work_budget) + ++work_budget->sweep_namespaces_used; + view_it->second.coverage = coverageOf(adopted_seal, catalog_cut, parsed->ns); + if (catalog_entry && !catalog_recovery_authoritative) + { + LOG_DEBUG(getLogger("CasOrphanManifestSweep"), + "CAS orphan sweep: retaining {} -- caller did not authorize catalog-named recovery", + parsed->key); + errored_namespaces.insert(parsed->ns.string()); + ++result.retained_no_coverage; + ++result.skipped; + decided_through = listed.key; + continue; + } + if (catalog_entry && catalog_entry->state == NsState::Creating) + { + LOG_DEBUG(getLogger("CasOrphanManifestSweep"), + "CAS orphan sweep: retaining {} -- catalog namespace is Creating and has no recovery authority", + parsed->key); + errored_namespaces.insert(parsed->ns.string()); + ++result.retained_no_coverage; + ++result.skipped; + decided_through = listed.key; + continue; + } + /// A corrupt snapshot or invalid transaction means the protection view is unavailable. Skip + /// this namespace's deletions and surface the error; never substitute an empty owner set. + try + { + if (catalog_entry) + { + const NamespaceLifeId life = NamespaceLifeId::fromCatalogEntry( + catalog_entry->ns, catalog_entry->incarnation); + const std::optional ckpt = readCkpt(backend, layout, life); + if (!ckpt) + { + LOG_WARNING(getLogger("CasOrphanManifestSweep"), + "CAS orphan sweep: namespace {} is catalog-named but its required _ckpt is absent; skipping", + parsed->ns.string()); + errored_namespaces.insert(parsed->ns.string()); + } + else + { + NamespaceProtection protection = activeManifestKeys( + store, *catalog_entry, ckpt->ckpt, view_it->second.coverage, work_budget); + if (protection.recovery_incomplete) + { + /// The committed-tail walk stopped early: `active`/`tail_removal_targets` + /// are a PARTIAL view. Discard them and retain the whole namespace on this + /// page instead of deciding from an incomplete protection set (fail-closed). + LOG_DEBUG(getLogger("CasOrphanManifestSweep"), + "CAS orphan sweep: retaining {} -- committed-tail recovery walk's work " + "budget exhausted before it could confirm the namespace's protection view", + parsed->key); + errored_namespaces.insert(parsed->ns.string()); + ++result.retained_work_budget; + } + else + { + view_it->second.tail_removal_targets = std::move(protection.tail_removal_targets); + active_it->second = std::move(protection.active); + } + } + } + + } + catch (const Exception & e) + { + LOG_WARNING(getLogger("CasOrphanManifestSweep"), + "CAS orphan sweep: namespace {} protection view unavailable ({}); skipping", + parsed->ns.string(), e.message()); + errored_namespaces.insert(parsed->ns.string()); + } + } + if (errored_namespaces.contains(parsed->ns.string())) + { + ++result.skipped; + decided_through = listed.key; + continue; + } + if (catalog_entry && active_it->second.contains(parsed->key)) + { + ++result.skipped; + decided_through = listed.key; + continue; + } + + /// If the post-observation catalog cut has no row, this candidate was necessarily created by a + /// now-dead life: creation publishes its row before any life-owned object. No current life can + /// name it, and a concurrent later creation can only replace its token after this observation. + /// A catalog-named row instead needs the ordinary coverage/owner proof below. + if (catalog_entry) + { + /// THE §6 SAFETY FLOOR, the same predicate `sweepNamespace` calls, under the same watermark + /// eligibility. A refusal is recorded rather than silent (Constraint 10). + String retain_reason; + SweepRetainClass retain_class = SweepRetainClass::None; + if (!manifestDeletionPremise(view_it->second, ManifestKey{parsed->key, parsed->prefix}, + &retain_reason, &retain_class)) + { + /// The sentence goes to the debug log for whoever is chasing ONE object; the class goes to + /// a counter, which is the only form in which this path can report itself (see + /// `ManifestSweepResult`). Both come from the predicate; neither is derived from the other. + LOG_DEBUG(getLogger("CasOrphanManifestSweep"), + "CAS orphan sweep: retaining {} -- {}", parsed->key, retain_reason); + switch (retain_class) + { + case SweepRetainClass::NoCoverage: ++result.retained_no_coverage; break; + case SweepRetainClass::Hold: ++result.retained_hold; break; + case SweepRetainClass::UnconsumedSeal: ++result.retained_unconsumed_seal; break; + case SweepRetainClass::TailRemoval: ++result.retained_tail_removal; break; + case SweepRetainClass::WorkBudgetExhausted: ++result.retained_work_budget; break; /// unreachable: the premise never returns this class itself + case SweepRetainClass::None: break; /// unreachable: the premise refused + } + ++result.skipped; + decided_through = listed.key; + continue; + } + } + + /// This exact token and bytes were captured before the catalog cut (see above). A missing body + /// has no deletion authority; a later replacement loses the old token at `deleteExact`. + /// + /// A well-formed key can legitimately be ABSENT here: the freeze loop above caps + /// fan-out at `nomination_budget` candidates, so a key beyond that cap was never frozen. Treat + /// it exactly like nomination-count exhaustion -- retain, and do NOT advance the cursor past + /// it, so the very next page/round examines it with a fresh budget instead of losing it. + const auto observed_it = observed_candidates.find(parsed->key); + if (observed_it == observed_candidates.end()) + { + budget_exhausted = true; + ++result.skipped; + continue; + } + const std::optional & got = observed_it->second; + if (!got) + { + ++result.skipped; + decided_through = listed.key; + continue; + } + const PartManifest body = decodePartManifest(openObject(FormatId::PartManifest, got->bytes)); + const ManifestId id{parsed->ns, parsed->ref}; + if (!refMatchesBody(id.ref, body) || !manifestNamespaceMatches(id.root_namespace, body)) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS orphan sweep: manifest identity mismatch at {} while deriving exact source edges", + parsed->key); + + ManifestSweepResult::Nomination nomination{ + .id = id, + .key = parsed->key, + .token = got->token, + .source_retirements = {}}; + for (const ManifestEntry & entry : body.entries) + if (entry.placement == EntryPlacement::Blob) + nomination.source_retirements.push_back(BlobSourceRetirement{ + .ref = entry.ref, + .source_id = sourceEdgeId(id, entry.path)}); + result.nominations.push_back(std::move(nomination)); + decided_through = listed.key; + } + + if (budget_exhausted) + { + /// Resume strictly after the last DECIDED key, leaving every undecided candidate ahead of the + /// cursor. `wrapped` stays false because this page did not reach the end of the keyspace — it + /// stopped early, and reporting a wrap would tell the caller the sweep had made a full circuit + /// over keys it never looked at. + result.next_cursor = decided_through; + result.wrapped = false; + return result; + } + + result.next_cursor = page.next_cursor; + result.wrapped = page.next_cursor.empty(); + return result; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.h new file mode 100644 index 000000000000..91714dacc7c8 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.h @@ -0,0 +1,206 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// One writer build prefix under `cas/manifests//`: the canonical hex `-/` +/// directory encoded as canonical hexadecimal epoch and sequence components. +struct BuildPrefix +{ + uint64_t writer_epoch = 0; + uint64_t build_sequence = 0; +}; + + +/// One manifest object the sweep is considering, carrying both halves the §6 deletion premise needs: +/// the full object key (what the tail's removal targets are named by) and the build prefix it lives +/// under (whose `writer_epoch` is the epoch whose closing seal must be consumed). +struct ManifestKey +{ + String key; + BuildPrefix prefix; +}; + +/// WHY the §6 deletion premise refused one manifest, as a CLASS rather than a sentence. The sentence +/// (`retain_reason`) names one object and is what an operator reads; the class is what a counter can +/// aggregate, and aggregating is the only way "the sweep retained every manifest in the pool, all for +/// the same reason" becomes visible at all. Deriving the class by matching substrings of the sentence +/// would make the prose load-bearing, so the predicate reports both and neither is parsed out of the +/// other. +enum class SweepRetainClass : uint8_t +{ + None = 0, /// the premise admitted the deletion; no retention happened + NoCoverage, /// no sealed coverage row for the namespace (a classification-0 row counts here) + Hold, /// the namespace is held, or is classified clamped + UnconsumedSeal, /// rule (1): the cursor has not consumed the build epoch's closing seal + TailRemoval, /// rule (2): an unconsumed tail record names this manifest as a removal target + WorkBudgetExhausted, /// the round's per-namespace or recovery-op work budget was spent before this + /// namespace's protection view could be built (or built completely); retained + /// rather than decided without a complete view (fail-closed, never a partial one) +}; + +/// The class as a short stable word, for log lines and metric names. +std::string_view sweepRetainClassName(SweepRetainClass c); + +/// The durable per-namespace fold state the §6 deletion premise reads. It is taken from the ADOPTED +/// fold seal — the `gc/state` -> `fold_seal` pair the sweep already read to learn its cursor, kept +/// whole instead of reduced to that one field. +/// +/// IT IS NOT DERIVED FROM A LISTING, and the type exists to keep it that way. Arithmetic ref intake +/// demoted the listing to a hint precisely because a store may omit a durable ref-log key from an +/// enumeration; a premise that re-derived its answer by listing the tail would inherit that hole and +/// license exactly the deletion it exists to withhold. +struct NamespaceFoldView +{ + /// The namespace's shard-0 coverage row. `nullopt` means the adopted seal carries no row for this + /// namespace at all: no round has ever sealed a ref cursor for it, so no epoch's closing seal is + /// proven consumed and every manifest under it is retained. + std::optional coverage; + + /// Manifest object keys the tail ABOVE the cursor names as REMOVAL targets, as the namespace's + /// protection view collected them. Rule (2) is a POSITIVE test against this set: a key found here + /// retains. Its negative direction proves nothing on its own — the set is assembled from the same + /// enumeration arithmetic intake distrusts — which is why rule (1) and not this set is what makes + /// the tail decidable. See `manifestDeletionPremise`. + std::set tail_removal_targets; +}; + +/// Spec §6, the sweep deletion premise, as ONE predicate both sweep paths call. A manifest of an +/// epoch-`E` build is deletable only when: +/// (1) the namespace cursor has consumed epoch `E`'s seal, AND +/// (2) no unconsumed tail record above the cursor names it as a removal target +/// (removals cross epochs; grants do not). +/// ANY uncertainty — an unreached frontier, an exhausted budget, a hold — means RETAIN. Delay is never +/// damage: a body kept one round longer costs storage, while a body deleted under an unproven cut is +/// either data loss (an unfolded `+1` still names it) or a fold that clamps forever on the missing body. +/// +/// Rule (1) is what makes rule (2) decidable rather than a second guess at the same enumeration. +/// Grants do not cross epochs, so every `+1` that could name an epoch-`E` build lives among epoch `E`'s +/// own records; and an epoch is left ONLY over its consumed `EpochSeal` (INV-2), so a sealed cursor in +/// a STRICTLY HIGHER epoch is durable proof that every one of those records is folded. Removals do +/// cross epochs, which is why rule (2) exists at all and why it is a separate test. +/// +/// `retain_reason`, when non-null, receives the reason the premise refused — it feeds the sweep's +/// `warnings` out-param so a retained manifest is a visible decision rather than a silent one. +/// `retain_class`, when non-null, receives the same refusal as a `SweepRetainClass` — what the cursor +/// page counts, since it has no `warnings` to carry the sentence. +bool manifestDeletionPremise(const NamespaceFoldView & view, const ManifestKey & manifest, + String * retain_reason, SweepRetainClass * retain_class = nullptr); + +/// Read one namespace's fold view out of the adopted fold seal. `tail_removal_targets` is left empty: +/// the callers that have a protection view fill it from theirs, and a caller that has none gets the +/// coverage half alone, which is the half rule (1) needs. +NamespaceFoldView namespaceFoldView(Pool & store, const RootNamespace & ns); + + + +/// Counters returned by one bounded cursor page. `listed` counts keys in the backend page, `skipped` +/// counts malformed, protected, ineligible, budget-exhausted, or race-spared keys, and `deleted` counts +/// only successful exact-token deletions. `next_cursor` and `wrapped` describe the backend cursor; the +/// cursor is a cleanup-progress hint and is never used as reachability authority. +/// +/// THE `retained_*` COUNTERS ARE THIS PATH'S ONLY VOICE. They break the §6 premise's share of +/// `skipped` out by reason class. Unlike `sweepNamespace`, the cursor page has no `warnings` +/// out-param and nothing downstream of it reads a per-object sentence, so without these a background +/// sweep retaining every manifest in the pool is indistinguishable, at any production log level, from +/// a sweep that had nothing to do. In Stage A that is not an edge case: rule (1) is satisfiable only +/// for a closed-and-folded epoch, so retention is the NORMAL outcome and these counters are very +/// nearly the whole story of what the sweep did and why. +struct ManifestSweepResult +{ + String next_cursor; + bool wrapped = false; + uint64_t listed = 0; + uint64_t deleted = 0; + uint64_t skipped = 0; + uint64_t retained_no_coverage = 0; + uint64_t retained_hold = 0; + uint64_t retained_unconsumed_seal = 0; + uint64_t retained_tail_removal = 0; + uint64_t retained_work_budget = 0; + + /// Exact-GET/decode candidates. The reducer must adopt every `source_retirements` entry before the + /// caller may exact-token-delete `key` with `token`. + struct Nomination + { + ManifestId id; + String key; + Token token; + std::vector source_retirements; + }; + std::vector nominations; + + /// The reason class that retained the most manifests on this page and how many — the rollup that + /// answers "why is manifest debris not shrinking?". `{None, 0}` means the premise retained nothing. + /// Ties resolve to the first class in enum order, which is stable across passes so an unchanged + /// pool reports an unchanged verdict. + std::pair topRetainReason() const; +}; + +/// Per-namespace pre-precommit orphan sweep. Deletes +/// manifest bodies written before `PrecommitAdd` and never named by any live owner, scoped to ONE +/// namespace + ONE build prefix. Rules: +/// - eligibility from the durable watermark fact only: the retired sentinel +/// (`min_active == UINT64_MAX`), or `min_active > build_sequence`, or a replaced incarnation — +/// NEVER a frozen-seq / judged-dead heuristic alone (a missing watermark => not eligible); +/// - the active `ManifestId` set comes from the namespace's committed + live-precommit owner view; +/// - delete only bodies whose `ManifestId` is ABSENT from the active set, by exact token; +/// - emits NO blob deltas (a pre-precommit body never contributed `+1`); +/// - a 404 between listing and deletion is record-and-continue, never a throw; +/// - never GETs a condemned body to revive it — eligibility + +/// exact-token delete only. +/// Returns the number of bodies actually deleted (a `DeleteClass::Deleted`-classified exact-token +/// delete only, never a spared `NotFound`/`TokenMismatch`) — the decommission manifest-debris drain +/// (`Core/CasDecommission.cpp`) sums this across every eligible build prefix into +/// `DecommissionReport::manifest_debris_removed`. +/// +/// `warnings`, when non-null, opts in to the decommission drain's tolerate-and-continue contract: a +/// per-key transient failure (a thrown backend exception on `head`/`deleteExact`) +/// is pushed onto `*warnings` and the sweep continues with the next key, instead of throwing out of +/// this call; likewise a protection-view-unavailable namespace (the pre-existing corrupt-snapshot skip +/// below) also pushes a "cannot confirm emptiness" warning, not just a `LOG_WARNING`. `warnings == +/// nullptr` (the default, every pre-existing caller) preserves the original behaviour exactly: a +/// per-key failure propagates as an exception (fail-close default), and the protection-view skip is +/// log-only. `NotFound`/`TokenMismatch` delete outcomes stay silently spared either way — those are the +/// normal "a fresh owner reclaimed it" race the periodic sweep expects, not a failure to warn about. +/// This direct decommission path relies on the caller's held server-root claim/fence: while that claim +/// is held, a same-server-root rebirth cannot become live between its catalog cut and exact-token delete. +/// It still rejects every catalog cut with an ambiguous current life id before making any deletion decision. +uint64_t sweepNamespace(Pool & store, const RootNamespace & ns, const BuildPrefix & prefix, + std::vector * warnings = nullptr); + +/// Whether `prefix` is sweep-eligible by the durable watermark fact alone. The floor is read from the +/// mount lease identified by the namespace's server-root prefix, not inferred from the manifest key or a +/// judged-dead heuristic. A missing lease provides no deletion authority, so the prefix is not eligible. +bool prefixEligible(Pool & store, const RootNamespace & ns, const BuildPrefix & prefix); + +/// Plan one cursor page without deleting. Every candidate is exact-GET, decoded and identity-validated; +/// its exact manifest-source edges are returned for accounting-neutral retirement in the next fold. +/// Catalog-named namespaces are retain-only unless the caller explicitly authorizes recovery from its +/// frozen catalog cut and the exact `_ckpt` frontier of the life named there. +/// +/// `work_budget`, when set, bounds the body-GET/retention fan-out to `nomination_budget` well-formed +/// candidates (never the whole `list_budget`-sized page), caps how many DISTINCT namespaces this page +/// may build a fresh protection view for, and caps the committed-tail recovery walk's ref-log GET +/// count cumulatively across the round (shared with every other destructive-work family via the same +/// `GcRoundWorkBudget` instance). Exhausting either cap retains every remaining candidate belonging to +/// the affected namespace on THIS page rather than deciding it without a complete protection view; +/// `nullptr` (the default) reproduces the pre-budget unbounded behavior. +ManifestSweepResult planManifestCursorPage( + Pool & store, + const String & cursor, + uint64_t list_budget, + uint64_t nomination_budget, + bool catalog_recovery_authoritative, + GcRoundWorkBudget * work_budget = nullptr); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CatalogLifecycleReconciler.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CatalogLifecycleReconciler.cpp new file mode 100644 index 000000000000..899f93504fde --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CatalogLifecycleReconciler.cpp @@ -0,0 +1,121 @@ +#include +#include + +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} +} + +namespace DB::Cas +{ + +CatalogLifecycleReconciler::CatalogLifecycleReconciler( + Backend & backend_, const Layout & layout_, const CasFoldSeal & adopted_parent_, + uint64_t admitted_generation_, + std::function check_fence_) + : backend(backend_) + , layout(layout_) + , adopted_parent(adopted_parent_) + , admitted_generation(admitted_generation_) + , check_fence(std::move(check_fence_)) +{ +} + +std::optional CatalogLifecycleReconciler::selectEligible( + const CasRefCatalog::Snapshot & catalog) const +{ + for (const CatalogEntry & entry : catalog.catalog.entries) + { + if (entry.state != NsState::Removing) + continue; + + const auto parent_row = adopted_parent.ref_lives.find(entry.incarnation); + if (parent_row == adopted_parent.ref_lives.end() + || !parent_row->second.cleanup_evidence + || parent_row->second.coverage.hold) + continue; + + return entry; + } + return std::nullopt; +} + +CatalogResolution CatalogLifecycleReconciler::resolveExactRow( + const CasRefCatalog::Snapshot & catalog, const CatalogEntry & observed) +{ + const auto current = std::find_if( + catalog.catalog.entries.begin(), + catalog.catalog.entries.end(), + [&](const CatalogEntry & entry) { return entry.ns.string() == observed.ns.string(); }); + if (current == catalog.catalog.entries.end()) + return CatalogResolution::ExactRowAbsent; + if (current->incarnation != observed.incarnation) + return CatalogResolution::ExactRowReplaced; + return CatalogResolution::ExactRowStillPresent; +} + +CatalogLifecycleReconcileResult CatalogLifecycleReconciler::reconcile() +{ + CatalogLifecycleReconcileResult result{ + .authority_status = AuthorityStatus::Authoritative, + .catalog_resolution = CatalogResolution::DrainComplete, + .retired_lives = {}, + .final_catalog_cut = std::nullopt, + .deleted = 0}; + CasRefCatalog::Snapshot catalog = CasRefCatalog::read(backend, layout); + + for (;;) + { + const std::optional eligible = selectEligible(catalog); + if (!eligible) + { + if (check_fence(admitted_generation) == CasRefCatalog::LeaderFenceStatus::Moved) + { + result.authority_status = AuthorityStatus::FencedOut; + return result; + } + result.catalog_resolution = CatalogResolution::DrainComplete; + result.final_catalog_cut = std::move(catalog); + return result; + } + + CasRefCatalog::CompletedRemovingDeleteResult delete_result + = CasRefCatalog::deleteCompletedRemovingAtSnapshot( + backend, layout, std::move(catalog), *eligible, adopted_parent, + admitted_generation, check_fence); + if (!delete_result.catalog_snapshot) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS catalog lifecycle reconciliation returned no catalog resolution snapshot"); + + catalog = std::move(*delete_result.catalog_snapshot); + result.catalog_resolution = resolveExactRow(catalog, *eligible); + if (delete_result.invalidated_life) + result.retired_lives.push_back(*delete_result.invalidated_life); + + if (delete_result.outcome == CasRefCatalog::CompletedRemovingDeleteOutcome::FencedOut) + { + result.authority_status = AuthorityStatus::FencedOut; + return result; + } + if (delete_result.outcome == CasRefCatalog::CompletedRemovingDeleteOutcome::ProofRefused) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS catalog lifecycle reconciliation selected namespace '{}' without matching no-hold cleanup evidence", + eligible->ns.string()); + if (result.catalog_resolution == CatalogResolution::ExactRowStillPresent) + throwCasWriteRetryLater(fmt::format( + "CAS catalog lifecycle reconciliation left completed-removal namespace '{}' present", + eligible->ns.string())); + if (delete_result.outcome == CasRefCatalog::CompletedRemovingDeleteOutcome::Deleted) + ++result.deleted; + } +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CatalogLifecycleReconciler.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CatalogLifecycleReconciler.h new file mode 100644 index 000000000000..fd34011c64ef --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CatalogLifecycleReconciler.h @@ -0,0 +1,64 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace DB::Cas +{ + +enum class AuthorityStatus : uint8_t +{ + Authoritative, + FencedOut, +}; + +enum class CatalogResolution : uint8_t +{ + DrainComplete, + ExactRowAbsent, + ExactRowReplaced, + ExactRowStillPresent, +}; + +/// The catalog-only result produced before a GC round may enumerate or publish successor state. +struct CatalogLifecycleReconcileResult +{ + AuthorityStatus authority_status; + CatalogResolution catalog_resolution; + std::vector retired_lives; + std::optional final_catalog_cut; + uint64_t deleted = 0; +}; + +/// Settles catalog rows that an already-adopted parent fold seal proved safe to remove. +/// +/// This component owns only deterministic eligible-row selection and the catalog `N + 1` drain. +/// It neither discovers the parent seal nor performs a hot ref LIST, ref walk, fold, publication, +/// runtime invalidation, or physical deletion. +class CatalogLifecycleReconciler +{ +public: + CatalogLifecycleReconciler( + Backend & backend_, const Layout & layout_, const CasFoldSeal & adopted_parent_, + uint64_t admitted_generation_, + std::function check_fence_); + + CatalogLifecycleReconcileResult reconcile(); + +private: + std::optional selectEligible(const CasRefCatalog::Snapshot & catalog) const; + static CatalogResolution resolveExactRow( + const CasRefCatalog::Snapshot & catalog, const CatalogEntry & observed); + + Backend & backend; + const Layout & layout; + const CasFoldSeal & adopted_parent; + uint64_t admitted_generation; + std::function check_fence; +}; + +} From 379e94821030fe68a2e5009c6149e4c6c20b30b2 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:35 +0200 Subject: [PATCH 19/30] CAS subsystem: Tools layer fsck (integrity checking), inspect, and pool-member decommission. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../Tools/CasDecommission.cpp | 476 +++++++ .../ContentAddressed/Tools/CasDecommission.h | 55 + .../ContentAddressed/Tools/CasFsck.cpp | 1180 +++++++++++++++++ .../ContentAddressed/Tools/CasFsck.h | 285 ++++ .../ContentAddressed/Tools/CasInspect.cpp | 638 +++++++++ .../ContentAddressed/Tools/CasInspect.h | 30 + 6 files changed, 2664 insertions(+) create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasFsck.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasFsck.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasInspect.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasInspect.h diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.cpp new file mode 100644 index 000000000000..416f5c327663 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.cpp @@ -0,0 +1,476 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +namespace +{ + +uint64_t nowMs() +{ + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); +} + +/// Delete every object listed under `prefix` by its listed (or, absent a list-token backend, HEAD'd) +/// token. This backs the staging and roots drain phases below: the victim's writers are fenced by the +/// decommission claim (`Pool::openForDecommission`), so nothing should be racing these deletes, and a +/// plain exact-token delete of every listed object is race-free. +/// +/// A per-object failure — a backend exception, a `TokenMismatch` or `NotFound` outcome, or an object +/// disappearing between `LIST` and `HEAD` — is recorded as a warning and does not prevent the remaining +/// objects from being attempted. The caller keeps the pool slot whenever warnings are present, so the +/// terminated slot remains available as a resume anchor instead of being deleted after an unconfirmed +/// drain. Returns only the objects whose exact-token delete was reported as `Deleted`. +uint64_t deleteListedPrefix(Backend & backend, const String & prefix, std::vector & warnings) +{ + uint64_t deleted = 0; + forEachListedKey(backend, prefix, [&](const ListedKey & listed) + { + try + { + Token token; + if (listed.token) + token = *listed.token; + else + { + const HeadResult head = backend.head(listed.key); + if (!head.exists) + { + warnings.push_back("decommission drain: " + listed.key + " vanished before delete"); + return; + } + token = head.token; + } + + const DeleteOutcome outcome = backend.deleteExact(listed.key, token); + const DeleteClass outcome_class = classifyDeleteOutcome(outcome); + if (outcome_class == DeleteClass::Deleted) + ++deleted; + else + warnings.push_back("decommission drain: " + listed.key + " delete outcome " + + String(deleteClassName(outcome_class))); + } + catch (...) + { + warnings.push_back("decommission drain: " + listed.key + " delete failed: " + + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + } + }); + return deleted; +} + +/// Delete one slot control object by a token captured at the protocol-defined fence point. Slot +/// retirement is fail-closed: unlike the debris drains above, any non-`Deleted` outcome or exception +/// stops the tail before it can touch the next control object. +bool deleteSlotObject(Backend & backend, const String & key, const Token & token, std::vector & warnings) +{ + try + { + const DeleteOutcome outcome = backend.deleteExact(key, token); + const DeleteClass outcome_class = classifyDeleteOutcome(outcome); + if (outcome_class == DeleteClass::Deleted) + return true; + + warnings.push_back("slot delete failed: " + key + ": delete outcome " + + String(deleteClassName(outcome_class))); + } + catch (...) + { + warnings.push_back("slot delete failed: " + key + ": " + + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + } + return false; +} + +} + +DecommissionReport decommissionPoolMember(BackendPtr backend, PoolConfig config, + const String & victim_srid, const CasEventSink & sink, + const std::function & request_gc_round) +{ + DecommissionReport report; + report.srid = victim_srid; + bool gc_round_needed = false; + /// A namespace may have reached `Removing` before a later namespace fails closed. Preserve the + /// already-earned liveness signal on every exit: the callback only wakes the existing serialized + /// GC worker and cannot perform catalog work itself. + SCOPE_EXIT({ + if (gc_round_needed && request_gc_round) + request_gc_round(); + }); + + /// Validate one required immutable ownership cut before impersonating the victim. The admin open + /// performs its own fresh catalog observation for mount safety, but namespace selection below + /// must reuse this exact pre-mutation decision rather than read a later authority set. + const Layout catalog_layout(config.pool_prefix); + const CasRefCatalog::Snapshot catalog_cut = CasRefCatalog::read(*backend, catalog_layout); + catalog_cut.life_index.throwIfAmbiguous("CAS decommission"); + + config.event_sink = sink; + PoolPtr admin = Pool::openForDecommission(std::move(backend), std::move(config), victim_srid); + + EventEmitter{*admin}.emit([&](CasEvent & e) + { + e.type = CasEventType::MemberDecommission; + e.outcome = "begin"; + e.reason = "operator decommission of pool member"; + e.detail = {{"server_root_id", victim_srid}}; + }); + + /// The pre-impersonation catalog cut is the complete ownership universe. Physical life keys carry + /// no logical path, and raw string prefixes such as `victim` must not select the distinct owner + /// `victim2`; the slash makes `victim` one canonical path component. + const String victim_namespace_prefix = victim_srid + "/"; + std::vector> owned_lives; + for (const CatalogEntry & entry : catalog_cut.catalog.entries) + { + if (entry.ns.string() != victim_srid && !entry.ns.string().starts_with(victim_namespace_prefix)) + continue; + const auto life = catalog_cut.life_index.resolve(entry.incarnation); + if (!life) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "ca-decommission: catalog entry '{}' has no physical life resolution", entry.ns.string()); + owned_lives.emplace_back(entry, *life); + } + + for (const auto & [selected_entry, life] : owned_lives) + { + const RootNamespace & ns = life.ns; + const String & ns_str = ns.string(); + + /// Refuse a same-name lifecycle move that landed after the immutable selection cut. The + /// exact-life overloads below also pin recovery to `life`, closing the race after this check: + /// a later replacement can never redirect a removal to its new incarnation. + const CasRefCatalog::Snapshot current_catalog = CasRefCatalog::read(admin->backend(), admin->layout()); + const auto current_entry = std::find_if( + current_catalog.catalog.entries.begin(), current_catalog.catalog.entries.end(), + [&](const CatalogEntry & entry) { return entry.ns.string() == ns_str; }); + if (current_entry == current_catalog.catalog.entries.end() || *current_entry != selected_entry) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "ca-decommission: namespace '{}' changed incarnation after the validated catalog cut; " + "refusing destructive work", + ns_str); + + if (selected_entry.state == NsState::Removing) + { + if (!admin->backend().head(admin->layout().refCkptKey(life)).exists) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "ca-decommission: namespace '{}' is Removing but its exact checkpoint is absent; " + "the catalog row remains owned and the victim slot cannot be retired", + ns_str); + + /// `dropNamespace` is the sole terminal writer. On an already-complete removal this is an + /// idempotent observation; on a pre-terminal `Removing` life it resumes the exact terminal + /// append under the administrative writer fence. Catalog deletion remains GC's job. + (void)admin->dropNamespace(life); + ++report.namespaces_already_removed; + gc_round_needed = true; + continue; + } + + const auto stats = admin->dropNamespace(life); + ++report.namespaces_removed; + report.committed_refs_removed += stats.committed_refs; + report.precommits_removed += stats.precommits; + report.edge_deltas_emitted += stats.committed_refs + stats.precommits; + if (selected_entry.state != NsState::Creating) + gc_round_needed = true; + + EventEmitter{*admin}.emit([&](CasEvent & e) + { + e.type = CasEventType::MemberDecommission; + e.outcome = "namespace_removed"; + e.reason = "decommission dropped a victim namespace"; + e.detail = {{"server_root_id", victim_srid}, {"namespace", ns_str}, + {"committed", std::to_string(stats.committed_refs)}, + {"precommits", std::to_string(stats.precommits)}}; + }); + } + + /// Manifest debris must be removed before the mount slot: deleting the mount body removes the + /// watermark authority, after which `floorForNamespace` returns no value and the ordinary orphan + /// sweep cannot prove that old-epoch debris is eligible. The decommission claim has advanced the + /// writer epoch, so every build prefix with `prefix.writer_epoch < w.writer_epoch` is eligible here. + /// Group the listed keys by namespace and build prefix so each group can use the exact-token orphan + /// sweep while the mount body still supplies its authority. + { + const String debris_prefix = admin->layout().casManifestsServerPrefix(victim_srid); + std::set> groups; /// (namespace, writer epoch, build sequence) + forEachListedKey(admin->backend(), debris_prefix, [&](const ListedKey & listed) + { + if (const auto parsed = admin->layout().parseManifestKey(listed.key)) + groups.emplace(parsed->root_namespace.string(), parsed->ref.writer_epoch, parsed->ref.build_sequence); + }); + for (const auto & [ns_str, writer_epoch, build_sequence] : groups) + report.manifest_debris_removed += sweepNamespace( + *admin, RootNamespace(ns_str), BuildPrefix{writer_epoch, build_sequence}, &report.warnings); + } + + /// Drain the victim's own `/staging//` area. The live-mount staging helper uses + /// an `IObjectStorage`, while this command intentionally works at the `Backend` layer, so the same + /// prefix is listed and deleted directly. The claim fences the victim's writers during this sweep. + report.staging_objects_removed += deleteListedPrefix( + admin->backend(), admin->poolConfig().pool_prefix + "/staging/" + victim_srid + "/", report.warnings); + + /// Drain the victim's mountpoint objects. These are loose, non-content-addressed files under + /// `Layout::serverRootDataPrefix`; they have no writer epoch of their own, so the claim is what + /// prevents a returning victim from racing this deletion. + report.mountpoint_objects_removed += deleteListedPrefix( + admin->backend(), admin->layout().serverRootDataPrefix(victim_srid), report.warnings); + + /// The catalog, not physical debris, owns the slot-retirement decision. A terminal append only + /// moves a row to `Removing`; GC must fold/prune/delete it before the member's ownership anchor can + /// disappear. Capture one exact whole-catalog cut after every drain, then revalidate its token and + /// canonical value immediately before entering the retirement tail. The administrative claim fences + /// the victim writer between those observations. + std::optional retirement_catalog_cut; + if (report.warnings.empty()) + { + retirement_catalog_cut = CasRefCatalog::read(admin->backend(), admin->layout()); + const uint64_t victim_owned_count = std::count_if( + retirement_catalog_cut->catalog.entries.begin(), retirement_catalog_cut->catalog.entries.end(), + [&](const CatalogEntry & entry) + { + return entry.ns.string() == victim_srid + || entry.ns.string().starts_with(victim_namespace_prefix); + }); + if (victim_owned_count > 0) + report.warnings.push_back( + "pool member decommission underway: " + std::to_string(victim_owned_count) + + " namespace(s) are still owned by this member; upcoming GC rounds perform the final " + "cleanup — re-run this command afterwards to retire the slot"); + } + + /// Retire the slot strictly last and only after a clean drain. Copy the layout and shared backend + /// before `admin.reset()`: graceful close destroys the `Pool`, while the backend must remain alive to + /// retire the slot objects afterwards. + const Layout layout = admin->layout(); + const BackendPtr pool_backend = admin->poolBackendPtr(); + if (report.warnings.empty()) + { + const CasRefCatalog::Snapshot fresh_retirement_catalog + = CasRefCatalog::read(admin->backend(), admin->layout()); + if (!retirement_catalog_cut + || fresh_retirement_catalog.token != retirement_catalog_cut->token + || fresh_retirement_catalog.catalog != retirement_catalog_cut->catalog) + { + report.warnings.push_back( + "catalog changed after the victim ownership check; refusing slot retirement against a stale cut"); + } + } + if (report.warnings.empty()) + { + const String mount_key = layout.mountKey(victim_srid); + const String epoch_key = layout.epochKey(victim_srid); + const String owner_key = layout.ownerKey(victim_srid); + + /// Capture both the epoch value and its exact token while the decommission claim still fences + /// the victim. A successor can only bump this object after the farewell below releases the + /// claim, so this token is the epoch-side successor fence for the retirement tail. + std::optional claimed_epoch; + try + { + claimed_epoch = pool_backend->get(epoch_key); + if (!claimed_epoch) + report.warnings.push_back("slot capture failed: " + epoch_key + " is absent under the admin claim"); + } + catch (...) + { + report.warnings.push_back("slot capture failed: " + epoch_key + ": " + + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + } + + /// Graceful close stamps an already-expired lease and the watermark farewell + /// (`min_active = UINT64_MAX`), making the slot `terminated` before its mutable control objects + /// are removed and its owner anchor is tombstoned. + admin.reset(); + + /// Read the farewell immediately after `finishTeardown` wrote it. Its exact token is the + /// mount-side fence: deleting by this token can remove only THIS decommission's farewell, not + /// a successor reclaim. Validate the body against the epoch value captured under the claim so + /// a successor that completed before this GET is also recognized and left untouched. + std::optional farewell_mount; + try + { + farewell_mount = pool_backend->get(mount_key); + if (!farewell_mount) + report.warnings.push_back("slot capture failed: " + mount_key + " farewell is absent"); + } + catch (...) + { + report.warnings.push_back("slot capture failed: " + mount_key + ": " + + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + } + + bool captures_match = claimed_epoch && farewell_mount; + if (captures_match) + { + try + { + const ServerEpoch epoch_value = decodeServerEpoch(claimed_epoch->bytes); + const MountLease mount_value = decodeMountLease(farewell_mount->bytes); + captures_match = epoch_value.next_writer_epoch != 0 + && mount_value.writer_epoch == epoch_value.next_writer_epoch - 1 + && mount_value.min_active == std::numeric_limits::max() + && !mount_value.gc_fenced; + if (!captures_match) + { + report.warnings.push_back( + "slot capture failed: " + mount_key + + " is not this decommission's farewell for the epoch captured under the admin claim"); + } + } + catch (...) + { + report.warnings.push_back("slot capture failed while validating " + mount_key + " and " + epoch_key + ": " + + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + captures_match = false; + } + } + + /// Mount first: if a successor reclaimed it after the farewell capture, the stale farewell + /// token yields `TokenMismatch` and the tail stops before touching epoch or owner. Epoch second: + /// its under-claim token similarly detects a successor allocation. Before touching owner, re-read + /// both mutable objects: a same-UUID successor can recreate them after both deletes without + /// rewriting the owner identity anchor. Mere presence proves that the slot is live again. Every + /// delete must be explicitly confirmed as `Deleted`, and the final owner tombstone rewrite must + /// succeed against the exact token read immediately before it. + /// + /// ACCEPTED RESIDUAL WINDOW (final review, not closed by this recheck): a same-UUID successor + /// can still recreate epoch/mount in the narrow gap strictly AFTER this liveness recheck but + /// BEFORE the owner CAS below reads its own token -- the successor's owner anchor (same + /// server_uuid, not yet retired) then gets tombstoned by this decommission run. The successor's + /// live process is not deleted (only its owner anchor is marked retired), but a LATER restart of + /// that same identity would refuse to reclaim it (claimOwnerOrThrow's tombstone guard). This is + /// a narrow, low-probability window, deliberately not closed here: T5's owner-tombstone design + /// (finding #9) intentionally stopped short of making concurrent decommission-vs-recreate + /// airtight to the microsecond, since that was explicitly not the priority for this fix. + report.slot_removed = false; + if (captures_match && deleteSlotObject(*pool_backend, mount_key, farewell_mount->token, report.warnings) + && deleteSlotObject(*pool_backend, epoch_key, claimed_epoch->token, report.warnings)) + { + std::optional current_mount; + std::optional current_epoch; + bool liveness_recheck_succeeded = true; + try + { + current_mount = pool_backend->get(mount_key); + } + catch (...) + { + report.warnings.push_back("slot liveness recheck failed: " + mount_key + ": " + + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + liveness_recheck_succeeded = false; + } + try + { + current_epoch = pool_backend->get(epoch_key); + } + catch (...) + { + report.warnings.push_back("slot liveness recheck failed: " + epoch_key + ": " + + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + liveness_recheck_succeeded = false; + } + + if (liveness_recheck_succeeded && (current_mount || current_epoch)) + { + report.warnings.push_back( + "slot delete aborted: successor reappeared after mutable control-object deletion; owner kept"); + } + else if (liveness_recheck_succeeded) + { + try + { + if (const auto owner = pool_backend->get(owner_key)) + { + OwnerObject tombstoned = decodeOwner(owner->bytes); + tombstoned.retired_at_ms = nowMs(); + /// Controlled, not a bare putOverwrite: a transient transport error here (or + /// one whose response was simply lost) must not be reported as a hard failure + /// when the write actually landed. A standalone controller (decommission is an + /// administrative, non-hot-path operation; no mount-lease fence applies to it + /// -- the exact-token CAS itself is the safety mechanism, same as the mount/ + /// epoch deletes above) resolves an ambiguous attempt with one GET: unchanged + /// token means the write never applied (legitimately retryable within budget); + /// matching bytes means this exact tombstone already landed (Committed, not a + /// failure); anything else is a genuine successor reclaim (Conflict). + CasRequestController controller(pool_backend, CasRequestBudget{}); + const CasOverwriteResult result = controller.putOverwriteControlled( + owner_key, encodeOwner(tombstoned), owner->token, [] { return true; }); + if (result.outcome == CasOverwriteOutcome::Committed) + report.slot_removed = true; + else if (result.outcome == CasOverwriteOutcome::Conflict) + report.warnings.push_back( + "slot tombstone failed: " + owner_key + + ": successor reclaimed the owner anchor before this decommission's tombstone write"); + else + report.warnings.push_back( + "slot tombstone failed: " + owner_key + + ": tombstone write outcome could not be resolved (retry budget exhausted " + "or the resolve GET itself failed) -- rerun the command to retry"); + } + else + report.warnings.push_back( + "slot tombstone failed: " + owner_key + ": object absent before tombstone write"); + } + catch (...) + { + report.warnings.push_back("slot tombstone failed: " + owner_key + ": " + + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + } + } + } + } + else + { + report.slot_removed = false; + LOG_WARNING(getLogger("CasDecommission"), + "CAS decommission '{}': drain incomplete ({} warnings) — mount slot kept (terminated); " + "re-run the command to finish", victim_srid, report.warnings.size()); + admin.reset(); /// Graceful close still stamps the farewell, leaving the slot `terminated`. + } + + /// The `end` event is emitted via `sink` directly, not `EventEmitter{*admin}`: `admin` is gone by + /// now. This also means its `warnings` count reflects the FINAL total, including a slot-retirement + /// failure appended just above -- `EventEmitter`'s own zero-cost-when-absent guard is reproduced by + /// the `if (sink)` below. + if (sink) + { + CasEvent e; + e.type = CasEventType::MemberDecommission; + e.outcome = "end"; + e.reason = "decommission finished"; + e.detail = {{"server_root_id", victim_srid}, + {"namespaces_removed", std::to_string(report.namespaces_removed)}, + {"warnings", std::to_string(report.warnings.size())}, + {"slot_removed", report.slot_removed ? "1" : "0"}}; + sink(std::move(e)); + } + return report; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.h new file mode 100644 index 000000000000..a86edf9286c4 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Counts the work performed by `decommissionPoolMember` for one pool member. The namespace counters +/// describe metadata and ref-log transitions; the object counters describe physical objects deleted by +/// the manifest, staging, and mountpoint drains. Blob bytes are intentionally not reported: removing +/// ref edges makes them eligible for ordinary GC, but this operation does not synchronously reclaim +/// shared content. +/// +/// A decommission is resumable. A previous run may already have moved namespaces to `Removing`, and a +/// warning means that the corresponding drain was not confirmed. In either case the report lets the +/// caller distinguish work done by this invocation from work observed from an earlier invocation. +struct DecommissionReport +{ + String srid; /// The decommissioned member's `server_root_id`. + uint64_t namespaces_removed = 0; /// Namespaces erased by this invocation. + uint64_t namespaces_already_removed = 0; /// Namespaces already `Removing` on entry. + uint64_t committed_refs_removed = 0; /// Committed ref records removed by namespace drops. + uint64_t precommits_removed = 0; /// Precommit records removed by namespace drops. + uint64_t edge_deltas_emitted = 0; /// The sum of `committed_refs_removed` and `precommits_removed`. + uint64_t manifest_debris_removed = 0; /// Eligible manifest objects deleted from old build prefixes. + uint64_t staging_objects_removed = 0; /// Objects deleted from the member's staging prefix. + uint64_t mountpoint_objects_removed = 0; /// Objects deleted from the member's roots/mountpoint prefix. + bool slot_removed = false; /// Whether mount and epoch were deleted and the owner was tombstoned. + std::vector warnings; /// Drain or slot-retirement failures; a non-empty list keeps the slot. +}; + +/// Erases all content owned by a permanently dead pool member. The operation first claims the member's +/// slot as an administrative writer; a live lease is refused, and the claim fences the dead member from +/// writing while cleanup runs. It then drops each table namespace through `Pool::dropNamespace`, drains +/// eligible manifest debris, staging objects, and mountpoint objects, and retires the slot only after all +/// drains are confirmed. Namespace drops are idempotent: a rerun resumes any missing terminal append +/// and leaves exact catalog-row deletion to GC. The member slot remains while any catalog entry still +/// belongs to the victim. +/// +/// This is a writer operation, not GC: it emits the normal ref-edge deltas and does not invent ref +/// transitions. Per-object drain failures are recorded in `DecommissionReport::warnings` and leave the +/// terminated slot as a resume anchor; other failures, including refusal to claim the member, propagate +/// as exceptions. When set, `sink` receives `MemberDecommission` audit events for the run's begin, +/// per-namespace, and end milestones. +DecommissionReport decommissionPoolMember(BackendPtr backend, PoolConfig config, + const String & victim_srid, const CasEventSink & sink = {}, + const std::function & request_gc_round = {}); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasFsck.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasFsck.cpp new file mode 100644 index 000000000000..7848ae591f27 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasFsck.cpp @@ -0,0 +1,1180 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int TIMEOUT_EXCEEDED; +} +} + +namespace DB::Cas +{ + +namespace +{ +constexpr uint64_t PROGRESS_PAGES = 16; + +using Deadline = std::optional; + +/// Enforce the optional overall scan deadline between backend operations. A timeout is propagated as +/// `TIMEOUT_EXCEEDED`; the public `runFsck` wrapper may convert that exception into a partial report when +/// explicitly requested. +void checkDeadline(const Deadline & deadline, std::string_view phase) +{ + if (deadline && std::chrono::steady_clock::now() > *deadline) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, + "fsck: exceeded the deadline during '{}' — run against a QUIESCED pool or raise --timeout.", phase); +} + +void listAll(Backend & backend, const String & prefix, std::unordered_map & out, + const FsckProgress & on_progress, const Deadline & deadline, std::string_view phase) +{ + static constexpr size_t kPageLimit = 1000; + uint64_t pages = 0; + size_t count_in_page = 0; + forEachListedKey(backend, prefix, [&](const ListedKey & k) + { + out[k.key] = k.size; + if (++count_in_page == kPageLimit) + { + count_in_page = 0; + ++pages; + checkDeadline(deadline, phase); + if (on_progress && pages % PROGRESS_PAGES == 0) + on_progress(phase, out.size(), pages); + } + }, kPageLimit); + /// The walk's `backend.list` lands at least once even for an empty/undersized final page -- + /// check it here, mirroring the original per-page loop (deadline checked after every physical page). + if (count_in_page > 0 || pages == 0) + { + ++pages; + checkDeadline(deadline, phase); + } + if (on_progress) + on_progress(phase, out.size(), pages); +} + +/// Parse (writer_epoch, build_sequence) from a manifest object key. Delegates to the one shared +/// `Layout::parseManifestKey` instead of hand-rolling a second parser; returns false on a +/// malformed or foreign key. +bool parseBuildPrefix(const Layout & layout, const String & key, BuildPrefix & out) +{ + const auto parsed = layout.parseManifestKey(key); + if (!parsed) + return false; + out.writer_epoch = parsed->ref.writer_epoch; + out.build_sequence = parsed->ref.build_sequence; + return true; +} + +/// The ref-walk (which builds `reachable_blobs`/`blob_labels`) and the HEAD-confirm below run minutes +/// apart with no snapshot between them. A ref that gets republished (now names a +/// different manifest) or DROPPED in that window, combined with a legitimate GC delete of the OLD +/// blob, makes the stale walk look like a genuine dangle (a "phantom dangling") — this made the fsck +/// oracle dishonest and falsely report a dangle during long-running validation. +/// +/// Before counting a HEAD-absent blob as `Dangling`, re-resolve every `"ns/ref"` label under the same +/// immutable catalog row, using a fresh exact `_ckpt` from that original physical life. This admits a +/// same-life repoint/drop while refusing a competing rebirth. `label` is split on the LAST '/' — +/// mirroring exactly how the walk built it (`ns_str + "/" + ref_name`): `ref_name` never contains '/', +/// but `ns_str` may, so the join separator is always the rightmost one. +/// +/// Fails CLOSED on any ambiguity (a malformed label, a recovery error, a corrupt manifest): treated as +/// "still referenced", i.e. the original conservative verdict. +/// The fix can only SHRINK false positives — it must never hide a real one. +struct FsckRecoveryAuthority +{ + NamespaceLifeId life; + CatalogEntry catalog_entry; + std::optional checkpoint; +}; + +using FsckRecoveryAuthorities = std::unordered_map; +using RecordRecoveryUnchecked = std::function; + +/// Recheck one ref table against a newer `_ckpt` from the SAME physical life selected by fsck's +/// original catalog cut. The catalog row and life id never move; only the monotone checkpoint may +/// advance, which is how a same-life drop/repoint that completed during a long scan becomes visible +/// without admitting a competing rebirth. A missing or unreadable checkpoint cannot prove that an +/// old owner went away, so the caller records lost coverage and keeps the conservative verdict. +std::optional recoverLateRefTable( + Backend & backend, const Layout & layout, const FsckRecoveryAuthority & authority, + const RecordRecoveryUnchecked & record_unchecked) +{ + try + { + const std::optional sampled = readCkpt(backend, layout, authority.life); + if (!sampled) + { + record_unchecked(authority.life.ns, layout.refCkptKey(authority.life), + "late ref recheck: the original life checkpoint is absent"); + return std::nullopt; + } + return recoverRefTableDetailedFromAuthority( + backend, layout, authority.catalog_entry, sampled->ckpt).state; + } + catch (const Exception & e) + { + record_unchecked(authority.life.ns, layout.refCkptKey(authority.life), + "late ref recheck: the original life checkpoint or replay is unreadable: " + e.message()); + return std::nullopt; + } + catch (...) + { + record_unchecked(authority.life.ns, layout.refCkptKey(authority.life), + "late ref recheck: the original life checkpoint or replay could not be read"); + return std::nullopt; + } +} + +bool blobStillReferenced(Pool & store, const Layout & layout, + const FsckRecoveryAuthorities & authorities, const String & bkey, + const std::vector & labels, const Deadline & deadline, + const RecordRecoveryUnchecked & record_unchecked) +{ + if (labels.empty()) + return true; + for (const String & label : labels) + { + checkDeadline(deadline, "re-resolving refs at HEAD-absent"); + const size_t slash = label.rfind('/'); + if (slash == String::npos) + return true; /// malformed label — cannot re-resolve, fail closed + const String ns_part = label.substr(0, slash); + const String ref_name = label.substr(slash + 1); + try + { + /// Never read a second catalog cut here. A later rebirth may name the same logical namespace + /// but it is not the life whose original row made this blob reachable in this fsck pass. + const auto authority_it = authorities.find(ns_part); + if (authority_it == authorities.end()) + { + record_unchecked(RootNamespace{ns_part}, layout.refCatalogKey(), + "late blob recheck: no original Live/Removing authority was retained"); + return true; /// no original Live/Removing authority -- fail closed + } + const RootNamespace rns{ns_part}; + const std::optional table = recoverLateRefTable( + store.backend(), layout, authority_it->second, record_unchecked); + if (!table) + return true; + const auto rit = table->getCommitted().find(ref_name); + if (rit == table->getCommitted().end()) + continue; /// the ref was DROPPED since the walk — this label no longer applies + const PartManifest body = store.readManifest(ManifestId{rns, rit->second.manifest_ref}); + for (const ManifestEntry & e : body.entries) + { + if (e.placement != EntryPlacement::Blob) + continue; + if (layout.blobKey(e.ref) == bkey) + return true; /// an original-life ref still names this exact blob — a real dangle + } + } + catch (...) + { + return true; /// cannot confirm the ref moved away — keep the conservative verdict + } + } + return false; /// no original-life label names this blob — the stale-walk artifact is gone +} + +/// The manifest sibling of the `blobStillReferenced` recheck above. The ref-walk captures each committed +/// `(ref_name -> manifest_ref)` from a FRESH per-namespace recovery, but the `backend.get(mkey)` that +/// confirms the manifest body runs LATER in the same (possibly long) namespace loop. A ref republished to +/// a DIFFERENT manifest — or DROPPED — in that window, combined with a legitimate GC delete of the OLD +/// manifest body, makes the stale captured row look like a committed ref over a missing manifest (a +/// "phantom dangling manifest"), the same dishonest-oracle failure `blobStillReferenced` kills for blobs. +/// +/// Before counting a missing manifest body as `Dangling`, re-resolve the EXACT ref from the SAME frozen +/// catalog row with a fresh exact `_ckpt` from that original physical life, then check whether the +/// committed row still names THIS exact manifest key. A later catalog cut must not replace that row, +/// but a same-life checkpoint advance must be visible. Fails CLOSED on any ambiguity (a throw, a corrupt +/// table): treated as "still referenced", the original conservative verdict — the fix can only SHRINK +/// false positives, never hide a real loss. +bool manifestStillReferenced(Backend & backend, const Layout & layout, const RootNamespace & ns, + const FsckRecoveryAuthorities & authorities, const String & ref_name, + const String & mkey, const Deadline & deadline, + const RecordRecoveryUnchecked & record_unchecked) +{ + checkDeadline(deadline, "re-resolving ref at missing-manifest"); + try + { + const auto authority_it = authorities.find(ns.string()); + if (authority_it == authorities.end()) + { + record_unchecked(ns, layout.refCatalogKey(), + "late manifest recheck: no original Live/Removing authority was retained"); + return true; /// no original Live/Removing authority -- fail closed + } + const std::optional table = recoverLateRefTable( + backend, layout, authority_it->second, record_unchecked); + if (!table) + return true; + const auto rit = table->getCommitted().find(ref_name); + if (rit == table->getCommitted().end()) + return false; /// the ref was DROPPED since the walk — no longer a committed owner + /// A republish moved the ref to a different manifest key: this old key is no longer owned. + return layout.manifestKey(ManifestId{ns, rit->second.manifest_ref}) == mkey; + } + catch (...) + { + return true; /// cannot confirm the ref moved away — keep the conservative verdict + } +} + +String renderId(const RefTxnId & id) +{ + return std::to_string(id.writer_epoch) + "-" + std::to_string(id.ref_sequence); +} + +/// Per-NAMESPACE verdicts of the stream audit. Both counters count namespaces, not rows: a namespace +/// has exactly one answer about its stream even when several checks reach it. +/// +/// A namespace PROVEN broken is never also counted `unchecked`. "Proved broken" and "could not prove" +/// are different answers, and letting the second overwrite or accompany the first would turn a fatal +/// into an ambiguity — the recovery path throws on a holed stream, so a chain-broken namespace reliably +/// produces a downstream failure too, and that failure must not dilute the verdict that explains it. +struct NsVerdicts +{ + std::set chain_broken; + std::set unchecked; + + void recordChainBroken(FsckReport & report, const RootNamespace & ns, const String & key, String note) + { + chain_broken.insert(ns.string()); + unchecked.erase(ns.string()); + push(report, key, FsckClass::ChainBroken, std::move(note)); + } + + void recordUnchecked(FsckReport & report, const RootNamespace & ns, const String & key, String note) + { + if (chain_broken.contains(ns.string())) + return; + unchecked.insert(ns.string()); + push(report, key, FsckClass::Unchecked, std::move(note)); + } + + /// Both classes are emitted in EVERY mode, not just `detail`: they are namespace verdicts, bounded + /// by the namespace count, and a summary run that hid them would report a number nobody could act on. + void push(FsckReport & report, const String & key, FsckClass cls, String note) const + { + FsckObject o; + o.key = key; + o.kind = ObjectKind::Blob; /// ref objects have no ObjectKind; reuse Blob as the generic kind + o.size = 0; + o.cls = cls; + o.reachable_from = {std::move(note)}; + report.objects.push_back(std::move(o)); + } + + void publish(FsckReport & report) const + { + report.chain_broken = chain_broken.size(); + report.unchecked = unchecked.size(); + } +}; + +/// THE ARITHMETIC STREAM WALK (spec §7). Read-only, one namespace. +/// +/// The frozen catalog row and exact `_ckpt` define the complete finite walk. LIST supplies no genesis, +/// witness, frontier or stop condition, and the walker never probes the position after +/// `_ckpt.committed_through`. Every required id is point-read from the checkpoint base's successor (or +/// `{life_epoch, 1}`) through that inclusive frontier. A missing required id is therefore a proven hole; +/// no above-hole listing witness is needed. An epoch seal advances directly to the next epoch's first id, +/// exactly as authoritative read-only recovery does. +void checkRefStream(Backend & backend, const Layout & layout, const NamespaceLifeId & life, + const CatalogEntry & catalog_entry, const std::optional & checkpoint_sample, + const Deadline & deadline, FsckReport & report, NsVerdicts & verdicts) +{ + checkDeadline(deadline, "ref stream"); + const RootNamespace & ns = life.ns; + const std::optional checkpoint + = checkpoint_sample ? std::optional{checkpoint_sample->ckpt} : std::nullopt; + const RecoveryGrounding grounding = chooseRecoveryGrounding(catalog_entry, checkpoint); + if (grounding.base) + { + try + { + /// Even when the base IS the frontier and there is no replay tail, a checkpoint may not + /// turn an `EpochSeal` into a state snapshot by naming a same-id `_snap`. + (void)readCheckpointSnapshotBase(backend, layout, life, *checkpoint); + } + catch (const Exception & e) + { + const String key = layout.refSnapshotKey(life, *grounding.base); + const String note = "ref stream: checkpoint snapshot base " + renderId(*grounding.base) + + " is invalid: " + e.message(); + if (e.code() != ErrorCodes::CORRUPTED_DATA) + { + verdicts.recordUnchecked(report, ns, key, note); + return; + } + + /// A concurrent checkpoint advance may retire the sampled base between these exact reads. + /// Only the SAME checkpoint incarnation turns a missing/invalid member of its required + /// triple into durable corruption. A changed, absent, or unreadable authority proves no + /// such thing and remains the honest `Unchecked` answer. + checkDeadline(deadline, "checkpoint-base authority revalidation"); + try + { + const std::optional current = readCkpt(backend, layout, life); + if (!current || !checkpoint_sample || current->token != checkpoint_sample->token) + { + verdicts.recordUnchecked(report, ns, key, + note + "; checkpoint authority changed while validating its snapshot base"); + return; + } + } + catch (const Exception & revalidation_error) + { + verdicts.recordUnchecked(report, ns, key, + note + "; checkpoint authority could not be revalidated: " + revalidation_error.message()); + return; + } + catch (...) + { + verdicts.recordUnchecked(report, ns, key, + note + "; checkpoint authority could not be revalidated"); + return; + } + + verdicts.recordChainBroken(report, ns, key, note); + return; + } + } + if (!grounding.walk_from || !grounding.committed_through) + return; + + RefTxnId expected = *grounding.walk_from; + while (expected <= *grounding.committed_through) + { + checkDeadline(deadline, "ref stream"); + const auto got = backend.get(layout.refLogKey(life, expected)); + if (!got) + { + verdicts.recordChainBroken(report, ns, layout.refLogKey(life, expected), + "ref stream: checkpoint requires id " + renderId(expected) + " at or below inclusive frontier " + + renderId(*grounding.committed_through) + ", but its exact key is absent"); + return; + } + + bool is_seal = false; + try + { + is_seal = refLogTxnIsEpochSeal( + decodeRefLogTxn(openObject(FormatId::RefLog, got->bytes), ns.string(), expected)); + } + catch (const Exception & e) + { + verdicts.recordUnchecked(report, ns, layout.refLogKey(life, expected), + "ref stream: the checkpoint-required record at " + renderId(expected) + + " could not be decoded: " + e.message()); + return; + } + ++report.ref_records_walked; + + try + { + if (const std::optional next = nextRefLogIdWithinCommittedFrontier( + expected, is_seal, *grounding.committed_through)) + expected = *next; + else + break; + } + catch (const Exception & e) + { + verdicts.recordChainBroken(report, ns, layout.refLogKey(life, expected), + "ref stream: " + e.message()); + return; + } + } +} + +/// Perform the scan and accumulate into `report`. This helper owns the read-only traversal: it first +/// recovers authoritative refs, then checks physical objects and GC labels, while preserving the +/// distinction between a missing live object and expected in-flight cleanup. Deadline exceptions are +/// intentionally left to `runFsck`, which decides whether partial results were requested. +void runFsckImpl(Pool & store, bool detail, const FsckProgress & on_progress, const Deadline & deadline, + const String & namespace_prefix, FsckReport & report) +{ + const Layout & layout = store.layout(); + Backend & backend = store.backend(); + /// Path-derived per-object algorithm parsing: every listed blob-tree key -- across every + /// admitted algo, not just the pool's node-local write algo -- is classified via + /// `Layout::parseBlobKey`, which derives the `BlobRef` from the key's OWN `` path segment + /// (and its `.meta` sibling). A foreign/malformed key (unknown algo segment, wrong-width hex, a + /// non-`.meta`/non-blob shape) parses to `std::nullopt` and is classified as debris, never an + /// exception. + + /// Reachability is recomputed from the authoritative refs (never from GC state): + /// for each namespace, each committed ref resolves to a ManifestId; read its body; a committed ref + /// naming a MISSING body is an ERROR (Dangling); a present body whose blobs are missing is an ERROR. + std::set reachable_blobs; /// blob object keys named by a live owner + std::set owned_manifest_keys; /// manifest object keys named by a committed owner + /// blob key -> "ns/ref" labels of the refs that named it. Always populated (not just under + /// `detail`) — the HEAD-absent re-resolve below needs it in every mode. + std::unordered_map> blob_labels; + + uint64_t refs_walked = 0; + NsVerdicts verdicts; + SCOPE_EXIT({ verdicts.publish(report); }); + const RecordRecoveryUnchecked record_recovery_unchecked = + [&](const RootNamespace & ns, const String & key, const String & detail_text) + { + verdicts.recordUnchecked(report, ns, key, detail_text); + }; + + /// RECORD AND CONTINUE for a key that belongs to no namespace at all. fsck is the forensic tool an + /// operator reaches for once something is already wrong, so a key it cannot attribute must become a + /// FINDING and not an abort: an audit that died on the first bad key would report nothing about the + /// healthy namespaces it never reached, which is the wrong failure order for a read-only diagnostic. + /// + /// `seen` is what makes the count a count of DEFECTS: each sweep below enumerates namespaces again + /// and sees the same offending key, and only the first sighting is recorded. + std::set lifeless_seen; + auto recordLifelessKeys = [&](const NamespaceListing & listing) + { + for (const UnattributableNamespaceKey & bad : listing.skipped) + { + if (!lifeless_seen.insert(bad.key).second) + continue; + ++report.lifeless_keys; + FsckObject o; + o.key = bad.key; + o.kind = ObjectKind::Blob; /// a lifeless key has no ObjectKind; reuse Blob as the generic kind + o.cls = FsckClass::LifelessKey; + o.size = 0; + o.reachable_from = {bad.reason}; + report.objects.push_back(std::move(o)); + } + }; + + /// One immutable cut owns every physical-id join in this walk. `Creating` participates in that + /// attribution (its physical keys may exist) but is never recovered: only Live/Removing rows have a + /// durable publication frontier. A diagnostic records duplicate ids and keeps walking unrelated + /// unique lives. + const CasRefCatalog::Snapshot catalog_cut = CasRefCatalog::read(backend, layout); + struct FsckWalkLife + { + NamespaceLifeId life; + CatalogEntry catalog_entry; + }; + std::vector walk_lives; + walk_lives.reserve(catalog_cut.catalog.entries.size()); + for (const CatalogEntry & entry : catalog_cut.catalog.entries) + { + if (!entry.ns.string().starts_with(namespace_prefix)) + continue; + if (entry.state == NsState::Creating) + continue; + try + { + if (const auto life = catalog_cut.life_index.resolve(entry.incarnation)) + walk_lives.push_back(FsckWalkLife{.life = *life, .catalog_entry = entry}); + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::CORRUPTED_DATA) + throw; + recordLifelessKeys(NamespaceListing{{}, {{ + layout.refCatalogKey() + "#" + renderIncarnation(entry.incarnation), e.message()}}}); + } + } + + /// Physical life-owned keys carry no logical name. Classify each COMPLETE, canonical key against a + /// catalog cut taken AFTER this physical listing finishes (observe-then-cut), not the earlier + /// `catalog_cut` above: `NamespaceJanitor::runOnePage` (the only real deleter of this debris) uses + /// the identical ordering, and it is what makes "life absent from a LATER cut" sound -- creation + /// always admits a `Creating` catalog row before writing any life-owned object (spec §2), so a life + /// that is absent from a cut taken after the listing cannot be a concurrent birth this listing raced. + /// A malformed shape (the parser refuses, or the reserved segment names no clean relative file) is + /// classified immediately as it cannot become residue no matter which cut resolves it. + if (namespace_prefix.empty()) + { + struct CanonicalNamespaceKey + { + String key; + uint64_t size; + NamespaceLifePhysicalId life_id; + }; + std::vector canonical_candidates; + + forEachListedKey(backend, layout.namespaceRootPrefix(), [&](const ListedKey & listed) + { + std::optional physical_id; + try + { + if (const auto ref_object = layout.parseRefObjectKey(listed.key)) + physical_id = ref_object->life_id; + else if (const auto checkpoint = layout.parseRefCkptKey(listed.key)) + physical_id = *checkpoint; + else if (const auto namespace_file = layout.parseNamespaceFileKey(listed.key)) + physical_id = namespace_file->life_id; + else + { + recordLifelessKeys(NamespaceListing{{}, {{listed.key, "unrecognized key under the namespace ownership tree"}}}); + return; + } + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::CORRUPTED_DATA) + throw; + recordLifelessKeys(NamespaceListing{{}, {{listed.key, e.message()}}}); + return; + } + canonical_candidates.push_back(CanonicalNamespaceKey{listed.key, listed.size, *physical_id}); + }); + + /// The post-observation cut. All three catalog states -- `Creating`, `Live`, `Removing` -- + /// protect a life for this purpose; only a life absent from every one of them is residue. + const CasRefCatalog::Snapshot post_listing_cut = CasRefCatalog::read(backend, layout); + std::unordered_set pending_lives; + for (const CanonicalNamespaceKey & candidate : canonical_candidates) + { + try + { + if (post_listing_cut.life_index.resolve(candidate.life_id)) + continue; /// protected by some catalog state as of the later cut -- not residue + } + catch (const Exception & e) + { + /// The reverse life index throws `CORRUPTED_DATA` when the post-listing cut carries a + /// duplicated life id: a catalog defect, not evidence about THIS key. Record and keep + /// walking, same as every other catalog-authority failure in this scan -- an audit that + /// aborted on the first bad key would report nothing about the healthy candidates + /// still queued behind it. + if (e.code() != ErrorCodes::CORRUPTED_DATA) + throw; + recordLifelessKeys(NamespaceListing{{}, {{candidate.key, e.message()}}}); + continue; + } + ++report.namespace_janitor_pending; + report.namespace_janitor_pending_bytes += candidate.size; + pending_lives.insert(candidate.life_id); + FsckObject o; + o.key = candidate.key; + o.kind = ObjectKind::Blob; /// no ObjectKind names namespace-life debris; reuse Blob as the generic kind + o.cls = FsckClass::JanitorPending; + o.size = candidate.size; + o.reachable_from = {"physical life id is absent from a catalog cut taken after this listing; " + "janitor-pending, not corruption"}; + report.objects.push_back(std::move(o)); + } + report.namespace_janitor_pending_lives = pending_lives.size(); + } + + /// Every replay and late recheck below reuses the same exact catalog row and physical life. The + /// primary walk also retains its checkpoint sample; a late recheck exact-reads `_ckpt` again at that + /// SAME life so a concurrent same-life drop/repoint is visible without ever accepting a rebirth. + FsckRecoveryAuthorities recovery_authorities; + recovery_authorities.reserve(walk_lives.size()); + + for (const FsckWalkLife & walk_life : walk_lives) + { + const NamespaceLifeId & life = walk_life.life; + const RootNamespace & ns = life.ns; + const String & ns_str = ns.string(); + /// RECORD AND CONTINUE, NEVER WEDGE. Everything below is per-namespace, and every one of these + /// steps can raise `CORRUPTED_DATA` on a namespace whose stream is damaged — the replay refuses a + /// non-contiguous tail, the codecs refuse an invalid body. For RECOVERY that throw is the correct + /// fail-close; for a read-only diagnostic it is a bug, because the audit then reports NOTHING + /// about the namespaces it never reached, including the healthy ones. So one namespace's failure + /// becomes that namespace's verdict and the sweep goes on. + /// + /// `TIMEOUT_EXCEEDED` is deliberately NOT caught: the deadline is a property of the whole scan, + /// and `runFsck`'s `partial` handling owns it. + try + { + /// One materialized `_ckpt` body is part of this namespace's frozen audit authority. The + /// recovery API receives exactly these bytes; `checkRefStream` receives the same decoded + /// value, so the two legs cannot quietly choose different frontiers after a concurrent CAS. + const std::optional checkpoint_sample = readCkpt(backend, layout, life); + const std::optional checkpoint + = checkpoint_sample ? std::optional{checkpoint_sample->ckpt} : std::nullopt; + const auto [authority_it, inserted] = recovery_authorities.emplace( + ns.string(), FsckRecoveryAuthority{ + .life = life, .catalog_entry = walk_life.catalog_entry, .checkpoint = checkpoint}); + chassert(inserted); + + /// The arithmetic stream audit runs FIRST, so a holed stream gets the verdict that EXPLAINS + /// it (`chain-broken`) rather than the downstream `CORRUPTED_DATA` the replay below would + /// raise about the same hole. + checkRefStream( + backend, layout, life, walk_life.catalog_entry, checkpoint_sample, deadline, report, verdicts); + + /// This recovery's finite range comes from the original catalog row and exact `_ckpt`, never + /// from a stream listing, a self-resolved name, or an F+1 probe. + const RefTableState table = recoverRefTableDetailedFromAuthority( + backend, layout, authority_it->second.catalog_entry, authority_it->second.checkpoint).state; + for (const auto [ref_name, row] : table.getCommitted()) + { + const ManifestId id{ns, row.manifest_ref}; + const String mkey = layout.manifestKey(id); + owned_manifest_keys.insert(mkey); + const String label = ns_str + "/" + ref_name; + + const auto got = backend.get(mkey); + if (!got) + { + /// A committed ref naming a missing manifest body would be an INV-NO-DANGLE violation — + /// but the per-ref GET runs later than the namespace's ref recovery, so a stale captured + /// row plus a legitimate GC delete of a since-superseded manifest can masquerade as one, + /// and a bare GET can lag a present object. Revalidate exactly like the blob `Dangling` + /// recheck below: HEAD the exact object AND re-resolve under the original catalog row + /// plus a fresh checkpoint from its physical life. Count the dangle ONLY when the exact + /// object is HEAD-absent AND that life still names THIS exact manifest — otherwise it is + /// LIST/GET lag or a phantom stale-row, never a loss. + if (!backend.head(mkey).exists + && manifestStillReferenced(backend, layout, ns, recovery_authorities, ref_name, mkey, + deadline, record_recovery_unchecked)) + { + ++report.dangling; + FsckObject o; + o.key = mkey; + o.kind = ObjectKind::Blob; /// manifests have no ObjectKind; reuse Blob as the generic kind + o.size = 0; + o.cls = FsckClass::Dangling; + o.reachable_from = {label}; + report.objects.push_back(std::move(o)); + } + /// A present object is GET lag. A row not named by the original-life authority is a + /// stale-walk artifact, not a dangle; its original owner cannot contribute blobs. + ++refs_walked; + continue; + } + + PartManifest body = decodePartManifest(openObject(FormatId::PartManifest, got->bytes)); + if (!refMatchesBody(id.ref, body) || !manifestNamespaceMatches(id.root_namespace, body)) + { + ++report.dangling; + FsckObject o; + o.key = mkey; + o.kind = ObjectKind::Blob; + o.size = got->bytes.size(); + o.cls = FsckClass::Dangling; + o.reachable_from = {label}; + report.objects.push_back(std::move(o)); + ++refs_walked; + continue; + } + + for (const ManifestEntry & e : body.entries) + { + if (e.placement != EntryPlacement::Blob) + continue; + const String bkey = layout.blobKey(e.ref); + reachable_blobs.insert(bkey); + ++report.total_blob_refs; + report.referenced_logical_bytes += e.blob_size; + blob_labels[bkey].push_back(label); + } + + ++refs_walked; + checkDeadline(deadline, "walking refs"); + if (on_progress && refs_walked % 64 == 0) + on_progress("walking refs", reachable_blobs.size(), refs_walked); + } + } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::TIMEOUT_EXCEEDED) + throw; + verdicts.recordUnchecked(report, ns, + layout.namespaceStreamPrefix(life), + "fsck could not examine this namespace: " + e.message()); + } + } + report.distinct_blobs = reachable_blobs.size(); + + /// Scoped mode skips the GLOBAL physical classification below: it is meaningless under a + /// filter (blobs owned by other namespaces would read as unreachable) and would cost a + /// pool-wide LIST for what should be O(scoped refs). + if (namespace_prefix.empty()) + { + /// Physical listing: blobs + manifest bodies. The per-hash `.meta` descriptor sibling + /// (`blobMetaKey(id) == blobKey(id) + ".meta"`) lives under the SAME + /// `blobsPrefix()` as the body, so partition the raw LIST into bodies vs `.meta` objects up + /// front — a `.meta` key must never be classified as a content body (it would otherwise be + /// misread as an unreferenced blob and fall into the dangling/pending/unaccounted pipeline + /// below), and a body must never be misread as a `.meta`. + std::unordered_map present_all; + listAll(backend, layout.blobsPrefix(), present_all, on_progress, deadline, "listing blobs"); + std::unordered_map present_blobs; + std::unordered_set present_meta_hashes; + present_blobs.reserve(present_all.size()); + for (const auto & [key, sz] : present_all) + { + if (key.ends_with(".meta")) + { + if (const std::optional ref = layout.parseBlobKey(key)) + present_meta_hashes.insert(*ref); + /// else: foreign key shape under blobs/ — not ours to pair + } + else + present_blobs.emplace(key, sz); + } + for (const auto & [_, sz] : present_blobs) + report.physical_bytes += sz; + + /// Reachable blobs must be present (HEAD-confirm against LIST lag before declaring loss). + for (const String & bkey : reachable_blobs) + { + auto it = present_blobs.find(bkey); + bool exists = it != present_blobs.end(); + uint64_t size = exists ? it->second : 0; + if (!exists) + { + const HeadResult h = backend.head(bkey); + if (h.exists) + { + exists = true; + size = h.size; + report.physical_bytes += h.size; + } + } + + const auto lit = blob_labels.find(bkey); + if (!exists) + { + /// Before declaring a loss, re-resolve the referencing refs from the original audit + /// authority. A later rebirth must not replace the old owner while this verdict is being + /// decided. + const bool still_referenced = blobStillReferenced(store, layout, recovery_authorities, bkey, + lit != blob_labels.end() ? lit->second : std::vector{}, deadline, + record_recovery_unchecked); + if (!still_referenced) + continue; /// stale-walk artifact: neither reachable nor dangling — skip entirely + } + + if (exists) + ++report.reachable; + else + ++report.dangling; + if (detail || !exists) + { + FsckObject o; + o.key = bkey; + o.kind = ObjectKind::Blob; + o.size = size; + o.cls = exists ? FsckClass::Reachable : FsckClass::Dangling; + if (detail && lit != blob_labels.end()) + o.reachable_from = lit->second; + report.objects.push_back(std::move(o)); + } + } + + /// Present-but-unreferenced blobs: classify through the GC pipeline view instead of one + /// suspicious "unreachable" lump (the multi-stage graduation keeps a nonzero churning + /// set here on ANY active pool, and beta testers read "unreachable" as a leak). The GC state is + /// read for LABELING ONLY — reachability above never consults it. + std::unordered_map retired_by_hash; + std::unordered_set unref_hashes; + std::unordered_set in_run_hashes; + /// The NON-SENTINEL source edges the snapshot still holds on each unreferenced blob, collected in + /// `detail` mode only. `in_run_hashes` alone answers "does GC still see this blob at all"; the + /// stale-edge cross-check below needs the edge IDENTITIES so it can ask whether their source + /// manifests still exist. Sentinel rows (`source_id == 0` — `kZeroMarker`/`kCondemned`) are not + /// edges and are excluded. + std::unordered_map, BlobRefHash> unref_edge_sources; + bool have_gc_state = false; + + for (const auto & [bkey, sz] : present_blobs) + if (!reachable_blobs.contains(bkey)) + { + if (const std::optional ref = layout.parseBlobKey(bkey)) + unref_hashes.insert(*ref); + } + + if (!unref_hashes.empty()) + { + if (const auto state_got = backend.get(layout.gcStateKey())) + { + have_gc_state = true; + const GcState gc_state = decodeGcState(state_got->bytes); + /// The adopted fold seal names the snapshot runs; resolution is by ref, never by key + /// construction. Every row whose hash is in our candidate set marks "known to GC" — + /// edges still counted (drop unfolded), an explicit zero-marker mid-pipeline, or a + /// `kCondemned` sentinel row that carries the condemned state (retired-in-snapshot): + /// the `kCondemned` rows feed `retired_by_hash` (the `PendingGc` classification) in the + /// SAME pass, replacing the removed `retired_refs`/`decodeRetiredSet` loop. + /// + /// These sets are keyed by the full `BlobRef`, not a narrowed digest. The run's own + /// algorithm-prefixed key is parsed by `SourceEdgeKeyCodec` and compared directly with + /// the full identity parsed from the listed blob key. This is required for mixed-algorithm + /// pools: a 64-hex digest must not be truncated or compared as though it used the pool's + /// local write algorithm, or its true GC state could be hidden as `Unaccounted`. + if (const auto seal_got = backend.get(layout.foldSealKey(gc_state.snap_generation, gc_state.snap_attempt))) + { + uint64_t rows = 0; + for (const RunRef & run : decodeFoldSeal(seal_got->bytes, gc_state.snap_generation).blob_target_runs) + { + checkDeadline(deadline, "reading gc snapshot runs"); + /// Typed open: the source-edge run reader goes through openSourceEdgeRun (the NDJSON + /// header gates type == cas_run + kind == source_edge). Fsck keys off the row's hash + /// (the record's own algo-prefixed key, never from pool meta). + SourceEdgeRunView reader = openSourceEdgeRun(backend, run.key); + String key; + String payload; + while (reader.next(key, payload)) + { + BlobRef ref; + UInt128 source_id; + SourceEdgeKeyCodec::parse(key, ref, source_id); // throws CORRUPTED_DATA on malformed (fail-closed) + if (unref_hashes.contains(ref)) + { + in_run_hashes.insert(ref); + if (detail && source_id != UInt128{0}) + unref_edge_sources[ref].push_back(source_id); + if (!payload.empty() && payload[0] == kCondemned) + { + const CondemnedRow row = decodeCondemnedRow(payload); + RetiredEntry e; + e.kind = ObjectKind::Blob; + e.ref = ref; + e.token = row.token; + e.size = row.size; + e.condemn_round = row.condemn_round; + e.delete_pending = row.delete_pending; + retired_by_hash.emplace(ref, std::move(e)); + } + } + if (on_progress && ++rows % 65536 == 0) + on_progress("reading gc snapshot runs", in_run_hashes.size(), rows); + } + /// Whole-file seal checksum: compare the drained run's accumulated + /// checksum to the seal's `RunRef::checksum`. Fsck is a read-only auditor — instead of + /// throwing (which would abort the whole scan on the first corrupt run), catalogue the + /// mismatch as a `CorruptedRun` finding (with the run key) and continue so the audit + /// enumerates every problem in one pass. The deletion-deriving consumers + /// (`fold`/`zeroInDegree`/`previewDeletes`) still fail closed on the same mismatch. + if (reader.accumulatedChecksum() != run.checksum) + { + ++report.corrupted_runs; + if (detail) + report.objects.push_back(FsckObject{.key = run.key, .cls = FsckClass::CorruptedRun, .reachable_from = {}}); + } + } + } + } + } + + /// STALE-EDGE cross-check. A residual `+1` whose matching `-1` never folded pins its blob at + /// in-degree 1 forever: every GC round recomputes the same nonzero in-degree and never nominates + /// the blob, so the `AwaitingGc` "expected, no action needed" label is a lie — nothing will ever + /// reclaim it. The edge names its source, so the check is to ask whether that source still exists: + /// build the set of source ids that every manifest body PRESENT in the pool would contribute, and + /// treat an edge outside that set as one whose source manifest is gone. + /// + /// COST: one LIST per namespace plus one GET per manifest body. It is therefore gated on `detail` + /// — the cheap summary path (the ca-soak fixpoint poll calls it in a loop) must not gain a single + /// extra request — and additionally on some unreferenced blob actually carrying a real edge, so a + /// pool with nothing to cross-check pays nothing. + /// + /// `stale_edge_check_available` is the fail-closed switch: a manifest body we cannot decode would + /// silently withhold its edges from the live set and turn every blob it owns into a false hard + /// finding, so one undecodable body disables the whole cross-check for this scan rather than + /// manufacture an error. The check may only ever SHRINK to silence, never invent a finding. + std::unordered_set live_source_ids; + bool stale_edge_check_available = detail && !unref_edge_sources.empty(); + if (stale_edge_check_available) + { + const NamespaceListing stale_edge_listing = store.listNamespaces(namespace_prefix); + recordLifelessKeys(stale_edge_listing); + for (const String & ns_str : stale_edge_listing.namespaces) + { + const RootNamespace ns{ns_str}; + std::unordered_map manifest_bodies; + listAll(backend, layout.manifestNamespacePrefix(ns), manifest_bodies, on_progress, deadline, + "listing manifests for the stale-edge check"); + for (const auto & [mkey, _] : manifest_bodies) + { + checkDeadline(deadline, "reading manifests for the stale-edge check"); + const std::optional id = layout.parseManifestKey(mkey); + if (!id) + continue; /// foreign/malformed key under `manifests/` — contributes no source edge + const auto got = backend.get(mkey); + if (!got) + continue; /// gone between the LIST and the GET — genuinely not a live source + try + { + const PartManifest body = decodePartManifest(openObject(FormatId::PartManifest, got->bytes)); + for (const ManifestEntry & e : body.entries) + if (e.placement == EntryPlacement::Blob) + live_source_ids.insert(sourceEdgeId(*id, e.path)); + } + catch (...) + { + stale_edge_check_available = false; /// incomplete live set — do not accuse anyone + break; + } + } + if (!stale_edge_check_available) + break; + } + } + + for (const auto & [bkey, sz] : present_blobs) + { + if (reachable_blobs.contains(bkey)) + continue; + ++report.unreachable; + + /// A foreign/malformed key (`parseBlobKey` -> `nullopt`) falls back to the default `BlobRef{}`, + /// which cannot match a real `retired_by_hash`/`in_run_hashes` entry — it lands in the generic + /// `Unaccounted` bucket below, exactly the "debris, not ours" classification `parseBlobKey` + /// documents: foreign algorithm segments are debris, not pool objects. + const BlobRef hash = layout.parseBlobKey(bkey).value_or(BlobRef{}); + + FsckClass cls = FsckClass::Unaccounted; + String note; + if (const auto rit = retired_by_hash.find(hash); rit != retired_by_hash.end() + && backend.head(bkey).token == rit->second.token) + { + /// The PRESENT incarnation is the condemned one — deletion is scheduled. A token + /// mismatch means the listed entry belongs to a displaced older incarnation and says + /// nothing about this object; fall through to the snapshot check. + cls = FsckClass::PendingGc; + note = rit->second.delete_pending + ? "delete_pending: exact-token delete executes next GC round" + : "condemned at round " + std::to_string(rit->second.condemn_round) + + "; graduates once every writer acks past it (expected)"; + } + else if (in_run_hashes.contains(hash)) + { + /// `in_run_hashes` only says the GC snapshot still holds SOMETHING for this blob. Split on + /// whether any of it is still actionable. One edge whose source manifest is PRESENT keeps + /// the ordinary `AwaitingGc` verdict — that manifest's removal still folds its `-1`, and an + /// unowned-but-present manifest is reclaimed by the orphan sweep, so the blob is genuinely + /// mid-pipeline. When EVERY edge names a manifest that no longer exists, no `-1` is left to + /// fold: the in-degree is pinned above zero for good and only a rebuild can clear it. + uint64_t stale_edges = 0; + bool all_edges_stale = false; + if (const auto eit = unref_edge_sources.find(hash); + stale_edge_check_available && eit != unref_edge_sources.end() && !eit->second.empty()) + { + for (const UInt128 & source_id : eit->second) + if (!live_source_ids.contains(source_id)) + ++stale_edges; + all_edges_stale = stale_edges == eit->second.size(); + } + + if (all_edges_stale) + { + cls = FsckClass::StaleEdge; + note = "all " + std::to_string(stale_edges) + " source edges name manifests that no longer " + "exist — unreclaimable by the incremental GC (needs `cas-gc-rebuild`); NOT expected, investigate"; + } + else + { + cls = FsckClass::AwaitingGc; + note = "edges still in the GC snapshot; the drop has not folded yet (expected)"; + } + } + else if (!have_gc_state) + { + cls = FsckClass::AwaitingGc; + note = "GC has not run on this pool yet"; + } + else + { + note = "not in the current GC view — transient for a fast create+drop between rounds; " + "PERSISTENT occurrences violate INV-2 (reachability-before-content), investigate"; + } + + switch (cls) + { + case FsckClass::PendingGc: ++report.pending_gc; break; + case FsckClass::AwaitingGc: ++report.awaiting_gc; break; + case FsckClass::StaleEdge: ++report.stale_edge; break; + default: ++report.unaccounted; break; + } + if (detail) + { + FsckObject o; + o.key = bkey; + o.kind = ObjectKind::Blob; + o.size = sz; + o.cls = cls; + o.reachable_from = {std::move(note)}; + report.objects.push_back(std::move(o)); + } + } + + /// Meta <-> body pairing: a `.meta` object with no + /// body is an INV-META-BODY violation (the fixed meta/body lifecycle never leaves a meta + /// orphaned of its body) — a real ERROR, distinct from `dangling` (which is reachability-driven). + /// A body with no `.meta` is a benign not-yet-adopted (or interrupted-birth) artifact, NOT a dangle + /// — it still classifies through the ordinary present-but-unreferenced pipeline above. + std::unordered_set present_body_hashes; + present_body_hashes.reserve(present_blobs.size()); + for (const auto & [bkey, _] : present_blobs) + if (const std::optional ref = layout.parseBlobKey(bkey)) + present_body_hashes.insert(*ref); + /// else: foreign key shape under blobs/ — not ours to pair + for (const BlobRef & hash : present_meta_hashes) + if (!present_body_hashes.contains(hash)) + ++report.meta_without_body; + for (const BlobRef & hash : present_body_hashes) + if (!present_meta_hashes.contains(hash)) + ++report.body_without_meta; + } + else + { + /// Scoped mode: dangling-only for the selected namespaces. Each blob named by a scoped ref + /// is HEAD-verified (O(scoped refs), no pool-wide LIST); the unreachable/pending pipeline + /// classification needs the whole pool and is intentionally skipped. + for (const String & bkey : reachable_blobs) + { + checkDeadline(deadline, "head-checking scoped blobs"); + const HeadResult h = backend.head(bkey); + const auto lit = blob_labels.find(bkey); + bool exists = h.exists; + if (!exists) + { + /// Use the same HEAD-absent re-resolve as the global-mode loop above. + const bool still_referenced = blobStillReferenced(store, layout, recovery_authorities, bkey, + lit != blob_labels.end() ? lit->second : std::vector{}, deadline, + record_recovery_unchecked); + if (!still_referenced) + continue; /// stale-walk artifact — neither reachable nor dangling + } + if (exists) + { + ++report.reachable; + report.physical_bytes += h.size; + } + else + ++report.dangling; + if (detail || !exists) + { + FsckObject o; + o.key = bkey; + o.kind = ObjectKind::Blob; + o.size = exists ? h.size : 0; + o.cls = exists ? FsckClass::Reachable : FsckClass::Dangling; + if (detail && lit != blob_labels.end()) + o.reachable_from = lit->second; + report.objects.push_back(std::move(o)); + } + } + } + + /// Pre-precommit manifest debris: a `cas/manifests/` body with no committed owner. An ELIGIBLE prefix's + /// orphan is reclaimable debris => INFO (Unreachable); a non-eligible (in-flight) one is also info, + /// never an error. The owner-visible missing-body case is the error above. + const NamespaceListing manifest_debris_listing = store.listNamespaces(namespace_prefix); + recordLifelessKeys(manifest_debris_listing); + for (const String & ns_str : manifest_debris_listing.namespaces) + { + const RootNamespace ns{ns_str}; + const String manifests_prefix = layout.manifestNamespacePrefix(ns); + std::unordered_map manifest_bodies; + listAll(backend, manifests_prefix, manifest_bodies, on_progress, deadline, "listing manifests"); + for (const auto & [mkey, sz] : manifest_bodies) + { + if (owned_manifest_keys.contains(mkey)) + continue; /// owned by a committed ref — accounted above + ++report.unreachable; + if (detail) + { + BuildPrefix prefix; + const bool parsed = parseBuildPrefix(layout, mkey, prefix); + FsckObject o; + o.key = mkey; + o.kind = ObjectKind::Blob; + o.size = sz; + o.cls = FsckClass::Unreachable; + if (parsed && prefixEligible(store, ns, prefix)) + o.reachable_from = {"reclaimable-pre-precommit"}; + else + o.reachable_from = {"in-flight-pre-precommit"}; + report.objects.push_back(std::move(o)); + } + } + } + +} + +} + +FsckReport runFsck(Pool & store, bool detail, FsckProgress on_progress, + std::optional deadline, + bool partial_on_deadline, const String & namespace_prefix) +{ + FsckReport report; + try + { + runFsckImpl(store, detail, on_progress, deadline, namespace_prefix, report); + } + catch (const Exception & e) + { + if (!partial_on_deadline || e.code() != ErrorCodes::TIMEOUT_EXCEEDED) + throw; + report.partial = true; + report.partial_reason = e.message(); + } + return report; +} + +String formatFsckSummary(const FsckReport & report) +{ + /// Field order is load-bearing for humans only; every consumer parses `key=value` tokens. `partial` + /// and its free-text reason go LAST because the reason can contain spaces and quotes, so a parser + /// splitting on whitespace has to trim from the tail (see the harness's `parse_fsck_summary`). + /// `std::ostringstream`, not a ClickHouse write buffer: this reproduces the exact `std::cout` + /// formatting the line has always had, `dedup_ratio`'s default double precision included, so + /// extracting the line from the command changes nothing a parser can observe. + std::ostringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + out << "reachable=" << report.reachable + << " dangling=" << report.dangling + << " unreachable=" << report.unreachable + << " pending_gc=" << report.pending_gc + << " awaiting_gc=" << report.awaiting_gc + << " unaccounted=" << report.unaccounted + << " stale_edge=" << report.stale_edge + << " corrupted_runs=" << report.corrupted_runs + << " chain_broken=" << report.chain_broken + << " lifeless_keys=" << report.lifeless_keys + << " janitor_pending=" << report.namespace_janitor_pending + << " janitor_pending_bytes=" << report.namespace_janitor_pending_bytes + << " janitor_pending_lives=" << report.namespace_janitor_pending_lives + << " unchecked=" << report.unchecked + << " ref_records_walked=" << report.ref_records_walked + << " physical_bytes=" << report.physical_bytes + << " referenced_logical_bytes=" << report.referenced_logical_bytes + << " distinct_blobs=" << report.distinct_blobs + << " total_blob_refs=" << report.total_blob_refs + << " dedup_ratio=" << report.dedupRatio(); + if (report.partial) + out << " partial=1 reason='" << report.partial_reason << "'"; + return out.str(); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasFsck.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasFsck.h new file mode 100644 index 000000000000..620904f4c37f --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasFsck.h @@ -0,0 +1,285 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace DB::Cas +{ + +/// Optional progress sink for `runFsck`: called periodically during the listing and reachability +/// walk so a long scan over a large/slow pool is visibly progressing (not hung). `phase` names the +/// current step; `objects`/`pages` are running counts. Default {} = no progress (existing callers). +using FsckProgress = std::function; + +/// Classification assigned to each object examined by `runFsck`. +/// +/// The reachability classes are derived only from authoritative refs and the physical object listing. +/// The GC-related classes are an additional explanation for present-but-unreferenced blobs; GC state +/// is used for labeling only and can never make a referenced object appear safe. Integrity classes are +/// hard findings: the report remains unclean when any of them is present. +enum class FsckClass : uint8_t +{ + Reachable, /// reachable from a live ref AND present in the object store + Dangling, /// reachable from a live ref but the object is MISSING — INV-NO-LOSS violation + Unreachable, /// pre-precommit manifest debris (labeled reclaimable / in-flight) + /// The GC pipeline deletes present-but-unreferenced blobs in explicit stages, so these classes + /// distinguish expected in-flight work from an object outside the GC view. They are labels only, + /// never inputs to reachability. + PendingGc, /// listed in the retired set (condemned / delete_pending) — deletion is scheduled; EXPECTED + AwaitingGc, /// edges still in the GC snapshot (drop/reclaim not folded yet) or GC never ran — EXPECTED + Unaccounted, /// absent from the whole GC view — transient for a fast create+drop between rounds; + /// PERSISTENT occurrences should be impossible (INV-2 reachability-before-content) + StaleEdge, /// every source edge the GC snapshot still holds on this blob names a manifest that no + /// longer exists anywhere in the pool, so the matching `-1` can never fold: the blob's + /// in-degree can never reach zero and the incremental GC can never reclaim it. Only a + /// full rebuild of the in-degree state can. ERROR — never an `AwaitingGc` "expected" + /// backlog, which is exactly the label that used to hide it. + CorruptedRun, /// a GC source-edge run's whole-file seal checksum (`RunRef::checksum`) disagrees with + /// the stored bytes — cataloged so the read-only audit enumerates every finding in one + /// pass; deletion-deriving consumers (`fold`, `zeroInDegree`, `previewDeletes`) still + /// fail closed on the same mismatch. ERROR + /// The two verdicts of the arithmetic ref-stream walk (spec §7). They are about a NAMESPACE, not an + /// object; the row's `key` identifies the exact log where the walk stopped or the checkpoint-named + /// snapshot base whose required triple could not be validated. + ChainBroken, /// the exact checkpoint authority is durably inconsistent: its required snapshot-base + /// triple is corrupt, or a ref-log id is absent below its confirmed frontier. Ids are + /// dense `1..T` within `(namespace, epoch)` (INV-1), so neither is a stream end. ERROR + Unchecked, /// the walk could not prove this namespace's stream EITHER WAY (an unprovable epoch + /// crossing, an undecodable body, or unstable authority/transport). Not a finding and + /// not a clean bill of health: the honest third answer, reported so nobody reads a + /// silence as a proof. + LifelessKey, /// a namespace-tree key the `Layout` parsers refuse (a malformed/non-canonical shape, + /// including the un-incarnated Stage A layout), OR a catalog incarnation that is + /// ambiguous or otherwise unreadable. Neither a current writer nor the catalog's own + /// reverse life index can produce this key's meaning, so it belongs to no namespace + /// and no per-namespace verdict can carry it. ERROR + JanitorPending,/// a COMPLETE, canonical namespace-life key (parses via the exact writer grammar, + /// nonzero 32-hex life id) whose life is simply absent from a catalog cut taken AFTER + /// the physical listing. This is the protocol-produced interval between a fenced GC + /// exact-deleting a `Removing` catalog row and the perpetual `NamespaceJanitor` + /// reaching this key on a later bounded page -- inert debris, not damage. Reported + /// as a soft finding: NOT in `kFsckHardFindings`, does not fail the report. +}; + +/// One object or integrity finding emitted in detailed mode, or emitted for every missing reachable +/// object even in summary mode. `key` identifies the physical or logical object; `size` is its listed +/// size and is zero for a missing object. `reachable_from` contains `"namespace/ref"` owners for +/// reachable and dangling objects, or a diagnostic note for other classifications. +struct FsckObject +{ + String key; + ObjectKind kind = ObjectKind::Blob; + uint64_t size = 0; /// on-disk object size (0 when dangling) + FsckClass cls = FsckClass::Reachable; + std::vector reachable_from; /// "ns/ref" labels (populated for reachable/dangling when detail) +}; + +/// Aggregate result of a read-only `runFsck` scan. +/// +/// Reachability and byte counters describe the scan's authoritative-ref view. `unreachable` is the +/// total of all present-but-unreferenced objects, including the GC pipeline classes and manifest debris, +/// and is intentionally retained as one monotone number for residual-settling monitoring. The detailed +/// `objects` list is populated according to the scan's `detail` mode. In partial mode all counters are +/// lower bounds over the portion walked before the deadline; `clean` must not be used as a claim about +/// the unvisited part of the pool. +struct FsckReport +{ + uint64_t reachable = 0; + uint64_t dangling = 0; + /// TOTAL of everything present-but-unreferenced (blob pipeline classes below + manifest debris). + /// Kept as the sum so residual-settling loops (soak) keep one monotone number to watch. + uint64_t unreachable = 0; + uint64_t pending_gc = 0; /// blobs in the retired set — deletion scheduled (expected) + uint64_t awaiting_gc = 0; /// blobs whose drop is not folded yet / GC never ran (expected) + uint64_t unaccounted = 0; /// blobs outside the GC view (transient or anomaly) + /// Blobs whose every remaining source edge names a manifest that no longer exists — permanently + /// stuck at a nonzero in-degree, unreclaimable by the incremental GC. A hard ERROR (see + /// `FsckClass::StaleEdge`). Populated only in `detail` mode: naming the live sources costs one GET + /// per manifest body, and the cheap summary path must stay request-for-request unchanged. + uint64_t stale_edge = 0; + + /// The per-hash `.meta` descriptor sibling of a blob body: + /// pairing check between the `blobs/` physical listing's `.meta` keys and its body keys. + /// ADVISORY, not a hard finding: GC deletes the body FIRST and then drops the `.meta` on a bounded, + /// error-suppressed advisory pool that runs strictly after (and may drop the op — see `CasGc`), so a + /// single raw LIST legitimately observes a body-less `.meta` mid-graduation and NO finite grace makes + /// a persistent one hard evidence. Counted and reported; excluded from `clean()`. + uint64_t meta_without_body = 0; /// a `.meta` object with no body — INV-META-BODY advisory + uint64_t body_without_meta = 0; /// a body with no `.meta` — a not-yet-adopted or interrupted-birth + /// artifact; benign, NOT a dangle + + /// GC source-edge runs whose whole-file seal checksum did not match the stored bytes. Cataloged + /// with the run key in `objects`; the audit CONTINUES — a read-only auditor + /// enumerates all problems in one pass rather than aborting on the first corrupt run. + uint64_t corrupted_runs = 0; + + /// The arithmetic ref-stream walk (spec §7). fsck reads each namespace's stream by EXACT KEY from + /// `_ckpt.checkpoint`'s successor upward — never from a listing, which may omit durable records — + /// and reports one verdict per namespace. + /// + /// `chain_broken` counts namespaces with a proven hole (see `FsckClass::ChainBroken`) and is a HARD + /// ERROR: part of `clean`, and the command exits nonzero on it. `unchecked` counts namespaces the + /// walk could not prove either way; it is COVERAGE, not a finding, so it + /// is reported and printed but does not make a report unclean — exactly like `partial`. A pool with + /// nothing wrong reads `chain_broken=0 unchecked=0`, so `unchecked` is never a resting state. + /// + /// `ref_records_walked` is how many ref-log records the walk actually read and proved, summed over + /// namespaces. It is what makes "the tail above the checkpoint was walked" observable rather than + /// inferred from the absence of a complaint. + uint64_t chain_broken = 0; + uint64_t unchecked = 0; + uint64_t ref_records_walked = 0; + + /// Keys the namespace enumeration could not attribute to any namespace, OR a catalog incarnation + /// that is ambiguous or unreadable (see `FsckClass::LifelessKey` and `Cas::NamespaceListing`). Does + /// NOT include a complete, canonical namespace-life key whose life is simply absent from the catalog + /// -- that is `namespace_janitor_pending`, counted separately and not a hard finding. Counted + /// DISTINCT by key: the scan enumerates namespaces several times and every sweep sees the same + /// offending key, so a per-sweep count would multiply one defect. A hard finding: no current writer + /// can produce this key's meaning, and an audit is where an operator finds out about it. + uint64_t lifeless_keys = 0; + + /// Canonical namespace-life keys whose life is absent from a catalog cut taken AFTER the physical + /// listing (see `FsckClass::JanitorPending`). SOFT: never in `kFsckHardFindings`, never fails the + /// report. Persistent non-convergence across authorized janitor cycles is an operational leak + /// question (`CASGCNamespaceCleanupLeaks`, the `namespace_cleanup` GC-log phase), not an integrity + /// finding this counter can answer on its own -- one snapshot cannot prove an unbounded leak. + uint64_t namespace_janitor_pending = 0; + uint64_t namespace_janitor_pending_bytes = 0; + uint64_t namespace_janitor_pending_lives = 0; /// distinct life ids counted above + + uint64_t physical_bytes = 0; + uint64_t referenced_logical_bytes = 0; + uint64_t total_blob_refs = 0; + uint64_t distinct_blobs = 0; + + /// Set when the scan hit its deadline in partial mode: counts cover only what was walked + /// before the deadline — a lower bound, not the pool truth. + bool partial = false; + String partial_reason; + + std::vector objects; + + /// Return logical blob references per distinct reachable blob, or zero when no distinct blob was seen. + double dedupRatio() const { return distinct_blobs ? double(total_blob_refs) / double(distinct_blobs) : 0.0; } + + /// Return whether the scan found no missing reachable object or hard integrity violation. Expected + /// GC backlog classes do not make a report unclean, and `meta_without_body` is advisory (see its + /// field: GC's body-then-meta delete ordering makes a body-less `.meta` a legitimate transient with + /// no finite hard horizon); a partial report only covers the visited subset. `stale_edge` is a hard + /// finding, but it is only ever nonzero in `detail` mode — a clean summary report says nothing about + /// stale edges, exactly as a partial report says nothing about the unvisited part of the pool. + /// `chain_broken` is a hard finding in every mode. `unchecked` deliberately is NOT one: it says the + /// walk proved nothing about those namespaces, which is a statement about COVERAGE, and folding it + /// in here would make "cannot prove" indistinguishable from "found broken". + /// Defined out-of-line below, over `kFsckHardFindings`, so that "a term of `clean`" and "a row of + /// that list" are the same thing rather than two lists that can drift. + bool clean() const; +}; + +/// ONE hard finding: the name every surface renders it under, and the counter it reads. +struct FsckHardFinding +{ + std::string_view name; + uint64_t FsckReport::* value; +}; + +/// THE HARD FINDINGS, and the single authority on what they are. `FsckReport::clean` is computed from +/// this list, so adding a term means adding a row here. +/// +/// The name is the one the text summary line and the SQL result column both use, which is what lets a +/// test check a rendering surface by iterating this list instead of restating its contents. +/// The SIZE IS DEDUCED, deliberately. A fixed `std::array` rejects an added row with +/// an "excess elements in ..." diagnostic -- which stops the build, but its text carries none of the +/// guidance the assert below does, so the author learns only that they miscounted. (Which noun that +/// diagnostic uses depends on the brace form, so it is not quoted here.) Deduced, an added row compiles +/// and the assert is what speaks. +inline constexpr std::array kFsckHardFindings{ + FsckHardFinding{"dangling", &FsckReport::dangling}, + FsckHardFinding{"corrupted_runs", &FsckReport::corrupted_runs}, + FsckHardFinding{"stale_edge", &FsckReport::stale_edge}, + FsckHardFinding{"chain_broken", &FsckReport::chain_broken}, + FsckHardFinding{"lifeless_keys", &FsckReport::lifeless_keys}, +}; + +/// TRIPWIRE. A hard finding has to reach three CODE surfaces, and each has been forgotten at least once: +/// the text summary line (`formatFsckSummary`), `CommandFsck::executeImpl`'s nonzero-exit set, and the +/// SQL result row (`contentAddressedFsckColumns` + `appendContentAddressedFsckRow`). It has happened +/// repeatedly, on more than one occasion and to more than one term, each time with the rule written down +/// in prose and each time the prose not holding. (No count is given: the records that document those +/// episodes do not support one number, and a tally nobody can reconstruct is the same defect as the rest.) +/// +/// ONE ROW OF THIS LIST IS DELIBERATELY NOT IN THE EXIT SET, so the "three surfaces" rule has a named +/// exception rather than a silent violation: `stale_edge` is nonzero only under `--detail`, and +/// `CommandFsck::executeImpl` prints it as a `note:` and never throws. What licenses that is the pair -- +/// a documented reason AND a compensating gate elsewhere (`stale_edge_verdict` in +/// `utils/ca-soak/soak/fsck.py`, asserted by the soak checkpoint in `soak/run.py`, which fails closed +/// when the key is absent). A new finding may take the same exception only WITH both halves; without +/// them it belongs in the exit set. +/// +/// WHAT THIS ASSERT CHECKS, precisely: that the number of hard findings still equals the number written +/// here. Nothing more. It does NOT check that any surface renders them -- it cannot see the renderers, +/// which is the whole reason it lives with the struct: this header is included by the summary formatter, +/// by `programs/disks/CommandFsck.cpp`, and by `src/Interpreters/InterpreterSystemQuery.cpp`, so changing +/// the list breaks the build in every TU that owes an update, including the two no unit test can reach. +/// +/// The summary line is checked for real, by a test that iterates the list +/// (`CasFsckSummary.EveryHardFindingAppearsOnTheSummaryLine`). The exit set and the SQL row are NOT -- +/// for those, this assert plus the list below it is the whole of the mechanism, so bumping the number +/// without visiting them defeats it. Bump it only after all three are done. +/// +/// AND IT REACHES NO PROSE. The rule is also restated in `docs/superpowers/cas/AGENTS.md` +/// and in the soak harness's comments and messages; those restatements have gone stale before -- more +/// than once, about the exit set -- and nothing here can break a build over them. (No count is given, +/// for the same reason the paragraph above gives none: nobody keeping a tally of restatements can +/// promise its own count will not go stale next.) They are a fourth surface, unfenced by construction. +static_assert(kFsckHardFindings.size() == 5, + "A hard finding was added to or removed from `kFsckHardFindings`, which is `FsckReport::clean`. " + "Before updating this count, render it in ALL THREE code surfaces: `formatFsckSummary`'s line, " + "`CommandFsck::executeImpl`'s nonzero-exit set, and `contentAddressedFsckColumns` + " + "`appendContentAddressedFsckRow`. Two of the three have no test that can fail for you -- the " + "comment above this assert says which. A finding may be left out of the exit set only the way " + "`stale_edge` is: with a documented reason AND a compensating soak assert."); + +inline bool FsckReport::clean() const +{ + for (const FsckHardFinding & finding : kFsckHardFindings) + if (this->*finding.value != 0) + return false; + return true; +} + +/// Independently recompute reachability from authoritative refs (never from GC state or snapshots) and +/// diff it against a raw object listing. The operation is read-only; `detail` populates per-object rows. +/// `deadline`, if set, bounds the WHOLE scan: it is checked between list pages and reachability +/// refs, throwing `TIMEOUT_EXCEEDED` if exceeded (a slow-but-progressing scan surfaces a clear +/// error instead of an opaque hang) — unless `partial_on_deadline` is set, in which case the +/// accumulated lower-bound counts are returned instead, flagged via `FsckReport::partial`. A single +/// LIST page stuck in S3-client retries is bounded separately by the disk's S3 retry/timeout +/// settings, not here. `namespace_prefix`, if non-empty, scopes the scan to namespaces with this +/// prefix and skips the pool-wide unreachable classification (dangling-only mode). +FsckReport runFsck(Pool & store, bool detail, FsckProgress on_progress = {}, + std::optional deadline = {}, + bool partial_on_deadline = false, const String & namespace_prefix = {}); + +/// Render the single machine-parseable summary line (no trailing newline). This is the ONLY view of a +/// report most consumers ever get -- the soak harness parses it, CI greps it, an operator reads it -- so +/// it lives here, next to the report and under test, rather than inline in the command where nothing +/// could reach it. Every term of `FsckReport::clean` MUST appear: a hard finding the line omits is a +/// finding no run will ever report, which is how `corrupted_runs` stayed invisible from the day it was +/// first counted. That requirement is CHECKED for this surface, not merely stated: +/// `CasFsckSummary.EveryHardFindingAppearsOnTheSummaryLine` iterates `kFsckHardFindings` and looks for +/// each name in the line, so a term added to the list and not rendered here fails that test. Zeros are +/// printed, never omitted: "absent" and "zero" are different facts, and consumers (e.g. the harness's +/// `stale_edge_verdict`) fail closed on absence by design. +String formatFsckSummary(const FsckReport & report); + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasInspect.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasInspect.cpp new file mode 100644 index 000000000000..1dbf597e8a9b --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasInspect.cpp @@ -0,0 +1,638 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; +} +} + +namespace DB::Cas +{ + +namespace +{ + +/// Escapes `s` as a JSON string LITERAL (including the surrounding quotes). Handles the standard +/// two-char escapes plus a `\uXXXX` fallback for any other control byte; everything else (including +/// raw multi-byte UTF-8) passes through unchanged. This is a debug/inspection rendering, not a wire +/// format, so it deliberately does not attempt full Unicode validation. +String jsonEscape(std::string_view s) +{ + String out; + out.reserve(s.size() + 2); + out += '"'; + for (unsigned char c : s) + { + switch (c) + { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (c < 0x20) + out += fmt::format("\\u{:04x}", c); + else + out += static_cast(c); + } + } + out += '"'; + return out; +} + +/// u128 fields (hashes, ids, tokens-as-u128) render as a lowercase-hex JSON string, matching +/// `u128ToHex` — never as a nested {high,low} object or a decimal number. +String jsonHex(const UInt128 & v) { return jsonEscape(u128ToHex(v)); } +String jsonUInt(uint64_t v) { return std::to_string(v); } +String jsonBool(bool b) { return b ? "true" : "false"; } + +/// A minimal JSON object builder: each `add` takes a key and an already-rendered JSON fragment +/// (a quoted string, a number, `true`/`false`/`null`, or a nested `{...}`/`[...]`) and joins them +/// with commas. No pretty-printing — this is a debug/inspection tool, not a wire format. +class JsonObj +{ +public: + JsonObj & add(std::string_view key, const String & raw_value) + { + if (!first) + out += ","; + first = false; + out += jsonEscape(key); + out += ":"; + out += raw_value; + return *this; + } + + String str() const { return "{" + out + "}"; } + +private: + String out; + bool first = true; +}; + +String jsonArray(const std::vector & items) +{ + String out = "["; + for (size_t i = 0; i < items.size(); ++i) + { + if (i) + out += ","; + out += items[i]; + } + out += "]"; + return out; +} + +String renderManifestRef(const ManifestRef & r) +{ + return JsonObj() + .add("writer_epoch", jsonUInt(r.writer_epoch)) + .add("build_sequence", jsonUInt(r.build_sequence)) + .add("manifest_ordinal", jsonUInt(r.manifest_ordinal)) + .str(); +} + +/// Snapshot and log ref objects use `RefTxnId` values with `writer_epoch` and `ref_sequence` fields. +/// `renderRefTxnIdObj` renders those raw numeric fields rather than the canonical hex form, which +/// rejects a zero field, so inspection can dump any object, including a malformed one, without +/// failing while rendering its identifiers. +String renderRefTxnIdObj(const RefTxnId & id) +{ + return JsonObj() + .add("writer_epoch", jsonUInt(id.writer_epoch)) + .add("ref_sequence", jsonUInt(id.ref_sequence)) + .str(); +} + +String refOwnerKindName(RefOwnerKind k) +{ + switch (k) + { + case RefOwnerKind::Committed: return "Committed"; + case RefOwnerKind::Precommit: return "Precommit"; + } + return "Unknown"; +} + +String renderRefOwnerBinding(const RefOwnerBinding & b) +{ + return JsonObj() + .add("kind", jsonEscape(refOwnerKindName(b.kind))) + .add("ref_name", jsonEscape(b.ref_name)) + .add("manifest_ref", renderManifestRef(b.manifest_ref)) + .str(); +} + +String renderRefCommittedRow(const RefCommittedRow & r) +{ + return JsonObj() + .add("ref_name", jsonEscape(r.ref_name)) + .add("manifest_ref", renderManifestRef(r.manifest_ref)) + .add("published_at_ms", jsonUInt(r.published_at_ms)) + .str(); +} + +String renderRefTableSnapshot(const RefTableSnapshot & s) +{ + std::vector committed; + committed.reserve(s.committed.size()); + for (const auto & row : s.committed) + committed.push_back(renderRefCommittedRow(row)); + + std::vector precommits; + precommits.reserve(s.precommits.size()); + for (const auto & b : s.precommits) + precommits.push_back(renderRefOwnerBinding(b)); + + return JsonObj() + .add("object", jsonEscape("ref_snapshot")) + .add("namespace", jsonEscape(s.ns)) + .add("snapshot_id", renderRefTxnIdObj(s.snapshot_id)) + .add("committed", jsonArray(committed)) + .add("precommits", jsonArray(precommits)) + .str(); +} + +/// The namespace's checkpoint (spec INV-4). Every field is optional and each absence means something +/// different an operator needs to see: no `life_epoch` means no writer that knew this namespace's +/// genesis epoch has written here yet, no `committed_through` means the life has no committed +/// transaction, no `checkpoint_snapshot_id` means recovery has no snapshot base, and no +/// `last_epoch_seal` means no epoch of this namespace has been closed. They are rendered as explicit +/// `null`s rather than omitted keys so all four cases are visible. +/// `ns` comes from the KEY -- unlike the log and snapshot objects, a `_ckpt` body does not name its +/// namespace, so there is no key-to-body binding to cross-check here. +String renderRefCkpt(const RootNamespace & ns, const RefCkpt & c) +{ + return JsonObj() + .add("object", jsonEscape("ref_ckpt")) + .add("namespace", jsonEscape(ns.string())) + .add("life_epoch", c.life_epoch ? jsonUInt(*c.life_epoch) : "null") + .add("committed_through", c.committed_through ? renderRefTxnIdObj(*c.committed_through) : "null") + .add("checkpoint_snapshot_id", + c.checkpoint_snapshot_id ? renderRefTxnIdObj(*c.checkpoint_snapshot_id) : "null") + .add("last_epoch_seal", c.last_epoch_seal ? renderRefTxnIdObj(*c.last_epoch_seal) : "null") + .str(); +} + +String refOpKindName(RefOpKind k) +{ + switch (k) + { + case RefOpKind::NamespaceBirth: return "NamespaceBirth"; + case RefOpKind::OwnerTransition: return "OwnerTransition"; + case RefOpKind::SetPublishedAt: return "SetPublishedAt"; + case RefOpKind::RemoveNamespace: return "RemoveNamespace"; + case RefOpKind::EpochSeal: return "EpochSeal"; + } + return "Unknown"; +} + +String renderRefOp(const RefOp & op) +{ + return JsonObj() + .add("kind", jsonEscape(refOpKindName(op.kind))) + .add("old_binding", op.old_binding ? renderRefOwnerBinding(*op.old_binding) : "null") + .add("new_binding", op.new_binding ? renderRefOwnerBinding(*op.new_binding) : "null") + .add("ref_name", jsonEscape(op.ref_name)) + .add("expected_manifest_ref", renderManifestRef(op.expected_manifest_ref)) + .add("published_at_ms", jsonUInt(op.published_at_ms)) + .str(); +} + +String renderRefLogTxn(const RefLogTxn & t) +{ + std::vector ops; + ops.reserve(t.ops.size()); + for (const auto & op : t.ops) + ops.push_back(renderRefOp(op)); + + return JsonObj() + .add("object", jsonEscape("ref_log")) + .add("namespace", jsonEscape(t.ns)) + .add("txn_id", renderRefTxnIdObj(t.txn_id)) + .add("ops", jsonArray(ops)) + .add("prev_epoch_seal", t.prev_epoch_seal ? renderRefTxnIdObj(*t.prev_epoch_seal) : "null") + .str(); +} + +String placementName(EntryPlacement p) +{ + switch (p) + { + case EntryPlacement::Inline: return "Inline"; + case EntryPlacement::Blob: return "Blob"; + } + return "Unknown"; +} + +/// `inline_bytes` renders as its LENGTH only, not its content — an inline file's bytes are payload +/// data, not part-manifest identity, and may be arbitrarily large / non-UTF8. +String renderManifestEntry(const ManifestEntry & e) +{ + /// Render `blobIdOf(e.ref)` (":"). The algorithm must remain part of the + /// rendered identity: a bare digest is ambiguous in a pool containing algorithms with different + /// digest widths, and each entry's own `ref.algo` determines its width. + return JsonObj() + .add("path", jsonEscape(e.path)) + .add("placement", jsonEscape(placementName(e.placement))) + .add("blob", jsonEscape(blobIdOf(e.ref))) + .add("blob_size", jsonUInt(e.blob_size)) + .add("inline_bytes_size", jsonUInt(e.inline_bytes.size())) + .str(); +} + +String renderPartManifest(const PartManifest & m) +{ + std::vector entries; + entries.reserve(m.entries.size()); + for (const auto & e : m.entries) + entries.push_back(renderManifestEntry(e)); + + return JsonObj() + .add("ref", renderManifestRef(m.ref)) + .add("root_namespace_id", jsonEscape(m.root_namespace_id.string())) + .add("payload_digest", jsonHex(m.payload_digest)) + .add("entries", jsonArray(entries)) + .str(); +} + +String renderMountLease(const MountLease & m) +{ + return JsonObj() + .add("server_uuid", jsonHex(m.server_uuid)) + .add("writer_epoch", jsonUInt(m.writer_epoch)) + .add("hostname", jsonEscape(m.hostname)) + .add("pid", jsonUInt(m.pid)) + .add("started_at_ms", jsonUInt(m.started_at_ms)) + .add("seq", jsonUInt(m.seq)) + .add("expires_at_ms", jsonUInt(m.expires_at_ms)) + .add("min_active", jsonUInt(m.min_active)) + .add("gc_fenced", jsonBool(m.gc_fenced)) + .str(); +} + +String renderGcLease(const GcLease & l) +{ + return JsonObj() + .add("owner", jsonHex(l.owner)) + .add("seq", jsonUInt(l.seq)) + .str(); +} + +String renderGcState(const GcState & s) +{ + return JsonObj() + .add("round", jsonUInt(s.round)) + .add("gc_shards", jsonUInt(s.gc_shards)) + .add("snap_generation", jsonUInt(s.snap_generation)) + .add("snap_pruned_through", jsonUInt(s.snap_pruned_through)) + .add("snap_attempt", jsonUInt(s.snap_attempt)) + .add("manifest_sweep_cursor", jsonEscape(s.manifest_sweep_cursor)) + .add("lease", renderGcLease(s.lease)) + .str(); +} + +String tokenTypeName(TokenType t) +{ + switch (t) + { + case TokenType::ETag: return "ETag"; + case TokenType::Generation: return "Generation"; + case TokenType::Emulated: return "Emulated"; + } + return "Unknown"; +} + +/// `Token::value` is an opaque backend-native string (e.g. an S3 ETag) — NOT a 128-bit hash — so it +/// renders verbatim (escaped), not hex-converted; `type` names which backend family minted it. +String renderToken(const Token & t) +{ + return JsonObj() + .add("value", jsonEscape(t.value)) + .add("type", jsonEscape(tokenTypeName(t.type))) + .str(); +} + +String objectKindName(ObjectKind k) +{ + switch (k) + { + case ObjectKind::Blob: return "Blob"; + } + return "Unknown"; +} + +String renderRunRef(const RunRef & r) +{ + return JsonObj() + .add("key", jsonEscape(r.key)) + .add("checksum", jsonHex(r.checksum)) + .add("shard", jsonUInt(r.shard)) + .add("generation", jsonUInt(r.generation)) + .str(); +} + +String renderRefCoverage(const RefCoverage & c) +{ + return JsonObj() + .add("classification", jsonUInt(c.classification)) + .add("last_folded_ref_id", renderRefTxnIdObj(c.last_folded_ref_id)) + .str(); +} + +String renderFoldSeal(const CasFoldSeal & seal) +{ + JsonObj ref_lives; + for (const auto & [life_id, state] : seal.ref_lives) + ref_lives.add(renderIncarnation(life_id), JsonObj() + .add("coverage", renderRefCoverage(state.coverage)) + .add("cleanup_evidence", state.cleanup_evidence + ? JsonObj().add("remove_txn_id", renderRefTxnIdObj(state.cleanup_evidence->remove_txn_id)).str() + : "null") + .str()); + + std::vector blob_target_runs; + blob_target_runs.reserve(seal.blob_target_runs.size()); + for (const auto & r : seal.blob_target_runs) + blob_target_runs.push_back(renderRunRef(r)); + + /// A fold seal carries per-GC-shard totals for `kCondemned` rows in its source runs. Render the + /// summary from the seal itself; the older separate retired-reference object is no longer part + /// of the current layout. + JsonObj condemned_summary; + for (const auto & [shard, cs] : seal.condemned_summary) + condemned_summary.add(std::to_string(shard), JsonObj() + .add("condemned_total", jsonUInt(cs.condemned_total)) + .add("pending_total", jsonUInt(cs.pending_total)) + .add("oldest_nonpending_condemn_round", jsonUInt(cs.oldest_nonpending_condemn_round)) + .str()); + + return JsonObj() + .add("generation", jsonUInt(seal.generation)) + .add("parent_generation", jsonUInt(seal.parent_generation)) + .add("ref_lives", ref_lives.str()) + .add("blob_target_runs", jsonArray(blob_target_runs)) + .add("condemned_summary", condemned_summary.str()) + .str(); +} + +String provenanceOpName(ProvenanceOp op) +{ + switch (op) + { + case ProvenanceOp::Other: return "Other"; + case ProvenanceOp::Insert: return "Insert"; + case ProvenanceOp::Merge: return "Merge"; + case ProvenanceOp::Mutation: return "Mutation"; + case ProvenanceOp::Attach: return "Attach"; + case ProvenanceOp::Repack: return "Repack"; + } + return "Unknown"; +} + +String renderProvenance(const Provenance & p) +{ + return JsonObj() + .add("created_at_ms", jsonUInt(p.created_at_ms)) + .add("creator_server_id", jsonHex(p.creator_server_id)) + .add("ch_version", jsonUInt(p.ch_version)) + .add("op", jsonEscape(provenanceOpName(p.op))) + .str(); +} + +String metaStateName(MetaState s) +{ + switch (s) + { + case MetaState::Clean: return "clean"; + case MetaState::Condemned: return "condemned"; + } + return "unknown"; +} + +/// The per-hash `.meta` descriptor is the blob body's sibling and records its freshness state +/// (`Clean` or `Condemned`), not its payload. It is rendered separately from `renderEnvelopeHeader`: +/// the body remains an enveloped object, while the descriptor has its own format. +String renderBlobMeta(const BlobMeta & m) +{ + return JsonObj() + .add("object", jsonEscape("blob_meta")) + .add("version", jsonUInt(m.version)) + .add("state", jsonEscape(metaStateName(m.state))) + .add("condemn_round", jsonUInt(m.condemn_round)) + .add("size", jsonUInt(m.size)) + .str(); +} + +String renderEnvelopeHeader(const EnvelopeHeader & h) +{ + return JsonObj() + .add("kind", jsonEscape(objectKindName(h.kind))) + /// The blob identity is carried by the object key, so the envelope keeps only the provenance + /// fields needed for forensics (`ch` and `bld`) together with its compatibility version. + .add("compatibility_version", jsonUInt(h.compatibility_version)) + .add("incarnation_tag", jsonHex(h.incarnation_tag)) + .add("build_id", jsonHex(h.build_id)) + .add("header_len", jsonUInt(h.header_len)) + .add("provenance", h.provenance ? renderProvenance(*h.provenance) : "null") + .add("intended_ref", h.intended_ref ? jsonEscape(*h.intended_ref) : "null") + .str(); +} + +/// The word vocabulary a row's marker byte renders as, matching the `cas_run` NDJSON's own `"m"` field +/// words (`CasRecordStreamFormat.cpp`'s private `markerToWord`) so cas-inspect speaks the same vocabulary +/// as the on-disk format rather than inventing a second one. +String sourceEdgeRowKindName(char marker) +{ + switch (marker) + { + case kEdgeActive: return "edge"; + case kZeroMarker: return "zero"; + case kCondemned: return "condemned"; + default: return "unknown"; + } +} + +String renderCondemnedRow(const CondemnedRow & r) +{ + return JsonObj() + .add("delete_pending", jsonBool(r.delete_pending)) + .add("token", renderToken(r.token)) + .add("size", jsonUInt(r.size)) + .add("condemn_round", jsonUInt(r.condemn_round)) + .add("marker_confirmed", jsonBool(r.marker_confirmed)) + .str(); +} + +/// Renders one blob-target source-edge run segment (`Layout::blobTargetRunKey`): every row (edge, +/// zero-marker, or condemned sentinel), plus a summary. `parsed` carries the run's own coordinates +/// recovered from the key; `bytes` is decoded with the same typed `SourceEdgeRunView` reader the fold / +/// `zeroInDegree` / `fsck` consumers use (the memory overload, since `caInspectToJson` is a pure +/// function of (key, bytes) with no backend access here). A malformed key or payload propagates the +/// codec's own `CORRUPTED_DATA` (`SourceEdgeKeyCodec::parse`, `decodeCondemnedRow`) -- rows are never +/// silently skipped. +String renderBlobTargetRun(const ParsedBlobTargetRunKey & parsed, std::string_view bytes) +{ + SourceEdgeRunView reader = openSourceEdgeRun(bytes); + + std::vector rows; + std::set distinct_blobs; + uint64_t edge_count = 0; + uint64_t condemned_count = 0; + uint64_t zero_marker_count = 0; + + String key; + String payload; + while (reader.next(key, payload)) + { + BlobRef ref; + UInt128 source_id; + SourceEdgeKeyCodec::parse(key, ref, source_id); // throws CORRUPTED_DATA on a malformed key (fail-closed) + if (payload.empty()) + throw DB::Exception(DB::ErrorCodes::CORRUPTED_DATA, + "cas-inspect: source-edge run row for blob {} has an empty payload", blobIdOf(ref)); + const char marker = payload[0]; + + distinct_blobs.insert(ref); + JsonObj row; + row.add("blob", jsonEscape(blobIdOf(ref))) + /// `source_id` is a `CityHash128` of (namespace, writer_epoch, build_sequence, + /// manifest_ordinal, path) -- not invertible here, so it renders as plain hex, exactly like + /// every other opaque u128 identifier in this file. + .add("source_id", jsonHex(source_id)) + .add("kind", jsonEscape(sourceEdgeRowKindName(marker))); + + switch (marker) + { + case kEdgeActive: + ++edge_count; + break; + case kZeroMarker: + ++zero_marker_count; + break; + case kCondemned: + ++condemned_count; + row.add("condemned", renderCondemnedRow(decodeCondemnedRow(payload))); // CORRUPTED_DATA on malformed (fail-closed) + break; + default: + throw DB::Exception(DB::ErrorCodes::CORRUPTED_DATA, + "cas-inspect: source-edge run row for blob {} has an unknown marker 0x{:02x}", + blobIdOf(ref), static_cast(marker)); + } + rows.push_back(row.str()); + } + + return JsonObj() + .add("object", jsonEscape("blob_target_run")) + .add("generation", jsonUInt(parsed.generation)) + .add("attempt", jsonUInt(parsed.attempt)) + .add("shard", jsonUInt(parsed.shard)) + .add("seq", jsonUInt(parsed.seq)) + .add("rows", jsonArray(rows)) + .add("summary", JsonObj() + .add("rows", jsonUInt(rows.size())) + .add("distinct_blobs", jsonUInt(distinct_blobs.size())) + .add("edges", jsonUInt(edge_count)) + .add("condemned", jsonUInt(condemned_count)) + .add("zero_markers", jsonUInt(zero_marker_count)) + .str()) + .str(); +} + +} + +String caInspectToJson(const Layout & layout, const String & key, std::string_view bytes, + const std::optional & resolved_life) +{ + /// Most-specific first: `cas/manifests/.../NNNNNN.zst` before the pool-wide `cas/ns/stream/` + /// prefix, the `/mount` and `/fold_seal` suffixes before the pool-wide `gc/state` exact match, + /// and the `.meta` sibling suffix before the bare `blobs/` prefix it also matches. + if (key.starts_with(layout.casManifestsPrefix()) && key.ends_with(storedSuffix(FormatId::PartManifest))) + return renderPartManifest(decodePartManifest(openObject(FormatId::PartManifest, bytes))); + + const auto requireResolvedLife = [&](NamespaceLifePhysicalId life_id) -> const NamespaceLifeId & + { + if (!resolved_life || resolved_life->incarnation != life_id) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "cas-inspect: life_id {} has no unique resolution in the supplied catalog cut", + renderIncarnation(life_id)); + return *resolved_life; + }; + + if (key.starts_with(layout.namespaceStateRootPrefix())) + { + if (const auto life_id = layout.parseRefCkptKey(key)) + return renderRefCkpt(requireResolvedLife(*life_id).ns, decodeRefCkpt(bytes)); + } + + if (key.starts_with(layout.casRefsPrefix())) + { + + const auto parsed = layout.parseRefObjectKey(key); + if (!parsed) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "cas-inspect: key under cas/ns/stream is not a recognized ref-object key '{}'", key); + const NamespaceLifeId & life = requireResolvedLife(parsed->life_id); + if (parsed->kind == RefObjectKind::Snap) + return renderRefTableSnapshot(decodeRefTableSnapshot( + openObject(FormatId::RefSnapshot, bytes), life.ns.string(), parsed->txn_id)); + if (parsed->kind == RefObjectKind::Log) + return renderRefLogTxn(decodeRefLogTxn( + openObject(FormatId::RefLog, bytes), life.ns.string(), parsed->txn_id)); + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "cas-inspect: unhandled ref-object kind for key '{}'", key); + } + + if (key == layout.gcStateKey()) + return renderGcState(decodeGcState(bytes)); + + if (key.ends_with("/mount")) + return renderMountLease(decodeMountLease(bytes)); + + if (key.ends_with("/fold_seal")) + return renderFoldSeal(decodeFoldSeal(bytes)); + + /// Blob-target source-edge run segments (`Layout::blobTargetRunKey`) are the ground truth for + /// every in-degree question, so they get a typed decode too, not just the fold seal that names + /// them. Checked before the pool-wide `blobs/` prefix below (disjoint anyway -- these keys live + /// under `gc/gen/`, never `blobs/` -- but most-specific-first stays the dispatch's rule). + if (const auto parsed = layout.parseBlobTargetRunKey(key)) + return renderBlobTargetRun(*parsed, bytes); + + /// `blobMetaKey(id) == blobKey(id) + ".meta"`, so a meta descriptor also matches + /// `blobsPrefix()` below. Check it first or it would be decoded incorrectly as an envelope. A + /// non-`.meta` blob body still carries its envelope. + if (key.starts_with(layout.blobsPrefix()) && key.ends_with(".meta")) + return renderBlobMeta(decodeBlobMeta(bytes)); + + if (key.starts_with(layout.blobsPrefix())) + return renderEnvelopeHeader(decodeEnvelopeHeader(bytes, bytes.size(), ObjectKind::Blob)); + + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "cas-inspect: unrecognized key layout '{}' (recognized: cas/ns/stream, cas/ns/state, cas/manifests, " + "gc/server-roots/*/mount, gc/state, gc/gen/*/fold_seal, gc/gen/*/attempt/*/blob_target/*/*, " + "retired, blobs, blobs/*.meta)", key); +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasInspect.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasInspect.h new file mode 100644 index 000000000000..0c6bfa3e0cee --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasInspect.h @@ -0,0 +1,30 @@ +#pragma once +#include +#include +#include + +namespace DB::Cas +{ + +/// Read-only decode-to-JSON dispatch for `clickhouse-disks cas-inspect` (and its unit tests): given +/// any key that could live in a content-addressed pool plus the raw bytes stored at it, decode with +/// the matching codec and render the struct's fields as human-readable JSON. `layout` supplies the +/// pool's key shapes (there is no live pool/backend access here — pure function of (key, bytes)), so +/// it can be exercised directly against encoder output in unit tests, with no disk / object storage +/// involved. +/// +/// Dispatch is by KEY SHAPE, most-specific first (`cas/manifests/.../NNNNNN.zst` before the +/// `cas/ns/stream/` and `cas/ns/state/` roots, `/mount` and `/fold_seal` suffixes, the +/// `gc/gen/*/attempt/*/blob_target/*/*` source-edge run segments, then the pool-wide `gc/state` +/// and `blobs/` prefix). u128 and hash fields render as lowercase hex strings (matching +/// `u128ToHex`), while backend-native `Token` values render as escaped strings. Neither is exposed +/// as an array of bytes or a raw struct dump. +/// +/// Throws `ErrorCodes::BAD_ARGUMENTS` when `key` matches none of the recognized CA layouts. Any +/// decode failure of a matched key (invalid header, corrupted bytes, future format version, ...) +/// propagates as-is from the underlying `decode*` function (typically `CORRUPTED_DATA` or +/// `UNKNOWN_FORMAT_VERSION`) — this function performs no fallback decode and swallows nothing. +String caInspectToJson(const Layout & layout, const String & key, std::string_view bytes, + const std::optional & resolved_life = std::nullopt); + +} From 1c5ecf225111f387d94a9b37f44215173609239a Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:35 +0200 Subject: [PATCH 20/30] CAS subsystem: microbenchmarks Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../benchmarks/CMakeLists.txt | 4 + .../benchmarks/benchmark_cas_ref_protocol.cpp | 553 ++++++++++++++++++ 2 files changed, 557 insertions(+) create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/benchmarks/CMakeLists.txt create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/benchmarks/benchmark_cas_ref_protocol.cpp diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/benchmarks/CMakeLists.txt b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/benchmarks/CMakeLists.txt new file mode 100644 index 000000000000..0f792624cea1 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/benchmarks/CMakeLists.txt @@ -0,0 +1,4 @@ +clickhouse_add_executable(benchmark_cas_ref_protocol benchmark_cas_ref_protocol.cpp) +target_link_libraries (benchmark_cas_ref_protocol PRIVATE + ch_contrib::gbenchmark_all + dbms) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/benchmarks/benchmark_cas_ref_protocol.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/benchmarks/benchmark_cas_ref_protocol.cpp new file mode 100644 index 000000000000..f23f4dc06bae --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/benchmarks/benchmark_cas_ref_protocol.cpp @@ -0,0 +1,553 @@ +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +/// Pure measurement, no pass/fail assertions -- see the cas-gc-rebuild BACKLOG.md entries +/// "OPTIMIZATION OPPORTUNITY -- ref-ledger JSON encoding writes byte-by-byte" and the (now +/// RESOLVED) "admits() re-encodes the WHOLE ref table once per state-growing op" entry for the +/// investigation these benchmarks measure. Build with `-DENABLE_BENCHMARKS=ON` and run the +/// resulting `benchmark_cas_ref_protocol` binary directly; never wired into `ninja test` +/// or CI. +/// +/// BM_Admits history (synthetic RefTableState, time/call, this binary): +/// Before incremental admits() (2026-07-19) -- full O(N) rebuild+encode per call: +/// N=100: 48.8 us N=1,000: 476 us N=10,000: 5,018 us N=100,000: 55,976 us +/// Google Benchmark complexity fit: O(N log N), RMS 2%. +/// After incremental admits() (2026-07-20) -- O(1) via incremental body-byte counters on +/// RefTableState: +/// N=100: 1842 ns N=1,000: 1875 ns N=10,000: 1864 ns N=100,000: 1919 ns +/// Google Benchmark complexity fit: O(1), RMS 1-2%. +/// +/// BM_EncodeRefLogTxn history (this binary; acceptance gate for the CasJsonWriter migration): +/// Before CasJsonWriter, field-by-field WriteBuffer calls (baseline): 753 ns. +/// After CasJsonWriter bulk-append migration (2026-07-20): 333 ns -- this is the shipped code. +/// BM_MemcpyTxnBytes floor (same bytes, plain String appends of 16-byte fragments): 30.7 ns. +/// Ratio EncodeRefLogTxn / MemcpyTxnBytes = 333 / 30.7 ~= 10.8x -- above the 3x acceptance gate. +/// A `keyLiteral` "rung-1" contingency variant (merging separator+key text into one literal +/// append for the fixed unprefixed keys in writeOp/writeCommittedRow) was also measured: 325 ns +/// ~= 10.8x -- a negligible ~2.5% move, not worth a third key-rendering path. It was NOT shipped; +/// writeOp/writeCommittedRow keep the single `writeKey` path for clarity. Per the contingency +/// ladder, rung 2 was NOT attempted either (it trades readability and needs a human decision); +/// reported as DONE_WITH_CONCERNS. CasEncodingPins.* stayed byte-identical (green) throughout. +/// +/// Phase B baselines, 2026-07-21, pre-encapsulation (this binary; `--benchmark_repetitions=3 +/// --benchmark_report_aggregates_only=true`; medians reported). Recorded ahead of the +/// `RefTableState` encapsulation refactor so later phases can re-run this exact suite unchanged and +/// diff against these numbers. +/// BM_Admits (promote op; stays O(1) via the incremental budget counters, untouched by this round): +/// N=100: 963 ns N=1,000: 979 ns N=10,000: 988 ns N=100,000: 1,029 ns +/// Complexity fit: O(1), RMS 2%. +/// BM_AdmitsAddPrecommit (add op -- THE production hotspot shape: `manifestAlreadyOwned`'s linear +/// value scan AT THIS BASELINE; O(1) via the owned-manifest index since E2 -- see the Final block +/// below): +/// N=100: 995 ns N=1,000: 4,266 ns N=10,000: 38,771 ns N=100,000: 400,222 ns +/// Complexity fit: O(N), ~4.0 ns/row, RMS 2%. +/// BM_ApplyRefLogTxn (scratch copy + validate + apply + install of one promote): +/// N=100: 724 ns N=1,000: 738 ns N=10,000: 784 ns N=100,000: 788 ns +/// Complexity fit: O(1), RMS 4%. +/// BM_ReplayHistory (fold/recovery profile: snapshot of size N, 256 tail txns, 2 ops each): +/// N=100: 6.15 ms N=1,000: 46.1 ms N=10,000: 454.0 ms N=100,000: 4.93 s +/// Complexity fit: O(N), ~48,859 ns/row, RMS 3%. +/// BM_ScratchCopy (one full RefTableState copy off a materialized state -- the isolation floor): +/// N=100: 45.7 ns N=1,000: 46.0 ns N=10,000: 46.7 ns N=100,000: 46.8 ns +/// Complexity fit: O(1), RMS 1%. +/// BM_SnapshotEncode (encodeRefTableSnapshot(snapshotOf(state))): +/// N=100: 14,955 ns N=1,000: 150,061 ns N=10,000: 1,508,586 ns N=100,000: 15,885,841 ns +/// Complexity fit: O(N), ~159 ns/row, RMS 1%. +/// BM_MergedIteration (full base + 10%-overlay merged iteration, post-copy pre-materialize shape): +/// N=100: 759 ns N=1,000: 7,719 ns N=10,000: 81,073 ns N=100,000: 864,552 ns +/// Complexity fit: O(N), ~8.6 ns/row, RMS 4%. +/// BM_Materialize (RefCowMap::materialize after one overlay insert on an N-row base): +/// N=100: 12,069 ns N=1,000: 126,687 ns N=10,000: 1,296,326 ns N=100,000: 18,145,559 ns +/// Complexity fit: O(N log N), RMS 2%. +/// +/// Final, 2026-07-21, shipped tree (post E1+E2+E3; E4 tried and REVERTED -- full per-phase tables in +/// `bench_t5_e3.log`): +/// BM_AdmitsAddPrecommit: ~692-714 ns FLAT across N=100..100,000 -- O(1), RMS 1% +/// (the owned-manifest index replaced the linear scan; ~571x at N=100k). +/// BM_ReplayHistory: 1,725.58 ns/row (was 48,859) -- in-place `TrustedReplay` apply, -96.5%. +/// BM_ApplyRefLogTxn: ~778-822 ns O(1). BM_Admits (promote): ~996-1,056 ns O(1). +/// BM_ScratchCopy: ~58 ns O(1) (+~11 ns vs baseline: one more shared_ptr copy for the index). +/// BM_SnapshotEncode / BM_MergedIteration / BM_Materialize: unchanged from baseline (E4 reverted). +/// +/// Implementation note for later phases: `makeSyntheticState` calls `RefCowMap::materialize()` +/// after `replay` (which never does -- it is the pure state-machine equation, and +/// `stateFromSnapshot` loads every row through `emplace`, which only ever touches the overlay). +/// Skipping that call makes every `RefTableState` copy in this suite (including `admits`'s and +/// `applyRefLogTxn`'s own internal scratch copies) an O(N) deep-copy of an un-materialized overlay +/// map instead of an O(1) shared-base copy -- this was caught during this round because it made +/// BM_Admits regress from the documented O(1) to visibly O(N log N), contradicting its own history +/// above. Production's RETAINED states are all materialized before reuse (the live table materializes +/// once per flush; post-consult the recovery-install site in CasRefLedger.cpp materializes the +/// replayed state before retaining it -- it previously did not, which is the recovery-latency cliff +/// BM_FlushInstall now measures against), so the fix was to materialize in the helper, not to accept +/// the contaminated numbers. (replay's own internal per-txn states are never materialized mid-fold; +/// BM_ReplayHistory models that path on purpose.) + +using namespace DB::Cas; + +namespace +{ + +/// A ref-ledger key shape as actually written on the wire: table_uuid + database + table + part_name. +constexpr std::string_view kSafeKeyLikeString + = "eeeb74a2-606a-4ee9-840a-1aac7b5ac25b_ca_stress_default_part_20260719_0_89811_538"; + +RefLogTxn makeSamplePromoteTxn() +{ + RefLogTxn txn; + txn.ns = "roots/ca_soak_ch1"; + txn.txn_id = RefTxnId{1, 12345}; + + RefOp op; + op.kind = RefOpKind::OwnerTransition; + op.old_binding = RefOwnerBinding{RefOwnerKind::Precommit, "20260719_0_89811_538_89818", ManifestRef{1, 1, 999999}}; + op.new_binding = RefOwnerBinding{RefOwnerKind::Committed, "20260719_0_89811_538_89818", ManifestRef{1, 1, 999999}}; + txn.ops.push_back(op); + return txn; +} + +/// A synthetic snapshot of `n` committed rows plus one pending precommit ready to promote. +/// Built as a RefTableSnapshot and materialized via the public `replay` entry point, so this +/// helper keeps compiling unchanged when RefTableState's fields become private (Phase A). +RefTableSnapshot makeSyntheticSnapshot(size_t n) +{ + RefTableSnapshot snapshot; + snapshot.ns = "roots/bench"; + snapshot.snapshot_id = RefTxnId{1, 1}; + for (size_t i = 0; i < n; ++i) + { + RefCommittedRow row; + row.ref_name = "part_" + std::to_string(i) + "_20260719_0_1000_1"; + row.manifest_ref = ManifestRef{1, 1, static_cast(i + 1)}; + snapshot.committed.push_back(row); + } + std::sort(snapshot.committed.begin(), snapshot.committed.end(), + [](const auto & a, const auto & b) { return a.ref_name < b.ref_name; }); + snapshot.precommits.push_back(RefOwnerBinding{RefOwnerKind::Precommit, "new_part_x", ManifestRef{1, 1, 999999}}); + return snapshot; +} + +/// A synthetic committed-ref table of `n` rows, plus one pending precommit ready to promote -- +/// exactly the shape `admits()` previews on every state-growing ref op. Rebuilt through `replay` +/// (the public state-machine entry point) rather than by poking `RefTableState` fields directly, +/// so this helper survives Phase A's encapsulation of `RefTableState`. +/// +/// `replay` (the pure state-machine equation) never materializes: `stateFromSnapshot` loads every +/// committed row through `RefCowMap::emplace`, which only ever touches the overlay. Left alone, +/// every subsequent `RefTableState` copy here (`admits`'s and `applyRefLogTxn`'s own internal +/// scratch copies, and every benchmark's own scratch copy below) would deep-copy an N-row overlay +/// map instead of sharing an immutable base pointer -- silently turning "the cost of the operation +/// under test" into "the cost of copying an un-materialized map" and swamping the O(1) `admits` +/// result the header history documents. The RETAINED long-lived states production keeps are all +/// materialized: the writer's live table materializes once per flush, and -- post-consult -- the +/// recovery-install site in `CasRefLedger.cpp` now calls `materializeCommitted()` on the replayed +/// state before retaining it (it previously did NOT, so the first flush copied an N-row overlay -- +/// exactly the cliff this fix removed and the reason `BM_FlushInstall` below measures the fully +/// materialized flush cost). So this helper materializes too, matching what every real caller does +/// immediately after building or replaying a state it will keep. (Note that `replay`'s own INTERNAL +/// per-transaction states are never materialized mid-fold -- `BM_ReplayHistory` deliberately models +/// that, feeding `replay(snapshot, tail)` an un-materialized base on purpose.) +RefTableState makeSyntheticState(size_t n) +{ + RefTableState state = replay(makeSyntheticSnapshot(n), {}); + state.materializeCommitted(); + return state; +} + +} + +/// Floor comparison: writeJSONString's per-character escaping loop (WriteHelpers.h) on a string +/// that needs no escaping at all (a real ref-ledger key shape) vs a raw bulk write of the same +/// bytes. See BM_RawBulkWriteSafe below for the delta. +static void BM_WriteJSONStringSafe(benchmark::State & state) +{ + DB::FormatSettings settings; + DB::PODArray buf; + for (auto _ : state) + { + buf.clear(); + DB::WriteBufferFromVector> out(buf); + DB::writeJSONString(kSafeKeyLikeString, out, settings); + benchmark::DoNotOptimize(buf.data()); + } +} +BENCHMARK(BM_WriteJSONStringSafe); + +static void BM_RawBulkWriteSafe(benchmark::State & state) +{ + DB::PODArray buf; + for (auto _ : state) + { + buf.clear(); + DB::WriteBufferFromVector> out(buf); + DB::writeChar('"', out); + out.write(kSafeKeyLikeString.data(), kSafeKeyLikeString.size()); + DB::writeChar('"', out); + benchmark::DoNotOptimize(buf.data()); + } +} +BENCHMARK(BM_RawBulkWriteSafe); + +/// Absolute cost of encoding one ref-log transaction (a single promote op) with +/// `encodeRefLogTxn`'s migrated `CasJsonWriter` bulk-append implementation (see the history +/// comment at the top of this file and the BACKLOG resolution). `BM_MemcpyTxnBytes` right below +/// is the floor to diff this against. +static void BM_EncodeRefLogTxn(benchmark::State & state) +{ + const RefLogTxn txn = makeSamplePromoteTxn(); + for (auto _ : state) + benchmark::DoNotOptimize(encodeRefLogTxn(txn)); +} +BENCHMARK(BM_EncodeRefLogTxn); + +/// The "near-memcpy" floor for BM_EncodeRefLogTxn: the SAME encoded bytes assembled from +/// precomputed 16-byte fragments by plain String appends -- approximating the writer's append +/// granularity with zero formatting/escaping work. Originally an acceptance gate for the +/// CasJsonWriter migration; measurement showed the <=3x-of-floor target is physically unreachable for a validating, +/// JSON-escaping encoder (BM_EncodeRefLogTxn lands at ~10.8x this floor even after the 2.26x +/// CasJsonWriter speedup -- see the BACKLOG resolution for the profiled breakdown). Kept as a +/// documented reference floor, not a pass/fail gate. +static void BM_MemcpyTxnBytes(benchmark::State & state) +{ + const RefLogTxn txn = makeSamplePromoteTxn(); + const String encoded = encodeRefLogTxn(txn); + std::vector fragments; + constexpr size_t kFragment = 16; + for (size_t off = 0; off < encoded.size(); off += kFragment) + fragments.push_back(std::string_view(encoded).substr(off, kFragment)); + + String buf; + buf.reserve(encoded.size()); + for (auto _ : state) + { + buf.clear(); + for (const auto f : fragments) + buf.append(f.data(), f.size()); + benchmark::DoNotOptimize(buf.data()); + } +} +BENCHMARK(BM_MemcpyTxnBytes); + +/// admits() used to re-derive and re-encode the WHOLE committed-ref snapshot on every call +/// (CasRefProtocol.cpp), showing O(N log N) growth with table size; it now maintains +/// incremental body-byte counters on RefTableState instead, so this should show flat (O(1)) +/// time/call across the range. ->Complexity() has Google Benchmark fit and print the +/// empirical big-O across the range. +static void BM_Admits(benchmark::State & state) +{ + const size_t n = static_cast(state.range(0)); + const RefTableState table = makeSyntheticState(n); + RefOp op; + op.kind = RefOpKind::OwnerTransition; + op.old_binding = RefOwnerBinding{RefOwnerKind::Precommit, "new_part_x", ManifestRef{1, 1, 999999}}; + op.new_binding = RefOwnerBinding{RefOwnerKind::Committed, "new_part_x", ManifestRef{1, 1, 999999}}; + + for (auto _ : state) + benchmark::DoNotOptimize(admits(table, op, 1ull << 40, 1ull << 40)); + + state.SetComplexityN(static_cast(n)); +} +BENCHMARK(BM_Admits)->RangeMultiplier(10)->Range(100, 100000)->Complexity(); + +/// THE production hotspot shape: add-precommit runs `manifestAlreadyOwned` (a linear value scan +/// today). Expected O(N) before the experiments, O(1) after the winning combination. Unlike +/// BM_Admits (a promote, which never calls `manifestAlreadyOwned`), this previews a pure add -- +/// the op every part publication starts with -- so it is the shape production traces show as +/// linear even after the incremental-budget fix landed for BM_Admits' promote shape. +static void BM_AdmitsAddPrecommit(benchmark::State & state) +{ + const size_t n = static_cast(state.range(0)); + const RefTableState table = makeSyntheticState(n); + RefOp op; + op.kind = RefOpKind::OwnerTransition; + op.new_binding = RefOwnerBinding{RefOwnerKind::Precommit, "brand_new_part", ManifestRef{2, 1, 1}}; + + for (auto _ : state) + benchmark::DoNotOptimize(admits(table, op, 1ull << 40, 1ull << 40)); + + state.SetComplexityN(static_cast(n)); +} +BENCHMARK(BM_AdmitsAddPrecommit)->RangeMultiplier(10)->Range(100, 100000)->Complexity(); + +/// One transaction end-to-end: scratch copy + validate + apply + install (a promote of the +/// staged precommit). The copy is part of the measured cost on purpose -- it is what E3 attacks. +static void BM_ApplyRefLogTxn(benchmark::State & state) +{ + const size_t n = static_cast(state.range(0)); + const RefTableState table = makeSyntheticState(n); + + RefLogTxn txn; + txn.ns = "roots/bench"; + txn.txn_id = RefTxnId{1, 2}; + RefOp promote; + promote.kind = RefOpKind::OwnerTransition; + promote.old_binding = RefOwnerBinding{RefOwnerKind::Precommit, "new_part_x", ManifestRef{1, 1, 999999}}; + promote.new_binding = RefOwnerBinding{RefOwnerKind::Committed, "new_part_x", ManifestRef{1, 1, 999999}}; + txn.ops.push_back(promote); + + for (auto _ : state) + { + RefTableState scratch = table; + applyRefLogTxn(scratch, txn); + benchmark::DoNotOptimize(&scratch); + } + + state.SetComplexityN(static_cast(n)); +} +BENCHMARK(BM_ApplyRefLogTxn)->RangeMultiplier(10)->Range(100, 100000)->Complexity(); + +/// End-to-end FLUSH-INSTALL cost: apply one state-growing transaction (add a fresh precommit, then +/// promote it -- touching BOTH the committed map AND the owned-manifest index) and then +/// `materializeCommitted()`, which folds BOTH COW overlays into fresh shared bases. THIS is the O(N) +/// critical section production holds `state_mutex` for, once per ref-log flush -- the number the +/// "writer path is flat" claim (drawn from `BM_ApplyRefLogTxn`, which stops before materialize) must be +/// weighed against. `BM_ApplyRefLogTxn` measures apply-without-install; the shipped-report +/// `BM_Materialize` measures only `RefCowMap`'s half; this measures the whole install including the +/// second (`owned_manifests`) container the index added, over the same N range. +static void BM_FlushInstall(benchmark::State & state) +{ + const size_t n = static_cast(state.range(0)); + const RefTableState table = makeSyntheticState(n); // materialized, as a live table is at a flush boundary + + /// add + promote of a fresh ref: the add inserts into `owned_manifests`, the promote grows + /// `committed` -- so materialize below folds a nonempty overlay in BOTH containers. Manifest {4,1,1} + /// and ref name are unique against the synthetic snapshot's {1,1,*} rows and "new_part_x" precommit. + RefLogTxn txn; + txn.ns = "roots/bench"; + txn.txn_id = RefTxnId{1, 2}; + RefOp add; + add.kind = RefOpKind::OwnerTransition; + add.new_binding = RefOwnerBinding{RefOwnerKind::Precommit, "flush_install_new_part", ManifestRef{4, 1, 1}}; + txn.ops.push_back(add); + RefOp promote; + promote.kind = RefOpKind::OwnerTransition; + promote.old_binding = RefOwnerBinding{RefOwnerKind::Precommit, "flush_install_new_part", ManifestRef{4, 1, 1}}; + promote.new_binding = RefOwnerBinding{RefOwnerKind::Committed, "flush_install_new_part", ManifestRef{4, 1, 1}}; + txn.ops.push_back(promote); + + for (auto _ : state) + { + RefTableState working = table; // O(1): shared base + applyRefLogTxn(working, txn); // O(ops): bounded overlay + working.materializeCommitted(); // O(N): the critical-section fold this benchmark exists to measure + benchmark::DoNotOptimize(&working); + } + + state.SetComplexityN(static_cast(n)); +} +BENCHMARK(BM_FlushInstall)->RangeMultiplier(10)->Range(100, 100000)->Complexity(); + +/// Same flush-install as `BM_FlushInstall`, but exercising the E5 uniquely-owned-base fast path that +/// production actually hits. `BM_FlushInstall` copies a shared fixture (`working = table`), so at +/// `materializeCommitted()` the base still has `use_count() == 2` and the fold must build a fresh +/// base -- O(N). Production's live table has NO outstanding scratch copy at the install point: +/// `CasRefLedger::flushRefBatch` EXPLICITLY releases its trial-validation copy (`working = RefTableState{}`) +/// before allocating the id and doing the post-PUT install, so at `materializeCommitted()` the live +/// base is uniquely owned and the fold happens in place -- O(overlay). This variant models that by +/// rebuilding a private, +/// materialized state each iteration (its base `use_count()` is 1), timing only the apply + in-place +/// materialize. The per-iteration rebuild AND the prior iteration's O(N) teardown are excluded from +/// the measurement by hoisting `working` out of the loop and rebuilding it via move-assignment under +/// Pause/ResumeTiming (the reassignment both destroys the previous grown state and installs a fresh +/// materialized one, all untimed). The residual per-iteration Pause/Resume overhead is a constant +/// floor, so the signal to read is FLATNESS across N (O(overlay)), not the absolute small-N number. +static void BM_FlushInstallUniqueOwner(benchmark::State & state) +{ + const size_t n = static_cast(state.range(0)); + + RefLogTxn txn; + txn.ns = "roots/bench"; + txn.txn_id = RefTxnId{1, 2}; + RefOp add; + add.kind = RefOpKind::OwnerTransition; + add.new_binding = RefOwnerBinding{RefOwnerKind::Precommit, "flush_install_new_part", ManifestRef{4, 1, 1}}; + txn.ops.push_back(add); + RefOp promote; + promote.kind = RefOpKind::OwnerTransition; + promote.old_binding = RefOwnerBinding{RefOwnerKind::Precommit, "flush_install_new_part", ManifestRef{4, 1, 1}}; + promote.new_binding = RefOwnerBinding{RefOwnerKind::Committed, "flush_install_new_part", ManifestRef{4, 1, 1}}; + txn.ops.push_back(promote); + + /// Hoisted out of the loop so the O(N) teardown of the previous iteration's grown state is folded + /// into the untimed move-assignment below, not charged to the timed apply + materialize region. + RefTableState working; + for (auto _ : state) + { + state.PauseTiming(); + working = makeSyntheticState(n); // private, materialized: base use_count() == 1 + state.ResumeTiming(); + + applyRefLogTxn(working, txn); // O(ops): bounded overlay + working.materializeCommitted(); // O(overlay): uniquely-owned base folded IN PLACE (the E5 win) + benchmark::DoNotOptimize(&working); + } + + state.SetComplexityN(static_cast(n)); +} +/// Fixed iteration count: the E5 fast path makes the timed apply + in-place-materialize region tiny +/// and N-independent, so google-benchmark's default min-time targeting would demand millions of +/// iterations at every N -- each paying an untimed O(N) `makeSyntheticState` rebuild, which explodes +/// at large N. A fixed, modest count keeps every point cheap while still averaging enough samples to +/// read the flatness across N (the whole point of this variant). +BENCHMARK(BM_FlushInstallUniqueOwner)->RangeMultiplier(10)->Range(100, 100000)->Iterations(500)->Complexity(); + +/// The fold/recovery profile: K transactions replayed over a size-N snapshot. Each txn creates +/// and promotes one new ref (two ops), so each add pays today's `manifestAlreadyOwned` scan. +/// K fixed at 256; complexity fit is over N. +static void BM_ReplayHistory(benchmark::State & state) +{ + const size_t n = static_cast(state.range(0)); + const RefTableSnapshot snapshot = makeSyntheticSnapshot(n); + + constexpr size_t kTailTxns = 256; + std::vector tail; + tail.reserve(kTailTxns); + for (size_t k = 0; k < kTailTxns; ++k) + { + RefLogTxn txn; + txn.ns = "roots/bench"; + txn.txn_id = RefTxnId{1, 2 + k}; + + /// Refs unique per k, and namespaced under writer_epoch 3 so they collide with nothing in + /// the snapshot's own {1,1,i} committed series or its {1,1,999999} precommit. + const String ref_name = "replay_part_" + std::to_string(k); + const ManifestRef manifest_ref{3, 1, static_cast(k + 1)}; + + RefOp add; + add.kind = RefOpKind::OwnerTransition; + add.new_binding = RefOwnerBinding{RefOwnerKind::Precommit, ref_name, manifest_ref}; + txn.ops.push_back(add); + + RefOp promote; + promote.kind = RefOpKind::OwnerTransition; + promote.old_binding = RefOwnerBinding{RefOwnerKind::Precommit, ref_name, manifest_ref}; + promote.new_binding = RefOwnerBinding{RefOwnerKind::Committed, ref_name, manifest_ref}; + txn.ops.push_back(promote); + + tail.push_back(std::move(txn)); + } + + for (auto _ : state) + benchmark::DoNotOptimize(replay(snapshot, tail)); + + state.SetComplexityN(static_cast(n)); +} +BENCHMARK(BM_ReplayHistory)->RangeMultiplier(10)->Range(100, 100000)->Complexity(); + +/// The isolation primitive on its own: one full state copy (COW committed + std::set precommits +/// + counters). Overlay is empty (state fresh from replay+materialize), so this is the floor. +static void BM_ScratchCopy(benchmark::State & state) +{ + const size_t n = static_cast(state.range(0)); + RefTableState table = makeSyntheticState(n); + table.materializeCommitted(); /// makeSyntheticState already materializes; repeated here + /// defensively (a no-op on an empty overlay) so this benchmark's + /// floor claim does not silently depend on that helper's internals. + + for (auto _ : state) + { + RefTableState copy = table; + benchmark::DoNotOptimize(©); + } + + state.SetComplexityN(static_cast(n)); +} +BENCHMARK(BM_ScratchCopy)->RangeMultiplier(10)->Range(100, 100000)->Complexity(); + +/// Canonical snapshot encoding for size N (per-flush cost, expected O(N) -- the question is the +/// constant, which E4's contiguous scan attacks). +static void BM_SnapshotEncode(benchmark::State & state) +{ + const size_t n = static_cast(state.range(0)); + const RefTableState table = makeSyntheticState(n); + + for (auto _ : state) + benchmark::DoNotOptimize(encodeRefTableSnapshot(snapshotOf(table, "roots/bench"))); + + state.SetComplexityN(static_cast(n)); +} +BENCHMARK(BM_SnapshotEncode)->RangeMultiplier(10)->Range(100, 100000)->Complexity(); + +/// Full merged iteration with a 10% overlay (post-copy, pre-materialize shape): an N-row +/// materialized base, then a fresh overlay of N/10 rows layered on top with `materialize()` +/// deliberately not called again -- so iteration must merge base and overlay in sorted order the +/// way the cold full-scan paths (snapshotOf, listRefs, dropNamespace) do against an in-flight batch. +/// Benchmarks `RefCowMap` directly (like `BM_Materialize` below) rather than through +/// `RefTableState::getCommitted()`: this isolates the merge-iteration primitive itself, and building +/// the overlay via `RefTableState`'s promote/precommit transactions would additionally measure the +/// state machine's own per-op bookkeeping, which is not what this benchmark is about. +static void BM_MergedIteration(benchmark::State & state) +{ + const size_t n = static_cast(state.range(0)); + + RefCowMap map; + for (size_t i = 0; i < n; ++i) + { + RefCommittedRow row; + row.ref_name = "part_" + std::to_string(i) + "_20260719_0_1000_1"; + row.manifest_ref = ManifestRef{1, 1, static_cast(i + 1)}; + map.emplace(row.ref_name, row); + } + map.materialize(); + + const size_t overlay_n = std::max(1, n / 10); + for (size_t i = 0; i < overlay_n; ++i) + { + RefCommittedRow row; + row.ref_name = "overlay_part_" + std::to_string(i) + "_20260719_0_1000_1"; + row.manifest_ref = ManifestRef{2, 1, static_cast(i + 1)}; + map.insert_or_assign(row.ref_name, row); + } + + for (auto _ : state) + { + size_t total = 0; + for (const auto [ref_name, row] : map) + total += row.ref_name.size(); + benchmark::DoNotOptimize(total); + } + + state.SetComplexityN(static_cast(n)); +} +BENCHMARK(BM_MergedIteration)->RangeMultiplier(10)->Range(100, 100000)->Complexity(); + +/// RefCowMap::materialize after one overlay insert on an N-row base (per-flush install cost). +/// Benchmarks RefCowMap directly -- it is a public class. +static void BM_Materialize(benchmark::State & state) +{ + const size_t n = static_cast(state.range(0)); + RefCowMap base_map; + for (size_t i = 0; i < n; ++i) + { + RefCommittedRow row; + row.ref_name = "part_" + std::to_string(i) + "_20260719_0_1000_1"; + row.manifest_ref = ManifestRef{1, 1, static_cast(i + 1)}; + base_map.emplace(row.ref_name, row); + } + base_map.materialize(); + + for (auto _ : state) + { + RefCowMap copy = base_map; + RefCommittedRow new_row; + new_row.ref_name = "brand_new_part_20260719_0_1000_1"; + new_row.manifest_ref = ManifestRef{2, 1, 1}; + copy.insert_or_assign(new_row.ref_name, new_row); + copy.materialize(); + benchmark::DoNotOptimize(©); + } + + state.SetComplexityN(static_cast(n)); +} +BENCHMARK(BM_Materialize)->RangeMultiplier(10)->Range(100, 100000)->Complexity(); + +BENCHMARK_MAIN(); From edb79bed0488dfad301ae31a1acfdbea7a184302 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:35 +0200 Subject: [PATCH 21/30] CAS integration: disk surface and registration The content-addressed metadata storage, its disk transaction and part staging (the top-level subsystem glue), registration of the content_addressed metadata storage type, capability predicates on IDisk/DiskObjectStorage/IMetadataStorage, the conditional-object-storage API on IObjectStorage, the FileView read-pipeline stage, write-ETag surfacing, cache-over-CA, and the atomic-file-write short-circuit for txn_version.txt. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .../DiskObjectStorage/DiskObjectStorage.cpp | 52 +- .../DiskObjectStorage/DiskObjectStorage.h | 13 +- .../DiskObjectStorageCache.cpp | 14 +- .../MetadataStorageFromCacheObjectStorage.cpp | 7 + .../MetadataStorageFromCacheObjectStorage.h | 1 + .../ContentAddressedExchange.cpp | 170 ++ .../ContentAddressedExchange.h | 260 ++ .../ContentAddressedMetadataStorage.cpp | 2299 +++++++++++++++++ .../ContentAddressedMetadataStorage.h | 774 ++++++ .../ContentAddressedSettings.cpp | 214 ++ .../ContentAddressedSettings.h | 89 + .../ContentAddressedTransaction.cpp | 1966 ++++++++++++++ .../ContentAddressedTransaction.h | 420 +++ .../ContentAddressed/README.md | 198 ++ .../MetadataStorages/IMetadataStorage.h | 51 + .../MetadataStorageFactory.cpp | 39 + .../ObjectStorages/IObjectStorage.h | 72 + .../RegisterDiskObjectStorage.cpp | 23 +- src/Disks/DiskType.cpp | 2 + src/Disks/DiskType.h | 1 + src/Disks/IDisk.h | 7 + src/Disks/ReadOnlyDiskWrapper.h | 5 + src/IO/ReadPipeline.cpp | 39 +- src/IO/ReadPipeline.h | 21 +- src/IO/WriteBufferFromFileBase.h | 7 + src/IO/WriteBufferFromFileDecorator.h | 9 + src/IO/WriteSettings.h | 39 + .../VersionMetadataOnDisk.cpp | 12 + src/Storages/StorageMergeTree.cpp | 15 +- src/Storages/StorageReplicatedMergeTree.cpp | 12 + 30 files changed, 6816 insertions(+), 15 deletions(-) create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedExchange.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedExchange.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.cpp create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.h create mode 100644 src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md diff --git a/src/Disks/DiskObjectStorage/DiskObjectStorage.cpp b/src/Disks/DiskObjectStorage/DiskObjectStorage.cpp index 211f73468449..191fc83b9b65 100644 --- a/src/Disks/DiskObjectStorage/DiskObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/DiskObjectStorage.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -743,6 +744,7 @@ bool DiskObjectStorage::isSharedCompatible() const { case MetadataStorageType::Plain: case MetadataStorageType::PlainRewritable: + case MetadataStorageType::CAS: case MetadataStorageType::StaticWeb: return true; default: @@ -752,9 +754,33 @@ bool DiskObjectStorage::isSharedCompatible() const bool DiskObjectStorage::supportsHardLinks() const { + /// MergeTree consults `supportsHardLinks` at exactly two pure capability gates + /// (`MergeTreeData::checkAlterIsPossible` and `checkMutationIsPossible`) to decide whether + /// mutations / lightweight `DELETE` / data-`ALTER`s are possible. On a content-addressed pool + /// these are supported: `MutateTask` builds the new part through ONE whole-part transaction in + /// which `createHardLink` carries unchanged columns forward BY REFERENCE (the new manifest entry + /// points at the same blob hash — no re-upload) and changed columns are written as fresh blobs, + /// committed atomically. No code branches on this flag to choose a per-file-autocommit path, so + /// advertising true does NOT open the per-file clone hazard: the corrupting whole-part clone + /// paths (partition clone, BACKUP hard-link, replication) are gated by their own independent + /// checks (`checkAlterPartitionIsPossible`, the BACKUP CA rejection, the `Replicated*MergeTree` + /// CA rejection), which remain in force. + if (metadata_storage->isContentAddressed()) + return true; + return !metadata_storage->isWriteOnce() && !metadata_storage->isPlain(); } +bool DiskObjectStorage::isContentAddressed() const +{ + return metadata_storage->isContentAddressed(); +} + +bool DiskObjectStorage::supportsAtomicFileWrites() const +{ + return metadata_storage->supportsAtomicFileWrites(); +} + String DiskObjectStorage::getReadResourceName() const { @@ -790,7 +816,26 @@ void DiskObjectStorage::prepareRead( std::optional read_hint, ReadPipeline & pipeline) const { - const auto storage_objects = metadata_storage->getStorageObjects(path); + /// Content-addressed reads: in-manifest bytes come from memory (no object exists); a + /// blob-backed part file translates to its physical blob object + a payload window, which + /// rides the STANDARD pipeline below (gather/caches/async prefetch — same chain as plain + /// object-storage disks, so right-mark bounds reach the object reader and its range requests + /// stay drainable, B116) and is bounded by the FileView stage at the end. + std::optional ca_blob_view; + if (metadata_storage->isContentAddressed()) + { + const auto * ca = dynamic_cast(metadata_storage.get()); + if (ca) + { + if (ca->prepareInManifestRead(path, settings, pipeline)) + return; + ca_blob_view = ca->getBlobViewPlan(path); + } + } + + const auto storage_objects = ca_blob_view + ? StoredObjects{ca_blob_view->object} + : metadata_storage->getStorageObjects(path); auto read_settings = updateIOSchedulingSettings(settings, getReadResourceName(), getWriteResourceName()); auto global_context = Context::getGlobalContextInstance(); @@ -870,6 +915,11 @@ void DiskObjectStorage::prepareRead( global_context->getAsyncReadCounters(), global_context->getFilesystemReadPrefetchesLog()); } + + /// A content-addressed blob-backed file is a payload window inside its blob: bound the + /// chain to it (and skip the CHCA envelope header in front). + if (ca_blob_view) + pipeline.needFileView(path, ca_blob_view->payload_offset, ca_blob_view->payload_end); } std::unique_ptr DiskObjectStorage::readFileIfExists( diff --git a/src/Disks/DiskObjectStorage/DiskObjectStorage.h b/src/Disks/DiskObjectStorage/DiskObjectStorage.h index 663fb0c3f215..09ec478b02d0 100644 --- a/src/Disks/DiskObjectStorage/DiskObjectStorage.h +++ b/src/Disks/DiskObjectStorage/DiskObjectStorage.h @@ -48,7 +48,14 @@ friend class DiskObjectStorageReservation; DataSourceDescription getDataSourceDescription() const override { return data_source_description; } - bool supportZeroCopyReplication() const override { return metadata_storage->getType() != MetadataStorageType::Keeper; } + /// A content-addressed pool deduplicates blobs across parts and has no per-replica unique blob + /// ids; the zero-copy subsystem (B1) is explicitly out of scope for M1, so advertise it as + /// unsupported (honest capability — B31). Other object-storage metadata types keep the old rule. + bool supportZeroCopyReplication() const override + { + return metadata_storage->getType() != MetadataStorageType::Keeper + && metadata_storage->getType() != MetadataStorageType::CAS; + } bool supportParallelWrite() const override { return object_storages->takePointingTo(cluster->getLocalLocation())->supportParallelWrite(); } @@ -210,6 +217,10 @@ friend class DiskObjectStorageReservation; bool supportsHardLinks() const override; + bool isContentAddressed() const override; + + bool supportsAtomicFileWrites() const override; + /// Get structure of object storage this disk works with. Examples: /// DiskObjectStorage(S3ObjectStorage) /// DiskObjectStorage(CachedObjectStorage(S3ObjectStorage)) diff --git a/src/Disks/DiskObjectStorage/DiskObjectStorageCache.cpp b/src/Disks/DiskObjectStorage/DiskObjectStorageCache.cpp index 5f144c7dd993..f3588a84b6cb 100644 --- a/src/Disks/DiskObjectStorage/DiskObjectStorageCache.cpp +++ b/src/Disks/DiskObjectStorage/DiskObjectStorageCache.cpp @@ -18,10 +18,22 @@ DiskObjectStoragePtr DiskObjectStorage::wrapWithCache(FileCachePtr cache, const auto local_location = cluster->getLocalLocation(); registry[local_location] = std::make_shared(registry[local_location], cache, cache_settings, layer_name); + /// A content-addressed disk cannot be fronted by the generic MetadataStorageFromCacheObjectStorage + /// passthrough: that wrapper hides isContentAddressed and the concrete CA metadata/transaction + /// types the CA read/write paths dynamic_cast to, so a cache-wrapped CA disk would take the generic + /// write path and throw NOT_IMPLEMENTED at startup. Reuse the CA metadata storage directly; only the + /// object storage is cached. Safe because ContentAddressedMetadataStorage::startup()/shutdown() are + /// idempotent, so the base disk and this cache disk share one mount/lease with no conflict. + /// Immutable content-hash blobs then cache through the CachedObjectStorage above; the control plane + /// keeps using the CA metadata storage's own raw object-storage pointer and bypasses the cache. + MetadataStoragePtr cache_metadata_storage = metadata_storage->isContentAddressed() + ? metadata_storage + : std::make_shared(metadata_storage); + auto cache_disk = std::make_shared( layer_name, std::make_shared(layer_name, cluster->getConfiguration()), - std::make_shared(metadata_storage), + cache_metadata_storage, std::make_shared(std::move(registry)), std::dynamic_pointer_cast(shared_from_this()), Context::getGlobalContextInstance()->getConfigRef(), diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/Cache/MetadataStorageFromCacheObjectStorage.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/Cache/MetadataStorageFromCacheObjectStorage.cpp index f1d1af2ae2c2..783ebc6f174c 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/Cache/MetadataStorageFromCacheObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/Cache/MetadataStorageFromCacheObjectStorage.cpp @@ -169,6 +169,13 @@ bool MetadataStorageFromCacheObjectStorage::isReadOnly() const return underlying->isReadOnly(); } +bool MetadataStorageFromCacheObjectStorage::isContentAddressed() const +{ + /// Defense-in-depth: never lie about content-addressing if this wrapper is ever constructed over a + /// CA storage. The primary fix (DiskObjectStorage::wrapWithCache) bypasses this wrapper for CA disks. + return underlying->isContentAddressed(); +} + bool MetadataStorageFromCacheObjectStorage::isTransactional() const { return underlying->isTransactional(); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/Cache/MetadataStorageFromCacheObjectStorage.h b/src/Disks/DiskObjectStorage/MetadataStorages/Cache/MetadataStorageFromCacheObjectStorage.h index 5cc03281bcba..f5e70e1e16ff 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/Cache/MetadataStorageFromCacheObjectStorage.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/Cache/MetadataStorageFromCacheObjectStorage.h @@ -63,6 +63,7 @@ class MetadataStorageFromCacheObjectStorage : public IMetadataStorage std::optional getStorageObjectsIfExist(const std::string & path) const override; bool isReadOnly() const override; + bool isContentAddressed() const override; bool isTransactional() const override; bool isPlain() const override; bool isWriteOnce() const override; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedExchange.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedExchange.cpp new file mode 100644 index 000000000000..513b0ae0a753 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedExchange.cpp @@ -0,0 +1,170 @@ +#include + +#include + +namespace DB +{ + +namespace +{ + +/// The version tag. A peer that knows a different token shape must be refused rather than +/// misparsed, and a bare field count is not a version: two shapes can agree on it by accident. +constexpr std::string_view kTokenVersion = "car1"; +/// `|` is a legal cookie octet (RFC 6265 excludes only CTLs, whitespace, `"`, `,`, `;` and `\`) +/// and never appears in an encoded field, because encoding leaves only the unreserved set and `%`. +constexpr char kFieldSeparator = '|'; +constexpr size_t kFieldCount = 6; +/// Sized for the widest real field -- a namespace is `/store//@cas@` and a +/// ref name is a part name, possibly `detached/`-prefixed. The cap is a bound on what a peer can make +/// this server hold and log, not a schema: a field that needs more than this is refused, and a refused +/// token costs a byte fetch. +constexpr size_t kMaxFieldBytes = 256; +constexpr size_t kMaxTokenBytes = 1024; + +bool isUnreserved(unsigned char c) +{ + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') + || c == '-' || c == '.' || c == '_' || c == '~'; +} + +/// Control characters never travel: they would break the cookie header and they are the classic way to +/// smuggle a forged line into whatever log prints the token. +bool isControl(unsigned char c) +{ + return c < 0x20 || c == 0x7F; +} + +void appendPercentEncoded(std::string_view field, String & out) +{ + static constexpr std::string_view hex_digits = "0123456789ABCDEF"; + for (const char ch : field) + { + const auto c = static_cast(ch); + if (isUnreserved(c)) + { + out += ch; + } + else + { + out += '%'; + out += hex_digits[c >> 4]; + out += hex_digits[c & 0x0F]; + } + } +} + +std::optional hexNibble(char ch) +{ + const auto c = static_cast(ch); + if (c >= '0' && c <= '9') + return static_cast(c - '0'); + if (c >= 'A' && c <= 'F') + return static_cast(c - 'A' + 10); + if (c >= 'a' && c <= 'f') + return static_cast(c - 'a' + 10); + return std::nullopt; +} + +/// Strict inverse of `appendPercentEncoded`: a `%` must be followed by exactly two hex digits, and no +/// byte outside the unreserved set may appear unescaped. Lenient decoding is how a percent-encoding +/// pair stops being a bijection, and this one has to be a bijection -- the encoded form is what routes +/// the confirm, the decoded form is what it compares. +std::optional percentDecodeStrict(std::string_view field) +{ + String out; + out.reserve(field.size()); + for (size_t i = 0; i < field.size(); ++i) + { + if (field[i] != '%') + { + if (!isUnreserved(static_cast(field[i]))) + return std::nullopt; + out += field[i]; + continue; + } + + if (i + 2 >= field.size()) + return std::nullopt; + const auto hi = hexNibble(field[i + 1]); + const auto lo = hexNibble(field[i + 2]); + if (!hi || !lo) + return std::nullopt; + const auto decoded = static_cast((*hi << 4) | *lo); + if (isControl(decoded)) + return std::nullopt; + out += static_cast(decoded); + i += 2; + } + return out; +} + +} + +std::optional encodeCasRelinkSourceToken(const CasRelinkSourceToken & token) +{ + const std::array fields{ + &token.pool_uuid, &token.server_root_id, &token.root_namespace, + &token.ref_name, &token.part_name, &token.manifest_ref_text}; + + String out{kTokenVersion}; + for (const String * field : fields) + { + /// Every field is required: a token with a hole in it can only route somewhere it was not meant + /// to, and `resolveContentAddressedConfirm` would have to re-discover that for itself. + if (field->empty() || field->size() > kMaxFieldBytes) + return std::nullopt; + for (const char ch : *field) + if (isControl(static_cast(ch))) + return std::nullopt; + + out += kFieldSeparator; + appendPercentEncoded(*field, out); + } + + if (out.size() > kMaxTokenBytes) + return std::nullopt; + return out; +} + +std::optional decodeCasRelinkSourceToken(std::string_view text) +{ + if (text.size() > kMaxTokenBytes) + return std::nullopt; + + std::array segments; + size_t count = 0; + size_t pos = 0; + while (true) + { + if (count == segments.size()) + return std::nullopt; /// more separators than the shape has fields + const size_t sep = text.find(kFieldSeparator, pos); + if (sep == std::string_view::npos) + { + segments[count++] = text.substr(pos); + break; + } + segments[count++] = text.substr(pos, sep - pos); + pos = sep + 1; + } + + if (count != segments.size() || segments[0] != kTokenVersion) + return std::nullopt; + + CasRelinkSourceToken token; + const std::array fields{ + &token.pool_uuid, &token.server_root_id, &token.root_namespace, + &token.ref_name, &token.part_name, &token.manifest_ref_text}; + + for (size_t i = 0; i < kFieldCount; ++i) + { + auto decoded = percentDecodeStrict(segments[i + 1]); + if (!decoded || decoded->empty() || decoded->size() > kMaxFieldBytes) + return std::nullopt; + *fields[i] = std::move(*decoded); + } + return token; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedExchange.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedExchange.h new file mode 100644 index 000000000000..ac05a13bc78d --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedExchange.h @@ -0,0 +1,260 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ + +class ReadPipeline; +struct ReadSettings; + +/// The answer of the relink confirm as it crosses the exchange seam (spec §confirm-primitive). Declared +/// here, on the narrow interface, so `DataPartsExchange` can carry the answer without including any +/// content-addressed header; `ContentAddressedMetadataStorage` maps `Cas::ConfirmAnswer` onto it. +/// +/// ONLY `Yes` AUTHORIZES ANYTHING. `No` and `Unknown` are one outcome for every caller -- both mean +/// "not proven", both are `SourceProofFailed` in the spec's failure taxonomy, and neither may be used +/// to conclude anything about the source. That is not a simplification, it is a coupling: gate 1 +/// evaluates the mount fence LAST (`CasRefLedger::confirmExactRef` rule 6), so a mount that has already +/// lost its fence -- and can therefore no longer speak for the namespace at all -- still answers `No` +/// for a token that does not match its last-known row. Any code that ever treats `No` as authoritative +/// knowledge (say, to skip a retry or to conclude the part is gone) makes that ordering wrong and must +/// hoist rule 6 above the row comparison first. +enum class CasConfirmAnswer : uint8_t +{ + Yes, + No, + Unknown, +}; + +/// The sender's confirm token for one relink offer (spec §wire-protocol). It rides back to the receiver +/// as a response cookie on the offer and returns verbatim as the only argument of the confirm request, +/// so the receiver never has to understand a field of it and the sender never has to remember anything +/// between the two requests. Every field is minted by the sender out of its OWN committed state; when +/// it comes back it is untrusted peer input and is used only as a lookup key -- `resolveContentAddressedConfirm` +/// answers whatever the fields happen to select, and selects nothing at all when they match nothing. +/// +/// `pool_uuid` and `server_root_id` route the question to the one mount entitled to answer it (a pool +/// UUID is shared by every server root writing into the pool, so it cannot select a mount on its own); +/// `root_namespace` and `ref_name` name the binding; `manifest_ref_text` is the exact manifest the +/// offer carried; `part_name` is gate 0's key into the sender's parts set. +struct CasRelinkSourceToken +{ + String pool_uuid; + String server_root_id; + String root_namespace; + String ref_name; /// the ref the sender published the part under (`detached/` for B66b) + String part_name; /// the MergeTree part name + String manifest_ref_text; /// canonical `writer_epoch:build_sequence:manifest_ordinal` +}; + +/// The token's wire form: `car1||…|`, each field percent-encoded down to the RFC 3986 +/// unreserved set. The encoding exists because two of the fields are not character-safe as they stand +/// -- a namespace carries `/` and `@`, and `server_root_id` is whatever the operator configured -- and +/// the token has to survive both an HTTP cookie value and a URL query parameter. Percent-encoding is +/// what makes that true by construction, instead of by a character allowlist that would silently +/// disable relink for a legal-but-unusual `server_root_id`. +/// +/// `nullopt` in either direction means "not a token": an empty or over-long field on the way out, and +/// on the way in a wrong version tag, a wrong field count, a malformed escape, a control character or +/// an over-long field. A refusal is never an answer about the source -- the sender simply makes no +/// offer, and the receiver's confirm is simply unproven. +std::optional encodeCasRelinkSourceToken(const CasRelinkSourceToken & token); +std::optional decodeCasRelinkSourceToken(std::string_view text); + +/// Receiver side, the outcome of staging a relink (spec §failure-taxonomy). Two values, because the +/// receiver has exactly two safe responses to a relink that did not commit, and they are NOT +/// interchangeable: +/// - `Prepared` -- the receiver's `+1` is durable and the caller now OWES the handle a terminal +/// operation. It has published nothing yet. +/// - `MechanismFallbackAllowed` -- relink cannot work here, nothing was staged, and the SENDER STILL +/// HAS THE PART, so re-requesting the bytes from that same sender is a sound recovery. +/// +/// A failure to prove the source is deliberately NOT representable here. That is the whole reason this +/// is typed at all: `adoptPartFromManifest` used to catch every `Exception` and return `false`, which +/// turned "the source could not prove it still holds the manifest" into "ask the source for the bytes" +/// -- the one recovery that is unsound, because the doubt is about the source itself. +enum class CaRelinkPrepare : uint8_t +{ + Prepared, + MechanismFallbackAllowed, +}; + +/// Receiver side, the outcome of promoting a prepared relink. THREE values, because a promote has +/// three outcomes and only two of them are knowledge: +/// - `Committed` -- the receiver's ref is committed. +/// - `MechanismFallbackAllowed` -- the promote is PROVEN to have committed nothing (it was rejected +/// before its ref-log append: a body-absent precommit, a precommit that is no longer the live owner, +/// a ref conflict), which leaves the sender's copy the only authority and its bytes a sound recovery. +/// - `Unresolved` -- the ref-log append was attempted and did not come back with a verdict, so the ref +/// MAY be committed. This is NOT a mechanism fallback and must never be reported as one: fetching +/// the bytes after a relink that actually committed publishes the same part twice. The only sound +/// recovery is to retry the whole fetch later, which is what the caller does with it. +/// +/// Anything else PROPAGATES as an exception rather than becoming a value here -- an unclassified local +/// failure is not evidence that a byte fetch would fare better, and hiding it is exactly the catch-all +/// this boundary replaced. +enum class CaRelinkPromote : uint8_t +{ + Committed, + MechanismFallbackAllowed, + Unresolved, +}; + +/// Receiver side: a relink that is DURABLE BUT NOT PROMOTED, as the exchange sees it. It exists because +/// the confirm has to interpose between the receiver's `+1` becoming durable and the promote (spec +/// §relink-handle), and it is abstract so `DataPartsExchange` can own that window without owning any +/// content-addressed type. +/// +/// Exactly one terminal operation is owed, `promote` or `abort`. Destruction is a backstop, not a +/// substitute: the underlying transaction's precommit binding is live-epoch, so no sweep and no GC +/// reclaims it -- only the removal that `abort` appends does. +class ICaPreparedRelink +{ +public: + virtual ~ICaPreparedRelink() = default; + + /// Commits the receiver's ref over the shared-pool blobs. Call ONLY after the source has proven it + /// still holds exactly the offered manifest. `Unresolved` is a real outcome, not a defensive + /// leftover: the commit may be durable, so the caller must neither publish the part nor fetch its + /// bytes. + virtual CaRelinkPromote promote() = 0; + + /// Releases the durable `+1` by appending the exact precommit removal. NEVER THROWS, and that is a + /// requirement rather than a convenience: this is what a scope guard runs while an exception is + /// already in flight (the receiver's own retry-later error), and it must neither replace that error + /// nor terminate the process. An append that fails is logged here and retried by the underlying + /// handle's own destructor. Calling it after a `promote` -- successful or not -- is a no-op: a + /// promote that failed has already discharged the duty on its way out. + virtual void abort() noexcept = 0; +}; + +/// Purpose-built seam for `DataPartsExchange`: it exposes everything replication needs from a +/// content-addressed disk and nothing else. `ContentAddressedMetadataStorage` implements it, and +/// the exchange obtains the interface by casting the disk's `IMetadataStorage` to this interface +/// rather than depending on the concrete storage class. Keeping this boundary narrow prevents the +/// replication path from becoming coupled to content-addressed storage internals. +/// +/// Relink wire contract: the sender transmits `{pool_uuid, encoded PartManifest body, confirm token}`. +/// The replica-internal `part_id` cookie carries the opaque manifest bytes and a response cookie carries +/// the token, so this exchange adds no protocol field of its own. The manifest's sender-specific identity (`ManifestRef`, `root_namespace_id`, +/// and `payload_digest`) is not authoritative: the receiver uses only the entries, adopts references +/// to blobs in the shared pool by hash without reading blob bodies from the sender, and publishes a +/// fresh manifest in its own namespace. Every per-part file is an ordinary manifest entry, so the +/// manifest is self-contained and there is no separate sidecar or metadata-version wire field. +/// When the local manifest cannot be committed — for example a required blob body is absent at +/// precommit — adoption publishes nothing and the caller falls back to fetching the part bytes. +class IContentAddressedExchange +{ +public: + virtual ~IContentAddressedExchange() = default; + + /// The pool's stable identity (Cas::PoolMeta::pool_id, hex). Two replicas may relink iff equal + /// — endpoint/prefix string-matching is unsafe (false positives => mis-relink). Empty before + /// the storage started up. + virtual const String & getPoolUUID() const = 0; + + /// Sender side, routing predicate for the confirm action (spec §wire-protocol "Routing contract"): + /// does THIS instance own `root_namespace` under `server_root_id`? A pool UUID is shared by every + /// server root writing into the pool, so `getPoolUUID` alone cannot select the mount that is + /// entitled to answer for a namespace; the caller pairs the two and requires EXACTLY one match + /// (zero or several are both `Unknown`). I/O-free and never throws in any lifecycle state -- a + /// routing predicate that could fail would turn a misrouted question into an error instead of an + /// unproven answer. + virtual bool ownsNamespace(const String & server_root_id, const String & root_namespace) const = 0; + + /// Sender side, gate 1 of the relink confirm (spec §confirm-primitive), forwarded to the ledger: + /// does `ref_name` in `root_namespace` still name EXACTLY the manifest rendered by + /// `manifest_ref_text` (the canonical `writer_epoch:build_sequence:manifest_ordinal` form) in this + /// writer's committed view? Read-only, performs ZERO object-store I/O, and creates nothing: a cold, + /// evicted, recovering, busy, wedged, poisoned or unfenced table answers `Unknown` rather than + /// doing work, because a remote peer drives this query. Never throws: an unparsable token and a + /// disk that is not started or has reached a terminal lifecycle are `Unknown` too. + virtual CasConfirmAnswer confirmExactRef(const String & root_namespace, const String & ref_name, + const String & manifest_ref_text) const = 0; + + /// Sender side: everything one relink offer puts on the wire. The manifest body is opaque to the + /// exchange caller and is decoded by the receiver; the token is what the receiver hands back to + /// confirm the offer before it promotes. + struct RelinkOffer + { + String manifest_bytes; + String confirm_token; + }; + + /// Sender side: build the relink offer for this server's committed part at the given disk-relative + /// path. `nullopt` means the path is not a committed content-addressed part here, or the token + /// could not be minted, so the sender must make no offer and streams the part bytes instead. + /// + /// The manifest body and the token come out of ONE resolution of the part, and that is the point of + /// returning them together rather than as two calls: a repoint between them would hand the receiver + /// a token naming a manifest whose entries it never adopted, and a `Yes` for that manifest would + /// protect the wrong blobs. + virtual std::optional getRelinkOffer(const String & part_path) const = 0; + + /// Receiver side, the FIRST half of a relink: decode the transferred manifest, perform a normal + /// local build from shared-pool blob references without reading a single blob body from the sender, + /// stage a fresh manifest in the receiver namespace, `precommitAdd` it -- and STOP. The sender's + /// `root_namespace_id`, `ManifestRef` and `payload_digest` are ignored; only the entries are used. + /// + /// `part_path` is the RECEIVER's disk-relative path of the part directory being built, addressed the + /// same way `getRelinkOffer` addresses the sender's: the namespace and the ref name come from + /// routing it, so the caller never composes a ref name and a relink into `TABLE/detached/DIR` + /// (B66b) lands on the `detached/` ref for free, through the one router every other read and + /// write of that part uses. A path that does not route to a part DIRECTORY of a live table -- a + /// table dir, a file inside a part, a FREEZE shadow path -- is a caller error and throws. + /// + /// On `Prepared` the receiver's `+1` is DURABLE and `out` holds the handle that owes the terminal + /// operation; nothing is committed and no ref is live yet. The promote is deferred because the + /// receiver must first ask the source whether it still holds exactly this manifest (spec + /// §core-idea): the `+1` has to be durable BEFORE that question is asked, or the answer proves + /// nothing about the window that follows it. + /// + /// On `MechanismFallbackAllowed` nothing was staged and `out` is null: the manifest did not decode, + /// or the local staging hit the retryable class (a body-absent precommit, a precommit that is no + /// longer the live owner, a ref conflict -- `ABORTED`/`NETWORK_ERROR`). Any OTHER error propagates. + /// + /// Promotion trusts the adopted references through the durable manifest edge -- it does not re-read + /// each blob body -- the same interserver trust as an ordinary `ReplicatedMergeTree` fetch. A blob + /// that becomes absent or condemned after adoption is not caught here; it is an fsck-detectable + /// invariant violation. + virtual CaRelinkPrepare prepareAdoptFromManifest( + const String & part_path, + const String & manifest_bytes, + std::unique_ptr & out) = 0; + + /// ==== `DiskObjectStorage::prepareRead` hooks ==== + /// The two CA-only reads `DiskObjectStorage::prepareRead` needs before it composes the standard + /// object-storage pipeline. Exposed on this narrow seam (rather than only on the concrete + /// `ContentAddressedMetadataStorage`) so `prepareRead` casts to the interface instead of coupling + /// to the concrete storage class. + + /// The CA read entry called by `DiskObjectStorage::prepareRead` before the generic + /// storage-objects path: serves in-manifest bytes (mutable per-part files, inline entries, + /// verbatim namespace files) from memory. Returns false when the path is not in-manifest. + virtual bool prepareInManifestRead(const std::string & path, const ReadSettings & settings, ReadPipeline & pipeline) const = 0; + + /// Translates a blob-backed part file to the physical blob + /// object plus the payload WINDOW inside it (the CHCA envelope header occupies + /// [0, payload_offset)). DiskObjectStorage::prepareRead composes the STANDARD object-storage + /// pipeline over `object` (gather/caches/async prefetch — the same chain plain s3 disks get, + /// so `MergeTreeReaderStream` right-mark bounds reach the object reader and its range + /// requests stay drainable) and bounds it with the pipeline's FileView stage. + /// nullopt = the path is not a blob-backed part file (caller falls through; absent paths + /// then fail in getStorageObjects exactly as before). + struct BlobViewPlan + { + StoredObject object; /// physical blob key; logical path; readable extent (envelope + payload) + size_t payload_offset = 0; /// view left bound inside the blob + size_t payload_end = 0; /// view right bound (payload_offset + payload length) + }; + /// Resolves a blob-backed path to its physical object and payload window. Returns nullopt for + /// in-manifest, loose, directory, or otherwise unresolved paths. + virtual std::optional getBlobViewPlan(const std::string & path) const = 0; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.cpp new file mode 100644 index 000000000000..31a988bfa038 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.cpp @@ -0,0 +1,2299 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int FILE_DOESNT_EXIST; + extern const int LOGICAL_ERROR; + extern const int CORRUPTED_DATA; + extern const int READONLY; + extern const int BAD_ARGUMENTS; + extern const int ABORTED; + extern const int NETWORK_ERROR; + extern const int NOT_IMPLEMENTED; + extern const int INVALID_STATE; +} + +namespace ContentAddressedSetting +{ + extern const ContentAddressedSettingsString scratch_path; + extern const ContentAddressedSettingsString server_root_id; + extern const ContentAddressedSettingsBool gc_enabled; + extern const ContentAddressedSettingsUInt64 gc_interval_sec; + extern const ContentAddressedSettingsUInt64 deduplication_cache_bytes; + extern const ContentAddressedSettingsUInt64 deduplication_head_first_min_bytes; + extern const ContentAddressedSettingsUInt64 gc_snapshot_generations_to_keep; + extern const ContentAddressedSettingsUInt64 gc_shards; + extern const ContentAddressedSettingsUInt64 manifest_sweep_list_budget_keys; + extern const ContentAddressedSettingsUInt64 manifest_sweep_delete_budget_keys; + extern const ContentAddressedSettingsUInt64 gc_round_graduation_budget; + extern const ContentAddressedSettingsUInt64 gc_round_redelete_budget; + extern const ContentAddressedSettingsUInt64 gc_round_sweep_namespace_budget; + extern const ContentAddressedSettingsUInt64 gc_round_sweep_recovery_op_budget; + extern const ContentAddressedSettingsUInt64 gc_round_ref_cleanup_budget; + extern const ContentAddressedSettingsUInt64 gc_round_prefix_wholesale_budget; + extern const ContentAddressedSettingsUInt64 gc_round_handoff_prefix_wholesale_budget; + extern const ContentAddressedSettingsUInt64 gc_round_outcome_entry_budget; + extern const ContentAddressedSettingsUInt64 gcs_max_conditional_put_bytes; + extern const ContentAddressedSettingsUInt64 part_folder_cache_bytes; + extern const ContentAddressedSettingsUInt64 part_folder_cache_max_entries; + extern const ContentAddressedSettingsUInt64 part_folder_cache_max_entry_bytes; + extern const ContentAddressedSettingsUInt64 manifest_decode_cache_bytes; + extern const ContentAddressedSettingsUInt64 gc_meta_pool_size; + extern const ContentAddressedSettingsBool blob_hash_allow_new; + extern const ContentAddressedSettingsBool skip_access_check; +} + +namespace +{ +/// The `lifecycle` column value for `system.cas_mounts` (spec §7): the pool lifecycle +/// condition collapsed to the operator-facing vocabulary. The two `Vanished*` sub-states both map to the +/// bare `vanished` -- the sub-state (replaced/forgotten) lives in the `lifecycle_reason` column, +/// so a `NULL`-free `lifecycle` stays a small, stable enumerated set. +const char * casLifecycleToString(Cas::PoolLifecycle lc) +{ + switch (lc) + { + case Cas::PoolLifecycle::Live: return "live"; + case Cas::PoolLifecycle::TransientNotLive: return "not_live"; + case Cas::PoolLifecycle::IdentityLost: return "identity_lost"; + case Cas::PoolLifecycle::VanishedReplaced: + case Cas::PoolLifecycle::VanishedForgotten: return "vanished"; + } + return "unknown"; +} + +/// The ENUM-CLEAN `lifecycle_reason` word: the `vanished` sub-state (replaced/forgotten) so a +/// downstream `lifecycle || '(' || lifecycle_reason || ')'` yields exactly e.g. `vanished(forgotten)`. +/// Empty for every non-`vanished` state (the `lifecycle` column already fully names those). The rich [D5] +/// text is carried separately in `lifecycle_detail`. +const char * casLifecycleReasonWord(Cas::PoolLifecycle lc) +{ + switch (lc) + { + case Cas::PoolLifecycle::Live: + case Cas::PoolLifecycle::TransientNotLive: + case Cas::PoolLifecycle::IdentityLost: return ""; + case Cas::PoolLifecycle::VanishedReplaced: return "replaced"; + case Cas::PoolLifecycle::VanishedForgotten: return "forgotten"; + } + return ""; +} +} + +/// ============================================================================================ +/// The method -> operation-class inventory (rev.7 spec §1; the review artifact of Task 8). +/// +/// EVERY public method of `ContentAddressedMetadataStorage` and `ContentAddressedTransaction` is listed +/// with the `CasOpClass` it routes through `checkOpAdmitted` (or "Factory" for the never-gated I/O-free +/// surface). The gate is consulted at the method's entry; `store()`/`partAccess()`/`poolAccess()` keep +/// their own terminal check (Task 5) as low-level defense, reached only in the `Live` case. +/// +/// ---- ContentAddressedMetadataStorage ---- +/// Factory (never gated): getType, getPath, supportsChmod, supportsStat, isReadOnly, isContentAddressed, +/// transactionIsStagingOverlay, supportsAtomicFileWrites, supportsTransactionalMutableFiles, +/// areBlobPathsRandom, getHardlinkCount, createTransaction (I/O-free -- allocates a txn), getPoolUUID, +/// serverRootId, scratchPath, stagingBackend, conditionalCopySupported, objectStorage, gcHealth, +/// lifecycleSnapshot (both non-store()-gated introspection reads for system.cas_mounts -- +/// readable in EVERY lifecycle state including a not-live/vanished/null pool, spec §7), +/// parseStagingBackend/parsePartFolderValidate/ +/// tryFromDisk (static), checkNotReadOnly, the *ForTest seams, serverPrefix/liveNamespace/ +/// shadowNamespace/route/classifyDirectory (pure path computation, no pool I/O), +/// ownsNamespace (the relink-confirm routing predicate -- a string comparison against +/// `server_root_id`, deliberately answerable in EVERY lifecycle state). +/// Probe: existsFile, existsDirectory, existsFileOrDirectory, listDirectory, iterateDirectory, +/// isDirectoryEmpty, getStorageObjectsIfExist, liveTreeDirHasChildren, listLiveTreeChildren. +/// EMPTY-PROOF RULE (Task 9, spec §1 [B3]): on a NON-terminal (Live/read-only) pool, an +/// empty `listDirectory` answer at a `TableDir`/`DetachedContainer` root additionally runs +/// `confirmPoolIdentityForEmptyEnumeration` (one authoritative, UNCACHED `_pool_meta` probe) +/// -- a `KeyAbsent`/`ContainerAbsent`/transport result throws the typed 668 instead of the +/// empty answer. `iterateDirectory`/`isDirectoryEmpty` inherit it (both funnel through +/// `listDirectory`). A `Vanished` pool never reaches it (the gate short-circuits `Probe`). +/// ContentRead: getFileSize, getLastModified, getStorageObjects, getBlobViewPlan, readBlobPayload, +/// prepareInManifestRead, tryGetInManifestBytes, getRelinkOffer, confirmExactRef +/// (the ONE ContentRead entry that converts the gate's refusal into its own typed +/// `Unknown` answer instead of propagating it -- a confirm never throws at its caller). +/// Write: adoptPartFromManifest. +/// Admin: runOneGcRoundForTest, runGarbageCollectionRoundNow, runGcRebuildNow, runFsckNow (the +/// rev.8 FSCK-on-running path -- an FSCK of a not-live disk is refused by the Admin gate). +/// Lifecycle/uncgated drivers (NOT op-gated -- they DRIVE the state): startup, shutdown, forgetDisk, +/// gcStop, gcStart. store()/partAccess()/poolAccess() are the internal accessors, not public op entries. +/// readableNamespaceFilesLife, stagingKeyPrefix, detachedRefNames, movingRefNames are post-gate helpers +/// (their public callers gate first; they reach store() only in the Live case). +/// confirmPoolIdentityForEmptyEnumeration is a post-gate helper too (the EMPTY-PROOF RULE, Task 9): +/// listDirectory calls it only after the gate admitted a Probe and only on an empty table-root answer. +/// +/// ---- ContentAddressedTransaction (routes through metadata_storage.checkOpAdmitted) ---- +/// Write: writeFile (and tryCreateWriteBuffer, which funnels into it), createDirectory, +/// createDirectoryRecursive, createHardLink, moveDirectory, moveFile, replaceFile, +/// setLastModified, setReadOnly. +/// Remove: removeDirectory, removeRecursive, unlinkFile. +/// commit/tryCommit: Remove when there is nothing to publish (parts empty -- the DROP/rename path, which +/// applied its ref mutations immediately), Write when it must publish staged parts. This is what +/// lets a vanished-disk table's DROP finish (Remove -> no-op success) while a publishing commit +/// throws the typed Vanished [D5] refusal. +/// Unsupported (always throw, state-independent -- no gate needed): createMetadataFile, +/// generateObjectKeyForPath, chmod, truncateFile. +/// Factory / overlay-only (no committed-pool I/O): supportsChmod, getSubmittedForRemovalBlobs, and the +/// read-your-writes overlay readers tryGetInFlightStorageObjects/tryReadFileInFlight/ +/// tryGetInFlightFileSize/hasInFlightDirectory/listInFlightDirectory (they read THIS transaction's own +/// in-memory staging; a vanished-disk transaction has none because its writes threw at writeFile). +/// +/// Null-pool rule: a storage with no published pool (before `startup` or after `shutdown` -- the +/// storage-level Constructing/ShutDown lifecycle) fails loud for EVERY class, `Probe` included, via +/// `throwStorageNotStarted`. There is no benign "absent" answer for a storage that has never published a +/// pool (or torn one down); the pool lifecycle below is the sole authority for a published pool. +/// ============================================================================================ + +namespace +{ + +/// Canonical disk-relative path: components joined by single '/', no leading/trailing slashes. +/// Callers hand paths in both shapes (the Unfreezer walks shadow dirs WITH a trailing slash); +/// namespace strings and prefix matching need the canonical form. +std::string canonicalDiskPath(const std::string & path) +{ + std::string result; + std::string component; + auto flush = [&] + { + if (component.empty()) + return; + if (!result.empty()) + result += '/'; + result += component; + component.clear(); + }; + for (char c : path) + { + if (c == '/') + flush(); + else + component.push_back(c); + } + flush(); + return result; +} + +/// "/" -> {first, rest} ({whole, ""} when there is no '/'). +std::pair splitFirstComponent(const std::string & s) +{ + const auto slash = s.find('/'); + if (slash == std::string::npos) + return {s, ""}; + return {s.substr(0, slash), s.substr(slash + 1)}; +} + +void addFirstComponent(std::unordered_set & out, const std::string & name) +{ + const auto slash = name.find('/'); + out.emplace(slash == std::string::npos ? name : name.substr(0, slash)); +} + +/// Drop a trailing `@cas@` content-addressing boundary marker from a mirrored path segment, so a +/// table-dir surfaces under its logical (unsuffixed) name in directory listings. +std::string stripCasArchiveSuffix(std::string s) +{ + const auto & suffix = Cas::kCasArchiveSuffix; + if (s.size() >= suffix.size() && std::string_view(s).ends_with(suffix)) + s.resize(s.size() - suffix.size()); + return s; +} + +std::vector toVector(std::unordered_set && set) +{ + return std::vector(std::make_move_iterator(set.begin()), std::make_move_iterator(set.end())); +} + +/// The server uuid string (with dashes) -> the core's UInt128 server id. +UInt128 serverIdToU128(const std::string & server_id) +{ + String hex; + hex.reserve(32); + for (char c : server_id) + if (c != '-') + hex += c; + if (hex.size() == 32) + return Cas::hexToU128(hex); + /// Unit-test ids ("srv1") are not uuids — hash them stably. + UInt128 r{}; + for (char c : server_id) + r = r * 131 + static_cast(c); + return r == UInt128(0) ? UInt128(1) : r; +} + +} + +ContentAddressedMetadataStorage::ContentAddressedMetadataStorage( + ObjectStoragePtr object_storage_, + String storage_path_prefix_, + String server_id_, + String disk_name_, + ContextPtr context_, + const ContentAddressedSettings & settings_) + : object_storage(std::move(object_storage_)) + , storage_path_prefix(std::move(storage_path_prefix_)) + , storage_path_full(fs::path(object_storage->getRootPrefix()) / storage_path_prefix) + , server_id(std::move(server_id_)) + , server_root_id(settings_[ContentAddressedSetting::server_root_id].value) + , disk_name(!disk_name_.empty() ? disk_name_ : storage_path_prefix) + , local_scratch_path(settings_[ContentAddressedSetting::scratch_path].value) + , context(context_) + , gc_enabled(settings_[ContentAddressedSetting::gc_enabled].value) + , gc_interval(std::chrono::seconds(settings_[ContentAddressedSetting::gc_interval_sec].value)) + , deduplication_cache_bytes(settings_[ContentAddressedSetting::deduplication_cache_bytes].value) + , deduplication_head_first_min_bytes(settings_[ContentAddressedSetting::deduplication_head_first_min_bytes].value) + , gc_snapshot_generations_to_keep(settings_[ContentAddressedSetting::gc_snapshot_generations_to_keep].value) + , gc_shards(settings_[ContentAddressedSetting::gc_shards].value) + , manifest_sweep_list_budget_keys(settings_[ContentAddressedSetting::manifest_sweep_list_budget_keys].value) + , manifest_sweep_delete_budget_keys(settings_[ContentAddressedSetting::manifest_sweep_delete_budget_keys].value) + , gc_round_graduation_budget(settings_[ContentAddressedSetting::gc_round_graduation_budget].value) + , gc_round_redelete_budget(settings_[ContentAddressedSetting::gc_round_redelete_budget].value) + , gc_round_sweep_namespace_budget(settings_[ContentAddressedSetting::gc_round_sweep_namespace_budget].value) + , gc_round_sweep_recovery_op_budget(settings_[ContentAddressedSetting::gc_round_sweep_recovery_op_budget].value) + , gc_round_ref_cleanup_budget(settings_[ContentAddressedSetting::gc_round_ref_cleanup_budget].value) + , gc_round_prefix_wholesale_budget(settings_[ContentAddressedSetting::gc_round_prefix_wholesale_budget].value) + , gc_round_handoff_prefix_wholesale_budget(settings_[ContentAddressedSetting::gc_round_handoff_prefix_wholesale_budget].value) + , gc_round_outcome_entry_budget(settings_[ContentAddressedSetting::gc_round_outcome_entry_budget].value) + , gcs_max_conditional_put_bytes(settings_[ContentAddressedSetting::gcs_max_conditional_put_bytes].value) + , cas_part_folder_cache_bytes(settings_[ContentAddressedSetting::part_folder_cache_bytes].value) + , cas_part_folder_cache_max_entries(settings_[ContentAddressedSetting::part_folder_cache_max_entries].value) + , cas_part_folder_cache_max_entry_bytes(settings_[ContentAddressedSetting::part_folder_cache_max_entry_bytes].value) + , manifest_decode_cache_bytes(settings_[ContentAddressedSetting::manifest_decode_cache_bytes].value) + , gc_meta_pool_size(settings_[ContentAddressedSetting::gc_meta_pool_size].value) + , staging_backend(settings_.stagingBackend()) + , blob_hash_algo(settings_.blobHashAlgo()) + , blob_hash_allow_new(settings_[ContentAddressedSetting::blob_hash_allow_new].value) + , skip_access_check(settings_[ContentAddressedSetting::skip_access_check].value) + , part_folder_validate(settings_.partFolderValidate()) +{ +} + +Cas::StagingBackend ContentAddressedMetadataStorage::parseStagingBackend(const std::string & value) +{ + if (value == "local") + return Cas::StagingBackend::Local; + if (value == "s3") + return Cas::StagingBackend::S3; + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Unknown staging_backend value '{}' (expected 'local' or 's3')", value); +} + +Cas::StagingBackend ContentAddressedMetadataStorage::parseStagingBackend( + const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) +{ + return parseStagingBackend(config.getString(config_prefix + ".staging_backend", "local")); +} + +Cas::PartFolderValidate ContentAddressedMetadataStorage::parsePartFolderValidate(const std::string & value) +{ + using PartFolderValidate = Cas::PartFolderValidate; + if (value == "always") + return {PartFolderValidate::Mode::Always, 0}; + if (value == "never") + return {PartFolderValidate::Mode::Never, 0}; + if (value.starts_with("age ")) + { + /// `std::from_chars` against an UNSIGNED type never accepts a leading '-' (unlike + /// `std::stoull`, which silently negates modulo 2^64) -- a malformed/negative/non-digit/empty + /// suffix falls through to the terminal throw below instead of wrapping into an astronomical + /// age_seconds that behaves as skip-forever. + const std::string age_str = value.substr(4); + uint64_t age_seconds = 0; + const auto [ptr, ec] = std::from_chars(age_str.data(), age_str.data() + age_str.size(), age_seconds); + if (ec == std::errc{} && ptr == age_str.data() + age_str.size()) + return {PartFolderValidate::Mode::Age, age_seconds}; + } + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Unknown part_folder_validate value '{}' (expected 'always', 'never', or 'age ')", value); +} + +Cas::PartFolderValidate ContentAddressedMetadataStorage::parsePartFolderValidate( + const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) +{ + return parsePartFolderValidate(config.getString(config_prefix + ".part_folder_validate", "always")); +} + +ContentAddressedMetadataStorage * ContentAddressedMetadataStorage::tryFromDisk(const DiskPtr & disk) +{ + /// The cheap predicate FIRST, never an exception probe: for every non-object-storage disk + /// (DiskLocal & co.) `getMetadataStorage` throws NOT_IMPLEMENTED, and merely CONSTRUCTING that + /// exception increments `system.errors` even when the throw is caught. This function runs on + /// every asynchronous-metrics tick for every configured disk, so the old exception-as-control- + /// flow probe polluted `system.errors` with a steady stream of NOT_IMPLEMENTED on pure-local + /// servers — caught as a stray-error failure by strict-error tests (`test_cancel_backup`'s + /// NoTrashChecker, Altinity PR#2073). `isContentAddressed` is a throw-free virtual (IDisk + /// defaults to false; wrappers forward it — see ReadOnlyDiskWrapper). + if (!disk || !disk->isContentAddressed()) + return nullptr; + /// A content-addressed disk always implements getMetadataStorage (it IS the CA metadata + /// storage), so no NOT_IMPLEMENTED handling is needed past the predicate. + return dynamic_cast(disk->getMetadataStorage().get()); +} + +void ContentAddressedMetadataStorage::runOneGcRoundForTest() +{ + /// Admin class (rev.7 spec §1): refuse on a transient / IdentityLost / Vanished pool (a terminal disk + /// throws the typed Vanished [D5] refusal; an uncertain one throws 668) before touching the scheduler. + /// The later `!cas_store` re-check under `gc_scheduler_mutex` still guards the not-mounted race. + checkOpAdmitted(CasOpClass::Admin); + /// The pacing scheduler must be STABLE across calls: the lease's observation-window steal + /// protocol compares consecutive observations of the SAME observer (gc_id), so an ad-hoc + /// scheduler per call would acquire the lease on the first call and then back off forever + /// ("incumbent alive" - its own previous incarnation). Recreating the scheduler for every call + /// would therefore make every round after the first a silent no-op. + /// Hold gc_scheduler_mutex for the whole round: a concurrent `shutdown` waits for the round to + /// finish because clean GC completion takes priority over fast shutdown. pointer_mutex (a + /// separate, briefly-held mutex) only guards the scheduler snapshot/creation below, so + /// gcHealth/store/partAccess never block behind this round. + /// Test-only seam (inert in production): lets a test interleave a concurrent FORGET into the window + /// between the pre-lock admission check above and the lock acquisition below -- the exact I-1 TOCTOU. + if (gc_verb_admit_window_hook_for_test) + gc_verb_admit_window_hook_for_test(); + std::lock_guard round_lock(gc_scheduler_mutex); + /// Re-run the admission gate UNDER `gc_scheduler_mutex` (mirroring `gcStart`'s lock-then-gate): the + /// pre-lock check above is only a fast-fail. A round admitted while `Live` can block on this mutex behind + /// a concurrent FORGET (which holds it for its whole teardown); once FORGET settles the pool `Vanished`, + /// this re-check throws the typed [D5] refusal instead of resurrecting a scheduler on a decommissioned + /// pool. `checkOpAdmitted` takes only the brief `pointer_mutex` -- lock order gc_scheduler -> pointer. + checkOpAdmitted(CasOpClass::Admin); + if (shutdown_called) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot run garbage collection after ContentAddressedMetadataStorage shutdown has begun"); + std::shared_ptr snapshot; + { + std::lock_guard ptr_lock(pointer_mutex); + if (!gc_scheduler) + { + /// `checkOpAdmitted(Admin)` above already fails loud on a null pool; this is the defensive + /// re-check for a concurrent `shutdown` reset between the two `pointer_mutex` acquisitions. + /// Surface the same `INVALID_STATE` refusal `poolAccess()` gives, never `LOGICAL_ERROR` + /// (which would abort under debug/ASan builds). + if (!cas_store) + throwStorageNotStarted(); + gc_scheduler = std::make_shared( + cas_store, gc_interval, fmt::format("{}::ContentAddressedGC", storage_path_full), + disk_name, makeGcRoundLogger()); + } + snapshot = gc_scheduler; + } + snapshot->runOneRoundNow(); +} + +void ContentAddressedMetadataStorage::requestGcRoundSoon() +{ + std::shared_ptr snapshot; + { + std::lock_guard lock(pointer_mutex); + snapshot = gc_scheduler; + } + if (snapshot) + snapshot->requestRoundSoon(); +} + +std::optional ContentAddressedMetadataStorage::gcHealth() const +{ + /// A brief pointer_mutex snapshot only -- this must NEVER wait behind gc_scheduler_mutex (an + /// in-flight round can hold that for a long time; an unprivileged SELECT on + /// system.cas_mounts must not stall behind it). The snapshot keeps the scheduler + /// alive via its own refcount even if `shutdown` concurrently resets the member, and + /// CasGcScheduler::gcHealth() is itself lock-free (atomic reads), so calling it outside any lock + /// here is safe. + std::shared_ptr snapshot; + { + std::lock_guard lock(pointer_mutex); + snapshot = gc_scheduler; + } + if (!snapshot) + return std::nullopt; + return snapshot->gcHealth(); +} + +CasLifecycleSnapshot ContentAddressedMetadataStorage::lifecycleSnapshot() const +{ + /// Factory-class (spec §7): I/O-free and reachable in EVERY state. NEVER calls store()/poolAccess() + /// (which refuse a not-mounted disk) or touches the backend -- that is the whole point, so the disk + /// that vanished is still visible. Only a brief pointer_mutex snapshot of the pool pointer, plus reads + /// of storage members that are immutable after startup. + CasLifecycleSnapshot snap; + snap.server_root_id = server_root_id; + /// The last-known pool identity. `pool_uuid` is written once (single-threaded) at the end of `startup` + /// and never reset by `shutdown`, so it is empty ONLY before the first successful startup and stable + /// thereafter -- the disk stays introspectable under its identity even once the pool is torn down. + snap.pool_id = pool_uuid; + + Cas::PoolPtr pool; + { + std::lock_guard lock(pointer_mutex); + pool = cas_store; + } + if (!pool) + { + /// No pool published: the storage-level lifecycle (spec §1's Constructing/ShutDown). Distinguish + /// the two by whether startup ever ran, which `pool_uuid` records (empty => never started). A + /// disk that was started then torn down (shutdown) reports `shutdown`. reason/detail/since stay + /// empty/0 -- no terminal cause. + snap.lifecycle = snap.pool_id.empty() ? "constructing" : "shutdown"; + return snap; + } + + const Cas::Pool::LifecycleSnapshot ps = pool->lifecycleSnapshot(); + snap.lifecycle = casLifecycleToString(ps.lifecycle); + snap.reason = casLifecycleReasonWord(ps.lifecycle); + snap.detail = ps.detail; + snap.since = ps.since; + return snap; +} + +Cas::GcRoundLogger ContentAddressedMetadataStorage::makeGcRoundLogger() const +{ + /// Unit tests pass a null context (no system logs); the scheduler then runs without a sink. + if (!context) + return {}; + auto ctx = context; + /// The configured disk name (threaded from the metadata-storage factory); falls back to + /// storage_path_prefix for callers that don't supply one (e.g. unit tests). + const String disk = disk_name; + return [ctx, disk](const Cas::GcRoundLogRecord & r) + { + auto log = ctx->getContentAddressedGarbageCollectionLog(); + if (!log) + return; + ContentAddressedGarbageCollectionLogElement e; + const auto now = std::chrono::system_clock::now(); + e.event_time = std::chrono::system_clock::to_time_t(now); + e.event_time_microseconds = timeInMicroseconds(now); + switch (r.event_type) + { + case Cas::GcRoundLogRecord::EventType::Start: + e.event_type = ContentAddressedGarbageCollectionLogElement::START; + break; + case Cas::GcRoundLogRecord::EventType::Finish: + e.event_type = ContentAddressedGarbageCollectionLogElement::FINISH; + break; + case Cas::GcRoundLogRecord::EventType::Phase: + e.event_type = ContentAddressedGarbageCollectionLogElement::PHASE; + break; + } + e.disk_name = r.disk_name.empty() ? disk : r.disk_name; + e.srid = r.srid; + e.gc_id = r.gc_id; + e.trigger = r.trigger == Cas::GcRoundLogRecord::Trigger::Manual + ? ContentAddressedGarbageCollectionLogElement::MANUAL + : ContentAddressedGarbageCollectionLogElement::SCHEDULED; + switch (r.outcome) + { + case Cas::GcRoundLogRecord::Outcome::Unknown: + e.outcome = ContentAddressedGarbageCollectionLogElement::UNKNOWN; + break; + case Cas::GcRoundLogRecord::Outcome::Success: + e.outcome = ContentAddressedGarbageCollectionLogElement::SUCCESS; + break; + case Cas::GcRoundLogRecord::Outcome::NotALeader: + e.outcome = ContentAddressedGarbageCollectionLogElement::NOT_A_LEADER; + break; + case Cas::GcRoundLogRecord::Outcome::Failed: + e.outcome = ContentAddressedGarbageCollectionLogElement::FAILED; + break; + case Cas::GcRoundLogRecord::Outcome::Deferred: + e.outcome = ContentAddressedGarbageCollectionLogElement::DEFERRED; + break; + } + e.round = r.round; + e.candidates_marked = r.candidates_marked; + e.objects_deleted = r.objects_deleted; + e.objects_absent = r.objects_absent; + e.objects_replaced = r.objects_replaced; + e.objects_spared = r.objects_spared; + e.manifests_deleted = r.manifests_deleted; + e.entries_condemned = r.entries_condemned; + e.entries_graduated = r.entries_graduated; + e.entries_redeleted = r.entries_redeleted; + e.fence_outs = r.fence_outs; + e.anomalies = r.anomalies; + e.duration_ms = r.duration_ms; + e.error = r.error; + e.profile_events = r.profile_events; + e.round_id = r.round_id; + e.phase = r.phase; + e.phase_duration_microseconds = r.phase_duration_microseconds; + e.phase_metrics = r.phase_metrics; + /// Best-effort: SystemLog::add never blocks GC; a full queue drops the row with a warning. + log->add(std::move(e)); + }; +} + +Cas::CasEventSink ContentAddressedMetadataStorage::makeCasEventSink() const +{ + /// Unit tests pass a null context (no system logs); the Pool then runs without a sink. + if (!context) + return {}; + auto ctx = context; + /// The configured disk name (threaded from the metadata-storage factory); falls back to + /// storage_path_prefix for callers that don't supply one (e.g. unit tests). + const String disk = disk_name; + return [ctx, disk](Cas::CasEvent ev) + { + auto log = ctx->getContentAddressedLog(); + if (!log) + return; + ContentAddressedLogElement e; + const auto now = std::chrono::system_clock::now(); + e.event_time = std::chrono::system_clock::to_time_t(now); + e.event_time_microseconds = timeInMicroseconds(now); + e.event_type = toString(ev.type); + e.disk_name = disk; + e.namespace_ = std::move(ev.namespace_); + e.ref_name = std::move(ev.ref_name); + e.object_kind = toString(ev.object_kind); + e.object_hash = std::move(ev.object_hash); + e.token = std::move(ev.token); + e.round = ev.round; + e.gen = ev.gen; + e.at_version = ev.at_version; + e.outcome = std::move(ev.outcome); + e.reason = std::move(ev.reason); + e.thread_id = getThreadId(); + e.query_id = CurrentThread::getQueryId(); + e.detail = std::move(ev.detail); + /// Best-effort: SystemLog::add never blocks the Core; a full queue drops the row with a warning. + log->add(std::move(e)); + }; +} + +Cas::RoundReport ContentAddressedMetadataStorage::runGarbageCollectionRoundNow() +{ + checkNotReadOnly("GC round"); + if (!gc_enabled) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Garbage collection is not enabled on this content-addressed disk"); + /// Admin class (rev.7 spec §1): refuse on a transient / IdentityLost / Vanished pool before touching + /// the scheduler -- `SYSTEM CAS GC RUN` reaches here directly. + checkOpAdmitted(CasOpClass::Admin); + /// Mirror runOneGcRoundForTest: a STABLE scheduler instance across calls (the lease's + /// observation-window steal protocol compares consecutive observations of the same gc_id). + /// Hold gc_scheduler_mutex for the whole round: a concurrent `shutdown` waits for the round to + /// finish because clean GC completion takes priority over fast shutdown. pointer_mutex only + /// guards the scheduler snapshot/creation below, so gcHealth/store/partAccess never block behind + /// this round. + /// Test-only seam (inert in production): lets a test interleave a concurrent FORGET into the window + /// between the pre-lock admission check above and the lock acquisition below -- the exact I-1 TOCTOU. + if (gc_verb_admit_window_hook_for_test) + gc_verb_admit_window_hook_for_test(); + std::lock_guard round_lock(gc_scheduler_mutex); + /// Re-run the admission gate UNDER `gc_scheduler_mutex` (mirroring `gcStart`'s lock-then-gate): the + /// pre-lock check above is only a fast-fail. A round admitted while `Live` can block on this mutex behind + /// a concurrent FORGET (which holds it for its whole teardown); once FORGET settles the pool `Vanished`, + /// this re-check throws the typed [D5] refusal instead of resurrecting a scheduler on a decommissioned + /// pool. `checkOpAdmitted` takes only the brief `pointer_mutex` -- lock order gc_scheduler -> pointer. + checkOpAdmitted(CasOpClass::Admin); + if (shutdown_called) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot run garbage collection after ContentAddressedMetadataStorage shutdown has begun"); + std::shared_ptr snapshot; + { + std::lock_guard ptr_lock(pointer_mutex); + if (!gc_scheduler) + { + /// Same reasoning as `runOneGcRoundForTest` above: `checkOpAdmitted(Admin)` already failed + /// loud on a null pool, so this is the defensive re-check for a concurrent `shutdown` reset + /// -- a normal `INVALID_STATE` refusal, never a `LOGICAL_ERROR` abort. + if (!cas_store) + throwStorageNotStarted(); + gc_scheduler = std::make_shared( + cas_store, gc_interval, fmt::format("{}::ContentAddressedGC", storage_path_full), + disk_name, makeGcRoundLogger()); + } + snapshot = gc_scheduler; + } + return snapshot->runOneRoundNow(Cas::GcRoundLogRecord::Trigger::Manual); +} + +Cas::RebuildReport ContentAddressedMetadataStorage::runGcRebuildNow(bool force) const +{ + checkNotReadOnly("GC rebuild"); + if (!gc_enabled) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Garbage collection is not enabled on this content-addressed disk"); + /// Admin class (rev.7 spec §1): refuse on a transient / IdentityLost / Vanished pool. (`store()` below + /// already throws on a terminal pool, but not on the merely-transient one -- this closes that gap and + /// keeps the refusal uniform with the other GC entry points.) Pre-lock: only a fast-fail. + checkOpAdmitted(CasOpClass::Admin); + /// Serialize the whole rebuild under `gc_scheduler_mutex`, exactly as a synchronous round and FORGET's + /// teardown are (both hold this same mutex). Held for the rebuild's DURATION so a concurrent FORGET waits + /// it out (fail-closed) instead of reporting the disk decommissioned while the rebuild's one-shot + /// `Cas::Gc` is still issuing durable `gc/`-plane writes. `Gc::rebuildBaseline` holds only the + /// Pool/backend (no back-reference to this storage), so it never re-takes this mutex -- no deadlock; and + /// `store()` takes only the brief `pointer_mutex` (lock order gc_scheduler -> pointer), so + /// gcHealth/store/partAccess never block behind this rebuild. + std::lock_guard round_lock(gc_scheduler_mutex); + /// Re-run the admission gate under the lock (mirroring the round verbs): a rebuild admitted while `Live` + /// but blocked here behind a FORGET refuses once the pool is `Vanished` (the later `store()` is also + /// fail-closed, so this is a fast-fail before minting the GC identity). + checkOpAdmitted(CasOpClass::Admin); + /// Test-only seam (inert in production): fires WHILE this rebuild holds `gc_scheduler_mutex` -- the + /// in-flight window a concurrent FORGET must serialize behind (I-2). Lets a test hold the lock here and + /// observe that FORGET blocks until the rebuild releases it. + if (gc_verb_admit_window_hook_for_test) + gc_verb_admit_window_hook_for_test(); + /// A one-shot Gc instance is fine here (unlike the scheduler's stable-instance requirement for + /// the lease's observation-window steal protocol): rebuildBaseline does its own lease + /// acquire/steal check internally and this command runs exactly one round. + const UInt128 gc_id = (static_cast(thread_local_rng()) << 64) | thread_local_rng(); + const auto cas_store_snapshot = store(); + Cas::Gc gc(cas_store_snapshot, gc_id, {}, {}, + getLogger(fmt::format("CasGc({})", cas_store_snapshot->poolConfig().server_root_id))); + return gc.rebuildBaseline(force); +} + +ContentAddressedMetadataStorage::PoolView ContentAddressedMetadataStorage::openPoolView() const +{ + /// Native mode rides real conditional ops (probed fail-closed by Pool::open); Local object + /// storage has none, so the backend emulates exact token semantics in-process (single server). + const auto mode = object_storage->getType() == ObjectStorageType::Local + ? Cas::ObjectStorageBackend::Mode::EmulatedSingleProcess + : Cas::ObjectStorageBackend::Mode::Native; + auto backend = std::make_shared(object_storage, mode, gcs_max_conditional_put_bytes); + + /// EmulatedSingleProcess emulates the conditional-op / exact-token semantics in-process (local + /// object storage has none). That emulation is per-process: two servers pointed at the SAME local + /// pool (e.g. an NFS/shared mount) each keep independent token state and would silently violate + /// the CAS invariants — the capability probe cannot detect this (each process passes it alone). + /// Make a shared-pool misconfiguration visible at INFO, not WARNING. + /// An inline `disk = disk(... object_storage_type=local ...)` opens the disk on the QUERY thread, so + /// a WARNING is forwarded to the client at the functional-test default `send_logs_level=warning` and + /// fails EVERY such query (clickhouse-test fails a test on ANY client stderr). At INFO the message + /// still lands in the server log for operator visibility but is not forwarded to client queries, so + /// the ~15 CA-over-local stateless tests stop failing on a benign single-server note. (A genuinely + /// shared local pool is a niche risk that would also surface via CAS/GC corruption; a future + /// `system.warnings` entry could restore a louder, test-safe signal.) + if (mode == Cas::ObjectStorageBackend::Mode::EmulatedSingleProcess) + LOG_INFO( + getLogger("ContentAddressedMetadataStorage"), + "Content-addressed disk over LOCAL object storage uses emulated in-process conditional " + "operations — safe ONLY for a single server. Do NOT share this pool path between multiple " + "ClickHouse servers (e.g. a shared/NFS mount): the CAS/GC invariants would break silently. " + "Use an S3-backed pool for multi-server / shared deployments."); + + /// Key spaces per mode: the Emulated (Local) backend maps bare pool keys under + /// getCommonKeyPrefix (the disk root dir), so the POOL prefix must be bucket-relative - strip + /// the common prefix when the configured prefix carries it (the local factory passes the root + /// path). Native passes keys through, so the configured prefix is used as-is (for S3 it + /// already embeds the endpoint sub-path). + String pool_prefix = storage_path_prefix; + /// The configured prefix is an endpoint sub-path and usually carries a TRAILING slash + /// ("content_addressed_s3/"); Cas::Layout joins components with '/', and a doubled slash in + /// keys is backend-hostile (RustFS rejects "p//_probe" LIST prefixes with InvalidArgument - + /// Some backends reject such prefixes while others merely tolerate them). + while (!pool_prefix.empty() && pool_prefix.back() == '/') + pool_prefix.pop_back(); + String physical_key_prefix_local; + if (mode == Cas::ObjectStorageBackend::Mode::EmulatedSingleProcess) + { + physical_key_prefix_local = object_storage->getCommonKeyPrefix(); + /// Slash-tolerant strip: the common prefix usually ends with '/', the pool prefix was + /// just trimmed of trailing slashes - compare canonical forms. + String common_trimmed = physical_key_prefix_local; + while (!common_trimmed.empty() && common_trimmed.back() == '/') + common_trimmed.pop_back(); + if (!common_trimmed.empty()) + { + if (pool_prefix == common_trimmed) + pool_prefix.clear(); + else if (pool_prefix.starts_with(common_trimmed + "/")) + pool_prefix = pool_prefix.substr(common_trimmed.size() + 1); + } + if (pool_prefix.empty()) + pool_prefix = "ca"; + } + + Cas::PoolConfig pool_config; + pool_config.pool_prefix = pool_prefix; + pool_config.server_id = serverIdToU128(server_id); + pool_config.server_root_id = server_root_id; + /// A read-only (``) disk opens with no background watermark and no write probe. + pool_config.background_watermark = (context != nullptr) && !read_only; + pool_config.read_only = read_only; + pool_config.skip_access_check = skip_access_check; + /// The node-local write algorithm: `PoolMeta::createOrValidate` accepts it with no write once + /// it is a member of the pool's `algos_used`; a not-yet-admitted algo is admitted via + /// `blob_hash_allow_new` or refused (BAD_ARGUMENTS, the default). + pool_config.blob_hash_algo = blob_hash_algo; + pool_config.blob_hash_allow_new = blob_hash_allow_new; + pool_config.deduplication_cache_bytes = deduplication_cache_bytes; + pool_config.deduplication_head_first_min_bytes = deduplication_head_first_min_bytes; + pool_config.manifest_decode_cache_bytes = manifest_decode_cache_bytes; + pool_config.gc_snapshot_generations_to_keep = gc_snapshot_generations_to_keep; + pool_config.gc_shards = gc_shards; + pool_config.manifest_sweep_list_budget_keys = manifest_sweep_list_budget_keys; + pool_config.manifest_sweep_delete_budget_keys = manifest_sweep_delete_budget_keys; + pool_config.gc_round_graduation_budget = gc_round_graduation_budget; + pool_config.gc_round_redelete_budget = gc_round_redelete_budget; + pool_config.gc_round_sweep_namespace_budget = gc_round_sweep_namespace_budget; + pool_config.gc_round_sweep_recovery_op_budget = gc_round_sweep_recovery_op_budget; + pool_config.gc_round_ref_cleanup_budget = gc_round_ref_cleanup_budget; + pool_config.gc_round_prefix_wholesale_budget = gc_round_prefix_wholesale_budget; + pool_config.gc_round_handoff_prefix_wholesale_budget = gc_round_handoff_prefix_wholesale_budget; + pool_config.gc_round_outcome_entry_budget = gc_round_outcome_entry_budget; + pool_config.gc_meta_pool_size = gc_meta_pool_size; + pool_config.event_sink = makeCasEventSink(); + + PoolView view; + view.physical_key_prefix = physical_key_prefix_local; + view.pool_prefix = pool_prefix; + view.pool = Cas::Pool::open(std::move(backend), std::move(pool_config)); + return view; +} + +void ContentAddressedMetadataStorage::startup() +{ + if (cas_store) + return; + + /// Observe-only mode (the disk's config): skip the probe (a probe write would fail on + /// a read-only backend), run no watermark, start no GC, and fail the mutating surface closed. + read_only = object_storage->isReadOnly(); + + /// Everything below builds into LOCALS -- nothing is published to `cas_store`/`part_access`/ + /// `gc_scheduler`/`pool_uuid`/`conditional_copy_supported` until the single publish step at the + /// very end. This makes a mid-startup throw leave the object exactly as unstarted as it was on + /// entry (the `if (cas_store) return;` head above still sees an empty pool), so a caller can + /// retry `startup` after a transient failure instead of being stuck with a half-built mount. + PoolView view = openPoolView(); + physical_key_prefix = view.physical_key_prefix; + auto pool = std::move(view.pool); + auto uuid = Cas::u128ToHex(pool->poolMeta().pool_id); + auto facade = std::make_shared(pool, + Cas::CachedPartFolderAccess::CacheParams{ + .cache_bytes = cas_part_folder_cache_bytes, + .max_entries = cas_part_folder_cache_max_entries, + .max_entry_bytes = cas_part_folder_cache_max_entry_bytes, + .validate = part_folder_validate}); + + /// The optional mount-time capability probe for a write-once conditional server-side copy. + /// Only relevant when this disk opted in to `staging_backend=s3`; `Local` (the default, + /// global constraint: OFF BY DEFAULT) takes NO probe here — `conditional_copy_supported` simply + /// stays at its `false` default and is never consulted on the local path. Skipped in + /// observe-only/readonly mode: a probe write would fail on a read-only backend, exactly like the + /// mandatory battery (`runCapabilityProbe`) above skips a read-only mount. + /// + /// Fail-close, never fail-open: an unsupported or non-enforcing backend just falls back to local + /// staging (`conditional_copy_supported` stays `false`) — this is NOT a mount failure, unlike the + /// mandatory battery, because `local` staging remains fully functional. + bool copy_supported = false; + if (staging_backend == Cas::StagingBackend::S3 && !read_only) + { + const String probe_prefix = physicalKey(view.pool_prefix + "/staging/" + server_root_id + "/probe"); + copy_supported = Cas::probeConditionalCopy(*object_storage, probe_prefix); + if (!copy_supported) + LOG_INFO( + getLogger("ContentAddressedMetadataStorage"), + "staging_backend=s3 requested but the object storage does not enforce conditional " + "copy; falling back to local staging"); + + /// Reclaim this mount's own leaked `staging//` debris (a promote whose staging-delete never + /// ran, or an aborted transaction's never-promoted staging object — see + /// `cleanupPendingTempFiles`) at mount start. Only runs when the S3 path is actually usable + /// (`conditional_copy_supported`) — an unsupported/fail-closed-to-local mount never wrote any + /// S3 staging objects under this prefix in the first place. LEASE-FENCE: the prefix below is + /// keyed by THIS mount's own `server_root_id` (the SAME prefix construction the probe above + /// and every staging key this mount ever mints use -- and the same formula `stagingKeyPrefix()` + /// uses post-startup; computed from `view.pool_prefix` here rather than via + /// `stagingKeyPrefix()` itself, because that helper calls `store()`, which is fail-closed and + /// would throw before the pool is published), so this sweep can never reach a different + /// mount's in-flight staging (`Cas::sweepOwnMountStaging`'s own doc comment). GC excludes + /// `staging/` entirely (a distinct top-level prefix from `blobs/` — see `CasLayout.h`), so this + /// sweeper is the ONLY reclaimer of `staging/` debris. + if (copy_supported) + Cas::sweepOwnMountStaging(*object_storage, physicalKey(view.pool_prefix + "/staging/" + server_root_id) + "/"); + } + + /// The background GC scheduler runs only on the disk-factory path (context non-null) and when + /// enabled - the lease makes concurrent schedulers across mounters safe (work dedup), so no + /// further gating is needed because the scheduler's lease coordinates concurrent mounters. + /// `CasGcScheduler` holds its own `PoolPtr` (see its `store` member), so starting it against the + /// LOCAL `pool` before publish is safe -- the scheduler keeps the pool alive on its own. It is + /// also safe on the unwind path below: `scheduler` here is a local `shared_ptr`, so if something + /// after this point throws (only the fault-injection hook can, in production nothing does), + /// its destructor runs during stack unwinding, which drops the last reference and destroys the + /// `CasGcScheduler`; its destructor calls `stop()`, which joins both worker threads before the + /// exception continues propagating. No explicit `SCOPE_EXIT` is needed for that. + std::shared_ptr scheduler; + if (context && gc_enabled && !read_only) + { + scheduler = std::make_shared( + pool, gc_interval, fmt::format("{}::ContentAddressedGC", storage_path_full), + disk_name, makeGcRoundLogger()); + scheduler->start(); + } + + /// Test-only: lets a test prove that a failure here (after everything above has succeeded, but + /// before publish) leaves nothing published and a retry can still succeed. A no-op in production. + if (startup_fault_injection_for_test) + startup_fault_injection_for_test(); + + /// The single publish step: as the LAST action of `startup`, atomically hand the fully-built + /// pool, part-folder facade, and GC scheduler to the members other threads observe through + /// `store`/`partAccess`/the GC entry points, in ONE `pointer_mutex` acquisition -- so no caller of + /// `poolAccess()` can ever observe a half-published mount. Everything above only ever touched locals, + /// so any throw before this point (including from the fault-injection hook above) leaves those + /// members (a null `cas_store`) exactly as they were on entry. + { + std::lock_guard lock(pointer_mutex); + cas_store = std::move(pool); + part_access = std::move(facade); + gc_scheduler = std::move(scheduler); + } + pool_uuid = std::move(uuid); + conditional_copy_supported = copy_supported; +} + +void ContentAddressedMetadataStorage::shutdown() +{ + /// Wait for any in-flight synchronous round to finish cleanly first (gc_scheduler_mutex is held + /// for a round's whole duration) -- unchanged priority: clean GC completion over fast shutdown. + std::lock_guard round_lock(gc_scheduler_mutex); + shutdown_called = true; + std::shared_ptr old_scheduler; + { + std::lock_guard ptr_lock(pointer_mutex); + old_scheduler = std::move(gc_scheduler); + gc_scheduler.reset(); + part_access.reset(); + /// Terminal server-shutdown semantics: a one-way trip (no server-lifecycle "remount" after + /// shutdown). Nulling `cas_store` puts the storage back into the null-pool (ShutDown) lifecycle, + /// so `poolAccess()`/the gate report the same operational refusal post-shutdown as pre-startup. + cas_store.reset(); + } + /// `stop` joins the background threads. Runs outside pointer_mutex (no reset left to race: + /// gc_scheduler is already null) but still inside round_lock, so a NEW round can't start here. + /// old_scheduler keeps the object alive regardless. + if (old_scheduler) + old_scheduler->stop(); +} + +namespace +{ +/// A human-readable UTC decommission stamp for the FORGET [D5] message (an operator asserted this, so the +/// message must be traceable to the audit log by wall time). Format: "YYYY-MM-DD HH:MM:SS UTC". +String utcStampNow() +{ + const std::time_t now = std::time(nullptr); + std::tm tm_utc{}; + gmtime_r(&now, &tm_utc); + char buf[32]; + const size_t n = std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S UTC", &tm_utc); + return String(buf, n); +} +} + +void ContentAddressedMetadataStorage::forgetDisk() +{ + /// SYSTEM CAS FORGET (spec §5): the operator force-Vanish. A lifecycle verb, NOT a + /// store()-class op — it must work on a NOT-live disk (a stuck transient / IdentityLost pool), so it + /// reaches the pool DIRECTLY, never through `poolAccess()`/`checkOpAdmitted` (which refuse a not-live + /// disk). Serialized against FSCK / GC STOP / GC START by `lifecycle_mutex`, and against a concurrent + /// synchronous GC round by `gc_scheduler_mutex` (a round holds the latter, so this waits it out). + std::lock_guard lifecycle(lifecycle_mutex); + std::lock_guard round_lock(gc_scheduler_mutex); + + Cas::PoolPtr pool; + std::shared_ptr scheduler; + { + std::lock_guard lock(pointer_mutex); + pool = cas_store; + /// Detach the scheduler from the member under `pointer_mutex` (as shutdown/unmount do): no new + /// synchronous round can adopt it, and `gcHealth` reports "no GC" for a forgotten disk immediately. + /// The actual stop()+join runs below, inside the pool's protocol + /// (OUTSIDE `pointer_mutex`, since it joins threads). + scheduler = std::move(gc_scheduler); + } + + if (!pool) + { + /// No published pool to forget (never started / shut down). The disk is already not serving; a + /// restart re-registers the name. Idempotent no-op. (The detached `scheduler` is null here too — + /// `cas_store`/`gc_scheduler` are published and cleared together.) + LOG_WARNING(getLogger("ContentAddressedMetadataStorage"), + "SYSTEM CAS FORGET on content-addressed disk '{}': no published pool — nothing " + "to decommission (a restart re-registers the name).", disk_name); + return; + } + + /// The [D5] forgotten message, carrying the actual decommission timestamp (an operator ASSERTION, not + /// an erasure proof — the wording says so). `Pool::throwIfLifecycleTerminal` surfaces it verbatim to + /// every store-class caller after the transition, and the WARN at the transition logs it too. + const String reason = fmt::format( + "decommissioned by SYSTEM CAS FORGET at {} — erasure was NOT verified; if this was a " + "mistake the data may be intact (restart re-registers the name)", utcStampNow()); + + /// Run the fence-first protocol on the pool. The GC-stop callback stops+joins the (detached) scheduler + /// at spec §5 step 3/4; the scheduler is destroyed when the local `scheduler` leaves this scope. + pool->forgetDisk([&scheduler] { if (scheduler) scheduler->stop(); }, reason); +} + +void ContentAddressedMetadataStorage::gcStop() +{ + /// SYSTEM CAS GC STOP (spec §6): stop ONLY the background GC scheduler. STOP-IN-PLACE -- + /// the scheduler object is RETAINED in the member (contrast `forgetDisk`/`shutdown`, which `std::move` + /// it out and destroy it): a later `gcStart` must re-enter the SAME instance so its `gc_id` + lease + /// observation history survive. Keeping it in the member also keeps `gcHealth` reading the (stopped) + /// state truthfully, rather than "no GC". + /// A lifecycle-control verb: serialized against FSCK / forget / GC START by `lifecycle_mutex`, and + /// against a concurrent synchronous GC round by `gc_scheduler_mutex` (a round holds the latter, so this + /// waits it out). It does NOT consult `checkOpAdmitted` -- stopping GC works on ANY disk state, including + /// a not-live/Vanished one (stopping the reclaimer on a sick disk is a legitimate operator action). + std::lock_guard lifecycle(lifecycle_mutex); + std::lock_guard round_lock(gc_scheduler_mutex); + + /// Snapshot the scheduler under `pointer_mutex` by COPY (never `std::move`): leave it in the member. + std::shared_ptr snapshot; + { + std::lock_guard lock(pointer_mutex); + snapshot = gc_scheduler; + } + if (!snapshot) + { + /// No scheduler at all (GC disabled / read-only / not started / already forgotten). Stopping GC on a + /// disk that runs none is a no-op success -- the operator's intent ("no GC background activity") + /// already holds. + LOG_INFO(getLogger("ContentAddressedMetadataStorage"), + "SYSTEM CAS GC STOP on content-addressed disk '{}': no GC scheduler " + "(disabled/read-only/not started) -- nothing to stop.", disk_name); + return; + } + /// `stop()` joins the worker + heartbeat threads and clears the in-process leadership hint. Runs OUTSIDE + /// `pointer_mutex` (it joins threads). Idempotent: a second STOP finds an already-stopped scheduler and + /// `stop()` is a safe no-op. + snapshot->stop(); +} + +void ContentAddressedMetadataStorage::gcStart() +{ + /// SYSTEM CAS GC START (spec §6): restart the background GC scheduler stopped by `gcStop`. + /// Serialized like `gcStop`. Unlike it, START refuses on a decommissioned/uncertain pool: restarting GC + /// there would only spin failing rounds, so it goes through the uniform GC gate. + std::lock_guard lifecycle(lifecycle_mutex); + std::lock_guard round_lock(gc_scheduler_mutex); + + checkNotReadOnly("GC start"); + if (!gc_enabled) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Garbage collection is not enabled on this content-addressed disk"); + /// Admin class (rev.7 spec §1): refuse on a transient / `IdentityLost` / `Vanished` pool (typed 668 / + /// [D5]) and on a null pool (`throwStorageNotStarted`). Only a `Live` pool proceeds -- the same uniform + /// gate every GC entry point uses (`runGarbageCollectionRoundNow` / `runGcRebuildNow`). + checkOpAdmitted(CasOpClass::Admin); + if (shutdown_called) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot start garbage collection after ContentAddressedMetadataStorage shutdown has begun"); + + std::shared_ptr snapshot; + { + std::lock_guard ptr_lock(pointer_mutex); + if (!gc_scheduler) + { + /// `Live` + `gc_enabled` + not read-only but no scheduler: reachable only when the disk was + /// started in a context that started none (e.g. a unit-test null context) or after a GC RUN that + /// created a lazy one was never started. Create a STABLE instance now, mirroring the GC RUN entry + /// points, so START is meaningful. (`checkOpAdmitted` above already proved `cas_store` is live.) + if (!cas_store) + throwStorageNotStarted(); + gc_scheduler = std::make_shared( + cas_store, gc_interval, fmt::format("{}::ContentAddressedGC", storage_path_full), + disk_name, makeGcRoundLogger()); + } + snapshot = gc_scheduler; + } + /// `start()` is a no-op if already running (idempotent) and re-enters the SAME instance after a stop -- + /// the persistent `gc` observer + `gc_id` are preserved, and leadership is re-acquired only by the next + /// round's normal `gc/state` acquisition, never restored here. Runs outside `pointer_mutex` for symmetry + /// with `stop()` (it spawns threads but joins nothing, so it does not block). + snapshot->start(); +} + +Cas::FsckReport ContentAddressedMetadataStorage::runFsckNow(bool detail) const +{ + /// Outermost lock (see its own doc comment): held for the WHOLE scan, so a concurrent lifecycle-control + /// verb (FORGET / GC STOP / GC START) cannot race the disk out from under an in-flight FSCK. + std::lock_guard lifecycle(lifecycle_mutex); + + /// FSCK scans the LIVE running pool directly (rev.8). Admin class -- refuse on a transient / + /// IdentityLost / Vanished / null pool before touching it, exactly like the GC entry points + /// (`runGarbageCollectionRoundNow`/`runGcRebuildNow`). The scan is read-only and its findings are + /// revalidated against a fresh authoritative read (`CasFsck`'s Dangling / missing-manifest rechecks), + /// so concurrent writers never yield a phantom finding. + checkOpAdmitted(CasOpClass::Admin); + return Cas::runFsck(*store(), detail); +} + +ContentAddressedMetadataStorage::PoolAccessSnapshot ContentAddressedMetadataStorage::poolAccess() const +{ + PoolAccessSnapshot snap; + { + std::lock_guard lock(pointer_mutex); + snap.pool = cas_store; + snap.part_access = part_access; + } + /// A null pool covers before-first-startup and after-shutdown uniformly -- the storage-level + /// Constructing/ShutDown lifecycle. Fail loud (spec §1's null-pool fail-loud); there is no benign + /// answer for a storage that has not published a pool. + if (!snap.pool) + throwStorageNotStarted(); + /// rev.7 §1: a published pool that has entered a terminal lifecycle condition (`IdentityLost` or any + /// `Vanished`) must ALSO refuse store()-class access, so nothing silently proceeds against an erased + /// or replaced data root. This is the store()-class terminal check; the full six-class operation gate + /// (which also gates the transient state and answers truth-absent on removes/enumeration) is + /// `checkOpAdmitted`. + snap.pool->throwIfLifecycleTerminal(); + return snap; +} + +void ContentAddressedMetadataStorage::throwStorageNotStarted() const +{ + /// No pool is published: the storage-level lifecycle is Constructing (before `startup`) or ShutDown + /// (after `shutdown`). `pool_uuid` is empty ONLY before the first successful startup (written once at + /// its end, never reset by `shutdown`), so it distinguishes the two, exactly as `lifecycleSnapshot()` + /// reports `constructing`/`shutdown`. Immutable-after-startup, so read without `pointer_mutex`. + const char * phase = pool_uuid.empty() ? "constructing" : "shutdown"; + throw Exception(ErrorCodes::INVALID_STATE, + "content-addressed disk '{}' is not started (storage lifecycle: {})", disk_name, phase); +} + +Cas::PoolPtr ContentAddressedMetadataStorage::store() const +{ + return poolAccess().pool; +} + +std::shared_ptr ContentAddressedMetadataStorage::partAccess() const +{ + return poolAccess().part_access; +} + +void ContentAddressedMetadataStorage::checkNotReadOnly(std::string_view what) const +{ + if (read_only) + throw Exception(ErrorCodes::READONLY, + "Content-addressed disk is opened read-only: {} is rejected", what); +} + +CasOpAdmission ContentAddressedMetadataStorage::checkOpAdmitted(CasOpClass op) const +{ + /// `Factory` is never routed here (its call sites are I/O-free and work in every state); a Factory + /// arg is a call-site bug. This unreachable path is genuinely unreachable, so the LOGICAL_ERROR never + /// constructs (never aborts a debug/ASan build). + if (op == CasOpClass::Factory) + throw Exception(ErrorCodes::LOGICAL_ERROR, "checkOpAdmitted must not be called for the Factory class"); + + Cas::PoolPtr pool; + { + std::lock_guard lock(pointer_mutex); + pool = cas_store; + } + + /// A null pool = the storage-level lifecycle is Constructing (before `startup`) or ShutDown (after + /// `shutdown`): fail loud for EVERY class, `Probe` included. There is no benign "absent" answer for a + /// storage that has never published a pool (or torn one down) -- only a genuinely `Vanished` POOL + /// (below) answers truth-absent. This is the spec §1 null-pool fail-loud contract. + if (!pool) + throwStorageNotStarted(); + + /// The rev.7 six-class gate keyed on the pool lifecycle condition (spec §1). + const Cas::PoolLifecycle lc = pool->lifecycle(); + if (lc == Cas::PoolLifecycle::Live) + return CasOpAdmission::Proceed; + + if (lc == Cas::PoolLifecycle::TransientNotLive) + /// A lease blip: uncertain but AUTO-RECOVERING. No class but Factory proceeds, and the refusal is + /// minted TRANSIENT -- this is the READ plane, where a consumer that cannot tell unavailability + /// from damage acts destructively: `ReplicatedMergeTreePartCheckThread` detaches a part whose read + /// throws an error its retryable-classifier does not list. `IdentityLost` gets its own richer, + /// TERMINAL 668 below -- it does not auto-recover, so both "temporarily unreachable" and the + /// retryable class would misdiagnose it. The wait-and-retry guidance is actionable for every caller + /// here, and specifically for `SYSTEM CAS GC START` run mid-recovery by an operator + /// who STOPped GC pre-maintenance: this is a wait, not a dead end. + Cas::throwCasTransientUnavailable( + fmt::format("content-addressed disk '{}'", disk_name), + "mount lease not held; backing may be temporarily unreachable; the operation is admitted " + "again once the disk recovers to Live"); + + /// `IdentityLost` and the terminal `Vanished*` states carry the typed per-reason [D5] message the pool + /// owns (single source). A SETTLED `Vanished` pool answers `Probe`/`Remove` truthfully without touching + /// it; `IdentityLost` (sentinels absent, no auto-recovery) has NO benign answer -- every class + /// fails loud with the "recover by restart or FORGET; a matching-sentinel restore does not auto-revive" + /// diagnosis. So the truth-absent short-circuit is gated on the Vanished states specifically. + const bool settled_vanished = lc == Cas::PoolLifecycle::VanishedReplaced + || lc == Cas::PoolLifecycle::VanishedForgotten; + if (settled_vanished && (op == CasOpClass::Probe || op == CasOpClass::Remove)) + return CasOpAdmission::TruthAbsent; + pool->throwIfLifecycleTerminal(); + throw Exception(ErrorCodes::LOGICAL_ERROR, + "checkOpAdmitted: unreachable -- non-Live pool did not throw for content-addressed disk '{}'", disk_name); +} + +void ContentAddressedMetadataStorage::confirmPoolIdentityForEmptyEnumeration(const std::string & path) const +{ + /// EMPTY-PROOF RULE (rev.7 spec §1 [B3]). Reached ONLY when `listDirectory` computed an EMPTY listing + /// at a `TableDir`/`DetachedContainer` root on a NON-terminal pool -- `checkOpAdmitted` already ran + /// (admitted as `Live`, and NOT a settled `Vanished` state, which would have short-circuited `Probe` to + /// `TruthAbsent` before any classification). This is the last silent-empty-load killer: an empty table + /// root is exactly what a silently-erased backing looks like, and a read-only pool (no lease, no + /// erasure observer) has no other line of defense. So the empty answer is authorized ONLY by an + /// AUTHORITATIVE, UNCACHED positive on the pool identity object -- a cached positive never suffices. + const Cas::PoolPtr pool = store(); /// Live here (past the op gate's Live, non-terminal admission). + + ++empty_proof_probe_count_for_test; + const Cas::SentinelProbeResult probe = empty_proof_probe_override_for_test + ? empty_proof_probe_override_for_test() + : Cas::probeSentinel(pool->backend(), pool->layout().poolMetaKey()); + + switch (probe.outcome) + { + case Cas::ProbeOutcome::Present: + /// The pool identity is authoritatively present -- the empty listing is the truth. + return; + case Cas::ProbeOutcome::KeyAbsent: + case Cas::ProbeOutcome::ContainerAbsent: + /// A clean authoritative miss on `_pool_meta`: the backing is (or is being) erased. Refuse + /// the empty answer rather than silently attaching an empty table over an erased pool. + throw Exception(ErrorCodes::INVALID_STATE, + "content-addressed disk '{}' -- pool identity object absent while enumerating '{}' -- " + "refusing the empty answer; the backing may be erased", + disk_name, path); + case Cas::ProbeOutcome::AccessDenied: + case Cas::ProbeOutcome::Indeterminate: + /// Absence was NEVER established (a transport/permission fault). Fail closed and TRANSIENT -- + /// never promote an unproven probe into an empty answer, and never let a consumer read an + /// unreachable pool identity as damage. The arm above, where absence IS proven, keeps its + /// terminal 668: an erased backing does not heal by retrying. This arm promises no particular + /// recovery either: `AccessDenied` is a credential/policy fault that a return to `Live` does + /// not clear, so "retry" is the only honest guidance for the pair. + Cas::throwCasTransientUnavailable( + fmt::format("content-addressed disk '{}'", disk_name), + fmt::format("pool identity object could not be confirmed while enumerating '{}' " + "(transport or permission fault) -- refusing the empty answer; retry", path)); + } +} + +MetadataTransactionPtr ContentAddressedMetadataStorage::createTransaction() +{ + checkNotReadOnly("writes"); + return std::make_shared(*this); +} + +String ContentAddressedMetadataStorage::stagingKeyPrefix() const +{ + /// Mirrors the probe's own prefix construction (`startup`'s `probe_prefix` above), minus + /// the probe's own `/probe` leaf — this is the writer-owned sibling subtree of the SAME + /// `staging//` area. `store()` throws INVALID_STATE when no pool is published + /// (pre-`startup`/post-`shutdown`); every caller (writeFile, via a transaction) runs post-startup. + return physicalKey(store()->poolConfig().pool_prefix + "/staging/" + server_root_id); +} + +/// ==== namespace mapping ==== + +std::string ContentAddressedMetadataStorage::serverPrefix() const +{ + /// Live namespaces and mirrored live-tree files are rooted by the configured + /// `server_root_id`, not by the ClickHouse ServerUUID-derived token. `ServerUUID` is only the + /// mount owner token; `server_root_id` is the persistent layout identity. + return server_root_id; +} + +std::vector ContentAddressedMetadataStorage::listLiveTreeChildren(const std::string & path) const +{ + /// Probe: a Vanished disk enumerates empty (truth). Its callers (`listDirectory`) already gate, but + /// this public helper is gated too so a direct call is truthful. + if (checkOpAdmitted(CasOpClass::Probe) == CasOpAdmission::TruthAbsent) + return {}; + const std::string canonical = canonicalDiskPath(path); + const std::string scope = serverPrefix() + "/" + (canonical.empty() ? "" : canonical + "/"); + std::unordered_set result; + for (const auto & child : store()->listMirroredChildren(scope)) + result.emplace(stripCasArchiveSuffix(child)); + return toVector(std::move(result)); +} + +bool ContentAddressedMetadataStorage::liveTreeDirHasChildren(const std::string & path) const +{ + /// Probe gate FIRST, ahead of the hardcoded disk-root short-circuit below (the rev.7 offender): on a + /// Vanished disk even the disk root reads absent (truth), never the unconditional `true`. + if (checkOpAdmitted(CasOpClass::Probe) == CasOpAdmission::TruthAbsent) + return false; + const std::string canonical = canonicalDiskPath(path); + /// The disk root always exists; otherwise a non-empty server-root-scoped mirrored LIST is the signal. + if (canonical.empty()) + return true; + const std::string scope = serverPrefix() + "/" + canonical + "/"; + return !store()->listMirroredChildren(scope).empty(); +} + +Cas::RootNamespace ContentAddressedMetadataStorage::liveNamespace(const std::string & table_uuid) const +{ + /// Path mirroring: the namespace is the table's canonical disk path with the + /// content-addressed boundary marked by `@cas@` on the table-dir segment, prefixed by the + /// configured `server_root_id`. e.g. `/store/3f2/3f2a…@cas@`. + return Cas::RootNamespace{serverPrefix() + "/" + Cas::mirroredArchiveNamespace(table_uuid)}; +} + +std::optional +ContentAddressedMetadataStorage::readableNamespaceFilesLife(const Cas::RootNamespace & ns) const +{ + return store()->namespaceFilesLifeIfReadable(ns); +} + +Cas::RootNamespace ContentAddressedMetadataStorage::shadowNamespace(const std::string & shadow_table_dir) +{ + /// The LITERAL shadow table dir (shadow//store// or .../data//): + /// bijective with the disk path for both layouts, pool-global (backups are read by any + /// replica), and the shadow tree enumerates from Pool::listNamespaces("shadow/"). + /// Canonicalize because the unfreezer can hand the directory a trailing slash. + return Cas::RootNamespace{canonicalDiskPath(shadow_table_dir)}; +} + + +std::optional +ContentAddressedMetadataStorage::route(const Cas::PartFilePath & p) const +{ + Route r; + if (!p.backup_name.empty()) + { + r.ns = shadowNamespace(p.shadow_table_dir); + r.ref = p.part_name; + r.file = p.file; + return r; + } + if (p.part_name == Cas::kDetachedDirName) + { + /// The parser reports detached paths with part_name == "detached" and the real detached + /// part dir as the first component of `file`. Detached parts share the table namespace and + /// INTO the table's OWN archive namespace: each detached part is a ref keyed by + /// `detached/` (vs a live ``), so the re-split here keeps the table namespace + /// and prepends the `detached/` ref prefix. An empty `p.file` (the bare `
/detached` + /// container dir) yields an empty ref → the filtered-container listing path. + r.ns = liveNamespace(p.table_uuid); + auto [part, file] = splitFirstComponent(p.file); + r.ref = part.empty() ? "" : std::string(Cas::kDetachedRefPrefix) + part; + r.file = file; + return r; + } + if (p.part_name == Cas::kMovingDirName) + { + /// L1 (MOVE-to-CA fix): re-split exactly like detached, folding onto a `moving/`-PREFIXED + /// ref (kMovingRefPrefix) -- NOT the part's final ref directly. Publishing the clone under + /// the final ref before the mover's swap would break move crash-atomicity: a crash between + /// the clone publication and swapClonedPart would leave a committed LIVE ref that never went + /// through the swap, and moving/'s own startup cleanup couldn't distinguish that premature + /// ref from a real live part. The staging ref keeps the pre-swap clone un-live; the mover's + /// rename does a real ref repoint moving/ -> (the same committed-ref-repoint + /// path merge-result/delete_tmp renames already use). An empty p.file (the bare + ///
/moving container dir) yields an empty ref, same convention as detached. + r.ns = liveNamespace(p.table_uuid); + auto [part, file] = splitFirstComponent(p.file); + r.ref = part.empty() ? "" : std::string(Cas::kMovingRefPrefix) + part; + r.file = file; + return r; + } + r.ns = liveNamespace(p.table_uuid); + r.ref = p.part_name; + r.file = p.file; + return r; +} + +std::vector ContentAddressedMetadataStorage::detachedRefNames(const Cas::RootNamespace & ns) const +{ + std::vector refs; + for (const auto & [ref, _] : store()->listRefs(ns)) + if (ref.starts_with(Cas::kDetachedRefPrefix)) + refs.push_back(ref); + return refs; +} + +std::vector ContentAddressedMetadataStorage::movingRefNames(const Cas::RootNamespace & ns) const +{ + std::vector refs; + for (const auto & [ref, _] : store()->listRefs(ns)) + if (ref.starts_with(Cas::kMovingRefPrefix)) + refs.push_back(ref); + return refs; +} + +/// ==== read surface ==== + +bool ContentAddressedMetadataStorage::existsFile(const std::string & path) const +{ + /// Probe gate (rev.7 §1): real while live, throws while uncertain (incl. a null/unstarted pool), + /// truthfully absent once Vanished. + if (checkOpAdmitted(CasOpClass::Probe) == CasOpAdmission::TruthAbsent) + return false; + if (!Cas::isPartFilePath(path)) + { + if (auto tf = Cas::parseTableFilePath(path)) + { + const auto life = readableNamespaceFilesLife(liveNamespace(tf->table_uuid)); + return life && store()->getNamespaceFile(*life, tf->tail).has_value(); + } + /// A loose mountpoint object is a plain object at roots//. + /// Use a HEAD-based existence check (directory-safe), NOT a body read: the traversal in + /// system.remote_data_paths probes existsFile on directory-shaped pool paths (e.g. `store`), and a + /// body read (getMountpointObject) throws "Is a directory". A directory is not a file. + return store()->mountpointObjectExists(serverPrefix() + "/" + path); + } + + auto p = Cas::parsePartFilePath(path); + if (!p || p->file.empty()) + return false; + auto r = route(*p); + if (!r || r->file.empty()) + return false; + + /// Per-part files flow through the ordinary content path like any other file; no ForceFresh + /// special case is needed. + /// Safe to serve a CACHED view here: every committed-ref write that could have moved this entry + /// (`repointRef`/`promoteBuild`) erases the cached view on success, so a stale hit is impossible + /// by construction, not by freshness policy. + auto view = partAccess()->getView(r->refKey(), Cas::Freshness::CachedForLoad); + return view && view->findFile(r->file); +} + +ContentAddressedMetadataStorage::DirRoute ContentAddressedMetadataStorage::classifyDirectory(const std::string & path) const +{ + DirRoute dr; + + /// FREEZE shadow namespace — routed BEFORE the live branches (a shadow table dir also + /// satisfies parseTableUuid). + if (Cas::isShadowPath(path)) + { + if (auto p = Cas::parsePartFilePath(path); p && !p->backup_name.empty() && p->file.empty()) + { + dr.shape = DirShape::ShadowPart; + dr.p = std::move(p); + return dr; + } + if (Cas::endsWithTableUuidPair(path)) + { + dr.shape = DirShape::ShadowTable; + return dr; + } + dr.shape = DirShape::ShadowIntermediate; + return dr; + } + + /// The Atomic `store/` shard dir (see listDirectory): route to the generic existence signal + /// before parseTableUuid/parseTableFilePath misclaim it as a non-Atomic table. + if (Cas::isAtomicShardDir(path)) + { + dr.shape = DirShape::AtomicShard; + return dr; + } + + if (auto uuid = Cas::parseTableUuid(path)) + { + dr.shape = DirShape::TableDir; + dr.uuid = std::move(uuid); + return dr; + } + + if (auto p = Cas::parsePartFilePath(path)) + { + auto r = route(*p); + /// The detached CONTAINER dir
/detached. + if (r && r->ref.empty() && p->part_name == Cas::kDetachedDirName) + { + dr.shape = DirShape::DetachedContainer; + dr.p = std::move(p); + dr.r = std::move(r); + return dr; + } + /// The moving CONTAINER dir
/moving (MOVE-to-CA fix): the mover's crash-cleanup + /// (MergeTreeData.cpp, MOVING_DIR_NAME) existsDirectory/removeRecursive's this bare path + /// at every table load to reclaim a staging ref left behind by an interrupted move. + if (r && r->ref.empty() && p->part_name == Cas::kMovingDirName) + { + dr.shape = DirShape::MovingContainer; + dr.p = std::move(p); + dr.r = std::move(r); + return dr; + } + /// A part dir (live, detached, or shadow). + if (r && !r->ref.empty() && r->file.empty()) + { + dr.shape = DirShape::PartDir; + dr.p = std::move(p); + dr.r = std::move(r); + return dr; + } + /// A projection dir. + if (r && !r->ref.empty()) + { + if (auto prefix = Cas::PartFolderView::projectionDirPrefix(r->file)) + { + dr.shape = DirShape::ProjectionDir; + dr.p = std::move(p); + dr.r = std::move(r); + dr.projection_prefix = std::move(prefix); + return dr; + } + } + /// No sub-shape matched: fall through, identical to today's post-`if (p)` continuation. + } + + /// A table-level SUBDIRECTORY (deduplication_logs/...). + if (auto tf = Cas::parseTableFilePath(path)) + { + dr.shape = DirShape::TableSubdir; + dr.tf = std::move(tf); + return dr; + } + + /// A generic INTERMEDIATE live-tree directory (disk root, `store`, ...). + dr.shape = DirShape::GenericIntermediate; + return dr; +} + +bool ContentAddressedMetadataStorage::existsDirectory(const std::string & path) const +{ + if (checkOpAdmitted(CasOpClass::Probe) == CasOpAdmission::TruthAbsent) + return false; + const DirRoute dr = classifyDirectory(path); + switch (dr.shape) + { + case DirShape::ShadowPart: + return partAccess()->existsRef(Route{shadowNamespace(dr.p->shadow_table_dir), dr.p->part_name, ""}.refKey(), + Cas::Freshness::CachedForLoad); + case DirShape::ShadowTable: + return store()->hasAnyRefWithPrefix(shadowNamespace(path), ""); + case DirShape::ShadowIntermediate: + { + /// Intermediate dir (shadow/, shadow//store, ...): exists iff SOME shadow namespace + /// under this path still has a LIVE ref. A raw object LIST of the mirrored subtree would count + /// tombstoned-but-not-yet-GC'd shard/manifest objects — CA removal is tombstone + deferred GC + /// (`removeRecursive`/`dropNamespace` only tombstone; `Cas::Gc` physically deletes later) — so a + /// just-`UNFREEZE`d backup dir would spuriously "exist" until a GC round runs. Instead + /// enumerate the namespaces exactly as `removeRecursive` does (`listNamespaces(scope)`) and + /// consult the tombstone-aware `listRefs` (as the `endsWithTableUuidPair` case above does), so + /// existence is consistent with the ref-level signal and independent of GC timing. + const std::string canonical = canonicalDiskPath(path); + const std::string scope = canonical.empty() ? "shadow/" : canonical + "/"; + const Cas::NamespaceListing listing = store()->listNamespaces(scope); + for (const auto & ns : listing.namespaces) + if (store()->hasAnyRefWithPrefix(Cas::RootNamespace{ns}, "")) + return true; + /// A key this scope's enumeration could not attribute leaves emptiness UNPROVEN, and the + /// fail-close answer for an existence probe is "present": reporting absent is what would let + /// a caller treat the subtree as gone. Answering present is bounded -- an already-unfrozen + /// directory keeps showing up until the key is cleared -- and it never claims absence that + /// was not established. + /// + /// The answer alone would leave an operator with a directory that will not go away and + /// nothing naming the key that holds it, so the key and the refusal are LOGGED here too -- + /// the other three consumers of this enumeration each surface the skip to a human, and a + /// boolean is not that. Rate-limited because this is an existence probe on a browse path: + /// `LogSeriesLimiter` keys on the LOGGER NAME, so one message per window prints regardless of + /// which key it was about. + if (!listing.skipped.empty()) + { + LogSeriesLimiter log(getLogger("CasShadowScopeLifelessKey"), /*allowed_count=*/1, /*interval_s=*/60); + LOG_WARNING(log, + "existsDirectory('{}'): {} key(s) under this scope name no namespace life, so it " + "cannot be proven empty and is reported as PRESENT. First such key: '{}' ({}). Run " + "`cas-fsck` to enumerate them all.", + path, listing.skipped.size(), listing.skipped.front().key, listing.skipped.front().reason); + return true; + } + return false; + } + case DirShape::AtomicShard: + return liveTreeDirHasChildren(path); + case DirShape::TableDir: + /// A table directory exists iff its logical namespace still has foreground removal work + /// outstanding, or has never proven completion: present while `Creating`, for every `Live` + /// row (even zero parts and zero namespace files -- an empty live table is still a table), + /// and while `Removing` before its terminal `remove_namespace` transaction is durable; + /// absent only once no catalog row exists at all, or the terminal is durably proven. This is + /// deliberately NOT "has a committed ref": a table that removed its last part, or that never + /// wrote one, must stay present until an actual namespace-drop admits and completes removal + /// -- otherwise `DROP TABLE` on such a table would silently skip physical cleanup and leak + /// its catalog row forever. + return store()->namespaceStillLogicallyPresent(liveNamespace(*dr.uuid)); + case DirShape::DetachedContainer: + /// Exists iff it has at least one reference. + return store()->hasAnyRefWithPrefix(dr.r->ns, Cas::kDetachedRefPrefix); + case DirShape::MovingContainer: + /// Exists iff it has at least one staging ref (MOVE-to-CA fix, mirrors DetachedContainer). + return store()->hasAnyRefWithPrefix(dr.r->ns, Cas::kMovingRefPrefix); + case DirShape::PartDir: + /// Exists iff its ref is present. + return partAccess()->existsRef(dr.r->refKey(), Cas::Freshness::CachedForLoad); + case DirShape::ProjectionDir: + { + /// At least one tree entry (or mutable file) under its prefix. + auto view = partAccess()->getView(dr.r->refKey(), Cas::Freshness::CachedForLoad); + return view && view->hasDirectory(*dr.projection_prefix); + } + case DirShape::TableSubdir: + { + /// At least one verbatim file under it. + const auto life = readableNamespaceFilesLife(liveNamespace(dr.tf->table_uuid)); + if (!life) + return false; + const std::string prefix = dr.tf->tail + "/"; + for (const auto & name : store()->listNamespaceFiles(*life)) + if (name.starts_with(prefix)) + return true; + return false; + } + case DirShape::GenericIntermediate: + /// Exists iff a server-root-scoped mirrored LIST finds any object. Keeps `cd`/existence + /// consistent with listDirectory so `clickhouse-disks` traversal behaves like a normal disk. + return liveTreeDirHasChildren(path); + } + return liveTreeDirHasChildren(path); /// unreachable +} + +bool ContentAddressedMetadataStorage::existsFileOrDirectory(const std::string & path) const +{ + if (checkOpAdmitted(CasOpClass::Probe) == CasOpAdmission::TruthAbsent) + return false; + if (Cas::isPartFilePath(path)) + { + auto p = Cas::parsePartFilePath(path); + auto r = p ? route(*p) : std::nullopt; + if (r && !r->ref.empty() && !r->file.empty()) + { + auto view = partAccess()->getView(r->refKey(), Cas::Freshness::CachedForLoad); + if (!view) + return false; + return view->hasFile(r->file) || view->hasDirectory(r->file + "/"); + } + } + return existsFile(path) || existsDirectory(path); +} + +uint64_t ContentAddressedMetadataStorage::getFileSize(const std::string & path) const +{ + /// ContentRead: a size query resolves a specific file; on a Vanished disk it fails loud with the typed + /// error rather than silent-absent (never let a reader mistake erased backing for "file not there"). + checkOpAdmitted(CasOpClass::ContentRead); + if (!Cas::isPartFilePath(path)) + { + if (auto bytes = tryGetInManifestBytes(path)) /// verbatim table-level file + return bytes->size(); + if (auto bytes = store()->getMountpointObject(serverPrefix() + "/" + path)) + return bytes->size(); + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: no object for {}", path); + } + + auto p = Cas::parsePartFilePath(path); + if (!p || p->file.empty()) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: not a part file path: {}", path); + auto r = route(*p); + if (!r || r->file.empty()) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: not a part file path: {}", path); + + auto view = partAccess()->getView(r->refKey(), Cas::Freshness::CachedForLoad); + if (!view) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: no ref for {}", path); + if (auto size = view->fileSize(r->file)) + return *size; + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: file {} not in manifest of {}", r->file, path); +} + +Poco::Timestamp ContentAddressedMetadataStorage::getLastModified(const std::string & path) const +{ + /// ContentRead: resolves a specific part's stamp; loud typed error on a Vanished disk. + checkOpAdmitted(CasOpClass::ContentRead); + /// Timestamps are DERIVED for content addressing: the part's publish wall-clock, stamped by + /// the transaction into the typed `RefPayload.published_at_ms` field (epoch milliseconds). + /// Every shape (part dir, detached part dir, projection dir, part file) reports its part's + /// stamp; a part published without a stamp (published_at_ms == 0) reports the epoch (harmless: + /// stamps only feed cleanup TTLs and system tables). + auto resolve_stamp = [&](const Route & r) -> Poco::Timestamp + { + auto resolved = partAccess()->resolve(r.refKey(), Cas::Freshness::CachedForLoad); + if (!resolved) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: no ref for {}", path); + if (resolved->published_at_ms == 0) + return Poco::Timestamp(0); + /// published_at_ms is epoch milliseconds; Poco::Timestamp::fromEpochTime takes seconds. + return Poco::Timestamp::fromEpochTime(static_cast(resolved->published_at_ms / 1000)); + }; + + if (auto p = Cas::parsePartFilePath(path)) + { + auto r = route(*p); + if (r && !r->ref.empty()) + return resolve_stamp(*r); + } + /// Table-level / generic verbatim files: no per-object mtime is kept — epoch. + if (existsFile(path)) + return Poco::Timestamp(0); + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: no object for {}", path); +} + +std::vector ContentAddressedMetadataStorage::listDirectory(const std::string & path) const +{ + if (checkOpAdmitted(CasOpClass::Probe) == CasOpAdmission::TruthAbsent) + return {}; + const DirRoute dr = classifyDirectory(path); + switch (dr.shape) + { + case DirShape::ShadowPart: + { + /// Shadow PART dir: the frozen part's file names (first components). + auto view = partAccess()->getView(Route{shadowNamespace(dr.p->shadow_table_dir), dr.p->part_name, ""}.refKey(), + Cas::Freshness::CachedForLoad); + return view ? view->listChildren("") : std::vector{}; + } + case DirShape::ShadowTable: + { + /// Shadow TABLE dir: the frozen part names. + std::vector result; + for (const auto & [ref, _] : store()->listRefs(shadowNamespace(path))) + result.push_back(ref); + return result; + } + case DirShape::ShadowIntermediate: + { + /// Enumerate children via a scoped LIST of the mirrored subtree. A + /// mirrored LIST naturally surfaces intermediate path segments AND `@cas@`-suffixed + /// table dirs; strip the trailing `@cas@` for the logical view. Loose LIST is fine: the + /// existing listRefs re-check filters out dropped-but-registered archives so they don't + /// appear as false children. + const std::string canonical = canonicalDiskPath(path); + const std::string scope = canonical.empty() ? "shadow/" : canonical + "/"; + std::unordered_set result; + for (const auto & child : store()->listMirroredChildren(scope)) + result.emplace(stripCasArchiveSuffix(child)); + return toVector(std::move(result)); + } + case DirShape::AtomicShard: + /// A pure intermediate dir whose only child is the uuid-anchored table dir. Its path + /// shape collides with the non-Atomic `data/` fallback of both parseTableUuid and + /// parseTableFilePath, so it MUST be routed to the generic mirrored LIST BEFORE those + /// branches claim it (see classifyDirectory). + return listLiveTreeChildren(path); + case DirShape::TableDir: + { + /// Part names (live and `detached/` references) plus table-level verbatim + /// file names; addFirstComponent collapses both to their first path segment (live part + /// names and the single `detached` subdir, exactly like a nested verbatim file). + const auto ns = liveNamespace(*dr.uuid); + std::unordered_set result; + for (const auto & [ref, _] : store()->listRefs(ns)) + addFirstComponent(result, ref); + /// A dropped table lists empty for its namespace files too (its refs are already gone + /// via the ref state); only surface verbatim file names while the table is not removed. + if (const auto life = readableNamespaceFilesLife(ns)) + for (const auto & name : store()->listNamespaceFiles(*life)) + addFirstComponent(result, name); + /// EMPTY-PROOF RULE (Task 9, spec §1 [B3]): an empty table root is exactly what a + /// silently-erased backing looks like -- authorize the empty answer only against an + /// authoritative, uncached `_pool_meta` positive (see the helper). + if (result.empty()) + confirmPoolIdentityForEmptyEnumeration(path); + return toVector(std::move(result)); + } + case DirShape::DetachedContainer: + { + /// Detached part names (prefix stripped; never files). + std::vector result; + for (const auto & ref : detachedRefNames(dr.r->ns)) + result.push_back(ref.substr(Cas::kDetachedRefPrefix.size())); + /// EMPTY-PROOF RULE (Task 9, spec §1 [B3]): same for an empty detached container root. + if (result.empty()) + confirmPoolIdentityForEmptyEnumeration(path); + return result; + } + case DirShape::MovingContainer: + { + /// Staging part names (prefix stripped), mirrors DetachedContainer. + std::vector result; + for (const auto & ref : movingRefNames(dr.r->ns)) + result.push_back(ref.substr(Cas::kMovingRefPrefix.size())); + return result; + } + case DirShape::PartDir: + { + /// A part dir (live, detached part, shadow handled separately): logical file names, + /// nested keys collapsed to their first component (projections surface as ONE + /// .proj entry). + auto view = partAccess()->getView(dr.r->refKey(), Cas::Freshness::CachedForLoad); + return view ? view->listChildren("") : std::vector{}; + } + case DirShape::ProjectionDir: + { + /// Inner names with the .proj/ prefix stripped. + auto view = partAccess()->getView(dr.r->refKey(), Cas::Freshness::CachedForLoad); + return view ? view->listChildren(*dr.projection_prefix) : std::vector{}; + } + case DirShape::TableSubdir: + { + /// Verbatim files under /, first-component collapsed. + std::unordered_set result; + if (const auto life = readableNamespaceFilesLife(liveNamespace(dr.tf->table_uuid))) + for (const auto & name : store()->listNamespaceFiles(*life)) + if (name.starts_with(dr.tf->tail + "/")) + addFirstComponent(result, name.substr(dr.tf->tail.size() + 1)); + return toVector(std::move(result)); + } + case DirShape::GenericIntermediate: + /// The disk root "", `store`, or any loose-file container above a table dir: a + /// server-root-scoped mirrored LIST. (`store/` is handled by AtomicShard above, + /// since its non-Atomic-table ambiguity would otherwise misroute it here too late, + /// after parseTableUuid/parseTableFilePath have already claimed it.) + return listLiveTreeChildren(path); + } + return listLiveTreeChildren(path); /// unreachable +} + +DirectoryIteratorPtr ContentAddressedMetadataStorage::iterateDirectory(const std::string & path) const +{ + if (checkOpAdmitted(CasOpClass::Probe) == CasOpAdmission::TruthAbsent) + return std::make_unique(std::vector{}); + /// Mirror MetadataStorageFromPlainObjectStorage: iterateDirectory includes the path. + auto names = listDirectory(path); + std::vector fs_paths; + fs_paths.reserve(names.size()); + for (const auto & child : names) + fs_paths.push_back(fs::path(path) / child); + return std::make_unique(std::move(fs_paths)); +} + +bool ContentAddressedMetadataStorage::isDirectoryEmpty(const std::string & path) const +{ + /// A Vanished disk reports every directory empty too (truth): the ref-unlink removal path then + /// proceeds, letting a vanished-disk table's DROP complete rather than throwing CANNOT_RMDIR. + if (checkOpAdmitted(CasOpClass::Probe) == CasOpAdmission::TruthAbsent) + return true; + /// A part directory's files are virtual (derived from the tree): report it EMPTY so + /// DiskObjectStorage::removeDirectory proceeds straight to the ref-unlink instead of throwing + /// CANNOT_RMDIR per removal. The same applies to a projection subdirectory. The detached + /// CONTAINER and TABLE dirs keep the listing-based emptiness (DROP TABLE's non-empty guard). + if (auto p = Cas::parsePartFilePath(path)) + { + auto r = route(*p); + if (r && !r->ref.empty() && r->file.empty()) + return true; + if (r && !r->ref.empty() && Cas::PartFolderView::projectionDirPrefix(r->file)) + return true; + } + return !iterateDirectory(path)->isValid(); +} + +StoredObjects ContentAddressedMetadataStorage::getStorageObjects(const std::string & path) const +{ + /// ContentRead: resolves an object; loud typed error on a Vanished disk (never silent-empty). + checkOpAdmitted(CasOpClass::ContentRead); + /// In-manifest bytes (mutable per-part files, inline entries, verbatim namespace files) have + /// no object of their own: DiskObjectStorage::prepareRead serves them via tryGetInManifestBytes + /// BEFORE asking for storage objects. The sized empty-key placeholder below keeps size-only + /// consumers working and makes any bypassing reader fail LOUDLY (never silently wrong bytes). + if (auto bytes = tryGetInManifestBytes(path)) + return {StoredObject("", path, bytes->size())}; + + if (!Cas::isPartFilePath(path)) + { + if (Cas::parseTableFilePath(path)) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "ContentAddressed: table-level verbatim file is in-manifest, not a storage object: {}", path); + /// A loose mountpoint object: a real plain object at roots//. The + /// StoredObject key must be the PHYSICAL path (physicalKey-adjusted for Local backends). + /// Probe with a HEAD (directory-safe), not a body read: `system.remote_data_paths` + /// may reach here on a directory-shaped pool path and a GET would throw "Is a directory". + const std::string pool_key = store()->layout().mountpointObjectKey(serverPrefix() + "/" + path); + if (store()->mountpointObjectExists(serverPrefix() + "/" + path)) + return {StoredObject(physicalKey(pool_key), path, getFileSize(path))}; + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: no object for {}", path); + } + + auto p = Cas::parsePartFilePath(path); + if (!p || p->file.empty()) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: not a part file path: {}", path); + auto r = route(*p); + if (!r || r->file.empty()) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: not a part file path: {}", path); + + /// ONE snapshot for both the facade lookup and the pool `locate` below, so the two can never + /// straddle two different mount generations (see `poolAccess()`). + const auto snap = poolAccess(); + auto view = snap.part_access->getView(r->refKey(), Cas::Freshness::CachedForLoad); + if (!view) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: no ref for {}", path); + if (const auto * entry = view->findFile(r->file)) + { + const auto location = snap.pool->locate(*entry); + /// StoredObject carries no range (the recorded upstream delta) — the PAYLOAD length is the + /// size (what every size consumer wants); the header offset is applied by + /// getBlobViewPlan's view window, the only byte-reading path. + return {StoredObject(location.key, path, location.length)}; + } + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: file {} not in manifest of {}", r->file, path); +} + +std::optional ContentAddressedMetadataStorage::getStorageObjectsIfExist(const std::string & path) const +{ + /// A Vanished disk answers absent (truth). Probe first so the non-part `getStorageObjects` fallback + /// below (ContentRead) is never reached on a Vanished disk. + if (checkOpAdmitted(CasOpClass::Probe) == CasOpAdmission::TruthAbsent) + return std::nullopt; + /// Non-part shapes (verbatim table files, loose mountpoint objects) are rare paths — the + /// generic two-step is fine for them. + if (!Cas::isPartFilePath(path)) + { + if (existsFile(path)) + return getStorageObjects(path); + return std::nullopt; + } + auto p = Cas::parsePartFilePath(path); + if (!p || p->file.empty()) + return std::nullopt; + auto r = route(*p); + if (!r || r->file.empty()) + return std::nullopt; + + /// ONE snapshot for both the facade lookup and the pool `locate` below (see `poolAccess()`). + const auto snap = poolAccess(); + auto view = snap.part_access->getView(r->refKey(), Cas::Freshness::CachedForLoad); + if (!view) + return std::nullopt; + const auto * entry = view->findFile(r->file); + if (!entry) + return std::nullopt; + if (entry->placement == Cas::EntryPlacement::Inline) + return StoredObjects{StoredObject("", path, entry->size())}; + const auto location = snap.pool->locate(*entry); + return StoredObjects{StoredObject(location.key, path, location.length)}; +} + +std::optional ContentAddressedMetadataStorage::tryGetInManifestBytes(const std::string & path) const +{ + /// Speculative in-manifest probe: `DiskObjectStorage::prepareRead`/`getStorageObjects` call it before + /// falling back to a real storage-object lookup, and both treat "not in-manifest" (`std::nullopt`) as a + /// normal outcome. But this is a ContentRead: a disk in a terminal/uncertain/unstarted lifecycle must + /// PROPAGATE the typed 668 rather than convert it into a silent-absent `std::nullopt` -- so the gate + /// REPLACES the old catch-all that swallowed `poolAccess()`'s `INVALID_STATE` (which had hidden an + /// erased/replaced/transient backing behind a FILE_DOESNT_EXIST-shaped answer). + checkOpAdmitted(CasOpClass::ContentRead); + const PoolAccessSnapshot snap = poolAccess(); + + if (!Cas::isPartFilePath(path)) + { + if (auto tf = Cas::parseTableFilePath(path)) + { + const auto life = readableNamespaceFilesLife(liveNamespace(tf->table_uuid)); + return life ? snap.pool->getNamespaceFile(*life, tf->tail) : std::nullopt; + } + return std::nullopt; /// loose files are plain objects, not in-manifest bytes + } + + auto p = Cas::parsePartFilePath(path); + if (!p || p->file.empty()) + return std::nullopt; + auto r = route(*p); + if (!r || r->file.empty()) + return std::nullopt; + + auto view = snap.part_access->getView(r->refKey(), Cas::Freshness::CachedForLoad); + if (!view) + return std::nullopt; + return view->inlineBytes(r->file); +} + +bool ContentAddressedMetadataStorage::prepareInManifestRead( + const std::string & path, const ReadSettings & settings, ReadPipeline & pipeline) const +{ + /// In-manifest bytes (mutable per-part files, inline entries, verbatim namespace files): + /// served from memory — there is no object to read. + auto bytes = tryGetInManifestBytes(path); + if (!bytes) + return false; + + const auto size = bytes->size(); + auto creator = [path, data = std::move(*bytes)]( + const StoredObject &, const ReadSettings &, bool, bool) -> std::unique_ptr + { + return std::make_unique(path, data); + }; + pipeline.setSource(std::move(creator), {StoredObject("", path, size)}, settings); + return true; +} + +std::optional ContentAddressedMetadataStorage::getBlobViewPlan( + const std::string & path) const +{ + /// ContentRead: resolves a blob-backed path to its physical window; loud typed error on a Vanished + /// disk rather than a silent `std::nullopt` (which a reader would take as "not blob-backed"). + checkOpAdmitted(CasOpClass::ContentRead); + if (!Cas::isPartFilePath(path)) + return std::nullopt; + auto p = Cas::parsePartFilePath(path); + if (!p || p->file.empty()) + return std::nullopt; + auto r = route(*p); + if (!r || r->file.empty()) + return std::nullopt; + /// ONE snapshot for both the facade lookup and the pool `locate` below, instead of the previous + /// `partAccess()` then `store()` pair (each an independent `pointer_mutex` acquisition) -- see + /// `poolAccess()`. + const auto snap = poolAccess(); + auto view = snap.part_access->getView(r->refKey(), Cas::Freshness::CachedForLoad); + if (!view) + return std::nullopt; + if (const auto * entry = view->findFile(r->file)) + { + const auto location = snap.pool->locate(*entry); + BlobViewPlan plan; + /// bytes_size is the readable extent of THIS file's window, NOT the whole blob: a + /// right-bounded read stops at payload_end, and a shared blob's bytes beyond it belong + /// to other files. The caches key on the physical blob key, so payload ranges are + /// shared between every part that references the same blob. + plan.object = StoredObject(physicalKey(location.key), path, location.offset + location.length); + plan.payload_offset = location.offset; + plan.payload_end = location.offset + location.length; + return plan; + } + return std::nullopt; +} + +std::unique_ptr ContentAddressedMetadataStorage::readBlobPayload( + const Cas::BlobLocation & location, const std::string & path, const ReadSettings & settings) const +{ + /// ContentRead: the actual byte read; loud typed error on a Vanished disk instead of a raw backend + /// "no such key" from the erased object. + checkOpAdmitted(CasOpClass::ContentRead); + auto impl = object_storage->readObject( + StoredObject(physicalKey(location.key), path, location.offset + location.length), settings); + return std::make_unique( + std::move(impl), path, location.offset, location.offset + location.length); +} + +/// ==== `IContentAddressedExchange` ==== + +bool ContentAddressedMetadataStorage::ownsNamespace(const String & other_server_root_id, const String & root_namespace) const +{ + /// Routing for the relink confirm (spec §wire-protocol). `pool_uuid` says which POOL a token refers + /// to and is compared by the caller; every server root writing into that pool shares it, so the + /// namespace's owner is decided here. `liveNamespace` builds live and detached namespaces as + /// `/`, so ownership is exactly "rooted at MY server root". + /// The strict prefix (not a bare equality, not `starts_with(server_root_id)`) is what keeps + /// `srv1` from claiming `srv10/...`, and it deliberately does not match a pool-global shadow + /// namespace: a FREEZE tree belongs to no single mount and is never a relink source. + /// + /// Factory-class: no `store()`, no gate, no I/O, no throw. A misrouted question must come back as + /// an unproven answer, never as an error. + if (other_server_root_id.empty() || other_server_root_id != server_root_id) + return false; + return root_namespace.starts_with(server_root_id + "/"); +} + +CasConfirmAnswer ContentAddressedMetadataStorage::confirmExactRef( + const String & root_namespace, const String & ref_name, const String & manifest_ref_text) const +{ + /// Gate 1 of the relink confirm: a thin forward to the ledger, whose declaration + /// (`CasRefLedger::confirmExactRef`) carries the six-rule snapshot and the zero-I/O contract. This + /// layer adds exactly two things: the token text is decoded here, and the disk's own lifecycle is + /// answered as `Unknown` instead of as an exception. + const auto manifest_ref = Cas::tryParseManifestRef(manifest_ref_text); + if (!manifest_ref) + { + LOG_DEBUG(getLogger("ContentAddressedMetadataStorage"), + "Relink confirm for ref '{}' in namespace '{}' is unanswerable: manifest reference '{}' is not " + "the canonical epoch:build:ordinal form", ref_name, root_namespace, manifest_ref_text); + return CasConfirmAnswer::Unknown; + } + + Cas::ConfirmAnswer answer = Cas::ConfirmAnswer::Unknown; + try + { + /// ContentRead: the confirm reads this disk's committed view. A disk that never started, was shut + /// down, is transiently not live, or has reached a terminal lifecycle has no committed view to + /// speak for -- and `checkOpAdmitted` says so by throwing. + checkOpAdmitted(CasOpClass::ContentRead); + answer = store()->confirmExactRef(Cas::RootNamespace{root_namespace}, ref_name, *manifest_ref); + } + catch (const Exception & e) + { + /// This is NOT a fallback path: `Unknown` is the typed refusal this primitive is built around, + /// not an alternate behavior substituted for a failed one. Nothing consequential happens on it -- + /// the receiver aborts its prepared relink and retries later -- so swallowing the lifecycle + /// refusal costs a retry and can never authorize anything. Only `Yes` authorizes, and no `catch` + /// can produce a `Yes`. + LOG_DEBUG(getLogger("ContentAddressedMetadataStorage"), + "Relink confirm for ref '{}' in namespace '{}' is unanswerable on disk '{}': {}", + ref_name, root_namespace, disk_name, e.message()); + return CasConfirmAnswer::Unknown; + } + + switch (answer) + { + case Cas::ConfirmAnswer::Yes: + return CasConfirmAnswer::Yes; + case Cas::ConfirmAnswer::No: + return CasConfirmAnswer::No; + case Cas::ConfirmAnswer::Unknown: + return CasConfirmAnswer::Unknown; + } +} + +std::optional +ContentAddressedMetadataStorage::getRelinkOffer(const String & part_path) const +{ + /// Sender side: the committed part's encoded `PartManifest` body — the opaque payload the + /// receiver decodes — and the confirm token for it. Resolve the part path to its (ns, ref) exactly + /// as the read surface does (route), resolve the committed ref to its ManifestId, read the + /// immutable manifest, and re-encode it canonically. nullopt when the path is not a committed + /// content-addressed part here (no ref => no relink offer; the sender streams bytes). A live ref to + /// a missing/corrupt manifest throws (INV-NO-DANGLE surfaced, never substituted) — the same + /// fail-loud contract as partAccess()->getView. + /// ContentRead: reading a committed manifest; loud typed error on a Vanished disk. + checkOpAdmitted(CasOpClass::ContentRead); + auto p = Cas::parsePartFilePath(part_path); + if (!p) + return std::nullopt; + auto r = route(*p); + if (!r || r->ref.empty()) + return std::nullopt; + + auto view = partAccess()->getView(r->refKey(), Cas::Freshness::ForceFresh); + if (!view) + return std::nullopt; + + /// The token names the manifest THIS view resolved, so the offer and the question the receiver will + /// ask are about one and the same object by construction. `manifestId` is the journal identity the + /// ledger compares in gate 1, and it is already proven to agree with the body: `readManifest` + /// enforces `refMatchesBody`/`manifestNamespaceMatches` and throws `CORRUPTED_DATA` otherwise, so + /// there is no disagreement left for this function to discover or to quietly turn into a byte fetch. + /// + /// `ref_name` is what this mount publishes the part under and `part_name` is what gate 0 looks up + /// in the parts set; they coincide for every offer the sender can actually make, because it offers + /// only a live part. A staging path (`detached/`, `moving/`) reports the reserved directory name as + /// `part_name`, so a token minted for one selects no part at all — `Unknown`, which is the safe + /// direction — rather than selecting the wrong one. + const auto token = encodeCasRelinkSourceToken(CasRelinkSourceToken{ + .pool_uuid = pool_uuid, + .server_root_id = server_root_id, + .root_namespace = r->ns.string(), + .ref_name = r->ref, + .part_name = p->part_name, + .manifest_ref_text = Cas::manifestRefDebugString(view->manifestId().ref)}); + if (!token) + return std::nullopt; + + return RelinkOffer{.manifest_bytes = Cas::encodePartManifest(*view->manifest()), .confirm_token = *token}; +} + +namespace +{ + +/// The exchange's view of one durable-but-unpromoted relink: a `Cas::PreparedPartWrite` with the two +/// content-addressed-free verbs `DataPartsExchange` is allowed to know about. +/// +/// It also OWNS a `shared_ptr` snapshot of the part-folder facade, and that is load-bearing rather than +/// tidy: `PreparedPartWrite` holds its owner as a raw pointer, and this handle deliberately outlives +/// the call that made it -- it spans an interserver round trip -- so a concurrent `shutdown` resetting +/// the disk's facade would dangle that pointer. The snapshot is declared BEFORE the write, so the write +/// (and any abort its destructor runs) is destroyed first, while the facade is still alive. +class PreparedRelinkOverPartWrite : public ICaPreparedRelink +{ +public: + PreparedRelinkOverPartWrite(std::shared_ptr access_, Cas::PreparedPartWrite write_, + String ref_name_) + : access(std::move(access_)), write(std::move(write_)), ref_name(std::move(ref_name_)) + { + } + + CaRelinkPromote promote() override + { + try + { + write.promote(); + return CaRelinkPromote::Committed; + } + catch (const Exception & e) + { + /// UNCERTAINTY IS CHECKED FIRST, and it outranks the error code. A `NETWORK_ERROR` out of + /// the promote means one of two entirely different things: the promote was rejected before + /// its ref-log append (nothing committed), or the append itself was attempted and did not + /// resolve -- in which case the promotion PUT may have landed and the ref may be live. The + /// error code cannot tell them apart, which is why the transaction records the distinction + /// as it happens. Reporting the second case as a mechanism fallback would have the receiver + /// fetch the bytes and publish a second time over a relink that already committed. + if (write.commitIsUnresolved()) + { + LOG_INFO(getLogger("ContentAddressedMetadataStorage"), + "Relink of part {} could not be resolved: the promotion append may or may not have " + "committed ({}); the caller must retry the whole fetch later, NOT fetch the bytes", + ref_name, e.message()); + return CaRelinkPromote::Unresolved; + } + /// The same retryable class the staging half classifies: a body-absent precommit, a + /// precommit binding that is no longer the live owner, or a ref conflict. `promote` has + /// already abandoned the build on its way out, so the `+1` is released and the sender's + /// bytes are a sound recovery. Anything else propagates -- it is not a known-safe + /// mechanism failure, and the receiver must not silently turn it into a byte fetch. + if (e.code() != ErrorCodes::ABORTED && e.code() != ErrorCodes::NETWORK_ERROR) + throw; + LOG_INFO(getLogger("ContentAddressedMetadataStorage"), + "Relink of part {} could not be promoted (body-absent precommit, precommit not the live " + "owner, or a ref conflict): {}; the caller may fetch the bytes from the same source", + ref_name, e.message()); + return CaRelinkPromote::MechanismFallbackAllowed; + } + } + + void abort() noexcept override + { + /// Not defensive noise: a `promote` that FAILED discharges the duty itself (its catch abandons + /// the build), so the scope guard that always runs finds a terminal handle on exactly that path. + if (write.isTerminal()) + return; + try + { + write.abort(); + } + catch (...) + { + /// The removal append did not land. `PreparedPartWrite` stays non-terminal, so its own + /// destructor retries it; beyond that the durable backstop is the ref lane's wedge, exactly + /// as for every other abandon path. + tryLogCurrentException(getLogger("ContentAddressedMetadataStorage"), + fmt::format("aborting the prepared relink of part {}", ref_name)); + } + } + +private: + std::shared_ptr access; + Cas::PreparedPartWrite write; + String ref_name; +}; + +} + +/// TRUST MODEL: adopting a part from a peer-supplied manifest is exactly as trusted as an ordinary +/// ReplicatedMergeTree interserver part fetch. The interserver HTTP channel — not a per-blob ACL — is +/// the trust boundary: a malicious or MITM peer on that channel can already serve arbitrary part bytes +/// that the receiver adopts, in both the byte-streaming and the relink path. Table-level RBAC never +/// defended against a hostile peer, so relink-by-manifest adds no new trust surface. (See the retracted +/// umbrella "RBAC bypass" finding.) +CaRelinkPrepare ContentAddressedMetadataStorage::prepareAdoptFromManifest( + const String & part_path, const String & manifest_bytes, + std::unique_ptr & out) +{ + checkNotReadOnly("prepareAdoptFromManifest (interserver relink receiver)"); + /// Write class: publishing a receiver-local ref; throws typed on a Vanished disk, 668 while uncertain. + checkOpAdmitted(CasOpClass::Write); + + /// Receiver side. Sender identity is non-authoritative: we ignore the decoded ManifestRef, + /// root_namespace_id and payload_digest, and use ONLY the entries. We run a normal LOCAL build over + /// the SHARED-pool blobs — adopted by hash via adoptEvidence, NO blob body transferred — then stage a + /// FRESH receiver-local ManifestId in the receiver namespace and `precommitAdd` it. + /// + /// The promote does NOT happen here, and that split is the fix for the commit-before-release gap + /// this function used to carry (codex-6). The sender's relink response is fire-and-forget: it + /// releases the source part when `processQuery` returns, so if this `+1` were not yet durable while + /// the source's now-`Outdated` part was collected, the receiver would commit a manifest whose blobs + /// are gone. Publishing FIRST and asking SECOND closes THAT window: any removal of the source + /// binding is appended strictly after this `+1` is durable in the ref log. + /// + /// It does NOT establish that every subsequent GC fold SEES the `+1`, and the ordering must not be + /// read as if it did: `CaRelinkConfirmCore.tla` config `_sab_holeylist` shows a fold that misses it + /// (BACKLOG `{#list-as-journal-dataloss-2026-07-25}`), which is why the caller-side taxonomy states + /// plainly that a confirmed relink is not proven dangle-free (`DataPartsExchange.cpp`, "What a + /// `yes` does NOT prove"). Durable-before-asking is necessary, not sufficient. + /// + /// Promotion trusts the adopted leaves via the durable manifest edge (no per-file HEAD/loadMeta + /// probe); a genuinely-absent adopted blob is an invariant violation caught by fsck, not here — the + /// ordinary ReplicatedMergeTree interserver trust. + out.reset(); + + /// The RECEIVER's own (namespace, ref) for the target path — never the sender's `root_namespace_id`, + /// which is foreign to this server's path-mirroring identity. Routing rather than composing is what + /// gives B66b its detached target for free: `TABLE/detached/DIR` folds onto `detached/DIR` in + /// the table's OWN namespace, and a live target onto `DIR`, through the same `route` the reads use. + /// A path that is not a part DIRECTORY here is a caller error, not a fallback: a byte fetch to the + /// same place would be just as wrong, so it must not be quietly substituted. Shadow (FREEZE) paths + /// are rejected for the same reason — a backup namespace is never a fetch target. + auto p = Cas::parsePartFilePath(part_path); + auto r = p ? route(*p) : std::nullopt; + if (!p || !r || !p->backup_name.empty() || r->ref.empty() || !r->file.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Relink target '{}' does not address a content-addressed part directory of a live table", part_path); + + Cas::PartManifest decoded; + try + { + decoded = Cas::decodePartManifest(manifest_bytes); + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::CORRUPTED_DATA) + throw; + LOG_INFO(getLogger("ContentAddressedMetadataStorage"), "Relink of part {} not possible: transferred manifest failed to decode ({}); " + "caller falls back to a byte fetch", part_path, e.message()); + return CaRelinkPrepare::MechanismFallbackAllowed; + } + + auto access = partAccess(); + try + { + out = std::make_unique( + access, access->prepareEntries(r->refKey(), decoded.entries, Cas::ProvenanceOp::Attach), r->ref); + return CaRelinkPrepare::Prepared; + } + catch (const Exception & e) + { + /// `ABORTED` or `NETWORK_ERROR` means a body-absent precommit, a precommit binding that is no + /// longer the live owner, a ref conflict, or — since the transient-classifier round — this node's + /// own mount fence refusing the work (`throwCasTransientUnavailable`): all retryable, and the + /// sender still has the part, so the caller may fetch its bytes. `prepareEntries` abandons its own + /// build before propagating, so nothing is staged and no `+1` is left behind. The fence case needs + /// no special handling and stays fail-close by construction: the byte fetch it falls back to writes + /// through the SAME fenced disk and is refused in turn, so the fallback cannot smuggle a write past + /// a lost incarnation. Any other error propagates — an unclassified local failure is not evidence + /// that a byte fetch would do better. + if (e.code() != ErrorCodes::ABORTED && e.code() != ErrorCodes::NETWORK_ERROR) + throw; + LOG_INFO(getLogger("ContentAddressedMetadataStorage"), "Relink of part {} deferred (body-absent precommit, " + "precommit not the live owner, or a ref conflict): {}; caller falls back to a byte fetch", part_path, e.message()); + return CaRelinkPrepare::MechanismFallbackAllowed; + } +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.h new file mode 100644 index 000000000000..8c461f5f4f6d --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.h @@ -0,0 +1,774 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Poco::Util { class AbstractConfiguration; } + +namespace DB +{ +class IDisk; +using DiskPtr = std::shared_ptr; +} + +namespace DB::Cas +{ + +/// Selects where a content-addressed blob is staged before it is published. +/// +/// `Local` is the default and preserves the existing local scratch-file path byte for byte; it does +/// not run the conditional-copy probe. `S3` is opt-in and can stream large blobs to an object-store +/// staging key. That path is usable only after the mount-time probe has demonstrated write-once +/// conditional copy semantics. If the backend does not enforce those semantics, callers must stay +/// on `Local`: an unconditional copy could overwrite a live content-addressed blob. +enum class StagingBackend +{ + Local, + S3, +}; + +} + +namespace DB +{ + +/// The SIX operation classes the central `ContentAddressedMetadataStorage::checkOpAdmitted` gate keys on +/// (rev.7 spec §1). Every public metadata/transaction entry declares its class so a single place decides +/// what happens per pool-lifecycle condition: +/// - `Factory` — I/O-free construction / capability / introspection (`createTransaction`, the +/// `getType`/`getPath`/capability getters, `gcHealth`, `lifecycleSnapshot`). NEVER +/// gated: it works in every state. Such call sites do not invoke `checkOpAdmitted`. +/// - `Probe` — existence / enumeration (`existsFile`/`existsDirectory`/`listDirectory`/ +/// `iterateDirectory`/`isDirectoryEmpty`/`getStorageObjectsIfExist`). Answers the +/// truth: real while live, throws while uncertain, absent/empty once `Vanished`. +/// - `ContentRead` — resolving/serving bytes or per-file metadata (`getStorageObjects`/`getFileSize`/ +/// `getLastModified`/`getBlobViewPlan`/`readBlobPayload`/`getRelinkOffer`/ +/// `tryGetInManifestBytes`/`prepareInManifestRead`). Never silent-absent on `Vanished` +/// — a loud typed error instead. +/// - `Write` — create / write / rename, INCLUDING the previously-no-op sites and a publishing +/// `commit`. Throws typed on `Vanished`. +/// - `Remove` — ref/file removal (`removeRecursive`/`removeDirectory`/`unlinkFile`, and an +/// empty/pure-remove `commit`). No-op SUCCESS on `Vanished` so a vanished-disk table's +/// `DROP` completes. +/// - `Admin` — `store()`-reaching admin + GC round entry points. +enum class CasOpClass : uint8_t +{ + Factory, + Probe, + ContentRead, + Write, + Remove, + Admin, +}; + +/// The disposition `checkOpAdmitted` returns for the classes that have a truthful short-circuit answer in +/// the terminal `Vanished` state (Probe / Remove). `Proceed` means run the operation normally against the +/// live pool; `TruthAbsent` means answer the truth WITHOUT touching the pool (absent/empty for a Probe, +/// no-op success for a Remove). Every other outcome is a throw, so a caller only ever sees these two. +enum class CasOpAdmission : uint8_t +{ + Proceed, + TruthAbsent, +}; + +/// A non-gated lifecycle snapshot of one content-addressed disk for `system.cas_mounts` +/// (rev.7 spec §7, [C5]-visibility). It is a Factory-class read (§1): I/O-free, no `store()`/`poolAccess`, +/// truthful in EVERY state — including a not-live / vanished pool the store()-class surface refuses, and a +/// null pool (before `startup` / after `shutdown`). This is what keeps a disappearing disk VISIBLE to the +/// operator instead of silently missing from the table. +/// - `lifecycle` — one of `live` / `not_live` / `identity_lost` / `vanished` (a live pool), or +/// `constructing` / `shutdown` (no pool published). +/// - `reason` — the ENUM-CLEAN sub-state word: `replaced` / `forgotten` for a +/// `vanished` pool, empty otherwise. Kept a small closed vocabulary so a downstream +/// `lifecycle || '(' || reason || ')'` yields e.g. exactly `vanished(forgotten)` — +/// the [D5] free text lives in `detail`, never here. +/// - `detail` — the full [D5] reason text naming the actual failure (the replaced +/// diagnosis, the timestamped `FORGET` message, or the identity-loss message); spec §1 +/// requires it appear verbatim in the snapshot. Empty while `live` and for a null pool. +/// - `since` — wall-clock second the current non-`live` state was entered; 0 while `live`/no pool. +/// - `pool_id` — last-known pool UUID (empty before the first `startup`); the disk stays +/// introspectable under its identity even once the pool is gone. +/// - `server_root_id` — this server's node-local root id owning the mount slot. +struct CasLifecycleSnapshot +{ + String lifecycle; + String reason; + String detail; + time_t since = 0; + String pool_id; + String server_root_id; +}; + +/// Adapts ClickHouse's `IMetadataStorage` path-based interface to the content-addressed pool. +/// +/// The class owns the pool and its cached part-folder facade for the lifetime of an opened disk. It +/// parses disk paths, maps them to pool namespaces and references, and translates manifest entries +/// into `StoredObjects` or in-memory read sources. Transaction and GC entry points are exposed here +/// because disk lifecycle and system-query code own those operations; the CAS protocol itself stays +/// in `Cas::Pool`, `Cas::PartWriteTxn`, and `Cas::Gc`. +/// +/// Namespace mapping: +/// live part SERVER_ID/TABLE_UUID ref = PART_DIR +/// detached part SERVER_ID/TABLE_UUID ref = detached/DETACHED_PART_DIR +/// FREEZE shadow the LITERAL shadow table dir ref = PART_DIR +/// (shadow/BACKUP/store/U3/UUID or shadow/BACKUP/data/DB/TBL — bijective with +/// the disk path for both Atomic and non-Atomic layouts, so the shadow tree +/// enumerates from `Pool::listNamespaces("shadow/...")`) +/// generic files SERVER_ID/_disk verbatim namespace files (access probes) +/// +/// Small per-part files (`uuid.txt`, `metadata_version.txt`, `txn_version.txt`, `checksums.txt`, ...) +/// are inline-placement manifest tree entries, not sidecar objects. Their bytes are served through +/// `DiskObjectStorage::prepareRead`'s CA branch via `tryGetInManifestBytes`; `getStorageObjects` returns a +/// sized placeholder with an EMPTY remote key for them (any consumer bypassing the prepareRead branch +/// fails loudly, never reads wrong bytes). +class ContentAddressedMetadataStorage final : public IMetadataStorage, public IContentAddressedExchange +{ +public: + /// Constructs an unopened storage adapter. `settings_` carries every tunable that used to be a + /// positional parameter (see `ContentAddressedSettings`) -- it is the single source of defaults, so + /// the constructor itself declares none. `server_root_id`/`scratch_path` (the local-scratch + /// directory used when a write buffer must spill before hashing and upload, independent of the + /// object-storage key prefix) are read from `settings_` rather than taken as their own parameters. A + /// non-null `context_` enables the background GC scheduler on the disk-factory path; tests may pass + /// null to disable system-log integration and scheduling. `disk_name_` falls back to + /// `storage_path_prefix_` when empty, exactly as before this constructor collapsed. + ContentAddressedMetadataStorage( + ObjectStoragePtr object_storage_, + String storage_path_prefix_, + String server_id_, + String disk_name_, + ContextPtr context_, + const ContentAddressedSettings & settings_); + + /// Parses a `staging_backend` value (`local` | `s3`). Throws `BAD_ARGUMENTS` for an unrecognized + /// value rather than silently selecting a backend. + static Cas::StagingBackend parseStagingBackend(const std::string & value); + + /// Reads `staging_backend` from `config`, defaulting to `local`, and parses it. Kept only as a + /// thin wrapper around the string-taking overload for callers that still hold a config reference. + static Cas::StagingBackend parseStagingBackend(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); + + /// Parses a `part_folder_validate` value (`always` | `never` | `age `). The `age` form + /// accepts only a non-negative integer number of seconds; malformed input and unknown modes throw + /// `BAD_ARGUMENTS` instead of silently selecting a policy. + static Cas::PartFolderValidate parsePartFolderValidate(const std::string & value); + + /// Reads `part_folder_validate` from `config`, defaulting to `always`, and parses it. Kept only as + /// a thin wrapper around the string-taking overload for callers that still hold a config reference. + static Cas::PartFolderValidate parsePartFolderValidate(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); + + /// Returns the content-addressed metadata storage backing `disk`, or nullptr if `disk` is not + /// content-addressed. Plain (non-object-storage) disks do not implement `getMetadataStorage` at + /// all and throw `NOT_IMPLEMENTED`; that is treated as "not content-addressed" rather than + /// propagated. Any other exception from `getMetadataStorage` is rethrown. Centralizes the + /// detection lambda duplicated across `InterpreterSystemQuery` and + /// `StorageSystemContentAddressedMounts`; callers there have not yet been migrated to it. + static ContentAddressedMetadataStorage * tryFromDisk(const DiskPtr & disk); + + /// Runs one synchronous GC round for tests and diagnostics. If the scheduler is not running, + /// this lazily creates one so repeated calls retain the same lease-observation history. + void runOneGcRoundForTest(); + + /// Runs one synchronous GC round on the caller's thread and emits Start and Finish rows to + /// `system.cas_gc_log`. Throws `BAD_ARGUMENTS` when GC is disabled + /// by read-only mode or configuration. + Cas::RoundReport runGarbageCollectionRoundNow(); + + /// Coalesce an administrative liveness hint into the existing periodic GC worker. If this disk has + /// no running scheduler, the normal operator-visible retry path remains unchanged. + void requestGcRoundSoon(); + + /// Rebuilds the GC baseline for the `SYSTEM` disaster-recovery command. Each invocation uses a + /// fresh GC identity because `rebuildBaseline` performs its own lease check. A refused rebuild + /// (`report.performed == false`) writes nothing; the `SYSTEM` interpreter surfaces the refusal. + /// Throws `BAD_ARGUMENTS` when GC is disabled by read-only mode or configuration. + Cas::RebuildReport runGcRebuildNow(bool force) const; + + /// The `SYSTEM CAS FSCK` handler: a read-only, independent reachability audit. + /// + /// FSCK scans the LIVE running pool directly: Admin class -- it routes through + /// `checkOpAdmitted(CasOpClass::Admin)` (refuses on a transient / `IdentityLost` / `Vanished` pool, + /// consistent with `SYSTEM CAS GC RUN`), because an FSCK of a not-live disk is + /// meaningless -- the operator has the snapshot / FORGET path. The scan tolerates concurrent writers: + /// its ref-walk findings (missing manifest, dangling blob) are revalidated against a FRESH + /// authoritative read before being reported, so a legitimate concurrent republish/drop + GC delete + /// never surfaces as a phantom. Held under `lifecycle_mutex` for the whole scan so a concurrent + /// lifecycle-control verb (FORGET / GC STOP / GC START) cannot race it. + Cas::FsckReport runFsckNow(bool detail) const; + + /// Returns per-disk GC health for `system.cas_mounts`. Returns nullopt when this + /// disk has no scheduler because GC is disabled, the disk is read-only, or startup has not run. + /// Holds `gc_scheduler_mutex` for the entire call so a concurrent system-table query cannot + /// observe a scheduler while `shutdown` destroys it. + std::optional gcHealth() const; + + /// The non-gated lifecycle snapshot for `system.cas_mounts` (spec §7, Factory class): + /// I/O-free, reachable in EVERY state — a live pool (forwards `Pool::lifecycleSnapshot`), a terminal + /// pool the store()-class surface refuses, and a null pool (before `startup`/after `shutdown`, reported + /// as `constructing`/`shutdown`). Never calls `store()`/`poolAccess()`, never touches the backend, so + /// the very disk that vanished stays visible. Takes only a brief `pointer_mutex` snapshot of the pool + /// pointer; the identity fields are read from immutable-after-startup storage members. + CasLifecycleSnapshot lifecycleSnapshot() const; + + MetadataStorageType getType() const override { return MetadataStorageType::CAS; } + const std::string & getPath() const override { return storage_path_full; } + bool supportsChmod() const override { return false; } + bool supportsStat() const override { return false; } + bool isReadOnly() const override { return read_only; } + bool isContentAddressed() const override { return true; } + + /// Fail-close gate shared by every mutating entry point (transactions, GC round, GC rebuild, + /// pool-member decommission): an observe-only (``) disk must reject them all. + void checkNotReadOnly(std::string_view what) const; + + /// THE central six-class operation gate (rev.7 spec §1), consulted at EVERY public metadata/ + /// transaction entry (see the `CasOpClass` doc above and the method->class inventory at the top of + /// the .cpp). Given `op`'s class it inspects the pool lifecycle ONCE and decides: + /// - `Live` -> `Proceed` for every class. + /// - null pool (the storage-level Constructing/ShutDown lifecycle -- before `startup`/after + /// `shutdown`) -> throws "not started" for EVERY class, `Probe` included: a storage that has + /// never published a pool (or torn one down) has no benign "absent" answer to give. + /// - `TransientNotLive` / `IdentityLost` -> throws 668 for every class but `Factory` (uncertain + /// backing; the sub-state distinction is surfaced in `system.cas_mounts`, not the + /// op error). + /// - `Vanished*` -> `Probe`/`Remove` answer `TruthAbsent` (absent-empty / no-op success); + /// `ContentRead`/`Write`/`Admin` throw the typed per-reason [D5] message (via + /// `Pool::throwIfLifecycleTerminal`, the single home of those strings). + /// `Factory` is never passed here. Public because `ContentAddressedTransaction` funnels its own + /// mutating entries through it. + CasOpAdmission checkOpAdmitted(CasOpClass op) const; + + /// Content-addressed transactions are eager staging overlays: each mutating disk-transaction + /// method reaches the metadata transaction immediately rather than entering the FIFO queue. + bool transactionIsStagingOverlay() const override { return true; } + bool supportsAtomicFileWrites() const override { return true; } + bool supportsTransactionalMutableFiles() const override { return true; } + bool areBlobPathsRandom() const override { return false; } + uint32_t getHardlinkCount(const std::string &) const override { return 0; } + + /// Creates a write transaction bound to this storage. Throws `READONLY` before allocating one + /// when the disk was opened read-only. + MetadataTransactionPtr createTransaction() override; + + /// Opens the pool, validates its format and starts the optional GC scheduler. Read-only disks + /// skip write probes and GC; failures in startup propagate. Runs exactly once, single-threaded, + /// strictly before this object is exposed to any other thread (no other method can be called + /// concurrently with it) -- TSA_NO_THREAD_SAFETY_ANALYSIS is deliberate here, not a bypass of a + /// real risk: pointer_mutex/gc_scheduler_mutex exist to guard concurrent access AFTER startup, + /// which is definitionally impossible during it. + void startup() TSA_NO_THREAD_SAFETY_ANALYSIS override; + /// Stops the GC scheduler before releasing the part-folder facade and pool. Their destruction is + /// synchronized with the accessors and synchronous GC entry points. + void shutdown() override; + + /// `SYSTEM CAS FORGET` handler (Task 10, spec §5): the operator force-Vanish. Drives the + /// live pool to `Vanished(forgotten)` node-locally via `Pool::forgetDisk`'s fence-first protocol, and + /// stops + joins the GC scheduler as part of it. Unlike the store()-class verbs, this must work on a + /// NOT-live disk (a stuck transient / `IdentityLost` pool) — that is its purpose — so it reaches the + /// pool DIRECTLY, never through `poolAccess()`/`checkOpAdmitted` (which refuse a not-live disk); it is a + /// lifecycle verb, like the Factory class. FORGET is an operator ASSERTION, not an erasure proof: the + /// resulting [D5] error message (which carries the decommission timestamp) says so. The disk stays + /// registered; the six-class gate then answers the truth (Probe/Remove truth-absent, reads throw the + /// [D5] message). Idempotent; a disk with no published pool is a no-op. Serialized by `lifecycle_mutex` + /// (against FSCK / GC STOP / GC START) and `gc_scheduler_mutex` (against a synchronous round). + void forgetDisk() TSA_NO_THREAD_SAFETY_ANALYSIS; + + /// `SYSTEM CAS GC STOP` handler (Task 11, spec §6): stops ONLY the background GC scheduler. + /// The disk stays fully usable -- reads/writes are unaffected; this is granular operator control of the + /// GC pacer alone, NOT a lifecycle transition. Unlike `forgetDisk` this is + /// STOP-IN-PLACE: the scheduler object is RETAINED in the member (not detached/destroyed), so `gcHealth` + /// keeps reading its (now stopped) state truthfully and a later `gcStart` re-enters the SAME instance + /// with its `gc_id` + lease-observation history preserved. `stop()` + /// joins the worker+heartbeat threads and clears the in-process leadership hint. Idempotent (a second + /// STOP is a no-op); a no-op success when no scheduler exists (GC disabled / read-only / not started). + /// Works on a not-live/Vanished disk too -- stopping GC on a sick disk is legitimate operator action, so + /// this does NOT consult `checkOpAdmitted`. Serialized by `lifecycle_mutex` and `gc_scheduler_mutex`. + void gcStop() TSA_NO_THREAD_SAFETY_ANALYSIS; + + /// `SYSTEM CAS GC START` handler (Task 11, spec §6): restarts the background GC scheduler + /// stopped by `gcStop`, re-entering the SAME instance (its `start()` is re-enterable after a join). + /// Leadership is NOT auto-restored -- the scheduler re-acquires the durable `gc/state` lease through the + /// next round's normal acquisition. Idempotent (a no-op on a running scheduler). Unlike `gcStop`, it goes + /// through the uniform GC gate (`checkOpAdmitted(Admin)`): it refuses on a transient / `IdentityLost` / + /// `Vanished` pool with the typed error (668 / [D5]) and on a not-mounted disk -- restarting GC on a + /// decommissioned/uncertain pool is meaningless and would only spin failing rounds. Serialized by + /// `lifecycle_mutex` and `gc_scheduler_mutex`. + void gcStart() TSA_NO_THREAD_SAFETY_ANALYSIS; + + /// Test-only fault-injection hook. When set, `startup` invokes it right before it publishes + /// `cas_store`/`part_access`/`gc_scheduler`/`pool_uuid`/`conditional_copy_supported` -- everything + /// up to that point (opening the pool, building the part-folder facade, running the capability + /// probe, starting the GC scheduler) has already happened into locals, so throwing here proves a + /// late startup failure publishes nothing and a retry can still succeed. Left empty (a no-op) in + /// production. + std::function startup_fault_injection_for_test; + + /// Tests whether a path is represented by an inline manifest entry, namespace file, or loose + /// mountpoint object. + bool existsFile(const std::string & path) const override; + /// Tests whether a path names a virtual part, table, shadow, or mirrored live-tree directory. + bool existsDirectory(const std::string & path) const override; + /// Tests both file and directory interpretations of a path. + bool existsFileOrDirectory(const std::string & path) const override; + /// Returns the logical payload size, excluding a blob envelope. + uint64_t getFileSize(const std::string & path) const override; + /// Returns the part publication time; other existing files use epoch time because their mtime is + /// not retained by the content-addressed namespace. + Poco::Timestamp getLastModified(const std::string & path) const override; + /// Lists logical children of a virtual or mirrored directory. + std::vector listDirectory(const std::string & path) const override; + /// Iterates over `listDirectory` results with each child joined to `path`. + DirectoryIteratorPtr iterateDirectory(const std::string & path) const override; + /// Reports virtual part and projection directories as empty so removal unlinks their ref; table + /// and container directories use their listing. + bool isDirectoryEmpty(const std::string & path) const override; + /// Maps a logical path to its storage object. Inline entries return a sized empty-key placeholder + /// and must be served by the CA read branch. + StoredObjects getStorageObjects(const std::string & path) const override; + /// Performs one manifest lookup for part files instead of the inherited `existsFile` plus + /// `getStorageObjects` sequence. + std::optional getStorageObjectsIfExist(const std::string & path) const override; + + /// ==== `IContentAddressedExchange` (interserver relinking facade) ==== + const String & getPoolUUID() const override { return pool_uuid; } + /// Routing predicate for the confirm action: this mount owns a namespace iff the namespace is rooted + /// at ITS `server_root_id` (`liveNamespace` builds every live/detached namespace as + /// `/`). Factory-class: I/O-free, ungated, never throws. + bool ownsNamespace(const String & other_server_root_id, const String & root_namespace) const override; + /// Gate 1 of the relink confirm, forwarded to the pool's ref ledger. Answers `Unknown` for an + /// unparsable token and for a disk that has not started or has reached a terminal lifecycle. + CasConfirmAnswer confirmExactRef(const String & root_namespace, const String & ref_name, + const String & manifest_ref_text) const override; + /// Returns the canonical encoded manifest for a committed part plus the confirm token minted from + /// that same resolution, or nullopt when the path is not a committed CA part or the token cannot be + /// minted. Missing or corrupt committed state propagates as an exception. + std::optional getRelinkOffer(const String & part_path) const override; + /// Stages a peer-supplied manifest into this server's namespace without transferring blob bodies and + /// hands back the durable-but-unpromoted handle. The receiver's `part_path` is routed exactly as any + /// other part path, so a live target and a `detached/` one (B66b) differ only in the ref the router + /// derives. Answers `MechanismFallbackAllowed` for a decode failure or a retryable staging failure so + /// the caller can byte-fetch instead; read-only disks throw `READONLY`. + CaRelinkPrepare prepareAdoptFromManifest( + const String & part_path, const String & manifest_bytes, + std::unique_ptr & out) override; + + /// ==== wiring-internal surface (the transaction + the disk's prepareRead CA branch) ==== + + /// Returns a shared-ownership snapshot of the opened pool. Throws `INVALID_STATE` when no pool is + /// published (before the first `startup`, or after `shutdown`). A thin wrapper over `poolAccess()`. + Cas::PoolPtr store() const; + /// Returns a shared-ownership snapshot of the cached part-folder facade. Throws `INVALID_STATE` + /// under the same not-started condition as `store()`. Committed part-folder reads and mutations + /// go through this facade so cache validation remains centralized. Returning a `shared_ptr` + /// snapshot (not a reference) means the returned handle keeps the facade alive via its own + /// refcount even if `shutdown` concurrently resets the member -- unlike a reference, which would + /// dangle the instant `shutdown`'s reset runs. A thin wrapper over `poolAccess()`. + std::shared_ptr partAccess() const; + const std::string & serverRootId() const { return server_root_id; } + const std::string & scratchPath() const { return local_scratch_path; } + /// Returns the configured staging backend. `Local` is the behavior-preserving default; callers + /// must also check `conditionalCopySupported` before using S3 promotion. + Cas::StagingBackend stagingBackend() const { return staging_backend; } + /// Returns the mount-time conditional-copy capability result. It starts false and becomes true + /// only after the backend proves write-once copy semantics, so S3 promotion fails closed. + bool conditionalCopySupported() const { return conditional_copy_supported; } + /// Returns the underlying object storage for an S3 staging writer. It is meaningful only when + /// `stagingBackend` is `S3` and `conditionalCopySupported` is true. + const ObjectStoragePtr & objectStorage() const { return object_storage; } + /// Returns the physical prefix for this pool's writer-owned staging area. It is the same + /// `pool_prefix/staging/server_root_id` subtree used by the capability probe; callers append a + /// unique leaf and must not use the probe object itself. + String stagingKeyPrefix() const; + + /// Bytes that live INSIDE pool metadata rather than as their own object: an Inline-placement + /// manifest tree entry, or a verbatim namespace file. nullopt = the path is blob-backed (a real + /// storage object). + std::optional tryGetInManifestBytes(const std::string & path) const; + + /// The CA read entry called by `DiskObjectStorage::prepareRead` before the generic + /// storage-objects path: serves in-manifest bytes (mutable per-part files, inline entries, + /// verbatim namespace files) from memory. Returns false when the path is not in-manifest. + /// Declared on `IContentAddressedExchange` (the narrow seam `prepareRead` casts to); `BlobViewPlan` + /// is likewise inherited from there. + bool prepareInManifestRead(const std::string & path, const ReadSettings & settings, ReadPipeline & pipeline) const override; + + /// Resolves a blob-backed path to its physical object and payload window. Returns nullopt for + /// in-manifest, loose, directory, or otherwise unresolved paths. + std::optional getBlobViewPlan(const std::string & path) const override; + + /// Creates a seekable reader over one blob payload, excluding its envelope. Transactions use + /// this for read-your-writes; committed reads use `getBlobViewPlan` and the normal pipeline. + std::unique_ptr readBlobPayload( + const Cas::BlobLocation & location, const std::string & path, const ReadSettings & settings) const; + + /// Maps a live table UUID to its pool namespace. Detached parts share that namespace and use + /// `detached/`-prefixed references rather than a sibling namespace. + Cas::RootNamespace liveNamespace(const std::string & table_uuid) const; + /// Canonicalizes a literal shadow-table directory into the pool namespace used by freeze and + /// unfreeze paths. A trailing slash is ignored. + static Cas::RootNamespace shadowNamespace(const std::string & shadow_table_dir); + + /// The LIFE under which `ns`'s table-level namespace files — `format_version.txt` and the other + /// verbatim files — must be read, or `nullopt` when there are none to read. + /// + /// It answers two things at once because they are one question. A dropped-and-not-recreated table + /// (ref-table lifecycle durably `Removed`) is GONE for readers: its files must read as absent even + /// while GC has not yet physically reclaimed them (namespace removal is deferred to GC), mirroring + /// how its parts already vanish via the ref state. A never-born namespace is likewise empty. And a + /// readable namespace's files live under ITS OWN incarnation (Stage B Task 4b), never under a + /// previous life's — which is why the readable answer is a life rather than a `true`: a reader that + /// has no life cannot form a key at all, so a previous life's surviving objects are unreachable by + /// construction rather than by remembering to check something. + std::optional readableNamespaceFilesLife(const Cas::RootNamespace & ns) const; + + /// Returns the root prefix for mirrored live-tree objects. The persistent layout identity is + /// `server_root_id`; `ServerUUID` remains only the mount-owner token. + std::string serverPrefix() const; + + /// Enumerate the children of a GENERIC intermediate live-tree directory (the disk root "", + /// `store`, the `store/` shard dir, or any loose-file container above a table dir) via a + /// server-root-scoped mirrored S3 LIST of `roots///`. `@cas@`-suffixed table-dir + /// segments are surfaced under their logical (unsuffixed) name. This is what makes top-down + /// `clickhouse-disks` traversal of the live tree behave like a normal disk; concrete + /// `store////` navigation is still served by the exact-shape branches. + std::vector listLiveTreeChildren(const std::string & path) const; + /// Tests whether the server-root-scoped mirrored subtree has at least one child. The disk root is + /// always considered present. + bool liveTreeDirHasChildren(const std::string & path) const; + + /// Resolves one parsed path to its namespace, reference, and in-tree file. Detached paths are + /// re-split here so their references remain in the table namespace with a `detached/` prefix; + /// shadow paths map to a namespace derived from the literal shadow directory. + struct Route + { + Cas::RootNamespace ns{""}; + /// empty => the path is the namespace's container dir. For a detached part this is + /// `detached/` (a ref inside the table namespace, not a separate namespace). + std::string ref; + std::string file; /// empty => the path is the part dir itself + + /// The (ns, ref) identity subset — what the part-folder access layer keys on. + Cas::PartRefKey refKey() const { return {ns, ref}; } + }; + /// Converts a parsed path into the namespace/reference/file tuple used by the part-folder + /// facade. Returns nullopt only when the parsed path cannot be routed. + std::optional route(const Cas::PartFilePath & p) const; + + /// Returns full `detached/` reference names in a namespace. + std::vector detachedRefNames(const Cas::RootNamespace & ns) const; + + /// Returns full `moving/` staging-reference names in a namespace. Move recovery enumerates + /// these names and removes entries left by an interrupted move. + std::vector movingRefNames(const Cas::RootNamespace & ns) const; + + /// `existsDirectory` and `listDirectory` use one fixed dispatch order to route a path through + /// (shadow -> atomic-shard -> table-uuid -> part -> subdir -> generic), previously implemented + /// twice and kept in sync by hand. `classifyDirectory` (private, below) computes it once; both + /// callers then switch on the resulting shape. `DirShape` and `DirRoute` remain public only so + /// `classifyDirectoryForTest` can expose the classification to wiring tests; the logic stays + /// private. + enum class DirShape + { + ShadowPart, + ShadowTable, + ShadowIntermediate, + AtomicShard, + TableDir, + DetachedContainer, + MovingContainer, + PartDir, + ProjectionDir, + TableSubdir, + GenericIntermediate, + }; + + struct DirRoute + { + /// Defaulted so a future classifyDirectory return path that forgets to set it fails safe + /// (a defined shape) instead of switching on an indeterminate enum (UB). Matches the + /// existing unreachable-fallthrough choice at the bottom of existsDirectory/listDirectory. + DirShape shape = DirShape::GenericIntermediate; + std::optional p; + std::optional r; + std::optional uuid; + std::optional tf; + std::optional projection_prefix; + }; + + /// Test-only accessor exposing the private directory classification so wiring tests can pin the + /// dispatch order directly. + DirRoute classifyDirectoryForTest(const std::string & path) const { return classifyDirectory(path); } + + /// Test seams for the EMPTY-PROOF RULE (Task 9, spec §1 [B3]). The counter is bumped on every + /// authoritative pool-identity probe the empty-proof issues, so a test can assert it fires EXACTLY + /// once per EMPTY `TableDir`/`DetachedContainer` enumeration and NEVER on the non-empty hot path, + /// a deeper part-dir, or a terminal (`Vanished`) pool. `setEmptyProofProbeOverrideForTest` replaces + /// the real backend probe so a test can inject a transport/permission fault (`Indeterminate`/ + /// `AccessDenied`) deterministically -- the storage builds its own `ObjectStorageBackend` internally, + /// so a backend decorator cannot reach it otherwise. Both are inert in production. + uint64_t emptyProofProbeCountForTest() const { return empty_proof_probe_count_for_test.load(); } + void resetEmptyProofProbeCountForTest() { empty_proof_probe_count_for_test.store(0); } + void setEmptyProofProbeOverrideForTest(std::function fn) + { + empty_proof_probe_override_for_test = std::move(fn); + } + + /// Test-only seam (inert in production): invoked at each manual GC verb's FORGET-race juncture so a test + /// can deterministically interleave a concurrent FORGET. For the round verbs + /// (`runOneGcRoundForTest`/`runGarbageCollectionRoundNow`) it fires PRE-lock, in the admission->lock + /// TOCTOU window (I-1: the call is admitted while `Live`, parks here until FORGET drives the pool + /// `Vanished`, then hits the under-lock re-check). For `runGcRebuildNow` it fires WHILE the rebuild HOLDS + /// `gc_scheduler_mutex` (I-2: the in-flight window a concurrent FORGET must serialize behind). Empty by + /// default; production installs none. + void setGcVerbAdmitWindowHookForTest(std::function fn) { gc_verb_admit_window_hook_for_test = std::move(fn); } + + /// Test-only fault-injection/hook seam for `ContentAddressedTransaction::publishStaging`'s + /// promote/repoint call, keyed by the full `(ns, ref)` routed identity via `PartRefKey::cacheKey()` + /// (mirrors `CasRefLedger::setRefPreCarveHookForTest`'s no-op-in-production shape) -- a bare ref + /// name would misfire across a future multi-namespace fixture where two namespaces coincidentally + /// share a ref name. `armPromoteFailureForTest` makes the NEXT `promoteBuild`/`repointRef` call for + /// `key` throw instead of committing, modeling a transient promote-time backend failure. + /// `setAfterPromoteHookForTest` installs a one-shot callback run synchronously immediately after a + /// successful promote/repoint for `key` (before the caller's own post-commit bookkeeping), modeling + /// a concurrent writer racing in right after this transaction's confirm -- e.g. repointing the same + /// ref to a different manifest so a later rollback's `dropRefIfMatches` must see it changed. + void armPromoteFailureForTest(const Cas::PartRefKey & key) { promote_failure_refs_for_test.insert(key.cacheKey()); } + bool shouldFailPromoteForTest(const Cas::PartRefKey & key) const { return promote_failure_refs_for_test.contains(key.cacheKey()); } + void setAfterPromoteHookForTest(const Cas::PartRefKey & key, std::function hook) + { + after_promote_hooks_for_test[key.cacheKey()] = std::move(hook); + } + /// Invokes and removes `key`'s one-shot hook, if any registered. A no-op when none is installed + /// (the production hot path never installs one). + void runAfterPromoteHookForTest(const Cas::PartRefKey & key) + { + auto it = after_promote_hooks_for_test.find(key.cacheKey()); + if (it == after_promote_hooks_for_test.end()) + return; + auto hook = std::move(it->second); + after_promote_hooks_for_test.erase(it); + hook(); + } + +private: + const ObjectStoragePtr object_storage; + const std::string storage_path_prefix; + const std::string storage_path_full; + const std::string server_id; + const std::string server_root_id; + const std::string disk_name; + const std::string local_scratch_path; + const ContextPtr context; + + const bool gc_enabled; + const std::chrono::seconds gc_interval; + const uint64_t deduplication_cache_bytes; /// P1 known-present cache byte cap (0=off) + const uint64_t deduplication_head_first_min_bytes; /// P2 HEAD-before-PUT size threshold (0=off) + const uint64_t gc_snapshot_generations_to_keep; /// Number of GC snapshots retained (0 means keep all). + const uint64_t gc_shards; /// Blob-hash-prefix reducer shard count, fixed at pool creation. + const uint64_t manifest_sweep_list_budget_keys; + const uint64_t manifest_sweep_delete_budget_keys; + const uint64_t gc_round_graduation_budget; + const uint64_t gc_round_redelete_budget; + const uint64_t gc_round_sweep_namespace_budget; + const uint64_t gc_round_sweep_recovery_op_budget; + const uint64_t gc_round_ref_cleanup_budget; + const uint64_t gc_round_prefix_wholesale_budget; + const uint64_t gc_round_handoff_prefix_wholesale_budget; + const uint64_t gc_round_outcome_entry_budget; + /// GCS single-PUT budget for conditional writes (generation-token stores only): threaded into + /// the ObjectStorageBackend construction site in startup(). Irrelevant on ETag stores (AWS et al). + const uint64_t gcs_max_conditional_put_bytes; + /// Part-folder view cache settings. `cas_part_folder_cache_bytes == 0` disables retention. + const uint64_t cas_part_folder_cache_bytes; + const uint64_t cas_part_folder_cache_max_entries; + const uint64_t cas_part_folder_cache_max_entry_bytes; + /// Byte bound for the manifest decode cache in `Cas::Pool`. Zero disables decode caching. + const uint64_t manifest_decode_cache_bytes; + /// Bounded pool size for GC's per-hash freshness-metadata writes. + const uint64_t gc_meta_pool_size; + /// Configured staging backend; `Local` preserves the existing write path. + const Cas::StagingBackend staging_backend; + /// Blob content-hash function passed to `Cas::PoolConfig`. + const Cas::BlobHashAlgo blob_hash_algo; + /// Whether `blob_hash_algo` may be admitted into the pool's persisted `algos_used` set. + const bool blob_hash_allow_new; + /// Per-disk `` policy passed to `Cas::PoolConfig`. + const bool skip_access_check; + /// Policy controlling when retained part-folder views revalidate their manifest body. + const Cas::PartFolderValidate part_folder_validate; + /// Set by the mount-time conditional-copy capability probe — not const because the result is + /// unavailable until startup. + /// Defaults to false (fail-close): assumed unsupported until the probe proves otherwise. + bool conditional_copy_supported = false; + + /// A single coherent snapshot of the pool and its cached part-folder facade, taken under ONE + /// `pointer_mutex` acquisition (see `poolAccess()`) so no caller can observe `pool` from one mount + /// generation and `part_access` from another -- the two used to be fetched by two separate calls + /// to `store()`/`partAccess()` at some call sites, each taking its own `pointer_mutex` snapshot. + struct PoolAccessSnapshot + { + Cas::PoolPtr pool; + std::shared_ptr part_access; + }; + + /// Set by startup (Pool::open is fail-closed; empty store == not started). shared_ptr so + /// store()/partAccess() can return a by-value snapshot under `pointer_mutex` (see below) instead + /// of a reference that could dangle across a concurrent `shutdown` reset. + Cas::PoolPtr cas_store TSA_GUARDED_BY(pointer_mutex); + /// The part-folder access facade: the normal path + /// for committed part/projection reads and committed part-ref mutations. Constructed in + /// startup right after Pool::open; reset in shutdown before cas_store. shared_ptr for the same + /// snapshot-safety reason as cas_store. + std::shared_ptr part_access TSA_GUARDED_BY(pointer_mutex); + String pool_uuid; + /// shared_ptr so `runGarbageCollectionRoundNow`/`runOneGcRoundForTest` can take a snapshot under + /// `pointer_mutex`, release it, and run the (long) round via the snapshot -- never holding + /// `pointer_mutex` itself for the round's duration, so `gcHealth`/`store`/`partAccess` never + /// block behind an in-flight round. + std::shared_ptr gc_scheduler TSA_GUARDED_BY(pointer_mutex); + /// Outermost lock, taken by the lifecycle-control verbs: the FSCK handler, `forgetDisk`, and + /// `gcStop`/`gcStart`. Serializes them against each other. Lock order when nested locks are needed: + /// `lifecycle_mutex` -> `gc_scheduler_mutex` -> `pointer_mutex`, never the reverse. + mutable std::mutex lifecycle_mutex; + /// Serializes ONE synchronous GC round at a time and makes `shutdown` wait for an in-flight round + /// to finish cleanly (clean GC completion has priority over fast shutdown) -- held for the WHOLE + /// round. Deliberately NOT the same mutex as `pointer_mutex` below: this one can be held for a + /// long time, so nothing that only needs a brief pointer snapshot may share it. + mutable std::mutex gc_scheduler_mutex; + bool shutdown_called TSA_GUARDED_BY(gc_scheduler_mutex) = false; + /// Guards ONLY reads/writes of `cas_store`/`part_access`/`gc_scheduler` themselves + /// (snapshot, create-if-absent, reset) -- always held briefly. Lock ordering when more than one of + /// these is needed (the round entry points, `shutdown`, and -- outermost -- `lifecycle_mutex`): + /// `lifecycle_mutex` first (if held at all), then `gc_scheduler_mutex`, then `pointer_mutex` + /// nested inside, never the reverse. + mutable std::mutex pointer_mutex; + /// Derived from object_storage->isReadOnly() at startup (the disk's config). When set: + /// the probe is skipped, no watermark, no GC scheduler, and the mutating surface fails closed. + bool read_only = false; + /// Joined in front of core keys for DIRECT object_storage reads. The Emulated (Local) backend + /// maps bare pool keys under getCommonKeyPrefix; Native passes keys through - this member + /// mirrors that rule so readBlobPayload reads exactly where the backend wrote ("" for Native). + String physical_key_prefix; + + /// Adds the local-backend common prefix to a pool key when direct object-storage I/O requires + /// the physical key; native backends use the key unchanged. + String physicalKey(const String & key) const + { + if (physical_key_prefix.empty()) + return key; + if (physical_key_prefix.back() == '/') + return physical_key_prefix + key; + return physical_key_prefix + "/" + key; + } + + /// The one place that takes a `{pool, facade}` snapshot under a SINGLE `pointer_mutex` + /// acquisition. Throws `INVALID_STATE` (via `throwStorageNotStarted`) when no pool is published + /// (before the first `startup` or after `shutdown` -- the storage-level Constructing/ShutDown + /// lifecycle), then refuses a terminal pool via `throwIfLifecycleTerminal`. `store()`/`partAccess()` + /// are thin wrappers over this; every other caller that needs BOTH the pool and the facade for one + /// logical operation must call this ONCE and use both fields from the same snapshot, rather than + /// calling `store()` and `partAccess()` separately -- otherwise it could straddle a startup/shutdown + /// that changes `cas_store`/`part_access`. + PoolAccessSnapshot poolAccess() const; + + /// Builds and throws the `INVALID_STATE` "disk is not started" exception `poolAccess()`, the gate, + /// and the synchronous GC round entry points throw when no pool is published -- the storage-level + /// Constructing (before `startup`) / ShutDown (after `shutdown`) lifecycle. `pool_uuid` (written once + /// at the end of a successful `startup`, never reset) distinguishes the two in the message, exactly + /// as `lifecycleSnapshot()` reports `constructing`/`shutdown`. A normal, operator-facing refusal, + /// never a `LOGICAL_ERROR` (which would abort under debug/ASan builds). + [[noreturn]] void throwStorageNotStarted() const; + + /// EMPTY-PROOF RULE (rev.7 spec §1 [B3]): called by `listDirectory` when a `TableDir`/ + /// `DetachedContainer` enumeration on a NON-terminal (Live or read-only) pool is about to answer + /// empty. "Empty at a table root" is exactly what a silently-erased backing looks like, and a + /// read-only pool has no lease/observer to detect that erasure any other way (MergeTree skips both + /// directory creation and the `format_version.txt` write on a read-only disk) -- enumeration is its + /// ONLY line of defense against ATTACHing an empty table over an erased pool. So before answering + /// empty, this confirms the pool identity object (`_pool_meta`) with an AUTHORITATIVE, UNCACHED + /// probe: `Present` authorizes the empty answer (the pool is genuinely there and genuinely empty); + /// `KeyAbsent`/`ContainerAbsent` throw the typed 668 "backing may be erased"; `AccessDenied`/ + /// `Indeterminate` throw the typed transient 668 (fail-closed, retryable). NEVER reached on a + /// `Vanished` pool -- `checkOpAdmitted`'s `Probe`->`TruthAbsent` short-circuit answers truth-empty + /// before any classification runs, so the terminal path never pays the probe. Cost: one extra backend + /// HEAD per EMPTY table-root enumeration only (attach/load-time); the non-empty hot path is untouched. + void confirmPoolIdentityForEmptyEnumeration(const std::string & path) const; + + /// A pool opened as a standalone, UNPUBLISHED view: never touches `cas_store`/`part_access`/ + /// `gc_scheduler` -- the caller owns it entirely and drops it when done. + struct PoolView + { + Cas::PoolPtr pool; + /// The direct-object-storage key prefix for this view's backend (see `physicalKey`'s own + /// doc comment): populated for the Emulated (Local) backend, empty ("") for Native. Returned + /// rather than written to the `physical_key_prefix` member so this helper stays side-effect-free + /// and callable from a `const` method (`runFsckNow`). + String physical_key_prefix; + /// The resolved (bucket-relative, trailing-slash-trimmed) pool prefix passed into + /// `Cas::PoolConfig::pool_prefix` -- `startup()`'s S3-staging capability probe below needs the + /// SAME resolved value to build its own probe key, so it is returned here rather than + /// recomputed a second time. + String pool_prefix; + }; + + /// Builds the backend + `Cas::PoolConfig` and opens a pool exactly as `startup()` does. A read-only + /// (``) disk opens with no write probe, no background watermark, and no GC scheduler. Never + /// touches `cas_store`/`part_access`/`gc_scheduler`/`physical_key_prefix`/`pool_uuid`/ + /// `conditional_copy_supported`; `startup()` applies its own result to those members itself, in its + /// single publish step. + PoolView openPoolView() const; + + /// Classifies `path`'s directory shape by running the fixed dispatch order once (shadow -> + /// atomic-shard -> table-uuid -> part -> subdir -> generic), including the part-branch + /// fall-through when no sub-shape matches. Pure path classification — consults no lifecycle + /// state (e.g. `readableNamespaceFilesLife`); `existsDirectory`/`listDirectory` apply that gate + /// themselves in their per-shape arms, exactly as before this refactor. + DirRoute classifyDirectory(const std::string & path) const; + + /// Build the GC round sink: the std::function the scheduler calls per Start/Finish. Captures the + /// ContextPtr, converts the POD GcRoundLogRecord into a ContentAddressedGarbageCollectionLogElement, + /// and appends it to the SystemLog (best-effort). Returns an empty sink when context is null. + Cas::GcRoundLogger makeGcRoundLogger() const; + + /// Builds the per-event CAS audit sink: the `std::function` the pool calls on every + /// content-addressed decision. Captures the ContextPtr, converts the decoupled Core POD + /// `Cas::CasEvent` into a ContentAddressedLogElement, and appends it to the SystemLog + /// (best-effort). Returns an empty sink when context is null (unit tests). + Cas::CasEventSink makeCasEventSink() const; + + /// Backing state for the `*ForTest` promote fault-injection/hook seam declared above. Empty in + /// production (no test ever arms them); consulted only by `ContentAddressedTransaction::publishStaging`. + std::unordered_set promote_failure_refs_for_test; + std::unordered_map> after_promote_hooks_for_test; + + /// Backing state for the EMPTY-PROOF RULE `*ForTest` seams (Task 9), declared above. The counter is + /// bumped on every empty-proof probe; the override, when set, replaces the real backend probe. Both + /// are inert in production (no test ever sets the override; the counter is write-only there). + mutable std::atomic empty_proof_probe_count_for_test{0}; + std::function empty_proof_probe_override_for_test; + + /// Backing state for the `setGcVerbAdmitWindowHookForTest` seam declared above (the I-1/I-2 admission + /// TOCTOU tests). Empty in production; a `const` GC verb reads it and calls the const-qualified + /// `std::function::operator()`, so it needs no `mutable`. + std::function gc_verb_admit_window_hook_for_test; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp new file mode 100644 index 000000000000..a4112fd57cb3 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp @@ -0,0 +1,214 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int NO_ELEMENTS_IN_CONFIG; +} + +/// The `cas` disk block is shared with the generic object-storage/disk layer, whose +/// keys are consumed elsewhere (`ObjectStorageFactory`, `S3Settings`, `MetadataStorageFactory`'s +/// `getObjectKeyCompatiblePrefix`, `IDisk`, `DiskFromAST` for the inline SQL `disk(...)` form, +/// `RegisterDiskObjectStorage`'s fake-transaction gate) and must be skipped here rather than rejected +/// as unknown. As of 2026-07-21 this list covers ALL in-repo CAS disk configs and inline `disk(...)` +/// definitions, enumerated from four sources, each filtered down to the keys that are direct +/// children of an actual `metadata_type=cas` disk block (the raw `rg -o` output below +/// also contains disk-name and policy/volume wrapper tags, which are not config keys at all): +/// 1) `rg -o "<([a-z_0-9]+)>" -r '$1' utils/ca-soak/configs/storage_conf*.xml utils/ca-soak/configs/storage_overrides*.xml` +/// 2) every CAS integration-test disk config, both the `storage_conf.xml` bodies and the +/// per-node `server_root_id_node*.xml` overrides (`git grep -l content_addressed +/// tests/integration | grep -E '\.xml$'`) -- and every integration `test.py` +/// (`git ls-files 'tests/integration/*/test.py' | xargs grep -l +/// 'metadata_type.*content_addressed'`) confirmed EMPTY: no integration test builds a CAS disk +/// via inline SQL `disk(...)`, only via these XML configs. +/// 3) every CAS disk config under `tests/config/config.d/cas_*.xml` (the +/// stateless-lane XML configs) -- these are the only place in the tree with a LOCAL +/// `object_storage_type` CAS disk, so they are the only source of the generic `path` key below. +/// 4) every inline `disk(...)` SQL construct across ALL `cas_*` +/// stateless tests, `04278`-`04300` (pre-dating this settings struct) through `05002`-`05015` +/// (current) -- these supply `name` (read by `DiskFromAST` to derive the ad-hoc disk's name) +/// and `use_fake_transaction` (validated generically in `RegisterDiskObjectStorage.cpp` against +/// EVERY metadata type that needs a real transaction, not a CAS-specific check -- exercised by +/// `05015_cas_reject_fake_transaction` deliberately setting it to assert the REJECTION, which +/// needs the key to reach that check rather than being rejected earlier as unknown). The +/// `04278`-`04300` range turned up no keys beyond what `05002`-`05015` already required. +/// Any new CAS config FAMILY -- a new XML config directory or a new inline-`disk()` test pattern -- +/// added to the tree needs the same four-way scan repeated against it and this note updated. +/// `skip_access_check` is deliberately NOT in this set: it is registered as a CAS setting +/// below (the same config key also has meaning to `IDisk::startupImpl`, which drops it before +/// `metadata_storage->startup()` runs, but that does not make it foreign here). +static const std::set non_cas_keys = { + "type", "object_storage_type", "metadata_type", "path", "name", "use_fake_transaction", + "endpoint", "access_key_id", "secret_access_key", "region", "use_environment_credentials", + "readonly", "expect_continue_min_bytes", "http_client", "key_compatibility_prefix", +}; + +/// Config-key convention: the disk block already scopes every key to this disk, so no +/// key below carries a redundant `cas_`/`ca_` prefix (e.g. `part_folder_cache_bytes`, not +/// `cas_part_folder_cache_bytes`). +#define LIST_OF_CONTENT_ADDRESSED_SETTINGS(DECLARE, ALIAS) \ + DECLARE(String, scratch_path, "", "Server-local scratch dir for the write-buffer spill; a relative value is anchored to the server data path", 0) \ + DECLARE(Bool, gc_enabled, true, "Run the background GC scheduler on this disk", 0) \ + DECLARE(UInt64, gc_interval_sec, 60, "Seconds between background GC rounds (>= 1)", 0) \ + DECLARE(String, blob_hash, "cityhash128", "Pool blob content-hash function (cityhash128 | xxh3-128 | sha256); fixed at pool creation", 0) \ + DECLARE(Bool, blob_hash_allow_new, false, "Explicit opt-in to admit a NEW hash algo into an existing pool's algos_used", 0) \ + DECLARE(Bool, skip_access_check, false, "Skip the boot-time capability probe (start now, fix later)", 0) \ + DECLARE(UInt64, deduplication_cache_bytes, 64ULL << 20, "Byte budget of the blob presence cache (0 disables)", 0) \ + DECLARE(UInt64, deduplication_head_first_min_bytes, 1ULL << 20, "Minimum blob size to try a HEAD before uploading the body", 0) \ + DECLARE(UInt64, gc_snapshot_generations_to_keep, 3, "GC snapshot generations retained", 0) \ + DECLARE(UInt64, gc_shards, 1, "Blob-hash-prefix reducer shards (>= 1); creation-time only", 0) \ + DECLARE(UInt64, manifest_sweep_list_budget_keys, 1000, "Orphan-manifest sweep LIST budget per round", 0) \ + DECLARE(UInt64, manifest_sweep_delete_budget_keys, 100, "Orphan-manifest sweep DELETE budget per round", 0) \ + DECLARE(UInt64, gc_round_graduation_budget, 5000, "Blob graduation (condemned -> delete_pending) cohort cap per round (0 = unbounded)", 0) \ + DECLARE(UInt64, gc_round_redelete_budget, 5000, "Blob redelete (exact-token delete of a prior delete_pending row) cohort cap per round (0 = unbounded)", 0) \ + DECLARE(UInt64, gc_round_sweep_namespace_budget, 20, "Orphan-manifest sweep: distinct namespaces per page whose protection view may be built (0 = unbounded)", 0) \ + DECLARE(UInt64, gc_round_sweep_recovery_op_budget, 5000, "Orphan-manifest sweep: committed-tail ref-log GET/decode ops the recovery walk may spend per round (0 = unbounded)", 0) \ + DECLARE(UInt64, gc_round_ref_cleanup_budget, 5000, "Ref-object cleanup (covered log/snapshot deletes) cap per round (0 = unbounded)", 0) \ + DECLARE(UInt64, gc_round_prefix_wholesale_budget, 20000, "Generation-prefix wholesale delete (prune only) object cap per round (0 = unbounded)", 0) \ + DECLARE(UInt64, gc_round_handoff_prefix_wholesale_budget, 5000, "Post-CAS hand-off generation-prefix reclaim object cap per round, reserved separately from gc_round_prefix_wholesale_budget so a prune-heavy round cannot starve the one-shot hand-off (0 = unbounded)", 0) \ + DECLARE(UInt64, gc_round_outcome_entry_budget, 5000, "GcOutcomes per-round entry cap across the redelete/spared audit log (0 = unbounded)", 0) \ + DECLARE(String, server_root_id, "", "REQUIRED explicit layout subtree identity; macros expand as in the s3 endpoint", 0) \ + DECLARE(UInt64, gcs_max_conditional_put_bytes, 1ULL << 30, "GCS single-PUT budget for conditional writes (generation-token stores only)", 0) \ + DECLARE(UInt64, part_folder_cache_bytes, 64ULL << 20, "Part-folder view cache byte budget (0 disables retention)", 0) \ + DECLARE(UInt64, part_folder_cache_max_entries, 10000, "Part-folder view cache entry cap", 0) \ + DECLARE(UInt64, part_folder_cache_max_entry_bytes, 16ULL << 20, "Oversized part-folder views bypass retention above this size", 0) \ + DECLARE(String, part_folder_validate, "always", "ForceFresh body re-proof policy (always | never | age )", 0) \ + DECLARE(UInt64, manifest_decode_cache_bytes, 128ULL << 20, "Manifest DECODE cache byte budget (0 disables)", 0) \ + DECLARE(UInt64, gc_meta_pool_size, 16, "Bounded pool size for GC per-hash freshness-meta writes", 0) \ + DECLARE(String, staging_backend, "local", "Blob staging backend (local | s3); s3 is opt-in", 0) \ + +DECLARE_SETTINGS_TRAITS(ContentAddressedSettingsTraits, LIST_OF_CONTENT_ADDRESSED_SETTINGS, CONTENT_ADDRESSED_SETTINGS_SUPPORTED_TYPES) + +struct ContentAddressedSettingsImpl : public BaseSettings +{ + /// Parsed by `validate` from the corresponding string setting; cached here (rather than + /// re-parsed on every access) because the public header only forward-declares + /// `Cas::StagingBackend` / `Cas::PartFolderValidate` and cannot store them by value. + Cas::BlobHashAlgo blob_hash_algo_cached = Cas::BlobHashAlgo::CityHash128; + Cas::StagingBackend staging_backend_cached = Cas::StagingBackend::Local; + Cas::PartFolderValidate part_folder_validate_cached{}; +}; + +IMPLEMENT_SETTINGS_TRAITS_CUSTOM_IMPL(ContentAddressedSettingsTraits, LIST_OF_CONTENT_ADDRESSED_SETTINGS, ContentAddressedSettings, ContentAddressedSetting) + +ContentAddressedSettings::ContentAddressedSettings() : impl(std::make_unique()) +{ +} + +ContentAddressedSettings::~ContentAddressedSettings() = default; + +CONTENT_ADDRESSED_SETTINGS_SUPPORTED_TYPES(ContentAddressedSettings, IMPLEMENT_SETTING_SUBSCRIPT_OPERATOR) + +ContentAddressedSettings::ContentAddressedSettings(const ContentAddressedSettings & settings) + : impl(std::make_unique(*settings.impl)) +{ +} + +void ContentAddressedSettings::loadFromConfig( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const std::string & scratch_path_anchor_if_relative, + const std::string & default_scratch_path, + const MacroExpander & expand_macros) +{ + Poco::Util::AbstractConfiguration::Keys config_keys; + config.keys(config_prefix, config_keys); + + for (const std::string & key : config_keys) + { + if (non_cas_keys.contains(key)) + continue; + impl->set(key, config.getString(config_prefix + "." + key)); + } + + auto & settings = *this; + + /// Server-local scratch dir for the write-buffer spill. Mirrors how other metadata storages + /// compute their local working dir: a real filesystem path, NEVER the object-storage key + /// prefix. A configured RELATIVE scratch path is anchored to `scratch_path_anchor_if_relative` + /// (the caller-provided server data path), NOT the process CWD (which varies by launch method) + /// and NOT `default_scratch_path` -- that default is itself a per-disk subdirectory of the + /// server data path (`.../disks//cas_scratch/`), so anchoring a relative override to it + /// instead of to the server data path directly would silently nest the override two levels + /// deeper than intended (review finding: this is exactly the pre-existing factory's anchor, + /// which callers already depend on in shipped configs). The default is already absolute; only + /// an explicit relative override needs anchoring, and only to the server-data-path anchor. + if (settings[ContentAddressedSetting::scratch_path].changed) + { + if (fs::path(settings[ContentAddressedSetting::scratch_path].value).is_relative()) + settings[ContentAddressedSetting::scratch_path] = (fs::path(scratch_path_anchor_if_relative) / settings[ContentAddressedSetting::scratch_path].value).string(); + } + else + { + settings[ContentAddressedSetting::scratch_path] = default_scratch_path; + } + + /// Phase 0 (mount safety): macros expand here exactly as in the s3 `endpoint` + /// (`ObjectStorageFactory`): on a multi-replica stand every replica mounts ONE shared pool + /// (same endpoint) and must own a DISTINCT subtree, so the natural single-template config is + /// `{replica}`. An unknown macro throws (fail closed, via the + /// caller-supplied `expand_macros`). Gated on `.changed`: assigning unconditionally would mark the + /// field changed even when the key was ABSENT from config, defeating `validate`'s `.changed` check + /// below (the ABSENT-vs-invalid `NO_ELEMENTS_IN_CONFIG`-vs-`BAD_ARGUMENTS` distinction) -- a missing + /// key must reach `validate` still unchanged, not silently expanded-in-place to the same empty string. + if (settings[ContentAddressedSetting::server_root_id].changed) + settings[ContentAddressedSetting::server_root_id] = expand_macros(settings[ContentAddressedSetting::server_root_id].value); + + validate(); +} + +void ContentAddressedSettings::validate() +{ + auto & settings = *this; + + if (settings[ContentAddressedSetting::gc_interval_sec] == 0 || settings[ContentAddressedSetting::gc_shards] == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "content_addressed disk: gc_interval_sec and gc_shards must be >= 1 (got {}, {})", + settings[ContentAddressedSetting::gc_interval_sec].value, settings[ContentAddressedSetting::gc_shards].value); + + /// The layout subtree identity is explicit and REQUIRED — no default, so an ABSENT key throws a + /// typed `NO_ELEMENTS_IN_CONFIG` (mirroring the `metadata_type` check in `MetadataStorageFactory`), + /// distinct from a PRESENT-but-invalid value, which falls through to `validateServerRootId`'s + /// `BAD_ARGUMENTS` below. `.changed` is exactly "the config had this key" (or a caller set it + /// explicitly via the subscript operator); an unset field never reaches here as anything but empty. + if (!settings[ContentAddressedSetting::server_root_id].changed) + throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, + "Expected `server_root_id` in config for a content-addressed disk"); + + Cas::validateServerRootId(settings[ContentAddressedSetting::server_root_id].value); + + impl->blob_hash_algo_cached = Cas::parseBlobHashAlgo(settings[ContentAddressedSetting::blob_hash].value); + impl->staging_backend_cached = ContentAddressedMetadataStorage::parseStagingBackend(settings[ContentAddressedSetting::staging_backend].value); + impl->part_folder_validate_cached = ContentAddressedMetadataStorage::parsePartFolderValidate(settings[ContentAddressedSetting::part_folder_validate].value); +} + +Cas::BlobHashAlgo ContentAddressedSettings::blobHashAlgo() const +{ + return impl->blob_hash_algo_cached; +} + +Cas::StagingBackend ContentAddressedSettings::stagingBackend() const +{ + return impl->staging_backend_cached; +} + +Cas::PartFolderValidate ContentAddressedSettings::partFolderValidate() const +{ + return impl->part_folder_validate_cached; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.h new file mode 100644 index 000000000000..09f189c0d9e2 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.h @@ -0,0 +1,89 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace Poco { namespace Util { class AbstractConfiguration; } } // NOLINT(cppcoreguidelines-virtual-class-destructor) + +namespace DB::Cas +{ +/// Forward declared to keep this header light: the full definitions live in +/// `ContentAddressedMetadataStorage.h` (`StagingBackend`) and `Parts/PartFolderAccess.h` +/// (`PartFolderValidate`), which are heavy and — in `ContentAddressedMetadataStorage.h`'s +/// case — will itself include this header once the metadata storage is rewired onto it. +/// Both are legal opaque declarations: `StagingBackend` fixes no explicit underlying type +/// (matching its definition, which leaves it as the implicit `int`), and `PartFolderValidate` +/// is only ever used here as an incomplete-type function return, never stored by value. +enum class StagingBackend; +struct PartFolderValidate; +} + +namespace DB +{ +struct ContentAddressedSettingsImpl; + +/// Resolves `{macro}` placeholders in a config value, e.g. `server_root_id`. Kept as a +/// type-erased callback (rather than a `Context`/`Macros` reference) so this header stays free +/// of `Interpreters/Context` — settings loading has no business knowing about the query context. +using MacroExpander = std::function; + +#define CONTENT_ADDRESSED_SETTINGS_SUPPORTED_TYPES(CLASS_NAME, M) \ + M(CLASS_NAME, String) \ + M(CLASS_NAME, Bool) \ + M(CLASS_NAME, UInt64) + +CONTENT_ADDRESSED_SETTINGS_SUPPORTED_TYPES(ContentAddressedSettings, DECLARE_SETTING_TRAIT) + +/// Declarative settings for the `cas` disk metadata storage, mirroring the +/// `FileCacheSettings` pimpl/traits shape (`Core/BaseSettings.h`). Replaces the ~25 inline +/// `config.getX` calls that used to live in `MetadataStorageFactory.cpp`'s +/// `registerContentAddressedMetadataStorage` lambda; that lambda's key names, defaults, and +/// per-key rationale are the authoritative source for `LIST_OF_CONTENT_ADDRESSED_SETTINGS`. +struct ContentAddressedSettings +{ + ContentAddressedSettings(); + ContentAddressedSettings(const ContentAddressedSettings &); + ~ContentAddressedSettings(); + + CONTENT_ADDRESSED_SETTINGS_SUPPORTED_TYPES(ContentAddressedSettings, DECLARE_SETTING_SUBSCRIPT_OPERATOR) + + /// Loads every key under `config_prefix`, rejecting unknown non-object-storage keys (fail + /// closed, mirrors `FileCacheSettings::loadFromConfig`'s `non_cache_keys` skip-set). A missing + /// `scratch_path` defaults to `default_scratch_path`; a relative OVERRIDE is anchored to + /// `scratch_path_anchor_if_relative` instead (never the process CWD, and never + /// `default_scratch_path` -- that default is a per-disk subdirectory, e.g. + /// `/disks//cas_scratch/`, and anchoring a relative override to it would + /// silently nest the override two levels deeper than the server data path the operator meant; + /// mirrors `FileCacheSettings::loadFromConfig`'s `cache_path_prefix_if_relative` / + /// `default_cache_path` split). `server_root_id` is passed through `expand_macros` before + /// `validate` runs. Ends by calling `validate`. + void loadFromConfig( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const std::string & scratch_path_anchor_if_relative, + const std::string & default_scratch_path, + const MacroExpander & expand_macros); + + /// Fail-closed checks: `gc_interval_sec` and `gc_shards` must both be >= 1; `server_root_id` must + /// be present (an ABSENT key throws a typed `NO_ELEMENTS_IN_CONFIG`, distinct from a + /// PRESENT-but-invalid value, which throws `Cas::validateServerRootId`'s `BAD_ARGUMENTS`); and the + /// three enum-valued string settings (`blob_hash`, `staging_backend`, `part_folder_validate`) must + /// parse. The parsed enum values are cached for the typed accessors below. + void validate(); + + /// Typed accessors for the enum-valued string settings, parsed and cached by `validate`. + Cas::BlobHashAlgo blobHashAlgo() const; + Cas::StagingBackend stagingBackend() const; + Cas::PartFolderValidate partFolderValidate() const; + +private: + /// The parsed enum values live inside `impl` (defined in the .cpp, where the forward-declared + /// `Cas::StagingBackend` / `Cas::PartFolderValidate` types are complete), not as members here. + std::unique_ptr impl; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.cpp new file mode 100644 index 000000000000..de00857ce488 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.cpp @@ -0,0 +1,1966 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event CASBlobUploadFanoutBatches; + extern const Event CASBlobUploadFanoutTasks; +} + +namespace fs = std::filesystem; + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ABORTED; + extern const int FILE_DOESNT_EXIST; + extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::Cas +{ + +namespace +{ + +bool hasSuffix(std::string_view s, std::string_view suffix) +{ + return s.size() >= suffix.size() && s.substr(s.size() - suffix.size()) == suffix; +} + +} + +bool partFileMustStayBlob(std::string_view file_name) +{ + if (file_name == "primary.idx") + return true; + for (std::string_view suffix : {".bin", ".mrk", ".mrk2", ".mrk3", ".cmrk", ".cmrk2", ".cmrk3"}) + if (hasSuffix(file_name, suffix)) + return true; + return false; +} + +} + +namespace DB +{ + +namespace +{ + +[[noreturn]] void notYet(const char * op) +{ + /// These operations are part of the generic disk-transaction interface but have no + /// content-addressed equivalent or are not wired for this storage yet. Keep the message + /// self-explanatory because it is visible to operators. + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "The operation '{}' is not implemented for a content-addressed disk: it belongs to the " + "generic disk-transaction surface that the content-addressed write path does not use. " + "Hitting it usually means the disk is wrapped by a layer that bypasses the " + "content-addressed write path.", op); +} + +/// Inline candidates above this size spill to a blob instead of riding the tree object — a tuning +/// knob (could become a disk setting later). Keeps the tree object bounded against an unexpectedly +/// large eager file. +constexpr size_t INLINE_CAP = 1024 * 1024; /// 1 MiB + +} + +ContentAddressedTransaction::ContentAddressedTransaction(ContentAddressedMetadataStorage & metadata_storage_) + : metadata_storage(metadata_storage_) +{ +} + +ContentAddressedTransaction::~ContentAddressedTransaction() +{ + /// Always clean up pending staging (whether committed or not). On the success path + /// cleanupPendingTempFiles was already called at the end of commit(); this call is the defensive + /// backstop for aborted/exception-unwound transactions whose publishStaging never ran. + cleanupPendingTempFiles(); + + /// An uncommitted transaction's uploads become min_active-spared debris: abandon every + /// still-open PartWriteTxn so its build_seq is retired. This replaces the former pin machinery. + if (committed) + return; + + /// No refs are published before `commit`; moving a part from a temporary to a final path is only + /// a re-key in this overlay. An abandoned transaction therefore has no early-published ref to + /// compensate for; it only needs to abandon still-open builds below. + for (auto & [key, st] : parts) + { + if (!st.build) + continue; + try + { + st.build->abandon(); + } + catch (...) + { + /// A destructor must not throw. But a failed abandon can leave a LIVE-epoch precommit + /// binding that neither GC nor the (prior-epoch-scoped) stale-precommit sweep reclaims + /// until this mount remounts -- that must be diagnosable, not silently swallowed. + tryLogCurrentException(getLogger("ContentAddressedTransaction"), + "abandoning a build during transaction destruction " + "(a live precommit binding may persist until remount)"); + } + } +} + +ContentAddressedTransaction::PartStaging & +ContentAddressedTransaction::stagingFor(const ContentAddressedMetadataStorage::Route & r) +{ + return parts[{r.ns.string(), r.ref}]; +} + +Cas::PartWriteTxn & ContentAddressedTransaction::buildFor( + const ContentAddressedMetadataStorage::Route & r, PartStaging & st) +{ + if (!st.build) + st.build = metadata_storage.store()->beginPartWrite( + Cas::PartWriteInfo{.intended_ref = r.ns.string() + "/" + r.ref, + .intended_namespace = r.ns, .op = Cas::ProvenanceOp::Insert}); + return *st.build; +} + +ContentAddressedTransaction::PartStaging * ContentAddressedTransaction::findStaging( + const ContentAddressedMetadataStorage::Route & r) +{ + auto it = parts.find({r.ns.string(), r.ref}); + return it == parts.end() ? nullptr : &it->second; +} + +void ContentAddressedTransaction::cleanupPendingTempFiles() noexcept +{ + for (auto & [key, st] : parts) + { + for (const auto & pb : st.pending_blobs) + { + if (pb.backend == Cas::StagingBackend::Local) + { + std::error_code ec; + std::filesystem::remove(pb.staging_key, ec); + } + else if (committed) + { + /// A successful commit deletes the S3 staging object + /// HERE — `committed` is only ever true when EVERY part's `publishStaging` ran to + /// completion (commit() sets it right before this call), which means every referenced + /// pending blob was already promoted (`PartWriteTxn::putBlob` → `promoteStaged`/`resurrect`) + /// or, for an orphaned pending blob (its entry removed by `unlinkFile`/`replaceFile` + /// before commit), was never going to be promoted at all — either way the staging object + /// is no longer needed as a resurrect source, so it is safe to reclaim now. + /// + /// An ABORTED/exception-unwound transaction (`committed == false`, including a partial + /// multi-part commit failure where an EARLIER part's blobs were already promoted) leaves + /// its S3 staging objects in place — `staging_key` is a remote object-storage key, never a + /// bare `fs::remove` target, and is reclaimed by the mount-lease-scoped sweeper + /// (`Cas::sweepOwnMountStaging`), never here. This mirrors the local path's own asymmetry: + /// `Local` staging is a private per-transaction scratch file removed unconditionally on + /// both commit and abort (nobody else can ever read it), whereas an `S3` staging object + /// is the sanctioned resurrect source for the promote gate and must outlive an aborted + /// transaction so a later attempt (or the sweeper) can still account for it. + try + { + metadata_storage.objectStorage()->removeObjectIfExists(StoredObject(pb.staging_key)); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// Best-effort (noexcept context): a stubborn delete just leaves debris for the + /// mount-lease sweeper to reclaim on a later mount. + } + } + /// else: an S3-mode pending blob of an ABORTED transaction — intentionally left in place + /// (see above); the mount-lease sweeper (`Cas::sweepOwnMountStaging`) is its reclaimer. + } + st.pending_blobs.clear(); + } +} + +const ContentAddressedTransaction::PartStaging::PendingBlob * +ContentAddressedTransaction::findPendingBlob(const PartStaging & st, const Cas::BlobRef & ref) const +{ + /// Locate a pending blob by ref. Returns nullptr when the blob has already been uploaded + /// (post-precommit, pending_blobs is cleared) or was never staged as pending. + for (const auto & pb : st.pending_blobs) + if (pb.ref == ref) + return &pb; + return nullptr; +} + +void ContentAddressedTransaction::adoptStagedBlob( + const PartStaging::PendingBlob * pb, const Cas::ManifestEntry & entry, + PartStaging & dst_st, Cas::PartWriteTxn & dst_build, bool copy_pending) +{ + if (pb) + { + /// Pending blob (not yet uploaded): record a tokenless dependency without any pool operation. + /// If copy_pending, push a copy of the pb record into dst_st so publishStaging uploads it + /// for the dst part too (hardlink = copy semantics). If !copy_pending, the record is already + /// in dst_st (moved by caller) — skip the push. + if (copy_pending) + dst_st.pending_blobs.push_back(*pb); + dst_build.recordPendingBlobDep(entry.ref, entry.blob_size); + } + else + { + /// Uploaded / committed: record a tokenless W-EVIDENCE dep — no pool HEAD/GET before precommit. + /// §4 manifest-trust: the publish gate (promote) TRUSTS this committed-source adopted leaf via the + /// durable manifest edge — it does NOT observe/resurrect it. Only tokened / pending-upload leaves + /// are resurrected (by putBlob, before promote); a genuinely-absent adopted blob is an fsck finding. + dst_build.adoptEvidence(entry); + } +} + +std::optional +ContentAddressedTransaction::routeOf(const std::string & path) const +{ + auto p = Cas::parsePartFilePath(path); + if (!p) + return std::nullopt; + return metadata_storage.route(*p); +} + +void ContentAddressedTransaction::uploadPendingBlobs(PartStaging & st) +{ + /// Build the set of blob hashes actually referenced by the staged manifest. Only Blob + /// entries represent pending content uploads — Inline are not pending blobs. A pending_blob whose + /// hash is NOT in this set had its entry removed by unlinkFile/replaceFile and must not be uploaded + /// (it is an orphan). Its temp file is still cleaned by cleanupPendingTempFiles at commit end. + std::unordered_set referenced_hashes; + for (const auto & entry : st.entries) + if (entry.placement == Cas::EntryPlacement::Blob) + referenced_hashes.insert(entry.ref); + + /// Build one upload request per referenced pending blob. Duplicate refs (staged-hardlink copies push + /// a copy of the record) are collapsed by `fanOutBlobUploads`' grouping, which SUBSUMES the former + /// duplicate-membership filter here — the fan-out launches one task per unique ref and merges one + /// dep. The upload primitive differs by staging backend exactly as before: + /// - `Cas::StagingBackend::Local`: `open` re-reads the local staged temp file and streams + /// it into a write-once `putIfAbsentStream` create; the local-staging path remains byte-for-byte + /// compatible with its previous behavior. + /// - `Cas::StagingBackend::S3`: the bytes already live in an S3 staging object (`pb.staging_key`); + /// `server_side_copy_from` drives a WRITE-ONCE conditional SERVER-SIDE COPY (and an unconditional + /// resurrect copy FROM the staging object for a condemned incarnation). No local read-back — + /// `open` is left unset. + std::vector requests; + requests.reserve(st.pending_blobs.size()); + for (const auto & pb : st.pending_blobs) + { + if (!referenced_hashes.contains(pb.ref)) + continue; /// The entry was removed by unlinkFile/replaceFile; skip this orphan. + Cas::BlobSource source; + source.size = pb.size; + if (pb.backend == Cas::StagingBackend::S3) + { + source.server_side_copy_from = pb.staging_key; + } + else + { + const std::string staging_key = pb.staging_key; + source.open = [staging_key]() -> std::unique_ptr + { + return std::make_unique(staging_key); + }; + } + /// `declared_size` mirrors `source.size` (both are `pb.size`); the fan-out fail-closes if they + /// ever diverge, so build them together from the one authority. + requests.push_back(Cas::BlobUploadRequest{pb.ref, std::move(source), pb.size}); + } + + if (requests.empty()) + return; + + /// Fan out the uploads on the server-wide pool (spec §1). The fan-out enforces + /// one-task-per-unique-ref grouping, the merge-nothing failure contract, and merges every result + /// into `st.build`'s dep set on THIS (the owning writer) thread after the join. + Cas::fanOutBlobUploads(*st.build, requests, Cas::blobUploadPool()); +} + +void ContentAddressedTransaction::publishStaging(const Cas::RootNamespace & ns, const std::string & ref, PartStaging & st, + std::optional & out_slot) +{ + if (st.published) + return; /// this staging was already published earlier in this commit loop — never re-publish + + if (!st.build && st.entries.empty() && st.content_removed.empty()) + { + /// Nothing staged for this ref this transaction -- a touched-but-empty PartStaging (e.g. the + /// harmless residue of a removeDirectory that already superseded this staging's marks, + /// content_removed cleared to empty). Benign no-op; `out_slot` stays `std::nullopt`. + st.published = true; + return; + } + + /// For committed-ref standalone writes and removal marks, `st.entries` holds only the + /// CHANGED/ADDED entries (see stageBlobPartFile / the inline writeFile path) — never the whole + /// part once the ref already exists; `st.content_removed` holds paths a same-transaction + /// unlinkFile staged for removal (§6). Carry every OTHER committed entry forward (minus any + /// content_removed path) and republish once via the repoint path, rather than letting the + /// PartWriteTxn path below replace the manifest with just the delta. That would either reject a + /// genuine content change or silently drop untouched files. This handles both sub-cases the + /// interface allows: entries staged + /// WITH a PartWriteTxn (this transaction uploaded new content) and WITHOUT one (a former mutable + /// per-part file that is now an ordinary tree entry, or a marks-only removal with no writes). + if (!st.entries.empty() || !st.content_removed.empty()) + { + if (auto view = metadata_storage.partAccess()->getView({ns, ref}, Cas::Freshness::ForceFresh)) + { + if (st.build) + { + /// EDGE-BEFORE-OBSERVE is still load- + /// bearing here: a fresh blob's hash must be durably NAMED by a live precommit's + /// manifest body before `putBlob` makes its first backend observation. `repointRef` + /// below promotes through its OWN internal build (`adoptEvidence`, no `putBlob`) — it + /// protects entries whose content ALREADY exists (the carried-forward ones, and this + /// transaction's uploads once they land), but cannot itself protect a brand-new upload + /// made mid-repoint. So THIS build stages+precommits a SCRATCH manifest over + /// `st.entries` (BEFORE it is merged/moved below) — exactly the same closure the normal + /// (non-repoint) path further down establishes with `st.build->stageManifest(st.entries)` + /// + `precommitAdd`, which already names every hash this transaction is about to upload + /// — purely to hold that edge across the upload loop. Once `repointRef`'s own promote + /// makes the real (merged) manifest live, this scratch precommit is abandoned; it never + /// gets promoted. A marks-only removal never enters this sub-block (`st.build` is null). + const Cas::ManifestId scratch_id = st.build->stageManifest(st.entries); + st.build->precommitAdd(ns, ref, scratch_id); + uploadPendingBlobs(st); + } + + std::vector merged; + for (const auto & e : view->manifest()->entries) + if (!st.content_removed.contains(e.path) + && std::none_of(st.entries.begin(), st.entries.end(), + [&](const Cas::ManifestEntry & s) { return s.path == e.path; })) + merged.push_back(e); + for (auto & s : st.entries) + merged.push_back(std::move(s)); + + /// Test-only fault seam (Task 3 TDD): simulate a promote-time backend failure for `ref` + /// right before the durable repoint call. A no-op in production (nothing ever arms it). + if (metadata_storage.shouldFailPromoteForTest({ns, ref})) + throw Exception(ErrorCodes::ABORTED, + "ContentAddressedTransaction: test-injected promote failure for {}/{}", ns.string(), ref); + + /// Capture the exact `CommitOutcome` IMMEDIATELY -- into the caller-provided slot, before + /// the scratch-build `abandon()` below (which can itself throw), and before the test-only + /// after-promote hook (which can run arbitrary test code) -- so a later throw in either + /// cannot lose it (Task 2/3's publish-before-any-throwable-post-commit-work ordering). + const Cas::CommitOutcome oc = metadata_storage.partAccess()->repointRef({ns, ref}, std::move(merged), Cas::ProvenanceOp::Other); + out_slot = oc; /// always created=false here: this block only runs once `view` already resolved + /// Test-only: models a concurrent writer racing in right after this transaction's own + /// confirm (e.g. repointing `ref` again). A no-op in production. + metadata_storage.runAfterPromoteHookForTest({ns, ref}); + if (st.build) + { + st.build->abandon(); /// scratch precommit's protecting job is done; the real manifest is live + st.build.reset(); /// never re-abandon this build from the destructor + } + st.published = true; + return; + } + } + + if (!st.build) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "ContentAddressedTransaction: staged entries or removal marks for {}/{} without a Build", ns.string(), ref); + + /// Write path (rev. 15): stage the part manifest body (mints a ManifestId), precommitAdd a + /// build-intent owner (closure now protected by reachability), upload the pending blobs, then + /// promote — an atomic owner move that revalidates every non-tokened blob fail-closed. + /// + /// ORDERING IS LOAD-BEARING (EDGE-BEFORE-OBSERVE): + /// precommitAdd's durable closure names EVERY blob hash BEFORE putBlob makes the first backend + /// observation. This is what lets promote skip re-validating tokened leaves (a condemnation in the + /// putBlob→promote window cannot graduate — the next fold sees the edge). Moving putBlob before + /// precommitAdd would adopt an incarnation with no protecting edge and + /// trips the EDGE-BEFORE-OBSERVE fail-closed throw in PartWriteTxn::observeAndAdmit; the TLA+ order + /// sabotage (Gate A) is the formal guard. + const Cas::ManifestId id = st.build->stageManifest(st.entries); + st.build->precommitAdd(ns, ref, id); + uploadPendingBlobs(st); + + /// Test-only fault seam (Task 3 TDD): simulate a promote-time backend failure for `ref` right + /// before the durable promote call. A no-op in production (nothing ever arms it). + if (metadata_storage.shouldFailPromoteForTest({ns, ref})) + throw Exception(ErrorCodes::ABORTED, + "ContentAddressedTransaction: test-injected promote failure for {}/{}", ns.string(), ref); + + /// The exact, in-lane-derived `created` from `promoteBuild` replaces the racy pre-check this used + /// to be (`existsRef` before promote, which a concurrent writer could invalidate in the window + /// before the promote's own append confirms). Captured into `out_slot` IMMEDIATELY -- before the + /// test-only after-promote hook below (which can run arbitrary test code) or `st.build.reset()` + /// -- so a later throw there cannot lose it. + const Cas::CommitOutcome oc = metadata_storage.partAccess()->promoteBuild(*st.build, {ns, ref}, st.build->buildId(), id); + out_slot = oc; + /// Test-only: models a concurrent writer racing in right after this transaction's own confirm. A + /// no-op in production. + metadata_storage.runAfterPromoteHookForTest({ns, ref}); + st.build.reset(); /// the build is consumed (promoted); never re-abandon it from the destructor + st.published = true; +} + +void ContentAddressedTransaction::commit(const TransactionCommitOptionsVariant &) +{ + if (failed) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "retrying a failed content-addressed transaction is not supported"); + + /// Operation gate (rev.7 §1). A commit with staged parts to publish is a `Write` (throws the typed + /// Vanished [D5] refusal on a Vanished disk); a commit with nothing to publish is the DROP/rename path -- its ref mutations + /// already applied immediately (`removeRecursive`/`dropNamespace`/`moveDirectory`), so it is a `Remove` + /// that no-op-succeeds on a Vanished disk, which is what lets a vanished-disk table's DROP finish. + /// Both throw 668 while the backing is uncertain (transient / IdentityLost). + const CasOpClass commit_class = parts.empty() ? CasOpClass::Remove : CasOpClass::Write; + if (metadata_storage.checkOpAdmitted(commit_class) == CasOpAdmission::TruthAbsent) + { + /// Vanished + nothing to publish: complete as a no-op so the DROP/rename finishes. There are no + /// staged parts to publish and no local staging to keep; run the same idempotent epilogue a + /// normal empty commit runs. + committed = true; + cleanupPendingTempFiles(); + force_fresh_validated_refs.clear(); + return; + } + + /// Publish each staged part. [TXN-ONE-PIPELINE] This is the ONLY place a ref becomes durable — the + /// tmp->final rename is a pure overlay re-key. Commit + /// atomicity: there is no multi-ref atomic publish, so a publish that throws after + /// earlier parts already published would leave a PARTIAL commit — some refs durably visible while + /// the transaction reports failure, diverging the durable pool from the disk layer's all-or-nothing + /// expectation. Track the refs THIS commit creates and, on any exception, best-effort unpublish + /// them before rethrowing. A partial commit is NOT a protocol violation (each publish/dropRef is + /// individually gate-checked and journalled; the leftover uploads are GC-reclaimable debris) — this + /// restores the wiring-layer transaction contract, not a CAS invariant. + /// + /// Fail-closed (CLAUDE.md): only refs that were ABSENT before we published them are rolled back. A + /// ref that already existed is pre-existing data this commit must never destroy on its error path. + /// Publishing over a live ref does not occur in the MergeTree write path (unique part names), but + /// the rollback must not assume it. updateRefPublishedAt mutations (autocommit one-shots on a + /// COMMITTED part) are individually durable by design and are deliberately NOT rolled back. + /// + /// Task 3: the rollback keys on the EXACT manifest each `publishStaging` call committed + /// (`Cas::CommitOutcome`), not merely on "this part's (ns, ref) name" -- an unconditional `dropRef` + /// would clobber a DIFFERENT writer's repoint of the same ref name that lands in the window between + /// this part's publish and a later part's failure (see `CasCommitRollback.RepointByOtherWriterSurvivesRollback`). + /// `part_outcomes` is snapshotted and preallocated up front (one allocation, index-addressed, no + /// per-part growth) so `publishStaging` can write `part_outcomes[i]` with a no-throw slot write -- + /// the precondition for the precise per-part rollback below. Parts are published SERIALLY by the loop + /// that follows; only the blob uploads within each part fan out (`fanOutBlobUploads`). The snapshot + /// preserves `parts`' own iteration order (the map's (ns, ref) sort order); there is no dependency + /// between parts that would require a different order. Concurrent cross-part publication is future + /// scope and is NOT done here. + struct IndexedPart { Cas::RootNamespace ns; std::string ref; PartStaging * st; }; + std::vector ordered; + ordered.reserve(parts.size()); + for (auto & [key, st] : parts) + ordered.push_back({Cas::RootNamespace{key.first}, key.second, &st}); + + std::vector> part_outcomes; + part_outcomes.assign(ordered.size(), std::nullopt); + + try + { + for (size_t i = 0; i < ordered.size(); ++i) + publishStaging(ordered[i].ns, ordered[i].ref, *ordered[i].st, part_outcomes[i]); + } + catch (...) + { + failed = true; + /// Compensating rollback. Best-effort: a ref we cannot unpublish becomes unreferenced debris + /// (GC-reclaimed); never mask the original failure with a rollback failure. Only a slot whose + /// outcome `created` is true names a ref THIS commit made durable for the first time; a + /// repoint of an already-committed ref (created=false) is pre-existing data and is never + /// dropped. `dropRefIfMatches` additionally guards against a concurrent repoint of the SAME ref + /// since this call's own publish: it removes the ref only if it still names the exact + /// `manifest_ref` this commit bound, leaving a newer binding untouched. + for (const auto & oc : part_outcomes) + if (oc && oc->created) + metadata_storage.partAccess()->dropRefIfMatches({oc->ns, oc->ref}, oc->manifest_ref); + throw; + } + committed = true; + /// All pending blobs have been uploaded in publishStaging; remove their staging resources now. + cleanupPendingTempFiles(); + /// This transaction's unlinkFile ForceFresh-proof memoization is scoped to this transaction only; + /// clear it alongside the other per-transaction state resets above. + force_fresh_validated_refs.clear(); +} + +TransactionCommitOutcomeVariant ContentAddressedTransaction::tryCommit(const TransactionCommitOptionsVariant & options) +{ + if (!std::holds_alternative(options)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "ContentAddressed transaction supports only tryCommit without options"); + commit(options); + return true; +} + +ObjectStorageKey ContentAddressedTransaction::generateObjectKeyForPath(const std::string &) +{ + notYet("generateObjectKeyForPath"); +} + +StoredObjects ContentAddressedTransaction::getSubmittedForRemovalBlobs() +{ + /// CA never hands shared backing objects to the disk layer for removal — reclamation is the + /// GC's. Empty unconditionally because shared content-addressed objects are never owned by the + /// disk-layer removal list. + return {}; +} + +const Cas::ManifestEntry * ContentAddressedTransaction::findStagedEntry( + const ContentAddressedMetadataStorage::Route & r) const +{ + auto it = parts.find({r.ns.string(), r.ref}); + if (it == parts.end()) + return nullptr; + auto eit = std::find_if(it->second.entries.begin(), it->second.entries.end(), + [&](const Cas::ManifestEntry & e) { return e.path == r.file; }); + return eit == it->second.entries.end() ? nullptr : &*eit; +} + +std::optional ContentAddressedTransaction::tryGetInFlightStorageObjects(const std::string & path) const +{ + /// Read-your-writes: a projection spill-and-merge reads back its own staged blocks before + /// the parent part's single commit. Staged content blobs may still be pending (not yet uploaded). + auto r = const_cast(this)->routeOf(path); + if (!r || r->file.empty()) + return {}; + auto it = parts.find({r->ns.string(), r->ref}); + if (it == parts.end()) + return {}; + if (const auto * entry = findStagedEntry(*r)) + { + if (entry->placement == Cas::EntryPlacement::Blob) + { + /// A pending blob has not been uploaded yet — its storage object does not exist in + /// the pool. Return empty so the caller falls back to tryReadFileInFlight (local temp read). + if (findPendingBlob(it->second, entry->ref)) + return {}; + const auto location = metadata_storage.store()->locate(*entry); + return StoredObjects{StoredObject(location.key, path, location.length)}; + } + /// An Inline entry carries its bytes in `inline_bytes`; `size()` (not `blob_size`, which is 0 + /// for an inline entry carried forward from a decoded source manifest — createHardLink) reports + /// the real inline byte count, so an in-flight read of a carried-forward inline sidecar (e.g. a + /// MATERIALIZE-PROJECTION projection marks file) resolves to its real size, matching the + /// committed getStorageObjects path. + return StoredObjects{StoredObject("", path, entry->size())}; + } + return {}; +} + +std::unique_ptr ContentAddressedTransaction::tryReadFileInFlight( + const std::string & path, const ReadSettings & settings, std::optional /*read_hint*/) const +{ + auto r = const_cast(this)->routeOf(path); + if (!r || r->file.empty()) + return nullptr; + auto it = parts.find({r->ns.string(), r->ref}); + if (it == parts.end()) + return nullptr; + if (const auto * entry = findStagedEntry(*r)) + { + if (entry->placement == Cas::EntryPlacement::Inline) + return std::make_unique(path, entry->inline_bytes); + if (entry->placement == Cas::EntryPlacement::Blob) + { + /// A pending blob has not been uploaded yet — serve reads from the staging area (the + /// same bytes that will be promoted to the pool in publishStaging post-precommit): a local + /// temp file for `Cas::StagingBackend::Local`, or the S3 staging object for `Cas::StagingBackend::S3` + /// (`staging_key` is a remote object key there, never a + /// local path, so `ReadBufferFromFile` would misinterpret it as a filesystem path). + if (const auto * pb = findPendingBlob(it->second, entry->ref)) + { + if (pb->backend == Cas::StagingBackend::S3) + { + /// The staging object holds `[header][payload]` + /// (the fixed-length `blob_header_len` CABL envelope, so the promote can stay a + /// verbatim server-side copy). Read-your-writes must serve the PAYLOAD ONLY — wrap the + /// object read in a `ReadBufferFromFileView` windowed to `[header_len, header_len+size)` + /// so position 0 is the payload start, else the reader would see 256 bytes of header + /// prepended to the payload (corruption). The LOCAL staging temp file holds the payload + /// verbatim (no header), so its path is unchanged. + const uint64_t header_len = metadata_storage.store()->poolMeta().blob_header_len; + const uint64_t payload_end = header_len + pb->size; + auto impl = metadata_storage.objectStorage()->readObject( + StoredObject(pb->staging_key, path, payload_end), settings); + return std::make_unique( + std::move(impl), path, header_len, payload_end); + } + return std::make_unique(pb->staging_key); + } + return metadata_storage.readBlobPayload(metadata_storage.store()->locate(*entry), path, settings); + } + } + return nullptr; +} + +std::optional ContentAddressedTransaction::tryGetInFlightFileSize(const std::string & path) const +{ + auto r = const_cast(this)->routeOf(path); + if (!r || r->file.empty()) + return {}; + auto it = parts.find({r->ns.string(), r->ref}); + if (it == parts.end()) + return {}; + if (const auto * entry = findStagedEntry(*r)) + /// `size()` (not `blob_size` directly, which is 0 for an inline entry carried forward via + /// createHardLink from a decoded source manifest). Without this, an in-flight size query for a + /// carried-forward inline sidecar returns 0 — the 02941 MATERIALIZE-PROJECTION "Empty marks + /// file: 0, must be: 144" corruption on a same-session read. + return entry->size(); + return {}; +} + +bool ContentAddressedTransaction::hasInFlightDirectory(const std::string & path) const +{ + /// The directory overlay is true iff at least one staged file lives under `path` for `path`'s + /// part - what makes a carried-forward projection dir visible to loadProjections. + auto r = const_cast(this)->routeOf(path); + /// INNER directories only: the overlay exists for staged projection dirs + /// The overlay is used by `loadProjections` during finalize. The PART DIR ITSELF answers FALSE - a + /// dedup-rejected temporary part still holds its uncommitted transaction at destruction, and + /// an overlay "exists" for the bare part dir sends removeIfNeeded into remove(), whose + /// bare-disk check then logs the "part to remove doesn't exist" warning. + if (!r || r->ref.empty() || r->file.empty()) + return false; + auto it = parts.find({r->ns.string(), r->ref}); + if (it == parts.end()) + return false; + const std::string prefix = r->file + "/"; + for (const auto & entry : it->second.entries) + if (entry.path.starts_with(prefix)) + return true; + return false; +} + +std::vector ContentAddressedTransaction::listInFlightDirectory(const std::string & path) const +{ + /// Immediate-child names staged directly under `path` (one level) - loadProjections' + /// withPartFormatFromDisk iterates a staged projection dir to find its mark file. + auto r = const_cast(this)->routeOf(path); + std::vector result; + if (!r || r->ref.empty()) + return result; + auto it = parts.find({r->ns.string(), r->ref}); + if (it == parts.end()) + return result; + const std::string prefix = r->file.empty() ? "" : r->file + "/"; + std::set names; + auto add = [&](const std::string & name) + { + if (!name.starts_with(prefix) || name.size() <= prefix.size()) + return; + const auto rest = name.substr(prefix.size()); + const auto slash = rest.find('/'); + names.insert(slash == std::string::npos ? rest : rest.substr(0, slash)); + }; + for (const auto & entry : it->second.entries) + add(entry.path); + return {names.begin(), names.end()}; +} + +void ContentAddressedTransaction::createMetadataFile(const std::string &, const StoredObjects &) +{ + notYet("createMetadataFile"); +} + +void ContentAddressedTransaction::stageBlobPartFile( + const ContentAddressedMetadataStorage::Route & route, + const Cas::BlobRef & ref, size_t size, const std::string & staging_key, Cas::StagingBackend backend) +{ + /// Do not upload here. Record the pending blob (uploaded post-precommit in publishStaging) + /// and a tokenless dependency; putBlob later overwrites it with the tokened dependency. + /// The staging bytes are kept (the transaction owns them) — a local temp file for + /// `Cas::StagingBackend::Local`, or an S3 staging object for `Cas::StagingBackend::S3`. + auto & st = stagingFor(route); + st.pending_blobs.push_back({ref, staging_key, size, backend}); + buildFor(route, st).recordPendingBlobDep(ref, size); + + Cas::ManifestEntry entry; + entry.path = route.file; + entry.placement = Cas::EntryPlacement::Blob; + entry.ref = ref; + entry.blob_size = size; + std::erase_if(st.entries, [&](const Cas::ManifestEntry & e) { return e.path == entry.path; }); + st.entries.push_back(std::move(entry)); +} + +std::string ContentAddressedTransaction::buildS3StagingBlobHeader( + const ContentAddressedMetadataStorage::Route & route) const +{ + /// Mirror `PartWriteTxn::uploadFromSource`'s `buildHeader` (minus the dropped `logical_size`/`logical_hash` + /// fields and minus `build_id`, which is not known until commit and is diagnostic-only). A FRESH + /// `incarnation_tag` per staging object keeps the incarnation zone unique; the header is padded to + /// the pool's fixed `blob_header_len` so the payload starts at a constant offset. + const Cas::PoolPtr & store = metadata_storage.store(); + const Cas::PoolMeta & meta = store->poolMeta(); + const Cas::PoolConfig & cfg = store->poolConfig(); + + Cas::EnvelopeHeader header; + header.kind = Cas::ObjectKind::Blob; + header.incarnation_tag = (static_cast(thread_local_rng()) << 64) | thread_local_rng(); + header.build_id = 0; /// not known at stream time; diagnostic-only (not read by GC/read paths) + /// ch = the real ClickHouse VERSION_INTEGER (diagnostic-only; consistent with `PartWriteTxn::buildHeader`). + /// The v3 envelope drops hash_algo/domain_id/writer_version, so forensics ride on ch + bld. + header.provenance = Cas::Provenance{ + /*created_at_ms*/ 0, cfg.server_id, VERSION_INTEGER, Cas::ProvenanceOp::Other}; + header.intended_ref = route.ns.string() + "/" + route.ref; + /// The v3 codec pads to the pool's fixed header length and TRUNCATES a too-long intended_ref + /// internally (it is diagnostic-only), so the old drop-and-retry is gone — one encode call. + return Cas::encodeEnvelopeHeader(header, static_cast(meta.blob_header_len)); +} + +std::unique_ptr ContentAddressedTransaction::tryCreateWriteBuffer( + const std::shared_ptr & owner, + const std::string & path, size_t buf_size, WriteMode mode, + const WriteSettings & settings, bool autocommit) +{ + /// This transaction owns the write because the blob key is known only after hashing the payload. + /// Append is serviceable (read-modify-rewrite) only for a non-part / table-level verbatim file + /// (handled inside writeFile). A part file is a content blob or a whole-rewritten inline entry, so + /// append on a part-file path is unsupported. + if (mode == WriteMode::Append && Cas::isPartFilePath(path)) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Disk does not support WriteMode::Append for content part files"); + + /// Autocommit cannot work for a CONTENT BLOB part file (column data/marks, primary.idx): a part's + /// blobs are always written together as one build, whose manifest + ref publish only when commit() + /// runs. A small INLINE-eligible part file IS autocommittable (a standalone one-shot write): the write + /// lands as an ordinary manifest entry and, if the ref is already committed, `publishStaging`'s repoint + /// branch carries the rest of the part forward and republishes once (the transactional-INSERT + /// creation-CSN fill-in / removal-TID rewrite / rollback path). Verbatim / table-level files (not part + /// files) are durable on finalize regardless of `autocommit`. + if (autocommit && Cas::isPartFilePath(path)) + { + auto p = Cas::parsePartFilePath(path); + if (!p || p->file.empty() || Cas::partFileMustStayBlob(p->file)) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Autocommit writes are not supported for content part files on a content-addressed disk"); + + auto inner = writeFile(path, buf_size, mode, settings); + auto commit_callback = [owner](size_t) mutable { owner->commit(); }; + return std::make_unique( + std::move(inner), std::move(commit_callback), path, /*create_blob_if_empty=*/true); + } + + /// Non-autocommit (or verbatim autocommit): pin the owning disk transaction for the returned buffer's + /// lifetime. The CA write buffers capture a bare `this` in their deferred finalize / pin-blob callbacks; + /// MergedBlockOutputStream may finalize them LATER (another thread, or after the part storage / + /// transaction would otherwise be torn down on async-insert / cancel / exception-unwind). Holding + /// `owner` (which owns this ContentAddressedTransaction by shared_ptr) keeps that `this` valid until the + /// buffer — and so this callback — is destroyed after finalize (the lifetime guarantee now + /// expressed generically via `owner`). No cycle: the transaction does not hold the buffer. + auto inner = writeFile(path, buf_size, mode, settings); + auto keep_alive_callback = [owner](size_t) mutable {}; + return std::make_unique( + std::move(inner), std::move(keep_alive_callback), path, /*create_blob_if_empty=*/true); +} + +std::unique_ptr ContentAddressedTransaction::writeFile( + const std::string & path, size_t buf_size, WriteMode mode, const WriteSettings & settings) +{ + /// Write gate (rev.7 §1): the single chokepoint every write buffer (both direct and via + /// `tryCreateWriteBuffer`) is created through -- refuse on a Vanished (typed [D5]) or transient/ + /// IdentityLost (668) disk before staging any content or opening a verbatim buffer. + metadata_storage.checkOpAdmitted(CasOpClass::Write); + /// Non-part files are VERBATIM namespace files, durable on finalize (no commit involvement - + /// the disk layer's autocommit contract for them rides exactly this). Append is serviced by + /// read-modify-rewrite: the existing bytes are carried forward (the MVCC mutation-entry CSN + /// append depends on this). The `carried` prefix below is read ONCE here, at buffer-open time, and + /// frozen into the write callback; `casPutObject`'s CAS loop (invoked from the callback via + /// `putNamespaceFile`/`putMountpointObject`) only re-reads the TOKEN on conflict, not this base + /// content — see the single-appender invariant documented at `CasPlainObjects::casPutObject`. Safe + /// only because the sole production appender (the mutation-entry CSN write) never has a second + /// concurrent appender on the same key. + if (!Cas::isPartFilePath(path)) + { + if (auto tf = Cas::parseTableFilePath(path)) + { + /// The LIFE is resolved once, here at buffer-open time, and captured by value below, so a + /// finalize that runs later writes to the incarnation this open was admitted under -- never + /// into whatever life the namespace name happens to denote when the callback fires. + const Cas::NamespaceLifeId life + = metadata_storage.store()->namespaceLife(metadata_storage.liveNamespace(tf->table_uuid)); + const std::string name = tf->tail; + std::string prefix_bytes; + if (mode == WriteMode::Append) + if (auto existing = metadata_storage.store()->getNamespaceFile(life, name)) + prefix_bytes = std::move(*existing); + return std::make_unique( + [this, life, name, carried = std::move(prefix_bytes)](std::string bytes) + { + metadata_storage.store()->putNamespaceFile(life, name, carried + bytes); + }); + } + /// A loose disk file, including the startup write probe, is a plain mountpoint object. + const std::string key = metadata_storage.serverRootId() + "/" + path; + std::string prefix_bytes; + if (mode == WriteMode::Append) + if (auto existing = metadata_storage.store()->getMountpointObject(key)) + prefix_bytes = std::move(*existing); + return std::make_unique( + [this, key, carried = std::move(prefix_bytes)](std::string bytes) + { + metadata_storage.store()->putMountpointObject(key, carried + bytes); + }); + } + + auto p = Cas::parsePartFilePath(path); + auto r = p ? metadata_storage.route(*p) : std::nullopt; + if (!r || r->file.empty()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "ContentAddressedTransaction::writeFile: not a part file path: {}", path); + + /// The former + /// mutable-per-part-file branch (uuid.txt/metadata_version.txt/txn_version.txt staging directly + /// into a separate mutable payload) is DELETED here — these three names fall through to the + /// ordinary content path below like any other tree file. There is no filename left to special-case: + /// `kMutablePerPartFiles`/`isMutablePerPartFile` predicate itself is gone too — there is no + /// filename left to special-case. During part build these files land in the initial manifest with + /// every other staged file; a standalone write on an already-committed part repoints. + + /// A CONTENT part file that must stay a blob (per-column data/marks, primary.idx): spill + hash, + /// then stage the blob as PENDING (precommit-first). The blob is NOT uploaded/promoted here; + /// `publishStaging` uploads it (Local) or promotes the S3 staging object post-precommit. + /// recordPendingBlobDep (inside stageBlobPartFile) records a tokenless dependency without any + /// pool operation at staging time. + if (Cas::partFileMustStayBlob(r->file)) + { + /// S3-native staging: + /// when this disk opted in (`staging_backend=s3`) AND the mount-time capability probe + /// a capability probe proved the object storage enforces write-once conditional copy, stream directly + /// to a fresh per-mount S3 staging object while hashing — no local-disk round trip. Otherwise + /// (the OFF BY DEFAULT global constraint, or a probe fail-close) fall through to the existing, + /// byte-for-byte-unchanged local-temp-file path below. + /// Hash with this pool's node-local write algorithm rather than a hardcoded city hash; + /// `PoolMeta` no longer records a single pool-wide algorithm -- + /// mixed-algo pools track `algos_used`; `writeAlgo()` is the write-mint accessor now). + const auto hash_algo = metadata_storage.store()->writeAlgo(); + /// `hash_hex` is rendered by the streaming write buffer at `hash_algo`'s own width — + /// parse it back at that SAME width via `Cas::codecFor(hash_algo)` (never a pool-wide + /// `DigestCodec`, which no longer exists) into a full `BlobRef` pair. + + if (metadata_storage.stagingBackend() == Cas::StagingBackend::S3 && metadata_storage.conditionalCopySupported()) + { + const std::string staging_key = metadata_storage.stagingKeyPrefix() + "/" + getRandomASCIIString(32) + ".tmp"; + auto object_sink = metadata_storage.objectStorage()->writeObject(StoredObject(staging_key), WriteMode::Rewrite); + /// Build the fixed-length CABL envelope header now (before + /// the payload is streamed) so the staging object holds `[header][payload]` and the promote + /// stays a verbatim server-side copy. The header carries a FRESH `incarnation_tag`; `build_id` + /// is left 0 (not known at stream time — diagnostic-only, not read by GC/read paths). The + /// buffer writes this header first, UNHASHED and excluded from the reported size, so the + /// content key stays the pool's hash of `payload` and `blob_size` stays the payload size. + std::string envelope_header = buildS3StagingBlobHeader(*r); + /// rev.7 [C2]: capture the fence generation now, re-checked immediately before the durable + /// `sink->finalize()` in `finalizeImpl` (the streaming upload becomes durable there). + const Cas::PoolPtr pool = metadata_storage.store(); + const uint64_t admitted_generation = pool->fenceGeneration(); + return std::make_unique( + std::move(object_sink), + staging_key, + std::move(envelope_header), + hash_algo, + buf_size, + settings.use_adaptive_write_buffer, + settings.adaptive_write_buffer_initial_size, + [this, route = *r, hash_algo](const std::string & hash_hex, size_t size, const std::string & key) + { + const Cas::BlobRef ref{hash_algo, Cas::codecFor(hash_algo).fromHex(hash_hex)}; + stageBlobPartFile(route, ref, size, key, Cas::StagingBackend::S3); + }, + [pool, admitted_generation] { pool->checkFenceOrThrow(admitted_generation); }); + } + + return std::make_unique( + metadata_storage.scratchPath(), + hash_algo, + buf_size, + settings.use_adaptive_write_buffer, + settings.adaptive_write_buffer_initial_size, + [this, route = *r, hash_algo](const std::string & hash_hex, size_t size, const std::string & temp_path) + { + const Cas::BlobRef ref{hash_algo, Cas::codecFor(hash_algo).fromHex(hash_hex)}; + stageBlobPartFile(route, ref, size, temp_path, Cas::StagingBackend::Local); + }); + } + + /// Inline candidate (small eager metadata): buffer in memory, decide at finalize. <= INLINE_CAP + /// rides the single tree object as an Inline entry (one-GET part open); an oversized + /// candidate spills to a blob (the safety net). + return std::make_unique( + [this, route = *r](std::string bytes) + { + /// Mint via the one write hash function, `Cas::poolContentHash` (algorithm, payload) -> BlobRef + /// (`CasPartWriteTxn.h`) -- the SAME mint the streaming blob path's callers use, so an inline file + /// and a standalone blob of identical content get the same ref (same content hash identity) + /// under EVERY algo, including sha256. + const auto hash_algo = metadata_storage.store()->writeAlgo(); + const Cas::BlobRef ref = Cas::poolContentHash(hash_algo, bytes); + if (bytes.size() <= INLINE_CAP) + { + auto & st = stagingFor(route); + /// An inline (no-blob) entry still requires a PartWriteTxn. `publishStaging` stages the + /// manifest body, precommits, and promotes the ref even for a part with NO blob uploads; + /// it asserts `st.build != nullptr` whenever `st.entries` is non-empty. Without this, a + /// part whose files are ALL inline (a tiny/empty merge output, every file <= INLINE_CAP) + /// reaches `publishStaging` with entries but no PartWriteTxn -> LOGICAL_ERROR "staged entries + /// without a PartWriteTxn" -> a logical-error exception under abort_on_logical_error + /// since the inline-files feature). The blob path already establishes the PartWriteTxn via + /// `buildFor`; the inline path must do the same. + buildFor(route, st); + Cas::ManifestEntry entry; + entry.path = route.file; + entry.placement = Cas::EntryPlacement::Inline; + entry.ref = ref; /// content hash identity (same for inline and blob of same content) + /// `blob_size` stays 0 (its default) for an Inline entry — matching decode, which never + /// fills it for Inline. `entry.size()` is the logical size, derived from `inline_bytes`. + entry.inline_bytes = std::move(bytes); + std::erase_if(st.entries, [&](const Cas::ManifestEntry & e) { return e.path == entry.path; }); + st.entries.push_back(std::move(entry)); + } + else + { + /// Safety fallback: an unexpectedly large candidate spills to a blob (preserves the + /// invariant that big files are not held inline). Write the buffered bytes to a unique + /// local temp file (same scratchPath + random-name scheme as CaContentWriteBuffer), then + /// stage exactly like a streaming blob. + std::filesystem::create_directories(metadata_storage.scratchPath()); + const std::string temp_path = + metadata_storage.scratchPath() + "/inline_overflow_" + getRandomASCIIString(32) + ".tmp"; + { + WriteBufferFromFile tmp(temp_path); + tmp.write(bytes.data(), bytes.size()); + tmp.finalize(); + } + /// Until stageBlobPartFile takes ownership, WE own the temp file — mirror the blob path, + /// where CaContentWriteBuffer's dtor removes it unless the callback succeeded. If + /// stageBlobPartFile throws, drop the orphan instead of leaking it into scratch. This + /// fallback is always `Cas::StagingBackend::Local` — an oversized inline candidate is rare + /// enough (a safety net, not a hot path) that S3-staging mode does not cover it. + bool staged = false; + SCOPE_EXIT({ if (!staged) { std::error_code ec; std::filesystem::remove(temp_path, ec); } }); + stageBlobPartFile(route, ref, bytes.size(), temp_path, Cas::StagingBackend::Local); + staged = true; + } + }); +} + +void ContentAddressedTransaction::createDirectory(const std::string &) +{ + /// Object storage has no real directories (mirrors the plain-rewritable transaction) -- but this is a + /// Write: the gate makes it throw typed on a Vanished disk / 668 while uncertain, so a mutation is + /// never silently accepted against an erased or unreachable backing (rev.7 §1 previously-no-op site). + metadata_storage.checkOpAdmitted(CasOpClass::Write); +} + +void ContentAddressedTransaction::createDirectoryRecursive(const std::string &) +{ + metadata_storage.checkOpAdmitted(CasOpClass::Write); +} + +void ContentAddressedTransaction::removeDirectory(const std::string & path) +{ + /// CONTRACT: `removeDirectory`/`moveDirectory` mutate durable refs at CALL TIME, not at commit — + /// this is the everything-immediate model, not a missed "defer to commit" opportunity. `renameParts` + /// is the actual commit point; anything that goes wrong after one of these calls is undone by a + /// COMPENSATING operation over already-committed state, the same way upstream MergeTree's own + /// `rollbackPartsToTemporaryState` and outdated-part cleanup run over committed disk state rather + /// than an in-memory intent log. Recording these as staged intents and applying them at commit would + /// duplicate that compensation machinery for no correctness gain. + /// + /// Remove gate (rev.7 §1): a Vanished disk answers no-op success (nothing to remove -- truth), so the + /// enclosing DROP completes; a transient / IdentityLost disk throws 668 (the DROP re-queues). + if (metadata_storage.checkOpAdmitted(CasOpClass::Remove) == CasOpAdmission::TruthAbsent) + return; + + /// The MergeTree fast-removal path unlinks a part's files one by one (no-ops here) and then + /// calls removeDirectory() - the SINGLE authoritative point at which the part's ref must + /// be unlinked. Part dirs route to dropRef; anything else is a no-op (object + /// storage has no real directories; tables/detached/shadow are removed via removeRecursive). + if (auto r = routeOf(path); r && !r->ref.empty() && r->file.empty()) + { + metadata_storage.partAccess()->dropRefIfPresent(r->refKey()); + /// This transaction's staged removal marks for the same ref (content_removed, populated by + /// unlinkFile's per-file unlinks that the MergeTree fast-removal path issues right before this + /// call) are superseded by the whole-part ref-drop just performed above — discard them so + /// publishStaging's committed-ref repoint branch never chases an already-dropped ref, and the + /// dominant removal path pays zero repoints (one ref-drop only). + if (auto * st = findStaging(*r)) + { + st->content_removed.clear(); + st->entries.clear(); + if (st->build) + { + st->build->abandon(); + st->build.reset(); + } + } + return; + } +} + +void ContentAddressedTransaction::removeRecursive(const std::string & path, const ShouldRemoveObjectsPredicate & /*should_remove_objects*/) +{ + /// Removal = pointer-unlink + deferred GC: only refs and verbatim files go; the shared + /// blobs/trees are reclaimed by Cas::Gc once unreachable. The predicate gates backing-object + /// deletion, which CA always defers, so it is intentionally ignored here. + + /// Remove gate (rev.7 §1): a Vanished disk answers no-op success (nothing to remove), so a + /// vanished-disk table's DROP -- which reaches here via `removeSharedRecursive` -- completes; a + /// transient / IdentityLost disk throws 668 (the DROP re-queues, drains after recovery/FORGET). + if (metadata_storage.checkOpAdmitted(CasOpClass::Remove) == CasOpAdmission::TruthAbsent) + return; + + /// FREEZE shadow shapes first (a shadow table dir also satisfies parseTableUuid). + if (Cas::isShadowPath(path)) + { + if (auto p = Cas::parsePartFilePath(path); p && !p->backup_name.empty() && p->file.empty()) + { + const auto ns = ContentAddressedMetadataStorage::shadowNamespace(p->shadow_table_dir); + metadata_storage.partAccess()->dropRefIfPresent({ns, p->part_name}); + return; + } + if (Cas::endsWithTableUuidPair(path)) + { + metadata_storage.partAccess()->dropNamespace(ContentAddressedMetadataStorage::shadowNamespace(path)); + return; + } + /// Backup root / intermediate dir (SYSTEM UNFREEZE WITH NAME): drop every shadow + /// namespace under it. Canonicalize because callers hand trailing-slash dirs. + std::string prefix = path; + while (!prefix.empty() && prefix.back() == '/') + prefix.pop_back(); + /// RECORD AND CONTINUE, and the reason is the shape of the alternative. Refusing the DROP would + /// make the obstacle permanent, blocked by the one operation that could have cleared it. The GC + /// round does not clear it either. For a ref-family key that is a test's claim rather than this + /// comment's: `CasRefGc.UnIncarnatedRefKeyAbortsRefFoldingWithoutWedgingTheRound` runs a round + /// over such a key and holds that the round deletes nothing and the key survives it. For a + /// `_files`-family key the round never gets the chance: `Cas::Gc`'s fold only LISTs + /// `casRefsPrefix()`, never `rootsPrefix()`, so such a key sits outside anything a round scans. + /// Continuing drops every namespace the enumeration DID name; the offending key stays as + /// reported debris. It cannot hide a namespace that has any well-formed key of its own, because + /// attribution is per key. + const Cas::NamespaceListing listing = metadata_storage.store()->listNamespaces(prefix + "/"); + for (const Cas::UnattributableNamespaceKey & bad : listing.skipped) + LOG_ERROR(getLogger("ContentAddressedTransaction"), + "removeRecursive('{}'): key '{}' names no namespace life and was left in place ({}). " + "Every namespace this enumeration did name is still dropped; run `cas-fsck` to enumerate " + "such keys.", path, bad.key, bad.reason); + for (const auto & ns : listing.namespaces) + metadata_storage.partAccess()->dropNamespace(Cas::RootNamespace{ns}); + return; + } + + /// Table dir: the table's namespace (live + folded-in detached refs) and every verbatim + /// file go in one dropNamespace. + if (auto uuid = Cas::parseTableUuid(path)) + { + metadata_storage.partAccess()->dropNamespace(metadata_storage.liveNamespace(*uuid)); + return; + } + + if (auto p = Cas::parsePartFilePath(path)) + { + auto r = metadata_storage.route(*p); + /// The detached CONTAINER dir (DROP DETACHED / table-detach): drop all detached refs. + if (r && r->ref.empty() && p->part_name == Cas::kDetachedDirName) + { + for (const auto & ref : metadata_storage.detachedRefNames(r->ns)) + metadata_storage.partAccess()->dropRefIfPresent({r->ns, ref}); + return; + } + /// The moving CONTAINER dir (MOVE-to-CA fix, mirrors detached): the mover's crash-cleanup + /// (MergeTreeData.cpp, MOVING_DIR_NAME) calls this at table load to reclaim every staging + /// ref an interrupted move left behind. + if (r && r->ref.empty() && p->part_name == Cas::kMovingDirName) + { + for (const auto & ref : metadata_storage.movingRefNames(r->ns)) + metadata_storage.partAccess()->dropRefIfPresent({r->ns, ref}); + return; + } + /// A single part dir (live or detached): drop its ref. + if (r && !r->ref.empty() && r->file.empty()) + { + metadata_storage.partAccess()->dropRefIfPresent(r->refKey()); + return; + } + /// A projection subdir: virtual (nested in the parent tree) - removal is a no-op; the + /// blobs go when the part's ref does. + if (r && !r->ref.empty()) + return; + } + + /// Table-level SUBDIRECTORY (deduplication_logs/): remove every verbatim file under it. + if (auto tf = Cas::parseTableFilePath(path)) + { + /// The READABLE resolution, which is the non-creating one (`namespaceFilesLifeIfReadable` answers + /// an uncataloged namespace from a catalog-only lookup and writes nothing): a removal must never + /// birth the namespace it is removing from. No life means there is nothing here to remove. + const auto life = metadata_storage.readableNamespaceFilesLife( + metadata_storage.liveNamespace(tf->table_uuid)); + if (!life) + return; + const std::string prefix = tf->tail + "/"; + for (const auto & name : metadata_storage.store()->listNamespaceFiles(*life)) + if (name.starts_with(prefix)) + metadata_storage.store()->removeNamespaceFile(*life, name); + return; + } +} + +void ContentAddressedTransaction::createHardLink(const std::string & path_from, const std::string & path_to) +{ + /// Write gate (rev.7 §1). + metadata_storage.checkOpAdmitted(CasOpClass::Write); + auto src = routeOf(path_from); + auto dst = routeOf(path_to); + if (!src || src->file.empty() || !dst || dst->file.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "ContentAddressed: createHardLink requires two part-file paths: {} -> {}", path_from, path_to); + + auto & dst_st = stagingFor(*dst); + + /// Content file. Prefer an entry staged earlier in THIS transaction (the destination PartWriteTxn + /// re-observes the blob via cold reuse — its own dependency); else carry forward from the + /// COMMITTED source part (adoptFromTree: tokenless evidence pinned by the witnessed live + /// source tree, W-EVIDENCE). + Cas::ManifestEntry entry; + if (auto * src_st = findStaging(*src)) + { + auto it = std::find_if(src_st->entries.begin(), src_st->entries.end(), + [&](const Cas::ManifestEntry & e) { return e.path == src->file; }); + if (it != src_st->entries.end()) + { + entry = *it; + if (entry.placement == Cas::EntryPlacement::Blob) + { + /// Unified adopt dispatch. copy_pending=(&dst_st != src_st) so the pending + /// blob record is copied into dst_st only when the destination is a different part + /// (hardlink = copy semantics; same-part is a self-ref that shouldn't duplicate the record). + const auto * pb = findPendingBlob(*src_st, entry.ref); + adoptStagedBlob(pb, entry, dst_st, buildFor(*dst, dst_st), /*copy_pending=*/(&dst_st != src_st)); + } + else if (entry.placement != Cas::EntryPlacement::Inline) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "ContentAddressed: staged hardlink of unsupported placement for {}", path_from); + entry.path = dst->file; + std::erase_if(dst_st.entries, [&](const Cas::ManifestEntry & e) { return e.path == entry.path; }); + dst_st.entries.push_back(std::move(entry)); + return; + } + } + + /// Carry forward from the COMMITTED source part: read the source manifest, find the named entry, + /// record a TOKENLESS W-EVIDENCE dep for its blob (no HEAD before precommit; promote re-proves it). + /// ForceFresh getView == resolveRef(allow_stale=false) + readManifestShared, so this is the same + /// request pattern as before, now instrumented via the facade. + auto view = metadata_storage.partAccess()->getView(src->refKey(), Cas::Freshness::ForceFresh); + if (!view) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "ContentAddressed: createHardLink source part missing: {}", path_from); + const auto * src_entry = view->findFile(src->file); + if (!src_entry) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, + "ContentAddressed: createHardLink source file missing in manifest: {}", path_from); + buildFor(*dst, dst_st).adoptEvidence(*src_entry); + entry = *src_entry; + entry.path = dst->file; + std::erase_if(dst_st.entries, [&](const Cas::ManifestEntry & e) { return e.path == entry.path; }); + dst_st.entries.push_back(std::move(entry)); +} + +void ContentAddressedTransaction::setLastModified(const std::string &, const Poco::Timestamp &) +{ + /// Timestamps are derived for content addressing (the publish stamp), so accept and ignore them -- but + /// gate as a Write (previously-no-op site, rev.7 §1): never silently accept it on a Vanished/uncertain + /// disk. + metadata_storage.checkOpAdmitted(CasOpClass::Write); +} + +void ContentAddressedTransaction::chmod(const String &, mode_t) +{ + notYet("chmod"); +} + +void ContentAddressedTransaction::setReadOnly(const std::string &) +{ + /// Read-only flags have no content-addressed representation — accept and ignore them -- but gate as a + /// Write (previously-no-op site, rev.7 §1). + metadata_storage.checkOpAdmitted(CasOpClass::Write); +} + +void ContentAddressedTransaction::moveDirectory(const std::string & path_from, const std::string & path_to) +{ + /// Write gate (rev.7 §1): mutates durable refs immediately -- throw before touching them on a + /// Vanished/uncertain disk. + metadata_storage.checkOpAdmitted(CasOpClass::Write); + /// Same call-time-durability-plus-compensation contract as `removeDirectory` above: this mutates + /// durable refs immediately rather than staging an intent for commit; see the contract note there. + auto src_p = Cas::parsePartFilePath(path_from); + auto dst_p = Cas::parsePartFilePath(path_to); + auto src = src_p ? metadata_storage.route(*src_p) : std::nullopt; + auto dst = dst_p ? metadata_storage.route(*dst_p) : std::nullopt; + + /// RENAME TABLE / cross-engine move: both endpoints are TABLE dirs. Republish every ref (live + /// and folded-in `detached/`-prefixed refs) plus every verbatim file under the new table + /// identity, then drop the old namespace (the blobs/trees are content-addressed and untouched). + /// + /// There is no native cross-namespace atomicity (object storage has no directory rename, unlike a + /// non-CAS disk where RENAME TABLE is a single atomic directory rename). This is a best-effort + /// multi-op move, but it is RE-DRIVABLE/IDEMPOTENT: `republishRef` no-ops when the source ref is + /// already gone (resolveRef miss after a prior drive moved it), `putNamespaceFile` is + /// last-writer-wins (re-putting identical bytes is a no-op), and `dropNamespace` of an + /// already-empty/absent namespace is a no-op. So a mid-loop throw leaves the table SPLIT across the + /// two namespaces, but re-driving the SAME rename completes it. There is no in-call compensation; + /// true atomicity would need a durable move-journal (deliberately out of scope — it would touch the + /// tested GC/journal layer). On partial failure we log loudly so the split state is diagnosable. + if (auto src_table = Cas::parseTableUuid(path_from)) + { + if (auto dst_table = Cas::parseTableUuid(path_to)) + { + const auto from_ns = metadata_storage.liveNamespace(*src_table); + const auto to_ns = metadata_storage.liveNamespace(*dst_table); + try + { + for (const auto & [ref, _] : metadata_storage.store()->listRefs(from_ns)) + metadata_storage.partAccess()->republishRef({from_ns, ref}, {to_ns, ref}); + /// Asymmetric by necessity: the SOURCE is read, so it resolves readably and contributes + /// nothing when it has no life; the DESTINATION is written, so it resolves the minting way + /// and is born here if the rename is what first creates it. Resolved only once the source + /// actually has files to move, so a rename of a table with none does not mint a life for a + /// destination nothing is written to. + const auto from_life = metadata_storage.readableNamespaceFilesLife(from_ns); + const std::vector file_names + = from_life ? metadata_storage.store()->listNamespaceFiles(*from_life) : std::vector{}; + if (!file_names.empty()) + { + const Cas::NamespaceLifeId to_life = metadata_storage.store()->namespaceLife(to_ns); + for (const auto & name : file_names) + if (auto bytes = metadata_storage.store()->getNamespaceFile(*from_life, name)) + metadata_storage.store()->putNamespaceFile(to_life, name, *bytes); + } + metadata_storage.partAccess()->dropNamespace(from_ns); + } + catch (...) + { + LOG_ERROR(getLogger("ContentAddressedTransaction"), + "RENAME TABLE move was only partially applied: the table is SPLIT across namespaces " + "'{}' and '{}'. The move is idempotent — retrying the same RENAME re-drives it to " + "completion (already-moved refs/files are no-ops). Underlying error: {}", + from_ns.string(), to_ns.string(), getCurrentExceptionMessage(/*with_stacktrace=*/false)); + throw; + } + return; + } + } + + if (!src || !dst) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "ContentAddressed: moveDirectory cannot classify {} -> {}", path_from, path_to); + + /// Projection MATERIALIZE/merge renames the staged _.tmp_proj subdir to .proj + /// inside the SAME staged part: re-key the staged entry-name prefixes so the published tree + /// carries the final keys. + if (!src->file.empty() && !dst->file.empty() + && src->ns.string() == dst->ns.string() && src->ref == dst->ref + && src->file.ends_with(".tmp_proj") && dst->file.ends_with(".proj")) + { + if (auto * st = findStaging(*src)) + { + const std::string old_prefix = src->file + "/"; + const std::string new_prefix = dst->file + "/"; + for (auto & entry : st->entries) + if (entry.path.starts_with(old_prefix)) + entry.path = new_prefix + entry.path.substr(old_prefix.size()); + return; + } + } + + /// Every remaining shape is a PART-DIR move: (ns, ref) -> (ns', ref') with empty files. This + /// uniformly covers tmp->final (staged), committed renames (delete_tmp_, merge results), + /// DETACH (live -> detached ns), detached renames (attaching_/deleting_), and ATTACH + /// (detached -> live ns) - in the new layout they are all the same two moves: re-key any + /// staging, then move any committed ref. + if (!src->ref.empty() && src->file.empty() && !dst->ref.empty() && dst->file.empty()) + { + const std::pair src_key{src->ns.string(), src->ref}; + const std::pair dst_key{dst->ns.string(), dst->ref}; + if (src_key == dst_key) + return; + + /// Re-key a STAGED source into the destination. A move carries the + /// SOURCE's content to the destination — the POSIX `rename` semantic the rest of MergeTree + /// assumes, and exactly what `moveFile` does (`dst[file] = src_bytes`). On the happy path the + /// destination staging is freshly-created/empty so there is no collision at all; this only + /// matters if some future op-order stages the same mutable file under BOTH keys. + bool had_staged_source = false; + if (auto src_it = parts.find(src_key); src_it != parts.end()) + { + had_staged_source = true; + PartStaging & dst_st = parts[dst_key]; + PartStaging & src_st = src_it->second; + for (auto & entry : src_st.entries) + { + /// A genuine collision (both src and dst independently + /// staged DIFFERING bytes for the SAME path) is a fail-loud LOGICAL_ERROR rather than a + /// silent lost-update — the same defensive rule this loop used to apply only to the + /// three legacy mutable names now applies uniformly to every entry (that scoping was + /// itself a leftover of the mutable-file/entry split; there is only one kind of staged + /// file left). Identical bytes are a benign idempotent re-key; distinct paths are the + /// ordinary source-wins merge (a genuine collision is not expected in normal operation + /// — only some future op-order re-keying the same file under both stagings). + if (const auto existing = std::find_if(dst_st.entries.begin(), dst_st.entries.end(), + [&](const Cas::ManifestEntry & e) { return e.path == entry.path; }); + existing != dst_st.entries.end() && !(*existing == entry)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "ContentAddressed: moveDirectory file collision on '{}' ({} -> {}): " + "source and destination staged different bytes for the same file", + entry.path, src->ref, dst->ref); + std::erase_if(dst_st.entries, [&](const Cas::ManifestEntry & e) { return e.path == entry.path; }); + dst_st.entries.push_back(std::move(entry)); + } + /// Carry any staged removal marks forward too — a re-key of the + /// staging key must not silently drop them. + for (const auto & file : src_st.content_removed) + dst_st.content_removed.insert(file); + /// Move pending blobs from src to dst — they will be uploaded in dst's publishStaging. + for (auto & pb : src_st.pending_blobs) + dst_st.pending_blobs.push_back(std::move(pb)); + src_st.pending_blobs.clear(); + if (!dst_st.build) + { + dst_st.build = std::move(src_st.build); + } + else if (src_st.build) + { + /// Two Builds for one destination part: keep the destination's; the source build's + /// deps ride the staged entries (re-observed by the destination build at adopt + /// time is unnecessary - entries staged via putBlob/adopt carry deps in the SOURCE + /// build... merge conservatively by abandoning nothing and re-observing): + for (const auto & entry : dst_st.entries) + if (entry.placement == Cas::EntryPlacement::Blob) + { + /// Unified adopt dispatch. Pending blob records were already moved + /// to dst_st.pending_blobs above (MOVE semantics), so copy_pending=false. + adoptStagedBlob(findPendingBlob(dst_st, entry.ref), entry, dst_st, *dst_st.build, /*copy_pending=*/false); + } + src_st.build->abandon(); + } + parts.erase(src_it); + + /// A freshly-written part finalized tmp->final is re-keyed in the + /// overlay above (entries/marks/pending blobs/build moved src->dst). The durable publish + /// happens only in this transaction's commit (the existing publishStaging loop), not in + /// this method. `MergeTree` calls that commit from `Transaction::renameParts` while off + /// the `data_parts` lock and before the + /// Keeper multi. No early-published ref to compensate on abort within this method + /// (see ~ContentAddressedTransaction). + } + + if (had_staged_source) + { + /// A nested text-index sub-storage (MergeTask/MutateTask createTemporaryTextIndexStorage) + /// may have DURABLY published a committed scratch ref at THIS part's own path holding only + /// `/text_index_tmp/` files. That ref is not ours and is not staged; drop it now so the + /// overlay we publish in commit() is the authoritative manifest. Independent of our publish + /// timing (it targets an already-committed foreign ref), so it stays a call-time drop. + metadata_storage.partAccess()->dropRefIfPresent(src->refKey()); + return; + } + + /// Move any COMMITTED source ref (a merge/mutation result rename, DETACH, ATTACH, a + /// delete_tmp_ rename, an early-committed child ref being renamed away). Absent = a pure + /// staged/tmp move - nothing durable to touch. + metadata_storage.partAccess()->republishRef(src->refKey(), dst->refKey()); + return; + } + + throw Exception(ErrorCodes::LOGICAL_ERROR, + "ContentAddressed: moveDirectory from {} to {} has an unsupported shape", path_from, path_to); +} + +void ContentAddressedTransaction::moveFile(const std::string & path_from, const std::string & path_to) +{ + /// Write gate (rev.7 §1). (`replaceFile` delegates here, so its own gate below is defense in depth.) + metadata_storage.checkOpAdmitted(CasOpClass::Write); + /// Verbatim table-level files and loose mountpoint files: physically move the object (the + /// mutation entry tmp_mutation_N.txt -> mutation_N.txt rename; already durable from its finalize). + if (!Cas::isPartFilePath(path_from) && !Cas::isPartFilePath(path_to)) + { + auto move_table_verbatim = [&](const Cas::TableFilePath & src_tf, + const Cas::TableFilePath & dst_tf) + { + const Cas::RootNamespace src_ns = metadata_storage.liveNamespace(src_tf.table_uuid); + const Cas::RootNamespace dst_ns = metadata_storage.liveNamespace(dst_tf.table_uuid); + if (src_ns.string() == dst_ns.string() && src_tf.tail == dst_tf.tail) + return; + /// A verbatim rename is emulated as get(src) -> put(dst) -> remove(src) because object + /// storage has no atomic rename. SINGLE-WRITER CONTRACT: only the owning server renames its + /// own table-level verbatim files (mutation entries), so there is no concurrent writer to + /// race the blind put(dst) against — the put's last-writer-wins is safe under that contract. + /// Idempotent re-drive: if the source is already gone but the destination is present, a + /// previous drive completed this move — treat as done (matches a re-driven FS rename, which + /// is an ENOENT-tolerant no-op) instead of throwing FILE_DOESNT_EXIST. An unrelated + /// pre-existing destination can never reach this branch: destination names derive + /// deterministically from source names, and the SINGLE-WRITER contract means only this + /// move's own prior drive can have produced it. + /// The source resolves readably (a move reads it) and the destination the minting way (a move + /// writes it). A source namespace with no readable life has no file to move, which is the same + /// outcome as an absent object and takes the identical already-moved / genuinely-missing split + /// below -- so absence of a life is not a separate error path. + const auto src_life = metadata_storage.readableNamespaceFilesLife(src_ns); + const auto src_bytes = src_life + ? metadata_storage.store()->getNamespaceFile(*src_life, src_tf.tail) + : std::nullopt; + if (!src_bytes) + { + const auto dst_probe = metadata_storage.readableNamespaceFilesLife(dst_ns); + if (dst_probe && metadata_storage.store()->getNamespaceFile(*dst_probe, dst_tf.tail)) + return; + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: moveFile source missing: {}", path_from); + } + metadata_storage.store()->putNamespaceFile( + metadata_storage.store()->namespaceLife(dst_ns), dst_tf.tail, *src_bytes); + metadata_storage.store()->removeNamespaceFile(*src_life, src_tf.tail); + }; + auto src_tf = Cas::parseTableFilePath(path_from); + auto dst_tf = Cas::parseTableFilePath(path_to); + if (src_tf && dst_tf) + { + move_table_verbatim(*src_tf, *dst_tf); + return; + } + /// Loose mountpoint files (rare): read + put + remove plain objects. The same single-writer + /// contract + idempotent re-drive as the table-verbatim branch above. + const std::string src_key = metadata_storage.serverRootId() + "/" + path_from; + const std::string dst_key = metadata_storage.serverRootId() + "/" + path_to; + if (src_key == dst_key) + return; + auto bytes = metadata_storage.store()->getMountpointObject(src_key); + if (!bytes) + { + if (metadata_storage.store()->getMountpointObject(dst_key)) + return; + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "ContentAddressed: moveFile source missing: {}", path_from); + } + metadata_storage.store()->putMountpointObject(dst_key, *bytes); + metadata_storage.store()->removeMountpointObject(src_key); + return; + } + + auto src = routeOf(path_from); + auto dst = routeOf(path_to); + /// A part-DIRECTORY rename reaching moveFile (PartsTemporaryRename::rollBackAll undoes an + /// attach via moveFile): delegate to moveDirectory, which owns directory shapes. + if (src && dst && !src->ref.empty() && src->file.empty() && !dst->ref.empty() && dst->file.empty()) + { + moveDirectory(path_from, path_to); + return; + } + if (!src || src->file.empty() || !dst || dst->file.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "ContentAddressed: moveFile requires two part-file paths: {} -> {}", path_from, path_to); + + auto & src_st = stagingFor(*src); + auto & dst_st = stagingFor(*dst); + + /// Staged content entry re-keys in place (cross-part included; dependencies follow the entries). + /// canonical policy: SOURCE-wins — a move/rename carries the source's content to the destination, + /// overwriting any prior dest bytes (the POSIX `rename` semantic, and what the atomic-write + /// `.tmp -> final` rename requires). moveDirectory's staged merge is aligned to this same policy. + auto it = std::find_if(src_st.entries.begin(), src_st.entries.end(), + [&](const Cas::ManifestEntry & e) { return e.path == src->file; }); + if (it != src_st.entries.end()) + { + auto entry = std::move(*it); + src_st.entries.erase(it); + entry.path = dst->file; + if (&src_st != &dst_st && entry.placement == Cas::EntryPlacement::Blob) + { + /// Unified adopt dispatch. MOVE semantics — physically move the pending blob + /// record from src_st to dst_st FIRST (so dst_st owns the upload), then call adoptStagedBlob + /// with copy_pending=false (the record is already in dst_st; no additional copy needed). + auto pb_it = std::find_if(src_st.pending_blobs.begin(), src_st.pending_blobs.end(), + [&](const PartStaging::PendingBlob & pb) { return pb.ref == entry.ref; }); + if (pb_it != src_st.pending_blobs.end()) + { + dst_st.pending_blobs.push_back(std::move(*pb_it)); + src_st.pending_blobs.erase(pb_it); + } + adoptStagedBlob(findPendingBlob(dst_st, entry.ref), entry, dst_st, buildFor(*dst, dst_st), /*copy_pending=*/false); + } + std::erase_if(dst_st.entries, [&](const Cas::ManifestEntry & e) { return e.path == entry.path; }); + dst_st.entries.push_back(std::move(entry)); + return; + } + /// Source not staged in this transaction: this would cover a standalone one-shot rename of a + /// committed `txn_version.txt` file. Atomic-write storages (including CA) bypass that rename: + /// `VersionMetadataOnDisk::storeInfoToDataPartStorage` writes `txn_version.txt` directly, with no + /// `.tmp` + `replaceFile` dance. This branch therefore has no live caller and is retained only as + /// a fail-loud guard for an unsupported mutation shape. + throw Exception(ErrorCodes::LOGICAL_ERROR, "ContentAddressed: moveFile source not staged: {}", path_from); +} + +void ContentAddressedTransaction::replaceFile(const std::string & path_from, const std::string & path_to) +{ + /// Write gate (rev.7 §1): refuse before dropping staged destination state on a Vanished/uncertain disk. + metadata_storage.checkOpAdmitted(CasOpClass::Write); + /// replaceFile = moveFile that overwrites the destination. Drop any staged destination state + /// first, then delegate (the verbatim branch's putNamespaceFile already overwrites). + if (auto dst = routeOf(path_to); dst && !dst->file.empty()) + { + auto & dst_st = stagingFor(*dst); + /// A matching pending_blobs record (if any) is left in place — its temp file is cleaned by + /// cleanupPendingTempFiles at commit end, and the orphaned record is filtered out of the + /// publish upload by the staged-tree-hash check in publishStaging. We do NOT purge it + /// eagerly because the same hash may still be referenced by another staged entry. + std::erase_if(dst_st.entries, [&](const Cas::ManifestEntry & e) { return e.path == dst->file; }); + } + moveFile(path_from, path_to); +} + +void ContentAddressedTransaction::unlinkFile(const std::string & path, bool if_exists, bool /*should_remove_objects*/) +{ + /// Part file. Two sub-cases: + /// 1. A file STAGED in this transaction (content entry or legacy mutable bytes): drop the + /// staged state so it never reaches the published tree. + /// 2. A COMMITTED CONTENT file (not staged here): stage a REMOVAL MARK + /// (`content_removed`). The mark is resolved at publish (`publishStaging`): a repoint + /// republishes the manifest minus the removed paths, UNLESS this same transaction also + /// drops the whole part directory (`removeDirectory`), in which case the mark is + /// superseded — see `removeDirectory` below. + /// + /// This is a load-bearing invariant; do not "fix" it with a blanket fail-closed assert: + /// On a content-addressed disk a committed part is ONE atomic ref (its manifest tree); the removal + /// UNIT is the whole-part ref-drop done by `removeDirectory()`, NOT per-file unlinks. The + /// MergeTree fast-removal path (IMergeTreeDataPart::remove) unlinks EVERY part file one-by-one and + /// THEN calls `removeDirectory` — so a batched per-file unlink storm immediately followed by a + /// ref-drop in the SAME transaction must cost exactly one ref-drop and zero repoints, not one + /// repoint per unlinked file. `removeDirectory` clears any marks staged here for the same ref + /// before the transaction publishes, which is what makes the storm-then-drop shape free. A lone + /// surgical unlink NOT followed by a ref-drop in the same transaction (ATTACH's + /// `removeVersionMetadata`, a future backfill/repair delete) resolves to one repoint-remove — + /// this closes the file's former fail-open (a committed content file could never actually be + /// deleted on its own; this behavior now closes that earlier fail-open. + /// + /// Remove gate (rev.7 §1): a Vanished disk answers no-op success; a transient / IdentityLost disk + /// throws 668. + if (metadata_storage.checkOpAdmitted(CasOpClass::Remove) == CasOpAdmission::TruthAbsent) + return; + if (auto r = routeOf(path); r && !r->file.empty()) + { + auto & st = stagingFor(*r); + const bool staged_here = std::any_of(st.entries.begin(), st.entries.end(), + [&](const Cas::ManifestEntry & e) { return e.path == r->file; }); + /// A matching pending_blobs record (if any) is left in place — its temp file is cleaned by + /// cleanupPendingTempFiles at commit end, and the orphaned record is filtered out of the + /// publish upload by the staged-tree-hash check in publishStaging. We do NOT purge it + /// eagerly because the same hash may still be referenced by another staged entry. + std::erase_if(st.entries, [&](const Cas::ManifestEntry & e) { return e.path == r->file; }); + if (!staged_here) + { + /// One mandatory body-HEAD per (transaction, ref), not per file: the MergeTree fast-removal + /// path unlinks every file of the part through THIS transaction right before removeDirectory. + /// The first unlink re-proves the body ForceFresh; the rest of the burst reuses that proof. + const String memo_key = r->refKey().cacheKey(); + const bool already_proven = force_fresh_validated_refs.contains(memo_key); + const auto view = metadata_storage.partAccess()->getView( + r->refKey(), already_proven ? Cas::Freshness::CachedForLoad : Cas::Freshness::ForceFresh); + if (view && !already_proven) + force_fresh_validated_refs.insert(memo_key); + if (!view || !view->hasFile(r->file)) + { + if (if_exists) + return; + throw Exception( + ErrorCodes::FILE_DOESNT_EXIST, + "ContentAddressed: unlinkFile target does not exist: {}", + path); + } + st.content_removed.insert(r->file); + } + return; + } + + /// Verbatim table-level / loose mountpoint file: reclaim the object NOW (GC never scans them; + /// a pruned mutation entry would otherwise leak until DROP. + if (auto tf = Cas::parseTableFilePath(path)) + { + /// Readable resolution, i.e. the non-creating one: an unlink must not birth a namespace -- and + /// `unlinkFile(..., if_exists = true)` is called from cleanup paths whose whole contract is to be + /// a no-op. No life means no such file, exactly the absent case the branch below already handles. + const auto life = metadata_storage.readableNamespaceFilesLife( + metadata_storage.liveNamespace(tf->table_uuid)); + if (!life || !metadata_storage.store()->getNamespaceFile(*life, tf->tail)) + { + if (if_exists) + return; + throw Exception( + ErrorCodes::FILE_DOESNT_EXIST, + "ContentAddressed: unlinkFile target does not exist: {}", + path); + } + metadata_storage.store()->removeNamespaceFile(*life, tf->tail); + return; + } + /// Loose mountpoint file: exact-token delete of the plain object. + const String key = metadata_storage.serverRootId() + "/" + path; + if (!metadata_storage.store()->getMountpointObject(key)) + { + if (if_exists) + return; + throw Exception( + ErrorCodes::FILE_DOESNT_EXIST, + "ContentAddressed: unlinkFile target does not exist: {}", + path); + } + metadata_storage.store()->removeMountpointObject(key); +} + +void ContentAddressedTransaction::truncateFile(const std::string &, size_t) +{ + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "truncateFile is not supported on a content-addressed disk (blobs are immutable; " + "whole-file rewrites replace the staged entry instead)"); +} + +} + +namespace DB::Cas +{ + +namespace +{ +/// Ceiling for a CAS write-buffer allocation. An extreme `max_compress_block_size` (or any other +/// out-of-range buffer-size setting -- fuzzed or misconfigured) flows into `writeFile`'s `buf_size` / +/// `adaptive_write_buffer_initial_size` and, unclamped, reaches `Memory::alloc` where the allocator's +/// `checkSize` (>= 0x8000000000000000) fires a `LOGICAL_ERROR` and aborts the server in +/// debug/sanitizer builds. The ordinary MergeTree writers clamp compress-block sizes to 256 MiB +/// (`MergeTreeWriterSettings::MAX_COMPRESS_BLOCK_SIZE`) for exactly this reason; the CAS write path +/// received the value unclamped. Mirror that ceiling here, at the allocation site, so no caller can +/// pass an absurd size to the allocator. 256 MiB is duplicated (not #included) to keep the Disks layer +/// free of a Storages/MergeTree dependency; the regression guard is +/// `04070_no_crash_extreme_compress_block_size` run on a content-addressed storage policy. +constexpr size_t kMaxCasWriteBufferBytes = 256ULL * 1024 * 1024; + +size_t clampCasWriteBufferSize(size_t size) +{ + return std::min(size, kMaxCasWriteBufferBytes); +} +} + +void fanOutBlobUploads( + PartWriteTxn & build, + std::span requests, + ThreadPool & pool, + const BlobUploadFanoutHooksForTest * hooks) +{ + /// Group by unique ref. Staged-hardlink copies push a DUPLICATE pending-blob record for one BlobRef, + /// and the fan-out must launch exactly ONE task per unique ref (spec §1 "One task per unique ref"). + /// An ordered map gives a DETERMINISTIC dispatch order (ascending `BlobRef`), which fixes the "first + /// error" of the merge-nothing contract to a stable task so a failure is reproducible. + std::map grouped; + for (const auto & req : requests) + { + /// Fail-close: the fan-out groups and conflict-checks on `declared_size`, while `source.size` is + /// the per-attempt streaming byte authority. A wiring bug that let them diverge would group on + /// one value while streaming another — reject it rather than upload a wrong-length body. + if (req.declared_size != req.source.size) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "fanOutBlobUploads: request for {} declares size {} but its source is sized {} -- the " + "grouping key and the streaming byte authority must agree", + blobIdOf(req.ref), req.declared_size, req.source.size); + const auto [it, inserted] = grouped.try_emplace(req.ref, req); + if (!inserted && it->second.declared_size != req.declared_size) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "fanOutBlobUploads: conflicting declared sizes for {} ({} vs {}) -- staged-hardlink " + "copies of one ref must agree on size (one task, one dep per unique ref)", + blobIdOf(req.ref), it->second.declared_size, req.declared_size); + } + + if (grouped.empty()) + return; + + ProfileEvents::increment(ProfileEvents::CASBlobUploadFanoutBatches); + + /// One result slot per unique ref, pre-sized so element addresses are STABLE: each task writes ONLY + /// its own slot (no data race on the vector) and the vector never reallocates. Declared BEFORE the + /// runner and the drain guard so it OUTLIVES them — the drain guard joins every scheduled task before + /// `results` is destroyed on EVERY path, including a throw raised during the dispatch loop below (the + /// B90 lesson, `threadPoolCallbackRunner.h:68`). Tasks capture only owning/value state: a stable slot + /// pointer, the request by value (its source is captured by value too), and the txn pointer whose + /// pointee outlives the runner. + std::vector results(grouped.size()); + + { + /// `pool` is disjoint from the S3 writer pool an upload may itself use, and the calling thread + /// only submits and joins — it never occupies a pool slot — so a size-1 pool degenerates to a + /// correct serial run and can never deadlock. `ThreadName::UNKNOWN`: stage-1 must not add a + /// CAS-specific `ThreadName` to the shared enum (`setThreadName.h` is outside the allowed file + /// set), and UNKNOWN is the no-op name; it also stays clear of the pool-thread self-join + /// assertions in `CasPool`. The query `ThreadGroup` is still propagated per task by the runner. + using RunnerTask = ThreadPoolCallbackRunnerLocal::Task; + ThreadPoolCallbackRunnerLocal runner(pool, ThreadName::UNKNOWN); + const PartWriteTxn * txn = &build; /// `uploadBlobDetached` is const + build-neutral: safe off-thread + /// We track scheduled tasks in OUR OWN vector (via `enqueueAndGiveOwnership`) rather than the + /// runner's `enqueueAndKeepTrack`: that helper schedules a task and only THEN appends its handle + /// to an UNRESERVED tracking vector, so a `bad_alloc` at the append would leave a + /// scheduled-but-untracked task the runner's destructor cannot join — it would run later against + /// the already-destroyed `results`/txn (a use-after-free; codex stage-1 review, Critical). + /// PRE-RESERVING `handles` to the exact task count makes the append after each schedule a + /// no-throw operation, so a task is NEVER scheduled without being tracked in the SAME expression. + std::vector> handles; + handles.reserve(grouped.size()); + /// Drain on EVERY path: the runner's destructor only joins tasks IT owns (we use + /// `enqueueAndGiveOwnership`, so its own set stays empty), so WE must join every scheduled task + /// before `results` and the upload sources are destroyed — including when the dispatch loop + /// throws (an `on_dispatch`/`after_enqueue` seam, or a scheduling failure mid-loop). Declared + /// AFTER `handles`/`runner` so it runs FIRST on scope exit, joining while both are still alive; + /// `waitForAllToFinish` only waits (never throws), so it is safe on the unwinding path (the B90 + /// lesson, `threadPoolCallbackRunner.h:68`). + SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); }); + size_t idx = 0; + for (const auto & [ref, req] : grouped) + { + BlobUploadResult * slot = &results[idx++]; + BlobUploadRequest task_req = req; + const BlobUploadFanoutHooksForTest * task_hooks = hooks; + if (hooks && hooks->on_dispatch) + hooks->on_dispatch(ref); /// may throw ⇒ the drain guard joins already-scheduled tasks + /// Schedule and track in ONE no-throw step: `enqueueAndGiveOwnership` returns the handle (the + /// task is now runnable) and the pre-reserved `emplace_back` records it without allocating, so + /// there is no window in which a scheduled task is untracked. + handles.emplace_back(runner.enqueueAndGiveOwnership([slot, txn, req_by_value = std::move(task_req), task_hooks] + { + if (task_hooks && task_hooks->in_task) + task_hooks->in_task(req_by_value.ref); + *slot = txn->uploadBlobDetached(req_by_value); + })); + if (task_hooks && task_hooks->after_enqueue) + task_hooks->after_enqueue(ref); /// task already tracked ⇒ the drain guard joins it too + ProfileEvents::increment(ProfileEvents::CASBlobUploadFanoutTasks); + } + /// Drain ALL tasks, then rethrow the FIRST (ascending-ref dispatch order) that failed. A rethrow + /// bypasses the merge below, so NOTHING is merged: `build` stays at its pre-fan-out pending-dep + /// state (merge-nothing). On success this clears `handles`, so the drain guard above then waits an + /// empty set; on any throw it leaves them and the guard joins the survivors (already all done). + ThreadPoolCallbackRunnerLocal::waitForAllToFinishAndRethrowFirstError(handles); + } + + /// Every task succeeded (else we rethrew above): fold all results into `build` on this (the owning + /// writer) thread, all-or-nothing (`mergeBlobUploadResults` prevalidates, then build-and-swaps). + build.mergeBlobUploadResults(results); +} + + +CaContentWriteBuffer::CaContentWriteBuffer( + std::string temp_dir, + Cas::BlobHashAlgo hash_algo, + size_t buf_size, + bool use_adaptive_buffer_size, + size_t adaptive_buffer_initial_size, + OnFinalized on_finalized_) + : WriteBufferFromFileBase(clampCasWriteBufferSize(use_adaptive_buffer_size ? adaptive_buffer_initial_size : buf_size), nullptr, 0) + , on_finalized(std::move(on_finalized_)) +{ + fs::create_directories(temp_dir); + temp_path = temp_dir + "/" + getRandomASCIIString(32) + ".tmp"; + + /// The spill buffer is a SECOND per-stream buffer; thread the adaptive flag into it too so a + /// wide part keeps its footprint small. Its IO is a local temp file, not the remote stream. + sink = std::make_unique( + temp_path, + clampCasWriteBufferSize(buf_size), + /*flags=*/-1, + /*throttler=*/nullptr, + /*mode=*/0666, + /*existing_memory=*/nullptr, + /*alignment=*/0, + use_adaptive_buffer_size, + clampCasWriteBufferSize(adaptive_buffer_initial_size)); + hashing = Cas::makeBlobHashingWriteBuffer(hash_algo, *sink); +} + +CaContentWriteBuffer::CaContentWriteBuffer( + std::unique_ptr object_store_sink, + std::string object_key, + std::string envelope_header, + Cas::BlobHashAlgo hash_algo, + size_t buf_size, + bool use_adaptive_buffer_size, + size_t adaptive_buffer_initial_size, + OnFinalized on_finalized_, + std::function check_fence_before_finalize_) + : WriteBufferFromFileBase(clampCasWriteBufferSize(use_adaptive_buffer_size ? adaptive_buffer_initial_size : buf_size), nullptr, 0) + , on_finalized(std::move(on_finalized_)) + , temp_path(std::move(object_key)) + , is_s3_staging(true) + , sink(std::move(object_store_sink)) + , check_fence_before_finalize(std::move(check_fence_before_finalize_)) +{ + /// The sink is ALREADY opened against the staging object by the caller (writeFile) — this + /// constructor wraps it in the hashing chain, exactly like the local-temp-file mode. + /// + /// Write the CABL envelope header to the sink first, directly — + /// bypassing `hashing` (so it is excluded from the content hash) and this outer buffer's `count()` + /// (so the reported size is the payload only). The staging object therefore holds `[header][payload]` + /// and the promote stays a verbatim server-side copy. Only the payload the caller subsequently writes + /// through THIS buffer flows through `hashing`. The header write precedes any payload write, so the + /// on-object byte order is header-then-payload. + if (!envelope_header.empty()) + sink->write(envelope_header.data(), envelope_header.size()); + + /// The adaptive-sizing params only affect THIS outer buffer (mirroring the Local ctor above); the + /// sink's own buffering was decided by the caller when it opened the object-store write. + hashing = Cas::makeBlobHashingWriteBuffer(hash_algo, *sink); +} + +CaContentWriteBuffer::~CaContentWriteBuffer() +{ + /// Best-effort cleanup if finalize was never reached (exception unwind / cancel). + cancel(); + /// If on_finalized ran successfully the transaction (Local mode) or a later promote + /// path (S3 mode) owns the staged bytes and cleans them up. Do not remove them here. S3-mode + /// staging objects are never removed by this class at all (see cancelImpl / removeTempFile). + if (!temp_ownership_transferred && !is_s3_staging) + removeTempFile(); +} + +void CaContentWriteBuffer::nextImpl() +{ + if (!offset()) + return; + hashing->write(working_buffer.begin(), offset()); +} + +void CaContentWriteBuffer::finalizeImpl() +{ + next(); + const size_t size = count(); + + /// getHashHex flushes the chain and returns the streaming digest (the pool's selected algo) of + /// everything written, as 32 lowercase hex chars. + const std::string hash_hex = hashing->getHashHex(); + + hashing->finalize(); + + /// rev.7 [C2]: re-check the fence-generation admission IMMEDIATELY before the durable backend call + /// (S3 mode's `sink->finalize()` completes the staging object -- Local mode never sets this + /// callback). A fence trip or re-arm since construction aborts here with the typed transient error, + /// before the upload becomes durable. + if (check_fence_before_finalize) + check_fence_before_finalize(); + + sink->finalize(); + + /// On successful finalize, ownership of temp_path (Local: the local temp path; S3: the + /// staging object key) transfers to the caller (the transaction uploads/promotes it and cleans + /// up). cancel() still removes/cancels it. + if (on_finalized) + { + on_finalized(hash_hex, size, temp_path); + temp_ownership_transferred = true; + } +} + +void CaContentWriteBuffer::cancelImpl() noexcept +{ + if (hashing) + hashing->cancel(); + if (sink) + sink->cancel(); + /// S3 mode: `temp_path` is a remote object key, not a path on this filesystem — do NOT attempt + /// to delete the (possibly partially-written) staging object here. Cancelling `sink` above is + /// enough to make sure no partial finalize happens; reclaiming an orphaned staging object is the + /// mount-lease sweeper's job. + if (!is_s3_staging) + removeTempFile(); +} + +void CaContentWriteBuffer::removeTempFile() noexcept +{ + std::error_code ec; + fs::remove(temp_path, ec); +} + +void CaContentWriteBuffer::sync() +{ + next(); + hashing->next(); + sink->sync(); +} + +std::string CaContentWriteBuffer::getFileName() const +{ + return temp_path; +} + +CaInlineWriteBuffer::CaInlineWriteBuffer(OnInlined on_inlined_) + : WriteBufferFromFileBase(DBMS_DEFAULT_BUFFER_SIZE, nullptr, 0) + , on_inlined(std::move(on_inlined_)) +{ +} + +CaInlineWriteBuffer::~CaInlineWriteBuffer() +{ + cancel(); +} + +void CaInlineWriteBuffer::nextImpl() +{ + if (!offset()) + return; + accumulated.append(working_buffer.begin(), offset()); +} + +void CaInlineWriteBuffer::finalizeImpl() +{ + next(); + if (on_inlined) + on_inlined(std::move(accumulated)); +} + +void CaInlineWriteBuffer::sync() +{ + next(); +} + +std::string CaInlineWriteBuffer::getFileName() const +{ + return "ca_inline"; +} + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.h new file mode 100644 index 000000000000..90fc00c01352 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.h @@ -0,0 +1,420 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +/// Owns the metadata-side overlay for one object-storage transaction. Part files are accumulated by +/// routed namespace/ref, then published as manifest trees and refs by `commit`; verbatim namespace +/// and mountpoint files are written immediately when their buffers finalize. A part uses one +/// `Cas::PartWriteTxn`, created lazily before the first staged dependency, so the durable manifest +/// edge is established before the pool observes or uploads a new blob. +/// +/// Content blobs are first represented by local or S3 staging objects and are uploaded only during +/// publication. Inline entries remain in the manifest tree when they fit `INLINE_CAP`. The +/// destructor removes private local staging, leaves aborted S3 staging for the mount-lease sweeper, +/// and abandons any open part builds; it never publishes an uncommitted ref. +class ContentAddressedTransaction : public IMetadataTransaction +{ +public: + /// Borrows the metadata storage for the transaction lifetime; staged builds and resources are + /// owned by this transaction and finalized or abandoned by `commit`/destruction. + explicit ContentAddressedTransaction(ContentAddressedMetadataStorage & metadata_storage_); + + bool supportsChmod() const override { return false; } + + /// Publishes every staged part. Publication is ordered so each new blob is named by a durable + /// precommit edge first; if a later part fails, only refs created by this call are compensated. + void commit(const TransactionCommitOptionsVariant & options) override; + + /// Accepts only `NoCommitOptions`, delegates to `commit`, and returns its successful outcome. + TransactionCommitOutcomeVariant tryCommit(const TransactionCommitOptionsVariant & options) override; + + /// Unsupported because content-addressed keys are generated from payload hashes, not paths. + ObjectStorageKey generateObjectKeyForPath(const std::string & path) override; + /// Returns no removal list: shared CAS objects are reclaimed only by garbage collection. + StoredObjects getSubmittedForRemovalBlobs() override; + + /// Resolves a staged entry to its committed-style storage description. Pending blobs return no + /// object because their bytes still reside in staging and must be read through the overlay path. + std::optional tryGetInFlightStorageObjects(const std::string & path) const override; + /// Opens an inline entry, pending local/S3 staging object, or already uploaded blob for a + /// read-your-writes operation; returns null when the path is outside this transaction's overlay. + std::unique_ptr tryReadFileInFlight( + const std::string & path, const ReadSettings & settings, std::optional read_hint) const override; + /// Returns the logical size of a staged entry, including inline bytes and pending blob payloads. + std::optional tryGetInFlightFileSize(const std::string & path) const override; + /// Reports only inner staged directories. The bare part directory intentionally remains absent + /// so cleanup of a temporary, dedup-rejected part does not treat it as a real directory. + bool hasInFlightDirectory(const std::string & path) const override; + /// Lists immediate child names visible in the staged directory overlay. + std::vector listInFlightDirectory(const std::string & path) const override; + + /// This legacy object-list operation has no content-addressed equivalent and throws. + void createMetadataFile(const std::string & path, const StoredObjects & objects) override; + + /// Creates the buffer used by the disk transaction for this path. The returned wrapper keeps + /// `owner` alive until deferred finalization, because its callback captures this transaction; + /// it also applies the content-addressed append and autocommit rules before selecting a blob, + /// inline, or verbatim-file buffer. Part blobs cannot be published independently of their + /// manifest, while an inline-eligible standalone part file may commit through the repoint path. + std::unique_ptr tryCreateWriteBuffer( + const std::shared_ptr & owner, + const std::string & path, size_t buf_size, WriteMode mode, + const WriteSettings & settings, bool autocommit) override; + + /// Creates the inner buffer for a content-addressed path. Part blobs are hashed while being + /// staged, small metadata files are accumulated in memory and classified at finalize, and + /// verbatim namespace or mountpoint files are read-modify-written when append is requested. + std::unique_ptr writeFile( + const std::string & path, + size_t buf_size, + WriteMode mode, + const WriteSettings & settings); + + /// Directory creation is a metadata no-op because object storage has no directory objects. + void createDirectory(const std::string & path) override; + /// Recursive directory creation is likewise a no-op; entries create their own prefixes. + void createDirectoryRecursive(const std::string & path) override; + /// Drops a part ref, or does nothing for a non-part directory. A whole-part drop supersedes any + /// per-file removal marks staged earlier in this transaction. + void removeDirectory(const std::string & path) override; + /// Removes refs, namespace files, and shadow objects while leaving shared CAS objects to GC. + void removeRecursive(const std::string & path, const ShouldRemoveObjectsPredicate & should_remove_objects) override; + /// Copies an entry between parts, preserving pending staging ownership and tokenless evidence. + void createHardLink(const std::string & path_from, const std::string & path_to) override; + /// These filesystem metadata operations are unsupported because CAS metadata is immutable and + /// object storage exposes neither POSIX timestamps nor mode bits. + void setLastModified(const std::string & path, const Poco::Timestamp & timestamp) override; + void chmod(const String & path, mode_t mode) override; + void setReadOnly(const std::string & path) override; + /// Re-keys or merges staged part entries without publishing early; non-part paths use the + /// corresponding object-storage copy/remove semantics. + void moveDirectory(const std::string & path_from, const std::string & path_to) override; + /// Moves a staged entry and its pending ownership, or copies/removes a verbatim object as needed. + void moveFile(const std::string & path_from, const std::string & path_to) override; + /// Replaces the destination while preserving the source's content-addressed ownership rules. + void replaceFile(const std::string & path_from, const std::string & path_to) override; + /// Unlinks a staged entry or records a removal mark for an already committed manifest; shared + /// blobs are never deleted directly. + void unlinkFile(const std::string & path, bool if_exists, bool should_remove_objects) override; + /// Truncation is not representable without rewriting the content and is unsupported. + void truncateFile(const std::string & path, size_t size) override; + + /// Abandons open builds and cleans owned staging without publishing an uncommitted ref. + ~ContentAddressedTransaction() override; + +protected: + ContentAddressedMetadataStorage & metadata_storage; + +private: + /// State for one routed part. The build is opened lazily at the first staged write so every + /// manifest dependency is recorded by the same `Cas::PartWriteTxn`; pending blobs remain in + /// staging until publication. + struct PartStaging + { + /// Created lazily when a staged blob, inline entry, or adopted entry first needs publication. + Cas::PartWriteTxnPtr build; + std::vector entries; /// staged manifest entries (uploads + adoptions) + /// Paths removed from a committed part's manifest by this transaction. Publication carries + /// forward all other committed entries. If `removeDirectory` drops the whole ref in the same + /// transaction, that ref-drop supersedes these marks and clears them. + std::set content_removed; + bool published = false; /// set by publishStaging during commit(); the commit + /// loop is idempotent (never re-publishes a staging). + + /// `staging_key` is either a private local temp path or an S3 staging object key returned by + /// `CaContentWriteBuffer` at finalize. The backend selects cleanup and read-your-writes: + /// local files are removed by this transaction, whereas aborted S3 objects must remain + /// available to the mount-lease sweeper and to recovery of the promote source. + struct PendingBlob { Cas::BlobRef ref; std::string staging_key; uint64_t size = 0; Cas::StagingBackend backend = Cas::StagingBackend::Local; }; + std::vector pending_blobs; /// Staged blobs uploaded after the manifest edge is precommitted. + }; + + /// Keyed by (namespace string, ref name) — the routed identity, so live/detached/shadow + /// stagings never collide. + std::map, PartStaging> parts; + bool committed = false; + bool failed = false; + + /// Memoizes, per (this transaction, ref), whether `unlinkFile` has already re-proven a committed + /// ref's manifest body `ForceFresh`. The MergeTree fast-removal path unlinks every file of a part + /// through ONE transaction right before `removeDirectory`: the first unlink's `ForceFresh` view + /// proves the body once; the rest of the burst reuse that proof (`Cas::Freshness::CachedForLoad`) + /// instead of paying one manifest-body HEAD per file. Cleared in `commit()`'s epilogue. + std::unordered_set force_fresh_validated_refs; + + /// Stage a CONTENT part file as a blob: record the pending upload + a tokenless dependency + /// and add/replace its manifest entry. Shared by the streaming-blob path + /// (Local or S3-staging, `backend` says which) and the always-Local inline-cap fallback. + void stageBlobPartFile(const ContentAddressedMetadataStorage::Route & route, + const Cas::BlobRef & ref, size_t size, const std::string & staging_key, + Cas::StagingBackend backend); + + /// Builds the fixed-length CABL envelope header for a staging blob, with a fresh `incarnation_tag`, + /// so the S3 staging object holds `[header][payload]` and the promote + /// stays a verbatim server-side copy. `build_id` is left 0 (not known at stream time; diagnostic-only). + std::string buildS3StagingBlobHeader(const ContentAddressedMetadataStorage::Route & route) const; + + /// Returns (and, when necessary, creates) the staging state for a routed namespace/ref. + PartStaging & stagingFor(const ContentAddressedMetadataStorage::Route & r); + /// Finds existing staging state without creating an entry; returns nullptr when untouched. + PartStaging * findStaging(const ContentAddressedMetadataStorage::Route & r); + /// Finds the staged manifest entry for a routed file without consulting committed storage. + const Cas::ManifestEntry * findStagedEntry(const ContentAddressedMetadataStorage::Route & r) const; + /// Returns the pending (staged but not yet uploaded) blob for `ref`, or nullptr when it has + /// already been uploaded or was never staged. + const PartStaging::PendingBlob * findPendingBlob(const PartStaging & st, const Cas::BlobRef & ref) const; + /// Returns the part build, creating it lazily with the routed ref as its intended destination. + Cas::PartWriteTxn & buildFor(const ContentAddressedMetadataStorage::Route & r, PartStaging & st); + /// Parses a disk path and maps a part-file path to its namespace/ref/file route. + std::optional routeOf(const std::string & path) const; + + /// Removes local staging files after commit or abort. On a successful commit it also removes + /// S3 staging objects; aborted S3 objects are intentionally retained for lease-scoped cleanup. + void cleanupPendingTempFiles() noexcept; + + /// Uploads through `st.build` only the pending blobs still referenced by `st.entries`. An entry + /// removed by `unlinkFile` or `replaceFile` is skipped, but its staging resource is still cleaned + /// by `cleanupPendingTempFiles`. Used for both new refs and committed-ref repoints. + void uploadPendingBlobs(PartStaging & st); + + /// Adopts a manifest entry into another part while preserving its storage state. For a pending + /// blob, `copy_pending` controls whether the staging record is copied (hardlink semantics) or + /// has already been moved by the caller; either way the destination records a dependency. For + /// an uploaded or committed blob, the destination records tokenless evidence and does not + /// perform a pool read before precommit. + /// + /// `pb != nullptr` (pending, not yet uploaded): + /// - `copy_pending=true` → push a copy of *pb into dst_st.pending_blobs (hardlink semantics: + /// both src and dst upload independently; src's copy is left in place by the caller). + /// - `copy_pending=false` → the pb record is already in dst_st (moved or already there); + /// just record the dep without any additional push. + /// In both cases: dst_build.recordPendingBlobDep(entry.file_hash, entry.file_size). + /// + void adoptStagedBlob(const PartStaging::PendingBlob * pb, const Cas::ManifestEntry & entry, + PartStaging & dst_st, Cas::PartWriteTxn & dst_build, bool copy_pending); + + /// Publishes one staged part, either by promoting a newly staged manifest or by repointing an + /// existing ref after carrying its unchanged entries forward. It is idempotent within the commit + /// loop and marks the staging as published. Writes the exact `Cas::CommitOutcome` into `out_slot` + /// the INSTANT `promoteBuild`/`repointRef` confirms -- before any further throwable work (the + /// scratch-build abandon, or a test hook) -- so `commit` can roll back precisely with + /// `dropRefIfMatches` even when this call later throws. `out_slot` is left `std::nullopt` when this + /// staging had nothing to publish or was already published earlier in this commit loop; `commit` + /// preallocates one slot per part so this write is a no-throw, index-addressed slot write that never + /// grows a container -- it exists for the serial publish loop's rollback bookkeeping. Parts are + /// published serially (`commit`'s loop); only the blob uploads WITHIN a part fan out. Concurrent + /// cross-part publication is future scope and is NOT done here. + void publishStaging(const Cas::RootNamespace & ns, const std::string & ref, PartStaging & st, + std::optional & out_slot); +}; + +} + +namespace DB::Cas +{ + +/// Part files that must NOT be inlined into the tree: per-column data (`.bin`) and marks (`.mrk*`/ +/// `.cmrk*`) — inlining them would force a full-part fetch and destroy column-read selectivity — plus +/// `primary.idx`, which can be large (a size-threshold inlining of small primary.idx is a follow-up). +/// Everything else (the small eager metadata files) is an inline candidate, subject to INLINE_CAP. +bool partFileMustStayBlob(std::string_view file_name); + +/// Test-only fault-injection seams for the intra-part blob-upload fan-out. Inert in production +/// (`ContentAddressedTransaction::uploadPendingBlobs` passes `nullptr`). +struct BlobUploadFanoutHooksForTest +{ + /// Invoked ONCE per DISPATCHED task, on the calling (dispatch) thread, immediately before the task + /// is enqueued, with the unique `BlobRef` the task will upload. Lets a test count tasks (proving the + /// one-task-per-unique-ref grouping) and inject a throw mid-dispatch (to prove the runner's scope + /// destructor drains every already-scheduled task before the stack unwinds). + std::function on_dispatch; + /// Invoked at the TOP of each pool task (on the pool thread) BEFORE `uploadBlobDetached`, with the + /// task's `BlobRef`. Lets a test rendezvous tasks on a latch (concurrent dedup-cache insertion, pool + /// saturation) or fail a specific task deterministically, all without a sleep. + std::function in_task; + /// Invoked on the DISPATCH thread immediately AFTER a task has been scheduled AND recorded in the + /// fan-out's own tracking vector, with that task's `BlobRef`. A throw here exercises the invariant + /// that no scheduled task is ever left untracked: because tracking publication is a no-throw append + /// into a pre-reserved vector, the task is already tracked when this fires, so the drain-on-every-path + /// guard joins it before the captured `results` storage is destroyed. + std::function after_enqueue; +}; + +/// Fan out a part's pending blob uploads across `pool` and merge the results into `build` +/// all-or-nothing (spec §1 "Parallel blob upload within a part"). It: +/// - groups `requests` by `BlobRef` (staged-hardlink copies push duplicate records for one ref), so +/// it launches EXACTLY ONE `uploadBlobDetached` task per unique ref and merges exactly one dep; +/// - rejects, before any task runs, a request whose `declared_size` disagrees with its `source.size` +/// (a wiring bug) and duplicate records for one ref that declare conflicting sizes (a staging bug), +/// both as `LOGICAL_ERROR`; +/// - runs each task's `uploadBlobDetached` (a `const`, build-neutral primitive) on `pool` while the +/// calling thread only submits and joins -- never occupying a pool slot -- so a size-1 pool +/// degenerates to a correct serial run and can never deadlock; +/// - honours the MERGE-NOTHING failure contract: the join always drains every task (including on a +/// throw raised during the dispatch loop, because a scope-exit drain guard joins every +/// already-scheduled task on the unwinding path); if ANY task threw, NOTHING is merged (`build` +/// stays byte-for-byte at its pre-fan-out state) and the FIRST task error in ascending-`BlobRef` +/// dispatch order is rethrown. +/// The query `ThreadGroup` is propagated to each task the `ThreadPoolCallbackRunnerLocal` way. +void fanOutBlobUploads( + PartWriteTxn & build, + std::span requests, + ThreadPool & pool, + const BlobUploadFanoutHooksForTest * hooks = nullptr); + +/// Writes a CONTENT part file while computing its content hash. The blob key is only known +/// once all bytes are written, so the buffer spills to a unique local temp file while hashing with +/// `hash_algo` (`Cas::makeBlobHashingWriteBuffer` — the pool's selectable blob-hash +/// function the wiring defines; the core never re-hashes payloads). `CityHash128` stays the thin +/// `HashingWriteBuffer` adapter (byte-for-byte unchanged); `XXH3_128` hashes with xxh3 instead. On +/// finalize it hands (hash_hex, size, temp_path) +/// to the owning transaction; the transaction owns the staging resource after finalize and uploads it +/// post-precommit, so finalizeImpl no longer removes it. cancelImpl and the destructor (on error +/// paths) still remove it. +/// +/// In S3 staging mode: +/// a SECOND constructor streams directly to an already-opened object-store sink (an S3 staging +/// object) while hashing, instead of spilling to a local temp file — see its own doc comment below. +/// The local-temp-file constructor above is UNCHANGED byte-for-byte; this is an independent mode +/// selected only by which constructor the caller uses. +class CaContentWriteBuffer : public WriteBufferFromFileBase +{ +public: + using OnFinalized = std::function; + + /// Local-staging mode (today's default; BYTE-FOR-BYTE unchanged behavior). Buffer sizing mirrors + /// the plain object-storage backends: with adaptive sizing on, the working buffer STARTS small + /// and grows (what min_columns_to_activate_adaptive_write_buffer toggles — a wide part keeps its + /// per-INSERT footprint small). + CaContentWriteBuffer( + std::string temp_dir, + Cas::BlobHashAlgo hash_algo, + size_t buf_size, + bool use_adaptive_buffer_size, + size_t adaptive_buffer_initial_size, + OnFinalized on_finalized_); + + /// S3-native staging mode: `object_store_sink` is an ALREADY-OPENED write buffer over the staging + /// object at `object_key` (e.g. `object_storage->writeObject(StoredObject(object_key), ...)`). + /// + /// `envelope_header` is the fixed-length (`blob_header_len`) CABL envelope header the transaction + /// built for this staging blob. It is written to the sink FIRST — + /// UNHASHED and NOT counted in the reported size — so the staging object holds `[header][payload]` + /// and the promote can stay a verbatim server-side copy. Excluding the header from the hash is + /// CRITICAL: the content key must be the pool's selected hash of `payload` alone (else the random + /// `incarnation_tag` in the header would make every blob's key unique ⇒ zero dedup), and the reported + /// blob size must be the + /// PAYLOAD size (else the manifest `blob_size` would be payload+`blob_header_len`). Only the PAYLOAD + /// bytes written through THIS buffer flow through `hashing` and `count()`. + /// + /// Bytes are hashed while streaming into `object_store_sink`; on finalize `on_finalized` receives + /// `object_key` as its third argument (in place of a local temp path) and `getFileName()` returns + /// it too. `cancelImpl` only cancels `object_store_sink` — it never attempts to delete the + /// (possibly partially-written) staging object; reclaiming an orphaned staging object after a + /// cancelled write belongs to the mount-lease sweeper, not this buffer. + /// + /// `check_fence_before_finalize_` is the rev.7 [C2] fence-generation admission for this durable write + /// (the ONLY durable backend effect of the Local-mode constructor above is a private scratch file, so + /// it takes none): it is invoked immediately before the durable `sink->finalize()` call in + /// `finalizeImpl`, aborting with the typed transient error on a fence trip or re-arm since admission. It + /// defaults empty so the existing direct-construction unit tests (`gtest_cas_s3_staging.cpp`), which + /// exercise the buffer mechanics without a real `CasMountRuntime`, are unaffected. + CaContentWriteBuffer( + std::unique_ptr object_store_sink, + std::string object_key, + std::string envelope_header, + Cas::BlobHashAlgo hash_algo, + size_t buf_size, + bool use_adaptive_buffer_size, + size_t adaptive_buffer_initial_size, + OnFinalized on_finalized_, + std::function check_fence_before_finalize_ = {}); + + ~CaContentWriteBuffer() override; + + void sync() override; + std::string getFileName() const override; + +private: + /// Feeds bytes to the hashing/staging sink while preserving the base write-buffer contract. + void nextImpl() override; + /// Finalizes the sink, computes the content hash, and transfers the staging resource through + /// `on_finalized`; after that callback succeeds, the transaction owns cleanup. + void finalizeImpl() override; + /// Cancels the sink and removes local staging. S3 staging is left for lease-scoped reclamation. + void cancelImpl() noexcept override; + /// Removes the local staging path when ownership has not been transferred to the transaction. + void removeTempFile() noexcept; + + OnFinalized on_finalized; + /// Local mode: the local temp file path (removed by removeTempFile). S3 mode: the staging + /// object's key (never fs::remove'd — see is_s3_staging below). + std::string temp_path; + /// Selects the S3-staging semantics in cancelImpl/the destructor (skip local-file cleanup, + /// since `temp_path` is a remote key, not a path on this filesystem). false (the default, + /// local-temp-file constructor) is the pre-existing, byte-for-byte-unchanged behavior. + bool is_s3_staging = false; + /// The spill sink: a local WriteBufferFromFile (Local mode) or the caller-supplied object-store + /// sink (S3 mode). Either way it is a SECOND per-stream buffer wrapped by `hashing` below. + std::unique_ptr sink; + /// Built via `Cas::makeBlobHashingWriteBuffer(hash_algo, *sink)`: + /// `CityHash128` is a thin adapter over the pre-existing `HashingWriteBuffer` convention (byte-for-byte + /// unchanged); `XXH3_128` hashes with the pool's selected algo instead. + std::unique_ptr hashing; + bool temp_ownership_transferred = false; /// Set after successful `on_finalized`; the destructor skips local cleanup. + + /// rev.7 [C2]: fence-generation re-check invoked immediately before `sink->finalize()` in `finalizeImpl` + /// (populated only by the S3-staging constructor; a no-op empty `std::function` for Local mode). + std::function check_fence_before_finalize; +}; + +/// Write buffer for bytes that live INSIDE pool metadata (a small inline part file staged into the +/// manifest tree, or a verbatim namespace file PUT on finalize). Accumulates in memory (the bytes are +/// tiny) and hands them to the callback at finalize; the callback decides where they go and whether +/// they are durable immediately (verbatim) or at commit (staged into the part's manifest entries). +class CaInlineWriteBuffer : public WriteBufferFromFileBase +{ +public: + using OnInlined = std::function; + + explicit CaInlineWriteBuffer(OnInlined on_inlined_); + ~CaInlineWriteBuffer() override; + + void sync() override; + std::string getFileName() const override; + +private: + /// Appends bytes to the in-memory payload under the base write-buffer contract. + void nextImpl() override; + /// Hands the complete inline payload to the callback; the callback decides whether it is staged + /// in a manifest or written immediately as a verbatim file. + void finalizeImpl() override; + + OnInlined on_inlined; + std::string accumulated; +}; + +} diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md new file mode 100644 index 000000000000..5c11d6ccdad7 --- /dev/null +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md @@ -0,0 +1,198 @@ +# Content-Addressed (CAS) metadata storage + +This directory implements the `cas` metadata storage for +`DiskObjectStorage`: a "git for `MergeTree`" content-addressed pool over +S3-family object storage. Instead of per-part metadata files pointing at +randomly-named remote objects, every unique payload is stored **once** in a +shared pool under a key derived from its **content hash**, a part is an +immutable **manifest** listing its files, and a per-table **ref table** maps +part names to manifests. Replicas that share one pool deduplicate bytes +structurally: fetching a part can *relink* the sender's manifest entries by +hash instead of copying data. This replaces zero-copy replication's shared +mutable state with immutable, hash-addressed objects plus a small CAS +(compare-and-swap) protocol. + +## The data model in five objects + +- **Blob** — the unit of payload. Keyed by its content digest + (`CasLayout::blobKey`); an envelope wraps the payload + (`Formats/CasBlobEnvelopeFormat`). Blobs are immutable and shared: the pool + is the only index, and identity is always established by (re-)hashing the + bytes — there are no trust-the-checksum shortcuts. A freshness sidecar + (`blobMetaKey` = `blobKey` + `.meta`) carries GC state per hash. +- **Manifest** — an immutable description of one part: the full file tree with + per-file blob references or inline bytes. Small per-part files + (`uuid.txt`, `checksums.txt`, ...) live *inside* the manifest as inline tree + entries, not as separate objects. Keyed under + `cas/manifests//-/.zst`. +- **Ref table** — the mutable naming layer, one namespace per table + (`SERVER_ID/TABLE_UUID`), keyed under one LIFE of that namespace: an + append-only transaction log plus periodic snapshots (`cas/ns/stream//...`), with mutable + checkpoints and namespace files under `cas/ns/state//...`. The catalog resolves each opaque + physical ID to its logical namespace life (see `NamespaceLifeId`); stream objects are replayed + into an in-memory table mapping + ref names (part directory names) to manifests. All mutations go through a + precommit/promote two-step so a manifest always has an owner while visible. +- **Server root** — one per server (`server_root_id` in the disk config): a + single-writer mount slot with lease + writer-epoch fencing + (`gc/server-roots//{owner,epoch,mount}`). A fenced (expired or stolen) + writer can never mutate the pool again with stale state. +- **GC records** — round-based garbage collection state under `gc/...`: + leader lease/state, per-generation source-edge runs (which manifests + reference which blobs), fold seals, and outcome logs. GC computes blob + in-degrees from the edges, *condemns* zero-in-degree blobs, and later + deletes them with **exact-token** conditional deletes, so a concurrent + writer re-uploading the same content always wins (resurrection is a fresh + re-upload — never a read of a condemned object). + +Everything persisted is a text format with a versioned header line; see +`Formats/README.md` for the registry, key map, and evolution rules. + +## Lifecycle walkthroughs + +- **Write**: `ContentAddressedTransaction` buffers or spills file writes + (`scratch_path`, or opt-in S3 staging), hashes the payload, and hands it to + `Pool/CasPartWriteTxn`: upload-or-adopt each blob (dedup by hash; a HEAD + proves presence before reuse), stage the manifest, `precommitAdd` the ref, + then promote it to committed in the ref log. +- **Read**: disk path → `Parts/PartPathParser` (namespace + part + file) → + ref table resolves the manifest → `Pool/CasManifestReader` locates the + entry → blob `StoredObject`s or in-manifest inline bytes. +- **Fetch between replicas** (`ContentAddressedExchange`): the sender ships + its manifest bytes; the receiver rebuilds the part by *adopting* the listed + blobs by hash in its own namespace (no payload transfer) and publishes a + fresh manifest. Any decode or promote failure just falls back to the + ordinary byte fetch. +- **GC round** (`Gc/CasGc`, paced by `Gc/CasGcScheduler`): take/renew the + leader lease, collect source edges, fold per-shard in-degrees, condemn new + zero-in-degree blobs, exact-token-delete blobs condemned earlier whose + state did not change, and seal the round with one CAS on `gc/state`. + +## Source layout + +The tree is **layered**: entry points at the top level, implementation in +per-subsystem subdirectories with a strict one-direction include rule. + +``` +Primitives → Formats → Backend → Pool → Gc → Tools ≈ Parts → facade (top level) +``` + +- **`Primitives/`** — the vocabulary, zero outward dependencies: `CasBlobDigest` + (`BlobHashAlgo` + `BlobDigest` + `DigestCodec` + `BlobRef` — blob identity), + `CasTypes.h` (the other identity types: `RootNamespace`, `Token`, + `ManifestId`, `RefTxnId`), `CasNamespaceLifeId` (`NamespaceLifeId` — one LIFE of + a namespace's ref layer, the pair every ref key is built from), + `CasBlobHashingWriteBuffer` (streaming + hash-and-passthrough machinery), `CasXxh3Streamer` (the isolated vendored + xxHash wrapper), `CasCodecUtil` (identifier/hex codec helpers), `CasEvent` + (audit-event POD + sink). +- **`Formats/`** — everything persisted: bytes **and** keys. The per-object + text/format files (`CasFormat`, `CasTextFormat`, `CasPartManifestFormat`, + `CasRefLogFormat`, …) plus `CasLayout` (the object-key schema). See + `Formats/README.md` for the format registry. +- **`Backend/`** — the token-aware storage seam: `CasBackend` (the contract: + get/put/`putIfAbsent`/`casPut`/`deleteExact` with CAS tokens), + `CasObjectStorageBackend`, `CasInMemoryBackend`, `CasInstrumentedBackend`, + `CasRequestControl` (single-attempt conditional writes, explicit + state-aware retries), `CasProbe` (mount-time capability probe). +- **`Pool/`** — the pool engine: `CasPool` (composition root), `CasPartWriteTxn` + (one-part write transaction), `CasRefLedger` + `CasRefProtocol` (ref-table + log/snapshot/replay + intake), `CasServerRoot` (mount-claim protocol + + single-writer slot + staging sweeper), `CasPoolMeta`, `CasBlobMeta`, + `CasManifestReader`, `CasPlainObjects` (the `roots/...` verbatim + passthrough), `CasMountRuntime` (fence state shared with lanes). +- **`Gc/`** — garbage collection: `CasGcScheduler` (pacing thread), `CasGc` + (the round engine), `CasGcShardPlan` (sharding math), `CasBlobInDegree`, + `CasOrphanManifestSweep`. +- **`Tools/`** — operator verbs (`clickhouse-disks`): `CasFsck`, + `CasDecommission`, `CasInspect`. +- **`Parts/`** — part semantics over the pool: `PartPathParser` (the + ClickHouse-path classifier), `PartFolderAccess` (`PartRefKey` + `Freshness` + + `PartFolderValidate` + `PartFolderView` + `CachedPartFolderAccess`). +- **Top level (facade)** — the entry points: `ContentAddressedMetadataStorage` + (the `IMetadataStorage` facade), `ContentAddressedTransaction` (the + `IMetadataTransaction`, including the write buffers), `ContentAddressedExchange` + (the replication seam). + +## Include-direction rule + +A file may include only its **own layer** and layers to its **left** in the +order above. `Tools` and `Parts` are siblings with no edges between them. This +is enforced by convention (README rule) — there is no CI check. + +**Named exceptions** (deliberate): + +- The staging sweeper (in `Pool/CasServerRoot`) and `probeConditionalCopy` + bypass `Backend` and reach straight into `IObjectStorage`. +- `Backend` may read `Formats` traits (the provider-metadata mirror). + +## Configuration + +A CAS disk is an `object_storage` disk with `metadata_type` = +`cas`. Minimal example (see +`tests/config/config.d/cas_storage_policy_for_merge_tree_by_default.xml` +and its `_s3_` sibling for the lane configs used in CI): + +```xml + + + object_storage + s3 + cas + + replica-1 + cas_pool/ + + cas_scratch/ + 1 + 60 + + +``` + +`1` opens the disk in observe-only mode: no mount-slot +claim, no capability probe, no writes — the mode `clickhouse-disks` tools and +post-mortem inspection use. The full knob set (staging backend, cache sizes, +GC sharding, hash algorithm, request budgets) is parsed in +`MetadataStorageFactory.cpp`; each knob is documented at its parse site. + +## Operations and observability + +- `clickhouse-disks` verbs (all require the disk opened read-only): `fsck` + (independent reachability audit of refs → manifests → blobs), `cas-inspect` + (decode one pool object by its raw key to JSON), `cas-gc-dryrun` (preview the + next GC round's deletes), `cas-gc-rebuild` (disaster-recovery rebuild of the + `gc/state` baseline), `cas-drop-member` (decommission a dead pool member). +- `system.cas_log` — one row per CAS protocol event + (uploads, adopts, promotes, condemns, deletes, mount-slot writes, ...); + the primary audit trail when investigating pool state. +- The GC and writer paths also emit `ProfileEvents` counters (grep + `ProfileEvents.cpp` for `Cas`). + +## Testing + +- **Unit tests** (`unit_tests_dbms`): every CAS suite name starts with `Cas`, so + `--gtest_filter='Cas*'` runs the whole set — including parameterized suites, + whose instantiation prefixes are `Cas`-prefixed too so the `/` + spelling still matches. `utils/cas-gate/generate_cas_suites.sh` fails loud on a + CAS suite that does not match, so a new suite cannot silently sit outside the + filter; `utils/cas-gate/run_cas_gate_per_suite.sh` runs them one process per + suite, so an abort cannot hide the suites after it. +- **Stateless lanes**: the functional-test jobs "`cas storage`" + (local object storage) and "`cas s3 storage`" run the whole + stateless suite with `MergeTree` defaulting to a CAS disk. Tests that + legitimately cannot run there carry the `no-cas-storage` tag. +- **Soak / chaos**: `utils/ca-soak/` — multi-replica docker-compose + harnesses (fault proxies, GC sharding variants, AWS S3/GCS backends) and + adversarial scenarios. + +## Reading order + +To understand a request end to end, read in this order: + +1. `ContentAddressedMetadataStorage` — the facade / routing. +2. `Parts/PartFolderAccess` (`PartRefKey` → the folder view / cache). +3. `Pool/CasPool` — the pool composition root and `open` protocol. +4. `Pool/CasPartWriteTxn` — one-part write transaction. +5. `Gc/CasGc` — the GC round engine. diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/IMetadataStorage.h b/src/Disks/DiskObjectStorage/MetadataStorages/IMetadataStorage.h index e82735548a4b..5b1a6ef6d1b0 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/IMetadataStorage.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/IMetadataStorage.h @@ -27,6 +27,8 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } +struct IDiskTransaction; + /// Tries to provide some "transactions" interface, which allow /// to execute (commit) operations simultaneously. We don't provide /// any snapshot isolation here, so no read operations in transactions @@ -115,6 +117,16 @@ class IMetadataTransaction : private boost::noncopyable throwNotImplemented(); } + /// [TXN-ONE-PIPELINE] Optional per-metadata write buffer. Returns a ready-to-use buffer when the + /// metadata implementation owns its write mechanism (e.g. a content-addressed hash-on-write buffer + /// whose blob key is known only after the last byte). `owner` is the disk transaction that must be + /// kept alive for the returned buffer's lifetime and, when `autocommit`, committed from the finalize + /// callback. Default nullptr: the caller uses the generic streaming write path unchanged. + virtual std::unique_ptr tryCreateWriteBuffer( + const std::shared_ptr & /*owner*/, + const std::string & /*path*/, size_t /*buf_size*/, WriteMode /*mode*/, + const WriteSettings & /*settings*/, bool /*autocommit*/) { return nullptr; } + /// Metadata related methods /// Generate blob name for passed absolute local path. @@ -141,6 +153,22 @@ class IMetadataTransaction : private boost::noncopyable throwNotImplemented(); } + /// In-flight read-your-writes for a part being assembled by THIS transaction (B59). A CA part-build + /// transaction stages blobs (uploaded) + mutable bytes before the single commit; these let a reader + /// that holds the transaction resolve those staged files before they are committed. Default: no + /// in-flight visibility (the committed metadata path is authoritative). + virtual std::optional tryGetInFlightStorageObjects(const std::string & /*path*/) const { return {}; } + virtual std::unique_ptr tryReadFileInFlight( + const std::string & /*path*/, const ReadSettings & /*settings*/, std::optional /*read_hint*/) const { return nullptr; } + virtual std::optional tryGetInFlightFileSize(const std::string & /*path*/) const { return {}; } + /// Directory-granularity counterpart of the file trio: true iff this transaction has STAGED at least one + /// file under `path` for `path`'s part. Used so a carried-forward projection dir is visible to + /// loadProjections during finalize. Default: no in-flight directory visibility. + virtual bool hasInFlightDirectory(const std::string & /*path*/) const { return false; } + /// Immediate-child names staged directly under `path` (one level). Used so loadProjections' + /// withPartFormatFromDisk can iterate a staged projection dir to find its mark file. Default: empty. + virtual std::vector listInFlightDirectory(const std::string & /*path*/) const { return {}; } + virtual ~IMetadataTransaction() = default; protected: @@ -289,6 +317,23 @@ class IMetadataStorage : private boost::noncopyable return false; } + /// Returns true if the metadata storage is content-addressed, i.e. blob keys are derived + /// from content hashes and are only known after all bytes have been written. Such a storage + /// cannot use the up-front-key streaming write path of `DiskObjectStorageTransaction`; the + /// disk transaction delegates writes to the metadata transaction's content-addressed buffer. + virtual bool isContentAddressed() const { return false; } + + /// [TXN-ONE-PIPELINE] True when a transaction from this storage stages every mutation into a + /// transaction-private overlay at call time (eager) rather than queuing effects for FIFO replay in + /// commit. When true, DiskObjectStorageTransaction routes every mutating method straight to the + /// metadata transaction and keeps its own operations_to_execute queue empty. Default false + /// (ordinary object storage). + virtual bool transactionIsStagingOverlay() const { return false; } + + /// True when a file write through this metadata storage publishes atomically, i.e. no partial + /// content is ever observable under the file's final name (see `IDataPartStorage::supportsAtomicFileWrites`). + virtual bool supportsAtomicFileWrites() const { return false; } + using BlobsToRemove = std::unordered_map; virtual BlobsToRemove getBlobsToRemove(const ClusterConfigurationPtr & /*cluster*/, int64_t /*max_count*/) { return {}; } virtual int64_t recordAsRemoved(const StoredObjects & /*blobs*/) { return 0; } @@ -325,6 +370,12 @@ class IMetadataStorage : private boost::noncopyable /// True if write with Append mode supported. virtual bool supportWritingWithAppend() const { return false; } + /// True iff this metadata storage can persist the per-part mutable transaction file (txn_version.txt) + /// under MVCC. Distinct from supportWritingWithAppend: transactions rewrite txn_version.txt (tmp + + /// replaceFile), they never WriteMode::Append, so append-capability is the wrong proxy. A + /// content-addressed disk supports the mutable txn file via its per-ref sidecar. + virtual bool supportsTransactionalMutableFiles() const { return false; } + protected: [[noreturn]] static void throwNotImplemented() { diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/MetadataStorageFactory.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/MetadataStorageFactory.cpp index 2b2d6a284a4e..9bb1b1fb355d 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/MetadataStorageFactory.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/MetadataStorageFactory.cpp @@ -7,8 +7,11 @@ #endif #include #include +#include +#include #include #include +#include #include @@ -21,6 +24,12 @@ namespace ErrorCodes extern const int UNKNOWN_ELEMENT_IN_CONFIG; extern const int INVALID_CONFIG_PARAMETER; extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; +} + +namespace ContentAddressedSetting +{ + extern const ContentAddressedSettingsString scratch_path; } namespace @@ -205,6 +214,35 @@ static void registerPlainRewritableMetadataStorage(MetadataStorageFactory & fact }); } +static void registerContentAddressedMetadataStorage(MetadataStorageFactory & factory) +{ + factory.registerMetadataStorageType("cas", []( + const std::string & name, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + const ClusterConfigurationPtr & cluster, + const ObjectStorageRouterPtr & object_storages) -> MetadataStoragePtr + { + checkSingleLocation(cluster); + + const auto local_object_storage = object_storages->takePointingTo(cluster->getLocalLocation()); + std::string key_compatibility_prefix = getObjectKeyCompatiblePrefix(local_object_storage, config, config_prefix); + + auto global_context = Context::getGlobalContextInstance(); + ContentAddressedSettings settings; + settings.loadFromConfig( + config, config_prefix, + /*scratch_path_anchor_if_relative=*/ global_context->getPath(), + /*default_scratch_path=*/ fs::path(global_context->getPath()) / "disks" / name / "cas_scratch" / "", + [&](const std::string & s) { return global_context->getMacros()->expand(s); }); + fs::create_directories(settings[ContentAddressedSetting::scratch_path].value); + + return std::make_shared( + local_object_storage, key_compatibility_prefix, toString(ServerUUID::get()), + name, global_context, settings); + }); +} + static void registerMetadataStorageFromStaticFilesWebServer(MetadataStorageFactory & factory) { factory.registerMetadataStorageType("web", []( @@ -230,6 +268,7 @@ void registerMetadataStorages() registerMetadataStorageFromDisk(factory); registerPlainMetadataStorage(factory); registerPlainRewritableMetadataStorage(factory); + registerContentAddressedMetadataStorage(factory); registerMetadataStorageFromStaticFilesWebServer(factory); #if CLICKHOUSE_CLOUD registerMetadataStorageFromKeeper(factory); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index 41b16b95e50b..ff77c27915f6 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -161,6 +161,23 @@ using ObjectKeysWithMetadata = std::vector; class IObjectStorageIterator; using ObjectStorageIteratorPtr = std::shared_ptr; +/// Outcome of a token-conditional single-object removal (content-addressed disks). +enum class ConditionalRemoveOutcome : uint8_t { Removed, TokenMismatch, NotFound }; +struct ConditionalRemoveResult +{ + ConditionalRemoveOutcome outcome = ConditionalRemoveOutcome::NotFound; + bool created_delete_marker = false; /// backend reported a versioning delete marker +}; + +/// Outcome of a write-once conditional server-side copy (content-addressed disks): `created == true` +/// means this call won the race and created `object_to`; `created == false` means the destination +/// already existed (the precondition was rejected) and `dest_etag` is left empty. +struct ConditionalCopyResult +{ + bool created = false; + String dest_etag; +}; + /// Base class for all object storages which implement some subset of ordinary filesystem operations. /// /// Examples of object storages are S3, Azure Blob Storage, HDFS. @@ -267,6 +284,15 @@ class IObjectStorage /// Remove objects on path if exists virtual void removeObjectsIfExist(const StoredObjects & object) = 0; + /// Remove `object` ONLY if its current entity tag equals `etag`. Backends without enforced + /// conditional removal MUST NOT override this: the content-addressed capability probe relies on the + /// default to fail closed. Supported: S3 (DeleteObject If-Match, GA 2025-09). + virtual ConditionalRemoveResult removeObjectIfTokenMatches(const StoredObject & /*object*/, const std::string & /*etag*/) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Conditional (token-exact) object removal is not implemented for {} object storage", getName()); + } + /// Copy object with different attributes if required virtual void copyObject( /// NOLINT const StoredObject & object_from, @@ -275,6 +301,36 @@ class IObjectStorage const WriteSettings & write_settings, std::optional object_to_attributes = {}) = 0; + /// Copy `object_from` to `object_to` WRITE-ONCE: the copy is conditional on `object_to` not + /// already existing (`If-None-Match: *`). Returns `created=true` with the destination ETag if + /// this call created the object, or `created=false` (empty `dest_etag`) if the destination + /// already existed — that is the expected "lost the race" signal, not an error. Any other + /// failure propagates as an exception. + /// + /// Backends without an enforced, native (server-side) conditional copy MUST NOT override this: + /// the content-addressed write-once staging promote relies on the default to fail closed rather + /// than silently falling back to an unconditional overwrite. Supported: S3 (native `CopyObject` + /// / `CompleteMultipartUpload` with `If-None-Match`). + virtual ConditionalCopyResult copyObjectConditional( + const StoredObject & /*object_from*/, + const StoredObject & /*object_to*/, + const ReadSettings & /*read_settings*/, + const WriteSettings & /*write_settings*/, + std::optional /*object_to_attributes*/) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Conditional (write-once) object copy is not implemented for {} object storage", getName()); + } + + ConditionalCopyResult copyObjectConditional( + const StoredObject & object_from, + const StoredObject & object_to, + const ReadSettings & read_settings, + const WriteSettings & write_settings) + { + return copyObjectConditional(object_from, object_to, read_settings, write_settings, {}); + } + /// Copy object to another instance of object storage /// by default just read the object from source object storage and write /// to destination through buffers. @@ -323,6 +379,22 @@ class IObjectStorage virtual bool supportParallelWrite() const { return false; } + /// True when the incarnation tokens this storage returns from writes/HEADs are GCS generation + /// numbers riding the ETag plumbing (http_client = gcs_hmac / gcp_oauth conditional dialect). + /// Consumers (the CAS backend) stamp TokenType::Generation and route conditional writes + /// through the single-PUT path (GCS enforces no preconditions on CompleteMultipartUpload). + virtual bool conditionalOpsUseGenerationTokens() const { return false; } + + /// Whether the underlying bucket has object versioning enabled; nullopt when unknown or not + /// applicable. Used by the CAS capability probe to fail closed on GCS: on a versioned bucket + /// a token-exact DELETE archives a noncurrent generation instead of reclaiming storage. + virtual std::optional isBucketVersioningEnabled() const { return std::nullopt; } + + /// True when this object storage can execute writes under the given retry profile. + /// A caller that sets a non-Default profile on WriteSettings MUST check this first and + /// fail closed if unsupported (the profile is advisory only to backends that opt in). + virtual bool supportsRetryProfile(ObjectStorageRetryProfile profile) const { return profile == ObjectStorageRetryProfile::Default; } + virtual ReadSettings patchSettings(const ReadSettings & read_settings) const; virtual WriteSettings patchSettings(const WriteSettings & write_settings) const; diff --git a/src/Disks/DiskObjectStorage/RegisterDiskObjectStorage.cpp b/src/Disks/DiskObjectStorage/RegisterDiskObjectStorage.cpp index c662bf71facc..304ca5daab4b 100644 --- a/src/Disks/DiskObjectStorage/RegisterDiskObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/RegisterDiskObjectStorage.cpp @@ -6,12 +6,19 @@ #include #include #include +#include +#include #include namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + void registerObjectStorages(); void registerMetadataStorages(); void registerDiskObjectStorage(DiskFactory & factory, bool global_skip_access_check); @@ -77,7 +84,21 @@ void registerDiskObjectStorage(DiskFactory & factory, bool global_skip_access_ch LOG_DEBUG(getLogger("registerDiskObjectStorage"), "Metadata type hint: {}", compatibility_metadata_type_hint); auto metadata_storage = MetadataStorageFactory::instance().create(name, config, config_prefix, cluster, object_storages, compatibility_metadata_type_hint); - bool use_fake_transaction = config.getBool(config_prefix + ".use_fake_transaction", metadata_storage->getType() != MetadataStorageType::Keeper); + /// Content-addressed metadata (like Keeper) requires real, deferred disk transactions: a part's + /// file->blob mappings are accumulated across the whole part write and the manifest + ref are + /// published atomically when the transaction commits. A fake (per-file autocommit) transaction + /// would write each file independently with no commit point for the manifest/ref publish. + const auto metadata_type = metadata_storage->getType(); + const bool needs_real_transaction = metadata_type == MetadataStorageType::Keeper + || metadata_type == MetadataStorageType::CAS; + /// An explicit `use_fake_transaction=true` on a metadata type that requires deferred + /// transactions would silently break the atomic manifest/ref publish (per-file autocommit, + /// no commit point). Reject it instead of honoring it. + if (needs_real_transaction && config.getBool(config_prefix + ".use_fake_transaction", false)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Disk '{}': `use_fake_transaction` cannot be enabled for metadata type '{}'", + name, magic_enum::enum_name(metadata_type)); + bool use_fake_transaction = config.getBool(config_prefix + ".use_fake_transaction", !needs_real_transaction); DiskPtr disk = std::make_shared( name, std::move(cluster), diff --git a/src/Disks/DiskType.cpp b/src/Disks/DiskType.cpp index bf4506b4cbf6..8aef6db87ee0 100644 --- a/src/Disks/DiskType.cpp +++ b/src/Disks/DiskType.cpp @@ -19,6 +19,8 @@ MetadataStorageType metadataTypeFromString(const String & type) return MetadataStorageType::Plain; if (check_type == "plain_rewritable") return MetadataStorageType::PlainRewritable; + if (check_type == "cas") + return MetadataStorageType::CAS; if (check_type == "web") return MetadataStorageType::StaticWeb; if (check_type == "keeper") diff --git a/src/Disks/DiskType.h b/src/Disks/DiskType.h index 726557d5575d..7a612d2336de 100644 --- a/src/Disks/DiskType.h +++ b/src/Disks/DiskType.h @@ -32,6 +32,7 @@ enum class MetadataStorageType : uint8_t Keeper, Plain, PlainRewritable, + CAS, StaticWeb, Memory, }; diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 478795f523f1..b79a3171bde7 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -472,6 +472,13 @@ class IDisk : public Space /// If the disk is plain object storage. virtual bool isPlain() const { return false; } + /// If the disk is a content-addressed object-storage pool (`metadata_type = cas`). + /// A clean predicate so callers do not have to reach through `getDataSourceDescription`. + virtual bool isContentAddressed() const { return false; } + + /// True when a file write on this disk publishes atomically (see `IDataPartStorage::supportsAtomicFileWrites`). + virtual bool supportsAtomicFileWrites() const { return false; } + virtual bool isWriteOnce() const { return false; } virtual bool supportsHardLinks() const { return true; } diff --git a/src/Disks/ReadOnlyDiskWrapper.h b/src/Disks/ReadOnlyDiskWrapper.h index e75dd8623d24..a0c28d1c0a18 100644 --- a/src/Disks/ReadOnlyDiskWrapper.h +++ b/src/Disks/ReadOnlyDiskWrapper.h @@ -85,6 +85,11 @@ class ReadOnlyDiskWrapper : public IDisk NameSet getCacheLayersNames() const override { return delegate->getCacheLayersNames(); } MetadataStoragePtr getMetadataStorage() override { return delegate->getMetadataStorage(); } + /// Forwarded alongside getMetadataStorage: callers that gate on this predicate before reaching + /// for the metadata storage (ContentAddressedMetadataStorage::tryFromDisk and friends) must see + /// the delegate's answer through the wrapper, or a wrapped content-addressed disk silently + /// drops out of the CAS introspection paths. + bool isContentAddressed() const override { return delegate->isContentAddressed(); } std::unordered_map getSerializedMetadata(const std::vector & file_paths) const override { return delegate->getSerializedMetadata(file_paths); } diff --git a/src/IO/ReadPipeline.cpp b/src/IO/ReadPipeline.cpp index fa2e1f075110..1d7a1be480a6 100644 --- a/src/IO/ReadPipeline.cpp +++ b/src/IO/ReadPipeline.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -157,6 +158,17 @@ void ReadPipeline::needDecryption(String path, size_t buffer_size, KeyFinderFunc .key_finder = std::move(key_finder)}); } +void ReadPipeline::needFileView(String file_name, size_t left_bound, size_t right_bound) +{ + if (right_bound < left_bound) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "ReadPipeline: file view right bound ({}) is below the left bound ({})", right_bound, left_bound); + file_view = FileViewStage{ + .file_name = std::move(file_name), + .left_bound = left_bound, + .right_bound = right_bound}; +} + std::unique_ptr ReadPipeline::build() const { if (!source) @@ -182,7 +194,8 @@ std::unique_ptr ReadPipeline::build() const impl = wrapMemoryCache(std::move(impl)); // Stage 4 impl = wrapAsyncPrefetch(std::move(impl)); // Stage 5 - impl = wrapDecryption(std::move(impl)); // Stage 6 (encryption) + impl = wrapFileView(std::move(impl)); // Stage 6 (byte window) + impl = wrapDecryption(std::move(impl)); // Stage 7 (encryption) return impl; } @@ -193,14 +206,15 @@ std::unique_ptr ReadPipeline::tryBuildReaderExecutor() c if (!settings.use_reader_executor) return nullptr; - /// The executor does not implement caches, decryption, async prefetch, or the - /// distributed cache, so fall back rather than silently drop a configured stage. + /// The executor does not implement caches, decryption, async prefetch, the + /// distributed cache, or a file_view byte window, so fall back rather than + /// silently drop a configured stage. if (distributed_cache || memory_cache || !filesystem_caches.empty() - || !decryption_stages.empty() || async_prefetch) + || !decryption_stages.empty() || async_prefetch || file_view) { LOG_DEBUG(log, "use_reader_executor: falling back to the legacy read path " - "(caches/decryption not yet supported by the executor)"); + "(caches/decryption/file_view not yet supported by the executor)"); return nullptr; } @@ -673,6 +687,19 @@ std::unique_ptr ReadPipeline::wrapAsyncPrefetch(std::uni async_prefetch->prefetches_log); } +std::unique_ptr ReadPipeline::wrapFileView(std::unique_ptr impl) const +{ + /// -- Stage 6: File view -- + /// The view translates the consumer's positions/right bounds by `left_bound` and forwards + /// them down the chain, so `MergeTreeReaderStream::adjustRightMark` bounds reach the + /// object-storage reader and its range requests stay drainable (connection-pool friendly). + if (!file_view) + return impl; + + return std::make_unique( + std::move(impl), file_view->file_name, file_view->left_bound, file_view->right_bound); +} + std::unique_ptr ReadPipeline::wrapDecryption(std::unique_ptr impl) const { /// -- Stage 6: Decryption (may have multiple layers for double encryption) -- @@ -733,6 +760,8 @@ String ReadPipeline::describe() const append("MemoryCache"); if (async_prefetch) append("AsyncPrefetch"); + if (file_view) + append("FileView"); if (!decryption_stages.empty()) append("Decrypt"); diff --git a/src/IO/ReadPipeline.h b/src/IO/ReadPipeline.h index 61b5fa506351..586212be4598 100644 --- a/src/IO/ReadPipeline.h +++ b/src/IO/ReadPipeline.h @@ -47,7 +47,8 @@ using FilesystemReadPrefetchesLogPtr = std::shared_ptr build() const; @@ -215,6 +225,13 @@ class ReadPipeline KeyFinderFunc key_finder; }; + struct FileViewStage + { + String file_name; + size_t left_bound = 0; + size_t right_bound = 0; + }; + struct DistributedCacheStage { @@ -228,6 +245,7 @@ class ReadPipeline std::optional distributed_cache; std::optional async_prefetch; VectorWithMemoryTracking decryption_stages; + std::optional file_view; LoggerPtr log = getLogger("ReadPipeline"); @@ -246,6 +264,7 @@ class ReadPipeline std::unique_ptr buildSingleObjectStage(const std::string & query_id) const; std::unique_ptr wrapMemoryCache(std::unique_ptr impl) const; std::unique_ptr wrapAsyncPrefetch(std::unique_ptr impl) const; + std::unique_ptr wrapFileView(std::unique_ptr impl) const; std::unique_ptr wrapDecryption(std::unique_ptr impl) const; }; diff --git a/src/IO/WriteBufferFromFileBase.h b/src/IO/WriteBufferFromFileBase.h index 47dd4f5ed7ae..b60e951d8edf 100644 --- a/src/IO/WriteBufferFromFileBase.h +++ b/src/IO/WriteBufferFromFileBase.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -15,6 +16,12 @@ class WriteBufferFromFileBase : public BufferWithOwnMemory void sync() override = 0; virtual std::string getFileName() const = 0; + + /// The object-storage ETag/token the write produced, if any (e.g. the S3 PutObject / + /// CompleteMultipartUpload response ETag). Empty for backends that do not return a write-time + /// ETag (local files, etc.). Valid only after a successful finalize(). Lets content-addressed + /// callers record the just-written incarnation's token WITHOUT a follow-up HEAD. + virtual std::optional getResultObjectETag() const { return {}; } }; } diff --git a/src/IO/WriteBufferFromFileDecorator.h b/src/IO/WriteBufferFromFileDecorator.h index 07f843986bb0..cc05743642f5 100644 --- a/src/IO/WriteBufferFromFileDecorator.h +++ b/src/IO/WriteBufferFromFileDecorator.h @@ -19,6 +19,15 @@ class WriteBufferFromFileDecorator : public WriteBufferFromFileBase void preFinalize() override; + /// Forward the wrapped buffer's write-time ETag (if it is a file buffer that produced one), so a + /// decorated S3 buffer still lets content-addressed callers skip the post-write HEAD. + std::optional getResultObjectETag() const override + { + if (const auto * file_buf = dynamic_cast(impl.get())) + return file_buf->getResultObjectETag(); + return {}; + } + const WriteBuffer & getImpl() const { return *impl; } protected: diff --git a/src/IO/WriteSettings.h b/src/IO/WriteSettings.h index a3a3cf204f7f..591f803e6400 100644 --- a/src/IO/WriteSettings.h +++ b/src/IO/WriteSettings.h @@ -4,9 +4,22 @@ #include #include +#include + namespace DB { +/// Per-write retry-behavior selector, resolved by the object storage that executes the write. +/// SingleAttempt: exactly one HTTP attempt, no SDK-transparent retries — for conditional writes +/// whose retry loop lives above the storage client (it must resolve an uncertain PUT before +/// reissuing). Backends without a SingleAttempt implementation report it via +/// IObjectStorage::supportsRetryProfile; writers must fail closed rather than fall through. +enum class ObjectStorageRetryProfile : uint8_t +{ + Default, + SingleAttempt, +}; + /// Settings to be passed to IDisk::writeFile() struct WriteSettings { @@ -23,6 +36,12 @@ struct WriteSettings size_t filesystem_cache_reserve_space_wait_lock_timeout_milliseconds = 1000; bool s3_allow_parallel_part_upload = true; + /// Overrides S3RequestSetting::check_objects_after_upload for this write (nullopt = no + /// override). Writers of CAS-MUTABLE keys (content-addressed shard manifests) set `false`: + /// such a key is legitimately replaced by a concurrent conditional PUT between this upload and + /// the check's HEAD, so the size comparison false-positives ("it's a bug in S3") under normal + /// contention. Integrity for those keys is the conditional PUT outcome + token, not a recheck. + std::optional s3_check_objects_after_upload_override; bool azure_allow_parallel_part_upload = true; bool use_adaptive_write_buffer = false; @@ -36,6 +55,26 @@ struct WriteSettings std::string object_storage_write_if_none_match; /// Supported only for S3-like object storages. std::string object_storage_write_if_match; /// Supported only for S3-like object storages. + /// A conditional write on a generation-token store (GCS) must never take the multipart path: + /// GCS enforces no preconditions on CompleteMultipartUpload (measured 2026-07-03). When set, + /// WriteBufferFromS3 throws instead of starting a multipart upload. + bool s3_force_single_part_upload = false; + /// Companion cap: raises max_single_part_upload_size / min_upload_part_size in the request + /// settings so bodies up to this size stay in ONE part (RAM-buffered). 0 = no override. + size_t s3_single_part_upload_max_bytes_override = 0; + + /// Overrides S3RequestSetting::max_unexpected_write_error_retries (default 4) for this write. + /// WriteBufferFromS3::makeSinglepartUpload/completeMultipartUpload run their OWN retry loop above + /// the S3 client that reissues the identical request (WITH its If-None-Match/If-Match condition) + /// on a NO_SUCH_KEY response — a second retry-affecting layer a client-level override + /// (a client-level profile override) does not reach. A CAS conditional write sets this to 1 for + /// exactly one attempt at this layer too (RFC cas-s3-timeout-retry-control). 0 = no override. + size_t s3_max_unexpected_write_error_retries_override = 0; + + /// Selects the retry profile the object storage should execute this write under; see + /// ObjectStorageRetryProfile. + ObjectStorageRetryProfile object_storage_retry_profile = ObjectStorageRetryProfile::Default; + bool operator==(const WriteSettings & other) const = default; }; diff --git a/src/Interpreters/MergeTreeTransaction/VersionMetadataOnDisk.cpp b/src/Interpreters/MergeTreeTransaction/VersionMetadataOnDisk.cpp index eb7b9df9f056..ab7061854ad0 100644 --- a/src/Interpreters/MergeTreeTransaction/VersionMetadataOnDisk.cpp +++ b/src/Interpreters/MergeTreeTransaction/VersionMetadataOnDisk.cpp @@ -326,6 +326,18 @@ void VersionMetadataOnDisk::storeInfoToDataPartStorage( static constexpr auto filename = TXN_VERSION_METADATA_FILE_NAME; static constexpr auto tmp_filename = TMP_TXN_VERSION_METADATA_FILE_NAME; + if (data_part_storage.supportsAtomicFileWrites()) + { + /// Single atomic write: storages that publish file writes atomically do not need + /// the tmp+replace dance (which exists only for partial-local-write crash safety). + auto write_settings = storage.getContext()->getWriteSettings(); + auto buf = data_part_storage.writeFile(filename, 256, write_settings); + new_info.writeToBuffer(*buf, /*one_line=*/false); + buf->finalize(); + buf->sync(); + return; + } + try { { diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index b58b4e0dd1f6..1143c40938dd 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -176,11 +177,15 @@ static bool supportTransaction(const Disks & disks, LoggerPtr log) { for (const auto & disk : disks) { - if (!supportWritingWithAppend(disk)) - { - LOG_DEBUG(log, "Disk {} does not support writing with append", disk->getName()); - return false; - } + if (supportWritingWithAppend(disk)) + continue; + /// A content-addressed disk does not support append, but persists the per-part mutable + /// transaction file (txn_version.txt) via its per-ref sidecar, which is all MVCC needs. + if (auto * obj = dynamic_cast(disk.get()); + obj && obj->getMetadataStorage()->supportsTransactionalMutableFiles()) + continue; + LOG_DEBUG(log, "Disk {} does not support transactions", disk->getName()); + return false; } return true; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ad52354eb849..017f742edd5d 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -464,6 +464,18 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( { if (disk->getDataSourceDescription().metadata_type == MetadataStorageType::Keeper) throw Exception(ErrorCodes::BAD_ARGUMENTS, "ReplicatedMergeTree doesn't work with 's3_with_keeper' disk type"); + + /// B33 (lifted, CAS replication 2b + Phase 3.2): ReplicatedMergeTree on a content-addressed disk + /// is allowed. INSERT/SELECT/merge/mutation and fetch-by-relink (the CA analogue of zero-copy + /// replication) route through the working whole-part CA transaction / the relink path. The + /// replication-queue CLONE paths (queue-driven REPLACE/MOVE/ATTACH PARTITION FROM, the + /// cloneAndLoadDataPart-on-the-queue path) were audited in Phase 3.2: they reach the SAME + /// whole-part ContentAddressedTransaction the non-replicated stack uses (see + /// `MergeTreeData::checkAlterPartitionIsPossible`, reached here by dynamic dispatch — the + /// Phase 3.2 fail-closed override in this class was a pure delegation and was deleted by the + /// tail de-patch), NOT the per-file-autocommit B21 mode, so they are now permitted. The + /// zero-copy lockSharedData/unlockSharedData calls these reach are safe no-ops on CA (they + /// early-return on !supportZeroCopyReplication, which CA is). } initializeDirectoriesAndFormatVersion(relative_data_path_, LoadingStrictnessLevel::ATTACH <= mode, date_column_name); From 1d92894f880da584d0560b6d09de2411893e4e7e Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:35 +0200 Subject: [PATCH 22/30] CAS integration: system logs and introspection The CAS system logs (definitions, SystemLog registration, Context getters), the CAS mounts system table, and the per-disk GC-health asynchronous metrics. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- src/Common/SystemLogBase.cpp | 2 + src/Common/SystemLogBase.h | 2 + .../ContentAddressedGarbageCollectionLog.cpp | 114 ++++++++ .../ContentAddressedGarbageCollectionLog.h | 63 ++++ src/Interpreters/ContentAddressedLog.cpp | 73 +++++ src/Interpreters/ContentAddressedLog.h | 47 +++ src/Interpreters/Context.cpp | 16 + src/Interpreters/Context.h | 4 + .../ServerAsynchronousMetrics.cpp | 33 +++ src/Interpreters/SystemLog.cpp | 2 + src/Interpreters/SystemLog.h | 2 + .../StorageSystemContentAddressedMounts.cpp | 273 ++++++++++++++++++ .../StorageSystemContentAddressedMounts.h | 38 +++ src/Storages/System/attachSystemTables.cpp | 4 + 14 files changed, 673 insertions(+) create mode 100644 src/Interpreters/ContentAddressedGarbageCollectionLog.cpp create mode 100644 src/Interpreters/ContentAddressedGarbageCollectionLog.h create mode 100644 src/Interpreters/ContentAddressedLog.cpp create mode 100644 src/Interpreters/ContentAddressedLog.h create mode 100644 src/Storages/System/StorageSystemContentAddressedMounts.cpp create mode 100644 src/Storages/System/StorageSystemContentAddressedMounts.h diff --git a/src/Common/SystemLogBase.cpp b/src/Common/SystemLogBase.cpp index 2ee953ff7ec6..8adc1a7465ea 100644 --- a/src/Common/SystemLogBase.cpp +++ b/src/Common/SystemLogBase.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/Common/SystemLogBase.h b/src/Common/SystemLogBase.h index d8db456a29dd..c5eeafa162f2 100644 --- a/src/Common/SystemLogBase.h +++ b/src/Common/SystemLogBase.h @@ -17,6 +17,8 @@ M(CrashLogElement) \ M(OpenTelemetrySpanLogElement) \ M(PartLogElement) \ + M(ContentAddressedGarbageCollectionLogElement) \ + M(ContentAddressedLogElement) \ M(BackgroundSchedulePoolLogElement) \ M(QueryLogElement) \ M(QueryThreadLogElement) \ diff --git a/src/Interpreters/ContentAddressedGarbageCollectionLog.cpp b/src/Interpreters/ContentAddressedGarbageCollectionLog.cpp new file mode 100644 index 000000000000..fde269c4c999 --- /dev/null +++ b/src/Interpreters/ContentAddressedGarbageCollectionLog.cpp @@ -0,0 +1,114 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +ColumnsDescription ContentAddressedGarbageCollectionLogElement::getColumnsDescription() +{ + auto type_enum = std::make_shared(DataTypeEnum8::Values{ + {"Start", static_cast(START)}, {"Finish", static_cast(FINISH)}, + {"Phase", static_cast(PHASE)}}); + auto outcome_enum = std::make_shared(DataTypeEnum8::Values{ + {"Unknown", static_cast(UNKNOWN)}, {"Success", static_cast(SUCCESS)}, + {"NotALeader", static_cast(NOT_A_LEADER)}, {"Error", static_cast(FAILED)}, + {"Deferred", static_cast(DEFERRED)}}); + auto trigger_enum = std::make_shared(DataTypeEnum8::Values{ + {"Scheduled", static_cast(SCHEDULED)}, {"Manual", static_cast(MANUAL)}}); + auto lc_string = std::make_shared(std::make_shared()); + + return ColumnsDescription + { + {"hostname", lc_string, "Host name of the server executing the round."}, + {"event_date", std::make_shared(), "Event date."}, + {"event_time", std::make_shared(), "Event time."}, + {"event_time_microseconds", std::make_shared(6), "Event time with microseconds."}, + {"event_type", type_enum, "Start or Finish of a GC round, or one Phase of it."}, + {"disk_name", lc_string, "Content-addressed disk the round ran on."}, + {"server_root_id", lc_string, "Identifies the mount whose GC scheduler ran this round. Distinguishes concurrent mounters of the same shared pool; join on this column when correlating rounds against `system.cas_mounts`."}, + {"gc_id", std::make_shared(), "GC scheduler instance id (which mounter)."}, + {"trigger", trigger_enum, "Scheduled (background tick) or Manual (SYSTEM command)."}, + {"round", std::make_shared(), "GC round number (0 on Start)."}, + {"outcome", outcome_enum, "Unknown (Start) / Success (led, folded, and completed) / NotALeader (another replica holds the GC lease) / Deferred (led but took the skip-unchanged fast path -- no fold ran) / Error (the round threw)."}, + {"candidates_marked", std::make_shared(), "Objects retired (marked) this round."}, + {"objects_deleted", std::make_shared(), "Objects physically deleted this round."}, + {"objects_absent", std::make_shared(), "Retire candidates found already absent."}, + {"objects_replaced", std::make_shared(), "412-saves (a resurrection won the race)."}, + {"objects_spared", std::make_shared(), "Candidates spared (in-degree > 0 at recheck)."}, + {"manifests_deleted", std::make_shared(), "Owner-removed manifest bodies physically deleted this round (counted separately from blob deletes, B11)."}, + {"entries_condemned", std::make_shared(), "Retired entries newly condemned this round (retired-cursor pipeline stage 1)."}, + {"entries_graduated", std::make_shared(), "Retired entries newly floor-passed and republished delete_pending this round (stage 2; deleted the NEXT round)."}, + {"entries_redeleted", std::make_shared(), "Pending exact-token blob deletes executed this round (stage 3)."}, + {"fence_outs", std::make_shared(), "Expired mounts fenced out by this round's heartbeat floor."}, + {"anomalies", std::make_shared(), "Fold clamps surfaced (and survived) this round; steady >0 warrants a look at the round log details."}, + {"duration_ms", std::make_shared(), "Round wall-clock duration (Finish)."}, + {"error", std::make_shared(), "Exception text when outcome = Error."}, + {"ProfileEvents", std::make_shared(lc_string, std::make_shared()), + "On a Start/Finish row: the per-round ProfileEvents delta (the Cas* counters and S3 events for this round). On a Phase row: THAT PHASE's delta, so `GROUP BY phase` over `ProfileEvents['S3ListObjects']` attributes the round's LIST budget to the phase that spent it. Empty on the `meta_pool_wait` row by construction — that phase's work runs on other threads (read its `phase_metrics` instead)."}, + {"round_id", std::make_shared(), + "Correlator for every row of one round attempt (its Start, each Phase, and its Finish). Minted per attempt; unlike `round` it exists even for a round that never committed and for a round that never led. Group by this column to reconstruct one round."}, + {"phase", lc_string, + "The GC phase this row describes (empty on Start/Finish), in execution order: lease, pre_fold_ref_drain, heartbeat_floor, defer_decision, parent_seal_read, fold_ref_group, fold_seal_read, fold_ref_intake, fold_reduce, fold_seal_write, pending_deletes, meta_pool_wait, round_commit, handoff_reclaim, manifest_deletes, namespace_cleanup, ref_object_cleanup, orphan_sweep. A round that defers, or that never acquires the lease, emits only the phases it reached."}, + {"phase_duration_microseconds", std::make_shared(), + "Wall-clock duration of this phase in microseconds (Phase rows only). Microseconds because several phases are routinely sub-millisecond and the point is to see when they are not. Phase durations do not sum to the round's `duration_ms`: the round also does untimed bookkeeping between phases."}, + {"phase_metrics", std::make_shared(lc_string, std::make_shared()), + "Phase-specific semantic counts a phase computes for itself and no ProfileEvent can supply (Phase rows only) — for example `changed_shards` on defer_decision, `logs_accounted`/`logs_applied` on fold_ref_intake, `transactions_unapplied` on fold_reduce, `jobs_scheduled`/`jobs_completed` on meta_pool_wait. The verb counts ride the `ProfileEvents` column of the same row."}, + }; +} + +void ContentAddressedGarbageCollectionLogElement::appendToBlock(MutableColumns & columns) const +{ + size_t i = 0; + columns[i++]->insert(getFQDNOrHostName()); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); + columns[i++]->insert(event_time); + columns[i++]->insert(event_time_microseconds); + columns[i++]->insert(static_cast(event_type)); + columns[i++]->insert(disk_name); + columns[i++]->insert(srid); + columns[i++]->insert(gc_id); + columns[i++]->insert(static_cast(trigger)); + columns[i++]->insert(round); + columns[i++]->insert(static_cast(outcome)); + columns[i++]->insert(candidates_marked); + columns[i++]->insert(objects_deleted); + columns[i++]->insert(objects_absent); + columns[i++]->insert(objects_replaced); + columns[i++]->insert(objects_spared); + columns[i++]->insert(manifests_deleted); + columns[i++]->insert(entries_condemned); + columns[i++]->insert(entries_graduated); + columns[i++]->insert(entries_redeleted); + columns[i++]->insert(fence_outs); + columns[i++]->insert(anomalies); + columns[i++]->insert(duration_ms); + columns[i++]->insert(error); + { + Map map; + map.reserve(profile_events.size()); + for (const auto & [k, v] : profile_events) + map.push_back(Tuple{k, v}); + columns[i++]->insert(map); + } + columns[i++]->insert(round_id); + columns[i++]->insert(phase); + columns[i++]->insert(phase_duration_microseconds); + { + Map map; + map.reserve(phase_metrics.size()); + for (const auto & [k, v] : phase_metrics) + map.push_back(Tuple{k, v}); + columns[i++]->insert(map); + } +} + +} diff --git a/src/Interpreters/ContentAddressedGarbageCollectionLog.h b/src/Interpreters/ContentAddressedGarbageCollectionLog.h new file mode 100644 index 000000000000..9cbdbd3525f6 --- /dev/null +++ b/src/Interpreters/ContentAddressedGarbageCollectionLog.h @@ -0,0 +1,63 @@ +#pragma once +#include +#include +#include +#include + +namespace DB +{ + +struct ContentAddressedGarbageCollectionLogElement +{ + /// `PHASE`: one row per GC phase, emitted between the round's `START` and `FINISH` and correlated + /// with them by `round_id`. + enum EventType : int8_t { START = 1, FINISH = 2, PHASE = 3 }; + /// `DEFERRED`: the round acquired the GC lease and took the skip-unchanged fast path -- no fold, no + /// pre-CAS deletes, no `gc/state` CAS. Kept distinct from `SUCCESS` so a query against this table can + /// tell a round that genuinely folded and found nothing apart from one that never folded at all. + enum Outcome : int8_t { UNKNOWN = 1, SUCCESS = 2, NOT_A_LEADER = 3, FAILED = 4, DEFERRED = 5 }; + enum Trigger : int8_t { SCHEDULED = 1, MANUAL = 2 }; + + time_t event_time = 0; + Decimal64 event_time_microseconds = 0; + + EventType event_type = START; + String disk_name; + String srid; /// server_root_id of the mount whose GC scheduler ran this round + String gc_id; + Trigger trigger = SCHEDULED; + + UInt64 round = 0; + Outcome outcome = UNKNOWN; /// UNKNOWN on START; set to SUCCESS/NOT_A_LEADER/FAILED on FINISH + UInt64 candidates_marked = 0; + UInt64 objects_deleted = 0; + UInt64 objects_absent = 0; + UInt64 objects_replaced = 0; + UInt64 objects_spared = 0; + UInt64 manifests_deleted = 0; /// owner-removed manifest bodies deleted (B11 — distinct from blob deletes) + UInt64 entries_condemned = 0; /// retired-cursor pipeline: entries newly condemned this round + UInt64 entries_graduated = 0; /// retired-cursor pipeline: entries newly round-passed (delete_pending) this round + UInt64 entries_redeleted = 0; /// retired-cursor pipeline: pending exact-token blob deletes executed this round + UInt64 fence_outs = 0; /// expired mounts fenced-out by the round's heartbeat floor + UInt64 anomalies = 0; /// fold clamps surfaced this round + UInt64 duration_ms = 0; + String error; + std::map profile_events; /// per-round delta (FINISH); per-phase delta (PHASE) + + String round_id; /// correlator for every row of one round attempt + String phase; /// empty on START/FINISH + UInt64 phase_duration_microseconds = 0; /// PHASE rows only + std::map phase_metrics; /// PHASE rows only + + static std::string name() { return "ContentAddressedGarbageCollectionLog"; } + static ColumnsDescription getColumnsDescription(); + static NamesAndAliases getNamesAndAliases() { return {}; } + void appendToBlock(MutableColumns & columns) const; +}; + +class ContentAddressedGarbageCollectionLog : public SystemLog +{ + using SystemLog::SystemLog; +}; + +} diff --git a/src/Interpreters/ContentAddressedLog.cpp b/src/Interpreters/ContentAddressedLog.cpp new file mode 100644 index 000000000000..9ae6506ea326 --- /dev/null +++ b/src/Interpreters/ContentAddressedLog.cpp @@ -0,0 +1,73 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +ColumnsDescription ContentAddressedLogElement::getColumnsDescription() +{ + auto lc_string = std::make_shared(std::make_shared()); + return ColumnsDescription + { + {"hostname", lc_string, "Host name of the server that emitted the event."}, + {"event_date", std::make_shared(), "Event date."}, + {"event_time", std::make_shared(), "Event time."}, + {"event_time_microseconds", std::make_shared(6), "Event time with microseconds."}, + {"event_type", lc_string, "The CA decision/event (blob_put, blob_reuse_adopt, root_remove, indegree_zero, gc_retire_decision, gc_recheck_verdict, blob_delete, dangling_access, corrupt_dangle, ...)."}, + {"disk_name", lc_string, "Content-addressed disk / pool the event belongs to."}, + {"namespace", std::make_shared(), "roots/ (server/table), empty if N/A."}, + {"ref_name", std::make_shared(), "Part name / ref the event concerns, empty if N/A."}, + {"object_kind", lc_string, "none/blob/manifest/root/snapshot."}, + {"object_hash", std::make_shared(), "Content hash (lowercase hex) of the object, empty if N/A."}, + {"token", std::make_shared(), "Incarnation token (ETag) involved, empty if N/A."}, + {"round", std::make_shared(), "GC round (0 if N/A)."}, + {"generation", std::make_shared(), "GC snapshot generation (0 if N/A)."}, + {"at_version", std::make_shared(), "Manifest shard_version of the driving journal record (0 if N/A)."}, + {"outcome", lc_string, "Decision outcome (ok/adopt/resurrect/deleted/replaced/spared/absent/zeroed/skipped/...)."}, + {"reason", lc_string, "Human-readable WHY of the decision (the rationale) -- templated across rows, so LowCardinality."}, + {"thread_id", std::make_shared(), "OS thread that emitted the event."}, + {"query_id", std::make_shared(), "Query id for correlation with system.query_log (empty if N/A)."}, + {"detail", std::make_shared(lc_string, std::make_shared()), + "Structured event-specific facts (e.g. condemn_round, superseded_token, code, site)."}, + }; +} + +void ContentAddressedLogElement::appendToBlock(MutableColumns & columns) const +{ + size_t i = 0; + columns[i++]->insert(getFQDNOrHostName()); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); + columns[i++]->insert(event_time); + columns[i++]->insert(event_time_microseconds); + columns[i++]->insert(event_type); + columns[i++]->insert(disk_name); + columns[i++]->insert(namespace_); + columns[i++]->insert(ref_name); + columns[i++]->insert(object_kind); + columns[i++]->insert(object_hash); + columns[i++]->insert(token); + columns[i++]->insert(round); + columns[i++]->insert(gen); + columns[i++]->insert(at_version); + columns[i++]->insert(outcome); + columns[i++]->insert(reason); + columns[i++]->insert(thread_id); + columns[i++]->insert(query_id); + { + Map map; + map.reserve(detail.size()); + for (const auto & [k, v] : detail) + map.push_back(Tuple{k, v}); + columns[i++]->insert(map); + } +} + +} diff --git a/src/Interpreters/ContentAddressedLog.h b/src/Interpreters/ContentAddressedLog.h new file mode 100644 index 000000000000..84c7ae3c251c --- /dev/null +++ b/src/Interpreters/ContentAddressedLog.h @@ -0,0 +1,47 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace DB +{ + +/// One row per content-addressed (CA) decision/event (B170). The decoupled Core POD `Cas::CasEvent` +/// is mapped to this element by `ContentAddressedMetadataStorage::makeCasEventSink` and forwarded to +/// the SystemLog. Optional (off by default); enabled for soak/CI. The set is exhaustive enough to +/// reconstruct an entity's whole lifetime; `reason`/`detail` carry each decision's rationale. +struct ContentAddressedLogElement +{ + time_t event_time = 0; + Decimal64 event_time_microseconds = 0; + + String event_type; /// Cas::CasEventType name (snake_case), LowCardinality in the table + String disk_name; + String namespace_; + String ref_name; + String object_kind; /// none/blob/manifest/root/snap + String object_hash; + String token; + UInt64 round = 0; + UInt64 gen = 0; + UInt64 at_version = 0; + String outcome; + String reason; + UInt64 thread_id = 0; + String query_id; + std::map detail; + + static std::string name() { return "ContentAddressedLog"; } + static ColumnsDescription getColumnsDescription(); + static NamesAndAliases getNamesAndAliases() { return {}; } + void appendToBlock(MutableColumns & columns) const; +}; + +class ContentAddressedLog : public SystemLog +{ + using SystemLog::SystemLog; +}; + +} diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 1edce5d06b45..a55da287815b 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -6238,6 +6238,22 @@ std::shared_ptr Context::getPartLog() const return shared->system_logs->part_log; } +std::shared_ptr Context::getContentAddressedGarbageCollectionLog() const +{ + SharedLockGuard lock(shared->mutex); + if (!shared->system_logs) + return {}; + return shared->system_logs->cas_gc_log; +} + +std::shared_ptr Context::getContentAddressedLog() const +{ + SharedLockGuard lock(shared->mutex); + if (!shared->system_logs) + return {}; + return shared->system_logs->cas_log; +} + std::shared_ptr Context::getBackgroundSchedulePoolLog() const { SharedLockGuard lock(shared->mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index e939d24e0e86..2ee2713caa70 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -128,6 +128,8 @@ class QueryMetricLog; class QueryThreadLog; class QueryViewsLog; class PartLog; +class ContentAddressedGarbageCollectionLog; +class ContentAddressedLog; class BackgroundSchedulePoolLog; class TextLog; class TraceLog; @@ -1619,6 +1621,8 @@ class Context: public ContextData, public std::enable_shared_from_this /// Returns an object used to log operations with parts if it possible. /// Provide table name to make required checks. std::shared_ptr getPartLog() const; + std::shared_ptr getContentAddressedGarbageCollectionLog() const; + std::shared_ptr getContentAddressedLog() const; std::shared_ptr getBackgroundSchedulePoolLog() const; diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 5f49f439e684..3447e9a70bea 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -11,6 +11,8 @@ #include +#include + #include #include #include @@ -366,6 +368,37 @@ void ServerAsynchronousMetrics::updateImpl(TimePoint update_time, TimePoint curr } } #endif + + /// Per-disk CAS GC health, for Prometheus scraping. `tryFromDisk` returns nullptr for a + /// disk whose metadata storage is not content-addressed (the common case); `gcHealth()` + /// returns nullopt for a content-addressed disk whose GC scheduler has not started yet + /// (still opening, read-only, or GC disabled by configuration) -- both are skipped + /// silently, same as the DiskUsed_/DiskTotal_ metrics above skip disks that don't report + /// space. This runs on every asynchronous-metrics tick for every configured disk and must + /// never throw. + try + { + if (auto * ca_storage = ContentAddressedMetadataStorage::tryFromDisk(disk)) + { + if (auto health = ca_storage->gcHealth()) + { + new_values[fmt::format("CASGCIsLeader_{}", name)] = { health->is_leader ? 1 : 0, + "Whether this server currently holds the content-addressed garbage-collection lease for the disk (1) or not (0, e.g. another replica is leading)." }; + new_values[fmt::format("CASGCPendingReclaim_{}", name)] = { health->pending_reclaim, + "Cumulative content-addressed objects condemned minus objects physically deleted by this process while it has held the GC lease on the disk. A persistently growing value indicates GC is not keeping up with reclaim." }; + new_values[fmt::format("CASGCLastSuccessAgeSeconds_{}", name)] = { health->last_success_age_seconds, + "Seconds since this process last completed a successful content-addressed GC round as leader on the disk (0 if it has never led one)." }; + new_values[fmt::format("CASGCWedgedNamespaces_{}", name)] = { health->wedged_namespace_count, + "Number of content-addressed namespaces on the disk currently stuck behind a wedged reference lane, unable to make GC progress." }; + } + } + } + catch (...) // NOLINT(bugprone-empty-catch) + { + /// Sampled on every server tick for every disk; a transient failure here (e.g. a + /// store health query hiccup) must never break the rest of asynchronous-metrics + /// collection. + } } } diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index 00c6f99595cd..a4ce6371abf6 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index dcf52c250fe2..684c1f3f4525 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -17,6 +17,8 @@ M(QueryLog, query_log, "Contains information about executed queries, for example, start time, duration of processing, error messages.") \ M(QueryThreadLog, query_thread_log, "Contains information about threads that execute queries, for example, thread name, thread start time, duration of query processing.") \ M(PartLog, part_log, "This table contains information about events that occurred with data parts in the MergeTree family tables, such as adding or merging data.") \ + M(ContentAddressedGarbageCollectionLog, cas_gc_log, "Per-round records of the content-addressed (CA) MergeTree garbage collector: a Start and a Finish row per GC round, with counts of objects marked/deleted, duration, outcome, and per-round ProfileEvents.") \ + M(ContentAddressedLog, cas_log, "Per-event content-addressed (CA) MergeTree audit log: one row per blob/ref/GC decision (put, reuse, retire, delete, root add/remove, in-degree-zero, fence, lease, ...) plus errors (dangling access, fail-closed). Enabled by default while the CA disk feature is experimental (see config.xml); it is the primary forensic instrument for a CA issue and costs nothing when no CA disk is configured.") \ M(BackgroundSchedulePoolLog, background_schedule_pool_log, "Contains history of background schedule pool task executions.") \ M(TraceLog, trace_log, "Contains stack traces collected by the sampling query profiler.") \ M(CrashLog, crash_log, "Contains information about stack traces for fatal errors. The table does not exist in the database by default, it is created only when fatal errors occur.") \ diff --git a/src/Storages/System/StorageSystemContentAddressedMounts.cpp b/src/Storages/System/StorageSystemContentAddressedMounts.cpp new file mode 100644 index 000000000000..2a1b82a01ee7 --- /dev/null +++ b/src/Storages/System/StorageSystemContentAddressedMounts.cpp @@ -0,0 +1,273 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INVALID_STATE; +} + +StorageSystemContentAddressedMounts::StorageSystemContentAddressedMounts(const StorageID & table_id_) + : StorageWithCommonVirtualColumns(table_id_) +{ + StorageInMemoryMetadata storage_metadata; + storage_metadata.setColumns(ColumnsDescription( + { + {"disk", std::make_shared(), "Name of the content-addressed disk."}, + {"server_root_id", std::make_shared(), "Server root id owning the mount slot."}, + {"server_uuid", std::make_shared(), "UUID of the server incarnation holding the lease."}, + {"hostname", std::make_shared(), "Hostname recorded in the lease body."}, + {"process_id", std::make_shared(), "Process id recorded in the lease body."}, + {"writer_epoch", std::make_shared(), "Fenced writer epoch of the incarnation."}, + {"renewal_sequence", std::make_shared(), "Lease renewal sequence number."}, + {"started_at", std::make_shared(3), "Time when the lease started."}, + {"expires_at", std::make_shared(3), "Time when the lease expires."}, + {"min_active_build_sequence", std::make_shared(), "Oldest in-flight build sequence (UINT64_MAX means the mount said farewell)."}, + {"gc_fenced", std::make_shared(), "1 if GC fenced this slot out (terminal)."}, + {"state", std::make_shared(), "Mount slot state: live, expired, terminated, fenced or corrupt."}, + {"is_leader", std::make_shared(std::make_shared()), "1 if this server's GC scheduler holds this disk's leadership lease. NULL on rows describing other servers' mounts."}, + {"pending_reclaim", std::make_shared(std::make_shared()), "Cumulative condemned-minus-deleted backlog observed by this process's GC on this disk. NULL on rows describing other servers' mounts."}, + {"last_success_age_seconds", std::make_shared(std::make_shared()), "Seconds since this disk's GC last led a round (0 if it never led). NULL on rows describing other servers' mounts."}, + {"wedged_namespace_count", std::make_shared(std::make_shared()), "Ref-append lanes currently wedged on this disk. NULL on rows describing other servers' mounts."}, + {"lifecycle", std::make_shared(), "This server's content-addressed pool lifecycle for the disk (non-gated snapshot, always populated so a not-live disk stays visible): live, not_live, identity_lost, vanished, constructing (never started) or shutdown (torn down)."}, + {"lifecycle_reason", std::make_shared(), "The enum-clean sub-state word for a vanished disk: replaced or forgotten. Empty for every other lifecycle (so lifecycle || '(' || lifecycle_reason || ')' reads e.g. vanished(forgotten))."}, + {"lifecycle_detail", std::make_shared(), "The full typed reason text naming the actual cause when not live: the vanish diagnosis (data root replaced by a foreign pool / decommissioned by SYSTEM CAS FORGET at
cas_log
+ toYYYYMM(event_date) + 7500 + 1048576 + 8192 + 524288 + false + + + + + system + cas_gc_log
+ toYYYYMM(event_date) + 7500 + 1048576 + 8192 + 524288 + false +
+ + + + + node1 + + + node1 + + + + diff --git a/tests/integration/test_cas_drop_pool_member/configs/server_root_id_node2.xml b/tests/integration/test_cas_drop_pool_member/configs/server_root_id_node2.xml new file mode 100644 index 000000000000..65473eac537a --- /dev/null +++ b/tests/integration/test_cas_drop_pool_member/configs/server_root_id_node2.xml @@ -0,0 +1,16 @@ + + + + + + node2 + + + node2 + + + + diff --git a/tests/integration/test_cas_drop_pool_member/configs/storage_conf.xml b/tests/integration/test_cas_drop_pool_member/configs/storage_conf.xml new file mode 100644 index 000000000000..4916ab840d81 --- /dev/null +++ b/tests/integration/test_cas_drop_pool_member/configs/storage_conf.xml @@ -0,0 +1,46 @@ + + + + + + object_storage + s3 + cas + http://rustfs1:11121/test/cas_dpm_data/ + clickhouse + clickhouse + + + 1 + 1 + + + + object_storage + s3 + cas + http://rustfs1:11121/test/cas_dpm_data/ + clickhouse + clickhouse + true + 0 + + + + + +
+ disk_cas_dpm +
+
+
+
+
+
diff --git a/tests/integration/test_cas_drop_pool_member/test.py b/tests/integration/test_cas_drop_pool_member/test.py new file mode 100644 index 000000000000..6ac60271348d --- /dev/null +++ b/tests/integration/test_cas_drop_pool_member/test.py @@ -0,0 +1,323 @@ +import re +import shlex +import time + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + +cluster = ClickHouseCluster(__file__) + +# Both servers mount the SAME content-addressed pool over RustFS (not MinIO -- MinIO cannot serve CA +# pools, see memory), distinct server_root_id (node1/node2) -- exactly the shared-pool model test's +# two-node topology (test_cas_shared_pool), just on rustfs instead of minio (the model +# test predates rustfs support; test_cas_ref_snaplog is the rustfs precedent copied here). +STORAGE_POLICY = "cas_dpm" +RO_DISK = "disk_ca_ro" +CA_DISK = "disk_cas_dpm" + +SRID1 = "node1" +SRID2 = "node2" + +POOL = "cas_dpm_data" +BLOBS_PREFIX = POOL + "/blobs/" + +NUM_ROWS = 20000 + +# Background GC: grace=2s, interval=1s (storage_conf.xml). After the drop-pool-member command removes +# node2's namespaces the content that was only reachable through them becomes unreferenced GC fodder; +# poll until it drains. Bounded wait on a known background process, not a race workaround. +RECLAIM_RETRIES = 120 +RECLAIM_SLEEP = 1.0 # total bound ~= 120s + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + cluster.add_instance( + "node1", + main_configs=["configs/storage_conf.xml", "configs/server_root_id_node1.xml"], + with_rustfs=True, + stay_alive=True, + ) + cluster.add_instance( + "node2", + main_configs=["configs/storage_conf.xml", "configs/server_root_id_node2.xml"], + with_rustfs=True, + stay_alive=True, + ) + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def _count(prefix): + return len( + list(cluster.rustfs_client.list_objects(cluster.rustfs_bucket, prefix, recursive=True)) + ) + + +def _disks(node, query): + # Run a clickhouse-disks command against the read-only CA window over the same pool — cas-fsck refuses + # a writable pool, so it must go through disk_ca_ro (the ref-snaplog integration test's idiom). + return node.exec_in_container( + [ + "bash", + "-c", + "/usr/bin/clickhouse disks -C /etc/clickhouse-server/config.xml " + "--disk {} --save-logs --query {}".format(RO_DISK, shlex.quote(query)), + ] + ) + + +def test_drop_dead_pool_member_heals_the_pool(): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + + node1.query("DROP TABLE IF EXISTS t1 SYNC") + node2.query("DROP TABLE IF EXISTS t2 SYNC") + + blobs_baseline = _count(BLOBS_PREFIX) + + create_tpl = ( + "CREATE TABLE {tbl} (id Int64, s String) ENGINE = MergeTree() ORDER BY id " + "SETTINGS storage_policy = '{policy}'" + ) + + # (2) node2 gets its own table with several parts -- this data must NOT survive (node2 is about to + # be killed and decommissioned). + node2.query(create_tpl.format(tbl="t2", policy=STORAGE_POLICY)) + for i in range(4): + node2.query( + "INSERT INTO t2 SELECT number + {off}, toString(number + {off}) " + "FROM numbers({rows})".format(off=i * NUM_ROWS, rows=NUM_ROWS) + ) + assert int(node2.query("SELECT count() FROM t2")) == 4 * NUM_ROWS + + # (3) node1 gets its own table -- this data MUST survive the whole flow untouched. + node1.query(create_tpl.format(tbl="t1", policy=STORAGE_POLICY)) + node1.query( + "INSERT INTO t1 SELECT number, toString(number) FROM numbers({})".format(NUM_ROWS) + ) + n1_count = int(node1.query("SELECT count() FROM t1")) + n1_sum = int(node1.query("SELECT sum(id) FROM t1")) + assert n1_count == NUM_ROWS + + assert _count(BLOBS_PREFIX) > blobs_baseline, "expected content blobs after both nodes' inserts" + + # (3b) T9: system.cas_mounts scopes the GC-health columns (is_leader et al.) to the + # row for THIS server's own server_root_id; peer rows read NULL. Background GC (1s interval) + # should have led at least one round on each node by now, but that is a background race, not + # something this test synchronizes on directly -- poll rather than assume. For every disk + # that reports any non-NULL is_leader row there must be exactly one such row, and it must + # belong to the querying node's own server_root_id -- never a peer's. + for node, own_srid in ((node1, SRID1), (node2, SRID2)): + rows = [] + for _ in range(30): + rows = ( + node.query( + "SELECT disk, server_root_id FROM system.cas_mounts " + "WHERE is_leader IS NOT NULL ORDER BY disk" + ) + .strip() + .splitlines() + ) + if rows: + break + time.sleep(1.0) + assert rows, "expected at least one GC-health row with is_leader populated on {}".format( + node.name + ) + seen_disks = set() + for row in rows: + disk, srid = row.split("\t") + assert srid == own_srid, "peer server_root_id '{}' leaked GC health on disk '{}': {}".format( + srid, disk, rows + ) + assert disk not in seen_disks, "duplicate non-NULL is_leader row for disk '{}': {}".format( + disk, rows + ) + seen_disks.add(disk) + + # (4) Hard-kill node2: SIGKILL, no graceful farewell -- node2's mount lease is left to expire + # naturally, exactly the scenario decommission exists for. + node2.stop_clickhouse(kill=True) + + # (5) Wait until node1 observes node2's mount as no longer live (expired once its lease's TTL + # elapses with no renewal, since there was no graceful farewell to mark it terminated instead). + # min() because node1 sees the pool through TWO disks (the writable disk + the disk_ca_ro + # fsck window), so the mounts table carries one row per disk view for the same server_root_id -- + # aggregate to a single row for the equality assert. + assert_eq_with_retry( + node1, + "SELECT min(state != 'live') FROM system.cas_mounts WHERE server_root_id = '{}'".format( + SRID2 + ), + "1", + retry_count=90, + sleep_time=1.0, + ) + + # (6) Decommission the dead member from node1 -- PHASE 1 of a two-phase heal. SYSTEM queries do + # not accept a FORMAT clause (ParserSystemQuery is not part of ParserQueryWithOutput), so parse + # the default TSV row. Column order matches the interpreter's ColumnsDescription: + # server_root_id, namespaces_removed, namespaces_already_removed, committed_refs_removed, + # precommits_removed, manifest_debris_removed, staging_objects_removed, + # mountpoint_objects_removed, slot_removed, warnings. + # + # t2 is still `Live` at this point, so THIS call's normal drop path is what appends its + # removal terminal and moves its catalog row to `Removing` -- catalog deletion stays GC's job + # (`224aacd8eb9`), never the decommission command's, so a row this same call just legitimately + # transitioned still counts as "owned" and the retirement fence correctly refuses the slot. + # This is a success with GC completion pending, not a failure. + # The mounts-table poll above and the decommission command judge liveness by DIFFERENT + # predicates on purpose: the table renders TTL arithmetic over the last observed mount row, + # while the command re-reads the mountpoint object and refuses while the lease could still + # be live under its conservative safety margin. The destructive side being stricter is the + # fail-close direction, so the table saying "not live" does not guarantee the command is + # ready yet -- under sanitizer slowdowns the gap is wide enough to hit. Retry the command + # itself through the documented "wait for its lease to lapse" refusal, bounded. + report_tsv = None + for _ in range(90): + try: + report_tsv = node1.query( + "SYSTEM CAS DROP POOL MEMBER '{}' FROM DISK '{}'".format(SRID2, CA_DISK) + ).rstrip("\n") + break + except Exception as e: + if "alive or contended" not in str(e): + raise + time.sleep(1.0) + assert report_tsv is not None, "decommission kept refusing: lease never lapsed within the bound" + fields = report_tsv.split("\t") + assert len(fields) == 10, report_tsv + assert fields[0] == SRID2, report_tsv + assert int(fields[1]) >= 1, report_tsv # namespaces_removed: the drop half did its work + assert int(fields[8]) == 0, report_tsv # slot_removed: not yet -- GC owns the row now + assert "pool member decommission underway" in fields[9], report_tsv + + # (6b) PHASE 2: drive GC and re-run the decommission until the slot retires. Folding a fresh + # terminal and pruning its catalog row are two separate GC rounds by design (a fold-then-prune + # handoff -- see `CasDecommissionCatalogDuties.FoldedTerminalRemainsGcOwnedAndOnlyRequestsAnotherRound`), + # so poll rather than assume one round suffices. The explicit `GC RUN` is the same idiom + # `test_cas_replicated_relink` uses; it runs a synchronous round regardless of the background + # cadence. Catalog-row pruning is Task 5's catalog-only pre-fold drain and does not consult + # Stage A's destructive-reclaim suppression, so this heals under the current Stage-A posture. + for _ in range(30): + node1.query("SYSTEM CAS GC RUN '{}'".format(CA_DISK)) + report_tsv = node1.query( + "SYSTEM CAS DROP POOL MEMBER '{}' FROM DISK '{}'".format(SRID2, CA_DISK) + ).rstrip("\n") + fields = report_tsv.split("\t") + assert len(fields) == 10, report_tsv + if int(fields[8]) == 1: + break + assert "pool member decommission underway" in fields[9], report_tsv + else: + pytest.fail("pool never healed after driving GC: {}".format(report_tsv)) + + assert fields[9] == "", report_tsv # warnings: the pool healed cleanly + + # (7) node1's own data survives the whole flow untouched. + assert int(node1.query("SELECT count() FROM t1")) == n1_count + assert int(node1.query("SELECT sum(id) FROM t1")) == n1_sum + + # (8) node2's server_root_id is gone from the mounts table (only true once the slot above actually + # retired -- checked after PHASE 2, not right after the first, still-pending call). + assert ( + node1.query( + "SELECT count() FROM system.cas_mounts WHERE server_root_id = '{}'".format(SRID2) + ).strip() + == "0" + ) + + # (9) Drive GC (node1's background GC is already running against the shared pool) to reclaim + # node2's now-unreferenced content, then poll for the blob count to drain back to baseline -- + # the authoritative "no content leftovers" proof, mirroring the ref-snaplog integration test's + # idiom. node1's own t1 is still alive at this point and its blobs legitimately stay in the + # pool, so a drain-to-baseline check is only meaningful after t1 is dropped too -- its survival + # was already proven byte-for-byte in step 7, so drop it now and demand the pool drain to + # EMPTY: node2's content via the decommission, t1's via the ordinary drop, no leftovers from + # either. Then a read-only fsck over the drained pool must report clean (no dangling, no + # unaccounted objects). + node1.query("DROP TABLE t1 SYNC") + at_drop = _count(BLOBS_PREFIX) + + # THE RECLAMATION. Both contributions must go: node2's content via the decommission, t1's via the + # ordinary drop. Polled with an early exit, then cross-checked against GC's own bookkeeping so that + # a pool which shrank for some other reason cannot pass for a round that reclaimed it. + final = _count(BLOBS_PREFIX) + for _ in range(RECLAIM_RETRIES): + if final <= blobs_baseline: + break + time.sleep(RECLAIM_SLEEP) + final = _count(BLOBS_PREFIX) + + assert final <= blobs_baseline, ( + "the drained pool did not return to its baseline: baseline={}, at_drop={}, final={}".format( + blobs_baseline, at_drop, final + ) + ) + + node1.query("SYSTEM FLUSH LOGS") + rounds = int( + node1.query( + "SELECT count() FROM system.cas_gc_log " + "WHERE event_type = 'Finish' AND outcome = 'Success'" + ).strip() + or 0 + ) + assert rounds > 0, "no successful GC round ran at all" + deleted = int( + node1.query( + "SELECT sum(objects_deleted + manifests_deleted) " + "FROM system.cas_gc_log WHERE event_type = 'Finish'" + ).strip() + or 0 + ) + assert deleted > 0, "the pool drained but GC's own bookkeeping reports no deletion" + + # node2's decommissioned-and-healed namespace (t2) left canonical dead-life residue behind: its + # catalog row is gone (that is what let the slot retire above), but its `_ckpt`/`_files`/`_log` + # objects are the perpetual namespace janitor's job, not decommission's or GC's own destructive + # round. t1's row is pruned the same way once dropped. That residue must DRAIN to zero -- the + # janitor deletes one bounded page per round, so this is polled rather than read once -- and it + # must never be hard corruption on the way there, which is why `lifeless_keys` is checked on every + # poll and not only at the end. + for _ in range(RECLAIM_RETRIES): + fsck = _disks(node1, "cas-fsck") + assert "lifeless_keys=0" in fsck, fsck + janitor_pending_match = re.search(r"\bjanitor_pending=(\d+)", fsck) + assert janitor_pending_match, "cas-fsck summary is missing the janitor_pending field: {}".format(fsck) + if int(janitor_pending_match.group(1)) == 0: + break + time.sleep(RECLAIM_SLEEP) + + assert "dangling=0" in fsck, fsck + assert "unaccounted=0" in fsck, fsck + assert int(janitor_pending_match.group(1)) == 0, ( + "the dead-life residue from the healed decommission and the t1 drop never drained: {}".format(fsck) + ) + + # (10) Re-run the same command: decommission tombstones the owner anchor in place rather than + # deleting it, so the slot is not "unknown" -- the tombstone is found and the re-run is refused + # with CORRUPTED_DATA and a message telling the operator this server-root was explicitly + # decommissioned and will not silently resume. + err = node1.query_and_get_error( + "SYSTEM CAS DROP POOL MEMBER '{}' FROM DISK '{}'".format(SRID2, CA_DISK) + ) + assert "explicitly decommissioned" in err, err + + +def test_drop_pool_member_rejected_on_readonly_disk(): + # disk_ca_ro is the fail-close guard's target: an observe-only window over the SAME pool (used + # elsewhere in this test only for fsck). Decommission is a mutating operation, so it must be + # rejected on this disk exactly like `createTransaction`/GC round/GC rebuild are -- READONLY, + # not a silent no-op or a crash further down the call chain. + node1 = cluster.instances["node1"] + err = node1.query_and_get_error( + "SYSTEM CAS DROP POOL MEMBER 'whatever' FROM DISK '{}'".format(RO_DISK) + ) + assert "read-only" in err, err diff --git a/tests/integration/test_cas_file_cache/__init__.py b/tests/integration/test_cas_file_cache/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_cas_file_cache/configs/storage_conf.xml b/tests/integration/test_cas_file_cache/configs/storage_conf.xml new file mode 100644 index 000000000000..fcf833676ffb --- /dev/null +++ b/tests/integration/test_cas_file_cache/configs/storage_conf.xml @@ -0,0 +1,34 @@ + + + + + + object_storage + s3 + cas + itest-cas-file-cache + http://rustfs1:11121/test/cas_cache_data/ + clickhouse + clickhouse + + + + cache + disk_ca_s3 + /tmp/cas_file_cache/ + 1000000000 + + + + + +
+ disk_ca_s3_cache +
+
+
+
+
+
diff --git a/tests/integration/test_cas_file_cache/test.py b/tests/integration/test_cas_file_cache/test.py new file mode 100644 index 000000000000..a08f173b2874 --- /dev/null +++ b/tests/integration/test_cas_file_cache/test.py @@ -0,0 +1,117 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +STORAGE_POLICY = "cas_cache" +NUM_ROWS = 100000 + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + cluster.add_instance( + "node", + main_configs=["configs/storage_conf.xml"], + with_rustfs=True, + stay_alive=True, + ) + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_cache_over_ca_startup_and_roundtrip(): + # Before the fix the server fails to register the cache-over-CA disk (NOT_IMPLEMENTED at + # checkAccess), so this whole module fails at cluster.start(). After the fix, startup + a + # write/read round-trip succeed. + node = cluster.instances["node"] + + node.query("DROP TABLE IF EXISTS cas_cache_test SYNC") + node.query( + """ + CREATE TABLE cas_cache_test (id Int64, data String) + ENGINE = MergeTree() ORDER BY id + SETTINGS storage_policy = '{}' + """.format( + STORAGE_POLICY + ) + ) + node.query( + "INSERT INTO cas_cache_test SELECT number, toString(number) FROM numbers({})".format( + NUM_ROWS + ) + ) + expected_sum = (NUM_ROWS - 1) * NUM_ROWS // 2 + assert int(node.query("SELECT count() FROM cas_cache_test")) == NUM_ROWS + assert int(node.query("SELECT sum(id) FROM cas_cache_test")) == expected_sum + + node.query("DROP TABLE cas_cache_test SYNC") + + +def _profile_event(node, query_id, event): + node.query("SYSTEM FLUSH LOGS") + v = node.query( + "SELECT sum(ProfileEvents['{}']) FROM system.query_log " + "WHERE query_id = '{}' AND type = 'QueryFinish'".format(event, query_id) + ).strip() + return int(v) if v else 0 + + +def test_cache_hits_on_repeated_reads(): + # The point of the feature: a second full scan of the same data is served from the local file + # cache instead of re-fetching immutable content blobs from object storage. + node = cluster.instances["node"] + + node.query("DROP TABLE IF EXISTS cas_cache_metrics SYNC") + node.query( + """ + CREATE TABLE cas_cache_metrics (id Int64, data String) + ENGINE = MergeTree() ORDER BY id + SETTINGS storage_policy = '{}' + """.format( + STORAGE_POLICY + ) + ) + node.query( + "INSERT INTO cas_cache_metrics SELECT number, toString(number % 1000) FROM numbers(1000000)" + ) + node.query("OPTIMIZE TABLE cas_cache_metrics FINAL") + + # Start from a cold cache. + node.query("SYSTEM DROP FILESYSTEM CACHE") + + q1 = "cas_cache_cold_scan" + node.query( + "SELECT sum(cityHash64(id, data)) FROM cas_cache_metrics", + query_id=q1, + settings={"enable_filesystem_cache": 1}, + ) + # The cold read must POPULATE the cache (read-through), not just read from source: pin the write + # side so a config where the cache never fills cannot pass on the warm-scan check alone. + assert int(node.query("SELECT count() FROM system.filesystem_cache")) > 0 + + q2 = "cas_cache_warm_scan" + node.query( + "SELECT sum(cityHash64(id, data)) FROM cas_cache_metrics", + query_id=q2, + settings={"enable_filesystem_cache": 1}, + ) + + cold_source = _profile_event(node, q1, "CachedReadBufferReadFromSourceBytes") + warm_source = _profile_event(node, q2, "CachedReadBufferReadFromSourceBytes") + warm_cache = _profile_event(node, q2, "CachedReadBufferReadFromCacheBytes") + + assert cold_source > 0, "cold scan should read from source" + assert warm_source * 10 < cold_source, ( + "warm scan should read far fewer source bytes (cold={}, warm={})".format( + cold_source, warm_source + ) + ) + assert warm_cache > 0, "warm scan should read from the filesystem cache" + + assert int(node.query("SELECT count() FROM system.filesystem_cache")) > 0 + + node.query("DROP TABLE cas_cache_metrics SYNC") diff --git a/tests/integration/test_cas_gc_s3/__init__.py b/tests/integration/test_cas_gc_s3/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_cas_gc_s3/configs/storage_conf.xml b/tests/integration/test_cas_gc_s3/configs/storage_conf.xml new file mode 100644 index 000000000000..ac933c421ff5 --- /dev/null +++ b/tests/integration/test_cas_gc_s3/configs/storage_conf.xml @@ -0,0 +1,33 @@ + + + + + object_storage + s3 + cas + + itest-content-addressed-gc-s3 + + http://rustfs1:11121/test/cas_gc_data/ + clickhouse + clickhouse + + 1 + 1 + + + + + +
+ disk_cas_gc_s3 +
+
+
+
+
+
diff --git a/tests/integration/test_cas_gc_s3/test.py b/tests/integration/test_cas_gc_s3/test.py new file mode 100644 index 000000000000..7827ab80dbf9 --- /dev/null +++ b/tests/integration/test_cas_gc_s3/test.py @@ -0,0 +1,189 @@ +import time + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +STORAGE_POLICY = "cas_gc_s3" + +# Endpoint is http://rustfs1:11121/test/cas_gc_data/, so the pool's blobs and part footers live +# under these key prefixes inside the `test` RustFS bucket. The authoritative "no S3 leftovers" +# proof checks BOTH: a dropped table must leave neither content blobs nor part footers behind. +BLOBS_PREFIX = "cas_gc_data/blobs/" +PARTS_PREFIX = "cas_gc_data/parts/" + +# Enough rows / inserts to materialise several distinct blobs in the pool. +NUM_ROWS = 100000 +NUM_INSERTS = 8 + +# The background GC runs with grace=1s, interval=1s. After DROP TABLE ... SYNC the dropped table's +# footers/blobs become unreferenced. How long we give the background GC to do its rounds; this is +# waiting on a known background process, not papering over a race. +RECLAIM_RETRIES = 60 +RECLAIM_SLEEP = 1.0 # seconds; total bound ~= RECLAIM_RETRIES * RECLAIM_SLEEP = 60s + +# The destructive phases of a GC round. Every one of them stamps `suppressed` into its phase_metrics, +# which is how a suppressed round says so in a queryable way rather than only in the text log. +DESTRUCTIVE_PHASES = ( + "handoff_reclaim", + "manifest_deletes", + "namespace_cleanup", + "ref_object_cleanup", + "orphan_sweep", +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + cluster.add_instance( + "node", + main_configs=["configs/storage_conf.xml"], + with_rustfs=True, + stay_alive=True, + ) + + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def count_prefix(prefix): + objects = cluster.rustfs_client.list_objects( + cluster.rustfs_bucket, prefix, recursive=True + ) + return len(list(objects)) + + +def count_pool_objects(): + # Both content blobs and part footers count: reclamation means BOTH drain. + return count_prefix(BLOBS_PREFIX) + count_prefix(PARTS_PREFIX) + + +def gc_log_scalar(node, query): + node.query("SYSTEM FLUSH LOGS") + return int(node.query(query).strip()) + + +def test_gc_reclaims_dropped_blobs(): + """ + The background GC reclaims a dropped table's blobs and part footers. + + A GC round may destroy only while holding a frontier proof for EVERY namespace that can hold a live + edge — reachability is a property of the whole pool, so deleting one blob asserts something about + every namespace at once, including the ones the round never looked at. The catalog supplies that set, + and each namespace's proof is one exact-key read at its cursor's successor. + + So the reclamation below is asserted TOGETHER WITH the gate's own reason for permitting it: rounds + ran, every namespace in the universe reached a proven frontier, and no destructive phase reported + itself suppressed. Without that, a pool that shrank for some unrelated reason would read as a pass. + """ + node = cluster.instances["node"] + + node.query("DROP TABLE IF EXISTS cas_gc_test SYNC") + + # (1) Baseline: how many objects (blobs + part footers) exist in the pool before our table. + baseline = count_pool_objects() + + node.query( + """ + CREATE TABLE cas_gc_test ( + id Int64, + data String + ) ENGINE = MergeTree() + ORDER BY id + SETTINGS storage_policy = '{}' + """.format( + STORAGE_POLICY + ) + ) + + # (2) Insert enough distinct rows across several inserts to produce several blobs. + for i in range(NUM_INSERTS): + node.query( + "INSERT INTO cas_gc_test " + "SELECT number + {offset}, toString(number + {offset}) " + "FROM numbers({rows})".format(offset=i * NUM_ROWS, rows=NUM_ROWS) + ) + + assert int(node.query("SELECT count() FROM cas_gc_test")) == NUM_INSERTS * NUM_ROWS + + after_insert = count_pool_objects() + assert ( + after_insert > baseline + ), "expected pool object count (blobs+parts) to rise above baseline {} after inserts, got {}".format( + baseline, after_insert + ) + + # (3) Drop the table: refs are unlinked synchronously; the blobs and part footers become + # unreferenced GC fodder. + node.query("DROP TABLE cas_gc_test SYNC") + + # (4) Poll for the reclamation, exiting as soon as it has happened (grace=1s, interval=1s, so + # plenty of rounds run within the window). + final = count_pool_objects() + for _ in range(RECLAIM_RETRIES): + if final <= baseline: + break + time.sleep(RECLAIM_SLEEP) + final = count_pool_objects() + + # (5) The dropped table's objects are GONE, back to the pre-table baseline. + assert final <= baseline, ( + "the dropped table's objects were not reclaimed: baseline={}, after_insert={}, final={} " + "(blobs={}, parts={})".format( + baseline, + after_insert, + final, + count_prefix(BLOBS_PREFIX), + count_prefix(PARTS_PREFIX), + ) + ) + + # (6) …AND FOR THE RIGHT REASON, which is what separates "the gate opened on a proven frontier" + # from "the pool shrank for some other reason". + # + # (a) Rounds actually ran and completed as the leader. + rounds = gc_log_scalar( + node, + "SELECT count() FROM system.cas_gc_log " + "WHERE event_type = 'Finish' AND outcome = 'Success'", + ) + assert rounds > 0, "no successful GC round ran at all — this is not suppression, it is a wedge" + + # (b) The rounds report the deletion on their OWN bookkeeping, not only on the S3 object count. + deleted = gc_log_scalar( + node, + "SELECT sum(objects_deleted + manifests_deleted) " + "FROM system.cas_gc_log WHERE event_type = 'Finish'", + ) + assert deleted > 0, "the pool shrank but no round reported deleting anything" + + # (c) At least one round proved EVERY namespace in its universe (frontier_proven == + # frontier_namespaces, both nonzero). A round held up by a clamp, a hold or an exhausted + # probe budget would have frontier_proven < frontier_namespaces and could not have opened + # the gate. + fully_proven_rounds = gc_log_scalar( + node, + "SELECT count() FROM system.cas_gc_log " + "WHERE phase = 'fold_ref_intake' " + " AND phase_metrics['frontier_namespaces'] > 0 " + " AND phase_metrics['frontier_proven'] = phase_metrics['frontier_namespaces']", + ) + assert fully_proven_rounds > 0, ( + "no round reached a fully proven frontier, so whatever removed those objects was not a round " + "acting on a complete frontier" + ) + + # (d) And no destructive phase reported itself suppressed — the gate really did open. + suppressed_phases = gc_log_scalar( + node, + "SELECT uniqExact(phase) FROM system.cas_gc_log " + "WHERE phase IN {} AND phase_metrics['suppressed'] = 1".format(DESTRUCTIVE_PHASES), + ) + assert suppressed_phases == 0, ( + "a destructive phase reported itself suppressed on a pool whose frontier was complete" + ) diff --git a/tests/integration/test_cas_gc_sharded/__init__.py b/tests/integration/test_cas_gc_sharded/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_cas_gc_sharded/configs/server_root_id_node1.xml b/tests/integration/test_cas_gc_sharded/configs/server_root_id_node1.xml new file mode 100644 index 000000000000..92aea0f9cc5d --- /dev/null +++ b/tests/integration/test_cas_gc_sharded/configs/server_root_id_node1.xml @@ -0,0 +1,12 @@ + + + + + + node1 + + + + diff --git a/tests/integration/test_cas_gc_sharded/configs/server_root_id_node2.xml b/tests/integration/test_cas_gc_sharded/configs/server_root_id_node2.xml new file mode 100644 index 000000000000..da16e054454a --- /dev/null +++ b/tests/integration/test_cas_gc_sharded/configs/server_root_id_node2.xml @@ -0,0 +1,12 @@ + + + + + + node2 + + + + diff --git a/tests/integration/test_cas_gc_sharded/configs/storage_conf.xml b/tests/integration/test_cas_gc_sharded/configs/storage_conf.xml new file mode 100644 index 000000000000..e4236453a6ee --- /dev/null +++ b/tests/integration/test_cas_gc_sharded/configs/storage_conf.xml @@ -0,0 +1,38 @@ + + + + + object_storage + s3 + cas + + + http://rustfs1:11121/test/cas_gc_sharded/ + clickhouse + clickhouse + + 2 + + 1 + 1 + + + + + +
+ disk_cas_gc_sharded +
+
+
+
+
+
diff --git a/tests/integration/test_cas_gc_sharded/test.py b/tests/integration/test_cas_gc_sharded/test.py new file mode 100644 index 000000000000..a7c56edf2705 --- /dev/null +++ b/tests/integration/test_cas_gc_sharded/test.py @@ -0,0 +1,362 @@ +"""Phase 4 integration soak: two-replica disjoint-shard GC with gc_shards=2. + +Two ClickHouse nodes mount the SAME CA pool (shared-pool mode). The pool is configured with +`gc_shards=2` — at first GC-state creation the coordinator writes two `blob_target/` +runs (one per shard) per GC generation. Blob hashes route to shard 0 or shard 1 by +`blobShard(blob_hash, 2) = high64(hash) % 2` (CasGcShardPlan::blobShard). Each generation +therefore produces keys under both `blob_target/0/` and `blob_target/1/` (assuming the workload +generates enough distinct blobs to cover both shard buckets — see the 2000-row inserts below). + +The pool runs on RustFS, not MinIO: the CA mount capability probe (`CasProbe::runCapabilityProbe`) +requires an S3-compatible backend that enforces `DeleteObject If-Match` (conditional delete); +MinIO OSS silently honors a mismatched-token DELETE instead of rejecting it, which the fail-closed +probe treats as a fatal capability gap. + +The soak drives a blob-churn workload (INSERT x3 + OPTIMIZE FINAL + DROP x3 x2 rounds) on +`node1`, then restarts `node2` (light chaos), quiesces (waits for GC to drain), and asserts: + + A) No dangle / no loss — after quiesce both replicas return the same row counts for the live + table; no CA-layer exception or fatal error appears in either server log. + + B) Single completion signal per generation — no partial-shard product was adopted before all + shards were committed. The pool's `gc/state` object names exactly one adopted + (generation, attempt) pair; that pair's fold-seal object must exist. A retry-created attempt + that never got adopted (and so is not named by `gc/state`) may have written its own seal too — + this is expected and must NOT be treated as a second completion signal; that is why this test + resolves the adopted pair from `gc/state` first, rather than counting every seal object it can + list under a generation. The soak waits for `gc/state` to adopt a nonzero generation before + asserting. + + C) Disjoint-shard reduce progress — over the whole soak, `blob_target` keys were physically + written under BOTH shard 0 and shard 1 (proving the sharded path executed, not just the + gc_shards==1 fast-path). This is checked across every generation/attempt seen under + `gc/gen/`, not only the currently-adopted one: a seal's `blob_target_runs` carry a PARENT's + runs forward as references, but the physical objects stay under the (generation, attempt) + prefix where they were originally written -- a late-soak adopted attempt whose own round + produced no new deltas can have an empty `blob_target/` prefix of its own even though earlier + attempts wrote plenty. Scanning the whole `gc/gen/` subtree is the only check that matches + where the objects actually physically live. +""" + +import re +import time + +import pytest +from minio.error import S3Error + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +STORAGE_POLICY = "cas_gc_sharded" + +# Pool bucket key prefixes (the endpoint is http://rustfs1:11121/test/cas_gc_sharded/). +POOL_PREFIX = "cas_gc_sharded" +GC_STATE_KEY = POOL_PREFIX + "/gc/state" + +# Workload parameters — enough rows to produce blobs routing to BOTH hash-mod-2 buckets. +NUM_ROWS_PER_INSERT = 2000 +NUM_INSERTS = 4 + +# GC quiesce: grace=3s, interval=1s. We poll up to 90 s for at least one completed generation. +GC_POLL_RETRIES = 90 +GC_POLL_SLEEP = 1.0 + +# Error patterns in server logs that must NOT appear in a healthy soak. +CA_FATAL_LOG_KEYWORDS = [ + "DANGLE", + "dangle", + "CorruptDangle", +] + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + cluster.add_instance( + "node1", + main_configs=["configs/storage_conf.xml", "configs/server_root_id_node1.xml"], + macros={"replica": "node1"}, + with_rustfs=True, + with_zookeeper=True, + stay_alive=True, + ) + cluster.add_instance( + "node2", + main_configs=["configs/storage_conf.xml", "configs/server_root_id_node2.xml"], + macros={"replica": "node2"}, + with_rustfs=True, + with_zookeeper=True, + stay_alive=True, + ) + + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +# --------------------------------------------------------------------------- +# RustFS helpers +# --------------------------------------------------------------------------- + +def list_rustfs_prefix(prefix, recursive=True): + """Return a list of object keys under `prefix` in the shared RustFS bucket.""" + objects = cluster.rustfs_client.list_objects( + cluster.rustfs_bucket, prefix, recursive=recursive + ) + return [o.object_name for o in objects] + + +def get_rustfs_object(key): + """Return the raw bytes of `key`, or None if it does not exist.""" + try: + resp = cluster.rustfs_client.get_object(cluster.rustfs_bucket, key) + try: + return resp.read() + finally: + resp.close() + resp.release_conn() + except S3Error: + return None + + +# `gc/state`'s wire format is a plain JSON-like text object (CasGcStateFormat.cpp), not a binary +# blob: the two fields this test needs are literally spelled `"sg":""` (snap_generation) +# and `"sa":""` (snap_attempt) in the object bytes, so a direct regex read is exact without +# needing the C++ decoder. This mirrors the production reader that resolves "the adopted seal" +# (Gc/CasOrphanManifestSweep.cpp): read gc/state, take (snap_generation, snap_attempt), then look up +# that exact fold seal -- the only two-hop lookup that names one authoritative adopted pair. +_SNAP_GENERATION_RE = re.compile(r'"sg":"(\d+)"') +_SNAP_ATTEMPT_RE = re.compile(r'"sa":"(\d+)"') + + +def read_adopted_generation_and_attempt(): + """ + Return the (generation, attempt) pair `gc/state` currently names as adopted, or None if + `gc/state` does not exist yet or its `snap_generation` is still the "nothing adopted yet" + sentinel (0, GcState's documented default). + """ + data = get_rustfs_object(GC_STATE_KEY) + if data is None: + return None + text = data.decode("utf-8", errors="replace") + sg_match = _SNAP_GENERATION_RE.search(text) + sa_match = _SNAP_ATTEMPT_RE.search(text) + if not sg_match or not sa_match: + return None + generation = int(sg_match.group(1)) + if generation == 0: + return None + return generation, int(sa_match.group(1)) + + +def adopted_fold_seal_key(generation, attempt): + """The one fold-seal key `gc/state` names as adopted for (generation, attempt).""" + return "{}/gc/gen/{}/attempt/{}/fold_seal".format(POOL_PREFIX, generation, attempt) + + +# Matches ".../gc/gen//attempt//blob_target//" -- the exact key +# shape `Layout::blobTargetRunKey` writes -- capturing only the shard id. Deliberately NOT scoped +# to one (generation, attempt): the objects a run key names stay physically where they were +# WRITTEN, and a seal only carries a REFERENCE to an earlier attempt's runs forward, so "did the +# sharded path ever write both shards over the whole soak" has to scan the whole gc/gen/ subtree. +_BLOB_TARGET_SHARD_RE = re.compile(r"/gc/gen/\d+/attempt/\d+/blob_target/(\d+)/") + + +def blob_target_shards_present(): + """Return (all blob_target keys found under gc/gen/, set of shard ids covered by them).""" + keys = list_rustfs_prefix(POOL_PREFIX + "/gc/gen/", recursive=True) + blob_target_keys = [] + shards = set() + for k in keys: + m = _BLOB_TARGET_SHARD_RE.search(k) + if m: + blob_target_keys.append(k) + shards.add(int(m.group(1))) + return blob_target_keys, shards + + +# --------------------------------------------------------------------------- +# Workload +# --------------------------------------------------------------------------- + +def run_blob_churn_workload(node, table_name, rounds=2): + """ + Insert rows + merge + drop in `rounds` cycles. Each cycle creates a fresh + `ReplicatedMergeTree`, inserts `NUM_INSERTS` batches of `NUM_ROWS_PER_INSERT` rows, + forces a merge, then drops the table. This generates blob churn: blobs are + referenced during the cycle, then become orphaned after the drop. + """ + for i in range(rounds): + full_name = "{}_{}".format(table_name, i) + node.query("DROP TABLE IF EXISTS {} SYNC".format(full_name)) + node.query( + "CREATE TABLE {name} (id Int64, v UInt64, s String) " + "ENGINE = ReplicatedMergeTree('/clickhouse/tables/{name}', '{{replica}}') " + "ORDER BY id " + "SETTINGS storage_policy = '{policy}'".format( + name=full_name, policy=STORAGE_POLICY + ) + ) + for j in range(NUM_INSERTS): + offset = (i * NUM_INSERTS + j) * NUM_ROWS_PER_INSERT + node.query( + "INSERT INTO {name} " + "SELECT number + {off}, number + {off}, toString(number + {off}) " + "FROM numbers({rows})".format( + name=full_name, off=offset, rows=NUM_ROWS_PER_INSERT + ) + ) + node.query("OPTIMIZE TABLE {} FINAL".format(full_name)) + node.query("DROP TABLE {} SYNC".format(full_name)) + + +# --------------------------------------------------------------------------- +# Main soak test +# --------------------------------------------------------------------------- + +def test_sharded_gc_soak(): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + + # Create a long-lived table to confirm row-count parity after the soak. + node1.query("DROP TABLE IF EXISTS live_table SYNC") + node2.query("DROP TABLE IF EXISTS live_table SYNC") + node1.query( + "CREATE TABLE live_table (id Int64, s String) " + "ENGINE = ReplicatedMergeTree('/clickhouse/tables/live_table', '{{replica}}') " + "ORDER BY id " + "SETTINGS storage_policy = '{}'".format(STORAGE_POLICY) + ) + node2.query( + "CREATE TABLE live_table (id Int64, s String) " + "ENGINE = ReplicatedMergeTree('/clickhouse/tables/live_table', '{{replica}}') " + "ORDER BY id " + "SETTINGS storage_policy = '{}'".format(STORAGE_POLICY) + ) + + # Insert into the live table so it has real content during the soak. + node1.query( + "INSERT INTO live_table " + "SELECT number, toString(number) FROM numbers({rows})".format( + rows=NUM_ROWS_PER_INSERT * NUM_INSERTS + ) + ) + node2.query("SYSTEM SYNC REPLICA live_table", timeout=60) + + live_count_before = int(node1.query("SELECT count() FROM live_table")) + assert live_count_before == NUM_ROWS_PER_INSERT * NUM_INSERTS + + # --- WORKLOAD: blob churn on node1 --- + run_blob_churn_workload(node1, "churn", rounds=2) + + # --- CHAOS: restart node2 --- + node2.restart_clickhouse(kill=True) + node2.query("SYSTEM SYNC REPLICA live_table", timeout=120) + + # --- QUIESCE: wait for gc/state to adopt a generation --- + adopted = None + for _ in range(GC_POLL_RETRIES): + adopted = read_adopted_generation_and_attempt() + if adopted: + break + time.sleep(GC_POLL_SLEEP) + + assert adopted, ( + "gc/state never adopted a generation (snap_generation stayed at the zero sentinel) within " + "{} s; gc_shards=2 soak cannot proceed (check server logs for GC errors)".format( + GC_POLL_RETRIES * GC_POLL_SLEEP + ) + ) + + # Allow one more GC interval for any in-progress round to finish, then re-resolve the adopted + # pointer (it may have advanced again). + time.sleep(5) + generation, attempt = read_adopted_generation_and_attempt() or adopted + + # ----------------------------------------------------------------------- + # ASSERTION A: no dangle / no loss + # ----------------------------------------------------------------------- + + # Both replicas must agree on the live row count. + count1 = int(node1.query("SELECT count() FROM live_table")) + count2 = int(node2.query("SELECT count() FROM live_table")) + assert count1 == live_count_before, ( + "node1 live_table count changed: before {} after {}".format( + live_count_before, count1 + ) + ) + assert count1 == count2, ( + "replica row-count divergence: node1={} node2={}".format(count1, count2) + ) + + # No CA-layer dangle errors in either server log. `system.text_log`'s underlying table can + # take a moment after node2's restart above to become queryable ("Unknown table" briefly), + # independent of whether any row has actually been flushed to it yet. Wait that out with a + # SEPARATE, keyword-free readiness probe first, THEN run the real keyword checks with a plain + # query (no retry): retrying the keyword query itself would be self-defeating -- each failed + # attempt logs its own `` line that echoes the failing query's text (which contains the + # search keyword as a LIKE pattern), and that line then lands in system.text_log itself, so a + # later successful attempt of the SAME keyword query would count its own failed predecessors + # as matches. + for inst_name, inst in [("node1", node1), ("node2", node2)]: + inst.query_with_retry("SELECT count() FROM system.text_log") + for kw in CA_FATAL_LOG_KEYWORDS: + log_count = inst.query( + "SELECT count() FROM system.text_log " + "WHERE level IN ('Error', 'Fatal') " + " AND message LIKE '%{}%'".format(kw) + ) + assert int(log_count) == 0, ( + "{} has CA fatal/error entries matching '{}' in system.text_log".format( + inst_name, kw + ) + ) + + # ----------------------------------------------------------------------- + # ASSERTION B: the adopted (generation, attempt) has a durable fold seal + # ----------------------------------------------------------------------- + + # gc/state names exactly one adopted (generation, attempt) pair; this is the ONLY seal this + # test looks at -- a non-adopted retry attempt's own seal (if any) is never named by gc/state + # and so cannot be mistaken for a second completion signal. + seal_key = adopted_fold_seal_key(generation, attempt) + assert get_rustfs_object(seal_key) is not None, ( + "gc/state adopted (generation={}, attempt={}) but its fold seal ('{}') does not exist -- " + "gc/state points at an attempt whose seal was never durably written".format( + generation, attempt, seal_key + ) + ) + + # ----------------------------------------------------------------------- + # ASSERTION C: disjoint-shard reduce progress + # ----------------------------------------------------------------------- + + # Over the WHOLE gc/gen/ subtree (every generation and attempt seen, not only the currently + # adopted one -- see the module docstring's point C for why), blob_target keys must cover + # BOTH shard 0 and shard 1. + blob_target_keys, shards_covered = blob_target_shards_present() + # Log the observed key set once: an assertion that passes on an empty listing (e.g. a further + # key-shape mismatch) would be a silent false pass, not evidence the sharded path ran. + print( + "test_sharded_gc_soak: blob_target keys under {}/gc/gen/: {}".format( + POOL_PREFIX, blob_target_keys + ) + ) + assert blob_target_keys, ( + "no blob_target keys found anywhere under '{}/gc/gen/'; before checking shard coverage " + "the listing itself must be non-empty".format(POOL_PREFIX) + ) + assert 0 in shards_covered and 1 in shards_covered, ( + "blob_target keys under '{}/gc/gen/' cover shards {}, expected BOTH 0 and 1; the sharded " + "gc_shards=2 fold path may not have executed. Keys observed: {}".format( + POOL_PREFIX, sorted(shards_covered), blob_target_keys + ) + ) + + # ----------------------------------------------------------------------- + # Cleanup + # ----------------------------------------------------------------------- + node1.query("DROP TABLE IF EXISTS live_table SYNC") + node2.query("DROP TABLE IF EXISTS live_table SYNC") diff --git a/tests/integration/test_cas_insert_fault_recovery/__init__.py b/tests/integration/test_cas_insert_fault_recovery/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_cas_insert_fault_recovery/configs/server_root_id_node1.xml b/tests/integration/test_cas_insert_fault_recovery/configs/server_root_id_node1.xml new file mode 100644 index 000000000000..dd63460deccc --- /dev/null +++ b/tests/integration/test_cas_insert_fault_recovery/configs/server_root_id_node1.xml @@ -0,0 +1,10 @@ + + + + + + node1 + + + + diff --git a/tests/integration/test_cas_insert_fault_recovery/configs/server_root_id_node2.xml b/tests/integration/test_cas_insert_fault_recovery/configs/server_root_id_node2.xml new file mode 100644 index 000000000000..e1be581a8c9f --- /dev/null +++ b/tests/integration/test_cas_insert_fault_recovery/configs/server_root_id_node2.xml @@ -0,0 +1,10 @@ + + + + + + node2 + + + + diff --git a/tests/integration/test_cas_insert_fault_recovery/configs/storage_conf.xml b/tests/integration/test_cas_insert_fault_recovery/configs/storage_conf.xml new file mode 100644 index 000000000000..0149d398aa1c --- /dev/null +++ b/tests/integration/test_cas_insert_fault_recovery/configs/storage_conf.xml @@ -0,0 +1,27 @@ + + + + + object_storage + s3 + cas + + http://rustfs1:11121/test/shared_pool/ + clickhouse + clickhouse + + + + + + +
+ disk_cas_shared +
+
+
+
+
+
diff --git a/tests/integration/test_cas_insert_fault_recovery/test.py b/tests/integration/test_cas_insert_fault_recovery/test.py new file mode 100644 index 000000000000..e1f3ff4ee23a --- /dev/null +++ b/tests/integration/test_cas_insert_fault_recovery/test.py @@ -0,0 +1,163 @@ +import time + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +# Two replicas of one ReplicatedMergeTree on a SHARED content-addressed pool. +STORAGE_POLICY = "cas_shared" + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + cluster.add_instance( + "node1", + main_configs=["configs/storage_conf.xml", "configs/server_root_id_node1.xml"], + macros={"replica": "node1"}, + with_rustfs=True, + with_zookeeper=True, + stay_alive=True, + ) + cluster.add_instance( + "node2", + main_configs=["configs/storage_conf.xml", "configs/server_root_id_node2.xml"], + macros={"replica": "node2"}, + with_rustfs=True, + with_zookeeper=True, + stay_alive=True, + ) + + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def _wait_until(predicate, timeout=180, interval=2, desc=""): + # Condition-based wait (systematic-debugging): the ordinary lost-part recovery is asynchronous + # (part-check retry/backoff), so gating on a fixed-timeout `SYSTEM SYNC REPLICA` is inherently flaky — + # that call blocks on the very recovery we are waiting for. Poll the actual OUTCOME instead, with a + # generous cap. Transient errors while node1 is mid-restart are swallowed and retried. + deadline = time.time() + timeout + last = None + while time.time() < deadline: + try: + last = predicate() + except Exception as e: # node briefly unavailable during restart, etc. + last = e + if last is True: + return + time.sleep(interval) + raise AssertionError("timed out after {}s waiting for: {} (last={!r})".format(timeout, desc, last)) + + +def test_post_multi_termination_uses_ordinary_lost_part_recovery(start_cluster): + # HISTORY: this test was authored (2026-07-16) against the OLD commit ordering, where the disk + # commit ran AFTER the Keeper multi — the failpoint then left a phantom ZK part entry and the + # assertion was "ordinary lost-part recovery runs (ReplicatedDataLoss bumps, empty cover)". + # One day later the R3 acked-data-loss fix (`77484196b0d`) deliberately REVERSED that order: + # `renameParts` closes the part's disk-storage transaction BEFORE the Keeper multi, so a part + # must be durable before its block_id/part znode is registered. Under the new ordering the + # failpoint (`disk_object_storage_fail_commit_metadata_transaction`, fired from inside + # `renameParts`) aborts the INSERT BEFORE anything reaches ZK — there is no phantom part, no + # lost part, and NOTHING to recover. The old predicate waited forever (600s timeouts on all + # three sanitizer CI lanes of PR#2073 and on a local release build). + # + # The test now asserts the NEW invariant, which is strictly stronger for the user: + # 1. the failed INSERT leaves NO trace: no ZK part entry, no replication-queue debris, + # count() stays 0 on both replicas after a node1 restart, and `ReplicatedDataLoss` does + # NOT bump (nothing was ever lost); + # 2. THE R3 GUARD: retrying the SAME insert (same bytes => same block_id) actually lands — + # a phantom block_id surviving the failed attempt would silently dedup the retry away + # (the acked-data-loss class the reordering exists to prevent); + # 3. no CA-specific wedge: no LOGICAL_ERROR in either server's log, queues drained. + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + + node1.query("DROP TABLE IF EXISTS t SYNC") + node2.query("DROP TABLE IF EXISTS t SYNC") + + create = ( + "CREATE TABLE t (a UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/t', '{{replica}}') " + "ORDER BY a SETTINGS storage_policy = '{policy}'" + ).format(policy=STORAGE_POLICY) + node1.query(create) + node2.query(create) + + def loss_count(): + return int( + node1.query( + "SELECT sum(value) FROM system.events WHERE event = 'ReplicatedDataLoss'" + ) + or 0 + ) + + loss_before = loss_count() + + # Force the disk commit to throw. Under the R3 ordering this fires inside `renameParts`, + # BEFORE the Keeper multi — the INSERT fails with nothing registered anywhere (ONCE failpoint). + node1.query("SYSTEM ENABLE FAILPOINT disk_object_storage_fail_commit_metadata_transaction") + node1.query_and_get_error("INSERT INTO t VALUES (1)") + + # No phantom state may exist even across a restart: ZK has no part entry, so startup's + # `checkPartsImpl` has nothing to reconcile and no recovery runs. + node1.restart_clickhouse() + + def node1_clean(): + # The failed INSERT left no trace: nothing to recover (ReplicatedDataLoss unchanged), + # no rows, no replication-queue debris. + cnt = node1.query("SELECT count() FROM t").strip() + queue = node1.query( + "SELECT count() FROM system.replication_queue WHERE table = 't'" + ).strip() + return loss_count() == loss_before and cnt == "0" and queue == "0" + + _wait_until( + node1_clean, + timeout=120, + desc="node1 restarts clean: no phantom part, no recovery triggered, queue empty", + ) + + # THE R3 GUARD (acked-data-loss class): retrying the SAME insert (same bytes => same block_id) + # must genuinely land. If the failed attempt had leaked its block_id into ZK, dedup would + # silently swallow this retry and count() would stay 0 — exactly the silent loss the + # renameParts-before-Keeper ordering exists to prevent. + node1.query("INSERT INTO t VALUES (1)") + + def retry_landed_everywhere(): + return ( + node1.query("SELECT count() FROM t").strip() == "1" + and node2.query("SELECT count() FROM t").strip() == "1" + ) + + _wait_until( + retry_landed_everywhere, + timeout=120, + desc="the retried identical INSERT lands and replicates (no phantom-block_id dedup)", + ) + + # The regression guard: no CA-specific exception / LOGICAL_ERROR left either server wedged. The + # expected `FILE_DOESNT_EXIST` interserver miss is tolerated (it is not a LOGICAL_ERROR). + for node in (node1, node2): + assert not node.contains_in_log( + "LOGICAL_ERROR" + ), "unexpected LOGICAL_ERROR in {}'s log — a CA-specific failure, not ordinary lost-part recovery".format( + node.name + ) + + # Server is healthy (no wedge): a fresh, different INSERT also succeeds end to end. + node1.query("INSERT INTO t VALUES (2)") + + def replicated_two_rows(): + return ( + node1.query("SELECT count() FROM t").strip() == "2" + and node2.query("SELECT count() FROM t").strip() == "2" + ) + + _wait_until(replicated_two_rows, timeout=120, desc="fresh INSERT replicates to both replicas") + + node1.query("DROP TABLE IF EXISTS t SYNC") + node2.query("DROP TABLE IF EXISTS t SYNC") diff --git a/tests/integration/test_cas_lazy_load_recovery/__init__.py b/tests/integration/test_cas_lazy_load_recovery/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_cas_lazy_load_recovery/configs/server_root_id_node1.xml b/tests/integration/test_cas_lazy_load_recovery/configs/server_root_id_node1.xml new file mode 100644 index 000000000000..dd63460deccc --- /dev/null +++ b/tests/integration/test_cas_lazy_load_recovery/configs/server_root_id_node1.xml @@ -0,0 +1,10 @@ + + + + + + node1 + + + + diff --git a/tests/integration/test_cas_lazy_load_recovery/configs/storage_conf.xml b/tests/integration/test_cas_lazy_load_recovery/configs/storage_conf.xml new file mode 100644 index 000000000000..0149d398aa1c --- /dev/null +++ b/tests/integration/test_cas_lazy_load_recovery/configs/storage_conf.xml @@ -0,0 +1,27 @@ + + + + + object_storage + s3 + cas + + http://rustfs1:11121/test/shared_pool/ + clickhouse + clickhouse + + + + + + +
+ disk_cas_shared +
+
+
+
+
+
diff --git a/tests/integration/test_cas_lazy_load_recovery/test.py b/tests/integration/test_cas_lazy_load_recovery/test.py new file mode 100644 index 000000000000..3eeae3debc7d --- /dev/null +++ b/tests/integration/test_cas_lazy_load_recovery/test.py @@ -0,0 +1,88 @@ +import time + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +STORAGE_POLICY = "cas_shared" + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + cluster.add_instance( + "node1", + main_configs=["configs/storage_conf.xml", "configs/server_root_id_node1.xml"], + macros={"replica": "node1"}, + with_rustfs=True, + with_zookeeper=True, + stay_alive=True, + ) + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def _create(node): + # lazy_load_tables=1: the CAS table attaches as a proxy and its real storage is built on first + # access. A transient object-store outage during that build is ridden out / retried on a later + # access instead of being cached as a permanently-FAILED AsyncLoader job (which, for a non-lazy + # database, would strand the table until a full server restart). + node.query("CREATE DATABASE IF NOT EXISTS lazy_db ENGINE = Atomic SETTINGS lazy_load_tables = 1") + node.query( + "CREATE TABLE IF NOT EXISTS lazy_db.t (k UInt64, v UInt64) " + "ENGINE = ReplicatedMergeTree('/clickhouse/tables/lazy_t', '{replica}') " + "ORDER BY k SETTINGS storage_policy = '%s', min_bytes_for_wide_part = 0" % STORAGE_POLICY + ) + + +def test_lazy_cas_table_self_heals_after_s3_recovery(start_cluster): + node = cluster.instances["node1"] + _create(node) + node.query("INSERT INTO lazy_db.t SELECT number, number FROM numbers(100)") + assert node.query("SELECT count() FROM lazy_db.t").strip() == "100" + + # Restart so the table re-attaches as a lazy proxy (its real storage is not yet constructed; the + # disk mounts at startup while S3 is up, the storage is built only on first access below). + node.restart_clickhouse() + + # Touch the table while S3 is unreachable: the lazy first-access build (its CAS ref-recovery LIST + # over the object store) cannot complete, so the client query fails within its bounded timeout. + # Note: the build does NOT fail fast server-side -- it blocks on the object store's own retry until + # S3 returns (see the BACKLOG "block-until-recovered" note); the client-side timeout is what makes + # this probe short. We assert the probe DID hit the outage (raised): the build needs several object- + # store round-trips, so the freezer (effective within milliseconds of `pause_container` returning) + # reliably catches it -- if this ever flakes, the pause raced a sub-millisecond full build, not a + # real self-heal regression. + with cluster.pause_container("rustfs1", wait_for_paused=False): + probe_raised = False + try: + node.query("SELECT count() FROM lazy_db.t", timeout=30) + except Exception: + probe_raised = True # expected while the object store is unreachable + assert probe_raised, "the probe should have failed while S3 was unreachable (did the pause race the build?)" + + # S3 is back (context exit unpaused rustfs). WITHOUT a server restart and WITHOUT any DETACH, a + # later access must make the table usable again. This proves the key Layer 2 property: a transient + # object-store outage during a lazy CAS table's first-access build leaves NO permanently-cached + # AsyncLoader FAILED state (a non-lazy table whose load failed would stay FAILED until a full server + # restart). What actually recovers here is the original in-flight build completing once S3 returns + # (the block-until-recovered path), which is sufficient for "usable again without restart"; this + # test does not (and, given block-until-recovered, cannot) assert a proxy retry of a THROWN build. + deadline = time.time() + 180 + last = None + while time.time() < deadline: + try: + last = node.query("SELECT count() FROM lazy_db.t").strip() + except Exception as e: + last = "err: " + str(e) + if last == "100": + break + time.sleep(3) + assert last == "100", ( + "lazy CAS table must become usable again on a later access after S3 returns, with no server " + "restart (last=%r)" % last + ) diff --git a/tests/integration/test_cas_ref_snaplog/__init__.py b/tests/integration/test_cas_ref_snaplog/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_cas_ref_snaplog/configs/storage_conf.xml b/tests/integration/test_cas_ref_snaplog/configs/storage_conf.xml new file mode 100644 index 000000000000..cf842fcd4f6c --- /dev/null +++ b/tests/integration/test_cas_ref_snaplog/configs/storage_conf.xml @@ -0,0 +1,43 @@ + + + + + + object_storage + s3 + cas + itest-ref-snaplog + http://rustfs1:11121/test/cas_snaplog_data/ + clickhouse + clickhouse + 1 + 1 + + + + object_storage + s3 + cas + itest-ref-snaplog + http://rustfs1:11121/test/cas_snaplog_data/ + clickhouse + clickhouse + true + 0 + + + + + +
+ disk_ca +
+
+
+
+
+
diff --git a/tests/integration/test_cas_ref_snaplog/test.py b/tests/integration/test_cas_ref_snaplog/test.py new file mode 100644 index 000000000000..0a9628673a26 --- /dev/null +++ b/tests/integration/test_cas_ref_snaplog/test.py @@ -0,0 +1,170 @@ +import shlex +import time + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +STORAGE_POLICY = "ref_snaplog" +RO_DISK = "disk_ca_ro" + +# Endpoint is http://rustfs1:11121/test/cas_snaplog_data/, so the pool lives under bucket `test`, +# prefix `cas_snaplog_data/`. The snapshot+log ref protocol keeps a table's immutable transaction logs +# and snapshots under cas/ns/stream/, part manifests under cas/manifests/, and content blobs under blobs/. +POOL = "cas_snaplog_data" +BLOBS_PREFIX = POOL + "/blobs/" +REFS_PREFIX = POOL + "/cas/ns/stream/" +MANIFESTS_PREFIX = POOL + "/cas/manifests/" + +NUM_ROWS = 20000 +NUM_INSERTS = 8 + +# Background GC runs every 1s with a 2s grace. After DROP TABLE ... SYNC the dropped namespace's content +# (blobs) and part manifests become GC fodder; we poll until they drain. Bounded wait on a known +# background process, not a race hack. +RECLAIM_RETRIES = 120 +RECLAIM_SLEEP = 1.0 # total bound ~= 120s + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + cluster.add_instance( + "node", + main_configs=["configs/storage_conf.xml"], + with_rustfs=True, + stay_alive=True, + ) + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def _count(prefix): + return len( + list( + cluster.rustfs_client.list_objects( + cluster.rustfs_bucket, prefix, recursive=True + ) + ) + ) + + +def _content_objects(): + # Content blobs + part manifests: the objects the GC fold + condemn/delete pipeline and the + # namespace-cleanup item reclaim after a namespace is removed. + return _count(BLOBS_PREFIX) + _count(MANIFESTS_PREFIX) + + +def _disks(node, query): + # Run a clickhouse-disks command against the read-only CA window over the same pool. + return node.exec_in_container( + [ + "bash", + "-c", + "/usr/bin/clickhouse disks -C /etc/clickhouse-server/config.xml " + "--disk {} --save-logs --query {}".format(RO_DISK, shlex.quote(query)), + ] + ) + + +def test_ref_snaplog_lifecycle_reclaims_and_fsck_clean(): + node = cluster.instances["node"] + + for t in ("ref_t1", "ref_t1_renamed", "ref_t2"): + node.query("DROP TABLE IF EXISTS {} SYNC".format(t)) + + content_baseline = _content_objects() + + # (1) Two tables on the CA/rustfs policy. + for t in ("ref_t1", "ref_t2"): + node.query( + "CREATE TABLE {} (id Int64, data String) ENGINE = MergeTree() ORDER BY id " + "SETTINGS storage_policy = '{}'".format(t, STORAGE_POLICY) + ) + + # (2) Several inserts each -> distinct content blobs + one immutable ref-log transaction per insert. + for t in ("ref_t1", "ref_t2"): + for i in range(NUM_INSERTS): + node.query( + "INSERT INTO {} SELECT number + {off}, toString(number + {off}) " + "FROM numbers({rows})".format(t, off=i * NUM_ROWS, rows=NUM_ROWS) + ) + + assert int(node.query("SELECT count() FROM ref_t1")) == NUM_INSERTS * NUM_ROWS + assert int(node.query("SELECT count() FROM ref_t2")) == NUM_INSERTS * NUM_ROWS + + # The snapshot+log ref format is actually in use: immutable ref objects exist under cas/ns/stream/. + assert _count(REFS_PREFIX) > 0, "expected ref log/snapshot objects under cas/ns/stream/" + assert ( + _content_objects() > content_baseline + ), "expected content objects to rise above baseline after inserts" + + # Read-only fsck agrees while data is present: no authoritative ref names a missing object. + live_fsck = _disks(node, "cas-fsck") + assert "dangling=0" in live_fsck, live_fsck + + # (3) Rename one table: data must survive (its ref namespace and its logs/snapshots are unaffected). + node.query("RENAME TABLE ref_t1 TO ref_t1_renamed") + assert ( + int(node.query("SELECT count() FROM ref_t1_renamed")) == NUM_INSERTS * NUM_ROWS + ) + + # (4) Drop both: the writer appends remove_namespace; background GC folds the -1 edges, condemns and + # deletes the now-unreferenced blobs, and runs the namespace-cleanup item that reclaims the + # removed namespace's physical @cas@ prefixes (part manifests + verbatim files). + node.query("DROP TABLE ref_t1_renamed SYNC") + node.query("DROP TABLE ref_t2 SYNC") + # The content count at the moment the refs are unlinked, so the reclamation below is measured + # against what was actually there to reclaim. + after_drop_content = _content_objects() + + # (5) THE RECLAMATION: the pool's CONTENT (blobs + part manifests) drains back to baseline. Polled + # with an early exit, then cross-checked against GC's own bookkeeping — a pool that shrank for + # some other reason must not pass for a round that reclaimed it. + final = _content_objects() + for _ in range(RECLAIM_RETRIES): + if final <= content_baseline: + break + time.sleep(RECLAIM_SLEEP) + final = _content_objects() + + assert final <= content_baseline, ( + "the dropped namespaces' content was not reclaimed: baseline={}, " + "at_drop={}, final={} (blobs={}, manifests={})".format( + content_baseline, + after_drop_content, + final, + _count(BLOBS_PREFIX), + _count(MANIFESTS_PREFIX), + ) + ) + + node.query("SYSTEM FLUSH LOGS") + rounds = int( + node.query( + "SELECT count() FROM system.cas_gc_log " + "WHERE event_type = 'Finish' AND outcome = 'Success'" + ).strip() + ) + assert rounds > 0, "no successful GC round ran at all" + + deleted = int( + node.query( + "SELECT sum(objects_deleted + manifests_deleted) " + "FROM system.cas_gc_log WHERE event_type = 'Finish'" + ).strip() + or 0 + ) + assert deleted > 0, "the pool's content drained but GC's own bookkeeping reports no deletion" + + # (6) Read-only consumers on the DRAINED pool. + final_fsck = _disks(node, "cas-fsck") + assert "dangling=0" in final_fsck, final_fsck + + # `cas-gc-dryrun` on a fully drained pool has nothing left to preview. + dryrun = _disks(node, "cas-gc-dryrun") + assert "preview_deletes=0" in dryrun, dryrun diff --git a/tests/integration/test_cas_replicated_relink/__init__.py b/tests/integration/test_cas_replicated_relink/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_cas_replicated_relink/configs/server_root_id_node1.xml b/tests/integration/test_cas_replicated_relink/configs/server_root_id_node1.xml new file mode 100644 index 000000000000..be23e47852bb --- /dev/null +++ b/tests/integration/test_cas_replicated_relink/configs/server_root_id_node1.xml @@ -0,0 +1,12 @@ + + + + + + node1 + + + + diff --git a/tests/integration/test_cas_replicated_relink/configs/server_root_id_node2.xml b/tests/integration/test_cas_replicated_relink/configs/server_root_id_node2.xml new file mode 100644 index 000000000000..e3b468aaa4e4 --- /dev/null +++ b/tests/integration/test_cas_replicated_relink/configs/server_root_id_node2.xml @@ -0,0 +1,12 @@ + + + + + + node2 + + + + diff --git a/tests/integration/test_cas_replicated_relink/configs/storage_conf.xml b/tests/integration/test_cas_replicated_relink/configs/storage_conf.xml new file mode 100644 index 000000000000..0ce475f4bc80 --- /dev/null +++ b/tests/integration/test_cas_replicated_relink/configs/storage_conf.xml @@ -0,0 +1,36 @@ + + + + + object_storage + s3 + cas + + http://rustfs1:11121/test/shared_pool/ + clickhouse + clickhouse + + + 1 + 1 + + + + + +
+ disk_cas_shared +
+
+
+
+
+
diff --git a/tests/integration/test_cas_replicated_relink/configs/storage_conf_other_pool.xml b/tests/integration/test_cas_replicated_relink/configs/storage_conf_other_pool.xml new file mode 100644 index 000000000000..416cbfca86ae --- /dev/null +++ b/tests/integration/test_cas_replicated_relink/configs/storage_conf_other_pool.xml @@ -0,0 +1,31 @@ + + + + + + object_storage + s3 + cas + http://rustfs1:11121/test/other_pool/ + clickhouse + clickhouse + node2_other + 0 + + + + + +
+ disk_cas_other +
+
+
+
+
+
diff --git a/tests/integration/test_cas_replicated_relink/test.py b/tests/integration/test_cas_replicated_relink/test.py new file mode 100644 index 000000000000..83d8d239ba92 --- /dev/null +++ b/tests/integration/test_cas_replicated_relink/test.py @@ -0,0 +1,905 @@ +import re +import shlex +import time + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +# Both replicas mount the SAME content-addressed pool (endpoint .../root/shared_pool/). A +# ReplicatedMergeTree part written on one replica is therefore ALREADY present (as content blobs + +# manifest) in the pool when the other replica needs it — so the "fetch" is a fetch-by-relink: the +# fetching replica publishes its own ref to the existing blobs instead of downloading any bytes (the CA +# analogue of zero-copy replication, spec §4). +STORAGE_POLICY = "cas_shared" +CA_DISK = "disk_cas_shared" + +# A second, independent pool mounted by node2 only (configs/storage_conf_other_pool.xml). Used for the +# cross-pool leg of B66b: relink is gated on both sides naming the same pool, so a fetch into this one +# must degrade to bytes. +OTHER_STORAGE_POLICY = "cas_other" +OTHER_CA_DISK = "disk_cas_other" + +# The shared pool's blob prefix inside the `test` RustFS bucket. The relink proof is that the fetch does +# NOT create new objects under here: relink publishes a ref (per-server, under store/), never a blob. +BLOBS_PREFIX = "shared_pool/blobs/" + +NUM_ROWS = 10000 + +# ---------------------------------------------------------------------------------------------------- +# WHY EVERY RELINK ASSERTION BELOW IS A POSITIVE ONE +# +# "The fetch created no new blobs" is NOT by itself evidence that a relink happened. On a +# content-addressed disk a BYTE fetch writes the very same content, which deduplicates against the +# blobs already in the pool, so its blob-count delta is zero too. A test that only counts blobs is +# therefore green whether the protocol worked or silently fell back — the single easiest worthless test +# on this path. +# +# So each relink test asserts a signal that is reachable ONLY through the intended path: +# +# RELINK RAN -> the receiver's `Relink of part

onto disk finished (no bytes transferred).` +# That line is the last statement of `Fetcher::relinkPartToDisk` and is reachable only +# after the confirm answered `yes` AND `promote()` returned `Committed` (taxonomy +# row 4). Every other row returns or throws before it. +# BYTES RAN -> the receiver's `Download of part

onto disk finished.` from +# `downloadPartToDisk`, plus the specific line naming WHY relink was declined. +# +# The blob-count / `CASBlobPut == 0` checks are kept as corroboration, never as the proof. +# ---------------------------------------------------------------------------------------------------- + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + cluster.add_instance( + "node1", + main_configs=["configs/storage_conf.xml", "configs/server_root_id_node1.xml"], + macros={"replica": "node1"}, + with_rustfs=True, + with_zookeeper=True, + stay_alive=True, + ) + cluster.add_instance( + "node2", + main_configs=[ + "configs/storage_conf.xml", + "configs/server_root_id_node2.xml", + "configs/storage_conf_other_pool.xml", + ], + macros={"replica": "node2"}, + with_rustfs=True, + with_zookeeper=True, + stay_alive=True, + ) + + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def blob_keys(): + """Every object key under the shared pool's blob prefix, as a set.""" + objects = cluster.rustfs_client.list_objects( + cluster.rustfs_bucket, BLOBS_PREFIX, recursive=True + ) + return {obj.object_name for obj in objects} + + +def count_blobs(): + return len(blob_keys()) + + +def log_lines(node, pattern): + """Server-log lines matching an extended regular expression. + + Deliberately NOT `instance.grep_in_log`: that one globs `clickhouse-server.log*`, which includes + `clickhouse-server.err.log`, so any warning-or-above line is counted twice. Several assertions here + are exact counts, and a doubled count is indistinguishable from a real second attempt. + """ + out = node.exec_in_container( + [ + "bash", + "-c", + "grep -a -E {} /var/log/clickhouse-server/clickhouse-server.log || true".format( + shlex.quote(pattern) + ), + ] + ) + return [line for line in out.splitlines() if line.strip()] + + +def wait_for_log_lines(node, pattern, timeout=60): + """Poll until at least one line matches, then return the matches. Fails loudly on timeout.""" + deadline = time.time() + timeout + while True: + found = log_lines(node, pattern) + if found: + return found + assert time.time() < deadline, "timed out waiting for log lines matching {!r} on {}".format( + pattern, node.name + ) + time.sleep(0.5) + + +def relink_finished_pattern(table, part, disk=CA_DISK): + """The receiver-side proof that the publish→confirm→promote path completed for this exact part.""" + return r"default\.{} .*Relink of part {} onto disk {} finished \(no bytes transferred\)".format( + table, re.escape(part), disk + ) + + +def download_finished_pattern(table, part, disk=CA_DISK): + """The receiver-side proof that the BYTE path completed for this exact part.""" + return r"default\.{} .*Download of part {} onto disk {} finished".format( + table, re.escape(part), disk + ) + + +def relink_offer_pattern(table, part): + """The SENDER-side line, one per relink offer actually made. The attempt counter.""" + return r"default\.{} .*Sending part {} by relink".format(table, re.escape(part)) + + +def assert_relinked(node, table, part, disk=CA_DISK, timeout=60): + wait_for_log_lines(node, relink_finished_pattern(table, part, disk), timeout=timeout) + assert not log_lines(node, download_finished_pattern(table, part, disk)), ( + "part {} of {} was relinked AND byte-downloaded on {} — the relink proof is not exclusive".format( + part, table, node.name + ) + ) + + +def assert_byte_downloaded(node, table, part, disk=CA_DISK, timeout=60): + wait_for_log_lines(node, download_finished_pattern(table, part, disk), timeout=timeout) + assert not log_lines(node, relink_finished_pattern(table, part, disk)), ( + "part {} of {} was expected to arrive as bytes but a relink completed on {}".format( + part, table, node.name + ) + ) + + +def assert_no_new_blobs(before_keys): + """Corroboration for a relink: the fetch added no object under the pool's blob prefix. + + Phrased as "no NEW key" rather than "the same count" on purpose — background GC may reclaim + unrelated debris at any moment on this fixture (`gc_interval_sec` is 1), and a count that went DOWN + says nothing about whether the fetch wrote anything. + """ + new_keys = sorted(blob_keys() - before_keys) + assert not new_keys, "the fetch wrote {} new blob(s), e.g. {}".format(len(new_keys), new_keys[:5]) + + +def cas_blob_puts(node): + return int(node.query("SELECT sum(value) FROM system.events WHERE event = 'CASBlobPut'") or 0) + + +def active_part_names(node, table): + return node.query( + "SELECT name FROM system.parts WHERE database = 'default' AND table = '{}' AND active " + "ORDER BY name".format(table) + ).split() + + +def any_state_part_count(node, table, part): + return int( + node.query( + "SELECT count() FROM system.parts WHERE database = 'default' AND table = '{}' " + "AND name = '{}'".format(table, part) + ) + ) + + +def wait_until(predicate, timeout, what): + deadline = time.time() + timeout + while True: + if predicate(): + return + assert time.time() < deadline, "timed out waiting for {}".format(what) + time.sleep(0.5) + + +def fsck(node, disk=CA_DISK): + """`SYSTEM CAS FSCK ` as a dict of column -> value. + + Driven through `clickhouse-client --format` rather than a trailing `FORMAT` clause: `ASTSystemQuery` + is not an `ASTQueryWithOutput`, so `SYSTEM ... FORMAT TSVWithNames` is a syntax error. Reading the + header is what keeps this from depending on the column ORDER of the summary. + """ + out = node.exec_in_container( + [ + "bash", + "-c", + "clickhouse client --format TSVWithNames --query {}".format( + shlex.quote("SYSTEM CAS FSCK '{}'".format(disk)) + ), + ] + ).splitlines() + header, row = out[0].split("\t"), out[1].split("\t") + summary = dict(zip(header, row)) + assert "dangling" in summary, "unexpected FSCK summary shape: {}".format(out) + return summary + + +def gc_round(node, disk=CA_DISK): + node.query("SYSTEM CAS GC RUN '{}'".format(disk)) + + +def drop_everywhere(table): + for node in (cluster.instances["node1"], cluster.instances["node2"]): + node.query("DROP TABLE IF EXISTS {} SYNC".format(table)) + + +def create_replicated(node, table, policy=STORAGE_POLICY, zk_path=None, extra_settings=""): + node.query( + "CREATE TABLE {table} (id Int64, v UInt64, s String) " + "ENGINE = ReplicatedMergeTree('{zk}', '{{replica}}') ORDER BY id " + "SETTINGS storage_policy = '{policy}'{extra}".format( + table=table, + zk=zk_path or "/clickhouse/tables/" + table, + policy=policy, + extra=(", " + extra_settings) if extra_settings else "", + ) + ) + + +def insert_rows(node, table, start, rows=NUM_ROWS): + node.query( + "INSERT INTO {table} SELECT number, number * 10, toString(number) " + "FROM numbers({start}, {rows})".format(table=table, start=start, rows=rows) + ) + + +def test_replicated_fetch_by_relink(): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + + node1.query("DROP TABLE IF EXISTS r SYNC") + node2.query("DROP TABLE IF EXISTS r SYNC") + + # Two replicas of ONE ReplicatedMergeTree table on the shared CA pool. Lifting B33 is what makes this + # CREATE succeed at all; the shared-pool mount is what makes the second replica start. + create_tpl = ( + "CREATE TABLE r (id Int64, v UInt64, s String) " + "ENGINE = ReplicatedMergeTree('/clickhouse/tables/r', '{{replica}}') " + "ORDER BY id SETTINGS storage_policy = '{policy}'" + ) + node1.query(create_tpl.format(policy=STORAGE_POLICY)) + node2.query(create_tpl.format(policy=STORAGE_POLICY)) + + # (1) INSERT on replica node1. node2 must replicate the part. + node1.query( + "INSERT INTO r SELECT number, number * 10, toString(number) FROM numbers({rows})".format( + rows=NUM_ROWS + ) + ) + + # Blob count after the insert, BEFORE node2 fetches. This is the relink baseline. + blobs_after_insert = count_blobs() + assert blobs_after_insert > 0, "insert must have written content blobs to the shared pool" + + # (2) node2 fetches the part. SYNC REPLICA blocks until the queue (the fetch) drains. + node2.query("SYSTEM SYNC REPLICA r", timeout=60) + + # (3) node2 reads the SAME rows back. + expected_sum_id = (NUM_ROWS - 1) * NUM_ROWS // 2 + assert int(node2.query("SELECT count() FROM r")) == NUM_ROWS + assert int(node2.query("SELECT sum(id) FROM r")) == expected_sum_id + assert int(node2.query("SELECT sum(v) FROM r")) == expected_sum_id * 10 + + # (4) THE RELINK PROOF: the fetch created NO new blob objects. node2 published a ref to the blobs + # node1 already wrote — it did not download/re-write them. (Relink, not byte download.) + blobs_after_fetch = count_blobs() + assert blobs_after_fetch == blobs_after_insert, ( + "fetch-by-relink must not create new blob objects: had {} after insert, {} after node2 fetched " + "(a byte download would have re-written blobs)".format( + blobs_after_insert, blobs_after_fetch + ) + ) + + # (5) A merge on node1 fetched-by-relink by node2: insert a second part on node1, OPTIMIZE to merge, + # and confirm node2 picks up the merged part with still no new blobs beyond the merge's own. + node1.query( + "INSERT INTO r SELECT number, number * 10, toString(number) FROM numbers({a}, {rows})".format( + a=NUM_ROWS, rows=NUM_ROWS + ) + ) + node2.query("SYSTEM SYNC REPLICA r", timeout=60) + blobs_before_merge = count_blobs() + + node1.query("OPTIMIZE TABLE r FINAL") + node1.query("SYSTEM SYNC REPLICA r", timeout=60) + blobs_after_merge_on_node1 = count_blobs() + + # node2 fetches the merged part. The merge itself may write new blobs on node1 (the merged content), + # but node2's FETCH of that merged part must add NOTHING further (relink). + node2.query("SYSTEM SYNC REPLICA r", timeout=60) + blobs_after_merge_fetch = count_blobs() + assert blobs_after_merge_fetch == blobs_after_merge_on_node1, ( + "fetch-by-relink of the merged part must not create new blobs: {} after node1 merged, {} after " + "node2 fetched".format(blobs_after_merge_on_node1, blobs_after_merge_fetch) + ) + + assert int(node2.query("SELECT count() FROM r")) == 2 * NUM_ROWS + assert int(node1.query("SELECT count() FROM r")) == 2 * NUM_ROWS + + node1.query("DROP TABLE IF EXISTS r SYNC") + node2.query("DROP TABLE IF EXISTS r SYNC") + + +def test_relink_happy_path_proof(): + """Task 16 step 2 — the happy path, proved POSITIVELY. + + Taxonomy row 4 (confirm `yes` -> `promote` -> `Committed`). The proof is the receiver's + `... finished (no bytes transferred)` line, which no other row can reach; `CASBlobPut == 0` and the + flat blob count are corroboration only (see the note at the top of this file). + """ + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + table = "relink_happy" + drop_everywhere(table) + + create_replicated(node1, table) + create_replicated(node2, table) + + node2.query("SYSTEM STOP FETCHES {}".format(table)) + insert_rows(node1, table, 0) + part = active_part_names(node1, table)[0] + + blobs_before = blob_keys() + puts_before = cas_blob_puts(node2) + + node2.query("SYSTEM START FETCHES {}".format(table)) + node2.query("SYSTEM SYNC REPLICA {}".format(table), timeout=60) + + # THE PROOF: reachable only after a confirm `yes` and a committed promote. + assert_relinked(node2, table, part) + + # Corroboration, in the plan's own terms: the receiver issued no blob PUT at all, and the pool's + # blob set is byte-identical to what the sender's insert left behind. + assert cas_blob_puts(node2) == puts_before + assert_no_new_blobs(blobs_before) + + assert int(node2.query("SELECT count() FROM {}".format(table))) == NUM_ROWS + assert int(node2.query("SELECT sum(v) FROM {}".format(table))) == int( + node1.query("SELECT sum(v) FROM {}".format(table)) + ) + + drop_everywhere(table) + + +def test_fetch_part_into_detached_relinks(): + """Task 16 step 5 — B66b, manual caller #1: `ALTER TABLE ... FETCH PART ... FROM`. + + Taxonomy row 4 with `to_detached=true`: the staged ref is `detached/tmp-fetch_` and the + finalization is `renameTo(detached/)`. Before B66b the relink capability was gated on + `!to_detached`, so this fetch could only ever be bytes. + """ + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + src, dst = "b66b_part_src", "b66b_part_dst" + drop_everywhere(src) + drop_everywhere(dst) + + create_replicated(node1, src) + create_replicated(node2, dst) + insert_rows(node1, src, 0) + part = active_part_names(node1, src)[0] + + blobs_before = blob_keys() + puts_before = cas_blob_puts(node2) + + node2.query( + "ALTER TABLE {dst} FETCH PART '{part}' FROM '/clickhouse/tables/{src}'".format( + dst=dst, part=part, src=src + ) + ) + + assert_relinked(node2, dst, part) + assert cas_blob_puts(node2) == puts_before + assert_no_new_blobs(blobs_before) + + # ... and the detached part is a real, readable part once attached. + node2.query("ALTER TABLE {} ATTACH PART '{}'".format(dst, part)) + assert int(node2.query("SELECT count() FROM {}".format(dst))) == NUM_ROWS + assert int(node2.query("SELECT sum(v) FROM {}".format(dst))) == int( + node1.query("SELECT sum(v) FROM {}".format(src)) + ) + + drop_everywhere(src) + drop_everywhere(dst) + + +def test_fetch_partition_into_detached_relinks(): + """Task 16 step 5 — B66b, manual caller #2: `ALTER TABLE ... FETCH PARTITION ... FROM`. + + Same taxonomy row as the FETCH PART leg; a separate test because it is a separate call site (it + fetches a whole partition through its own thread pool) and Task 15 changed both. + """ + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + src, dst = "b66b_partition_src", "b66b_partition_dst" + drop_everywhere(src) + drop_everywhere(dst) + + create_replicated(node1, src) + create_replicated(node2, dst) + insert_rows(node1, src, 0) + part = active_part_names(node1, src)[0] + + blobs_before = blob_keys() + puts_before = cas_blob_puts(node2) + + node2.query( + "ALTER TABLE {dst} FETCH PARTITION ID 'all' FROM '/clickhouse/tables/{src}'".format( + dst=dst, src=src + ) + ) + + assert_relinked(node2, dst, part) + assert cas_blob_puts(node2) == puts_before + assert_no_new_blobs(blobs_before) + + node2.query("ALTER TABLE {} ATTACH PARTITION ID 'all'".format(dst)) + assert int(node2.query("SELECT count() FROM {}".format(dst))) == NUM_ROWS + + drop_everywhere(src) + drop_everywhere(dst) + + +def test_detached_fetch_cross_pool_falls_back_to_bytes(): + """Task 16 step 5 — the cross-pool leg: relink is gated on ONE pool, so this must be bytes. + + Not a taxonomy row at all: the sender's pre-filter (`receiver_pool_uuid == getPoolUUID()`) declines + to make an offer, so the receiver never enters `relinkPartToDisk`. The positive signal is therefore + the byte path's own completion line plus the ABSENCE of any relink offer for this part. + """ + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + src, dst = "xpool_src", "xpool_dst" + drop_everywhere(src) + drop_everywhere(dst) + + create_replicated(node1, src) + create_replicated(node2, dst, policy=OTHER_STORAGE_POLICY) + insert_rows(node1, src, 0) + part = active_part_names(node1, src)[0] + + node2.query( + "ALTER TABLE {dst} FETCH PART '{part}' FROM '/clickhouse/tables/{src}'".format( + dst=dst, part=part, src=src + ) + ) + + # The bytes really moved: the receiver ran `downloadPartToDisk` onto the OTHER pool's disk. + assert_byte_downloaded(node2, dst, part, disk=OTHER_CA_DISK) + # ... and the sender never offered a relink for it, which is what makes the byte path the *intended* + # outcome here rather than an accident of some later failure. + assert not log_lines(node1, relink_offer_pattern(src, part)) + + node2.query("ALTER TABLE {} ATTACH PART '{}'".format(dst, part)) + assert int(node2.query("SELECT count() FROM {}".format(dst))) == NUM_ROWS + assert int(node2.query("SELECT sum(v) FROM {}".format(dst))) == int( + node1.query("SELECT sum(v) FROM {}".format(src)) + ) + + drop_everywhere(src) + drop_everywhere(dst) + + +def test_attach_partition_from_relinks_on_queue_fetch(): + """Task 16 step 6 (RPL-5) — `ATTACH PARTITION ... FROM` replicates as `REPLACE_RANGE`. + + The source table exists only on node1, so node2 cannot clone locally and its queue entry falls + through to `executeReplaceRange`'s `fetchSelectedPart` — a THIRD fetch call site, with its own + `tmp_replace_from_fetch_` prefix. Taxonomy row 4; the proof is the same relink-finished line. + """ + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + src, dst = "rpl5_attach_src", "rpl5_attach_dst" + drop_everywhere(src) + drop_everywhere(dst) + + node1.query( + "CREATE TABLE {src} (id Int64, v UInt64, s String) ENGINE = MergeTree ORDER BY id " + "SETTINGS storage_policy = '{policy}'".format(src=src, policy=STORAGE_POLICY) + ) + create_replicated(node1, dst) + create_replicated(node2, dst) + insert_rows(node1, src, 0) + + node1.query("ALTER TABLE {dst} ATTACH PARTITION tuple() FROM {src}".format(dst=dst, src=src)) + part = active_part_names(node1, dst)[0] + + blobs_before = blob_keys() + puts_before = cas_blob_puts(node2) + + node2.query("SYSTEM SYNC REPLICA {}".format(dst), timeout=90) + + assert_relinked(node2, dst, part) + assert cas_blob_puts(node2) == puts_before + assert_no_new_blobs(blobs_before) + assert int(node2.query("SELECT count() FROM {}".format(dst))) == NUM_ROWS + + node1.query("DROP TABLE IF EXISTS {} SYNC".format(src)) + drop_everywhere(dst) + + +def test_replace_partition_relinks_on_queue_fetch(): + """Task 16 step 6 (RPL-5) — `REPLACE PARTITION`, i.e. the same entry with a drop range attached. + + Separate from the ATTACH leg because the destination is non-empty: node2 must drop its own covering + part AND fetch the replacement, so the relink runs against a partition that already had a ref. + """ + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + src, dst = "rpl5_replace_src", "rpl5_replace_dst" + drop_everywhere(src) + drop_everywhere(dst) + + node1.query( + "CREATE TABLE {src} (id Int64, v UInt64, s String) ENGINE = MergeTree ORDER BY id " + "SETTINGS storage_policy = '{policy}'".format(src=src, policy=STORAGE_POLICY) + ) + create_replicated(node1, dst) + create_replicated(node2, dst) + + # Destination starts non-empty and replicated, so REPLACE really replaces something. + insert_rows(node1, dst, 0) + node2.query("SYSTEM SYNC REPLICA {}".format(dst), timeout=60) + + insert_rows(node1, src, 5 * NUM_ROWS) + node1.query("ALTER TABLE {dst} REPLACE PARTITION tuple() FROM {src}".format(dst=dst, src=src)) + part = active_part_names(node1, dst)[0] + + node2.query("SYSTEM SYNC REPLICA {}".format(dst), timeout=90) + + assert_relinked(node2, dst, part) + assert int(node2.query("SELECT count() FROM {}".format(dst))) == NUM_ROWS + assert int(node2.query("SELECT sum(v) FROM {}".format(dst))) == int( + node1.query("SELECT sum(v) FROM {}".format(dst)) + ) + + node1.query("DROP TABLE IF EXISTS {} SYNC".format(src)) + drop_everywhere(dst) + + +def interserver_request(node, target_host, params): + """One raw interserver request, straight at the sender's `DataPartsExchange` endpoint. + + The version-mix behaviour lives on the wire and nowhere else: which protocol version the peer + advertises is not configurable, so the only way to exercise a NON-confirm-capable peer against this + build's sender is to be that peer. Returns (headers, body_size). + """ + query = "&".join("{}={}".format(k, v) for k, v in params) + out = node.exec_in_container( + [ + "bash", + "-c", + "curl -sS -o /tmp/ca_ism_body -D /tmp/ca_ism_hdr -w '%{{http_code}} %{{size_download}}' " + "{url} >/tmp/ca_ism_stat; cat /tmp/ca_ism_hdr; echo '--STAT--'; cat /tmp/ca_ism_stat".format( + url=shlex.quote("http://{}:9009/?{}".format(target_host, query)) + ), + ] + ) + headers, stat = out.split("--STAT--") + http_code, size = stat.split() + assert http_code == "200", "interserver request failed: {}\n{}".format(stat, headers) + return headers, int(size) + + +def test_version_mix_legacy_peer_gets_bytes(): + """Task 16 step 7 — version mix: a peer that does not promise to confirm is served BYTES. + + This is the sender-side half of the mixed-build gate, and it is the half that is reachable without a + second binary: the offer is gated on `client_protocol_version >= 11` (`..._WITH_CA_CONFIRM`), so a + peer advertising 10 — a build that would relink WITHOUT confirming — must get the byte stream. + Degrading to bytes, never to an unconfirmed relink, is the whole point of moving the gate to 11. + + The control request (identical, but advertising 11) is what makes the negative meaningful: it proves + the request is otherwise perfectly relinkable, so the absence of an offer in the v10 case is the + version gate and not a malformed request. + + The receiver-side row-1 branch — a genuinely OLD sender that offers a relink with NO source token — + is NOT covered here; see the report accompanying this task. + """ + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + table = "vermix" + drop_everywhere(table) + + create_replicated(node1, table) + create_replicated(node2, table) + insert_rows(node1, table, 0) + node2.query("SYSTEM SYNC REPLICA {}".format(table), timeout=60) + part = active_part_names(node1, table)[0] + + # The pool identity as the SERVER reports it: taken from the sender's own offer line rather than + # re-derived from the pool metadata, so the value fed back in is exactly what `getPoolUUID` returns. + offers = wait_for_log_lines(node1, relink_offer_pattern(table, part)) + pool_uuid = re.search(r"shared pool ([0-9a-f]+)\)", offers[-1]).group(1) + + endpoint = "DataPartsExchange:/clickhouse/tables/{}/replicas/node1".format(table) + base = [ + ("endpoint", endpoint), + ("part", part), + ("compress", "false"), + ("cas_pool_uuid", pool_uuid), + ] + + # CONTROL — a confirm-capable peer: an offer, with a token, and a tiny manifest-only body. + headers_v11, size_v11 = interserver_request( + node2, "node1", base + [("client_protocol_version", "11")] + ) + assert "cas_relink=part_manifest_v2" in headers_v11, headers_v11 + assert "cas_source_token=" in headers_v11, headers_v11 + + # THE CASE UNDER TEST — a peer advertising the pre-confirm version: no offer, and the part's bytes. + headers_v10, size_v10 = interserver_request( + node2, "node1", base + [("client_protocol_version", "10")] + ) + assert "cas_relink" not in headers_v10, headers_v10 + assert "cas_source_token" not in headers_v10, headers_v10 + assert "server_protocol_version=10" in headers_v10, headers_v10 + + # Positive proof that bytes ACTUALLY moved rather than the request merely succeeding: the v10 + # response carries the whole part, orders of magnitude more than the manifest-only relink payload. + assert size_v10 > 20 * size_v11, ( + "the v10 peer should have received the part's bytes, got {} bytes against the relink offer's " + "{}".format(size_v10, size_v11) + ) + + drop_everywhere(table) + + +def test_recursion_brake_bounds_relink_to_one_attempt(): + """Task 16 step 4 — the `allow_ca_relink` recursion brake. + + A mechanism failure that is a property of the sender/receiver PAIR reproduces on every attempt, so + without the brake the byte-fetch fallback re-advertises the pool, is re-offered a relink, fails + again, and recurses until the stack is gone. The failpoint injects exactly that class of failure + (taxonomy rows 2 and 5 share this ACTION), because no configuration can produce one. + + The assertion is a COUNT, not termination: exactly ONE relink offer is made for this part, and then + the bytes arrive. Termination alone would also hold for a brake that merely bounded the recursion at + some larger depth, and it would hold vacuously if the relink path were never entered at all. + """ + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + table = "brake" + drop_everywhere(table) + + create_replicated(node1, table) + create_replicated(node2, table) + + node2.query("SYSTEM STOP FETCHES {}".format(table)) + insert_rows(node1, table, 0) + part = active_part_names(node1, table)[0] + + node2.query("SYSTEM ENABLE FAILPOINT cas_relink_receiver_force_mechanism_failure") + try: + node2.query("SYSTEM START FETCHES {}".format(table)) + node2.query("SYSTEM SYNC REPLICA {}".format(table), timeout=90) + + # The receiver hit the injected failure exactly once... + hits = log_lines( + node2, + r"Failpoint cas_relink_receiver_force_mechanism_failure: abandoning the relink of part {}".format( + re.escape(part) + ), + ) + assert len(hits) == 1, "expected exactly one relink attempt, got {}:\n{}".format( + len(hits), "\n".join(hits) + ) + + # ... and the SENDER, independently, made exactly one offer. This is the sharper of the two: the + # re-request is what would re-open the capability, and the sender is the only party that can say + # whether it did. + offers = log_lines(node1, relink_offer_pattern(table, part)) + assert len(offers) == 1, "expected exactly one relink offer, got {}:\n{}".format( + len(offers), "\n".join(offers) + ) + + # And the fetch still succeeded, over the byte path. + assert_byte_downloaded(node2, table, part) + assert int(node2.query("SELECT count() FROM {}".format(table))) == NUM_ROWS + assert int(node2.query("SELECT sum(v) FROM {}".format(table))) == int( + node1.query("SELECT sum(v) FROM {}".format(table)) + ) + finally: + node2.query("SYSTEM DISABLE FAILPOINT cas_relink_receiver_force_mechanism_failure") + + drop_everywhere(table) + + +# Settings that make node1 drop an outdated part — and with it the CA ref the confirm asks about — +# within a few seconds instead of the default eight minutes. +FAST_OLD_PART_REMOVAL = ( + "old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 1, " + "max_cleanup_delay_period = 1" +) + + +def open_publish_confirm_window(node1, node2, table, base): + """Drive a relink up to the paused point BETWEEN the receiver's durable `+1` and the confirm. + + Returns `(part, part_blobs)`: the name of the part whose relink is now stalled, and the blob keys + that its insert ADDED to the pool. The delta matters — debris from earlier tests in this module may + still be sitting in the pool and may legitimately be reclaimed while the window is open, so only the + keys this part created can be asserted about. The caller MUST resume the failpoint. + + `base` shifts the generated rows so this table's column data is unlike any other table's in this + module. Without it the content-addressed store deduplicates the insert against an earlier test's + identical blobs and the delta is EMPTY — which would make every blob assertion below vacuous. + """ + create_replicated(node1, table, extra_settings=FAST_OLD_PART_REMOVAL) + create_replicated(node2, table, extra_settings=FAST_OLD_PART_REMOVAL) + + node2.query("SYSTEM STOP FETCHES {}".format(table)) + before_insert = blob_keys() + insert_rows(node1, table, base) + part = active_part_names(node1, table)[0] + part_blobs = blob_keys() - before_insert + assert part_blobs, "the insert wrote no new blob into the shared pool" + + node2.query("SYSTEM ENABLE FAILPOINT cas_relink_receiver_pause_before_confirm") + node2.query("SYSTEM START FETCHES {}".format(table)) + # Blocks until the fetch thread is parked inside `relinkPartToDisk`, after `prepareAdoptFromManifest` + # made the receiver's `+1` durable and before the confirm request is built. + node2.query("SYSTEM WAIT FAILPOINT cas_relink_receiver_pause_before_confirm PAUSE", timeout=120) + return part, part_blobs + + +def merge_the_source_part_away(node1, table, part, base): + """While the receiver is parked: make the sender stop holding the exact binding it offered.""" + insert_rows(node1, table, base + NUM_ROWS) + node1.query("OPTIMIZE TABLE {} FINAL".format(table)) + # The confirm is answered from the sender's live state, so the test is only meaningful once the old + # part — and the ref naming its manifest — is really gone, not merely Outdated. + wait_until( + lambda: any_state_part_count(node1, table, part) == 0, + timeout=120, + what="node1 to drop the outdated part {}".format(part), + ) + + +def test_confirm_refuses_when_source_dropped_in_window(): + """Task 16 step 1 — the race the confirm exists to lose safely. + + Taxonomy row 3: the source cannot prove it still holds the offered manifest, so the receiver aborts + its durable `+1` and throws a retry-later `NETWORK_ERROR` INSTEAD of falling back to bytes. The two + assertions that matter are (a) the queue recovers by re-selecting — here, onto the covering part — + and (b) NO byte re-request ever went to the source whose state was in doubt. (b) is the entire + reason row 3 throws where rows 2 and 5 return `nullptr`. + """ + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + table = "race_confirm" + drop_everywhere(table) + + try: + part, _ = open_publish_confirm_window(node1, node2, table, base=1_000_000) + merge_the_source_part_away(node1, table, part, base=1_000_000) + finally: + node2.query("SYSTEM DISABLE FAILPOINT cas_relink_receiver_pause_before_confirm") + + # POSITIVE SIGNAL for row 3: the locally generated refusal, naming the source and the part. + wait_for_log_lines( + node2, + r"Source .* did not prove it still holds the manifest it offered for part {}".format( + re.escape(part) + ), + timeout=120, + ) + + # (a) the queue re-selects rather than losing the data. + node2.query("SYSTEM SYNC REPLICA {}".format(table), timeout=180) + assert int(node2.query("SELECT count() FROM {}".format(table))) == 2 * NUM_ROWS + assert int(node2.query("SELECT sum(v) FROM {}".format(table))) == int( + node1.query("SELECT sum(v) FROM {}".format(table)) + ) + assert active_part_names(node2, table) == active_part_names(node1, table) + + # (b) the abandoned part was never re-requested as bytes from the same source, and it was never + # promoted either — both would be a violation of the row-3 contract. + assert not log_lines(node2, download_finished_pattern(table, part)), ( + "row 3 must not fall back to a byte re-request against the source it could not confirm" + ) + assert not log_lines(node2, relink_finished_pattern(table, part)) + assert any_state_part_count(node2, table, part) == 0 + + drop_everywhere(table) + + +def test_stalled_publish_protects_source_blobs_and_commits_nothing(): + """Task 16 step 3 — the codex-6 regression, which is why publish-then-confirm exists at all. + + The receiver's `+1` is durable while the fetch is stalled. Across the stall the sender merges the + part away and GC runs to a fixpoint several times over: the offered manifest's blobs MUST survive, + because the stalled receiver's own binding protects them — that is what makes the later confirm a + meaningful question rather than a race against a sweep. And when the confirm finally answers + `unproven`, the stalled attempt must leave NOTHING committed. + + The soundness guard is the last assertion: once the attempt is abandoned and the sender no longer + holds the part, GC DOES reclaim those same blobs. Without it, "the blobs survived" would also be + satisfied by a GC that never deletes anything. + """ + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + table = "codex6_stall" + drop_everywhere(table) + + try: + # `part_blobs` is exactly what this part's insert added to the pool — see the helper for why it + # has to be the delta and not everything under the prefix. + part, part_blobs = open_publish_confirm_window(node1, node2, table, base=2_000_000) + + merge_the_source_part_away(node1, table, part, base=2_000_000) + + # Four full GC rounds on both mounters, spread well past the pool's 3-second condemn grace, so + # a blob that was NOT protected would have been condemned, aged out and deleted in the window. + for _ in range(4): + gc_round(node1) + gc_round(node2) + time.sleep(1.5) + + missing = sorted(part_blobs - blob_keys()) + assert not missing, ( + "the stalled receiver's durable +1 must protect the offered manifest's blobs across GC; " + "{} of {} were reclaimed, e.g. {}".format(len(missing), len(part_blobs), missing[:5]) + ) + finally: + node2.query("SYSTEM DISABLE FAILPOINT cas_relink_receiver_pause_before_confirm") + + wait_for_log_lines( + node2, + r"Source .* did not prove it still holds the manifest it offered for part {}".format( + re.escape(part) + ), + timeout=120, + ) + + # Nothing was committed by the stalled attempt. + assert any_state_part_count(node2, table, part) == 0 + assert not log_lines(node2, relink_finished_pattern(table, part)) + + node2.query("SYSTEM SYNC REPLICA {}".format(table), timeout=180) + assert int(node2.query("SELECT count() FROM {}".format(table))) == 2 * NUM_ROWS + + # No dangling reference anywhere in the pool, from either mounter's point of view. + for node in (node1, node2): + summary = fsck(node) + assert summary["dangling"] == "0", "{} fsck: {}".format(node.name, summary) + + # THE SOUNDNESS GUARD, and it is what makes the survival asserted earlier mean anything: with the + # part gone from both replicas and the stalled attempt abandoned, its unique blobs are unreachable, + # so GC reclaiming them proves their survival DURING the stall was the relink pin and not GC + # inactivity. Without this, "the blobs were still there" would also be what a GC that never ran + # produces. + reclaimed = set() + for _ in range(8): + gc_round(node1) + gc_round(node2) + reclaimed = part_blobs - blob_keys() + if reclaimed == part_blobs: + break + # Pool-wide: the GC lease is held by ONE server and it need not be node1. + rounds = 0 + for n in (node1, node2): + n.query("SYSTEM FLUSH LOGS") + rounds += int( + n.query( + "SELECT count() FROM system.cas_gc_log " + "WHERE event_type = 'Finish' AND outcome = 'Success'" + ).strip() + or 0 + ) + assert rounds > 0, "no successful GC round ran at all" + assert reclaimed, ( + "none of the abandoned attempt's {} blob(s) were reclaimed, so their survival during the " + "stall does not distinguish the relink pin from an inactive GC".format(len(part_blobs)) + ) + + drop_everywhere(table) diff --git a/tests/integration/test_cas_s3/__init__.py b/tests/integration/test_cas_s3/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_cas_s3/configs/storage_conf.xml b/tests/integration/test_cas_s3/configs/storage_conf.xml new file mode 100644 index 000000000000..fdfa19ff5409 --- /dev/null +++ b/tests/integration/test_cas_s3/configs/storage_conf.xml @@ -0,0 +1,28 @@ + + + + + object_storage + s3 + cas + + itest-content-addressed-s3 + + http://rustfs1:11121/test/cas_data/ + clickhouse + clickhouse + + + + + +

+ disk_cas_s3 +
+ + + + + diff --git a/tests/integration/test_cas_s3/test.py b/tests/integration/test_cas_s3/test.py new file mode 100644 index 000000000000..c5144bafb8a0 --- /dev/null +++ b/tests/integration/test_cas_s3/test.py @@ -0,0 +1,145 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +STORAGE_POLICY = "cas_s3" +NUM_ROWS = 1000 + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + cluster.add_instance( + "node", + main_configs=["configs/storage_conf.xml"], + with_rustfs=True, + stay_alive=True, + ) + + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_cas_s3(): + node = cluster.instances["node"] + + node.query("DROP TABLE IF EXISTS cas_test SYNC") + node.query( + """ + CREATE TABLE cas_test ( + id Int64, + data String + ) ENGINE = MergeTree() + ORDER BY id + SETTINGS storage_policy = '{}' + """.format( + STORAGE_POLICY + ) + ) + + # First insert of NUM_ROWS deterministic rows. + node.query( + "INSERT INTO cas_test SELECT number, toString(number) FROM numbers({})".format( + NUM_ROWS + ) + ) + + expected_sum = (NUM_ROWS - 1) * NUM_ROWS // 2 + assert int(node.query("SELECT count() FROM cas_test")) == NUM_ROWS + assert int(node.query("SELECT sum(id) FROM cas_test")) == expected_sum + + # A second identical insert: the row count doubles. Each part's content is identical, so the + # content-addressed disk deduplicates the blobs, but the logical row count must still double. + node.query( + "INSERT INTO cas_test SELECT number, toString(number) FROM numbers({})".format( + NUM_ROWS + ) + ) + assert int(node.query("SELECT count() FROM cas_test")) == 2 * NUM_ROWS + assert int(node.query("SELECT sum(id) FROM cas_test")) == 2 * expected_sum + + # Merge the two parts together. + node.query("OPTIMIZE TABLE cas_test FINAL") + assert int(node.query("SELECT count() FROM cas_test")) == 2 * NUM_ROWS + assert int(node.query("SELECT sum(id) FROM cas_test")) == 2 * expected_sum + + # Persistence: after a restart the refs/footers/blobs in S3 must still resolve the data. + node.restart_clickhouse() + + assert int(node.query("SELECT count() FROM cas_test")) == 2 * NUM_ROWS + assert int(node.query("SELECT sum(id) FROM cas_test")) == 2 * expected_sum + + # Drop must complete without error (ref unlink + deferred GC). + node.query("DROP TABLE cas_test SYNC") + assert ( + node.query( + "SELECT count() FROM system.tables WHERE database = currentDatabase() AND name = 'cas_test'" + ).strip() + == "0" + ) + + +def test_mutations_and_patch_parts_survive_restart(): + # A mutated part and a patch part are ordinary content-addressed parts published as refs. After a + # restart the active set must be rediscovered from the refs in S3, so the post-mutation / + # post-lightweight-delete state must survive (CAS M7). + node = cluster.instances["node"] + + node.query("DROP TABLE IF EXISTS cas_mut SYNC") + node.query( + """ + CREATE TABLE cas_mut ( + id Int64, + v UInt64, + s String + ) ENGINE = MergeTree() + ORDER BY id + SETTINGS storage_policy = '{}', enable_block_number_column = 1, enable_block_offset_column = 1 + """.format( + STORAGE_POLICY + ) + ) + + node.query( + "INSERT INTO cas_mut SELECT number, number * 10, toString(number) FROM numbers({})".format( + NUM_ROWS + ) + ) + + # Heavy mutation: UPDATE one column (id/s carry forward by reference on the content-addressed disk). + node.query( + "ALTER TABLE cas_mut UPDATE v = v + 1 WHERE id % 2 = 0 SETTINGS mutations_sync = 2" + ) + # Heavy mutation: DELETE. + node.query("ALTER TABLE cas_mut DELETE WHERE id % 100 = 0 SETTINGS mutations_sync = 2") + # Data-ALTER (column type change). Via a storage policy there is no inline-disk CustomType in + # settings_changes, so this works on the content-addressed disk (see backlog B53). + node.query("ALTER TABLE cas_mut MODIFY COLUMN v Int64 SETTINGS mutations_sync = 2") + # Patch part: a forced lightweight-update DELETE (throws if unsupported, so success == patch path). + node.query( + "DELETE FROM cas_mut WHERE id % 7 = 0 " + "SETTINGS enable_lightweight_update = 1, lightweight_delete_mode = 'lightweight_update_force', lightweight_deletes_sync = 2" + ) + + count_before = int(node.query("SELECT count() FROM cas_mut")) + sum_before = int(node.query("SELECT sum(v) FROM cas_mut")) + digest_before = node.query("SELECT sum(cityHash64(id, v, s)) FROM cas_mut").strip() + + # Persistence: rediscover the active set (incl. the mutated and patch parts) from S3 refs. + node.restart_clickhouse() + + assert int(node.query("SELECT count() FROM cas_mut")) == count_before + assert int(node.query("SELECT sum(v) FROM cas_mut")) == sum_before + assert node.query("SELECT sum(cityHash64(id, v, s)) FROM cas_mut").strip() == digest_before + + node.query("DROP TABLE cas_mut SYNC") + assert ( + node.query( + "SELECT count() FROM system.tables WHERE database = currentDatabase() AND name = 'cas_mut'" + ).strip() + == "0" + ) diff --git a/tests/integration/test_cas_shared_pool/__init__.py b/tests/integration/test_cas_shared_pool/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_cas_shared_pool/configs/server_root_id_node1.xml b/tests/integration/test_cas_shared_pool/configs/server_root_id_node1.xml new file mode 100644 index 000000000000..be23e47852bb --- /dev/null +++ b/tests/integration/test_cas_shared_pool/configs/server_root_id_node1.xml @@ -0,0 +1,12 @@ + + + + + + node1 + + + + diff --git a/tests/integration/test_cas_shared_pool/configs/server_root_id_node2.xml b/tests/integration/test_cas_shared_pool/configs/server_root_id_node2.xml new file mode 100644 index 000000000000..e3b468aaa4e4 --- /dev/null +++ b/tests/integration/test_cas_shared_pool/configs/server_root_id_node2.xml @@ -0,0 +1,12 @@ + + + + + + node2 + + + + diff --git a/tests/integration/test_cas_shared_pool/configs/storage_conf.xml b/tests/integration/test_cas_shared_pool/configs/storage_conf.xml new file mode 100644 index 000000000000..1620ff6d0797 --- /dev/null +++ b/tests/integration/test_cas_shared_pool/configs/storage_conf.xml @@ -0,0 +1,36 @@ + + + + + object_storage + s3 + cas + + + http://rustfs1:11121/test/shared_pool/ + clickhouse + clickhouse + + + 1 + 1 + + + + + +
+ disk_cas_shared +
+
+
+
+
+
diff --git a/tests/integration/test_cas_shared_pool/test.py b/tests/integration/test_cas_shared_pool/test.py new file mode 100644 index 000000000000..5becc8dc049a --- /dev/null +++ b/tests/integration/test_cas_shared_pool/test.py @@ -0,0 +1,347 @@ +import time + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +# Both servers mount the SAME content-addressed pool (endpoint .../root/shared_pool/). The blob pool +# (blobs/ + parts/) is shared across servers; refs are per-server under store//..., so the +# two servers dedup identical content while keeping independent ref roots. +STORAGE_POLICY = "cas_shared" + +# blobs/ holds content blobs, parts/ holds part footers. These are the shared pool's object prefixes +# inside the `root` MinIO bucket. "No leftovers" means BOTH drain back to baseline. +BLOBS_PREFIX = "shared_pool/blobs/" +PARTS_PREFIX = "shared_pool/parts/" + +# Deterministic data. Identical rows on both nodes => identical content blobs => cross-server dedup. +NUM_ROWS = 100000 + +# Background GC: grace=3s, interval=1s. After both DROP ... SYNC the pool's objects become +# unreferenced and a sweep (run by either server) reclaims them after grace. Bounded poll: this waits +# on a known background process, it is not papering over a race. +RECLAIM_RETRIES = 60 +RECLAIM_SLEEP = 1.0 # seconds; total bound ~= 60s + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + # RustFS (not MinIO) backs the pool: the CA mount capability probe requires enforced + # conditional-DELETE semantics, which MinIO OSS lacks — the fail-closed probe aborted server + # startup there (PR#2073 CI triage). Both instances reach the shared rustfs1 and load the + # identical storage_conf.xml, so both mount the SAME shared pool. + cluster.add_instance( + "node1", + main_configs=["configs/storage_conf.xml", "configs/server_root_id_node1.xml"], + with_rustfs=True, + stay_alive=True, + ) + cluster.add_instance( + "node2", + main_configs=["configs/storage_conf.xml", "configs/server_root_id_node2.xml"], + with_rustfs=True, + stay_alive=True, + ) + + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def count_prefix(prefix): + objects = cluster.rustfs_client.list_objects( + cluster.rustfs_bucket, prefix, recursive=True + ) + return len(list(objects)) + + + +def _gc_bookkeeping(*nodes): + """Pool-wide (successful rounds, objects deleted) from the CA GC log. The GC lease is held by ONE + server per pool and which one is not fixed, so both must be asked.""" + rounds = deleted = 0 + for n in nodes: + n.query("SYSTEM FLUSH LOGS") + rounds += int( + n.query( + "SELECT count() FROM system.cas_gc_log " + "WHERE event_type = 'Finish' AND outcome = 'Success'" + ).strip() + or 0 + ) + deleted += int( + n.query( + "SELECT sum(objects_deleted + manifests_deleted + entries_redeleted) " + "FROM system.cas_gc_log WHERE event_type = 'Finish'" + ).strip() + or 0 + ) + return rounds, deleted + +def count_pool_objects(): + # The shared pool is empty only when BOTH content blobs and part footers are gone. + return count_prefix(BLOBS_PREFIX) + count_prefix(PARTS_PREFIX) + + +def test_two_servers_share_one_pool(): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + + node1.query("DROP TABLE IF EXISTS t1 SYNC") + node2.query("DROP TABLE IF EXISTS t2 SYNC") + + # (0) Baseline pool object count before either table exists. + baseline = count_pool_objects() + + # (1) Each server creates its OWN MergeTree table on the shared pool. Distinct names => distinct + # table UUIDs => independent per-server refs, but the SAME shared blob pool. + create_tpl = ( + "CREATE TABLE {tbl} (id Int64, v UInt64, s String) " + "ENGINE = MergeTree() ORDER BY id " + "SETTINGS storage_policy = '{policy}'" + ) + node1.query(create_tpl.format(tbl="t1", policy=STORAGE_POLICY)) + node2.query(create_tpl.format(tbl="t2", policy=STORAGE_POLICY)) + + # (2) INSERT IDENTICAL deterministic data into both. The content blobs are byte-identical, so the + # shared pool dedups them across the two servers. Logical reads must still be correct on each. + insert_tpl = ( + "INSERT INTO {tbl} " + "SELECT number, number * 10, toString(number) FROM numbers({rows})" + ) + node1.query(insert_tpl.format(tbl="t1", rows=NUM_ROWS)) + node2.query(insert_tpl.format(tbl="t2", rows=NUM_ROWS)) + + expected_sum_id = (NUM_ROWS - 1) * NUM_ROWS // 2 + assert int(node1.query("SELECT count() FROM t1")) == NUM_ROWS + assert int(node2.query("SELECT count() FROM t2")) == NUM_ROWS + assert int(node1.query("SELECT sum(id) FROM t1")) == expected_sum_id + assert int(node2.query("SELECT sum(id) FROM t2")) == expected_sum_id + + # Cross-server dedup sanity: the two identical single-part inserts must NOT have doubled the pool's + # blob count. With dedup the blob count after both inserts is well below twice the per-server count. + after_insert = count_pool_objects() + assert after_insert > baseline, ( + "expected pool object count to rise above baseline {} after inserts, got {}".format( + baseline, after_insert + ) + ) + + # (3) Heavy mutations / merges on EACH server, in parallel ownership of the shared pool. + # UPDATE (id/s carry forward by reference), DELETE, then OPTIMIZE FINAL. + node1.query("ALTER TABLE t1 UPDATE v = v + 1 WHERE id % 2 = 0 SETTINGS mutations_sync = 2") + node2.query("ALTER TABLE t2 UPDATE v = v + 1 WHERE id % 2 = 0 SETTINGS mutations_sync = 2") + + node1.query("ALTER TABLE t1 DELETE WHERE id % 100 = 0 SETTINGS mutations_sync = 2") + node2.query("ALTER TABLE t2 DELETE WHERE id % 100 = 0 SETTINGS mutations_sync = 2") + + node1.query("OPTIMIZE TABLE t1 FINAL") + node2.query("OPTIMIZE TABLE t2 FINAL") + + # Post-mutation expected aggregates (identical recipe on both, so both must match). + count_after_mut = int(node1.query("SELECT count() FROM t1")) + sum_after_mut = int(node1.query("SELECT sum(v) FROM t1")) + digest_after_mut = node1.query("SELECT sum(cityHash64(id, v, s)) FROM t1").strip() + + assert int(node2.query("SELECT count() FROM t2")) == count_after_mut + assert int(node2.query("SELECT sum(v) FROM t2")) == sum_after_mut + assert node2.query("SELECT sum(cityHash64(id, v, s)) FROM t2").strip() == digest_after_mut + + # (4) Let the background GC (enabled on BOTH servers, short grace) run several sweep cycles while + # both tables are still live. The cross-server safety property: a sweep run by either server + # must NOT reclaim a blob that the OTHER server's live part references (deduped/shared blob). + # Sleeping here is waiting on the known background sweep cadence, not a race workaround. + time.sleep(3 * RECLAIM_SLEEP + 3) # > grace(3s) + a few interval(1s) cycles + + # Re-read on BOTH servers: no data lost to the other server's GC. + assert int(node1.query("SELECT count() FROM t1")) == count_after_mut + assert int(node1.query("SELECT sum(v) FROM t1")) == sum_after_mut + assert node1.query("SELECT sum(cityHash64(id, v, s)) FROM t1").strip() == digest_after_mut + + assert int(node2.query("SELECT count() FROM t2")) == count_after_mut + assert int(node2.query("SELECT sum(v) FROM t2")) == sum_after_mut + assert node2.query("SELECT sum(cityHash64(id, v, s)) FROM t2").strip() == digest_after_mut + + # (5) Both servers drop their tables. Refs are unlinked synchronously; the shared pool's blobs and + # footers become unreferenced GC fodder. Then poll until the pool drains back to baseline. + node1.query("DROP TABLE t1 SYNC") + node2.query("DROP TABLE t2 SYNC") + + at_drop = count_pool_objects() + + # THE RECLAMATION: both servers' content goes. Polled with an early exit, then cross-checked + # against GC's own bookkeeping so a pool that shrank for some other reason cannot pass for a round + # that reclaimed it. + final = count_pool_objects() + for _ in range(RECLAIM_RETRIES): + if final <= baseline: + break + time.sleep(RECLAIM_SLEEP) + final = count_pool_objects() + + assert final <= baseline, ( + "the shared pool did not drain after both servers dropped: " + "baseline={}, after_insert={}, at_drop={}, final={} (blobs={}, parts={})".format( + baseline, + after_insert, + at_drop, + final, + count_prefix(BLOBS_PREFIX), + count_prefix(PARTS_PREFIX), + ) + ) + + # Counted POOL-WIDE: exactly one server holds the GC lease for a shared pool, and it need not be + # node1 — asking only node1 yields 0 rounds whenever node2 is the leader, which is how this + # assertion first failed. + rounds, deleted = _gc_bookkeeping(node1, node2) + assert rounds > 0, "no successful GC round ran at all" + assert deleted > 0, "the shared pool drained but GC's own bookkeeping reports no deletion" + + +# Crash-resilience uses a SMALLER, DISTINCT dataset per node. Distinct content => node1's blobs are +# NOT deduped with node2's, so "node2's GC must not reclaim node1's blobs while node1 is down" is a +# real, observable invariant on the pool object count (node1's blobs cannot hide behind node2's). +CRASH_ROWS = 50000 + + +def test_pool_survives_node_crash(): + # Proves the bucket is self-describing and the pool survives a hard node crash: + # (a) the surviving node keeps running with background GC on and loses no data; + # (b) the hard-killed node recovers its data on restart (refs are durable in the bucket); + # (c) any orphaned write-session the crash left behind is eventually reclaimed (its lease + # expires; the pool drains to baseline after DROP). + # Lock-fencing safety (paused GC leader fenced by a peer's higher fence token) is covered by the + # gtest SweepStopsWhenLeadershipLost; this test focuses on crash-resilience. + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + + node1.query("DROP TABLE IF EXISTS crash1 SYNC") + node2.query("DROP TABLE IF EXISTS crash2 SYNC") + + # (0) Baseline pool object count before either table exists. + baseline = count_pool_objects() + + create_tpl = ( + "CREATE TABLE {tbl} (id Int64, v UInt64, s String) " + "ENGINE = MergeTree() ORDER BY id " + "SETTINGS storage_policy = '{policy}'" + ) + node1.query(create_tpl.format(tbl="crash1", policy=STORAGE_POLICY)) + node2.query(create_tpl.format(tbl="crash2", policy=STORAGE_POLICY)) + + # (1) DISTINCT deterministic data per node (different `v` recipe => different content blobs, so + # node1's blobs are NOT shared with node2's and cannot be hidden behind dedup). + node1.query( + "INSERT INTO crash1 SELECT number, number * 10, toString(number) " + "FROM numbers({rows})".format(rows=CRASH_ROWS) + ) + node2.query( + "INSERT INTO crash2 SELECT number, number * 7, concat('n2_', toString(number)) " + "FROM numbers({rows})".format(rows=CRASH_ROWS) + ) + + # Capture node1's authoritative aggregates BEFORE the crash; recovery must reproduce them exactly. + n1_count = int(node1.query("SELECT count() FROM crash1")) + n1_sum_id = int(node1.query("SELECT sum(id) FROM crash1")) + n1_digest = node1.query("SELECT sum(cityHash64(id, v, s)) FROM crash1").strip() + assert n1_count == CRASH_ROWS + + n2_count = int(node2.query("SELECT count() FROM crash2")) + n2_digest = node2.query("SELECT sum(cityHash64(id, v, s)) FROM crash2").strip() + assert n2_count == CRASH_ROWS + + # The pool now holds BOTH nodes' (distinct) blobs. Remember this high-water mark: after node1 is + # killed, node2's GC must NOT shrink the pool below the level needed to hold node1's blobs. + after_both_inserts = count_pool_objects() + assert after_both_inserts > baseline + + # (2) HARD-KILL node1 (SIGKILL via pkill -9 => simulated crash). node2 stays up. A crash mid-flight + # can leave node1 holding an unreleased write-session lease (an orphaned pin) on the pool. + node1.stop_clickhouse(kill=True) + + # (3) With node1 down, keep node2 working AND let node2's background GC run several sweep cycles. + # Two invariants: + # - node2 reads its OWN data correctly (no loss while it owns the pool alone); + # - node2's GC does NOT reclaim node1's blobs: node1's refs are durable roots in the bucket + # even though node1's process is gone. Sleeping here waits on the known sweep cadence. + node2.query( + "INSERT INTO crash2 SELECT number, number * 7, concat('n2b_', toString(number)) " + "FROM numbers({rows})".format(rows=CRASH_ROWS) + ) + node2.query("OPTIMIZE TABLE crash2 FINAL") + n2_count_after = int(node2.query("SELECT count() FROM crash2")) + assert n2_count_after == 2 * CRASH_ROWS + + time.sleep(3 * RECLAIM_SLEEP + 3) # > grace(3s) + a few interval(1s) cycles of node2's GC + + # node2 lost nothing. + assert int(node2.query("SELECT count() FROM crash2")) == n2_count_after + # node1's blobs were NOT swept by node2's GC: the pool still holds at least node1's portion. node1 + # contributed (after_both_inserts - baseline) objects on top of the empty baseline, so even if + # node2 had reclaimed every one of its own blobs the pool could not have dropped below that. + node1_contribution = after_both_inserts - baseline + pool_with_node1_down = count_pool_objects() + assert pool_with_node1_down >= baseline + node1_contribution, ( + "node2's GC appears to have reclaimed node1's durable refs while node1 was down: " + "baseline={}, after_both_inserts={}, node1_contribution={}, pool_now={}".format( + baseline, after_both_inserts, node1_contribution, pool_with_node1_down + ) + ) + + # (4) RESTART node1. The bucket is self-describing: node1 rebuilds its active set from the durable + # refs and must re-read its table with the EXACT pre-crash count/sum/digest. After a hard kill + # the harness reconnects on start_clickhouse via wait_start; use the instance object fresh. + # 150s, not the 60s default: a post-SIGKILL restart legitimately pays the unclean-reclaim + # cost before serving — the stale-token observation window over its own unexpired lease + # (~TTL + 5% + renew_period/2 ≈ 36.5s with defaults) plus the materialization grace + # (30s default) plus the lease re-write; ~71s observed end-to-end. A bounded wait on a + # known, by-design recovery protocol — not a race hack. + node1.start_clickhouse(start_wait_sec=150) + + assert int(node1.query("SELECT count() FROM crash1")) == n1_count + assert int(node1.query("SELECT sum(id) FROM crash1")) == n1_sum_id + assert node1.query("SELECT sum(cityHash64(id, v, s)) FROM crash1").strip() == n1_digest + + # node2 still consistent after node1 rejoined. + assert int(node2.query("SELECT count() FROM crash2")) == n2_count_after + + # (5) DROP both tables. Refs are unlinked synchronously; the orphaned write-session that node1's + # crash left behind no longer pins anything once its lease expires, so GC (run by either + # server) reclaims the lot. Bounded-poll the pool until it drains back to baseline. + node1.query("DROP TABLE crash1 SYNC") + node2.query("DROP TABLE crash2 SYNC") + + at_drop = count_pool_objects() + + # THE RECLAMATION, and the point of this test: the hard kill left nothing behind that survives the + # drop. Once both tables are gone and the orphaned write-session's lease expires, nothing pins the + # content and the pool returns to baseline. + final = count_pool_objects() + for _ in range(RECLAIM_RETRIES): + if final <= baseline: + break + time.sleep(RECLAIM_SLEEP) + final = count_pool_objects() + + assert final <= baseline, ( + "the shared pool did not drain after the crash + both DROPs: " + "baseline={}, after_both_inserts={}, at_drop={}, " + "final={} (blobs={}, parts={})".format( + baseline, + after_both_inserts, + at_drop, + final, + count_prefix(BLOBS_PREFIX), + count_prefix(PARTS_PREFIX), + ) + ) + + # Counted POOL-WIDE, for the same leader-may-be-either-node reason as the first test in this file. + rounds, deleted = _gc_bookkeeping(node1, node2) + assert rounds > 0, "no successful GC round ran at all" + assert deleted > 0, "the shared pool drained but GC's own bookkeeping reports no deletion" From 8ad9b861f491407ced44767ca835c99c3b79330a Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:41 +0200 Subject: [PATCH 28/30] CAS CI wiring and test tags CA-default stateless/integration lanes in praktika and workflows, the content-addressed default-disk test configs, clickhouse-test support, and tag edits to pre-existing tests (no-content-addressed-storage and friends). Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- .github/workflows/master.yml | 522 +++++++++++++++++- .github/workflows/pull_request.yml | 502 ++++++++++++++++- .github/workflows/pull_request_community.yml | 480 ++++++++++++++++ .github/workflows/release_builds.yml | 158 +++++- ci/defs/altinity_jobs.py | 51 ++ ci/jobs/functional_tests.py | 20 +- ci/jobs/scripts/check_style/various_checks.sh | 26 + ci/jobs/scripts/clickhouse_proc.py | 104 +++- ci/workflows/backport_branches.py | 8 +- ci/workflows/master.py | 7 +- ci/workflows/pull_request.py | 13 +- ci/workflows/pull_request_community.py | 12 +- ci/workflows/release_branches.py | 8 +- ci/workflows/release_builds.py | 7 +- tests/clickhouse-test | 42 +- ...orage_policy_for_merge_tree_by_default.xml | 66 +++ ...orage_policy_for_merge_tree_by_default.xml | 35 ++ tests/config/install.sh | 16 + .../01271_show_privileges.reference | 7 + .../0_stateless/02253_empty_part_checksums.sh | 3 +- .../02254_projection_broken_part.sh | 3 +- .../02255_broken_parts_chain_on_start.sh | 3 +- .../02369_lost_part_intersecting_merges.sh | 3 +- .../02370_lost_part_intersecting_merges.sh | 3 +- ...2444_async_broken_outdated_part_loading.sh | 3 +- .../02486_truncate_and_unexpected_parts.sql | 1 - .../02980_s3_plain_DROP_TABLE_MergeTree.sh | 5 +- ...s3_plain_DROP_TABLE_ReplicatedMergeTree.sh | 3 +- ...lter_table_fetch_partition_thread_pool.sql | 1 + .../03352_allow_suspicious_ttl.sql | 2 +- .../0_stateless/03541_rename_column_start.sql | 2 +- ...03829_insert_deduplication_info_memory.sql | 10 +- ...eplicated_missing_covered_part_on_start.sh | 3 +- .../04316_reader_executor_basic.sql | 7 +- .../04327_reader_executor_metrics.sql | 6 +- ...04328_reader_executor_kpi_async_metric.sql | 6 +- 36 files changed, 2109 insertions(+), 39 deletions(-) create mode 100644 tests/config/config.d/cas_s3_storage_policy_for_merge_tree_by_default.xml create mode 100644 tests/config/config.d/cas_storage_policy_for_merge_tree_by_default.xml diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 94c61cb6e696..f618bc8663c5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -2860,6 +2860,516 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, sequential)' --workflow "MasterCI" --ci --timestamp + stateless_tests_amd_binary_cas_s3_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBjYXMgczMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (amd_binary, cas s3 storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_binary, cas s3 storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_BINARY_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_BINARY_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas s3 storage, parallel)' --workflow "MasterCI" --ci --timestamp + + stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }} + name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)' --workflow "MasterCI" --ci --timestamp + + stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }} + name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)' --workflow "MasterCI" --ci --timestamp + + stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfdHNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }} + name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 1/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 1/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_tsan, cas s3 storage, parallel, 1/2)' --workflow "MasterCI" --ci --timestamp + + stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfdHNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }} + name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 2/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 2/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_tsan, cas s3 storage, parallel, 2/2)' --workflow "MasterCI" --ci --timestamp + + stateless_tests_amd_msan_cas_s3_storage_parallel_1_3: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzMp') }} + name: "Stateless tests (amd_msan, cas s3 storage, parallel, 1/3)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_msan, cas s3 storage, parallel, 1/3)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, cas s3 storage, parallel, 1/3)' --workflow "MasterCI" --ci --timestamp + + stateless_tests_amd_msan_cas_s3_storage_parallel_2_3: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzMp') }} + name: "Stateless tests (amd_msan, cas s3 storage, parallel, 2/3)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_msan, cas s3 storage, parallel, 2/3)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, cas s3 storage, parallel, 2/3)' --workflow "MasterCI" --ci --timestamp + + stateless_tests_amd_msan_cas_s3_storage_parallel_3_3: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAzLzMp') }} + name: "Stateless tests (amd_msan, cas s3 storage, parallel, 3/3)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_msan, cas s3 storage, parallel, 3/3)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, cas s3 storage, parallel, 3/3)' --workflow "MasterCI" --ci --timestamp + + stateless_tests_arm_binary_cas_s3_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] + needs: [build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYmluYXJ5LCBjYXMgczMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (arm_binary, cas s3 storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (arm_binary, cas s3 storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, cas s3 storage, parallel)' --workflow "MasterCI" --ci --timestamp + + stateless_tests_amd_binary_cas_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBjYXMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (amd_binary, cas storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_binary, cas storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_BINARY_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_BINARY_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas storage, parallel)' --workflow "MasterCI" --ci --timestamp + stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_llvm_coverage_per_test, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -5995,7 +6505,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, clickbench_amd_release, clickbench_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, clickbench_amd_release, clickbench_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6148,6 +6658,16 @@ jobs: - stateless_tests_amd_tsan_s3_storage_sequential_2_2 - stateless_tests_arm_binary_parallel - stateless_tests_arm_binary_sequential + - stateless_tests_amd_binary_cas_s3_storage_parallel + - stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2 + - stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2 + - stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2 + - stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2 + - stateless_tests_amd_msan_cas_s3_storage_parallel_1_3 + - stateless_tests_amd_msan_cas_s3_storage_parallel_2_3 + - stateless_tests_amd_msan_cas_s3_storage_parallel_3_3 + - stateless_tests_arm_binary_cas_s3_storage_parallel + - stateless_tests_amd_binary_cas_storage_parallel - stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8 - stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8 - stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2b77e85439e8..708552d3ef2f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -2405,6 +2405,496 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, sequential)' --workflow "PR" --ci --timestamp + stateless_tests_amd_binary_cas_s3_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_tsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBjYXMgczMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (amd_binary, cas s3 storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_binary, cas s3 storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_BINARY_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_BINARY_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas s3 storage, parallel)' --workflow "PR" --ci --timestamp + + stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }} + name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)' --workflow "PR" --ci --timestamp + + stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }} + name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_ASAN_UBSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_ASAN_UBSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)' --workflow "PR" --ci --timestamp + + stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfdHNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }} + name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 1/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 1/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_tsan, cas s3 storage, parallel, 1/2)' --workflow "PR" --ci --timestamp + + stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfdHNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }} + name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 2/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 2/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_TSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_TSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_tsan, cas s3 storage, parallel, 2/2)' --workflow "PR" --ci --timestamp + + stateless_tests_amd_msan_cas_s3_storage_parallel_1_3: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzMp') }} + name: "Stateless tests (amd_msan, cas s3 storage, parallel, 1/3)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_msan, cas s3 storage, parallel, 1/3)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, cas s3 storage, parallel, 1/3)' --workflow "PR" --ci --timestamp + + stateless_tests_amd_msan_cas_s3_storage_parallel_2_3: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzMp') }} + name: "Stateless tests (amd_msan, cas s3 storage, parallel, 2/3)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_msan, cas s3 storage, parallel, 2/3)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, cas s3 storage, parallel, 2/3)' --workflow "PR" --ci --timestamp + + stateless_tests_amd_msan_cas_s3_storage_parallel_3_3: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAzLzMp') }} + name: "Stateless tests (amd_msan, cas s3 storage, parallel, 3/3)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_msan, cas s3 storage, parallel, 3/3)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_MSAN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_MSAN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, cas s3 storage, parallel, 3/3)' --workflow "PR" --ci --timestamp + + stateless_tests_arm_binary_cas_s3_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYmluYXJ5LCBjYXMgczMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (arm_binary, cas s3 storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (arm_binary, cas s3 storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, cas s3 storage, parallel)' --workflow "PR" --ci --timestamp + + stateless_tests_amd_binary_cas_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_tsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBjYXMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (amd_binary, cas storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_binary, cas storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_BINARY_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_BINARY_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas storage, parallel)' --workflow "PR" --ci --timestamp + stateless_tests_arm_asan_ubsan_azure_parallel: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, ci_tests, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] @@ -5545,7 +6035,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -5687,6 +6177,16 @@ jobs: - stateless_tests_amd_tsan_s3_storage_sequential_2_2 - stateless_tests_arm_binary_parallel - stateless_tests_arm_binary_sequential + - stateless_tests_amd_binary_cas_s3_storage_parallel + - stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2 + - stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2 + - stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2 + - stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2 + - stateless_tests_amd_msan_cas_s3_storage_parallel_1_3 + - stateless_tests_amd_msan_cas_s3_storage_parallel_2_3 + - stateless_tests_amd_msan_cas_s3_storage_parallel_3_3 + - stateless_tests_arm_binary_cas_s3_storage_parallel + - stateless_tests_amd_binary_cas_storage_parallel - stateless_tests_arm_asan_ubsan_azure_parallel - stateless_tests_arm_asan_ubsan_azure_sequential_1_2 - stateless_tests_arm_asan_ubsan_azure_sequential_2_2 diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index c576bd5afaba..89ec4bca1be8 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -2073,6 +2073,486 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, sequential)' --workflow "Community PR" --ci --timestamp + stateless_tests_amd_binary_cas_s3_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_binary, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBjYXMgczMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (amd_binary, cas s3 storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_binary, cas s3 storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_BINARY + uses: actions/download-artifact@v8 + with: + name: CH_AMD_BINARY + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas s3 storage, parallel)' --workflow "Community PR" --ci --timestamp + + stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_asan_ubsan, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }} + name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_ASAN_UBSAN + uses: actions/download-artifact@v8 + with: + name: CH_AMD_ASAN_UBSAN + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)' --workflow "Community PR" --ci --timestamp + + stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_asan_ubsan, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }} + name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_ASAN_UBSAN + uses: actions/download-artifact@v8 + with: + name: CH_AMD_ASAN_UBSAN + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)' --workflow "Community PR" --ci --timestamp + + stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_debug, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfdHNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }} + name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 1/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 1/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_TSAN + uses: actions/download-artifact@v8 + with: + name: CH_AMD_TSAN + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_tsan, cas s3 storage, parallel, 1/2)' --workflow "Community PR" --ci --timestamp + + stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_debug, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfdHNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }} + name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 2/2)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_tsan, cas s3 storage, parallel, 2/2)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_TSAN + uses: actions/download-artifact@v8 + with: + name: CH_AMD_TSAN + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_tsan, cas s3 storage, parallel, 2/2)' --workflow "Community PR" --ci --timestamp + + stateless_tests_amd_msan_cas_s3_storage_parallel_1_3: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_debug, build_amd_msan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzMp') }} + name: "Stateless tests (amd_msan, cas s3 storage, parallel, 1/3)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_msan, cas s3 storage, parallel, 1/3)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_MSAN + uses: actions/download-artifact@v8 + with: + name: CH_AMD_MSAN + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, cas s3 storage, parallel, 1/3)' --workflow "Community PR" --ci --timestamp + + stateless_tests_amd_msan_cas_s3_storage_parallel_2_3: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_debug, build_amd_msan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzMp') }} + name: "Stateless tests (amd_msan, cas s3 storage, parallel, 2/3)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_msan, cas s3 storage, parallel, 2/3)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_MSAN + uses: actions/download-artifact@v8 + with: + name: CH_AMD_MSAN + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, cas s3 storage, parallel, 2/3)' --workflow "Community PR" --ci --timestamp + + stateless_tests_amd_msan_cas_s3_storage_parallel_3_3: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_debug, build_amd_msan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAzLzMp') }} + name: "Stateless tests (amd_msan, cas s3 storage, parallel, 3/3)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_msan, cas s3 storage, parallel, 3/3)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_MSAN + uses: actions/download-artifact@v8 + with: + name: CH_AMD_MSAN + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, cas s3 storage, parallel, 3/3)' --workflow "Community PR" --ci --timestamp + + stateless_tests_arm_binary_cas_s3_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] + needs: [build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYmluYXJ5LCBjYXMgczMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (arm_binary, cas s3 storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (arm_binary, cas s3 storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_ARM_BIN + uses: actions/download-artifact@v8 + with: + name: CH_ARM_BIN + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, cas s3 storage, parallel)' --workflow "Community PR" --ci --timestamp + + stateless_tests_amd_binary_cas_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_binary, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBjYXMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (amd_binary, cas storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_binary, cas storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_BINARY + uses: actions/download-artifact@v8 + with: + name: CH_AMD_BINARY + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas storage, parallel)' --workflow "Community PR" --ci --timestamp + integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan_ubsan, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] diff --git a/.github/workflows/release_builds.yml b/.github/workflows/release_builds.yml index 8f6f6f22525d..5e943c0cd19e 100644 --- a/.github/workflows/release_builds.yml +++ b/.github/workflows/release_builds.yml @@ -1334,9 +1334,162 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, sequential)' --workflow "Release Builds" --ci --timestamp + stateless_tests_amd_binary_cas_s3_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBjYXMgczMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (amd_binary, cas s3 storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_binary, cas s3 storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_BINARY_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_BINARY_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas s3 storage, parallel)' --workflow "Release Builds" --ci --timestamp + + stateless_tests_arm_binary_cas_s3_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c] + needs: [build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYmluYXJ5LCBjYXMgczMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (arm_binary, cas s3 storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (arm_binary, cas s3 storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_ARM_BIN_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_ARM_BIN_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, cas s3 storage, parallel)' --workflow "Release Builds" --ci --timestamp + + stateless_tests_amd_binary_cas_storage_parallel: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c] + needs: [build_amd_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBjYXMgc3RvcmFnZSwgcGFyYWxsZWwp') }} + name: "Stateless tests (amd_binary, cas storage, parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stateless tests (amd_binary, cas storage, parallel)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_BINARY_GH + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: CH_AMD_BINARY_GH + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas storage, parallel)' --workflow "Release Builds" --ci --timestamp + finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, sign_release_amd_release, sign_release_arm_release, source_upload, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] + needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, sign_release_amd_release, sign_release_arm_release, source_upload, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -1433,6 +1586,9 @@ jobs: - source_upload - stateless_tests_arm_binary_parallel - stateless_tests_arm_binary_sequential + - stateless_tests_amd_binary_cas_s3_storage_parallel + - stateless_tests_arm_binary_cas_s3_storage_parallel + - stateless_tests_amd_binary_cas_storage_parallel - finish_workflow - GrypeScanServer - GrypeScanKeeper diff --git a/ci/defs/altinity_jobs.py b/ci/defs/altinity_jobs.py index 1d736abaa17e..ab7db9c881be 100644 --- a/ci/defs/altinity_jobs.py +++ b/ci/defs/altinity_jobs.py @@ -1,6 +1,7 @@ from praktika import Artifact, Job from ci.defs.defs import TEMP_DIR, ArtifactNames, RunnerLabels +from ci.defs.job_configs import common_ft_job_config class AltinityArtifactNames: @@ -69,3 +70,53 @@ class AltinityJobConfigs: command="python3 ./ci/jobs/source_upload.py", timeout=3600, ) + # Stateless tests with a content-addressed disk as the default MergeTree storage. + cas_functional_tests_jobs = common_ft_job_config.parametrize( + # CAS over S3: RustFS, not MinIO OSS, because the incarnation pool needs + # enforced conditional deletes. + Job.ParamSet( + parameter="amd_binary, cas s3 storage, parallel", + runs_on=RunnerLabels.AMD_MEDIUM_CPU, + requires=[ArtifactNames.CH_AMD_BINARY_GH], + ), + # The sanitizer lanes are sharded because an unsharded one exceeds the 6h + # GitHub job timeout and is killed before it uploads any results. + *[ + Job.ParamSet( + parameter=f"amd_asan_ubsan, cas s3 storage, parallel, {batch}/{total_batches}", + runs_on=RunnerLabels.AMD_MEDIUM_CPU, + requires=[ArtifactNames.CH_AMD_ASAN_UBSAN_GH], + ) + for total_batches in (2,) + for batch in range(1, total_batches + 1) + ], + *[ + Job.ParamSet( + parameter=f"amd_tsan, cas s3 storage, parallel, {batch}/{total_batches}", + runs_on=RunnerLabels.AMD_MEDIUM, + requires=[ArtifactNames.CH_AMD_TSAN_GH], + ) + for total_batches in (2,) + for batch in range(1, total_batches + 1) + ], + *[ + Job.ParamSet( + parameter=f"amd_msan, cas s3 storage, parallel, {batch}/{total_batches}", + runs_on=RunnerLabels.FUNC_TESTER_AMD, + requires=[ArtifactNames.CH_AMD_MSAN_GH], + ) + for total_batches in (3,) + for batch in range(1, total_batches + 1) + ], + Job.ParamSet( + parameter="arm_binary, cas s3 storage, parallel", + runs_on=RunnerLabels.ARM_MEDIUM_CPU, + requires=[ArtifactNames.CH_ARM_BINARY_GH], + ), + # CAS over local object storage. + Job.ParamSet( + parameter="amd_binary, cas storage, parallel", + runs_on=RunnerLabels.AMD_MEDIUM_CPU, + requires=[ArtifactNames.CH_AMD_BINARY_GH], + ), + ) diff --git a/ci/jobs/functional_tests.py b/ci/jobs/functional_tests.py index fcde1575154b..d66150839527 100644 --- a/ci/jobs/functional_tests.py +++ b/ci/jobs/functional_tests.py @@ -142,6 +142,8 @@ def run_tests( "old analyzer": "--analyzer", "WasmEdge": "--wasm-engine wasmedge", "s3 storage": "--s3-storage", + "cas storage": "--cas-storage", + "cas s3 storage": "--cas-s3-storage", "DatabaseReplicated": "--db-replicated", "DatabaseOrdinary": "--db-ordinary", "wide parts enabled": "--wide-parts", @@ -155,6 +157,8 @@ def run_tests( OPTIONS_TO_TEST_RUNNER_ARGUMENTS = { "s3 storage": "--s3-storage --no-stateful", + "cas storage": "--cas-storage", + "cas s3 storage": "--cas-s3-storage", "ParallelReplicas": "--no-zookeeper --no-shard --no-parallel-replicas", "AsyncInsert": " --no-async-insert", "DatabaseReplicated": " --no-stateful --replicated-database", @@ -242,6 +246,7 @@ def main(): is_targeted_check = False is_bugfix_validation = False is_s3_storage = False + is_cas_s3 = False is_azure_storage = False is_database_replicated = False is_shared_catalog = False @@ -295,8 +300,13 @@ def main(): is_excluded_from_llvm = True if "per_test_coverage" in to: is_per_test_coverage = True - if "s3 storage" in to: + if "s3 storage" in to and "cas" not in to: + # The CAS-over-s3 variant ("cas s3 storage") installs + # only its own default policy and must not pull in the s3 stateful-data / encrypted + # storage machinery, so it is deliberately excluded from is_s3_storage. is_s3_storage = True + if "cas s3 storage" in to: + is_cas_s3 = True if "azure" in to: is_azure_storage = True if "DatabaseReplicated" in to: @@ -637,6 +647,14 @@ def main(): def start(): res = CH.start_minio(test_type="stateless") and CH.start_azurite() + if res and is_cas_s3: + # The CA-over-S3 pool lives on RustFS (M-W D-W8): the incarnation pool + # needs ENFORCED conditional deletes, which MinIO OSS lacks (the + # fail-closed capability probe rejects it). start_rustfs wipes its data + # dir per run, so no pool state bleeds between runs (the local-CA + # analogue is the per-run server-store wipe). MinIO keeps the non-CA + # s3 disks. + res = CH.start_rustfs() res = res and CH.start() res = res and CH.wait_ready() if res: diff --git a/ci/jobs/scripts/check_style/various_checks.sh b/ci/jobs/scripts/check_style/various_checks.sh index 3660b8acd193..1d208fff563d 100755 --- a/ci/jobs/scripts/check_style/various_checks.sh +++ b/ci/jobs/scripts/check_style/various_checks.sh @@ -230,6 +230,32 @@ done # CLICKHOUSE_URL already includes "?" git grep -P 'CLICKHOUSE_URL(|_HTTPS)(}|}/|/|)\?' $ROOT_PATH/tests/queries/0_stateless/*.sh && echo "CLICKHOUSE_URL already includes '?', use '&' to append query parameters" +# A bare double quote inside a `-q """ ... """` block ends the string early. +# Bash treats """ as an empty string followed by an open quote, so everything up to the next quote is +# one argument. A quote anywhere inside -- including in a SQL comment -- closes it there, and the rest +# of the block becomes shell words. The script stays syntactically valid, so `bash -n` and shellcheck +# both pass; the only symptom is that the client receives a truncated query and reports a syntax error +# pointing at whatever followed the quote. Escape it as \" or use single quotes. +python3 - "$ROOT_PATH" <<'PYEOF' +import glob, os, re, sys + +for path in sorted(glob.glob(os.path.join(sys.argv[1], "tests/queries/0_stateless/*.sh"))): + inside = False + with open(path, encoding="utf-8", errors="replace") as handle: + for number, line in enumerate(handle, 1): + if not inside: + if re.search(r'(-q|--query)\s+"""\s*$', line.strip()): + inside = True + continue + # The block can close mid-line (`... LIMIT 10;"""`), so only what precedes the + # closing delimiter is still inside it. + body, closed, _ = line.partition('"""') + if '"' in re.sub(r'\\"', "", body): + print(f"{path}:{number}: bare double quote inside a -q \"\"\" block ends the SQL early") + if closed: + inside = False +PYEOF + # Large files checked into git. # Every byte committed is cloned by every contributor forever and cannot be removed without history rewriting. # Binary blobs (JARs, archives, .so, datasets) should be downloaded at test time or built from source. diff --git a/ci/jobs/scripts/clickhouse_proc.py b/ci/jobs/scripts/clickhouse_proc.py index 88a4e56c97ba..cbc8ad5984b4 100644 --- a/ci/jobs/scripts/clickhouse_proc.py +++ b/ci/jobs/scripts/clickhouse_proc.py @@ -9,6 +9,7 @@ import threading import traceback import uuid +import zipfile from collections import defaultdict from pathlib import Path from typing import List @@ -43,6 +44,7 @@ class ClickHouseProc: MINIO_LOG = f"{temp_dir}/minio.log" AZURITE_LOG = f"{temp_dir}/azurite.log" KAFKA_LOG = f"{temp_dir}/kafka.log" + RUSTFS_LOG = f"{temp_dir}/rustfs.log" LOGS_SAVER_CLIENT_OPTIONS = "--max_memory_usage 10G --max_threads 1 --max_rows_to_read=0 --max_result_rows 0 --max_result_bytes 0 --max_bytes_to_read 0 --max_execution_time 0 --max_execution_time_leaf 0 --max_estimated_execution_time 0" DMESG_LOG = f"{temp_dir}/dmesg.log" # TODO: run servers in dedicated wds to keep trash localised @@ -163,6 +165,77 @@ def start_minio(self, test_type): print("Failed to start minio") return False + RUSTFS_VERSION = "1.0.0-beta.12" + + def download_rustfs(self, rustfs_bin): + machine = platform.machine() + if machine not in ("x86_64", "aarch64", "arm64"): + print(f"unsupported architecture for rustfs [{machine}]") + return False + arch = "aarch64" if machine in ("aarch64", "arm64") else "x86_64" + url = ( + f"https://github.com/rustfs/rustfs/releases/download/{self.RUSTFS_VERSION}" + f"/rustfs-linux-{arch}-musl-v{self.RUSTFS_VERSION}.zip" + ) + zip_path = f"{temp_dir}/rustfs.zip" + if not Shell.check( + f"curl -sSfL --retry 3 --retry-delay 5 -o {zip_path} {url}", verbose=True + ): + print(f"failed to download rustfs from {url}") + return False + # The release zip contains the single `rustfs` binary at its root. + with zipfile.ZipFile(zip_path) as archive: + archive.extract("rustfs", temp_dir) + os.remove(zip_path) + os.chmod(rustfs_bin, 0o755) + return True + + def start_rustfs(self): + # RustFS backs the CAS-over-S3 pool because the incarnation pool needs enforced + # conditional operations (a wrong-token DELETE must fail with 412) that MinIO OSS lacks; + # MinIO keeps serving the non-CAS s3 disks on its own port. Binary and data dir live + # under ci/tmp, which CI wipes per run, so no pool state bleeds between runs. + rustfs_bin = f"{temp_dir}/rustfs" + if not Path(rustfs_bin).is_file() and not self.download_rustfs(rustfs_bin): + print(f"rustfs binary not found at {rustfs_bin} and download failed") + return False + data_dir = f"{temp_dir}/rustfs_data" + Shell.check(f"rm -rf {data_dir} && mkdir -p {data_dir}", verbose=True) + # The background data-scanner and auto-heal manager do no useful work on a single-disk + # ephemeral pool, but their namespace locks produced multi-minute bursts of 503 + # ServiceUnavailable that stalled client I/O. Client GET/PUT/LIST/DELETE do not depend on + # either. The RUSTFS_ENABLE_* spellings are deprecated since 1.0.0-beta.8. + # Raise the open-files limit for the same reason start_azurite does: under parallel load + # the server holds thousands of S3 connections, and at the default soft limit (1024) + # rustfs runs out of fds and refuses new TCP connections in bursts. + command = ( + "(ulimit -n 1048576 2>/dev/null || ulimit -n $(ulimit -Hn)) && " + f"RUSTFS_SCANNER_ENABLED=false RUSTFS_HEAL_ENABLED=false " + f"{rustfs_bin} server --address 0.0.0.0:11121 " + f"--access-key clickhouse --secret-key clickhouse {data_dir}" + ) + with open(self.RUSTFS_LOG, "w") as log_file: + self.rustfs_proc = subprocess.Popen( + command, stdout=log_file, stderr=subprocess.STDOUT, shell=True + ) + print(f"Started rustfs asynchronously with PID {self.rustfs_proc.pid}") + + if not Shell.check( + "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:11121/ | grep -qE '403|200'", + verbose=False, + retries=6, + ): + print("Failed to start rustfs") + return False + # The `test` bucket the storage policy expects. + res = Shell.check( + "/mc alias set carustfs http://localhost:11121 clickhouse clickhouse && /mc mb --ignore-existing carustfs/test", + verbose=True, + ) + if not res: + print("Failed to create rustfs test bucket") + return res + def start_azurite(self): # Raise the open files limit before launching azurite-rs. # Each concurrent test query opens a TCP connection plus an in-memory @@ -938,6 +1011,8 @@ def prepare_logs(self, info, all=False): res.append(self.AZURITE_LOG) if Path(self.KAFKA_LOG).exists(): res.append(self.KAFKA_LOG) + if Path(self.RUSTFS_LOG).exists(): + res.append(self.RUSTFS_LOG) if Path(self.DMESG_LOG).exists(): res.append(self.DMESG_LOG) if Path(self.CH_LOCAL_ERR_LOG).exists(): @@ -1211,6 +1286,29 @@ def dump_system_tables(self): Shell.check( f"sed -i 's|.*|{self.CH_LOCAL_ERR_LOG}|' /etc/clickhouse-server/config.xml" ) + # Open any CAS disk read-only: a writable open claims server-root ownership and fails + # closed against the real server's persisted owner uuid, while a read-only open skips the + # claim and is all a dump needs. Keyed on the `cas` marker + # rather than on disk names, so it covers every CAS disk however this job names it. + # `grep -R` and `sed --follow-symlinks` are required: `tests/config/install.sh` symlinks + # these configs into `config.d`, and `-r`/plain `sed` would silently match nothing. + Shell.check( + "grep -Rl 'cas' /etc/clickhouse-server/ 2>/dev/null " + "| xargs -r sed -i --follow-symlinks 's|cas|castrue|g'" + ) + # Report loudly if the substitution stops matching: a declared but not read-only CAS disk + # means this scrape is about to die on ownership. Reports; does not abort the dump. + if Shell.check( + "grep -Rlq 'cas' /etc/clickhouse-server/", + verbose=False, + ) and not Shell.check( + "grep -Rlq 'castrue' /etc/clickhouse-server/", + verbose=False, + ): + print( + "WARNING: a CAS disk is declared but the read-only marker was not inserted " + "-- `clickhouse local` will claim server-root ownership and this scrape will fail" + ) # FIXME: Hack for s3_with_keeper (note, that we don't need the disk, # the problem is that whenever we need disks all disks will be # initialized [1]) @@ -1226,8 +1324,12 @@ def dump_system_tables(self): self.restore_system_metadata_files_from_remote_database_disk() + # `**`, not `*`: dynamic cache disks created by tests nest their path, e.g. + # `filesystem_caches/disks/cache_03517/status` — a one-level glob missed exactly that file, + # and the scrape died on its flock (`StatusFile.cpp` "Another server instance ... is already + # running") when the server had not released it. cache_status_files = glob.glob( - f"{self.ch_var_lib_dir}/filesystem_caches/*/status" + f"{self.ch_var_lib_dir}/filesystem_caches/**/status", recursive=True ) if cache_status_files: print( diff --git a/ci/workflows/backport_branches.py b/ci/workflows/backport_branches.py index 528a47cbe32e..5f891943d5c9 100644 --- a/ci/workflows/backport_branches.py +++ b/ci/workflows/backport_branches.py @@ -1,9 +1,15 @@ from praktika import Workflow from ci.defs.defs import DOCKERS, SECRETS, ArtifactConfigs +from ci.defs.altinity_jobs import AltinityJobConfigs from ci.defs.job_configs import JobConfigs from ci.jobs.scripts.workflow_hooks.filter_job import should_skip_job +FUNCTIONAL_TESTS_JOBS = [ + *JobConfigs.functional_tests_jobs, + *AltinityJobConfigs.cas_functional_tests_jobs, +] + workflow = Workflow.Config( name="BackportPR", event=Workflow.Event.PULL_REQUEST, @@ -24,7 +30,7 @@ JobConfigs.docker_keeper, *JobConfigs.install_check_jobs, *JobConfigs.compatibility_test_jobs, - *[job for job in JobConfigs.functional_tests_jobs if "amd_asan_ubsan" in job.name], + *[job for job in FUNCTIONAL_TESTS_JOBS if "amd_asan_ubsan" in job.name], *[ job for job in JobConfigs.unittest_jobs diff --git a/ci/workflows/master.py b/ci/workflows/master.py index b3ffe686f331..b98ffb96fa66 100644 --- a/ci/workflows/master.py +++ b/ci/workflows/master.py @@ -13,6 +13,11 @@ from ci.jobs.scripts.workflow_hooks.filter_job import should_skip_job from ci.workflows.pull_request import REGULAR_BUILD_NAMES +FUNCTIONAL_TESTS_JOBS = [ + *JobConfigs.functional_tests_jobs, + *AltinityJobConfigs.cas_functional_tests_jobs, +] + # Add long retention tags to subset of artifacts clickhouse_binaries_with_tags = [] for artifact in ArtifactConfigs.clickhouse_binaries + ArtifactConfigs.clickhouse_stripped_binaries: @@ -55,7 +60,7 @@ *JobConfigs.compatibility_test_jobs, *[ j - for j in JobConfigs.functional_tests_jobs + for j in FUNCTIONAL_TESTS_JOBS if "coverage" not in j.name ], # *JobConfigs.functional_test_llvm_coverage_jobs, diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py index ae88a3b3a244..a3d063400716 100644 --- a/ci/workflows/pull_request.py +++ b/ci/workflows/pull_request.py @@ -14,11 +14,16 @@ from ci.jobs.scripts.workflow_hooks.filter_job import should_skip_job from ci.jobs.scripts.workflow_hooks.trusted import can_be_tested -ALL_FUNCTIONAL_TESTS = [job.name for job in JobConfigs.functional_tests_jobs] +FUNCTIONAL_TESTS_JOBS = [ + *JobConfigs.functional_tests_jobs, + *AltinityJobConfigs.cas_functional_tests_jobs, +] + +ALL_FUNCTIONAL_TESTS = [job.name for job in FUNCTIONAL_TESTS_JOBS] FUNCTIONAL_TESTS_PARALLEL_BLOCKING_JOB_NAMES = [ job.name - for job in JobConfigs.functional_tests_jobs + for job in FUNCTIONAL_TESTS_JOBS if any( substr in job.name for substr in ( @@ -40,7 +45,7 @@ REGULAR_BUILD_NAMES = [job.name for job in JobConfigs.build_jobs] PLAIN_FUNCTIONAL_TEST_JOB = [ - j for j in JobConfigs.functional_tests_jobs if "amd_debug, parallel" in j.name + j for j in FUNCTIONAL_TESTS_JOBS if "amd_debug, parallel" in j.name ][0] workflow = Workflow.Config( @@ -91,7 +96,7 @@ if j.name not in FUNCTIONAL_TESTS_PARALLEL_BLOCKING_JOB_NAMES else [] ) - for j in JobConfigs.functional_tests_jobs + for j in FUNCTIONAL_TESTS_JOBS if "coverage" not in j.name ], *[ diff --git a/ci/workflows/pull_request_community.py b/ci/workflows/pull_request_community.py index af76e6ac9c4b..71fe9bbca8ed 100644 --- a/ci/workflows/pull_request_community.py +++ b/ci/workflows/pull_request_community.py @@ -2,12 +2,18 @@ from praktika import Workflow, Artifact from ci.defs.defs import BASE_BRANCH, DOCKERS, ArtifactConfigs, JobNames +from ci.defs.altinity_jobs import AltinityJobConfigs from ci.defs.job_configs import JobConfigs from ci.jobs.scripts.workflow_hooks.filter_job import should_skip_job +FUNCTIONAL_TESTS_JOBS = [ + *JobConfigs.functional_tests_jobs, + *AltinityJobConfigs.cas_functional_tests_jobs, +] + FUNCTIONAL_TESTS_PARALLEL_BLOCKING_JOB_NAMES = [ job.name - for job in JobConfigs.functional_tests_jobs + for job in FUNCTIONAL_TESTS_JOBS if any( substr in job.name for substr in ( @@ -24,7 +30,7 @@ ] PLAIN_FUNCTIONAL_TEST_JOB = [ - j for j in JobConfigs.functional_tests_jobs if "amd_debug, parallel" in j.name + j for j in FUNCTIONAL_TESTS_JOBS if "amd_debug, parallel" in j.name ][0] def _normalize_gh_aliases(items): @@ -74,7 +80,7 @@ def _normalize_gh_aliases(items): if j.name not in FUNCTIONAL_TESTS_PARALLEL_BLOCKING_JOB_NAMES else [] ) - for j in JobConfigs.functional_tests_jobs if 'coverage' not in j.name + for j in FUNCTIONAL_TESTS_JOBS if 'coverage' not in j.name ], *[ job.set_run_after(FUNCTIONAL_TESTS_PARALLEL_BLOCKING_JOB_NAMES) diff --git a/ci/workflows/release_branches.py b/ci/workflows/release_branches.py index 2969c8f10b30..78f43c15d600 100644 --- a/ci/workflows/release_branches.py +++ b/ci/workflows/release_branches.py @@ -1,9 +1,15 @@ from praktika import Workflow from ci.defs.defs import BINARIES_WITH_LONG_RETENTION, DOCKERS, SECRETS, ArtifactConfigs +from ci.defs.altinity_jobs import AltinityJobConfigs from ci.defs.job_configs import JobConfigs from ci.jobs.scripts.workflow_hooks.filter_job import should_skip_job +FUNCTIONAL_TESTS_JOBS = [ + *JobConfigs.functional_tests_jobs, + *AltinityJobConfigs.cas_functional_tests_jobs, +] + builds_for_release_branch = [ job for job in JobConfigs.build_jobs @@ -31,7 +37,7 @@ JobConfigs.docker_server, JobConfigs.docker_keeper, *JobConfigs.install_check_master_jobs, - *[job for job in JobConfigs.functional_tests_jobs if "asan" in job.name], + *[job for job in FUNCTIONAL_TESTS_JOBS if "asan" in job.name], *[job for job in JobConfigs.unittest_jobs if "fuzzer" not in job.name], *[ job diff --git a/ci/workflows/release_builds.py b/ci/workflows/release_builds.py index 5e6165f9a744..d11b1324968e 100644 --- a/ci/workflows/release_builds.py +++ b/ci/workflows/release_builds.py @@ -5,6 +5,11 @@ from ci.defs.job_configs import JobConfigs from ci.jobs.scripts.workflow_hooks.filter_job import should_skip_job +FUNCTIONAL_TESTS_JOBS = [ + *JobConfigs.functional_tests_jobs, + *AltinityJobConfigs.cas_functional_tests_jobs, +] + # Add long retention tags to subset of artifacts clickhouse_binaries_with_tags = [] for artifact in ArtifactConfigs.clickhouse_binaries + ArtifactConfigs.clickhouse_stripped_binaries: @@ -45,7 +50,7 @@ AltinityJobConfigs.source_upload_job, *[ job - for job in JobConfigs.functional_tests_jobs + for job in FUNCTIONAL_TESTS_JOBS if any(t in job.name for t in ("release", "binary")) ], ], diff --git a/tests/clickhouse-test b/tests/clickhouse-test index a8bca36f4d0e..9979a8619ff0 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -1352,6 +1352,7 @@ class FailureReason(enum.Enum): OBJECT_STORAGE = "object-storage" COVERAGE = "coverage" S3_STORAGE = "s3-storage" + CAS_STORAGE = "cas-storage" AZURE_BLOB_STORAGE = "azure-blob-storage" BUILD = "not running for current build" NO_PARALLEL_REPLICAS = "smth is not supported with parallel replicas" @@ -2877,18 +2878,32 @@ class TestCase: if tags and ("no-s3-storage" in tags) and args.s3_storage: return FailureReason.S3_STORAGE + if ( + tags + and ("no-cas-storage" in tags) + and args.cas_storage + ): + return FailureReason.CAS_STORAGE if tags and ("no-azure-blob-storage" in tags) and args.azure_blob_storage: return FailureReason.AZURE_BLOB_STORAGE if ( tags and ("no-object-storage" in tags) - and (args.azure_blob_storage or args.s3_storage) + and ( + args.azure_blob_storage + or args.s3_storage + or args.cas_storage + ) ): return FailureReason.OBJECT_STORAGE if ( tags and "no-object-storage-with-slow-build" in tags - and (args.s3_storage or args.azure_blob_storage) + and ( + args.s3_storage + or args.azure_blob_storage + or args.cas_storage + ) and BuildFlags.RELEASE not in args.build_flags ): return FailureReason.OBJECT_STORAGE @@ -5624,6 +5639,16 @@ def main(args): if args.s3_storage and (BuildFlags.RELEASE not in args.build_flags): args.no_random_settings = True + if args.cas_s3_storage: + # CA-over-S3 is a CAS disk AND it is backed by S3 (minio). + # Mirror the local-CA gating (no-cas-storage + no-object-storage) + # and additionally honour no-s3-storage, since this disk IS s3-backed. + # Done AFTER the no_random_settings block above on purpose: like the local-CA + # job we keep random (merge-tree) settings; we only want the s3/object-storage + # *gating*, not the s3 stateful-data / no-random-settings machinery. + args.cas_storage = True + args.s3_storage = True + if args.skip: args.skip = set(args.skip) @@ -6338,6 +6363,19 @@ def parse_args(): default=False, help="Run tests over s3 storage", ) + parser.add_argument( + "--cas-storage", + action="store_true", + default=False, + help="Run tests with a CAS disk as the default MergeTree storage", + ) + parser.add_argument( + "--cas-s3-storage", + action="store_true", + default=False, + help="Run tests with a CAS disk backed by S3 (minio) as the default MergeTree storage. " + "Implies --cas-storage (object-storage gating) and --s3-storage (s3-storage gating).", + ) parser.add_argument( "--distributed-cache", action="store_true", diff --git a/tests/config/config.d/cas_s3_storage_policy_for_merge_tree_by_default.xml b/tests/config/config.d/cas_s3_storage_policy_for_merge_tree_by_default.xml new file mode 100644 index 000000000000..c73dd01b2338 --- /dev/null +++ b/tests/config/config.d/cas_s3_storage_policy_for_merge_tree_by_default.xml @@ -0,0 +1,66 @@ + + + + + object_storage + s3 + cas + + stateless-ca-s3 + + http://localhost:11121/test/cas_s3/ + clickhouse + clickhouse + + cas_s3_scratch/ + + 1 + 5 + + + + cache + cas_s3 + cas_s3_cache/ + 5368709120 + + + + + +
+ cas_s3 +
+
+
+ + +
+ cas_s3_cache +
+
+
+
+
+ + cas_s3 + + +
diff --git a/tests/config/config.d/cas_storage_policy_for_merge_tree_by_default.xml b/tests/config/config.d/cas_storage_policy_for_merge_tree_by_default.xml new file mode 100644 index 000000000000..18a8aa0c824f --- /dev/null +++ b/tests/config/config.d/cas_storage_policy_for_merge_tree_by_default.xml @@ -0,0 +1,35 @@ + + + + + object_storage + local + cas + + stateless-ca-local + + cas_pool/ + + cas_scratch/ + + 1 + 5 + + + + + +
+ cas +
+
+
+
+
+ + cas + +
diff --git a/tests/config/install.sh b/tests/config/install.sh index 359cd9cfd6f8..3c3311a96271 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -29,6 +29,8 @@ while [[ "$#" -gt 0 ]]; do --fast-test) FAST_TEST=1 && EXPORT_S3_STORAGE_POLICIES=0 ;; --analyzer) USE_OLD_ANALYZER=1 ;; --s3-storage) EXPORT_S3_STORAGE_POLICIES=1 && USE_S3_STORAGE_FOR_MERGE_TREE=1 && RANDOMIZE_OBJECT_KEY_TYPE=1 ;; + --cas-storage) USE_CAS_STORAGE_FOR_MERGE_TREE=1 ;; + --cas-s3-storage) USE_CAS_S3_STORAGE_FOR_MERGE_TREE=1 ;; --parallel-rep) USE_PARALLEL_REPLICAS=1 ;; --db-replicated) USE_DATABASE_REPLICATED=1 ;; --distributed-plan) USE_DISTRIBUTED_PLAN=1 ;; @@ -373,6 +375,20 @@ if [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" == "1" ]]; then else ln -sf $SRC_PATH/config.d/s3_storage_policy_for_merge_tree_by_default.xml $DEST_SERVER_PATH/config.d/ fi +elif [[ "$USE_CAS_STORAGE_FOR_MERGE_TREE" == "1" ]]; then + # Content-addressed disk (over local object storage) as the default MergeTree policy. + # Installs ONLY this config so the suite runs with CA-as-default without disturbing + # other storage variants (no s3/azure default policies are set up in this mode). + ln -sf $SRC_PATH/config.d/cas_storage_policy_for_merge_tree_by_default.xml $DEST_SERVER_PATH/config.d/ +elif [[ "$USE_CAS_S3_STORAGE_FOR_MERGE_TREE" == "1" ]]; then + # Content-addressed disk over S3 (minio) as the default MergeTree policy. This is the + # real-S3 (north star) counterpart of --cas-storage, which uses local + # object storage. minio is started unconditionally by the stateless praktika job (see + # ci/jobs/functional_tests.py start_minio), so the disk just needs minio reachable at + # localhost:11111 with the `test` bucket and the clickhouse/clickhouse creds, which + # setup_minio.sh provides. Installs ONLY this policy to keep the variant clean (no s3/azure + # default policies are set up in this mode). + ln -sf $SRC_PATH/config.d/cas_s3_storage_policy_for_merge_tree_by_default.xml $DEST_SERVER_PATH/config.d/ elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then if [[ -n "$USE_ENCRYPTED_STORAGE" ]] && [[ "$USE_ENCRYPTED_STORAGE" -eq 1 ]]; then ln -sf $SRC_PATH/config.d/azure_encrypted_storage_policy_by_default.xml $DEST_SERVER_PATH/config.d/ diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index cc318eca54d0..b42b919113aa 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -158,6 +158,13 @@ SYSTEM RELOAD ASYNCHRONOUS METRICS ['RELOAD ASYNCHRONOUS METRICS'] GLOBAL SYSTEM SYSTEM RECONNECT ZOOKEEPER ['SYSTEM RECONNECT ZOOKEEPER','RECONNECT ZOOKEEPER'] GLOBAL SYSTEM SYSTEM RELOAD [] \N SYSTEM SYSTEM RESTART DISK ['SYSTEM RESTART DISK'] GLOBAL SYSTEM +SYSTEM CAS GC RUN ['SYSTEM CAS GC RUN'] GLOBAL SYSTEM +SYSTEM CAS GC REBUILD ['SYSTEM CAS GC REBUILD'] GLOBAL SYSTEM +SYSTEM CAS DROP POOL MEMBER ['SYSTEM CAS DROP POOL MEMBER'] GLOBAL SYSTEM +SYSTEM CAS FSCK ['SYSTEM CAS FSCK'] GLOBAL SYSTEM +SYSTEM CAS FORGET ['SYSTEM CAS FORGET'] GLOBAL SYSTEM +SYSTEM CAS GC STOP ['SYSTEM CAS GC STOP'] GLOBAL SYSTEM +SYSTEM CAS GC START ['SYSTEM CAS GC START'] GLOBAL SYSTEM SYSTEM WAIT BLOBS CLEANUP ['SYSTEM WAIT BLOBS CLEANUP'] GLOBAL SYSTEM SYSTEM MERGES ['SYSTEM STOP MERGES','SYSTEM START MERGES','STOP MERGES','START MERGES'] TABLE SYSTEM SYSTEM TTL MERGES ['SYSTEM STOP TTL MERGES','SYSTEM START TTL MERGES','STOP TTL MERGES','START TTL MERGES'] TABLE SYSTEM diff --git a/tests/queries/0_stateless/02253_empty_part_checksums.sh b/tests/queries/0_stateless/02253_empty_part_checksums.sh index ba45d7d0b177..af4e1d896658 100755 --- a/tests/queries/0_stateless/02253_empty_part_checksums.sh +++ b/tests/queries/0_stateless/02253_empty_part_checksums.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: zookeeper, no-replicated-database, no-shared-merge-tree +# Tags: zookeeper, no-replicated-database, no-shared-merge-tree, no-cas-storage +# no-cas-storage: test asserts system.parts.path is an absolute local FS path; on a cas disk the path is a relative object-storage key (orthogonal part-file path-shape) # no-replicated-database because it adds extra replicas # no-shared-merge-tree do something with parts on local fs # add_minmax_index_for_numeric_columns=0: Adds extra files, which changes the hashes diff --git a/tests/queries/0_stateless/02254_projection_broken_part.sh b/tests/queries/0_stateless/02254_projection_broken_part.sh index 04a0c4fb0a19..84f4ef8bfe53 100755 --- a/tests/queries/0_stateless/02254_projection_broken_part.sh +++ b/tests/queries/0_stateless/02254_projection_broken_part.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: long, zookeeper, no-shared-merge-tree +# Tags: long, zookeeper, no-shared-merge-tree, no-cas-storage +# no-cas-storage: test asserts system.parts.path is an absolute local FS path; on a cas disk the path is a relative object-storage key (orthogonal part-file path-shape) CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02255_broken_parts_chain_on_start.sh b/tests/queries/0_stateless/02255_broken_parts_chain_on_start.sh index 888ac73e4ab1..de16ba1a0bff 100755 --- a/tests/queries/0_stateless/02255_broken_parts_chain_on_start.sh +++ b/tests/queries/0_stateless/02255_broken_parts_chain_on_start.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: long, zookeeper, no-shared-merge-tree +# Tags: long, zookeeper, no-shared-merge-tree, no-cas-storage +# no-cas-storage: test asserts system.parts.path is an absolute local FS path; on a cas disk the path is a relative object-storage key (orthogonal part-file path-shape) CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02369_lost_part_intersecting_merges.sh b/tests/queries/0_stateless/02369_lost_part_intersecting_merges.sh index 68ff2222dfec..cc4a3b53b957 100755 --- a/tests/queries/0_stateless/02369_lost_part_intersecting_merges.sh +++ b/tests/queries/0_stateless/02369_lost_part_intersecting_merges.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: zookeeper, no-shared-merge-tree, long +# Tags: zookeeper, no-shared-merge-tree, long, no-cas-storage +# no-cas-storage: test asserts system.parts.path is an absolute local FS path; on a cas disk the path is a relative object-storage key (orthogonal part-file path-shape) # no-shared-merge-tree: depend on local fs CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/02370_lost_part_intersecting_merges.sh b/tests/queries/0_stateless/02370_lost_part_intersecting_merges.sh index dff5b5f5eccf..72a749f0ec8e 100755 --- a/tests/queries/0_stateless/02370_lost_part_intersecting_merges.sh +++ b/tests/queries/0_stateless/02370_lost_part_intersecting_merges.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: long, zookeeper, no-shared-merge-tree, no-parallel +# Tags: long, zookeeper, no-shared-merge-tree, no-parallel, no-cas-storage +# no-cas-storage: test asserts system.parts.path is an absolute local FS path; on a cas disk the path is a relative object-storage key (orthogonal part-file path-shape) # no-shared-merge-tree: depend on local fs (remove parts) CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/02444_async_broken_outdated_part_loading.sh b/tests/queries/0_stateless/02444_async_broken_outdated_part_loading.sh index ff756fb620e8..2f3f38cfba4c 100755 --- a/tests/queries/0_stateless/02444_async_broken_outdated_part_loading.sh +++ b/tests/queries/0_stateless/02444_async_broken_outdated_part_loading.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: long, zookeeper, no-shared-merge-tree +# Tags: long, zookeeper, no-shared-merge-tree, no-cas-storage +# no-cas-storage: test asserts system.parts.path is an absolute local FS path; on a cas disk the path is a relative object-storage key (orthogonal part-file path-shape) CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02486_truncate_and_unexpected_parts.sql b/tests/queries/0_stateless/02486_truncate_and_unexpected_parts.sql index 29946e315544..c9cec021a6ec 100644 --- a/tests/queries/0_stateless/02486_truncate_and_unexpected_parts.sql +++ b/tests/queries/0_stateless/02486_truncate_and_unexpected_parts.sql @@ -1,4 +1,3 @@ - create table rmt (n int) engine=ReplicatedMergeTree('/test/02468/{database}', '1') order by tuple() partition by n % 2 settings replicated_max_ratio_of_wrong_parts=0, max_suspicious_broken_parts=0, max_suspicious_broken_parts_bytes=0; create table rmt1 (n int) engine=ReplicatedMergeTree('/test/02468/{database}', '2') order by tuple() partition by n % 2 settings replicated_max_ratio_of_wrong_parts=0, max_suspicious_broken_parts=0, max_suspicious_broken_parts_bytes=0; diff --git a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh index 5648db32d189..fe3dbd37e4d2 100755 --- a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh +++ b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-random-settings, no-random-merge-tree-settings, no-encrypted-storage +# Tags: no-fasttest, no-random-settings, no-random-merge-tree-settings, no-encrypted-storage, no-cas-storage +# Tag no-cas-storage: the test uses an Ordinary database, whose BACKUP path goes via +# temporary hard links - not supported on a cas disk (Code 344 SUPPORT_IS_DISABLED; +# BACKUP/RESTORE, B16/B34). Re-checked on the T13 CA-S3 lane (2026-06-12): still fails for this reason. # Tag no-fasttest: requires S3 # Tag no-random-settings, no-random-merge-tree-settings: to avoid creating extra files like serialization.json, this test too exocit anyway diff --git a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh index 865a43d91ef3..023e6fd8c529 100755 --- a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh +++ b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-random-settings, no-random-merge-tree-settings, no-shared-merge-tree, no-encrypted-storage +# Tags: no-fasttest, no-random-settings, no-random-merge-tree-settings, no-shared-merge-tree, no-encrypted-storage, no-cas-storage +# no-cas-storage: BACKUP via temporary hard links is not supported on a cas disk (Code 344 SUPPORT_IS_DISABLED; BACKUP/RESTORE, B16/B34) # Tag no-fasttest: requires S3 # Tag no-random-settings, no-random-merge-tree-settings: to avoid creating extra files like serialization.json, this test too exocit anyway # Tag no-shared-merge-tree: use database ordinary diff --git a/tests/queries/0_stateless/03350_alter_table_fetch_partition_thread_pool.sql b/tests/queries/0_stateless/03350_alter_table_fetch_partition_thread_pool.sql index f5cbd809eef2..dacad1a30788 100644 --- a/tests/queries/0_stateless/03350_alter_table_fetch_partition_thread_pool.sql +++ b/tests/queries/0_stateless/03350_alter_table_fetch_partition_thread_pool.sql @@ -1,4 +1,5 @@ -- Tags: no-parallel, no-replicated-database, no-shared-merge-tree +-- no-cas-storage: FETCH PARTITION is supported on a cas disk (the gate is lifted, byte-fetch lands into detached/, see 05002), but this test fetches a 100-part partition CONCURRENTLY via the FETCH thread pool. Those parallel fetches all read-modify-write the SHARED "detached" ref object; the read side of that hot pointer object is not serialized against the truncating in-place rewrite of the local object storage, so a concurrent reader can see a torn ref/manifest (CANNOT_READ_ALL_DATA / NO_FILE_IN_DATA_PART). The atomic pointer-object publish needed to make concurrent fan-out safe is a deferred backlog item (B66a); single-part FETCH works (01650 + 05002). -- Tag: no-parallel - to avoid polluting FETCH PARTITION thread pool with other fetches -- Tag: no-replicated-database - replica_path is different diff --git a/tests/queries/0_stateless/03352_allow_suspicious_ttl.sql b/tests/queries/0_stateless/03352_allow_suspicious_ttl.sql index 5fd2bb3bf3a4..2a23baffe13c 100644 --- a/tests/queries/0_stateless/03352_allow_suspicious_ttl.sql +++ b/tests/queries/0_stateless/03352_allow_suspicious_ttl.sql @@ -1,4 +1,4 @@ - -- Tags: long, zookeeper +-- Tags: long, zookeeper -- Replicated diff --git a/tests/queries/0_stateless/03541_rename_column_start.sql b/tests/queries/0_stateless/03541_rename_column_start.sql index b5d4fa03f18a..0fa8af8b26b8 100644 --- a/tests/queries/0_stateless/03541_rename_column_start.sql +++ b/tests/queries/0_stateless/03541_rename_column_start.sql @@ -1,4 +1,4 @@ - -- Tags: zookeeper +-- Tags: zookeeper CREATE TABLE rmt (a UInt64, b UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/rmt', '1') diff --git a/tests/queries/0_stateless/03829_insert_deduplication_info_memory.sql b/tests/queries/0_stateless/03829_insert_deduplication_info_memory.sql index 2dc33bfd11c1..f03853a88cbe 100644 --- a/tests/queries/0_stateless/03829_insert_deduplication_info_memory.sql +++ b/tests/queries/0_stateless/03829_insert_deduplication_info_memory.sql @@ -8,9 +8,13 @@ DROP TABLE IF EXISTS t_dedup_memory; CREATE TABLE t_dedup_memory (x UInt32, fat FixedString(10000)) ENGINE = MergeTree ORDER BY x; -- 10 000 rows * 10 000 bytes FixedString ≈ 100 MB of column data. --- With the bug, original_block doubles this to ~200 MB, exceeding the limit. --- Without the bug, only the data columns are held, fitting within the limit. -SET max_memory_usage = '150M'; +-- With the bug, original_block doubles this to ~200 MB, which must exceed the limit. +-- Without the bug, only the data columns are held (peak ≈ 143 MB), which must fit under it. +-- The limit sits between those two peaks with enough headroom to absorb small, fixed write-path +-- buffering overhead (e.g. a content-addressed disk's insert path adds ~0.5 MB on top of the ~143 MB +-- baseline) so the test still runs unchanged on such a storage backend, while remaining well below the +-- ~200 MB doubling-bug peak it is meant to catch. +SET max_memory_usage = '170M'; INSERT INTO t_dedup_memory SELECT number, toString(number) FROM numbers(10000) SETTINGS max_insert_threads = 1, min_insert_block_size_rows = 0, min_insert_block_size_bytes = 0; diff --git a/tests/queries/0_stateless/04215_replicated_missing_covered_part_on_start.sh b/tests/queries/0_stateless/04215_replicated_missing_covered_part_on_start.sh index f7fb4b10817b..7c1a2df1c8fb 100755 --- a/tests/queries/0_stateless/04215_replicated_missing_covered_part_on_start.sh +++ b/tests/queries/0_stateless/04215_replicated_missing_covered_part_on_start.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Tags: long, zookeeper, no-shared-merge-tree +# Tags: long, zookeeper, no-shared-merge-tree, no-cas-storage # no-shared-merge-tree: depends on local fs +# no-cas-storage: test asserts system.parts.path is an absolute local FS path; on a cas disk the path is a relative object-storage key (orthogonal part-file path-shape) CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/04316_reader_executor_basic.sql b/tests/queries/0_stateless/04316_reader_executor_basic.sql index 950db07f0003..f063513d9199 100644 --- a/tests/queries/0_stateless/04316_reader_executor_basic.sql +++ b/tests/queries/0_stateless/04316_reader_executor_basic.sql @@ -1,9 +1,12 @@ --- Tags: no-distributed-cache, no-encrypted-storage +-- Tags: no-distributed-cache, no-encrypted-storage, no-cas-storage -- The executor does not implement the distributed cache or decryption, so it -- falls back on those storage configs and the activation check below would not -- hold. Those stages can't be turned off from the test (unlike async prefetch -- and the filesystem cache), so skip them; the test still runs on local disk and --- plain object storage where the executor engages. +-- plain object storage where the executor engages. Content-addressed storage +-- always adds a `file_view` stage (the payload is a byte window inside a shared +-- blob), which the executor falls back on the same way -- see +-- `ReadPipeline::tryBuildReaderExecutor` -- so it never engages there either. -- -- Smoke test for the experimental ReaderExecutor read path. Reads a MergeTree -- table with `use_reader_executor = 1`, checks the data comes back correct (full diff --git a/tests/queries/0_stateless/04327_reader_executor_metrics.sql b/tests/queries/0_stateless/04327_reader_executor_metrics.sql index 9ec8766021d8..fead6d70ab72 100644 --- a/tests/queries/0_stateless/04327_reader_executor_metrics.sql +++ b/tests/queries/0_stateless/04327_reader_executor_metrics.sql @@ -1,8 +1,10 @@ --- Tags: no-distributed-cache, no-encrypted-storage +-- Tags: no-distributed-cache, no-encrypted-storage, no-cas-storage -- Like 04316, the executor falls back on the distributed cache and decryption -- (which can't be disabled from the test), so its metrics would not be emitted on -- those storage configs. Skip them; the test still runs on local disk and plain --- object storage where the executor engages. +-- object storage where the executor engages. Content-addressed storage always +-- adds a `file_view` stage (byte window inside a shared blob), which the +-- executor falls back on the same way -- see `ReadPipeline::tryBuildReaderExecutor`. -- -- Checks that the experimental ReaderExecutor emits its observability metrics. -- Reads a MergeTree table with `use_reader_executor = 1` and verifies, via the diff --git a/tests/queries/0_stateless/04328_reader_executor_kpi_async_metric.sql b/tests/queries/0_stateless/04328_reader_executor_kpi_async_metric.sql index 32988b1ea311..71063e563c4d 100644 --- a/tests/queries/0_stateless/04328_reader_executor_kpi_async_metric.sql +++ b/tests/queries/0_stateless/04328_reader_executor_kpi_async_metric.sql @@ -1,7 +1,9 @@ --- Tags: no-distributed-cache, no-encrypted-storage +-- Tags: no-distributed-cache, no-encrypted-storage, no-cas-storage -- The executor falls back on the distributed cache and decryption (which can't be -- disabled from the test), so its metrics would not be emitted there; skip those --- configs (as in 04316 / 04327). +-- configs (as in 04316 / 04327). Content-addressed storage always adds a +-- `file_view` stage (byte window inside a shared blob), which the executor +-- falls back on the same way -- see `ReadPipeline::tryBuildReaderExecutor`. -- -- End-to-end check that the modeled-cost KPI asynchronous metric -- `ReaderExecutorModeledCostMsPerRequestedMiB` moves when the executor does work. From 14ecc1f9922a7be743c38a2d7a813ccbc87fc45c Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:41 +0200 Subject: [PATCH 29/30] CAS documentation: the Antalya CAS book The dedicated documentation set under docs/en/antalya/cas/: architecture (storage layout, manifests and refs, blob protocol, part lifecycle, read path, replication, mounts and leases, namespaces, garbage collection, correctness, design history) and operations (configuration, bucket requirements, monitoring, debugging, troubleshooting, migration). Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- docs/en/antalya/cas/architecture/backend.md | 100 +++++++ .../antalya/cas/architecture/blob-protocol.md | 232 +++++++++++++++ .../antalya/cas/architecture/correctness.md | 91 ++++++ .../cas/architecture/design-history.md | 61 ++++ .../cas/architecture/garbage-collection.md | 263 ++++++++++++++++++ docs/en/antalya/cas/architecture/index.md | 114 ++++++++ .../cas/architecture/manifests-and-refs.md | 260 +++++++++++++++++ .../cas/architecture/mounts-and-leases.md | 223 +++++++++++++++ .../en/antalya/cas/architecture/namespaces.md | 172 ++++++++++++ .../cas/architecture/part-lifecycle.md | 148 ++++++++++ docs/en/antalya/cas/architecture/read-path.md | 84 ++++++ .../antalya/cas/architecture/replication.md | 118 ++++++++ .../cas/architecture/storage-layout.md | 160 +++++++++++ docs/en/antalya/cas/bucket-requirements.md | 44 +++ docs/en/antalya/cas/configuration.md | 132 +++++++++ docs/en/antalya/cas/index.md | 89 ++++++ docs/en/antalya/cas/operations/debugging.md | 225 +++++++++++++++ docs/en/antalya/cas/operations/migration.md | 209 ++++++++++++++ docs/en/antalya/cas/operations/monitoring.md | 101 +++++++ .../antalya/cas/operations/troubleshooting.md | 29 ++ docs/en/antalya/cas/quick-start.md | 145 ++++++++++ docs/en/antalya/cas/roadmap.md | 108 +++++++ 22 files changed, 3108 insertions(+) create mode 100644 docs/en/antalya/cas/architecture/backend.md create mode 100644 docs/en/antalya/cas/architecture/blob-protocol.md create mode 100644 docs/en/antalya/cas/architecture/correctness.md create mode 100644 docs/en/antalya/cas/architecture/design-history.md create mode 100644 docs/en/antalya/cas/architecture/garbage-collection.md create mode 100644 docs/en/antalya/cas/architecture/index.md create mode 100644 docs/en/antalya/cas/architecture/manifests-and-refs.md create mode 100644 docs/en/antalya/cas/architecture/mounts-and-leases.md create mode 100644 docs/en/antalya/cas/architecture/namespaces.md create mode 100644 docs/en/antalya/cas/architecture/part-lifecycle.md create mode 100644 docs/en/antalya/cas/architecture/read-path.md create mode 100644 docs/en/antalya/cas/architecture/replication.md create mode 100644 docs/en/antalya/cas/architecture/storage-layout.md create mode 100644 docs/en/antalya/cas/bucket-requirements.md create mode 100644 docs/en/antalya/cas/configuration.md create mode 100644 docs/en/antalya/cas/index.md create mode 100644 docs/en/antalya/cas/operations/debugging.md create mode 100644 docs/en/antalya/cas/operations/migration.md create mode 100644 docs/en/antalya/cas/operations/monitoring.md create mode 100644 docs/en/antalya/cas/operations/troubleshooting.md create mode 100644 docs/en/antalya/cas/quick-start.md create mode 100644 docs/en/antalya/cas/roadmap.md diff --git a/docs/en/antalya/cas/architecture/backend.md b/docs/en/antalya/cas/architecture/backend.md new file mode 100644 index 000000000000..fab4d23e323b --- /dev/null +++ b/docs/en/antalya/cas/architecture/backend.md @@ -0,0 +1,100 @@ +--- +description: 'The Cas::Backend storage seam, its token contract, the per-provider conditional-write dialects, and the mount-time capability probe.' +sidebar_label: 'Backend abstraction' +sidebar_position: 11 +slug: /antalya/cas/architecture/backend +title: 'CAS Architecture — Backend Abstraction' +doc_type: 'reference' +--- + +# Backend abstraction {#backend-abstraction} + +Every protocol described elsewhere in this set — blobs, manifests, refs, mounts, GC — is written +against one interface, `Cas::Backend` (`Backend/CasBackend.h`). It is a token-aware storage seam: +every present key has exactly one current incarnation identified by an opaque `Token`, and +`putOverwrite`/`casPut` succeed only against the expected current token (or expected absence). + +## The interface {#interface} + +| Method | Contract | +|---|---| +| `get` / `getStream` | Read bytes (or a forward-only stream, for write-once objects) plus the token of the incarnation read | +| `head` | Existence, size, token, and metadata without reading the body | +| `putIfAbsent` / `putIfAbsentStream` | Create-if-absent (`If-None-Match: *`); `PreconditionFailed` is a returned outcome, never an exception | +| `putOverwrite` | Replace the current object only when its token equals `expected`; a mismatch is a returned outcome | +| `casPut` | `expected == nullopt` ⇒ create-if-absent CAS (used for the first write of a root object); a set `expected` conditionally replaces that exact incarnation | +| `deleteExact` | Delete only the incarnation named by `token`; a token mismatch (`TokenMismatch`) leaves the object untouched and is distinguished from `NotFound` | +| `list` | One page of keys under a prefix, resumed by the backend's own cursor | +| `supportsListTokens` | Whether `list` can surface a per-key incarnation token, letting GC discovery skip an unchanged root shard without a `GET` | +| `promoteStaged` / `resurrect` | `promoteStaged`: write-once server-side copy from S3 staging (optional, defaults to `NOT_IMPLEMENTED`). `resurrect`: unconditional re-upload displacing a condemned incarnation from a caller-supplied reader (streamed on remote object storage, materialized one-at-a-time on the local emulated mode); size-checked before publication, and fresh-tagged so pending deletes of the old incarnation cannot remove it | + +`deleteExact`, `putIfAbsent`/`putIfAbsentStream`, and `putOverwrite`/`casPut` are safety-critical: +they are what makes exact-token deletes, write-once creation, and mutual exclusion hold. Every +other method is protocol hygiene. + +**`TOKEN ⟹ CONTENT`** is the one contract item the capability probe cannot check: a token must +uniquely identify the byte content of the incarnation it labels, so that a repeated token never +means different bytes. The read-path decode cache skips a re-read on a token match, so a backend +that recycled tokens across different content would serve stale manifests — a wrong-result bug, not +merely inefficiency. `S3` `ETag`s are content-derived; the in-memory and emulated backends mint a +strictly monotonic sequence that is never reused. This remains a standing requirement of every +backend implementation, not a property the probe verifies. + +## Provider dialects {#dialects} + +`ObjectStorageBackend` (`Backend/CasObjectStorageBackend.cpp`) wraps one `IObjectStorage` and picks +its token dialect from `IObjectStorage::conditionalOpsUseGenerationTokens()`: + +| Dialect | Token type | How a conditional write is expressed | +|---|---|---| +| `AWS` (default) | `ETag` | `If-None-Match: *` / `If-Match: ` sent as-is | +| `GCS` | `Generation` | The backend rewrites conditional headers before the request goes out: `If-None-Match: *` becomes `x-goog-if-generation-match: 0`, and `If-Match: ` becomes `x-goog-if-generation-match: ` (`applyGcsConditionalDialectToRequest`, `IO/S3/GCSConditionalDialect.cpp`) | + +The GCS dialect is opted into by client configuration (`http_client = gcs_hmac` or `gcp_oauth`), not +auto-detected from the endpoint host. It also rejects one shape outright: a **conditional +`CompleteMultipartUpload`** throws rather than silently dropping the precondition, because GCS +ignores preconditions on that call — a measured, documented gap, not a hypothetical one. `CAS`'s +conditional writes therefore always take the single-`PUT` path on a generation-dialect backend. + +This bounds conditional writes only: the write-once create is therefore single-part and limited by +`gcs_max_conditional_put_bytes` on a generation dialect, while the unconditional resurrect takes the +ordinary multipart path and has no size limit on any backend. + +Every request carrying a rewritten header also has its AWS auth headers stripped and every +remaining `x-amz-*` header renamed to `x-goog-*`, since GCS rejects a mixed header set. + +Azure Blob Storage's REST API documents equivalent conditional headers (`If-None-Match`, +`If-Match`), but no third dialect exists in this backend yet — `IObjectStorage`'s Azure +implementation does not currently wire up a `CAS` conditional path, so Azure is untested by the +capability probe below, not merely a slower-verified third case. + +## Exact-token delete, per provider {#exact-token-delete} + +`deleteExact(key, token)` is realized as a conditional `DELETE` naming the token as a precondition: +an `If-Match`-style delete on `AWS` (`ETag`), a generation-match delete on `GCS`. A precondition +failure — `S3::isPreconditionFailedError` — is reported as `DeleteOutcome::TokenMismatch`, never as +an exception; the object is left untouched. `DeleteOutcome::created_delete_marker` reports whether +the backend created a delete marker instead of actually removing the object, which the capability +probe rejects: a bucket with versioning enabled would let `CAS` "delete" a blob without freeing any +storage, and GC would silently stop reclaiming. + +## The capability probe {#capability-probe} + +`runCapabilityProbe` (`Backend/CasProbe.cpp`) runs a throwaway-key battery against every writable +mount, described in full on the [bucket requirements](/antalya/cas/bucket-requirements) page. It is +fail-closed: any check that does not pass throws `NOT_IMPLEMENTED` naming the specific failure, and +the mount refuses to become writable. Two mount-time gates sit alongside it: + +- `checkPoolPreconditions` — on the `GCS`-dialect combination only, verifies bucket versioning is + off (a confirmed `Enabled` throws; an inconclusive check proceeds under an assumption, logged at + `WARNING`, that versioning is off). +- `checkConditionalWriteSingleAttemptSupport` — refuses to mount writable unless the underlying + object storage supports a single-HTTP-attempt retry profile for conditional writes. A hidden SDK + retry can outlive the writer's mount lease and obscure whether a conditional operation actually + committed, so retries on the conditional path must be explicit CAS state-machine transitions, not + transparent client behavior. + +A third, optional probe (`probeConditionalCopy`) checks whether the backend enforces a write-once +conditional server-side copy. It only matters for `staging_backend = s3`: when the probe reports +`false`, S3-native staging silently falls back to local staging rather than refusing to mount, since +enforcement here is an optimization, not a correctness requirement of the disk itself. diff --git a/docs/en/antalya/cas/architecture/blob-protocol.md b/docs/en/antalya/cas/architecture/blob-protocol.md new file mode 100644 index 000000000000..64a62511afba --- /dev/null +++ b/docs/en/antalya/cas/architecture/blob-protocol.md @@ -0,0 +1,232 @@ +--- +description: 'How CAS writes, deduplicates, and reclaims a blob: conditional-write sequencing, the writer-versus-GC race, and the deterministic-artifact adoption pin.' +sidebar_label: 'Blob protocol' +sidebar_position: 3 +slug: /antalya/cas/architecture/blob-protocol +title: 'CAS Architecture — Blob Protocol' +doc_type: 'reference' +--- + +# CAS architecture — blob protocol {#blob-protocol} + +A blob is the unit of content-addressed storage: one part file's bytes, keyed by a hash of its +own content. This page covers how a blob gets written exactly once, how a duplicate write is +turned into a no-op, and how a writer and a `GC` round racing over the same blob are kept safe +without ever comparing multi-gigabyte bodies. Object layout and the four durable object kinds +are covered on the [overview page](/antalya/cas/architecture/); `GC`'s fold and round structure +is covered on the GC page. + +## Conditional-write sequence {#conditional-write-sequence} + +Every blob body lives at a key derived purely from its content hash +(`blobs///`, `CasLayout::blobKey`), with a sidecar `.meta` object at the +same key plus `.meta`. Because the key already encodes the digest, the backend never needs a +compare-and-swap on content — only on *presence* (`PUT` with `If-None-Match: *`) or on a specific +prior incarnation (`PUT`/`DELETE` with `If-Match: `). + +```mermaid +sequenceDiagram + autonumber + participant Writer + participant S3 as Object store + + Writer->>Writer: hash source, derive key from digest + alt dedup cache hit OR size >= deduplication_head_first_min_bytes + Writer->>S3: HEAD blobs/algo/hex + alt body present + Writer->>S3: GET .meta (point read, body never streamed) + Writer->>Writer: adopt current token if Clean or absent + else body absent + Writer->>S3: putIfAbsentStream (If-None-Match: star) + end + else small, no cache hit + Writer->>S3: putIfAbsentStream (If-None-Match: star) directly + end + S3-->>Writer: Done -- fresh upload, write Clean meta + S3-->>Writer: PreconditionFailed -- someone occupies the key + opt on PreconditionFailed + Writer->>S3: HEAD blobs/algo/hex + Writer->>S3: GET .meta -- adopt the occupant's token as a dependency + end +``` + +Ordered steps (`Pool/CasPartWriteTxn.cpp:160-245` and `:427-779`): + +1. `requireAlive()` — the build is not abandoned, the namespace not dropped, the writer epoch + still live. +2. **Adaptive dedup gate.** `HEAD` first if the dedup cache reports the content present, or the + object is at least `deduplication_head_first_min_bytes` (default 1 MiB). Below that threshold + a speculative conditional `PUT` is cheaper than a `HEAD` plus a `PUT`. +3. On a `HEAD` hit, `observeAndAdmit` point-reads the `.meta` sidecar and adopts the live + incarnation — the body is never streamed for a dedup hit. +4. Otherwise a bounded retry loop (up to 8 attempts) around `uploadFromSource`, which mints a + **fresh `incarnation_tag` per attempt** and does either a conditional server-side `COPY` from + S3 staging or a streaming `putIfAbsentStream`. The byte count is verified against the declared + source size. +5. A 412 means someone occupies the key. Because the key embeds the content digest, **any + occupant is by definition the intended content** — ambiguity is resolved by one `HEAD` + (occupancy), never by comparing bodies. +6. `Unresolved` (timeout, 5xx, connection loss) never acks. It throws retry-later — nothing was + published, so a body that lands late is inert debris for the orphan sweep. + +**Two writers uploading identical content** both derive the same key and both send +`If-None-Match: *`. The object store serializes them: one gets `Done`, the other gets 412, +`HEAD`s, point-reads `.meta`, and adopts the winner's token as its own dependency. The loser never +published anything — a failed or cancelled sink publishes nothing — and its adopt is protected by +its own durable precommit edge (see [the writer-versus-GC race](#writer-gc-race)). Both writers +are safe; the only cost is one wasted upload attempt. + +## Dedup and the identity primitive {#dedup-identity} + +Two blobs are the same object if and only if they hash to the same digest under the pool's +configured algorithm. Nothing else — not size, not `LIST` order, not a cheap prefix compare — +is allowed to stand in for that check. This follows the same rule everywhere in CAS: identity +is *proven* by hash equality, never *inferred* by a cheap signal, and re-hashing on read is the +identity primitive wherever the correctness of a decision depends on it. + +The blob content hash is pluggable per pool, fixed at pool creation: `blob_hash` selects +`cityhash128` (default), `xxh3-128`, or `sha256` (`parseBlobHashAlgo`, +`Primitives/CasBlobDigest.h`). A blob is identified by the pair `BlobRef = (BlobHashAlgo, digest)`, +never by a bare digest — a bare digest is ambiguous once more than one algorithm can appear in a +pool. `blob_hash_allow_new` gates admitting a second algorithm into an already-populated pool's +`algos_used` set; it defaults to off. + +`cityhash128` is not cryptographically collision-resistant. A pool shared across mutually +untrusted writers should run `sha256` — CAS enforces no policy choice here; the operator picks +the threat model via `blob_hash`. This is why the dedup admission gate is a `HEAD` (occupancy) +rather than a body compare: it tells the writer *something* already claims this key, and the +digest is the only claim CAS trusts. + +## The writer-versus-GC race {#writer-gc-race} + +This is the interleaving that gets the most reviewer attention, because a writer and a `GC` round +can legitimately disagree about whether a blob is still needed. + +```mermaid +sequenceDiagram + autonumber + participant W as Writer + participant S3 as Object store + participant GC as GC leader + + Note over GC: round n -- fold finds in-degree 0 + GC->>S3: HEAD blob -- capture exact token t1 + GC->>S3: write .meta = Condemned round n + + rect rgba(120,160,255,0.12) + Note over W,S3: a writer arrives wanting this content + W->>S3: append ref-log PRECOMMIT (durable +1 edge) + W->>S3: HEAD blob (present, token t1) + W->>S3: GET .meta + alt meta is Clean + W->>W: adopt t1 as dependency + Note over GC: next fold sees in-degree >= 1 -- spared + else meta is Condemned + W->>S3: PUT blob unconditional re-upload of writer own source, fresh incarnation tag -- token t2 not t1 + W->>S3: CAS .meta back to Clean + end + end + + Note over GC: round n+1 -- graduation, only if still zero + GC->>S3: re-verify in-degree, requires confirmed durable Condemned evidence for hash+token t1 + Note over GC: publishes delete_pending + + Note over GC: round n+2 -- the single content-delete site + GC->>S3: deleteExact(blob, t1) + alt writer resurrected + S3-->>GC: TokenMismatch -- nothing deleted, blob is live at t2 + else genuinely dead + S3-->>GC: Deleted -- then drop the .meta + end +``` + +The invariant that makes every interleaving safe: **revival is re-upload only — never `GET` a +condemned object to revive it.** A writer that finds `Condemned` metadata does not resurrect the +existing body; it re-uploads its own source bytes under a fresh `incarnation_tag`, producing a +new token that no prior `deleteExact` call can name. `GC` never streams a body it might delete, +and a writer never trusts a body it did not itself just write. + +Why this closes the race in both directions: + +- A writer that **adopts** a token must have read a non-`Condemned` marker, and its precommit + edge was durable *before* that read. The next fold therefore sees in-degree ≥ 1 and spares the + blob. +- A writer that **resurrects** changes the token. A stale `deleteExact(t1)` then returns + `TokenMismatch` and reclaims nothing — the delete names an exact incarnation, never "the object + at this key". +- The delete lags condemnation by at least two full rounds, and publishing the one edge that + authorizes an irreversible delete requires confirmed durable `Condemned` evidence for that + exact `(hash, token)` pair. Without it `GC` never throws — it carries the entry and retries the + marker write on the next round. + +Both directions degrade to a spurious re-upload or a no-op delete. Neither can lose data or leave +a dangling manifest entry. + +**One asymmetry worth flagging:** on a local (emulated) disk the resurrect path materializes the +full `[header][payload]` in memory; resurrections are serialized, so at most one body is held whole +in RAM at a time. On remote object storage the resurrect streams and holds nothing. + +### The `.meta` sidecar {#meta-sidecar} + +`.meta` has exactly two states: `Clean` (body present, may be referenced) and `Condemned` +(`GC` observed zero in-degree; the body is still present and a writer may resurrect it). An +*absent* `.meta` reads exactly like `Clean` — there is no third "unaccounted" state in the +stored format; `unaccounted` is an `ca-fsck` classification, not something `GC` ever writes. + +The record carries `state`, `condemn_round`, and `size`, and deliberately carries **no token**: +it is a per-hash hint, not a per-incarnation fact. All safety comes from the body's in-envelope +`incarnation_tag` plus exact-token deletes; a stale marker costs at worst one spurious re-upload, +never a lost delete or a false revival. + +## Deterministic artifacts and the adoption pin {#deterministic-artifacts} + +Some CAS objects are a pure function of their inputs: the `GC` source-edge run files (`cas_run`) +and fold seals (`cas_fold_seal`). For these, `putDeterministicArtifact` +(`Gc/CasBlobInDegree.cpp:341-352`) is the write-once helper: + +```cpp +if (backend.putIfAbsent(key, bytes).outcome == PutOutcome::PreconditionFailed) +{ + const auto existing = backend.get(key); + if (!existing || existing->bytes != bytes) + throw Exception(ErrorCodes::CORRUPTED_DATA, ...); + /// byte-equal => our own deterministic replay; adopt (no-op). +} +``` + +The idempotency argument: identical inputs produce byte-identical output, so a replayed round — +leader deposed mid-round, round `CAS` aborted, crash-restart — re-derives exactly the same bytes. +A 412 therefore means "already occupied by our own replay", verified by comparing the fetched +bytes, not inferred from occupancy alone as blob uploads do. Divergent bytes are impossible under +correct operation and fail closed as `CORRUPTED_DATA`. + +This is the format-evolution **adoption pin**, documented in the persisted-format registry +(`Formats/README.md`): on a `putDeterministicArtifact` conflict, the writer re-encodes at the `v` +of the *existing* object rather than at its own current build's version, so two writers on +different builds replaying the same deterministic round still land on byte-identical output. + +The helper is explicitly **not** for observation-bearing artifacts — `GC` outcome logs carry +`HEAD`-observed tokens on which two observers may legitimately disagree, so those use +first-durable-write-wins byte-adopt semantics instead. And a blob body can never use this path: +the fresh-tag rule means two attempts at the same logical create are allowed to legitimately +differ, which is exactly what `putDeterministicArtifact`'s divergence check would reject. + +## Settings {#settings} + +All names below are unprefixed keys inside the disk's `cas` config block +(`ContentAddressedSettings.cpp`, `LIST_OF_CONTENT_ADDRESSED_SETTINGS`); none carry a `cas_`/`ca_` +prefix. + +| Setting | Controls | Default | +|---|---|---| +| `blob_hash` | Pool blob content-hash function (`cityhash128` \| `xxh3-128` \| `sha256`); fixed at pool creation | `cityhash128` | +| `blob_hash_allow_new` | Explicit opt-in to admit a new hash algorithm into an existing pool's `algos_used` | `false` | +| `deduplication_cache_bytes` | Byte budget of the blob-presence cache that feeds the dedup `HEAD`-first decision (`0` disables) | 64 MiB | +| `deduplication_head_first_min_bytes` | Minimum blob size to try a `HEAD` before uploading the body | 1 MiB | +| `staging_backend` | Blob staging backend (`local` \| `s3`); `s3` is opt-in | `local` | +| `scratch_path` | Server-local scratch directory for the local-staging write-buffer spill; a relative value is anchored to the server data path | `""` | +| `gcs_max_conditional_put_bytes` | Largest conditional write on a generation-token store (GCS forces those single-part); does not bound the unconditional resurrect | 1 GiB | + +`GC`-round budgets that gate condemnation and reclaim of these same blobs (graduation, redelete, +sweep budgets) live on the GC architecture page, not here — they govern the `GC` side of the race +in [Writer-versus-GC race](#writer-gc-race), not the write path. diff --git a/docs/en/antalya/cas/architecture/correctness.md b/docs/en/antalya/cas/architecture/correctness.md new file mode 100644 index 000000000000..1beb28b0039f --- /dev/null +++ b/docs/en/antalya/cas/architecture/correctness.md @@ -0,0 +1,91 @@ +--- +description: 'What the TLA+ model corpus proves about CAS safety, the counterexamples that shaped the design, and how model-checking and soak/chaos testing complement each other.' +sidebar_label: 'Correctness' +sidebar_position: 12 +slug: /antalya/cas/architecture/correctness +title: 'CAS Architecture — Correctness' +doc_type: 'reference' +--- + +# CAS architecture — correctness {#correctness} + +`CAS` treats formal modelling as a pre-implementation gate, not after-the-fact documentation. No +task that changes safety-relevant behavior starts until the relevant `TLA+` model is green, and +"green" means every safety and liveness stage holds **and** every deliberately sabotaged variant +(`sab_*`) violates the specific rule it targets. A sabotage that fails to reproduce its named +counterexample is treated as seriously as a real violation — it means the model was not actually +covering the case it claimed to cover. This is why every safety rule below ships with the +counterexample that appears when you remove it. + +The full model index (source `.tla` files and proof-run records) lives at +`docs/superpowers/models/`; this page is the reader-facing summary. + +## Model → invariant → counterexample {#model-invariant-counterexample} + +| Model (`docs/superpowers/models/`) | Invariant it proves | Counterexample it caught | +|---|---|---| +| `CaIncarnationCore.tla` | `INV_NO_DANGLE`, `INV_NO_LOSS`, `INV_NO_RETURN` — the safety spine for the whole GC core | `sab_unconddelete`: replacing the exact-token delete with an unconditional one lets a stale delete kill the live incarnation a resurrect just wrote | +| `CaBuildRootPrecommit.tla` | `INV_NO_DANGLE_COMMITTED` — a committed manifest never references an absent blob | Reproduces the dangling-manifest hazard exactly: `WriteBlob → AdoptBlob → BuildDie → GcDelete → Commit` with no presence re-check publishes a manifest over a deleted blob | +| `CaGcLeaseCore.tla` | `NoFalseSteal` — no leader steals leadership from a live, mid-round incumbent | Without the advisory heartbeat, a frozen `seq` during a round looks identical to a dead leader, and a second leader steals from the alive one | +| `CaCasMountCore.tla` | Reclaim exclusivity for an expired mount | `sab_wallclockreclaim`: trusting the foreign mount body's wall-clock timestamp (instead of observing a stable token on the reclaimer's own monotonic clock) breaks exclusivity | +| `CaB140DangleMerge.tla` | `INV_NO_LOSS` across a GC lease handoff | Trim-before-durable: a fold cursor trimmed from in-memory state (not the durable snapshot) skips a live edge across a lease handoff, and the referenced blob is deleted while still live | +| `CaGcRootLocalPartManifestCore.tla` | `INV_NO_DANGLE` over the root-local part-manifest fold | `sab_lazyfenceunsafe`: reusing a stale parent fence position instead of a fresh all-shard fence dangles a live object | +| `CaGcShardIncarnationCore.tla` | `INV_NO_DANGLING` — safety of registry-free namespace discovery | `sab_pathkeyedcursor`: dropping the per-shard incarnation from the fold cursor reintroduces an ABA hazard on delete-then-recreate at the same path | +| `CaGcAckFloorZombie.tla` | `INV_NO_DANGLE` under two fully-interleaved GC leaders | `sab_eagerdelete`: a leader deleting its own fresh (not-yet-pending) graduations — the pre-amendment single-phase behavior — dangles when a deposed leader's pass overlaps a live one | +| `CaGcRoundDeferCore.tla` | `NoOverDelete` — a deferred round may skip a rebuild only when nothing destructive is pending | `sab_graduate_on_stale`: dropping the "an unfolded delta covers this blob" guard lets a deferred round delete a blob its own unread history still protects | +| `CaEdgeBeforeObserve.tla` | The writer/`GC` publish order is safe to simplify | `sab_late_edge`: allowing adoption before the precommit closure is durable (the pre-fix order) dangles | +| `CaGcCondemnMarkerGate.tla` | Graduation requires confirmed durable `Condemned` evidence | Swallowing a failed asynchronous condemn-marker write let a writer adopt a token a later graduation was about to delete | +| `CaRefTableSnapshotLogCore.tla` | Dense, per-life ref ids with an in-band `_ckpt` recovery frontier | `sab_scanistruth`: trusting a listing as the source of truth for "acked" reproduces the real production incident where a `LIST` omitted two already-durable, already-acknowledged ref entries | + +## Soak and chaos: the empirical oracle {#soak-and-chaos} + +Model-checking and the soak/chaos harness (`utils/ca-soak/`) catch different classes of error, and +the design leans on both rather than either alone. `TLA+` proves a protocol's constraints before a +line of `C++` exists — the two-coordinate namespace-incarnation proof and the build-root necessity +proof were both design decisions made this way. The soak, running two `ReplicatedMergeTree` +replicas against one shared pool under a seeded workload and a seeded fault injector, finds what an +idealized model necessarily abstracts away: the dangling-manifest hazard and the resurrect-reupload orphan were +both first observed live in `system.cas_log` during soak runs, before either got a focused model. Each +quiesced soak checkpoint cross-checks `SQL` results against a model oracle and runs +`clickhouse-disks ca-fsck` plus `ca-gc-dryrun`, asserting `dangling=0`. + +The relationship runs in both directions: the resurrect-reupload orphan (`utils/ca-soak` scenario +S30, root-caused via `system.cas_log`) got a focused `TLA+` reproduction that proved the fix and +was then retired once a deterministic `gtest` (`CASGCLeak.ResurrectReplacedIncarnationReclaimed`) +covered the same scenario for less ongoing cost — the model did its job as a pre-implementation +gate and the regression coverage moved to the cheaper, faster tool. A model's proven-safe shape +also becomes the thing a later soak scenario is written to stress. The `0x1430c` +incident — a `LIST` that omitted two already-durable ref entries, caught live by an instrumented +probe rather than reproduced by brute-force enumeration — is the clearest example: it is what made +`sab_scanistruth` a permanent, named counterexample rather than a one-off incident report. + +## What this buys a reader {#what-this-buys} + +None of this proves the shipped `C++` is bug-free — a model proves its own abstraction, and several +entries in the index are annotated `MIXED` or `DRIFTED` where the concrete mechanism has moved on +from what a model checks, with the audit trail kept precisely so that gap is visible rather than +implied. What it does buy: every safety rule in the GC core has an explicit counterexample on +record for the world where that rule is missing, and the corpus is itself periodically re-audited +for faithfulness to the code — a model whose guarantee the code no longer needs is deleted rather +than kept as false comfort. + +## Test coverage {#test-coverage} + +The implementation was built test-first (TDD), and the coverage is correspondingly dense: + +| Layer | Volume | +|---|---| +| Unit tests (`gtest`, `CAS*` suites) | ~1,900 test cases across ~130 files, covering formats, the write and read paths, the ref machinery, `GC`, recovery, and the backend contract | +| Integration tests | 10 dedicated `test_cas_*` suites (shared pools, `GC` on S3, sharded `GC`, relink replication, fault-injected `INSERT` recovery, member decommission, and more) | +| Stateless tests | dozens of dedicated `CAS` tests (pool integrity, leftovers, fsck, GC), in addition to the whole standard suite running on a `CAS`-default server (below) | + +## The whole test suite, on CAS by default {#stateless-suite-on-cas} + +Beyond the model corpus and the soak harness, the standard ClickHouse **stateless test suite runs +green with `CAS` as the default `MergeTree` storage**: dedicated CI lanes +(the `cas storage` and `cas s3 storage` job families — the latter covering ASan/TSan/MSan/UBSan +and ARM against a real S3-compatible store) run every stateless test against a server whose default disk is +a `CAS` pool. A small set of tests carries the `no-cas-storage` tag and is skipped in those lanes — +tests that exercise a mechanism a content-addressed disk deliberately does not have (for example, +`s3_plain` layouts or deliberately corrupted on-disk part chains). Everything else — the thousands +of tests that define what `MergeTree` is supposed to do — passes unchanged on top of `CAS`. diff --git a/docs/en/antalya/cas/architecture/design-history.md b/docs/en/antalya/cas/architecture/design-history.md new file mode 100644 index 000000000000..1587b2fdd58a --- /dev/null +++ b/docs/en/antalya/cas/architecture/design-history.md @@ -0,0 +1,61 @@ +--- +description: 'A condensed record of the paths CAS explored and rejected, and the major design pivots that produced the current architecture.' +sidebar_label: 'Design history' +sidebar_position: 13 +slug: /antalya/cas/architecture/design-history +title: 'CAS Architecture — Design History' +doc_type: 'reference' +--- + +# CAS architecture — design history {#design-history} + +This page is a condensed record of the roads not taken: what was tried, why it was abandoned, and +the sequence of pivots that produced the architecture described elsewhere in this section. + +## Rejected paths {#rejected-paths} + +| What it was | Why it was abandoned | What replaced it | +|---|---|---| +| **Generation-in-the-key** (Epoch-Based Reclamation core; blob keys carried a generation, `blobs//`) | Required `O(files)` persistent `Keeper` writes per commit and colliding intent keys across writers building identical content; a stuck writer stalled reclamation pool-wide | The incarnation-token design: identity moved into the object body and delete precision into the backend token, removing the generation from every key | +| **Merkle tree layer** (a `Tree` object kind, `trees/` prefix, `child_gen` carried inside a tree's own identity) | Depended on the generation-in-the-key core: a reclaim at any child propagated a new generation up the entire tree chain, and the tree layer was itself an extra surface for the same class of bug | Removed entirely; trees became manifest-internal, and `Blob` is the sole durable object kind besides the manifest and the ref | +| **Integer in-degree refcount** (a mutable counter, incremented per reference, decremented per release) | The decide-to-reference-then-not-yet-durable window let the fold observe in-degree 0 for a still-live blob; a mutable counter also costs a `CAS` round-trip proportional to write volume | A derived count: `GC` folds a multiset of `+`/`-` source-edge deltas, so losing or duplicating a record can only delay reclamation, never accelerate one | +| **Extending zero-copy replication instead of a new mechanism** | Zero-copy's structural costs (a commit spanning local disk, S3, and `Keeper`; a mutable refcount) are inherent to its design, not a bug to patch | `CAS` is an alternative to zero-copy, not a replacement: both remain available, `metadata_type = cas` is opt-in per disk, and no existing deployment needs to migrate | +| **Per-incarnation body keys** (`blobs/xx/.`, an alternative to the in-body incarnation tag) | A resurrect reusing the condemned incarnation instead of minting a fresh one reintroduced the shared-key race; structurally this was generation-in-the-key again | The in-body `incarnation_tag` plus exact-token body delete, which keeps the generation out of every object key | +| **Meta as the lifecycle linearizer** (a per-hash `.meta` object whose presence/absence *was* the authority for a blob's lifetime) | The marker is a point-read hint only, never consulted by reads; treating it as the linearizer would assert a guarantee the design does not make | The meta stays advisory: the in-body incarnation tag and exact-token delete are the real authority, and an absent meta reads identically to `Clean` | +| **Raw immutable bodies with a three-state tombstone meta** | A resurrect displacing the body forced a terminal-tombstone handshake — a writer↔`GC` liveness coupling that could re-enable data loss | The settled one-key-per-hash design with an in-body incarnation tag | +| **A persistent, append-only namespace registry for `GC` discovery** | Never deregistered on drop, so it grew monotonically forever; its fence cost scaled with namespaces ever created, not namespaces live | Discovery from the ref data itself, made safe by two independent coordinates: a durable per-shard incarnation plus a pool-global round | +| **A separate all-shard fence-and-recheck phase per `GC` round** | Both phases cost `O(pool size)` GET+CAS every round regardless of churn — roughly 2.4 million requests at 100k tables | A causal ack-floor: one streaming merge per round, with no separate fence or recheck phase, cutting the request count by roughly three orders of magnitude | +| **A pool-wide sparse ref-id allocator with completeness certificates bolted on** | Successive additive fixes kept growing without closing the root cause: absence is undecidable in a sparse id space | The invariants were changed instead of patched: dense per-life ids derived from applied state, an in-band epoch seal, and a `_ckpt` head object carrying the exact acknowledged frontier | + +## Turns at a glance {#turns-at-a-glance} + +| Date | Turn | +|---|---| +| 2026-06-01 | Starting point: "content-addressed storage for `MergeTree`" thesis and a working proof of concept | +| 2026-06-07 – 10 | The generation-in-the-key core is abandoned; the incarnation-token design replaces it | +| 2026-06-11 | The incarnation model passes exhaustive model checking with zero violations | +| 2026-06-18 | A dangling manifest reference — a committed manifest naming an already-deleted blob — leads to replacing per-blob protection hints with structural build-root reachability | +| 2026-06-24 – 26 | Formats begin converging on a single self-describing envelope (completed in July as the all-text, JSON-based codec set) | +| 2026-06-26 | Root-local full-tree manifests collapse a forest of small `GC` objects into one hot/cold split | +| 2026-07-01 | The namespace registry is deleted; discovery moves to the two-coordinate incarnation-and-round scheme | +| 2026-07-02 | Fence-and-recheck `GC` rounds are replaced by the one-pass, causal ack-floor round | +| 2026-07-06 – 10 | Writer/`GC` simplification: promote-time revalidation of tokened dependencies is proved redundant | +| 2026-07-13 | Mount-lease handover becomes boundary-exclusive, closing the cross-epoch grace window without a timeout | +| 2026-07-15 | All part files become content-addressed: the mutable file set drops to empty, and disk-transaction dispatch collapses to one precommit contract | +| 2026-07-17 | An acknowledged `INSERT` that could be lost is traced to a removed durability guard and fixed | +| 2026-07-26 | A `LIST` omitting two already-durable ref entries is caught during a soak run — the incident that settles the trust model for listings | +| 2026-07-27 – 29 | The sparse-id certificate stack is abandoned; the invariants change instead — dense ids, an in-band epoch seal, and a `_ckpt` recovery frontier | +| 2026-08-01 – 03 | Recovery stops reading listings entirely and works from authoritative objects; the listing trust model is finalized | + +## The pattern underneath {#the-pattern} + +A few reflexes recur across these pivots and still apply to new design work: + +- **Re-derive the invariant, don't patch the mechanism.** Every durable fix came from asking what + property must hold, not from patching the specific failure observed. +- **Delay is acceptable, authorization is not.** A stale-but-honest observation can only ever + postpone a decision; the design consistently rejects any mechanism that could *accelerate* a + destructive action past its safety gates — a delay is a latency cost, a wrongful authorization is + data loss. +- **A model that no longer matches the code is worse than no model.** Superseded models are + removed rather than kept: an unfaithful proof is false comfort, not documentation. diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md new file mode 100644 index 000000000000..48be9c4e6b24 --- /dev/null +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -0,0 +1,263 @@ +--- +description: 'The CAS garbage collector: leadership as work de-duplication, the 18-phase round pipeline, condemnation and exact-token deletion, sharding, and observability.' +sidebar_label: 'Garbage collection' +sidebar_position: 8 +slug: /antalya/cas/architecture/garbage-collection +title: 'CAS Architecture — Garbage Collection' +doc_type: 'reference' +--- + +# CAS architecture — garbage collection {#garbage-collection} + +`GC` is the only place in `CAS` that ever deletes a blob body or a manifest body. It runs as a +background, lease-paced loop per mount (`Gc::runRegularRound`, `Gc/CasGc.cpp`), folding ref-log +history into blob in-degree, condemning what reaches zero, and deleting only after that +condemnation has survived a full extra round. This page covers leadership, the round's 18 phases, +condemnation and deletion, sharding, pruning, round cost, and observability. Manifest and ref +mechanics that `GC` folds are covered on the +[manifests-and-refs page](/antalya/cas/architecture/manifests-and-refs); the writer-versus-`GC` +race over one blob is covered on the +[blob-protocol page](/antalya/cas/architecture/blob-protocol#writer-gc-race). + +## Leadership {#leadership} + +There is **no separate `GC` lease object**. The lease lives inside `gc/state` itself as +`{owner, seq}`. + +```mermaid +stateDiagram-v2 + [*] --> Reading: GET gc/state + Reading --> Creating: object absent, never observed before + Creating --> Leader: casPut create-if-absent, gc_shards fixed here, once + Reading --> Renewing: lease owner is me + Renewing --> Leader: casPut seq+1, guarded by the observed token + Reading --> Evaluating: foreign owner + Evaluating --> NotLeader: incumbent lease moved, or heartbeat moved, or steal not allowed + Evaluating --> Stealing: both frozen across a full observation window + Stealing --> Leader: casPut owner=me seq+1, on the observed token + Stealing --> NotLeader: lost the CAS, re-read and re-arm + Leader --> [*]: run the round +``` + +Two independent liveness signals are consulted before a steal: whether `(owner, seq)` moved since +the last tick, and whether the separate `gc/hb` heartbeat moved. The heartbeat is compared only +under the same remembered heartbeat owner, deliberately not against `lease.owner` — a deposed +leader's heartbeat thread keeps pulsing, and that must not cause a live new leader's lease to be +stolen. The paced background loop may steal; a manual `SYSTEM CAS GC RUN` may not, because the +safety argument needs two observations separated by real wall time. Because every renew or steal +bumps `seq`, `seq` doubles as the round's attempt id. + +**A deposed leader that keeps running cannot corrupt anything**, and the argument does not rely on +exclusivity at all: + +1. `gc/state` is published by exactly **one** `CAS` per round; a deposed leader's `CAS` fails and + its entire round evaporates. +2. Every fold artifact is written under that leader's own attempt number, invisible to every + reader, and reclaimed later by wholesale generation pruning. +3. Destructive pre-`CAS` actions are justified only by previously published durable state, so they + are replay-idempotent. +4. Deletes are exact-token, so a stale leader can never delete a newer incarnation. + +The lease is therefore **work de-duplication, not mutual exclusion**. + +## The round {#the-round} + +A round is one pass of 18 named phases ending in exactly one `gc/state` `CAS` +(`Gc::runRegularRound`, `Gc/CasGc.cpp`). + +| # | Phase (`GcPhaseTimer` name) | What it does | +|---|---|---| +| 1 | `lease` | Acquire, renew or steal the lease inside `gc/state`. The only phase a not-a-leader round emits | +| 2 | `pre_fold_ref_drain` | Resolve catalog `Removing` rows whose cleanup evidence the adopted parent already sealed; exact-CAS-delete the completed ones before anything else can act | +| 3 | `heartbeat_floor` | One `LIST` of `gc/server-roots/`, one `GET` per mount slot, fence-out `PUT` for any mount whose write-token has held stable past the threshold | +| 4 | `defer_decision` | One full `LIST` of `cas/ns/stream/`, build the catalog-keyed ref walk plan; decide `DEFER` (nothing changed, no graduation due) or continue to a full fold. A `DEFER` verdict still runs one namespace-janitor page — the same work phase 16 does on a folding round — with its deletes suppressed | +| 5 | `parent_seal_read` | Capture the parent fold seal's run references before the fold mutates the in-memory generation/attempt, to detect a ref that moved off an already-pruned generation | +| 6 | `fold_ref_group` | Regroup the one `LIST` from phase 4 into per-table listings — no I/O, the keys are already in hand | +| 7 | `fold_seal_read` | `GET` and decode the adopted fold seal that anchors this fold's coverage | +| 8 | `fold_ref_intake` | `GET` every new ref-log record and every referenced manifest, extracting blob source edges | +| 9 | `fold_reduce` | The three-cursor merge over prior edges, new deltas and the parent's condemned rows: spare, condemn, graduate or redelete each candidate | +| 10 | `fold_seal_write` | Write the new fold seal once, write-once deterministic, adopting a byte-identical replay instead of rewriting it | +| 11 | `pending_deletes` | The single content-delete site: exact-token `deleteExact` of every entry the *previous* round marked `delete_pending`, plus the forensic outcome-log writes | +| 12 | `meta_pool_wait` | Drain the bounded pool of async `.meta` condemn-marker writes queued during the fold | +| 13 | `round_commit` | Retention-prune old generations, then publish the single `gc/state` `CAS` that adopts the whole round | +| 14 | `handoff_reclaim` | Post-`CAS`: reclaim any generation that a ref moved off during this very round, before the ordinary wholesale prune would reach it | +| 15 | `manifest_deletes` | Delete manifest bodies whose owner-removal minus-one edge the `CAS` in phase 13 just adopted | +| 16 | `namespace_cleanup` | One bounded page of the perpetual namespace janitor, reclaiming dead-life debris | +| 17 | `ref_object_cleanup` | Prune ref logs and snapshots once both fold coverage and a live snapshot make them safe to delete | +| 18 | `orphan_sweep` | One cursor-paced page of the [orphan-manifest sweep](/antalya/cas/architecture/manifests-and-refs#orphan-sweep); wrapped so it can never fail the round | + +Phases 5 through 18 run only when phase 4 decides to fold. A `DEFER` verdict is not a bare no-op: +it still runs one bounded namespace-janitor page with `suppress_destructive = true` — cursor +progress and diagnostics only, no deletes — and then returns, publishing no fold artifact and no +`gc/state` `CAS` at all: + +```mermaid +flowchart LR + D4{"4 defer_decision"} -->|"nothing changed, no graduation due"| DEF["DEFER: one suppressed
namespace-janitor page, then return"] + D4 -->|"changed shards, or graduation due"| FOLD["phases 5 through 18: full fold and round commit"] +``` + +Orderings that are load-bearing: + +- **2 before 4** — a row proved complete by the adopted parent is resolved before `DEFER` or any + successor plan can publish. +- **15 after 13** — manifest bodies are deleted only after the `CAS` adopted their decrements. +- **13's prune before the `CAS`** — a pre-`CAS` destructive action may rely only on already- + published state. + +**Clamp suppression.** `suppress_destructive = !anomalies.empty() || !carried_holds.empty() || +!frontier_complete` is computed once and threaded into the merge, current-life ref cleanup and the +perpetual namespace janitor, so they cannot desynchronize. Under suppression there is no +graduation, no redelete, and no ref or namespace deletion; condemnation and sparing continue, +because both are non-destructive. + +**Fail-closed aborts.** A throw before the `CAS` means nothing is adopted: unapplied transactions, +a cursor/apply mismatch, a missing adopted seal, a table with a snapshot but no surviving log and +no cursor, a non-total condemned summary, and an observed delete marker (bucket versioning is on). + +## The one-pass commit {#gc-state} + +`gc/state` is the durable safety and round-adoption state: `round`, `gc_shards`, +`snap_generation`, `snap_pruned_through`, `snap_attempt`, `manifest_sweep_cursor`, and the lease. +Exactly one `CAS` per round publishes it; the fold itself performs no `CAS` of its own. + +**The fold seal *is* the coverage record**: generation, parent generation, one `ref_lives` row per +catalog-admitted opaque life (coverage plus optional cleanup evidence), references to the +source-edge run segments, and a per-shard condemned summary. It is encoded deterministically, so a +replayed round produces byte-identical bytes and adopts its own output through the +`putDeterministicArtifact` adoption pin (see the [blob-protocol page](/antalya/cas/architecture/blob-protocol#deterministic-artifacts)). +There is **no separate retired-list object** — condemned entries ride the source-edge run as +sentinel rows at `source_id = 0` — and **no run-file list outside the seal**; runs are resolved +*through* the seal's references, never by key construction. + +## Finding orphans {#finding-orphans} + +In-degree is a set of source edges, not a refcount. A blob becomes a candidate when its edge set +becomes empty and it was touched this pass: one `HEAD` captures the exact incarnation token and +size that a future delete will name. A blob merely carried from the parent run pays no `HEAD`. + +**The grace period is measured in rounds, not acks:** an entry graduates once it has survived one +full round (`condemn_round < current_round`). The heartbeat floor is liveness only and **never** +gates graduation. + +**The 404 rule.** A body that is present but invalid is `CORRUPTED_DATA`, hard. A body that is +missing is **never** a throw — the fold records and continues, and the caller decides by position: +a precommit activation clamps as a barrier; a committed or removal fold clamps only that table. +Prunes are likewise fail-open on 404. + +## Condemnation and deletion {#condemn-delete} + +```mermaid +flowchart LR + A["round n: in-degree hits zero
HEAD -- exact token t"] --> B["write .meta = Condemned round n
async, bounded pool, drained pre-CAS"] + B --> C["retired with condemn_round = n+1"] + C --> D{"round n+1: re-verify"} + D -->|"in-degree recovered"| S["SPARED -- recovery wins, even past the floor"] + D -->|"still zero, confirmed durable Condemned evidence for hash and t"| G["GRADUATED -- delete_pending"] + D -->|"still zero, evidence unconfirmed"| C2["carried unchanged, retry the marker, never throw"] + D -->|"current token not equal to t"| SUP["SUPERSEDED -- a writer resurrected, re-condemn the CURRENT token"] + G --> E["round n+2, pre-CAS: deleteExact blob, t"] + E -->|"Deleted or Absent"| F["then drop the .meta"] + E -->|TokenMismatch| H["nothing deleted -- live at a newer token, leave the .meta alone"] +``` + +The `.meta` sidecar carries **no token** — it is a per-hash hint. The exact incarnation token lives +in the condemned sentinel row inside the run, together with the condemn round and two flags, +`delete_pending` and `marker_confirmed`. `GC`'s marker is add-only: `Clean → Condemned` yes, the +reverse never, not even when sparing — only a writer that has already displaced the body may clear +it. Minimum two full rounds separate condemnation from deletion, and `delete_pending` is terminal — +an entry is never un-pended. + +## Sharding {#sharding} + +`gc_shards` is fixed at first lease acquire and immutable; decoders reject `0`. A blob routes by +the **high** 64 bits of its digest, read big-endian. + +The role split is worth internalizing: the **coordinator** — the lease holder — owns discovery, +round visibility, the single global fence, and the generation advance, because a publish into +*one* namespace can protect a blob owned by *any* shard, so these span the whole universe and must +not be sharded. **Reducers** own only their disjoint shard; their run-key namespaces never +collide, so two servers could reduce different shards concurrently and reducer work needs no +lease. + +A shard with an empty delta bucket and no condemned entries in the parent summary copies the +parent's run references verbatim — zero run I/O, a "pure carry". A missing parent summary entry on +a non-fresh pool is `CORRUPTED_DATA`, never silently treated as zero. + +## Pruning old objects {#pruning} + +- **Current-life ref logs and snapshots** (phase 17) — a log is deletable only when covered by + both durable fold coverage and a durable live snapshot; snapshots strictly older than the newest + observed one are deletable. There is no batch delete; it is `HEAD` plus `deleteExact` per key. +- **Generations** (phase 13) — keep the last `gc_snapshot_generations_to_keep` (default 3; `0` + means keep everything, for forensics). Pruning is wholesale: `LIST` the generation prefix and + delete everything under it, including deposed-leader debris and attempt-scoped outcome sets. A + generation still referenced by the live seal is skipped, but the cursor still advances past it — + leak-freedom then rests on the post-`CAS` hand-off reclaim in phase 14. +- **Manifests** — owner-removed bodies delete in phase 15; never-precommitted bodies go through the + [orphan-manifest sweep](/antalya/cas/architecture/manifests-and-refs#orphan-sweep) in phase 18. + +## What a round costs {#round-cost} + +Per **folding** round, with `N` live mounts, `S` ref tables and `S_changed` tables carrying new +logs: + +| Operation | Count | +|---|---| +| `LIST cas/ns/stream/` | 1 full enumeration | +| `LIST gc/server-roots/` | 1, plus 1 `GET` per mount | +| `GET` the adopted fold seal | 5, explicitly instrumented | +| `GET` ref logs | 1 per new log | +| `GET` manifests | 1 per emitted edge — no manifest-body cache within a round | +| `PUT` run segments | 1 per non-pure-carry shard, plus 1 fold seal | +| `HEAD` blobs | 1 per newly condemned | +| `DELETE` | 1 per graduate | +| `CAS gc/state` | 1 | + +The measured `GET` formula is exact: total `GET`s equal ref-log body `GET`s plus manifest body +`GET`s, i.e. `1 + edges_per_log`. An idle round is one `LIST` sweep, `N` heartbeat `GET`s, and one +`CAS`. A deferred round is cheaper still: one `LIST`, three seal `GET`s, the lease `GET`/`PUT` and +the heartbeat floor — no `gc/state` `CAS` at all. + +The round's work is internally self-regulated: anything a pass cannot finish is carried and retried +by the next round's cursors, never dropped. The internal pacing knobs are deliberately not part of +the user-facing configuration surface. + +| Setting | Default | Bounds | +|---|---|---| +| `gc_meta_pool_size` | 16 | bounded pool for condemn-marker writes | + +## Observability {#observability} + +`system.cas_gc_log` emits `Start`, `Finish` and per-`Phase` rows, correlated by `round_id` — not +`round`, which is `0` on `Start` and does not exist at all on a not-a-leader round. Phase rows +carry no verb columns by design: per-phase operation counts ride the row's own `ProfileEvents` +delta, so grouping by phase over an S3 event attributes the LIST/GET/PUT/DELETE budget without +inventing schema. `phase_metrics` carries the semantic counts no counter can supply (clamped +tables, dead precommits skipped, pure-carry shards, generations visited). `Deferred` is kept +distinct from `Success` precisely so "folded and found nothing" is distinguishable from "never +folded". Every `GC`-related `ProfileEvent` carries the uppercase `CAS`/`CASGC` prefix — for example +`CASGCRetiredCondemned`, `CASGCRetiredGraduated`, `CASGCRetiredRedeleted`, +`CASGCClampSuppressedPasses`, `CASGCHeartbeatFenceOuts`. + +Alongside it, `system.cas_log` carries the audit trail: the condemn chain, fence-outs, anomalies +(capped per round, each carrying the true total), and manifest deletes. + +`ca-fsck` distinguishes two classes that are easy to conflate: `dangling` — referenced but missing, +data loss — versus `unreachable`/`awaiting-gc` — present, unreferenced, and +simply waiting for graduation. + +## Operational surface {#operational-surface} + +| Command | Effect | +|---|---| +| `SYSTEM CAS GC RUN ''` | One synchronous round on the contacted node; only the lease holder makes progress | +| `SYSTEM CAS GC STOP` / `SYSTEM CAS GC START` | Stop or resume future rounds on the same scheduler, preserving its identity | +| `SYSTEM CAS GC REBUILD` (`clickhouse-disks ca-gc-rebuild`) | Fail-closed disaster-recovery path that every "GC refuses to run" error points at; deliberately over-protects — it prefers bounded leaks over risking an under-count. It cannot delete live data directly: deletions it produces still flow through the normal round's condemn, graduate, exact-token path | +| `clickhouse-disks ca-gc-dryrun` | Opens the disk read-only, constructs a non-leader `GC`, and prints what would be deleted with a reason per entry. Write-free, resolves runs through the seal's references. Documented caveat: it does not fold new owner events, so away from quiescence it can **over-report** — the subset guarantee holds only at quiescence, and its output must never feed a real delete | + +`SYSTEM CAS DROP POOL MEMBER '' FROM DISK ''` — permanent removal of a dead +replica, distinct from ordinary `GC` — is covered on the +[mounts-and-leases page](/antalya/cas/architecture/mounts-and-leases#mount-lifecycle). `SYSTEM CAS +FSCK` and its `dangling`/`unreachable` vocabulary are a read-only diagnostic pass, not part of the +`GC` protocol itself. diff --git a/docs/en/antalya/cas/architecture/index.md b/docs/en/antalya/cas/architecture/index.md new file mode 100644 index 000000000000..15a9516c9ae7 --- /dev/null +++ b/docs/en/antalya/cas/architecture/index.md @@ -0,0 +1,114 @@ +--- +description: 'What CAS is, the Git-analogy mental model, the object model, and the safety invariants a reviewer should hold every CAS protocol against.' +sidebar_label: 'Architecture overview' +sidebar_position: 1 +slug: /antalya/cas/architecture/ +title: 'CAS Architecture — Overview' +doc_type: 'reference' +--- + +# CAS architecture — overview {#overview} + +`CAS` ("content-addressed storage") is a `MetadataStorage` back-end for object-storage disks +(`metadata_type = cas`) that stores every `MergeTree` part file once, addressed by the hash of +its content. Many servers share one object-storage pool with no byte duplication, no zero-copy +bookkeeping in `Keeper`, no per-replica local-disk reference state that grows with data volume, +and no mutable per-blob refcount. + +It is still experimental — that is deliberate, not a caveat to apologize for. Pre-release means +the format can still change cheaply, with zero compatibility scaffolding, and the design can +still be iterated on invariants rather than migrations. The bet: all you need underneath is a +good S3 bucket. No external coordinator, no metadata service, no Keeper state proportional to +data — the pool is self-describing, and everything CAS needs to agree on (refs, leases, GC +leadership, fencing tokens) is an object in the bucket. + +This page is the entry point of a 4-page set: it gives the mental model. Deeper detail on +storage layout, the write/read protocols, and GC lives in the other three pages. + +## The Git analogy {#git-analogy} + +The fastest way to load the model is Git, which most readers already carry: + +| Git | CAS | +|---|---| +| blob (file content by hash) | **blob** — one part file's bytes, keyed by content hash | +| tree (directory listing) | **part manifest** — the immutable file list of one part | +| ref (`refs/heads/main`) | **ref** — `part name → manifest id`, the only mutable state | +| `gc` / reachability | **GC round** — an in-degree fold over refs → manifests → blobs | + +Where the analogy breaks: Git's objects are locally addressed and GC runs against a single +repository with no concurrent writers; CAS objects are addressed inside a shared, multi-writer +object-storage pool, and its GC round has to reason about ambiguity (crashed writers, +in-flight precommits, eventually-consistent `LIST`) that a local Git repository never faces. +Git also has no equivalent of a CAS ref's precommit state — a CAS ref transition is durable +before the blob it names is guaranteed reachable, never the other way round. + +## The object model {#object-model} + +Four durable object kinds exist in a pool: one mutable (the ref), three immutable +(part manifest, blob, and a blob's condemnation-marker sidecar). + +```mermaid +graph TD + R["Ref: part name maps to manifest id"] + M["Part manifest: file list of one part"] + B["Blob: one part file's bytes, keyed by content hash"] + BM["Blob meta: condemnation marker sidecar"] + + R -->|names| M + M -->|entry references| B + B -.->|sidecar| BM +``` + +**The reachability rule, stated once:** a blob is live if and only if some live manifest names +it, and a manifest is live if and only if some ref — committed or precommitted — names it. `GC` +computes exactly this and nothing else. + +## Safety invariants {#safety-invariants} + +The full numbered list lives in the CAS agent guide; this is the reader-facing summary of the +substance: + +| Invariant | What it means | +|---|---| +| No silent data loss | No path may delete an object a committed reference still names | +| Revival is re-upload only | A condemned blob is never revived by copying it — only by re-uploading the original bytes under a fresh identity | +| Exact-token deletes | Every delete names the exact object incarnation it removes, never "the object at this key" | +| `TOKEN ⟹ CONTENT` | A repeated write token implies unchanged bytes — the backend must never let a token be reused over different content | +| Fail closed on ambiguity | An operation that may have landed is never treated as one that did not | +| One content-delete site | Exactly one place in the whole codebase ever deletes a blob body, gated on a previously published `GC` round | +| `GC` never invents history | Cleaning up an abandoned write is the writer's job, not `GC`'s | +| Over-count only | A lost or duplicated `GC` fold can only delay a reclaim, never bring one forward | +| No dangle / no loss / no return | A live ref always resolves through present objects; a delete requires proven unreachability at an exact token; a retired object identity is never valid again (though the same logical key can return under a new token) | + +## Positioning: shared-nothing, not shared-state {#positioning} + +Each server owns the catalog rows under its own identity and writes only its own state objects +— that part is shared-nothing, same as `ReplicatedMergeTree` today. What CAS adds is a single +**shared** resource: the blob content space, addressed purely by content hash, which is +write-once and conflict-free by construction — two servers writing the same content write the +same key with the same bytes, so there is nothing to reconcile. The only mutual exclusion CAS +needs anywhere is a conditional write (create-if-absent, or compare-and-swap on a token) against +a single object. + +That is deliberately not a coordinator or a serializable metadata service: there is no external +coordinator, and no `ZooKeeper`/`Keeper` usage inside the pool protocol itself. `Keeper` stays +exactly where `ReplicatedMergeTree` already used it — replication log and part-set consensus — +and its load does not grow with pool size, because the pool's own bookkeeping never touches it. + +## The subsystem pages {#subsystem-pages} + +| Page | Covers | +|---|---| +| [Storage layout](/antalya/cas/architecture/storage-layout) | Every S3 key shape, the object envelope, codecs, a worked example tree | +| [Namespaces](/antalya/cas/architecture/namespaces) | Namespaces, `life_id`, the catalog, and their lifetime | +| [Blob protocol](/antalya/cas/architecture/blob-protocol) | Conditional writes, deduplication, the writer-vs-GC race | +| [Part lifecycle](/antalya/cas/architecture/part-lifecycle) | Build, precommit, upload, promote; crash points and their cleaners | +| [Manifests and refs](/antalya/cas/architecture/manifests-and-refs) | Part manifests and the ref machinery: publish, fold, recovery | +| [Mounts and leases](/antalya/cas/architecture/mounts-and-leases) | Server identity, the owner claim, the mount lease, fencing | +| [Replication](/antalya/cas/architecture/replication) | Fetch-by-relink between replicas sharing one pool | +| [Read path](/antalya/cas/architecture/read-path) | Ref resolution, manifest reads, ranged blob reads, the caches | +| [Garbage collection](/antalya/cas/architecture/garbage-collection) | Leadership, the round, sharding, cost, observability | +| [Backend abstraction](/antalya/cas/architecture/backend) | Provider dialects for conditional writes, the capability probe | +| [Correctness](/antalya/cas/architecture/correctness) | TLA+ models, counterexamples, soak methodology, test coverage | +| [Design history](/antalya/cas/architecture/design-history) | The rejected designs and the major pivots | diff --git a/docs/en/antalya/cas/architecture/manifests-and-refs.md b/docs/en/antalya/cas/architecture/manifests-and-refs.md new file mode 100644 index 000000000000..bd23b51e9a5f --- /dev/null +++ b/docs/en/antalya/cas/architecture/manifests-and-refs.md @@ -0,0 +1,260 @@ +--- +description: 'Part manifest structure and lifecycle, the ref table as the only mutable state in a CAS pool, the publish protocol, and the orphan-manifest sweep.' +sidebar_label: 'Manifests and refs' +sidebar_position: 5 +slug: /antalya/cas/architecture/manifests-and-refs +title: 'CAS Architecture — Manifests and Refs' +doc_type: 'reference' +--- + +# CAS architecture — manifests and refs {#manifests-and-refs} + +A part manifest is the immutable file list of one `MergeTree` part; a ref is the mutable pointer +from a part name to the manifest that currently backs it. Together they are the two object kinds +that make a CAS pool's state machine: manifests never change, refs are the only place anything +moves. This page covers what a manifest contains, how a manifest becomes reachable or becomes an +orphan, how a ref mutation is published durably, and how a mounted server recovers a ref table +after a crash or a fresh mount. The write/promote sequence that drives these primitives is on the +[part-lifecycle page](/antalya/cas/architecture/part-lifecycle); how `GC` folds ref history into +blob liveness is on the [garbage-collection page](/antalya/cas/architecture/garbage-collection). + +## Part manifests {#part-manifests} + +A manifest (`cas_part_manifest`, `Formats/CasPartManifestFormat.h`) has four top-level fields: +`ref` (its own id, repeated in the body for fail-closed validation), `root_namespace_id` (the +owning namespace, likewise repeated), `payload_digest` (integrity/debug only — never a key, never +a dedup input, never a `GC` edge), and `entries` — strictly ascending by path after decode. Each +entry is `{path, placement, BlobRef, blob_size, inline_bytes}`; the hash algorithm travels **per +entry**, so one manifest may legitimately mix algorithms if the pool has more than one enabled. + +A manifest deliberately holds **no** offsets, no packed-file support, no projections field, no +codec info, no parent-manifest link, no source edges, and no incarnation token. One blob is one +file's bytes; a read window is `{blobKey, blob_header_len, blob_size}`. A projection is an +ordinary entry whose path has a `.proj` component. The incarnation token is the backend `ETag` +observed by a `HEAD`, never stored in the manifest. + +**The manifest id is neither a content hash nor random.** It is +`ManifestRef = {writer_epoch, build_sequence, manifest_ordinal}` — durable writer epoch times +monotone per-incarnation build sequence times monotone per-build ordinal — which gives "no +manifest id reuse" by construction with no randomness needed. The `GC`-level identity is the pair +`ManifestId = (RootNamespace, ManifestRef)`; two namespaces may legally carry the same +`ManifestRef`. + +Backpressure caps are enforced before the body is written (`Pool/CasPartWriteTxn.cpp`): + +| Cap | Limit | +|---|---| +| Entries per manifest | 1 048 576 | +| Encoded manifest text | 256 MiB | +| Total inline bytes | 16 MiB | +| Largest single inline entry | 1 MiB | + +A manifest is written once with a conditional create (`putIfAbsentStream`) and **never rewritten**. +A different object at that key would be an id collision and is `CORRUPTED_DATA`, fail-closed, +before any owner transition names it. Rewriting a part therefore writes a **new** manifest over +the **same** blobs and moves the ref in one ref-log record — a repoint, covered in full on the +[part-lifecycle page](/antalya/cas/architecture/part-lifecycle#repoint). + +## Manifest lifecycle and the orphan sweep {#manifest-lifecycle} + +```mermaid +stateDiagram-v2 + [*] --> Staged: stageManifest, body PUT write-once + Staged --> PrecommitOwned: precommitAdd, ref-log OwnerTransition, plus-one edges on fold + PrecommitOwned --> Committed: promote, Precommit to Committed, no edge, net zero + Committed --> OwnerRemoved: drop or repoint or namespace removal, minus-one edges + OwnerRemoved --> [*]: GC deletes the body after the decrements are sealed + + Staged --> OrphanA: writer died before precommitAdd + OrphanA --> [*]: writer best-effort delete, else the orphan sweep + + PrecommitOwned --> DanglingPrecommit: writer died before promote + DanglingPrecommit --> OwnerRemoved: binding removed by abandon or a successor stale-precommit sweep +``` + +Two disjoint failure classes matter here: + +- **Pre-precommit orphan.** The body exists but no ref-log record ever named it. It contributes no + edges and nobody protects it — this is exactly what the orphan sweep below reclaims. +- **Dangling precommit.** The transaction died between `precommitAdd` and `promote`. Nothing wakes + it up on its own: a `PartWriteTxn` is never persisted. The binding must be removed by a ref-log + transaction — either the live writer's own `abandon`, or a fenced successor's stale-precommit + sweep, which removes precommits whose `manifest_ref.writer_epoch < live_epoch` + (`Pool/CasRefLedger.cpp`). Only after that minus-one folds does `GC` delete the body, on the + ordinary owner-removal path. + +The writer's own best-effort cleanup deliberately **skips** the precommit target once a precommit +was even attempted — including an uncertain outcome — because deleting a body that turns out to be +a live precommit would clamp `GC`'s fold barrier forever. + +### The orphan-manifest sweep {#orphan-sweep} + +Runs as the last phase of the `GC` round, cursor-paced and budgeted, wrapped so it can never fail +a round (`Gc/CasOrphanManifestSweep.cpp`). Eligibility comes **exclusively** from the durable +watermark in the mount lease — there is no age threshold and no time-based grace period anywhere +in this protocol. No mount lease for the `server_root_id` means no deletion authority means +nothing is swept for that root. + +```mermaid +flowchart TD + A["LIST one page of cas/manifests/
budget: manifest_sweep_list_budget_keys"] --> B{"build-prefix eligible?
durable watermark fact only"} + B -->|"epoch less than lease epoch"| ELIG["eligible, old-epoch debris"] + B -->|"same epoch, min_active clears build_seq"| ELIG + B -->|"no lease, or epoch ahead, or build may be live"| SKIP["skip"] + ELIG --> C["protection view: committed manifests
plus live precommits
plus manifests with an unfolded minus-one"] + C -->|"key protected"| SKIP2["skip"] + C -->|"not protected"| D["deleteExact key, token"] + D -->|Deleted| E["emit ManifestDelete audit event"] + D -->|"NotFound or TokenMismatch"| SKIP3["spared, a fresh owner reclaimed the key"] + E --> F["CAS gc/state with the advanced cursor"] +``` + +The protection view is built from the **same complete replay** that writer recovery uses, and a +namespace whose view fails to build is added to an errored set with **all** of its deletions +skipped — an empty owner set is never substituted for a failed one. The sweep deletes only +manifest bodies and emits no blob deltas, correct precisely because a pre-precommit body never +contributed a `+1`. Contrast with the owner-removal path, which is ordered the other way: fold the +`-1` edges, adopt the decrements in the round `CAS`, *then* delete the body — a crash there leaks +a body to this sweep, never a dangle. + +## Source edges: how a manifest makes blobs live {#source-edges} + +Blob liveness is a **set of source edges**, not a counter (`Gc/CasBlobInDegree.h`) — which is what +makes `GC`'s fold idempotent. An edge id is `sourceEdgeId(ManifestId, path)`, a deterministic hash +over the namespace, epoch, build sequence, ordinal and path — an edge *identity*, deliberately not +a content hash and not reconstructable. + +Edges are never written at manifest-write time. They materialize only when `GC` folds a ref-log +transaction that changes ownership: add-precommit means `+1` per blob entry; either removal means +`-1`; **promote means no edge at all**, because the manifest never loses an owner, so it is net +zero. Inline entries produce no edges — they have no separate object to reclaim. + +## The ref table {#ref-table} + +A ref is the only mutable state in the whole system, so this is where the concurrency design is +concentrated. + +- **Name** — a canonical clean relative path, in practice the part directory name with an optional + `detached/` or `moving/` prefix. +- **Value** — `{ref_name, ManifestRef, published_at_ms}`. There is **no** token/`ETag` in a ref + row; the cross-server "confirm token" is the text form `epoch:build:ordinal`. +- **Scope** — one ref table per `RootNamespace`, i.e. per table per server root. +- **Ownership slots** — a `ManifestRef` has at most one owner across the table, in one of two + slots: `Committed` or `Precommit`. Precommits are keyed by the pair `(ref_name, manifest_ref)`, + so several in-flight builds may legitimately contend for one ref name. + +In memory, `RefTableState` holds a copy-on-write map of committed rows, a set of precommits, an +ownership index enforcing the one-owner rule, a lifecycle (`Live`/`Removed`), the greatest applied +transaction id, and byte-size counters used for admission. Copying a state is a refcount bump, so +a flush's trial and candidate copies cost proportional to touched rows, not the whole table. +Network I/O is never performed while holding the state lock, so a reader sees either a whole +transaction or none of it. + +Two immutable object kinds carry the durable form under `cas/ns/stream//` (see the +[storage-layout key table](/antalya/cas/architecture/storage-layout#key-table)): a log object +holds exactly one transaction, `{namespace, txn_id, ops[]}`; a snapshot object holds one live table image +— sorted committed rows plus precommits. Mutable/path-addressed state lives separately under +`cas/ns/state//`: the per-life `_ckpt` checkpoint and any namespace-owned `_files/`. + +`RefTxnId = {writer_epoch, ref_sequence}` renders as two fixed-width hex fields, so lexical key +order equals tuple order. Ids are per-namespace and contiguous: within one `(namespace, +writer_epoch)` they run `1, 2, 3, …` with no holes, and a new mount epoch restarts the sequence at +`1`. A hole is therefore corruption, not an allocation artifact, and a non-successor id is rejected +as `CORRUPTED_DATA`. + +The op vocabulary is deliberately tiny: `NamespaceBirth`, `OwnerTransition{old?, new?}`, +`SetPublishedAt`, `RemoveNamespace`. There are exactly four legal `OwnerTransition` shapes — add +precommit, remove precommit, remove committed, and promote — enumerated identically by the state +machine and by `GC`'s edge extractor, so the two readers of the format cannot drift. + +Logs are pure conditional creates on write-once keys. There is no append-to-object and no +`CAS`-swapped mutable pointer anywhere in the ref lane. The writer never deletes ref objects; only +`GC` does, once coverage and a live snapshot both make a log safe to remove. + +Snapshots publish in the background, best-effort, one in flight per table, when the tail exceeds a +log-count or log-byte threshold. + +## Publishing a ref mutation {#publish-protocol} + +All mutations funnel through one flat-combining lane, `CasRefLedger::appendRefOps`. A single flush +carves a batch out of the queue and commits it as one or more transactions. + +```mermaid +flowchart TD + Q["appendRefOps enqueues ops"] --> REC["ensure the table is recovered"] + REC --> FEN{"mount fence still live?"} + FEN -->|no| FAIL0["fail the whole carved queue, retry error"] + FEN -->|yes| W{"outstanding wedge?"} + W -->|yes| WR["resolve the wedge by its exact key first"] + WR -->|resolved durable| INST0["install candidate, clear wedge"] + WR -->|still unresolved| FAIL1["fail the queue, stay wedged, never allocate a new id"] + W -->|no| CARVE["two-phase carve: plan may throw, publish never throws"] + CARVE --> VAL["per-item validation: caps, shape, byte budget
a failing item fails alone"] + VAL --> PREP["build candidate state and the complete wedge before the PUT"] + PREP --> PUT["putIfAbsent the ref-log key"] + PUT -->|Committed| OK["allocation-free install: swap state, bump counters, complete waiters"] + PUT -->|DefiniteFailure| GAP["fail survivors, id not consumed"] + PUT -->|"Unresolved, provably nothing sent"| NOSEND["do not wedge"] + PUT -->|"Unresolved, otherwise"| WEDGE["install the prepared wedge, survivors fail Uncertain"] + OK --> SNAP["maybe schedule a snapshot publish"] +``` + +The **wedge** is the mechanism that makes fail-closed ambiguity concrete: at most one per table, +recording the single conditional `PUT` whose outcome is unknown, complete with the key and the +sealed bytes. The next flush must resolve *that exact key* before it may allocate a new transaction +id — an unresolved write can never silently become a gap, and the ledger never double-publishes. + +Crash points: between the `PUT` and the install, the object is durable and unapplied — the next +mount's recovery replays it. Between a precommit and its promote, a dangling precommit is reclaimed +by the successor's stale-precommit sweep, described above. + +## Recovery {#recovery} + +Recovery is lazy per table, on first touch (`Pool/CasRefLedger.cpp`), and reads only named, +authoritative objects — there is no `LIST` anywhere in this path: + +1. **Exact `GET` of `_ckpt`.** The durable checkpoint is the sole source of the recovery grounding: + `chooseRecoveryGrounding` derives the base (a snapshot id, or genesis if there is none) and the + exact transaction to walk from purely from the checkpoint's own fields + (`committed_through`/`checkpoint_snapshot_id`/`life_epoch`) — recovery never enumerates its own + stream to find them. +2. If the grounding names a snapshot, `GET` and decode it as the replay base. +3. Walk forward by exact key from there, one transaction resident at a time: `GET` + `cas/ns/stream//-`, decode, apply, discard, advance to the next + arithmetic id. Every key this walk touches is a dense, deterministic successor of the last — + never a listed or guessed one. +4. **Absence is a decision point, not an error.** Finding a slot empty is either the live epoch's + stream legitimately ending there, or — for a dead predecessor epoch — the exact slot where its + closing `EpochSeal` must be written before the table may be trusted; the two cases are + distinguished by whether the epoch being walked is still live, not by retrying a listing. +5. Recovery may itself advance `_ckpt` as it replays, each time via a conditional write against the + checkpoint it last read; the write is re-verified with a fresh exact `GET` afterward, and a + concurrent winner's farther frontier is honored by restarting from that newer checkpoint rather + than trusting the write blindly. +6. Transient network errors retry the whole attempt with capped backoff; corruption and logic + errors fail fast. + +For a mounted writer the recovered in-memory table is authoritative for reads of its own +namespaces — there is no other writer of that namespace. S3 is authoritative for durability: +in-memory state advances only after a durable `PUT`, and a caller's `appendRefOps` returns only +after the durable install. In-flight precommits are visible only through the precommit set, never +through an ordinary ref resolve. + +Two cross-process readers see a different, colder view, but only at the discovery boundary: `GC` +and `ca-fsck` `LIST` once to discover which namespaces exist, staleness-bounded by whatever was +durable at `LIST` time, so a namespace born after that `LIST` is invisible to this pass. Within +each discovered namespace, the replay itself is not `LIST`-driven — it is the same exact-`GET`, +`_ckpt`-grounded arithmetic walk described above, just called from a caller-supplied catalog entry +instead of a live mount. The relink-confirm handshake (see the +[replication page](/antalya/cas/architecture/replication#relink-gates)) does zero object-store I/O +and answers `Yes` only against the resident, warm, fence-live in-memory table — `No` is not proof +of the negative, only `Yes` is fence-gated. + +## Namespace removal {#namespace-removal} + +Namespace removal has no physical-empty handshake. The writer changes the catalog row from `Live` +to `Removing`, appends the exact removals plus `RemoveNamespace`, and deletes nothing itself. The +`GC` fold attaches cleanup evidence to that life row; a later invocation's pre-fold drain exact-CAS +-deletes the matching `Removing` catalog row before any successor plan publishes. A perpetual +namespace janitor and the orphan-manifest sweep reclaim physical debris independently — a same-name +birth waits only for the catalog row to disappear, never for physical emptiness. diff --git a/docs/en/antalya/cas/architecture/mounts-and-leases.md b/docs/en/antalya/cas/architecture/mounts-and-leases.md new file mode 100644 index 000000000000..468a783297a6 --- /dev/null +++ b/docs/en/antalya/cas/architecture/mounts-and-leases.md @@ -0,0 +1,223 @@ +--- +description: 'How a CAS server establishes identity, claims its mount slot, and holds a renewable lease that fences stale writers out of the pool.' +sidebar_label: 'Mounts and leases' +sidebar_position: 4 +slug: /antalya/cas/architecture/mounts-and-leases +title: 'CAS Architecture — Mounts and Leases' +doc_type: 'reference' +--- + +Page 4 of 4 in the CAS architecture set. Covers server identity, the mount lease that fences +writers, and the server-scoped control-plane objects. No external coordinator is involved: there +is no ZooKeeper/Keeper client anywhere in this protocol — `MountLeaseKeeper` is a local lease +*renewer*, not a Keeper client. + +## `server_root_id` — the identity {#server-root-id} + +Every content-addressed disk must be configured with an explicit `server_root_id`. It is +validated and immutable, and deliberately **not** derived from `ServerUUID` — two replicas can +otherwise regenerate the same `ServerUUID` from a wiped local state directory, which must not +silently steal an existing identity. + +Validation (`validateServerRootId`, `Pool/CasServerRoot.h`) is fail-closed `BAD_ARGUMENTS`, no +sanitizing fallback: non-empty, at most 255 bytes, no empty/`.`/`..` path segment, no `_files` or +`_manifests` segment. + +It roots four subtrees and owns catalog names at or below ``: + +| Subtree | Contents | +|---|---| +| `gc/server-roots//` | owner, epoch, mount — the three control-plane objects below | +| `roots//` | loose mountpoint objects, no namespace/catalog association | +| `cas/manifests//` | part manifests | +| `staging//` | S3-staging debris, outside every GC `LIST`, reclaimed only by this server's next mount | + +`blobs/` is **not** under the `server_root_id` — content is pool-global, which is what makes cross-server +dedup work. Ref/namespace keys are also deliberately opaque and do not embed the `server_root_id`. + +Each replica sharing a backend endpoint must use a distinct `server_root_id`; omitting the setting +is a startup error. + +## The owner claim {#owner-claim} + +`claimOwnerOrThrow` binds `server_root_id` ↔ `server_uuid` **permanently**. The owner object is never deleted +and never reassigned — decommission only tombstones it in place. + +| Observed at `gc/server-roots//owner` | Action | +|---|---| +| present, same `server_uuid`, not tombstoned | proceed | +| present, `retired_at_ms` set | `CORRUPTED_DATA` — explicitly decommissioned, refuses to resume | +| present, different `server_uuid` | `CORRUPTED_DATA` — names the regenerated-uuid-file cause | +| absent, subtree provably empty | `putIfAbsent` the owner (claim) | +| absent, subtree non-empty | `CORRUPTED_DATA` — identity lost over existing data | +| lost the `putIfAbsent` race | re-read; equal uuid proceeds, else `CORRUPTED_DATA` | + +"Provably empty" requires both an authoritative decoded catalog naming no life owned by `server_root_id` and +a 1-key `LIST` probe finding nothing under `cas/manifests//` or `roots//`. + +Two failure modes this closes: + +- A **second server with a different `server_uuid`** is refused at this gate and can never take + over, regardless of lease expiry. +- A **same-uuid live twin** (two processes sharing one uuid file and `server_root_id`) is caught separately, by + the mount claim's token-stability observation, and aborts with an operator-facing message rather + than corrupting the pool. + +## The mount lease {#mount-lease} + +One object, `gc/server-roots//mount`, carries **both** the liveness lease and the build +watermark — there is no separate watermark object. `MountLease` fields: `server_uuid`, +`writer_epoch`, `hostname`, `pid`, `started_at_ms`, renewal `seq`, `expires_at_ms`, `min_active` +(the build-watermark floor), and `gc_fenced`. + +- **Cadence.** Renew every `mount_renew_period` (default 10 s), TTL `mount_lease_ttl_ms` (default + 30 s, TTL/3 renewal ratio). Each beat is a token-guarded `putOverwrite` bumping `seq + 1` — + `MountLeaseKeeper` never re-mints the object. +- **Local fence clock.** `CLOCK_BOOTTIME`, not `CLOCK_MONOTONIC`, so a VM resumed from suspend + correctly observes itself expired. The deadline anchors at attempt-*start*, never at response + time. +- **Per-write recheck.** Every durable write or delete captures the fence generation at admission + and rechecks it immediately before the object-store call and on every conditional retry. Reads + are not gated. +- **Request-budget admission.** `refAppendFenceOk` refuses to *start* a ref-log attempt unless + `attempt_timeout + safety_margin` fits inside the remaining lease, rejecting with + `BAD_ARGUMENTS` at request-admission time rather than mid-flight. + +**Losing the lease is neither read-only mode nor an abort.** It trips the local fence (latches +`lost`, bumps the fence generation, moves the in-process runtime to `TransientNotLive`) and +schedules a self-remount with exponential backoff from 1 s to 30 s. Only a *foreign* `server_uuid` +observed on the mount body is `LOGICAL_ERROR` — the owner anchor makes a foreign claim +protocol-unreachable, so seeing one is an invariant violation, not a recoverable race. A +`putOverwrite` that threw *before* observing any outcome does not fence while the confirmed +deadline is still comfortably ahead — only a **confirmed** mismatch is immediately terminal. A real +fence still costs only an epoch: recovery re-claims with a fresh one, bounded at 3 attempts. This +is the fail-closed posture from the general CAS invariant: doubt about the source aborts, doubt +about the mechanism may retry. + +GC's own view of a dead server is symmetric and clock-skew-immune: a slot becomes fence-eligible +only after the leader observes the *same* renewal token hold stable, on its own monotonic clock, +for `TTL + TTL/20 + cadence` — the identical formula a re-mounting server uses to wait out a +predecessor. The stamped `expires_at_ms` never participates in that decision; wall-clock `now` is +audit-only. + +## The two monotone counters {#counters} + +| Counter | Storage | Scope | Protects against | +|---|---|---|---| +| `writer_epoch` | durable, `gc/server-roots//epoch` (`ServerEpoch::next_writer_epoch`, CAS-bumped by `allocateWriterEpoch`) | across crashes and restarts | a same-`(uuid, epoch)` twin: a present mount under a normal claim attempt is `CORRUPTED_DATA` | +| `build_seq` | in-memory only, `CasMountRuntime::next_build_seq`, reset to 1 on every process start | one process incarnation | orders builds *within* an epoch; combined with `writer_epoch` it gives GC a total order | + +The absent-epoch branch of `allocateWriterEpoch` is deliberately paranoid: absent with a +non-empty subtree is `CORRUPTED_DATA` (reset hazard); absent with an empty subtree decides by an +authoritative probe, never by plain-`get` absence, because a transport fault must not be flattened +into "not found". + +Global build ordering is the **pair** `(writer_epoch, build_seq)` compared lexicographically — the +exact comparison GC uses for eligibility. The durable authority for both is the mount object +itself: no mount means no deletion authority means nothing is swept. `min_active`, the oldest +in-flight `build_seq`, rides in the same mount object as the watermark floor; `UINT64_MAX` in +`min_active` is the farewell/retired sentinel, not a real build. + +## Mount claim outcomes {#claim-outcomes} + +The implementation does not expose a single named durable-slot enum; `claimMount` instead returns +a `MountClaimResult::Kind` together with a `MountPriorState` describing which certificate of death +(if any) justified a reclaim: + +| `Kind` | Meaning | +|---|---| +| `Claimed` | fresh claim (absent slot), same-`(uuid, epoch)` refresh, or a certified reclaim | +| `LiveDoubleStart` | same `server_uuid`, different `writer_epoch`, and no certificate of death yet — a live twin, wait it out | +| `ForeignOwner` | different `server_uuid` — refused unconditionally | +| `FencedSelf` | same `(uuid, epoch)`, but `gc_fenced` — terminal for *this* epoch; the caller must mint a fresh one | + +| `MountPriorState` | Certificate that justified the reclaim | +|---|---| +| `None` | no reclaim needed (fresh claim or same-epoch refresh) | +| `Clean` | the predecessor's own graceful farewell (`min_active == UINT64_MAX`) | +| `Fenced` | GC's own threshold-gated fence-out (`gc_fenced`) | +| `UncleanObserved` | this claimant's own token-stability observation held for the full `TTL + drift` window | + +## Behavioral mount-slot model {#mount-state-machines} + +Two coupled state pictures. Neither is a literal source enum — the durable slot is derived from +the claim outcomes above and is shown here as behavior, not as a type in the code: + +```mermaid +stateDiagram-v2 + [*] --> Absent + Absent --> Live: claimMount putIfAbsent, seq=1 + Live --> Live: keeper beat, putOverwrite seq+1 + Live --> Fenced: GC observes a stable token past threshold, gc_fenced=1, body preserved + Live --> Terminated: certified drain, terminal farewell (expires_at=now, min_active=MAX) + Fenced --> Live: same-uuid claim with a fresh writer_epoch, instant reclaim + Terminated --> Live: same-uuid claim with a fresh writer_epoch, instant reclaim + Live --> Live: same-uuid claim, proven-dead token via UncleanObserved + Fenced --> Fenced: same uuid and epoch claim, FencedSelf, no write + Live --> Absent: decommission tail, mount then epoch then owner tombstone + Terminated --> [*] +``` + +The in-process `PoolLifecycle` runtime, by contrast, is a literal enum (`CasMountRuntime.h`): + +```mermaid +stateDiagram-v2 + [*] --> Live: Pool constructed, fence unarmed + Live --> Live: mountWritable arms the fence + Live --> TransientNotLive: renewal failure, tripMountLost, lost=true + TransientNotLive --> Live: self-remount succeeds with a fresh epoch + TransientNotLive --> TransientNotLive: probe inconclusive, retry with backoff + TransientNotLive --> IdentityLost: pool meta and owner both authoritatively absent + TransientNotLive --> VanishedReplaced: foreign pool_id observed + Live --> VanishedForgotten: SYSTEM CAS FORGET + IdentityLost --> [*] + VanishedReplaced --> [*] + VanishedForgotten --> [*] +``` + +`IdentityLost`, `VanishedReplaced` and `VanishedForgotten` are terminal and absorbing: the remount +and GC threads self-exit, and there is deliberately no auto-revive — an identity disappearing +under a live mount is an operator-level event. + +## Mount, unmount, crash {#mount-lifecycle} + +**Writable open** runs in a strict order: bootstrap-residual proof, capability probe under a +random per-mount prefix, pool-meta create-or-validate, `validateServerRootId`, owner claim, +`allocateWriterEpoch`, mount claim and keeper adopt, materialization grace if the predecessor was +unclean (default 30 s), arm the fence, start background renewal. If the grace period consumed the +TTL, one fresh renewal re-anchors the deadline before the fence is armed. + +**Clean unmount:** stop and join the remount thread, drain the ref lanes, and only if the drain +*certified* quiescence write the terminal farewell (`expires_at_ms` already-expired, +`min_active = UINT64_MAX`). That sentinel is what lets a successor reclaim instantly. If the drain +did not certify, the keeper stops renewing and writes no farewell — an unearned farewell would let +a successor start mutating while a stale conditional write from the predecessor is still in +flight. + +**Crash:** no farewell; the renewal token freezes. Recovery is either the same server restarting +and waiting out the token-stability observation, or the GC leader fencing the slot first, after +which any reclaim is instant. + +**Permanent removal** of a dead replica (`Cas::decommissionPoolMember`, driven by +`SYSTEM CAS DROP POOL MEMBER '' FROM DISK ''`) claims the victim's mount slot +as an administrative writer with a no-wait policy (refuses immediately if the member is alive), +drops every ref-bearing namespace, sweeps manifest debris before the slot (deleting the mount +removes the watermark authority), drains staging and roots, then — only with zero warnings — +retires in order: mount, epoch, a final liveness re-check, owner tombstone. + +## `system.cas_mounts` {#mounts-table} + +A read-only view of the same heartbeat-floor computation GC uses: one `LIST` of +`gc/server-roots/` plus one `GET` per slot, zero writes, per-row fail-open (an undecodable body +becomes `state = 'corrupt'`, never an exception). Shows every `server_root_id` in the pool, including peers. + +| Column | Notes | +|---|---| +| `disk`, `server_root_id`, `server_uuid`, `hostname`, `process_id` | identity | +| `writer_epoch`, `renewal_sequence`, `started_at`, `expires_at`, `min_active_build_sequence`, `gc_fenced` | lease state (`DateTime64(3)` columns; the millisecond-integer field names live only in the internal `MountLease` struct and the on-disk body) | +| `state` | one of `live`, `expired`, `terminated`, `fenced`, `corrupt` | +| `is_leader`, `pending_reclaim`, `last_success_age_seconds`, `wedged_namespace_count` | GC health, process-local; **`NULL` on every peer row** — a process-local fact must never be stamped onto another server's row | +| `lifecycle`, `lifecycle_reason`, `lifecycle_detail`, `lifecycle_since` | the SQL surface for the in-process `PoolLifecycle` runtime above: `lifecycle` is one of `live`, `not_live`, `identity_lost`, `vanished`, `constructing`, `shutdown`; `lifecycle_reason` distinguishes `replaced` from `forgotten` for a `vanished` disk; `lifecycle_detail` carries the full diagnosis text; `lifecycle_since` is when the current non-live state began (`NULL` while live) | + +The lifecycle snapshot is I/O-free and ungated, so a not-live, never-started, or vanished disk +still produces a row instead of silently disappearing from the table. diff --git a/docs/en/antalya/cas/architecture/namespaces.md b/docs/en/antalya/cas/architecture/namespaces.md new file mode 100644 index 000000000000..2f622ff0e329 --- /dev/null +++ b/docs/en/antalya/cas/architecture/namespaces.md @@ -0,0 +1,172 @@ +--- +description: 'What a namespace is, the opaque life_id that qualifies every object it owns, the pool-wide namespace catalog, and a namespace lifetime end to end from first write to catalog-row deletion.' +sidebar_label: 'Namespaces' +sidebar_position: 10 +slug: /antalya/cas/architecture/namespaces +title: 'CAS Architecture — Namespaces' +doc_type: 'reference' +--- + +# CAS architecture — namespaces {#namespaces} + +A namespace (`Cas::RootNamespace`) is the opaque, per-table, per-server-root string under which one +table's part manifests and one ref table live — in practice something the wiring layer composes, +such as `srv1/` for an ordinary table or `shadow//` for a `FREEZE` +shadow. `CAS` never interprets its contents beyond a shape check (non-empty, no empty or reserved +path segment, at most 512 bytes). The [manifests-and-refs page](/antalya/cas/architecture/manifests-and-refs#ref-table) +covers the ref table one namespace owns; this page covers the namespace itself — its physical +identity, the catalog that is the sole authority for whether it exists, and its full lifetime from +first write to the catalog row's deletion. + +## `life_id`: the physical identity {#life-id} + +A namespace **name** can be reused — a table dropped and recreated keeps the same name. What must +never be reused is the **physical identity** any durable object under that name is keyed by, so +that a stale reader of the old incarnation can never be handed bytes belonging to the new one. That +identity is `life_id`: an opaque, pool-wide, randomly minted 128-bit value (two `thread_local_rng` +draws; retried on the astronomically unlikely zero draw, since `0` is reserved as "never a valid +life"). Internally it is the catalog's `incarnation` field, aliased as `NamespaceLifePhysicalId`; +paired with the namespace name it forms `NamespaceLifeId{ns, incarnation}` +(`Primitives/CasNamespaceLifeId.h`). + +`NamespaceLifeId` deliberately has no default construction and no conversion from a bare namespace +name: code holding only the name cannot address a ref object or a namespace file at all, so +forgetting the life qualifier is a compile error, not a runtime aliasing bug. The only legitimate +source of a `NamespaceLifeId` is `fromCatalogEntry` — reading it off one immutable catalog cut — +which is what makes "this life belongs to this name" a catalog fact rather than something a caller +could reconstruct incorrectly. + +`life_id` renders as 32 fixed-width lowercase hex digits and appears in exactly the two subtrees +that are life-owned (see the [storage-layout key table](/antalya/cas/architecture/storage-layout#key-table)): + +| Subtree | Contents | +|---|---| +| `cas/ns/stream//` | The immutable `_log`/`_snap` ref-transaction history | +| `cas/ns/state//` | The mutable `_ckpt` checkpoint and any namespace-owned `_files/` | + +Part manifests deliberately do **not** carry `life_id` — a manifest already has its own globally +unique identity (`{writer_epoch, build_sequence, manifest_ordinal}` under the server root, see the +[manifests-and-refs page](/antalya/cas/architecture/manifests-and-refs#part-manifests)) and needs no +further qualification. Loose mountpoint objects under `roots/` are outside namespace ownership +altogether and carry no `life_id` either. + +## The namespace catalog {#catalog} + +One pool-wide object, `cas/ref_catalog` (`Layout::refCatalogKey`), is the sole authority for which +namespaces exist. It is read on every fold round and every ref-table recovery, and mutated by one +token-`CAS` write per lifecycle transition. Its entries are canonically ordered by namespace bytes, +strictly ascending, with no duplicate name — both the encoder and the decoder enforce this, so an +out-of-order or duplicate-keyed catalog can never become durable. + +Each row (`CatalogEntry`) carries: + +| Field | Meaning | +|---|---| +| `ns` | The namespace name | +| `state` | `Creating`, `Live`, or `Removing` — see below | +| `incarnation` | The `life_id` for this row, nonzero, never reused | +| `creator` | The mounted writer's fence identity (server root, writer epoch, admission fence generation) that is creating this row — **required** iff `state == Creating`, **forbidden** otherwise | +| `removal_started_round` | The `GC` round observed when removal began — **required** iff `state == Removing`, absent otherwise | + +`NsState`'s three wire values (`Creating = 1`, `Live = 2`, `Removing = 3`) are append-only, exactly +like every other persisted enum in `CAS`: a catalog object written by one build is read by another, +so a value is never renumbered or repurposed. + +```mermaid +stateDiagram-v2 + [*] --> Creating: casAdmitEntry -- fresh random life_id, creator fence stamped + Creating --> Live: completeCreation -- publish genesis _ckpt, then flip, clear creator + Creating --> Creating: a live foreign creator fence -- retry later, no steal + Creating --> Live: reconcileStaleCreator finds the creator fence provably dead,
a fresh opener steals and completes it + Live --> Removing: beginRemoving -- table drop, stamps removal_started_round + Removing --> [*]: GC drains the row once a fold sealed positive cleanup evidence + [*] --> Creating: a fresh createNamespace call, only once the old row is fully absent -- brand new life_id +``` + +A row's own state machine is linear per row (`Creating → Live → Removing → gone`); what makes the +catalog non-linear as a whole is that a stalled `Creating` row can resolve two different ways +depending on whether its creator fence is still alive, and that a name only becomes creatable again +once its prior row is completely gone — both shown above. + +## Lifetime end to end {#lifetime} + +### Creation, on first write {#creation} + +There is no explicit "create namespace" statement; a namespace is born the first time anything +resolves its ref table (`CasRefLedger::resolveNamespaceLife`, bounded at 32 loop attempts). If the +catalog has no row for the name at all, the resolving mount admits a `Creating` entry stamped with +its own creator fence and a freshly minted `life_id` +(`CasRefCatalog::createNamespace` → `casAdmitEntry`). Two more steps make it usable: + +1. **Publish the genesis checkpoint.** The first `_ckpt` ever written for this `life_id` carries + `life_epoch = creator.writer_epoch` — the only writer that will ever know this namespace's + genesis epoch. +2. **Flip to `Live`.** One token-`CAS` moves the row from `Creating` to `Live` and clears `creator`. + +Both steps re-check the resolving mount's own fence before writing, so a mount that lost its lease +mid-creation reports `FencedOut` rather than silently completing. Several openers racing the same +brand-new name all observe "no entry", but only one wins the admit; the rest see `Superseded` and +simply re-read the catalog, landing on the winner's `Creating` row. + +A `Creating` row under a **different** mount's creator fence is not this opener's problem to force: +if that fence is still provably alive, the opener retries later; only once the fence is provably +dead (the same mount-lease terminality check `GC`'s heartbeat floor uses) does +`reconcileStaleCreator` let a fresh opener steal the row onto its own fence and finish the two steps +above itself. + +### Removal {#removal} + +Dropping a table (`DROP TABLE`, and every operation that reduces to it) calls +`CasRefLedger::dropNamespace`. It closes the namespace's local positive-mutation lane first — new +positive writers are refused while the in-flight ones drain — then transitions the catalog row from +`Live` to `Removing` in one token-`CAS` (`beginRemoving`, stamping `removal_started_round` from the +currently observed `GC` round), then appends **one** ref-log transaction that removes every current +committed and precommit binding and ends with a terminal `RemoveNamespace` op. Removal is never +refused by an admission check — Constraint 13 in the catalog's own spec — it always succeeds once +the fence holds. + +Nothing is deleted by the writer at this point. No blob, no manifest, no ref-log object physically +disappears here — only pointers move, exactly like an ordinary [`DROP TABLE`](/antalya/cas/architecture/part-lifecycle#operation-mapping) +on any other ref. + +### What `GC` does with a `Removing` namespace {#gc-and-removal} + +The terminal `RemoveNamespace` transaction is folded like any other ref-log record, during the +[round's fold phases](/antalya/cas/architecture/garbage-collection#the-round). Folding it stamps +positive **cleanup evidence** directly onto that `life_id`'s row in the new fold seal — there is no +physical listing and no `Pending`/`Completed` handshake; the evidence is a pure fact about which +ref-log transaction folded. + +The **next** round's `pre_fold_ref_drain` phase is what actually removes the catalog row: it reads +the just-adopted parent fold seal, and for every `Removing` row whose life carries durable cleanup +evidence, it exact-`CAS`-deletes the catalog entry before that round does anything else. This +two-round shape — evidence sealed in round *n*, catalog row deleted in round *n+1* — is why removal +needs no separate physical-emptiness proof: by the time the row is deleted, a fold has already +proven its ref history is fully drained. + +### What disappears, and when {#what-disappears} + +| Object class | Reclaimed by | When | +|---|---|---| +| Catalog row (`cas/ref_catalog` entry) | `GC` phase 2, `pre_fold_ref_drain` | The round after the fold that sealed cleanup evidence for this life | +| Part manifest bodies | Ordinary owner-removal ([phase 15](/antalya/cas/architecture/garbage-collection#the-round)) for anything that had a committed or precommit binding, the [orphan-manifest sweep](/antalya/cas/architecture/manifests-and-refs#orphan-sweep) for anything that never got that far | As each owning ref is dropped by the removal transaction itself, independent of the catalog row | +| Blob bodies | The ordinary condemn/graduate/delete pipeline | Whenever the manifests that named them stop being live, same as any other blob | +| Ref stream/state objects (`_log`, `_snap`, `_ckpt`, `_files`) under the dead `life_id` | The perpetual namespace janitor ([phase 16](/antalya/cas/architecture/garbage-collection#the-round)) | Best-effort, one bounded `LIST` page at a time, whenever it next lists a key whose `life_id` a fresh catalog cut no longer names — independent of, and not gated on, catalog-row deletion | + +The janitor is leak-only: it never fails a round, never blocks progress on an unreadable key, and a +crash mid-page simply leaves debris for its next page. + +### Recreate while removing {#recreate-while-removing} + +A fresh `createNamespace` call for a name whose catalog row is still `Live` or `Removing` is +refused outright — internally this is a misuse `LOGICAL_ERROR`, because the higher-level open loop +(`resolveNamespaceLife`) filters that case out first and reports a typed retry-later error instead: +"creation waits for its terminal fold and catalog removal to complete". A caller that keeps +resolving the same name simply keeps retrying until the row is gone. + +Once `pre_fold_ref_drain` has deleted the row, the name is free again, and the very next opener mints +a **brand new**, independently random `life_id` — never the retired one. That is the whole answer to +"what happens on recreate": the old physical identity is never revived, so every key ever written +under it — its `_log`, its `_snap`, its `_ckpt`, its `_files` — stays permanently addressed by a +value nothing will ever mint again, and a reader still holding the old `NamespaceLifeId` observes +only stale-or-absent data, never a byte that belongs to the new incarnation. diff --git a/docs/en/antalya/cas/architecture/part-lifecycle.md b/docs/en/antalya/cas/architecture/part-lifecycle.md new file mode 100644 index 000000000000..5a61d0490dc1 --- /dev/null +++ b/docs/en/antalya/cas/architecture/part-lifecycle.md @@ -0,0 +1,148 @@ +--- +description: 'The part-add protocol from local build through blob upload to promote, its nine crash points and their cleaners, and how each MergeTree operation maps onto it.' +sidebar_label: 'Part lifecycle' +sidebar_position: 6 +slug: /antalya/cas/architecture/part-lifecycle +title: 'CAS Architecture — Part Lifecycle' +doc_type: 'reference' +--- + +# CAS architecture — part lifecycle {#part-lifecycle} + +Publishing a `MergeTree` part on a `CAS` disk is one durable protocol, +`stageManifest → precommitAdd → putBlob → promote`, driven by `Cas::PartWriteTxn` +(`Pool/CasPartWriteTxn.cpp`). This page walks that protocol end to end: local build, the durable +order and why each step is where it is, every crash window and who cleans it up, and how each +`MergeTree`-level operation (insert, merge, mutation, detach, …) maps onto it. Manifest structure +and the ref table it writes into are covered on the +[manifests-and-refs page](/antalya/cas/architecture/manifests-and-refs); the fetch-side protocol +for replicated parts is on the [replication page](/antalya/cas/architecture/replication). + +## The protocol {#protocol} + +```mermaid +sequenceDiagram + autonumber + participant MT as MergeTree + participant TX as CA transaction overlay + participant PW as PartWriteTxn + participant S3 as Object store + + rect rgba(140,190,140,0.12) + Note over MT,S3: Phase A -- local build, nothing durable, nothing visible + MT->>TX: writeFile data.bin + TX->>TX: classify: blob class spills and hashes to scratch or S3 staging + MT->>TX: writeFile count.txt, columns.txt, ... + TX->>TX: buffer small files in memory as inline candidates + MT->>TX: moveDirectory tmp_insert to final name + Note over TX: pure overlay re-key, not a publish + end + + rect rgba(120,160,255,0.12) + Note over MT,S3: Phase B -- publish, per part, serially + MT->>TX: commit + TX->>PW: stageManifest entries + PW->>S3: PUT manifest, write-once, no preliminary HEAD + PW->>S3: append ref-log PRECOMMIT, plus NamespaceBirth if needed + Note over PW: precommit durable, the observe gate opens + TX->>PW: fan out blob uploads, one task per unique BlobRef + par blob 1 + PW->>S3: HEAD / conditional PUT / adopt + and blob 2 + PW->>S3: ... + end + PW->>PW: merge upload results on the owning thread, one no-throw swap + TX->>PW: promote + PW->>S3: GET and validate the precommit manifest body + PW->>S3: append ref-log txn: retire old committed, Precommit to Committed, SetPublishedAt + Note over PW: commit durable, then retire the build sequence + end +``` + +**Phase A — staging.** The transaction is an eager overlay, not a queue: `writeFile` immediately +classifies the path and either spills bytes to a hashing buffer or holds them in memory as an +inline candidate. Blob-class files stage to local scratch by default, or — when `staging_backend` +is `s3` and the mount-time conditional-copy probe passed — to an S3 staging object written as +`[header][payload]`, so that the later promote is a verbatim server-side copy. The `tmp_ → final` +rename is a pure overlay re-key; the durable publish happens only in `commit`. + +**Step 1 — `stageManifest`.** Caps (see the +[manifests-and-refs page](/antalya/cas/architecture/manifests-and-refs#part-manifests)) are +checked before the write; the id is minted as `{epoch, build_seq, ordinal++}`; the body goes out +with a conditional create and no preliminary `HEAD`. Both a definite failure and an unresolved +outcome throw retry-later. + +**Step 2 — `precommitAdd`.** The intent — target namespace, final ref name, manifest — is recorded +before the append, because an unresolved append may have landed anyway. One ref-log transaction +adds the precommit binding. A same-name birth is refused with retry-later while the catalog still +says `Removing`; once the predecessor row is absent, creation receives a new opaque life id and +starts its own stream. On return the precommit is durable, and only now may the writer adopt +existing blobs. + +**Step 3 — blob upload fan-out.** One task per unique `BlobRef`, deterministic dispatch order, one +pre-sized result slot per ref (see the write-path sequence on the +[blob-protocol page](/antalya/cas/architecture/blob-protocol#conditional-write-sequence)). The +calling thread only submits and joins, never occupies a pool slot, so a pool of size one degenerates +to a correct serial run and can never deadlock. The contract is merge-nothing: if any task threw, +nothing is merged and the first error in dispatch order is rethrown. Results are folded into the +dependency set on the owning thread, into a copy, committed by one no-throw swap. Pool size is the +server setting `cas_blob_upload_pool_size` (default 16). + +**Step 4 — `promote`.** Reads and revalidates the precommit manifest body once; sets the commit +state to Uncertain before the append — past that point, failure is no longer proof of the negative +— then checks that the precommit is still the live owner and revalidates leaves. Tokened leaves +are skipped because they are edge-protected; tokenless leaves must be evidence adopts, trusted +through the durable manifest edge with no per-file `HEAD`; anything else is a `LOGICAL_ERROR`. The +whole thing lands as one ref-log record: optional retirement of the old committed binding, the pure +Precommit-to-Committed owner move, and `SetPublishedAt`. Promotion emits no blob deltas — the +manifest never loses an owner, so it is net zero. + +## Crash points and their cleaners {#crash-points} + +This table is the single best summary of the design's crash-safety story: every row leaks +something recoverable; no row loses data or leaves a dangling reference. + +| # | Crash window | Left behind | Who cleans it | +|---|---|---|---| +| C1 | During staging | Local temp files, or S3 staging objects | Local: unconditional cleanup plus buffer destructor. S3: the mount's own staging sweep at next mount — never deleted on abort | +| C2 | After `stageManifest`, before `precommitAdd` | An unreferenced manifest body | Writer's best-effort exact-token delete; durable backstop is the orphan-manifest sweep | +| C3 | `precommitAdd` returned Unresolved | A possibly-live precommit binding | Intent recorded pre-append; `abandon` appends the exact removal, tolerating absence. The body is never writer-deleted | +| C4 | Between `precommitAdd` and `promote` | A live precommit plus uploaded blobs | No resume path exists. Removed by `abandon`, else by a fenced successor's stale-precommit sweep | +| C5 | Mid blob fan-out | Already-uploaded blobs | Nothing merged; blobs become `GC`-reclaimable debris; the part is not published | +| C6 | `promote` append Unresolved | The ref may or may not be committed | Commit state Uncertain — the relink layer maps this to "retry the whole fetch", never to a byte fetch | +| C7 | A later part throws after earlier parts published | A partial multi-part commit | Precise rollback: drop only the refs this call created, matching the exact manifest — never clobbers a concurrent writer's repoint | +| C8 | Transaction destroyed uncommitted | Open builds | Destructor abandons every build | +| C9 | Namespace dropped mid-build | — | One atomic flag; every further op fails closed at the alive check | + +## The repoint {#repoint} + +Writing into an already-committed part — an `ALTER`-style metadata rewrite, or any standalone +write against a committed source — never mutates the existing manifest. It writes a **new** +manifest over the (possibly partly reused) blob set and moves the ref to it in one ref-log record. +Unchanged columns are adopted by hash through a tokenless evidence dependency with no `HEAD` and no +`GET`; changed columns are fresh uploads. A repoint therefore costs zero bytes moved for the +carry-forward portion of the file set — only the changed content re-uploads. + +## How each MergeTree operation maps {#operation-mapping} + +| Operation | CAS mechanics | +|---|---| +| `INSERT` | The canonical path above. Projections ride the parent part's transaction | +| Merge | Identical for the output part. `.tmp_proj → .proj` is an entry-prefix re-key inside the staged manifest, not a rename | +| Mutation | `createHardLink` per unchanged file: a source staged in *this* transaction copies the entry and its pending-blob record; a **committed** source records a tokenless evidence dependency with no `HEAD` and no `GET`. A mutation is a manifest rewrite where zero bytes move for the carry-forward | +| `ALTER` / metadata rewrites | Standalone writes into a committed part, i.e. a repoint | +| `DROP PART` | `removeDirectory` drops the ref and clears any per-file removal marks — one ref-drop, zero repoints | +| `DROP TABLE` / `DETACHED` / `UNFREEZE` | A namespace or prefixed-ref drop. Blobs are never deleted here — removal is pointer-unlink plus deferred `GC` | +| `RENAME TABLE` | Republishes every ref and verbatim file into the new namespace, then drops the old one. Not atomic across namespaces, but idempotent and re-drivable — true atomicity would need a move journal and is out of scope | +| `FREEZE` / `BACKUP` / `RESTORE` / cross-disk `MOVE` | Each wraps the whole clone in one disk transaction, because a CAS part is one atomic unit | + +`FREEZE` is the one operation that materializes real bytes into a genuinely separate shadow +namespace rather than reusing a table's own ref names — that shadow namespace is a `GC` +reachability root, and `UNFREEZE` releases its refs. + +## Reads while a part is in flight {#in-flight-reads} + +Read-your-writes for a part still inside an open transaction is served by an explicit overlay +rather than by any durable object — `tryGetInFlightStorageObjects`, `tryReadFileInFlight`, +`listInFlightDirectory`. One deliberate subtlety: the bare part directory reports as absent in the +overlay, so cleanup of a deduplication-rejected temporary part does not mistake it for a real part. diff --git a/docs/en/antalya/cas/architecture/read-path.md b/docs/en/antalya/cas/architecture/read-path.md new file mode 100644 index 000000000000..c92634dd9e74 --- /dev/null +++ b/docs/en/antalya/cas/architecture/read-path.md @@ -0,0 +1,84 @@ +--- +description: 'How a CAS read resolves a ref to a manifest and then to ranged blob reads, and the two caches — manifest decode and part-folder view — that sit on that path.' +sidebar_label: 'Read path' +sidebar_position: 9 +slug: /antalya/cas/architecture/read-path +title: 'CAS Architecture — Read Path' +doc_type: 'reference' +--- + +# CAS architecture — read path {#read-path} + +A `CAS` read never touches a classical local-metadata path: there is no local directory listing to +consult, only a ref resolve followed by object-store reads. This page covers the three ways a file +access is served, the full chain for the common case, the two caches that sit on that chain, and +how a part still open inside a write transaction serves its own reads. + +## How a file access is served {#access-kinds} + +| Access kind | How it is served | S3 cost | +|---|---|---| +| Inline entry — small files such as `count.txt`, `columns.txt` | Decoded straight out of the manifest body | Zero additional operations | +| Blob-backed file — `.bin`, marks, large `primary.idx` | Ranged `GET` bounded by `[header_len, header_len + blob_size)` | One `GET` per column file per part open | +| Verbatim file — `roots/…` objects | Plain object read, no `CAS` indirection | One `GET` | + +The full chain for a blob-backed file is: resolve the ref, read the manifest, look up the path, +build a blob view plan, ranged `GET`, then `ReadBufferFromFileView`. Because the payload always +starts at a pool-constant offset (the manifest's `blob_header_len`), no header parse is needed to +locate content — see the [envelope format](/antalya/cas/architecture/storage-layout#envelope-format) +on the storage-layout page. + +Part manifests themselves are read whole after opening the object: there is no on-disk random +access, `seek`, or streaming requirement for their entry records — a manifest is small enough that +decoding the whole body is cheaper than any partial-read machinery would be. + +## The two caches {#caches} + +| Cache | Keyed by | Setting | Default | What still hits the network | +|---|---|---|---|---| +| Manifest decode cache | `(ManifestId, Token)` | `manifest_decode_cache_bytes` | 128 MiB | A mandatory `HEAD` on **every** access, cache hit or miss | +| Part-folder view cache (`Cas::CachedPartFolderAccess`, `Parts/PartFolderAccess.h`) | Part ref key | `part_folder_cache_bytes`, `part_folder_cache_max_entries`, `part_folder_cache_max_entry_bytes` | 64 MiB / 10 000 entries / 16 MiB | Its `ForceFresh` policy re-proves the manifest body via that same mandatory `HEAD`, paced by `part_folder_validate` (`always` \| `never` \| `age `) | + +**The `HEAD` is mandatory even on a cache hit** — the page's most counter-intuitive fact, because it +means a cache hit still costs one object-store round trip: + +```mermaid +flowchart TD + A["readManifestShared(ManifestId)"] --> B["HEAD the manifest key"] + B -->|"absent"| C["throw FILE_DOESNT_EXIST --
a live ref must never name a missing object"] + B -->|"present, token t"| D{"cache lookup (ManifestId, t)"} + D -->|hit| E["return the cached decode -- no GET"] + D -->|miss| F["GET the body"] + F --> G{"body's own ref and namespace
match the key?"} + G -->|no| H["throw CORRUPTED_DATA"] + G -->|yes| I["decode, insert into cache keyed by (ManifestId, t), return"] +``` + +The `HEAD` is what proves the live ref still names an existing object — the no-dangle invariant — +and it supplies the token that keys the cache; only then is the decode cache consulted. On a miss, +the `GET` is followed by the two identity checks in the diagram, each `CORRUPTED_DATA` on failure. +Only a fully validated decode enters the cache. Setting either cache's byte budget to `0` disables +retention while leaving the `HEAD`-and-validate sequence intact — a cache is purely an +optimization, never a trust boundary. + +The part-folder view cache is invalidated on every promote and repoint, and is single-flight on a +cold build: concurrent readers of the same not-yet-cached view coalesce into one build rather than +racing independent `GET`s. + +## Reads while a part is still being written {#in-flight-reads} + +An in-flight part inside an open write transaction is not yet visible through the ordinary ref +resolve — reading it goes through the same explicit overlay used for read-your-writes, covered on +the [part-lifecycle page](/antalya/cas/architecture/part-lifecycle#in-flight-reads). The bare part +directory itself reports as absent in that overlay, precisely so that cleanup of a rejected +temporary part is never mistaken for a real, resolvable part. + +## Diagnostic and read-only access {#read-only-access} + +A read-only or diagnostic opener of a `CAS` disk (`ca-fsck`, `ca-gc-dryrun`, and similar tools) +must not claim mount ownership, schedule `GC`, or mint writer state — read-only enforcement sits +below the ordinary facade checks, at the backend layer itself. A mounted `Pool` caches its ref +table and does not re-recover it on every read; a diagnostic tool that deliberately performs a +fresh cold recovery on each pass can therefore observe a **less** stale ref table than a live +mounted read, which is intentional for tools whose entire purpose is catching drift a live mount +would not notice. diff --git a/docs/en/antalya/cas/architecture/replication.md b/docs/en/antalya/cas/architecture/replication.md new file mode 100644 index 000000000000..cc767f7fe115 --- /dev/null +++ b/docs/en/antalya/cas/architecture/replication.md @@ -0,0 +1,118 @@ +--- +description: 'Fetch by relink between two replicas sharing a pool: the gates in order, what actually seals commit-before-release, and detach/attach/drop.' +sidebar_label: 'Replication' +sidebar_position: 7 +slug: /antalya/cas/architecture/replication +title: 'CAS Architecture — Replication' +doc_type: 'reference' +--- + +# CAS architecture — replication {#replication} + +When two `ReplicatedMergeTree` replicas share a `CAS` pool, a fetch should move **no bytes** — the +receiver already has access to the same blobs the sender does. The mechanism is a three-phase +handshake, fetch by relink, layered directly on the ordinary interserver part-fetch protocol. This +page covers the handshake, the gates that decide whether it fires, what actually makes it safe +against a concurrent `GC` round, and how detach/attach/drop reduce to the same primitives. The +writer owns table semantics and part publication (see the +[part-lifecycle page](/antalya/cas/architecture/part-lifecycle)); `GC` owns ref-log folding and +physical cleanup (see the [garbage-collection page](/antalya/cas/architecture/garbage-collection)) +— ordinary replication traffic never reads `gc/state` or waits on a `GC` round. + +## The handshake {#handshake} + +Only two of the three phases are round trips to the sender — the offer and the confirm. The +publish and the promote are the receiver's own writes to the pool. + +```mermaid +sequenceDiagram + autonumber + participant R as Receiver + participant Snd as Sender + participant S3 as Shared pool + + R->>Snd: GET part, cas_pool_uuid = R's pool uuid, client_protocol_version = 11 + Note over R: advertising 11 is a promise to confirm before promoting + Snd->>Snd: same disk pool uuid? identity, never endpoint plus prefix + Snd->>S3: resolve the offer once -- manifest bytes and confirm token from the SAME view + Snd-->>R: cookie cas_relink = part_manifest_v2, cookie cas_source_token = ..., body = manifest bytes + Note over Snd: sender is fire-and-forget -- it releases the part here + + rect rgba(120,160,255,0.12) + Note over R,S3: T1 -- publish, the plus-one lands first + R->>S3: adopt entries by evidence, no HEAD, no bytes, stageManifest fresh receiver-local id, precommitAdd + Note over R: the sender's ManifestRef, namespace and digest are ignored -- only entries are used + end + + rect rgba(255,190,120,0.15) + Note over R,Snd: T2 -- confirm + R->>Snd: POST cas_confirm = token + Snd->>Snd: confirmExactRef, zero object-store I/O, never throws + Snd-->>R: cookie cas_confirm_answer = yes or unproven + end + + alt answer is yes + R->>S3: T3 -- promote, ref published + else anything else -- unproven, missing cookie, timeout, transport error + R->>R: throw a locally generated NETWORK_ERROR, retry later + Note over R: never a byte re-request -- that would go back to the very source whose state is in doubt + end +``` + +## The gates, in order {#relink-gates} + +| # | Gate | What it enforces | +|---|---|---| +| 1 | Pool identity | The receiver advertises `cas_pool_uuid`; the sender offers relink only if its own disk's pool uuid is **equal**. Matching by endpoint and prefix was tried and rejected — a minted pool uuid is the identity | +| 2 | Protocol version 11 | On the receiver side, advertising it is a promise to run the confirm round trip before promoting | +| 3 | One resolution for two outputs | The manifest bytes and the confirm token come from the **same** view. Two separate calls would allow a repoint in between and hand the receiver a token naming a manifest whose entries it never adopted | +| 4 | The receiver trusts nothing from the wire but the entry list | The sender's manifest id, namespace and payload digest are ignored; the target namespace and ref come from the receiver's own router, and manifest path hygiene is validated at decode | +| 5 | The confirm is I/O-free and fail-closed | A cold, evicted, unfenced or terminal mount answers `Unknown`. `No` and `Unknown` both go on the wire as `unproven`, because the fence check is evaluated last, so a `No` cannot be distinguished from "cannot prove it right now" | +| 6 | Only the literal `yes` authorizes promotion | Everything else — including a timeout — is one outcome: throw and retry later | +| 7 | Promote outcomes are three-way | `Committed` proceeds; a **proven** not-committed state (body-absent precommit, precommit no longer live owner, ref conflict) falls back to a byte fetch; `Unresolved` **throws**, because returning "fall back" there would publish the part twice | + +The byte-fetch fallback is bounded: it re-invokes the fetch with relink disabled, which stops the +receiver advertising its pool uuid, which stops the sender offering relink — so the relink path +cannot be entered twice for one fetch. Byte-fetched files content-address and dedup on arrival +anyway, so falling back never loses the dedup property, only the zero-byte-move property for that +one fetch. + +## What actually seals "commit before release" {#relink-seal} + +The receiver's `+1` — its precommit binding — is durable **before** the sender is asked anything, +and any removal of the sender's own binding is appended strictly after that `+1` is in the ref +log. That ordering, steps T1 then T2 then T3, is the whole seal. + +This does **not** establish that every subsequent `GC` fold *sees* that `+1` under every listing +behavior: a configuration with one incomplete listing page can, in principle, let a fold miss a +freshly published edge. A confirmed relink therefore proves only "the source still holds exactly +this manifest right now", not "no future fold can ever miss this edge" — `ca-fsck`'s +reachable-but-absent scan is the backstop for that gap, not the relink protocol itself. Relink +also races `GC` in the ordinary sense any writer does: between the sender encoding its offer and +the receiver's promote, `GC` on the shared pool may condemn a blob that was live only through the +sender's own ref. The [writer-versus-GC race](/antalya/cas/architecture/blob-protocol#writer-gc-race) +on the blob-protocol page is what makes that interleaving safe — revival is re-upload only, and the +receiver's evidence-adopt is protected by its own durable precommit edge exactly like any other +writer's adopt. + +A fetch whose source part is still a live, held `DataPartPtr` on the sender's own replica — the +common case for a local, same-process relink — keeps the source pinned through the destination's +commit by ordinary part-lifetime rules, independent of the ref-log seal above. + +## Detach, attach, drop {#detach-attach-drop} + +A detached part is **not** a separate namespace — it is a ref in the table's own namespace with a +`detached/` prefix (the same is true of `moving/`). Only `FREEZE` uses a genuinely separate shadow +namespace, which the ownership check deliberately refuses to claim, so a frozen part can never be +relink-confirmed. + +`DETACH`, `ATTACH`, `delete_tmp_` cleanup, and merge-result renames all reduce to the same two +moves: re-key any *staged* source into the destination, then `republishRef(src → dst)` for any +*committed* source. `republishRef` re-reads the source manifest freshly, publishes an +equivalent-entry manifest under the destination ref — a **new** manifest id, with blobs untouched +and adopted by evidence — then drops the source ref. A destination that already exists with +identical entries just drops the source, an idempotent re-drive; one with different entries +throws. + +Manifests are therefore per-ref and never moved: a detach creates a new manifest for +`detached/` and retires the old one, and the blobs' net in-degree is unchanged. diff --git a/docs/en/antalya/cas/architecture/storage-layout.md b/docs/en/antalya/cas/architecture/storage-layout.md new file mode 100644 index 000000000000..9b4336a6bf2f --- /dev/null +++ b/docs/en/antalya/cas/architecture/storage-layout.md @@ -0,0 +1,160 @@ +--- +description: 'S3 key layout and on-disk text-object formats used by the content-addressed storage (CAS) MergeTree disk backend.' +sidebar_label: 'Storage layout' +sidebar_position: 2 +slug: /antalya/cas/architecture/storage-layout +title: 'CAS Architecture — Storage Layout' +doc_type: 'reference' +--- + +# CAS architecture — storage layout {#storage-layout} + +Every key in a pool is built by one class, `Cas::Layout` (`Formats/CasLayout.h`), which owns +exactly the pool prefix. Every persisted object opens with a one-line JSON envelope header, and +control-plane bodies are JSON Lines — one JSON object per line, sorted where the object is a log +or a set of entries (`Formats/README.md`; see [Envelope format](#envelope-format) below for which +parts are a single JSON object versus JSON Lines versus raw payload bytes). The format is +deliberately this plain: any object can be fetched and read with ordinary line-oriented tools +while debugging, and a new field is additive — a tolerant reader skips it — so the format evolves +without a migration. + +## Key table {#key-table} + +All key patterns are shown under the pool prefix. A **namespace** is the opaque per-table string +under which one `MergeTree` table's part manifests and ref history live: for a live table it is +the table's canonical disk path (`store//`, `@cas@`-marked) prefixed by the owning +server's `server_root_id`, and a backup gets its own `shadow/…` namespace instead; `Cas::Layout` +only validates a namespace's shape and never interprets its contents. + +| Key pattern | Object | Codec | Writer | +|---|---|---|---| +| `_pool_meta` | pool identity + floors | `cas_pool_meta` | pool create/admit | +| `blobs///` | blob envelope + payload | `cas_blob` | uploads | +| `blobs///.meta` | blob freshness sidecar | `cas_blob_meta` | dedup/GC | +| `cas/ns/stream//_log/-.zst` | ref transaction log | `cas_ref_log` | writer commit path | +| `cas/ns/stream//_snap/-.zst` | complete ref table snapshot | `cas_ref_snap` | writer/GC fold | +| `cas/ns/state//_ckpt` | mutable per-life checkpoint | `cas_ref_ckpt` | writer/GC fold | +| `cas/ns/state//_files/` | namespace-owned verbatim file | — (raw passthrough) | upper layers | +| `cas/manifests//-/.zst` | part manifest | `cas_part_manifest` | part build | +| `gc/state` | GC state (incl. GC lease) | `cas_gc_state` | GC | +| `gc/hb` | GC leader heartbeat | `cas_gc_hb` | GC | +| `gc/maintenance_state` | leak-only namespace-janitor cursor | `cas_gc_maintenance_state` | future janitor | +| `gc/gen//attempt//fold_seal` | fold seal (deterministic) | `cas_fold_seal` | GC | +| `gc/gen//attempt//blob_target//` | GC source-edge run segment | `cas_run` | GC | +| `gc/gen//attempt//outcomes//.zst` | GC outcome log | `cas_gc_outcomes` | GC | +| `gc/server-roots//owner` | server-root owner singleton | `cas_owner` | mount | +| `gc/server-roots//epoch` | server-root epoch singleton | `cas_epoch` | mount | +| `gc/server-roots//mount` | mount lease (incl. `min_active` watermark) | `cas_mount_lease` | mount | +| `roots/` | loose mountpoint object, verbatim | — (never interpreted) | upper layers | +| `staging//…` | S3-native upload staging scratch | — | writer, own mount only | + +`` is `ch128`, `xxh3`, or `sha256` — the hash algorithm is a path segment because one pool may +legally hold blobs under several algorithms at once. `` is a flat two-character S3 key +shard for request-fan-out, unrelated to the separate `gc_shards` GC-internal reduction fan-out +(which appears only inside `gc/gen/…` keys and routes by the digest's high 64 bits, read +big-endian). Discovery LISTs use fixed prefixes: `cas/ns/stream/`, `cas/ns/`, `cas/manifests/`, +`blobs/` (deliberately without the algorithm segment, so one recursive LIST covers every +algorithm), `roots/`, `gc/server-roots/`. `staging/` is a top-level sibling that no GC LIST ever +touches — it is reclaimed only by its own server's next mount. + +## Envelope format {#envelope-format} + +Every persisted CAS metadata object is text: a header line, a body, and an optional trailer. + +``` +{"type":"cas_","v":N} <- header line, always present + <- one JSON object, sorted NDJSON records, + or a descriptor + raw payload zone +{"n":…} <- optional trailer (record/entry count) +``` + +`v` is the only version field; a reader rejects `v` above what the build supports with +`UNKNOWN_FORMAT_VERSION`, checked before the body. A `.zst` key suffix means, exactly, that the +object kind's compression policy is `Always`: the object is stored as one zstd frame with the +checksum flag on, and its declared content size is checked against a per-kind cap before +allocation. Always-small and deterministic kinds (`cas_ref_ckpt`, `cas_blob_meta`, `cas_fold_seal`, +`cas_run`, …) are stored raw, with no `.zst` suffix. + +The blob envelope is a special case of the header/body shape: a JSON descriptor padded with ASCII +spaces to a pool-constant `blob_header_len` (256 bytes, a `cas_pool_meta` field), terminated by +`\n`, so the raw payload always starts at that fixed offset with no header parse needed to locate +it. The part manifest is the other `PayloadHybrid` kind: text header, descriptor, sorted NDJSON +entry records, `{"n":…}` trailer, then a banner-framed raw payload zone for small inline file +bytes. + +## Codec table {#codec-table} + +Condensed from the authoritative traits table in `CasFormat.cpp` (`TRAITS`, asserted complete by +`gtest_cas_text_format.cpp`). + +| Type string | Family | Key strictness | Compression | +|---|---|---|---| +| `cas_blob` | `PayloadHybrid` | tolerant | never (raw, fixed offset) | +| `cas_blob_meta` | `Control` | tolerant | never | +| `cas_pool_meta` | `Control` | tolerant | never | +| `cas_ref_log` | `Control` | tolerant | always (`.zst`) | +| `cas_ref_snap` | `Control` | tolerant | always (`.zst`) | +| `cas_ref_ckpt` | `Control` | strict | never | +| `cas_ref_catalog` | `Control` | strict | never | +| `cas_part_manifest` | `PayloadHybrid` | tolerant | always (`.zst`) | +| `cas_run` | `RecordStream` | strict | pinned raw | +| `cas_fold_seal` | `Control` | strict | pinned raw | +| `cas_gc_state` | `Control` | tolerant | never | +| `cas_gc_hb` | `Control` | tolerant | never | +| `cas_gc_outcomes` | `Control` | tolerant | always (`.zst`) | +| `cas_gc_maintenance_state` | `Control` | strict | never | +| `cas_owner` | `Control` | tolerant | never | +| `cas_epoch` | `Control` | tolerant | never | +| `cas_mount_lease` | `Control` | tolerant | never | + +"Strict" means unknown keys are rejected rather than skipped, used for objects where every field +decides a durability or cleanup decision (`cas_ref_ckpt`, `cas_ref_catalog`, `cas_fold_seal`, +`cas_run`, `cas_gc_maintenance_state`); a `!`-prefixed key is always critical regardless of the +kind's strictness. "Pinned raw" objects (`cas_run`, `cas_fold_seal`) need stable bytes across +re-encodes for deterministic-artifact adoption, so their bytes are never recompressed once +written. `cas_blob` and `cas_part_manifest` are the `PayloadHybrid` family: a text descriptor +followed by a raw payload zone, rather than a single JSON body. + +## Worked example tree {#worked-example} + +Pool prefix `ca-pool`, server root `srv1`, one `Atomic` table, one part `all_1_1_0` with one blob +column file, written at `writer_epoch = 1, sequence = 3`: + +``` +ca-pool/_pool_meta + +ca-pool/cas/ns/stream/0123456789abcdef0123456789abcdef/_log/0000000000000001-0000000000000003.zst +ca-pool/cas/ns/stream/0123456789abcdef0123456789abcdef/_snap/0000000000000001-0000000000000003.zst +ca-pool/cas/ns/state/0123456789abcdef0123456789abcdef/_ckpt + +ca-pool/cas/manifests/srv1/store/3f2/3f2a1b7c-…-abcdefabcdef@cas@/0000000000000001-0000000000000003/000001.zst + +ca-pool/blobs/xxh3/a1/a1b2c3d4e5f60708b1c2d3e4f5061728 +ca-pool/blobs/xxh3/a1/a1b2c3d4e5f60708b1c2d3e4f5061728.meta + +ca-pool/roots/srv1/clickhouse_access_check_8f3a1c2d + +ca-pool/gc/state +ca-pool/gc/hb +ca-pool/gc/server-roots/srv1/{owner,epoch,mount} +ca-pool/gc/gen/7/attempt/1/fold_seal +ca-pool/gc/gen/7/attempt/1/blob_target/0/1 +ca-pool/gc/gen/7/attempt/1/outcomes/1/0.zst + +ca-pool/staging/srv1/ +``` + +`0123456789abcdef0123456789abcdef` is the opaque physical `life_id` the catalog maps the table's +namespace to; the ref log and snapshot keys reuse the same `RefTxnId` rendering +(`0000000000000001-0000000000000003`) as the manifest's build-scoped directory, but they are +different counters with different semantics, not the same identifier. The `data.bin` entry inside +the part manifest names the blob by `{XXH3_128, a1b2…1728}`, which is what resolves to the +`blobs/xxh3/a1/…` key above. A small file such as `count.txt` has no object of its own — it is +inline inside the manifest's raw payload zone, not a separate key. + +## Notes {#notes} + +- `cas/ns/state//_ckpt` carries **no** `.zst` suffix: `cas_ref_ckpt`'s compression policy + is `never`, while its `_log`/`_snap` siblings in the same `cas/ns/` tree compress `always`. +- The namespace-stream tree is `cas/ns/stream/` (immutable `_log`/`_snap` objects) and + `cas/ns/state/` (mutable `_ckpt`, verbatim `_files/`). diff --git a/docs/en/antalya/cas/bucket-requirements.md b/docs/en/antalya/cas/bucket-requirements.md new file mode 100644 index 000000000000..ca05ad903aaf --- /dev/null +++ b/docs/en/antalya/cas/bucket-requirements.md @@ -0,0 +1,44 @@ +--- +description: 'The object-store contract a bucket must satisfy to host content-addressed storage, and which providers qualify.' +sidebar_label: 'Bucket requirements' +sidebar_position: 4 +slug: /antalya/cas/bucket-requirements +title: 'CAS Bucket Requirements' +doc_type: 'reference' +--- + +# Bucket requirements {#bucket-requirements} + +`CAS` is built on a small object-store contract (`Backend/CasBackend.h`), checked by a capability +probe that runs at every writable mount and fails closed: an object store that does not enforce +these conditions is refused rather than trusted. + +## The capability table {#capability-table} + +| Requirement | Interface method | Why it is needed | +|---|---|---| +| Read-after-write on a fresh key | `Backend::get` / `Backend::head` | Recovery listings and point reads must see what was just written | +| Conditional create (`If-None-Match: *`) | `Backend::putIfAbsent`, `Backend::putIfAbsentStream` | Write-once creation of blobs, manifests, and ref-log entries | +| Conditional overwrite (`If-Match: `) | `Backend::putOverwrite`, `Backend::casPut` | The one mutual-exclusion primitive: mount leases, `gc/state` | +| Exact-token delete | `Backend::deleteExact` | GC must delete only the incarnation it condemned, never a resurrected replacement | +| Ranged `GET` | `Backend::get` / `Backend::getStream` with a `Range` | Opening one column file of a part costs one bounded read, not a whole-object fetch | +| `LIST` with a resumable cursor | `Backend::list` | GC discovery and the orphan-manifest sweep page through the pool without a separate index | +| No versioning / no delete markers | probed by `runCapabilityProbe`; `created_delete_marker` on `DeleteOutcome` | A delete marker over a live key would break exact-token semantics — GC would archive instead of reclaim | +| `TOKEN ⟹ CONTENT` (a repeated token implies unchanged bytes) | standing requirement on every `Backend` implementation | Not probed — it cannot be tested cheaply. A backend that recycled tokens would serve stale manifests, i.e. wrong query results, not merely an inefficiency | + +Bucket **versioning is not required** — in fact it must be **disabled** on the generation-token +dialect (see below), because a token-exact delete on a versioned bucket archives a noncurrent +generation instead of reclaiming storage, silently stopping GC reclamation. + +## Platform support {#platform-support} + +| Platform | Status | Notes | +|---|---|---| +| AWS S3 | ✓ | Native `ETag`-based conditional dialect: `If-None-Match` / `If-Match` used directly | +| Google Cloud Storage | ✓ | Generation-token dialect: conditional headers are rewritten to `x-goog-if-generation-match`, opted into via `http_client = gcs_hmac` or `gcp_oauth` | +| Azure Blob Storage | probably | Azure's REST API documents the equivalent conditional headers, but ClickHouse's Azure object-storage backend does not yet wire up a `CAS` conditional dialect the way the S3 and GCS paths do — untested, not validated by the capability probe | +| Other S3-compatible stores | only with enforced conditional operations | The capability probe is the actual gate: a store that silently ignores `If-None-Match`/`If-Match` (accepting and applying the write regardless) fails the probe and is refused. `RustFS` passes the full battery and is used as the project's test backend; `Garage` was evaluated and rejected because it silently ignores conditional operations | + +The full mechanics of the two dialects — how the backend detects which one a given endpoint speaks, +what the capability probe actually checks, and how exact-token deletes map onto each provider's +primitives — are in [the Backend architecture page](/antalya/cas/architecture/backend). diff --git a/docs/en/antalya/cas/configuration.md b/docs/en/antalya/cas/configuration.md new file mode 100644 index 000000000000..8e551ef292be --- /dev/null +++ b/docs/en/antalya/cas/configuration.md @@ -0,0 +1,132 @@ +--- +description: 'Every disk-level and server-level setting content-addressed storage exposes, generated from ContentAddressedSettings and ServerSettings at HEAD.' +sidebar_label: 'Configuration' +sidebar_position: 3 +slug: /antalya/cas/configuration +title: 'CAS Configuration Reference' +doc_type: 'reference' +--- + +# Configuration reference {#configuration-reference} + +## The disk config block {#disk-config} + +A `CAS` disk is an `object_storage` disk with `metadata_type` set to `cas` and an explicit +`server_root_id`. The recommended shape layers a `type=cache` disk in front of it — the local +filesystem cache absorbs repeated reads of the same blob, while the `CAS` disk underneath stays the +single source of truth the pool's other members and GC also read from. The storage policy references +the **cached** disk, not the raw `CAS` disk directly: + +```xml + + + + + object_storage + s3 + cas + {replica} + https://bucket.s3.amazonaws.com/cas/ + ... + ... + + + cache + cas + /var/lib/clickhouse/cas_cache/ + 10Gi + + + + + +
+ cas_cache +
+
+
+
+
+
+``` + +`path` and `max_size` are ordinary `type=cache` disk settings (see +[external disk cache](/operations/storing-data#using-local-cache)), not `CAS`-specific — size the +cache to the working set of blobs a node reads repeatedly, not to the pool's total size. `type`, +`object_storage_type`, `metadata_type`, `endpoint`, `access_key_id`, `secret_access_key`, and the +other generic object-storage/disk keys (`path`, `name`, `region`, `use_environment_credentials`, +`readonly`, `use_fake_transaction`, and a handful more) belong to the shared disk layer, not to +`CAS` — they are accepted inside the `cas` disk's own block but are not `CAS` settings. Every key +below this line, and every key not in that shared set, is rejected as unknown. + +The bare, uncached form — a storage policy pointing directly at the `CAS` disk, as used by +[quick start](/antalya/cas/quick-start) — remains valid and is the minimal way to try `CAS` out: + +```xml + + + +
+ cas +
+
+
+
+``` + +## Disk-level settings {#disk-settings} + +None of these keys carry a `cas_`/`ca_` prefix — the disk block already scopes them. + +`CAS` is experimental: any setting below may change semantics, change its default, or disappear +entirely before release. Treat this table as a snapshot of the current build, not a stable contract. + +| Setting | Default | Description | +|---|---|---| +| `server_root_id` | — (required) | Explicit layout subtree identity; macros expand as in the `s3` `endpoint`. Anchored in the pool by a write-once owner claim — a colliding identity is refused at mount | +| `scratch_path` | server data path | Server-local scratch dir for the write-buffer spill; a relative value is anchored to the server data path | +| `gc_enabled` | `true` | Run the background GC scheduler on this disk. `false` is a debugging aid, not an operating mode: garbage then accumulates indefinitely and silently — watch `system.cas_gc_log` for round activity if you ever toggle it | +| `gc_interval_sec` | `60` | Seconds between background GC rounds (≥ 1) | +| `blob_hash` | `cityhash128` | Pool blob content-hash function (`cityhash128` \| `xxh3-128` \| `sha256`). Recorded in the pool at creation; a mismatching config is refused at mount | +| `blob_hash_allow_new` | `false` | Explicit opt-in to admit a new hash algorithm into an existing pool. One-way: once admitted, the pool carries both algorithms permanently | +| `skip_access_check` | `false` | Skip the boot-time capability probe (start now, fix later). Safer than the name suggests: only the preflight probe is skipped — the conditional-write correctness check still runs unconditionally on every writable mount | +| `deduplication_cache_bytes` | 64 MiB | Byte budget of the blob presence cache (`0` disables) | +| `deduplication_head_first_min_bytes` | 1 MiB | Minimum blob size to try a `HEAD` before uploading the body | +| `gc_snapshot_generations_to_keep` | `3` | GC snapshot generations retained | +| `gc_shards` | `1` | Blob-hash-prefix reducer shards (≥ 1). Recorded in the pool at creation; a mismatching config is refused at mount | +| `gcs_max_conditional_put_bytes` | 1 GiB | Largest conditional write on a generation-token store (GCS forces those single-part); does not bound the unconditional resurrect | +| `part_folder_cache_bytes` | 64 MiB | Part-folder view cache byte budget (`0` disables retention) | +| `part_folder_cache_max_entries` | `10000` | Part-folder view cache entry cap | +| `part_folder_cache_max_entry_bytes` | 16 MiB | Oversized part-folder views bypass retention above this size | +| `part_folder_validate` | `always` | Cache body re-proof policy (`always` \| `never` \| `age `). **Leave at `always`**: the other modes trade the fail-closed body-existence check for an optimization — this is a trust decision about unverified data, not a performance knob | +| `manifest_decode_cache_bytes` | 128 MiB | Manifest decode cache byte budget (`0` disables) | +| `gc_meta_pool_size` | `16` | Bounded pool size for GC per-hash freshness-meta writes | +| `staging_backend` | `local` | Blob staging backend (`local` \| `s3`); `s3` is opt-in | + +### Choosing `blob_hash` {#choosing-blob-hash} + +`blob_hash` is fixed at pool creation, so pick it deliberately. `blob_hash_allow_new` is the +escape hatch — it admits a second algorithm into an existing pool's `algos_used` rather than +requiring a fresh pool. + +| Algorithm | Pick it for | Trade-off | +|---|---|---| +| `sha256` | Maximum safety | No known collision classes; slightly slower than the other two | +| `xxh3-128` | Maximum speed | Fastest, 128-bit, no known collision classes | +| `cityhash128` (default) | ClickHouse-ecosystem compatibility, and a possible future hash-reuse mode that avoids recomputation | Fast, but has a known class of collisions that occurs far more often than an ideal hash function would predict | + +## Server-level settings {#server-settings} + +Source: `ServerSettings.cpp`. Unlike the disk-level list, these carry the `cas_` prefix because they +are process-wide, not scoped to one disk block. + +| Setting | Default | Description | +|---|---|---| +| `cas_blob_upload_pool_size` | `16` | Size of the dedicated server-wide thread pool used to upload blobs in parallel when committing a `CAS` part. Zero is rejected: the pool must have at least one thread | + +## `SYSTEM CAS` commands {#system-commands} + +`SYSTEM CAS GC RUN`, `SYSTEM CAS GC STOP`, `SYSTEM CAS GC START`, `SYSTEM CAS GC REBUILD`, +`SYSTEM CAS FSCK`, `SYSTEM CAS FORGET`, and `SYSTEM CAS DROP POOL MEMBER '' FROM +DISK ''` operate on a mounted `CAS` disk. Introspection lives in `system.cas_log`, +`system.cas_gc_log`, and `system.cas_mounts`. diff --git a/docs/en/antalya/cas/index.md b/docs/en/antalya/cas/index.md new file mode 100644 index 000000000000..2bc71046494b --- /dev/null +++ b/docs/en/antalya/cas/index.md @@ -0,0 +1,89 @@ +--- +description: 'What content-addressed storage is, the problem it solves, its current status, and where to go next.' +sidebar_label: 'Overview' +sidebar_position: 1 +slug: /antalya/cas +title: 'Content-Addressed Storage' +doc_type: 'guide' +--- + +# Content-addressed storage {#content-addressed-storage} + +`ReplicatedMergeTree` on object storage has two unattractive options today. Plain replication +stores a byte-identical copy of every part on every replica, so storage cost multiplies with the +replication factor. Zero-copy replication shares the bytes, but at a structural price: every +replica keeps local metadata referencing each shared S3 object, and that state grows with the +data; a commit spans three independent systems — local disk, S3, and `Keeper` — whose interleaving +is easy to get subtly wrong, and a failure in any one of the three hurts availability; sharing is +tracked by a numeric refcount, so a lost or duplicated retry can corrupt the count; and the +special cases supporting all of this are scattered widely through the `MergeTree` code. + +Content-addressed storage (`CAS`) is a `MetadataStorage` back-end for object-storage disks +(`metadata_type = cas`) that takes the same sharing goal and collapses it onto one system: every +`MergeTree` part file is stored once, keyed by the hash of its content, in the object-storage pool +itself. There is no `CAS` state in `Keeper` at all — a commit is one conditional write against a +single object in the pool — and the reachability accounting is a derived in-degree edge set folded +from append-only deltas, not a mutable refcount a lost message can corrupt. + +```mermaid +graph LR + subgraph today["Today: zero-copy replication"] + R1["Replica 1
local disk: object refs
(grows with data)"] -->|"in-flight ops only"| K["Keeper"] + R2["Replica 2
local disk: object refs
(grows with data)"] -->|"in-flight ops only"| K + R1 -.->|"shares bytes"| S1["S3"] + R2 -.->|"shares bytes"| S1 + end + subgraph cas["CAS: content-addressed pool"] + C1["Replica 1"] -->|"publish a ref"| P["S3 pool
(refs, leases, GC — all in-bucket)"] + C2["Replica 2"] -->|"publish a ref"| P + end +``` + +Every CAS bookkeeping object — refs, mount leases, GC leadership, fencing tokens — lives in the +bucket. There is no external coordinator, and no `Keeper` usage inside the pool protocol; `Keeper` +stays exactly where `ReplicatedMergeTree` already used it, for replication log and part-set +consensus, and its load does not grow with pool size. + +## Deployment guidance {#deployment-guidance} + +`GC` throughput is proportional to how much changes in the pool: a pool holding a very large +number of parts from many servers, or data that churns very quickly, means longer `GC` rounds. +Two consequences for planning: + +- **The preferred deployment is a second tier for cold data**: hot, fast-churning parts stay on + the local (or plain S3) tier, and `CAS` holds the large, slow-moving cold tail — where + deduplication pays the most and `GC` traffic is minimal. +- **At large scale, shard the pool by key prefix.** With tens of servers, or thousands of tables + and millions of parts, split the deployment into several independent pools by giving each shard + its own prefix — the shards can share one bucket: + + ```xml + https://bucket.s3.amazonaws.com/cas/{shard} + ``` + + Each prefix is a fully independent pool (its own refs, leases, and `GC`), so rounds stay short + regardless of the total fleet size. + +## Status {#status} + +`CAS` is **experimental**. It ships in Altinity Antalya builds. Experimental means the on-disk +format and the SQL surface can still change between releases — that is deliberate, not a caveat to +apologize for. Pre-release means the format can change cheaply, with zero compatibility +scaffolding, and the design can keep being iterated on invariants rather than migrations. The bet +underneath it: all you need is a good S3 bucket. See [bucket requirements](/antalya/cas/bucket-requirements) +for exactly what "good" means. + +`CAS` coexists with zero-copy replication; it does not replace it. `metadata_type = cas` is opt-in +per disk, so adopting it never requires migrating an existing deployment. + +## Where to go next {#nav} + +| Page | Covers | +|---|---| +| [Quick start](/antalya/cas/quick-start) | A minimal disk config and the first `CREATE TABLE` / `INSERT` / `SELECT` | +| [Configuration](/antalya/cas/configuration) | Every disk-level and server-level setting | +| [Bucket requirements](/antalya/cas/bucket-requirements) | What an object store must support, and which providers qualify | +| [Architecture overview](/antalya/cas/architecture/) | The object model, the Git analogy, and the safety invariants | +| [Correctness](/antalya/cas/architecture/correctness) | How the design was verified: TLA+ models, counterexamples, soak methodology | +| [Design history](/antalya/cas/architecture/design-history) | What earlier designs were tried and rejected, and why | +| [Roadmap](/antalya/cas/roadmap) | What is shipped, planned, and deliberately not pursued | diff --git a/docs/en/antalya/cas/operations/debugging.md b/docs/en/antalya/cas/operations/debugging.md new file mode 100644 index 000000000000..b1ed38791341 --- /dev/null +++ b/docs/en/antalya/cas/operations/debugging.md @@ -0,0 +1,225 @@ +--- +description: 'SQL-first CAS debugging: live investigation queries against cas_log/cas_gc_log/cas_mounts/blob_storage_log, SYSTEM CAS FSCK/GC RUN/GC STOP-START/FORGET, and the offline clickhouse-disks tools for when the server cannot answer.' +sidebar_label: 'Debugging' +sidebar_position: 4 +slug: /antalya/cas/operations/debugging +title: 'CAS Operations — Debugging' +doc_type: 'guide' +--- + +# Operations — debugging {#debugging} + +Debugging a content-addressed (`CAS`) incident starts on a **live server**, with SQL: the three +system tables plus `SYSTEM CAS` commands cover reachability checks, forced GC rounds, and +per-object/per-round forensics without ever touching the bucket directly. The offline +`clickhouse-disks` tools at the [end of this page](#offline-tools) are the fallback for when SQL +cannot reach the pool at all — the server is down, or the access is deliberately read-only forensic. + +## Investigating on a live server {#live-investigation} + +See [monitoring](/antalya/cas/operations/monitoring#system-tables) for the three system tables' +grain and general health queries; this section is investigation queries for a specific incident, +not a health dashboard. + +### What happened to this part or blob {#part-blob-history} + +`system.cas_log` carries one row per writer/GC decision, keyed by `ref_name` (a part name) or +`object_hash` (a blob's content hash): + +```sql +SELECT event_time_microseconds, event_type, outcome, reason, object_kind, object_hash, token, round, detail +FROM system.cas_log +WHERE disk_name = 'cas' AND ref_name = '' +ORDER BY event_time_microseconds; +``` + +```sql +SELECT event_time_microseconds, event_type, outcome, reason, ref_name, round, detail +FROM system.cas_log +WHERE disk_name = 'cas' AND object_kind = 'blob' AND object_hash = '' +ORDER BY event_time_microseconds; +``` + +`outcome` (`ok`, `adopt`, `resurrect`, `deleted`, `replaced`, `spared`, `absent`, `zeroed`, +`skipped`) and `reason` are the two columns to read first; `detail` is a +`Map(LowCardinality(String), String)` of decision-specific facts (`condemn_round`, +`superseded_token`, `code`, `site`) worth `arrayJoin(detail)` when the summary columns alone do not +explain the decision. See [`system.cas_log`](/operations/system-tables/cas_log) for the full column +reference. + +### Why GC is not reclaiming {#why-not-reclaiming} + +Two questions, in order: is this node's scheduler leading, and did its recent rounds actually fold? + +```sql +SELECT server_root_id, is_leader, state, last_success_age_seconds, pending_reclaim +FROM system.cas_mounts WHERE disk = 'cas'; + +SELECT event_time, outcome, candidates_marked, entries_condemned, entries_graduated, + entries_redeleted, anomalies +FROM system.cas_gc_log +WHERE event_type = 'Finish' AND disk_name = 'cas' +ORDER BY event_time DESC LIMIT 10; +``` + +A `0`/`false` `is_leader` means this node never reclaims for this disk — check the peer that holds +leadership instead. A steady `entries_condemned` with `entries_graduated` stuck at `0` means objects +are being found but never crossing the safety floor (recall the grace period is measured in full +rounds, not acks — see [condemnation and deletion](/antalya/cas/architecture/garbage-collection#condemn-delete)). +A specific blob's own story — was it ever condemned, spared, or is it not being seen at all — is the +per-object query in the previous section, filtered to `object_kind = 'blob'`. + +### What one GC round did {#gc-round-detail} + +Every round writes a `Start` and a `Finish` row to +[`system.cas_gc_log`](/operations/system-tables/cas_gc_log), correlated by `round_id` (not `round`, +which is `0` on `Start` and absent on a round that never led). One `Phase` row per phase reached +carries that phase's own `phase_duration_microseconds`, `ProfileEvents` delta, and `phase_metrics` — +group by `round_id` to reconstruct one round in order: + +```sql +SELECT event_type, outcome, phase, phase_duration_microseconds, duration_ms +FROM system.cas_gc_log +WHERE round_id = '' +ORDER BY event_time_microseconds; +``` + +### Who holds the mount {#who-holds-mount} + +```sql +SELECT server_root_id, hostname, process_id, state, writer_epoch, renewal_sequence, + expires_at, is_leader +FROM system.cas_mounts +WHERE disk = 'cas' +ORDER BY is_leader DESC; +``` + +Every `server_root_id` sharing the pool shows up here, not just this node's own — a `state` other +than `live` (`expired`, `terminated`, `fenced`, `corrupt`) on a member that should be up is the first +thing to check before assuming a lease problem is this node's own. `is_leader` and the other +process-local columns are `NULL` on every peer's row; run the query on that peer to see its own view. + +## SQL commands for live diagnosis {#sql-commands} + +### SYSTEM CAS FSCK {#sql-fsck} + +The online consistency check — unlike the offline tools below, this runs against a disk the server +already has **mounted and serving traffic**; the scan re-validates every finding against a fresh +authoritative read, so it needs no quiesce: + +```sql +SYSTEM CAS FSCK cas; +``` + +Returns one row: `disk`, `reachable`, `dangling`, `unreachable`, `pending_gc`, `awaiting_gc`, +`unaccounted`, `stale_edge`, `corrupted_runs`, `chain_broken`, `unchecked`, `lifeless_keys`, +`namespace_janitor_pending` (+`_bytes`/`_lives`), `ref_records_walked`, `physical_bytes`, +`referenced_logical_bytes`, `distinct_blobs`, `total_blob_refs`. `dangling` is the one column that +means data loss — `unreachable`, `pending_gc`, and `awaiting_gc` are objects still +moving through the normal condemn/graduate/delete pipeline, not a problem on their own. +`chain_broken` and `corrupted_runs` are the other two hard findings: a hole in a ref-log stream and a +GC source-edge run that failed its checksum, respectively. This summary-only form has no +per-object `--detail` equivalent yet — for that, the offline `cas-fsck --detail` below is still +needed. + +### SYSTEM CAS GC RUN {#sql-gc-run} + +Runs one round synchronously and returns exactly the shape of a `cas_gc_log` `Finish` row — driving +a round on demand while watching its outcome interactively is one of the most direct diagnostics +available: + +```sql +SYSTEM CAS GC RUN cas; +``` + +One row per disk it ran on: `disk`, `acquired_lease`, `deferred`, `round`, `candidates_marked`, +`objects_deleted`, `objects_absent`, `objects_replaced`, `objects_spared`, `manifests_deleted`, +`entries_condemned`, `entries_graduated`, `entries_redeleted`, `fence_outs`, `anomalies`, +`pending_candidates`, `pending_condemned`, `pending_retired`. Omitting +the disk name runs one round on every content-addressed disk on the node. A manual run executes +regardless of `SYSTEM CAS GC STOP` — `STOP` pauses only the background scheduler. + +### SYSTEM CAS GC STOP / START {#sql-gc-stop-start} + +Pause the background scheduler on one disk while investigating a suspect object, so it cannot be +condemned or deleted mid-investigation, then resume it: + +```sql +SYSTEM CAS GC STOP cas; +-- investigate, e.g. cas-inspect a specific blob's raw key +SYSTEM CAS GC START cas; +``` + +`STOP` is idempotent and stops-in-place (the same scheduler instance resumes on `START`, keeping its +`gc_id` and lease-observation history); it works even on a not-live disk. It does not stop a manual +`SYSTEM CAS GC RUN`. See the [operational surface](/antalya/cas/architecture/garbage-collection#operational-surface) +table for the full command list. + +### SYSTEM CAS FORGET {#sql-forget} + +Node-local operator assertion that a disk is permanently gone — the "fire marshal" verb for a stuck +disk (a transient/`IdentityLost` pool, an operator-asserted decommission): + +```sql +SYSTEM CAS FORGET cas; +``` + +It is an assertion, not a proof of erasure: the disk stays registered and answers further store-class +access with a typed error, and a server restart re-registers the name. This is different from +[`SYSTEM CAS DROP POOL MEMBER`](/antalya/cas/operations/migration#decommission), which permanently +retires one pool *member*'s identity across the whole shared pool — `FORGET` only affects this node's +own local view of one disk. + +## Offline tools {#offline-tools} + +When the server cannot answer — it is down, or the access needs to be read-only forensic against the +bucket directly, disaster recovery of `gc/state`, or a raw object decode — `clickhouse-disks` runs +these against the pool's backend without a live server. All five require the disk to be opened with +`true` in the `clickhouse-disks` config; they must never claim a live server's +mount. + +| Command | Use it for | +|---|---| +| `cas-fsck [--detail] [--timeout N] [--namespace PREFIX] [--partial]` | The same reachability scan as `SYSTEM CAS FSCK`, offline. `--detail` adds a per-object `\t\t` listing (`reachable`, `dangling`, `unreachable`, `pending-gc`, `awaiting-gc`, `unaccounted`, `stale-edge`, `corrupted-run`, `chain-broken`, `unchecked`, `lifeless-key`, `janitor-pending`) — the only way to get per-object, not just per-pool, findings. `--timeout`/`--partial` bound a scan on a large pool | +| `cas-gc-dryrun` | Previews the next round's deletes, read-only, no lease. Over-reports away from quiescence (does not fold new owner events) — a diagnostic only, never a delete source | +| `cas-inspect ''` | Decodes one raw object-storage key (as printed by `cas-fsck`/`cas-gc-dryrun`) straight to JSON | +| `cas-gc-rebuild [--force]` | Disaster recovery: rebuilds a `gc/state` baseline from raw owner state after the GC guard has refused every regular round. `--force` bypasses only the healthy-state refusal, never a competing leader or a failed `CAS`. See [`SYSTEM CAS GC REBUILD`](/sql-reference/statements/system#system-cas-gc-rebuild) for the destructive-tool caveats | + +```bash +clickhouse-disks -C config.xml --disk cas cas-fsck --detail +clickhouse-disks -C config.xml --disk cas cas-gc-dryrun +clickhouse-disks -C config.xml --disk cas cas-inspect '' +clickhouse-disks -C config.xml --disk cas cas-gc-rebuild --force +``` + +`cas-drop-member` — the offline twin of `SYSTEM CAS DROP POOL MEMBER` — is covered on the +[migration page](/antalya/cas/operations/migration#decommission) alongside the SQL form, since +decommissioning a pool member is a migration/scale-down operation, not an incident-time tool. + +## The CLICKHOUSE_USER_FILES gotcha when reproducing a test manually {#user-files-gotcha} + +Running a `CAS` stateless test directly with `tests/clickhouse-test` against a manually started +`clickhouse-server` (outside a configured praktika lane) requires exporting `CLICKHOUSE_USER_FILES` +to match the server's actual data path. The harness's default, +`/var/lib/clickhouse/user_files`, will not match a custom data path, which makes the pool directory +invisible to the server — the symptom is an `Unknown disk` error together with a diagnostic that +reads like an empty pool (e.g. `baseline=0 after_insert=0`) even though the server is otherwise +healthy. + +## What to collect before filing a bug {#filing-a-bug} + +- `SYSTEM CAS FSCK ''` output (or `clickhouse-disks cas-fsck --detail`, if the server cannot + answer or a per-object listing is needed) — the authoritative reachability snapshot at the time of + the incident. +- The `system.cas_gc_log` rows for the relevant `round_id`(s): `Start`, every `Phase`, and `Finish`. +- The `system.cas_log` rows for the specific ref name, blob hash, or object key involved, filtered by + `event_time` around the incident. +- `system.cas_mounts` output from every node sharing the pool, to capture lease/epoch state at + incident time — it is a live view and will not reflect a state that has since changed. +- For a suspected object-store issue, `system.blob_storage_log` rows for the affected `disk_name` + with a nonzero `error_code`, and the relevant `CAS*` `ProfileEvents` (`system.query_log`'s + `ProfileEvents` map for one query, or `system.metric_log`'s `ProfileEvent_*` columns for a window — + see [monitoring](/antalya/cas/operations/monitoring#key-metrics) for which counters matter and the + restart-resets-`system.events` caveat). +- The server version and, if the incident is reproducible, the exact `CREATE TABLE` / `INSERT` / + `ALTER` sequence that triggers it. diff --git a/docs/en/antalya/cas/operations/migration.md b/docs/en/antalya/cas/operations/migration.md new file mode 100644 index 000000000000..0d4f954c2dae --- /dev/null +++ b/docs/en/antalya/cas/operations/migration.md @@ -0,0 +1,209 @@ +--- +description: 'Adding a content-addressed disk to an existing deployment, moving a partition onto it with ALTER TABLE MOVE PARTITION, rolling back, and permanently decommissioning a pool member.' +sidebar_label: 'Migration' +sidebar_position: 1 +slug: /antalya/cas/operations/migration +title: 'CAS Operations — Migration' +doc_type: 'guide' +--- + +# Operations — migration {#migration} + +This page walks through moving `MergeTree` data onto a content-addressed (`CAS`) disk from an +existing disk, and the reverse. `metadata_type = cas` is opt-in per disk (see the +[overview](/antalya/cas)), so this is an additive change to a running deployment: the existing +disk and its data are untouched until a partition is explicitly moved. + +## Add a CAS disk alongside an existing one {#add-disk} + +A storage policy can carry both an ordinary disk and a `CAS` disk as separate volumes. `ALTER TABLE +... MOVE PARTITION ... TO DISK` then moves data between them without an `INSERT`/`DROP` cycle. As on +the [configuration](/antalya/cas/configuration#disk-config) page, the recommended shape layers a +`type=cache` disk over the `CAS` disk, and the policy's volume references the **cached** disk name: + +```xml + + + + + local + /var/lib/clickhouse/local_disk/ + + + object_storage + s3 + cas + {replica} + https://bucket.s3.amazonaws.com/cas/ + ... + ... + + + cache + cas + /var/lib/clickhouse/cas_cache/ + 10Gi + + + + + + + local_disk + + + cas_cache + + + + + + +``` + +See [configuration](/antalya/cas/configuration) for the full disk-level settings surface and +[bucket requirements](/antalya/cas/bucket-requirements) for what the target bucket needs to +support. A table does not need to be created for the first time on `CAS` to use it — an existing +table just needs its storage policy widened to include a volume backed by a `CAS` disk, which is a +metadata-only change (`ALTER TABLE ... MODIFY SETTING storage_policy = ...`, subject to the usual +constraint that the new policy must still contain every volume and disk of the old one — a storage +policy can only grow, never lose a disk it once had). + +## Move a partition onto CAS {#move-partition} + +`ALTER TABLE ... MOVE PARTITION ... TO DISK` moves every part of one partition to the named disk in +place — the ordinary `MergeTree` partition-move mechanism, unchanged by `CAS`: + +```sql +CREATE TABLE events (event_date Date, event_id UInt64, payload String) +ENGINE = MergeTree ORDER BY event_id PARTITION BY event_date +SETTINGS storage_policy = 'tiered'; + +INSERT INTO events VALUES ('2026-08-04', 1, 'hello'), ('2026-08-04', 2, 'world'); + +SELECT name, partition, disk_name FROM system.parts WHERE table = 'events' AND active; +``` + +```text +Row 1: +────── +name: 20260804_1_1_0 +partition: 2026-08-04 +disk_name: local_disk +``` + +The partition starts on `local_disk`, the first volume in the policy. Moving it onto `CAS` uploads +each part's files as content-addressed blobs, writes a part manifest, and publishes a ref — the same +write path an `INSERT` directly onto `CAS` takes (see +[what just happened](/antalya/cas/quick-start#what-happened) in the quick start). `TO DISK` names the +disk actually listed in the policy's volume — with a cache layered in front, that is the **cache** +disk's name (`cas_cache`), not the raw `CAS` disk's name (`cas`) underneath it; naming the raw disk +is refused, because it is not a member of the table's storage policy: + +```sql +ALTER TABLE events MOVE PARTITION '2026-08-04' TO DISK 'cas_cache'; + +SELECT name, partition, disk_name FROM system.parts WHERE table = 'events' AND active; +``` + +```text +Row 1: +────── +name: 20260804_1_1_0 +partition: 2026-08-04 +disk_name: cas_cache +``` + +```sql +SELECT * FROM events ORDER BY event_id; +``` + +```text +2026-08-04 1 hello +2026-08-04 2 world +``` + +`system.parts.disk_name` reports the cache disk's name, not the underlying `CAS` disk's — this is +the ordinary `type=cache` disk behavior (the same happens layering a cache over any other disk type) +and is not `CAS`-specific. `system.filesystem_cache` shows the part's files populated into the +`cas_cache` cache on this read-through. + +## Roll back {#rollback} + +The move is symmetric: `MOVE PARTITION ... TO DISK` back onto the original disk name returns the +partition to its previous location, with the data intact throughout: + +```sql +ALTER TABLE events MOVE PARTITION '2026-08-04' TO DISK 'local_disk'; + +SELECT name, partition, disk_name FROM system.parts WHERE table = 'events' AND active; +``` + +```text +Row 1: +────── +name: 20260804_1_1_0 +partition: 2026-08-04 +disk_name: local_disk +``` + +Moving a partition off `CAS` does not itself delete the blobs it stops referencing — dropping the +old ref makes them eligible for reclamation by the next +[GC round](/antalya/cas/architecture/garbage-collection), the same as dropping a part. + +This exact three-disk, cache-over-`CAS` configuration and the forward/rollback `ALTER TABLE ... MOVE +PARTITION` sequence above were run against a live server before publication, using the `local` +object-storage backend for the `cas` disk: `CREATE TABLE`, `INSERT`, both `MOVE PARTITION` +directions, the `system.parts` checks, the `system.filesystem_cache` check, and the `SELECT` all +completed with zero errors and the shown output. A prior attempt to move onto `TO DISK 'cas'` +directly (the raw disk, not the cache) was refused with `All parts of partition '20260804' are +already on disk 'cas_cache'. (UNKNOWN_DISK)` — a real error message from the run, kept here because +it is exactly what an operator sees after guessing the wrong disk name. + +## Permanently removing a pool member {#decommission} + +A `CAS` pool can be shared by several servers (see [`server_root_id`](/antalya/cas/architecture/mounts-and-leases#server-root-id)). +Scaling down — permanently removing a server that will never rejoin the pool — is a distinct, +irreversible operation from an ordinary restart or a temporary outage: it fences the member's +`server_root_id` and reclaims the storage attributable only to it. + +`SYSTEM CAS DROP POOL MEMBER` claims the victim's mount slot as an administrative writer (refusing +immediately if the member is still alive), drops every table namespace the member owned, sweeps +manifest debris, drains its staging and mountpoint objects, and — only once every drain is +confirmed — retires the mount slot itself. It emits ordinary ref-edge deltas rather than a GC +transition: it does not synchronously reclaim shared blob content, it only makes the now-unreferenced +blobs eligible for an ordinary GC round to reclaim later. + +```sql +SYSTEM CAS DROP POOL MEMBER 'server_root_id' FROM DISK 'disk_name' [ON CLUSTER cluster_name] +``` + +Both `server_root_id` and `disk_name` are required string literals. The offline CLI twin, +`clickhouse-disks cas-drop-member `, does the same work against a disk opened +read-only — the pool-admin claim happens internally, so the disk it runs against must not be the +live server's own mount: + +```bash +clickhouse-disks -C config.xml --disk cas cas-drop-member 'replica-2' +``` + +The command returns one row (or, offline, one line per field) with `namespaces_removed`, +`namespaces_already_removed`, `committed_refs_removed`, `precommits_removed`, +`manifest_debris_removed`, `staging_objects_removed`, `mountpoint_objects_removed`, and +`slot_removed`. It is resumable: a rerun skips namespaces already marked removed and reports them +under `namespaces_already_removed` rather than redoing the work. A per-object drain failure is +recorded as a `warning` rather than raised as an exception, leaving the slot terminated but not +fully drained so a later invocation can resume; a non-empty `warnings` means exactly that, and the +mount slot stays in place as a resume anchor rather than being fully retired. + +**Preconditions.** Confirm the member is actually and permanently dead before running this: the +operation fences that `server_root_id` out even if the server comes back online, and it deletes +namespace and drain state that cannot be recovered. Check `system.cas_mounts` for the member's +`state` and `last_success_age_seconds` first — a `live` row, or one with a recent lease renewal, +means the member is not a decommission candidate yet. + +**Verification.** After the command reports `slot_removed = true` with no warnings, the member's +`server_root_id` no longer appears as a row in `system.cas_mounts` on any peer, and a subsequent +`SYSTEM CAS GC RUN` on the pool will no longer wait on or fence its heartbeat. See +[mount, unmount, crash](/antalya/cas/architecture/mounts-and-leases#mount-lifecycle) for how the +claim, drain, and retirement steps fit into the mount-slot lifecycle. diff --git a/docs/en/antalya/cas/operations/monitoring.md b/docs/en/antalya/cas/operations/monitoring.md new file mode 100644 index 000000000000..6ab8b3dd9bb0 --- /dev/null +++ b/docs/en/antalya/cas/operations/monitoring.md @@ -0,0 +1,101 @@ +--- +description: 'The three content-addressed system tables, a key-metrics table with healthy ranges, and queries for reading GC health from cas_gc_log.' +sidebar_label: 'Monitoring' +sidebar_position: 2 +slug: /antalya/cas/operations/monitoring +title: 'CAS Operations — Monitoring' +doc_type: 'guide' +--- + +# Operations — monitoring {#monitoring} + +Content-addressed (`CAS`) storage exposes three system tables and a family of `CAS`-prefixed +`ProfileEvents`. This page is the entry point for day-to-day health checks; see +[debugging](/antalya/cas/operations/debugging) for incident-time tooling and +[troubleshooting](/antalya/cas/operations/troubleshooting) for symptom-driven diagnosis. + +## The three system tables {#system-tables} + +| Table | Grain | Use it for | +|---|---|---| +| [`system.cas_mounts`](/operations/system-tables/cas_mounts) | One row per mount slot in the pool, read live from the backend on every query | Who is in the pool right now, lease/epoch state, which node holds GC leadership | +| [`system.cas_gc_log`](/operations/system-tables/cas_gc_log) | One `Start`/`Finish` row per GC round, plus one `Phase` row per phase reached | GC round outcomes, duration, and where a round's `LIST`/`GET`/`PUT`/`DELETE` budget went | +| [`system.cas_log`](/operations/system-tables/cas_log) | One row per writer/GC decision (blob puts, dedup adoptions, retire decisions, dangling-access findings) | Fine-grained forensics for one part, one blob hash, or one round | + +`system.cas_mounts` is the only one of the three with no persisted backing log — it is a live view, +so a transient backend error on one disk is skipped rather than blinding the whole query. The other +two are ordinary `system.*_log` tables and follow the usual flush/retention settings. + +## Key metrics {#key-metrics} + +Every `CAS`-related `ProfileEvent` carries the uppercase `CAS`/`CASGC` prefix. This is a curated +subset for a first health pass; the full list groups by object class (`CASBlob*`, `CASManifest*`, +`CASRoot*`, `CASGC*`, `CASServer*`, `CASOther*`, `CASRef*`, `CASMeta*`) and is enumerated in +`src/Common/ProfileEvents.cpp`. + +| Metric | Healthy range | A spike or nonzero means | +|---|---|---| +| `CASBlobCompareSwapConflict` | Near zero relative to `CASBlobCompareSwap` | Concurrent-update contention on blob metadata | +| `CASBlobHeadFirst` vs `CASBlobBodyPutAvoided` | `CASBlobBodyPutAvoided` tracks `CASBlobHeadFirst` closely | A widening gap means the dedup `HEAD`-before-`PUT` gate is firing but not finding matches — expected for genuinely new content, worth checking if it dominates | +| `CASRefAppendWedged` | Zero | A ref-log append lane exhausted its retries after an uncertain `PUT`; ref-log progress on that namespace may be stalled | +| `CASRefNeedsRecovery` | Zero | A ref-append lane could not install a known-durable transaction and now refuses writes, snapshots, and confirmation until durable replay completes | +| `CASRefAppendSealRejected` | Occasional (a deposed writer losing a race is the protocol working); sustained growth is not | A writer keeps retrying after losing its mount and does not yet know it | +| `CASGCHeartbeatFenceOuts` | Zero on a healthy pool | GC fenced an expired mount; check `system.cas_mounts` for a member that should have cleanly unmounted | +| `CASGCUnmatchedRemoveDeltas` | Occasional (benign per-key no-op by design) | A persistent nonzero rate means removal deltas are reaching the reducer without their matching activation — a correctness signal worth a look, not an automatic false deletion | +| `CASGCCondemnMarkerUnconfirmedCarry` | Zero | A durable condemn marker could not be confirmed; deletion is safely postponed but investigate marker write/read failures | +| `CASGCMetaWriteAnomaly` | Zero | The bounded GC metadata pool failed an operation; backend or pool pressure may delay metadata convergence | +| `CASRefRollbackBestEffortDropFailed` | Zero | A rollback cleanup drop hit a backend failure; refs may remain live and GC may be delayed on that namespace | + +Two counter-reading caveats that apply to `system.events`-backed metrics generally, not only `CAS` +ones: a counter that has never incremented can be **absent** from `system.events` rather than +present at zero — query with `system_events_show_zero_values = 1` to tell "never happened" from "not +shown". A server restart resets `system.events` to zero, so a cumulative `CAS` total across a +restart has to be computed from summed per-second deltas in `system.metric_log`, not read directly +off `system.events`. + +## Reading GC health from cas_gc_log {#gc-health} + +Round outcomes over the last day, per disk: + +```sql +SELECT disk_name, outcome, count() AS rounds, avg(duration_ms) AS avg_ms +FROM system.cas_gc_log +WHERE event_type = 'Finish' AND event_time > now() - INTERVAL 1 DAY +GROUP BY disk_name, outcome +ORDER BY disk_name, rounds DESC; +``` + +A steady stream of `Success` and `Deferred` rows is healthy; `Deferred` means the round found no +changed shard needing a fold and no graduation was due — a cheap round, not a stuck one (see +[the round](/antalya/cas/architecture/garbage-collection#the-round)). Recurring `Error` rows, or +`NotALeader` outcomes for the disk's own scheduler, warrant investigation. `anomalies` in the +`Finish` row is worth a steady watch: it is fold clamps surfaced and survived, so a non-zero value +that persists across rounds is more interesting than an isolated one. + +Which phase dominates round duration or the `LIST` budget — reproduced from the +[per-phase rows](/operations/system-tables/cas_gc_log#per-phase-rows) reference: + +```sql +SELECT phase, + count() AS rounds, + quantile(0.99)(phase_duration_microseconds) AS p99_microseconds, + sum(ProfileEvents['S3ListObjects']) AS lists +FROM system.cas_gc_log +WHERE event_type = 'Phase' AND disk_name = 'cas' +GROUP BY phase +ORDER BY p99_microseconds DESC; +``` + +Pending-reclaim backlog and time since a disk's GC last led, from the live mount view: + +```sql +SELECT disk, server_root_id, is_leader, pending_reclaim, last_success_age_seconds, wedged_namespace_count +FROM system.cas_mounts +WHERE is_leader IS NOT NULL +ORDER BY disk, server_root_id; +``` + +`is_leader`, `pending_reclaim`, `last_success_age_seconds`, and `wedged_namespace_count` are +process-local — `NULL` on every row describing a peer's mount — so this query is only informative +run against the node whose GC leadership you are checking; run it on each node to see the whole +pool's view of itself. diff --git a/docs/en/antalya/cas/operations/troubleshooting.md b/docs/en/antalya/cas/operations/troubleshooting.md new file mode 100644 index 000000000000..94c29a8d077c --- /dev/null +++ b/docs/en/antalya/cas/operations/troubleshooting.md @@ -0,0 +1,29 @@ +--- +description: 'Symptom-to-action table for common content-addressed storage incidents: mount lease loss, stalled GC, startup failures, fsck timeouts, and read-only pools.' +sidebar_label: 'Troubleshooting' +sidebar_position: 3 +slug: /antalya/cas/operations/troubleshooting +title: 'CAS Operations — Troubleshooting' +doc_type: 'guide' +--- + +# Operations — troubleshooting {#troubleshooting} + +Start from the symptom, not the mechanism. Each row below names a concrete diagnostic query or +command and the action it points to; see [monitoring](/antalya/cas/operations/monitoring) for the +system tables referenced and [debugging](/antalya/cas/operations/debugging) for the underlying +tools. + +| Symptom | Diagnosis | Action | +|---|---|---| +| A server keeps losing its mount lease and self-remounting | Check `system.cas_mounts` for the server's own row's `state`/`expires_at`; losing the lease is neither read-only mode nor an abort, it trips a local fence and retries with backoff from 1s to 30s | Look for clock issues (the fence clock is `CLOCK_BOOTTIME`) or network latency exceeding `mount_lease_ttl_ms` (default 30s); see [the mount lease](/antalya/cas/architecture/mounts-and-leases#mount-lease) | +| Writes slow down or stall under load, with no exception reaching the client | S3 `SlowDown`/`ServiceUnavailable`/`RequestTimeout`/`InternalError` (5xx) responses are not on `CasRequestController`'s definite-failure whitelist (only malformed-request, entity-too-large, and access-denied are), so they classify as `Unresolved` and are retried automatically. Confirm with `sum(ProfileEvents['CASConditionalWriteUnresolved'])` rising alongside `sum(ProfileEvents['CASConditionalWriteAttempts'])` over `system.query_log` for the affected window (or `ProfileEvent_CASConditionalWriteUnresolved` in `system.metric_log` for a cumulative view across queries), and check `system.blob_storage_log` for `disk_name = ''` rows with a nonzero `error_code` around the same window | Nothing to configure per-request: the controller retries the same `(key, bytes)` with capped-exponential backoff (200ms initial, capped at 5s) for up to 16 attempts inside a 90-second operation deadline, and the mount-lease renewer keeps extending the fence across the disruption — this is the "blips, throttling, partial outages" case the write path is built to survive. Confirm the mount lease itself is still renewing (`system.cas_mounts.expires_at` moving forward, `last_success_age_seconds` not climbing) — if it is, this is expected and self-resolving. If `SlowDown` responses are sustained rather than transient, check the bucket's request-rate limits against the pool's actual PUT/GET rate (see [bucket requirements](/antalya/cas/bucket-requirements)) and consider lowering `cas_blob_upload_pool_size` to reduce concurrent upload traffic; a write only surfaces a client-visible `NETWORK_ERROR` if the 90-second deadline is exhausted before the store recovers, and that error is retried by the ordinary merge/insert backoff, not silently dropped | +| `GC` never seems to reclaim space after tables are dropped | `SELECT * FROM system.cas_gc_log WHERE event_type='Finish' ORDER BY event_time DESC LIMIT 5` — check `outcome`; also `SELECT is_leader FROM system.cas_mounts` on this node | If `outcome != 'Success'`/`'Deferred'`, see [reading GC health](/antalya/cas/operations/monitoring#gc-health); if this node is not the leader (`is_leader = 0`), it never reclaims for this disk — check the peer holding leadership. Reclamation also needs at least two full rounds past condemnation by design (the grace period is rounds, not acks) — a single manual `SYSTEM CAS GC RUN` will not finish it | +| A dangling-access exception or `CORRUPTED_DATA` on read | Run `clickhouse-disks cas-fsck --detail` and check `dangling` specifically — it is the one class that means data loss, distinct from `unreachable`/`awaiting-gc`, which are just waiting for graduation | A nonzero `dangling` count is a real incident: collect the `--detail` output (see [what to collect before filing a bug](/antalya/cas/operations/debugging#filing-a-bug)) before taking any destructive action | +| `SYSTEM CAS FSCK` or `clickhouse-disks cas-fsck` times out on a large pool | The scan is bounded by `--timeout` (default 600s / the `SYSTEM` form has no override); a large `roots/` prefix can make the scan slow | Retry with `--partial` to see the counts accumulated so far instead of aborting empty-handed, or `--namespace ` to scope the scan to a subset of namespaces | +| `SYSTEM CAS DROP POOL MEMBER` returns a non-empty `warnings` column | A per-object drain step could not confirm emptiness; the mount slot is left terminated but not fully drained, as a resume anchor | Rerun the same command — it is resumable and skips namespaces already marked removed, reporting them under `namespaces_already_removed` | +| Writes or `ALTER`s on a `CAS` disk fail with a `READONLY`-class error | The disk's metadata storage rejects every mutating entry point; this is deliberate for a disk opened with `true`, used by every offline `clickhouse-disks` tool | Confirm whether the disk was intentionally configured read-only (offline inspection, `cas-fsck`, `cas-gc-dryrun`, `cas-gc-rebuild`, `cas-drop-member` all require it); a production disk serving writes must not carry `true` | +| A table stays unavailable after a transient network error during startup | `AsyncLoader` has no retry/requeue path for a failed table load job: a transient S3 `NETWORK_ERROR` during `CAS` ref-table startup recovery can leave the job permanently `FAILED` | Restart the server, or issue a fresh load for the table; this is a one-shot job design, not a `CAS`-specific bug | +| A `CAS` server process aborts after its pool directory was removed or renamed while mounted | The lease-renewal thread observes a confirmed mismatch and raises `LOGICAL_ERROR`; in debug/sanitizer builds constructing that exception aborts the process | Never remove or rename a mounted pool's storage path while a server has it live; to retire a member permanently use [`SYSTEM CAS DROP POOL MEMBER`](/antalya/cas/operations/migration#decommission) instead of raw filesystem operations | +| Stale-looking part metadata after an out-of-band change to the pool | The part-folder view cache may be serving a retained (not re-validated) view | Set the disk-level `part_folder_cache_bytes = 0` as a diagnostic kill switch to disable retention, and run `fsck`/integrity checks with `part_folder_validate = always` so every read re-proves the body | +| A wide merge (many thousands of columns) fails with a port-exhaustion error from the network layer | Each column in a wide part can cost a separate object-store operation in one merge, and a very wide part can issue on the order of the column count in requests, exhausting local ephemeral TCP ports under load | Reduce concurrent merge parallelism on that table, or increase the host's ephemeral port range; this is a general high-fan-out-merge limit, not specific to content addressing | diff --git a/docs/en/antalya/cas/quick-start.md b/docs/en/antalya/cas/quick-start.md new file mode 100644 index 000000000000..b40e7d547121 --- /dev/null +++ b/docs/en/antalya/cas/quick-start.md @@ -0,0 +1,145 @@ +--- +description: 'A minimal content-addressed storage disk config and the first CREATE TABLE, INSERT, and SELECT against it, executed live before publication.' +sidebar_label: 'Quick start' +sidebar_position: 2 +slug: /antalya/cas/quick-start +title: 'CAS Quick Start' +doc_type: 'guide' +--- + +# Quick start {#quick-start} + +## The disk config {#disk-config} + +A `CAS` disk is an `object_storage` disk with `metadata_type` set to `cas` and an explicit, +per-server `server_root_id`. This example uses the `local` object-storage backend so it needs +nothing beyond a `ClickHouse` binary — no bucket, no credentials: + +```xml + + + + + object_storage + local + cas + quickstart-demo + cas_pool/ + + + cache + cas + cas_cache/ + 10Gi + + + + + +
+ cas_cache +
+
+
+
+
+
+``` + +The `cas_cache` disk layers a local filesystem cache over `cas`: it absorbs repeated reads of the +same blob while `cas` stays the source of truth, and the policy's volume points at the cached disk +— see [configuration](/antalya/cas/configuration#disk-config) for the sizing note. + +`server_root_id` must be unique per server sharing a pool. On a single, non-replicated server a +literal string, as above, is enough; on a replicated cluster where every replica shares one config, +`{replica}` expands through the same macro substitution an `s3` +disk's `endpoint` already uses, giving each replica a distinct subtree from one template. + +**S3 endpoint variant.** Swap `object_storage_type` to `s3` and add the usual object-storage +connection keys; nothing else in this config changes: + +```xml + + object_storage + s3 + cas + quickstart-demo + https://bucket.s3.amazonaws.com/cas/ + ... + ... + +``` + +`cas_cache` is unaffected by this swap — it wraps `disk cas` regardless of which object-storage +backend `cas` itself uses. See [bucket requirements](/antalya/cas/bucket-requirements) for what the +target bucket needs to support, and [configuration](/antalya/cas/configuration) for the full +settings surface. + +## First table {#first-table} + +```sql +CREATE TABLE events (event_date Date, event_id UInt64, payload String) +ENGINE = MergeTree ORDER BY event_id +SETTINGS storage_policy = 'cas'; + +INSERT INTO events VALUES ('2026-08-04', 1, 'hello'), ('2026-08-04', 2, 'world'); + +SELECT * FROM events ORDER BY event_id; +``` + +```text + ┌─event_date─┬─event_id─┬─payload─┐ +1. │ 2026-08-04 │ 1 │ hello │ +2. │ 2026-08-04 │ 2 │ world │ + └────────────┴──────────┴─────────┘ +``` + +An ordinary `MergeTree` table on a `CAS` disk. `INSERT`, `SELECT`, merges, and mutations all work +exactly as on any other `MergeTree` — the content-addressing is invisible at the SQL surface. + +## Checking the mount {#checking-the-mount} + +```sql +SELECT disk, server_root_id, state, is_leader FROM system.cas_mounts; +``` + +```text +Row 1: +────── +disk: cas +server_root_id: quickstart-demo +state: live +is_leader: 0 + +Row 2: +────── +disk: cas_cache +server_root_id: quickstart-demo +state: live +is_leader: 0 +``` + +`system.cas_mounts` shows every server currently sharing this pool, not just the local one. With a +cache layered in front, the same mount shows up **twice** — once under each configured disk name +(`cas` and `cas_cache`), both reporting the one underlying `server_root_id` — because the table +lists a row per configured disk, not per mount; this is the one visible change the cache layer adds +to this page's output. `is_leader` is `0` on both rows because `GC` leader election is asynchronous +and had not yet run at query time on this freshly mounted disk — see +[mounts and leases](/antalya/cas/architecture/mounts-and-leases) for the full column reference and +[garbage collection](/antalya/cas/architecture/garbage-collection) for leadership. + +## What just happened {#what-happened} + +The `INSERT` wrote two part files as content-addressed blobs, a part manifest listing them, and a +ref pointing the part name at that manifest — the only mutable object the write touched. On a +second replica sharing this same pool, inserting or fetching the identical content publishes a ref +without re-uploading a single byte; see +[garbage collection](/antalya/cas/architecture/garbage-collection) for how a dropped part's blobs +get reclaimed once nothing references them anymore. + +This exact cache-layered configuration and SQL were run against a live server before publication: +`CREATE TABLE`, `INSERT`, `SELECT`, and the `system.cas_mounts` query above all completed with zero +errors, with the two-row `system.cas_mounts` output shown above captured from that run. The +`INSERT`/`SELECT` output is unaffected by the cache — the one visible difference the cache layer +adds anywhere on this page is that second `system.cas_mounts` row. + diff --git a/docs/en/antalya/cas/roadmap.md b/docs/en/antalya/cas/roadmap.md new file mode 100644 index 000000000000..0c8b14110d89 --- /dev/null +++ b/docs/en/antalya/cas/roadmap.md @@ -0,0 +1,108 @@ +--- +description: 'What CAS ships today, what is still planned, known platform limitations, and design directions deliberately not taken.' +sidebar_label: 'Roadmap' +sidebar_position: 5 +slug: /antalya/cas/roadmap +title: 'CAS Roadmap' +doc_type: 'guide' +--- + +# CAS roadmap {#cas-roadmap} + +CAS is experimental (see [status](/antalya/cas/)): the format and SQL surface can still change. +This page tracks what already works, what is still ahead, and — since a project this deep in +adversarial verification collects real dead ends — what was tried and deliberately not shipped. + +## Shipped {#shipped} + +**Storage and object model.** Content-addressed blobs deduplicated across every replica sharing +a pool; immutable part manifests; a pluggable blob-hash algorithm (`cityhash128` default, +`xxh3-128`, or `sha256`) fixed per pool at creation; a JSON-text object format end to end (no +binary framing, no protobuf) so any object can be read with ordinary line-oriented tools. + +**Write path.** Conditional writes (create-if-absent, compare-and-swap) as the only mutual +exclusion primitive the pool needs; an adaptive HEAD-before-PUT dedup gate; a bounded thread pool +fanning out multi-blob part uploads in parallel; carry-forward on mutation for `Wide` parts (an +untouched column is re-referenced, not re-hashed). + +**Read path.** Ref resolution to manifest to ranged blob reads, with a manifest-decode cache and +a part-folder view cache sitting on that path. + +**Replication.** Fetch by relink between replicas sharing a pool — a replicated fetch publishes a +ref pointing at blobs the pool already has, at zero bytes on the wire — with a publish-then-confirm +protocol that closes the sender-crash and stale-cache races a naive relink would be exposed to. + +**Garbage collection.** An 18-phase round built on a causal ack-floor (no separate fence-and-recheck +phase); sharded folding (`gc_shards`); condemn/spare bookkeeping; generation pruning with a +configurable retention window; a dry-run mode and a rebuild path for recovery. + +**Mounts and identity.** Explicit `server_root_id` per disk; a renewable mount lease with +observation-based reclaim of an expired predecessor (never trusting a foreign body's wall-clock +timestamp); clean decommission of a permanently departed pool member +(`SYSTEM CAS DROP POOL MEMBER`). + +**Backends.** AWS S3 (`ETag`-based conditional dialect) and Google Cloud Storage (generation-token +dialect) both live-validated; a capability probe that runs at every writable mount and refuses to +proceed on a backend that does not enforce the conditions CAS depends on. + +**Operability.** `system.cas_log`, `system.cas_gc_log`, and `system.cas_mounts` for introspection; +`clickhouse-disks` commands `ca-fsck`, `ca-inspect`, `ca-gc-dryrun`, and `ca-gc-rebuild`; the +`SYSTEM CAS` SQL control surface (`GC RUN`/`STOP`/`START`/`REBUILD`, `FSCK`, `FORGET`, `DROP POOL +MEMBER`). + +**Coexistence.** `metadata_type = cas` is opt-in per disk; zero-copy replication keeps working +unmodified on disks that do not opt in — see [why CAS exists](/antalya/cas/) for the fuller +positioning. + +## In progress / planned {#in-progress} + +- **Azure real-store validation.** AWS and GCS are live-validated; Azure is not — see + [known limitations](#known-limitations) below. +- **WORM deployments.** A read-only disk mode exists today; a fuller write-once story — a pool + served immutably, with pinned snapshots for read-only replicas — has a draft design and is not + yet implemented. +- **Backup and restore.** See [Backups](#backups) below — this is further along as a design than as + an implementation. +- **First-class local-disk pools.** Today a pool over local paths runs a minimal best-effort + emulation of the conditional-write dialect (single-process, serialized resurrections). Making the + local mode efficient in its own right is under consideration: a local CAS tier is a natural target + for backups, pinned snapshots, and moving data between CAS tiers. + +## Known limitations {#known-limitations} + +- **Azure Blob Storage's REST API documents the equivalent conditional headers CAS needs, but no + CAS conditional-write dialect is wired up for it yet** — untested, not validated by the capability + probe. See [bucket requirements](/antalya/cas/bucket-requirements) and + [the backend page](/antalya/cas/architecture/backend) for the AWS/GCS dialects that are wired. +- **Other S3-compatible object stores qualify only if they pass the capability probe** — a store + that silently ignores conditional writes is refused at mount time rather than trusted. Bucket + versioning must be off; it is not required to be on. +- **The format and settings surface can still change.** CAS is pre-release: there is no persisted + production data to keep compatible, so a format change costs a version bump, not a migration. + Treat every detail on these pages as subject to change until the format is declared stable. + +## Backups {#backups} + +A `snapshot` / `mirror` / `fetch` / `restore` design is **approved but not implemented**. The +model is deliberately git-shaped: `snapshot` is instant and free (like `git tag` — it references +existing manifests, copies nothing); `mirror` is a continuous pull from a production pool into a +backup pool (like `git push --mirror`); `fetch` is a selective pull from a backup pool into a +fresh pool (a partial clone); `restore` is an in-pool relink (like `git checkout`, instant). One +closure-walk-and-hash-verification primitive is meant to serve all three pool-to-pool movements. +None of this is wired into the `BACKUP`/`RESTORE` SQL surface yet. + +## Deliberately rejected directions {#rejected} + +A short pointer list; the reasons and the counterexamples that drove each decision are in +[design history](/antalya/cas/architecture/design-history). + +- A Merkle tree layer as a distinct object kind. +- Epoch-based reclamation as the GC core. +- An integer in-degree refcount instead of a folded edge set. +- A persistent, append-only namespace registry for GC discovery. +- Per-incarnation body keys as an alternative to an in-body incarnation tag. +- Using a blob's freshness metadata as the authority for its lifecycle instead of an advisory hint. +- A separate all-shard fence-and-recheck phase per GC round. +- A sparse ref-id allocator with a certificate stack bolted on to prove completeness. +- Extending zero-copy replication instead of building a new mechanism — CAS is an alternative to + zero-copy, not a replacement; both remain available. From 056488b47a0b3f32ea2ebf83d508b697ff4c0c17 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 5 Aug 2026 00:07:41 +0200 Subject: [PATCH 30/30] CAS documentation User-facing documentation: content-addressed storage in storing-data, the SYSTEM statements, and the three system-table pages. Co-Authored-By: Claude Fable 5 Signed-off-by: Mikhail Filimonov --- docs/en/operations/storing-data.md | 102 +++++++++++- .../en/operations/system-tables/cas_gc_log.md | 156 ++++++++++++++++++ docs/en/operations/system-tables/cas_log.md | 62 +++++++ .../en/operations/system-tables/cas_mounts.md | 81 +++++++++ docs/en/sql-reference/statements/system.md | 145 ++++++++++++++++ 5 files changed, 545 insertions(+), 1 deletion(-) create mode 100644 docs/en/operations/system-tables/cas_gc_log.md create mode 100644 docs/en/operations/system-tables/cas_log.md create mode 100644 docs/en/operations/system-tables/cas_mounts.md diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index 8a2903cd1ebb..c26316f5cdff 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -46,7 +46,7 @@ It requires specifying:
-Optionally, `metadata_type` can be specified (it is equal to `local` by default), but it can also be set to `plain`, `web` and, starting from `24.4`, `plain_rewritable`. +Optionally, `metadata_type` can be specified (it is equal to `local` by default), but it can also be set to `plain`, `web`, `plain_rewritable` (starting from `24.4`) and `cas`. Usage of `plain` metadata type is described in [plain storage section](/operations/storing-data#plain-storage), `web` metadata type can be used only with `web` object storage type, `local` metadata type stores metadata files locally (each metadata files contains mapping to files in object storage and some additional meta information about them). For example: @@ -452,6 +452,106 @@ is equal to Starting from `24.5` it is possible to configure any object storage disk (`s3`, `azure`, `local`) using the `plain_rewritable` metadata type. +### Using Content-Addressed Storage {#content-addressed-storage} + +Setting `metadata_type` to `cas` turns a disk into a content-addressed (CAS) disk: every +object is addressed by the hash of its content rather than by a randomly generated blob name, so +identical content written by different parts (or different tables) is stored once and shared. A +background garbage collector reclaims objects once no part references them anymore; see +[`SYSTEM CAS GC RUN`](/sql-reference/statements/system#system-cas-gc-run), +[`SYSTEM CAS GC REBUILD`](/sql-reference/statements/system#system-cas-gc-rebuild), +[`SYSTEM CAS DROP POOL MEMBER`](/sql-reference/statements/system#system-cas-drop-pool-member), +and the [`system.cas_gc_log`](/operations/system-tables/cas_gc_log), +[`system.cas_mounts`](/operations/system-tables/cas_mounts), and +[`system.cas_log`](/operations/system-tables/cas_log) system tables. See the +[content-addressed storage documentation](/antalya/cas) for the architecture, operations +runbooks, and a live-validated quick start. + +Configuration: + +```xml + + object_storage + s3 + cas + https://s3.eu-west-1.amazonaws.com/clickhouse-eu-west-1.clickhouse.com/data/ + 1 + + server-{replica} + disks/s3_cas/cas_scratch/ + local + cityhash128 + true + 60 + 1 + 67108864 + 67108864 + always + +``` + +The CAS-specific settings are written directly inside the disk element, alongside +`object_storage` / `` and the connection settings, which are the +same as for any other `object_storage` disk. Since the disk element already scopes every key to this +disk, none of the keys below carry a redundant `cas_`/`ca_` prefix. + +#### Required parameters {#required-parameters-content-addressed} + +- `server_root_id` — the subtree of the shared pool that this server owns. When several replicas + mount the same pool (same `endpoint`), each one must own a distinct subtree, so this is normally + written with a macro, e.g. `server-{replica}`. Missing this key is + a startup error. + +#### Optional parameters {#optional-parameters-content-addressed} + +These are the commonly used settings; see [Configuration](/antalya/cas/configuration) for the full +disk-level and server-level settings surface. + +- `scratch_path` — a real, server-local filesystem directory used to spill the write buffer before it + is committed to the pool (never the object-storage key prefix). Defaults to + `/disks//cas_scratch/`. A relative override is anchored to the server + data path, not the process's current working directory. +- `staging_backend` — `local` (default) or `s3`. Selects where in-flight part data is staged before + being committed into the pool; `local` is byte-for-byte the original write path, `s3` enables + S3-native staging. +- `blob_hash` — `cityhash128` (default), `xxh3-128`, or `sha256`. Selects the pool's blob + content-hash function. The choice is fixed at pool creation; a reopen whose `blob_hash` disagrees + with the pool's recorded algorithm fails closed. See + [choosing `blob_hash`](/antalya/cas/configuration#choosing-blob-hash) for the trade-offs between + the three. +- `blob_hash_allow_new` — `false` by default. Admits a new hash algorithm into an existing pool's set + of recorded algorithms; without it, a `blob_hash` that disagrees with what the pool already recorded + fails closed instead of silently turning the pool mixed-algorithm. +- `gc_enabled` — `true` by default. Enables the background garbage collector for this disk. +- `gc_interval_sec` — `60` by default; must be `>= 1`. Interval between background GC rounds. +- `gc_shards` — `1` by default; must be `>= 1`. Number of blob-hash-prefix shards the GC reducer + splits work across. This is a creation-time-only setting: on reopen the pool's persisted GC state is + authoritative. +- `deduplication_cache_bytes` — `64` MiB by default. Size of the in-memory deduplication lookup cache. +- `deduplication_head_first_min_bytes` — `1` MiB by default. Minimum blob size at which a `HEAD` is sent + before the body, so that an upload of already-present content can be skipped. `0` disables it. +- `gc_snapshot_generations_to_keep` — `3` by default. Number of past GC snapshot generations retained. +- `gcs_max_conditional_put_bytes` — `1` GiB by default. On generation-token backends (Google Cloud + Storage), the body of a conditional write is RAM-buffered up to this size; a larger conditional + write throws `NOT_IMPLEMENTED`. Irrelevant on `ETag`-based backends such as AWS S3. +- `part_folder_cache_bytes` — `64` MiB by default. Size of the part-folder view cache. `0` disables + retention; this is a supported permanent operational configuration, not only a debug aid. +- `part_folder_cache_max_entries` — `10000` by default. Maximum number of entries in the part-folder + view cache. +- `part_folder_cache_max_entry_bytes` — `16` MiB by default. Maximum size of a single cached + part-folder view entry. +- `part_folder_validate` — `always` (default), `never`, or `age `. Controls how often a + `ForceFresh` read re-proves a cached manifest body via a `HEAD` request: `always` re-proves every + time (the original, pre-optimization behavior), `never` trusts the cache without re-proving, and + `age ` re-proves only once the cached entry is older than the given number of seconds. +- `manifest_decode_cache_bytes` — `128` MiB by default. Byte bound for the decoded-manifest cache. + `0` disables decode caching entirely (a diagnostic mode). +- `gc_meta_pool_size` — `16` by default. Bounded thread-pool size for the GC's per-hash freshness-meta + writes (condemn/spare/delete), so a mass `DROP` condemning millions of blobs does not run fully + sequentially. +- `skip_access_check` — `false` by default. Skips the disk's startup access check ("start now, fix + later"), unlike the generic disk-wide startup flag. + ### Using Azure Blob Storage {#azure-blob-storage} `MergeTree` family table engines can store data to [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/) diff --git a/docs/en/operations/system-tables/cas_gc_log.md b/docs/en/operations/system-tables/cas_gc_log.md new file mode 100644 index 000000000000..5fd04b4fb11d --- /dev/null +++ b/docs/en/operations/system-tables/cas_gc_log.md @@ -0,0 +1,156 @@ +--- +description: 'System table containing per-round records of the content-addressed (CAS) MergeTree garbage collector.' +sidebar_label: 'cas_gc_log' +sidebar_position: 30 +slug: /operations/system-tables/cas_gc_log +title: 'system.cas_gc_log' +doc_type: 'reference' +--- + +## Description {#description} + +The `system.cas_gc_log` table contains per-round records of the +content-addressed (CAS) MergeTree garbage collector. For every garbage-collection round it stores a +`Start` row and a `Finish` row (like `system.part_log` stores events per data part), with the counts +of objects marked and deleted, the round duration, the outcome, and a per-round `ProfileEvents` +delta. + +Between them it also stores one `Phase` row per GC phase the round reached, each carrying that +phase's own duration, its `ProfileEvents` delta, and its phase-specific counts. All rows of one round +share a `round_id`. See [Per-phase rows](#per-phase-rows). + +Rounds are emitted both by the background GC scheduler (`trigger = 'Scheduled'`) and by the +synchronous [`SYSTEM CAS GC RUN`](/sql-reference/statements/system#system-cas-gc-run) +command (`trigger = 'Manual'`). + +The table is created only if the `cas_gc_log` server setting is +specified (it is enabled by default in the shipped `config.xml`). + +## Columns {#columns} + +- `hostname` ([LowCardinality(String)](/sql-reference/data-types/lowcardinality)) — Host name of the server executing the round. +- `event_date` ([Date](/sql-reference/data-types/date)) — Event date. +- `event_time` ([DateTime](/sql-reference/data-types/datetime)) — Event time. +- `event_time_microseconds` ([DateTime64(6)](/sql-reference/data-types/datetime64)) — Event time with microseconds precision. +- `event_type` ([Enum8](/sql-reference/data-types/enum)) — `Start` or `Finish` of a GC round, or one `Phase` of it. +- `disk_name` ([LowCardinality(String)](/sql-reference/data-types/lowcardinality)) — The content-addressed disk the round ran on. +- `server_root_id` ([LowCardinality(String)](/sql-reference/data-types/lowcardinality)) — Identifies the mount whose GC scheduler ran this round. Distinguishes concurrent mounters of the same shared pool; join on this column when correlating rounds against [`system.cas_mounts`](/operations/system-tables/cas_mounts). +- `gc_id` ([String](/sql-reference/data-types/string)) — The GC scheduler instance id (which mounter ran the round). +- `trigger` ([Enum8](/sql-reference/data-types/enum)) — `Scheduled` (background tick) or `Manual` (`SYSTEM` command). +- `round` ([UInt64](/sql-reference/data-types/int-uint)) — The GC round number (`0` on a `Start` row). +- `outcome` ([Enum8](/sql-reference/data-types/enum)) — `Unknown` (on a `Start` row), `Success` (led, folded, and completed), `NotALeader` (another replica holds the GC lease), `Deferred` (led but took the skip-unchanged fast path — no fold ran, because no changed shard reached the fold threshold and no graduation was due), or `Error` (the round threw). +- `candidates_marked` ([UInt64](/sql-reference/data-types/int-uint)) — Objects retired (marked) this round. +- `objects_deleted` ([UInt64](/sql-reference/data-types/int-uint)) — Objects physically deleted this round. +- `objects_absent` ([UInt64](/sql-reference/data-types/int-uint)) — Retire candidates found already absent. +- `objects_replaced` ([UInt64](/sql-reference/data-types/int-uint)) — `412`-saves (a resurrection won the race against the delete). +- `objects_spared` ([UInt64](/sql-reference/data-types/int-uint)) — Candidates spared because their in-degree was greater than zero at recheck. +- `manifests_deleted` ([UInt64](/sql-reference/data-types/int-uint)) — Owner-removed manifest bodies physically deleted this round, counted separately from blob deletes. +- `entries_condemned` ([UInt64](/sql-reference/data-types/int-uint)) — Retired entries newly condemned this round (retired-cursor pipeline stage 1). +- `entries_graduated` ([UInt64](/sql-reference/data-types/int-uint)) — Retired entries newly floor-passed and republished `delete_pending` this round (pipeline stage 2; deleted the next round). +- `entries_redeleted` ([UInt64](/sql-reference/data-types/int-uint)) — Pending exact-token blob deletes executed this round (pipeline stage 3). +- `fence_outs` ([UInt64](/sql-reference/data-types/int-uint)) — Expired mounts fenced out by this round's heartbeat floor. +- `anomalies` ([UInt64](/sql-reference/data-types/int-uint)) — Fold clamps surfaced (and survived) this round. A steady non-zero value warrants a look at the round log details. +- `duration_ms` ([UInt64](/sql-reference/data-types/int-uint)) — The round wall-clock duration (on a `Finish` row). +- `error` ([String](/sql-reference/data-types/string)) — The exception text when `outcome = 'Error'`. +- `ProfileEvents` ([Map(LowCardinality(String), UInt64)](/sql-reference/data-types/map)) — On a `Start`/`Finish` row, the per-round `ProfileEvents` delta (the `CAS*` counters and S3/disk events for this round). On a `Phase` row, **that phase's** delta, so `GROUP BY phase` over `ProfileEvents['S3ListObjects']` attributes the round's `LIST` budget to the phase that spent it. +- `round_id` ([String](/sql-reference/data-types/string)) — The correlator for every row of one round attempt: its `Start`, each of its `Phase` rows, and its `Finish`. Minted per attempt, so unlike `round` it exists even for a round that never committed and for a round that never led. Group by this column to reconstruct one round. +- `phase` ([LowCardinality(String)](/sql-reference/data-types/lowcardinality)) — The GC phase this row describes; empty on `Start`/`Finish`. See [Per-phase rows](#per-phase-rows) for the phase list. +- `phase_duration_microseconds` ([UInt64](/sql-reference/data-types/int-uint)) — The wall-clock duration of this phase, in microseconds (`Phase` rows only). Microseconds rather than milliseconds because several phases are routinely sub-millisecond and the point of the row is to see when they are not. +- `phase_metrics` ([Map(LowCardinality(String), UInt64)](/sql-reference/data-types/map)) — Phase-specific semantic counts (`Phase` rows only) that a phase computes for itself and no `ProfileEvents` counter can supply. The verb counts ride the `ProfileEvents` column of the same row. + +## Per-phase rows {#per-phase-rows} + +Besides the `Start` and `Finish` row of each round, the collector emits one `Phase` row per GC phase. +Every row of one round attempt — `Start`, each `Phase`, and `Finish` — shares a `round_id`. A round +that defers, or that never acquires the GC lease, emits only the phases it actually reached; a round +that throws still emits the row of the phase it died in. + +The phases, in execution order: + +| `phase` | What it covers | Dominant I/O | +|---|---|---| +| `lease` | Acquire, renew, or observe the GC lease. The only phase a `NotALeader` round emits. | `gc/state` `GET` + compare-and-swap | +| `pre_fold_ref_drain` | Resolve catalog rows whose terminal fold evidence is already adopted before this invocation publishes or defers. | catalog `GET` + exact compare-and-swap | +| `heartbeat_floor` | Classify every mount slot and fence out the dead ones. | `LIST` of the mount prefix, one `GET` per mount, one `PUT` per fence | +| `defer_decision` | The skip-unchanged decision: graduation check plus the round's one enumeration of the ref prefix. | one full ref-prefix `LIST`, two fold-seal `GET`s | +| `parent_seal_read` | Capture the pre-fold seal's run refs for the hand-off reclaim. | one fold-seal `GET` | +| `fold_ref_group` | Regroup the round's enumeration into per-table listings — what this round will fold. | none | +| `fold_seal_read` | The adopted fold seal, read twice at the same generation and attempt. | two fold-seal `GET`s | +| `fold_ref_intake` | Read and fold every new ref log and the manifest bodies its edges name. | one `GET` per new log, one `GET` per manifest edge | +| `fold_reduce` | The per-shard in-degree merge: condemn, spare, graduate. | prior-run streaming `GET`s, one `HEAD` per zero-transition candidate, run `PUT`s | +| `fold_seal_write` | Publish the new fold seal. | one `PUT` | +| `pending_deletes` | The single content-delete site: exact-token deletes of previously published `delete_pending` entries, plus the outcome logs. | one `DELETE` per entry, one outcome-log `PUT` per shard | +| `meta_pool_wait` | Drain the round's per-hash freshness-meta writes. | none on this thread — see the caveat below | +| `round_commit` | The generation-retention prune and the round's single `gc/state` compare-and-swap. | prune `LIST`s and deletes, one compare-and-swap | +| `handoff_reclaim` | Wholesale-reclaim generations a moved run ref stranded below the retention cursor. | prefix `LIST`s and deletes | +| `manifest_deletes` | Exact-token deletes of owner-removed manifest bodies, after their decrements were adopted. | one `DELETE` per body | +| `namespace_cleanup` | Run one bounded `cas/ns/` page across the stream and state subtrees for the perpetual dead-life janitor. This phase is physical reclamation, not a lifecycle gate. | one namespace-root page `LIST`, catalog cut, exact-token deletes | +| `ref_object_cleanup` | Delete ref logs covered by both the durable fold cursor and a durable snapshot, plus superseded snapshots. | one `HEAD` + one `DELETE` per deletable object | +| `orphan_sweep` | The budgeted, cursor-paced orphan part-manifest backstop. | budgeted `LIST` and deletes | + +Which phase dominates a round: + +```sql +SELECT phase, + count() AS rounds, + quantile(0.5)(phase_duration_microseconds) AS p50_microseconds, + quantile(0.99)(phase_duration_microseconds) AS p99_microseconds, + sum(phase_duration_microseconds) AS total_microseconds +FROM system.cas_gc_log +WHERE event_type = 'Phase' AND disk_name = 'ca' +GROUP BY phase +ORDER BY total_microseconds DESC; +``` + +Which phase spends the `LIST` budget: + +```sql +SELECT phase, sum(ProfileEvents['S3ListObjects']) AS lists +FROM system.cas_gc_log +WHERE event_type = 'Phase' AND disk_name = 'ca' +GROUP BY phase +ORDER BY lists DESC; +``` + +One round, in order — including a round that failed, which is why the correlator is `round_id` and +not `round`: + +```sql +SELECT phase, phase_duration_microseconds, phase_metrics, ProfileEvents['S3ListObjects'] AS lists +FROM system.cas_gc_log +WHERE round_id = '...' AND event_type = 'Phase' +ORDER BY event_time_microseconds; +``` + +Two caveats when reading these rows: + +- Work scheduled onto the GC meta pool runs on other threads, so the `meta_pool_wait` row's + `ProfileEvents` delta is **empty by construction**. Read its `phase_metrics` `jobs_scheduled` / + `jobs_completed` next to its duration instead: they distinguish a deep queue from a slow endpoint. +- Phase durations do not sum to the round's `duration_ms`. The round also performs untimed + bookkeeping between phases, and the `Finish` row's `duration_ms` remains the authority on total + round time. + +## Example {#example} + +```sql +SELECT + event_type, + disk_name, + trigger, + outcome, + candidates_marked, + objects_deleted, + duration_ms +FROM system.cas_gc_log +ORDER BY event_time_microseconds DESC +LIMIT 2 +FORMAT Vertical; +``` + +## See Also {#see-also} + +- [`SYSTEM CAS GC RUN`](/sql-reference/statements/system#system-cas-gc-run) — run one GC round synchronously. +- [`system.cas_mounts`](/operations/system-tables/cas_mounts) — live per-`server_root_id` mount and GC-health state. +- [`system.cas_log`](/operations/system-tables/cas_log) — per-decision event log for the CAS garbage collector and writer. +- [`system.part_log`](/operations/system-tables/part_log) — the analogous per-part event log. diff --git a/docs/en/operations/system-tables/cas_log.md b/docs/en/operations/system-tables/cas_log.md new file mode 100644 index 000000000000..315842d3b9b4 --- /dev/null +++ b/docs/en/operations/system-tables/cas_log.md @@ -0,0 +1,62 @@ +--- +description: 'System table containing a per-decision event log for the content-addressed (CAS) MergeTree writer and garbage collector.' +sidebar_label: 'cas_log' +sidebar_position: 32 +slug: /operations/system-tables/cas_log +title: 'system.cas_log' +doc_type: 'reference' +--- + +## Description {#description} + +The `system.cas_log` table contains a per-decision event log for the content-addressed +(CAS) MergeTree storage engine: blob puts and dedup adoptions, root/ref transitions, in-degree changes, +garbage-collector retire decisions and recheck verdicts, blob deletes, and dangling-access/corruption +findings. It is a much finer-grained, per-event complement to +[`system.cas_gc_log`](/operations/system-tables/cas_gc_log), +which only records one `Start`/`Finish` row per GC round. + +## Columns {#columns} + +- `hostname` ([LowCardinality(String)](/sql-reference/data-types/lowcardinality)) — Host name of the server that emitted the event. +- `event_date` ([Date](/sql-reference/data-types/date)) — Event date. +- `event_time` ([DateTime](/sql-reference/data-types/datetime)) — Event time. +- `event_time_microseconds` ([DateTime64(6)](/sql-reference/data-types/datetime64)) — Event time with microseconds precision. +- `event_type` ([LowCardinality(String)](/sql-reference/data-types/lowcardinality)) — The CAS decision/event, e.g. `blob_put`, `blob_reuse_adopt`, `root_remove`, `indegree_zero`, `gc_retire_decision`, `gc_recheck_verdict`, `blob_delete`, `dangling_access`, `corrupt_dangle`. +- `disk_name` ([LowCardinality(String)](/sql-reference/data-types/lowcardinality)) — The content-addressed disk / pool the event belongs to. +- `namespace` ([String](/sql-reference/data-types/string)) — `roots/` (server/table); empty if not applicable. +- `ref_name` ([String](/sql-reference/data-types/string)) — Part name / ref the event concerns; empty if not applicable. +- `object_kind` ([LowCardinality(String)](/sql-reference/data-types/lowcardinality)) — One of `none`, `blob`, `manifest`, `root`, `snapshot`. +- `object_hash` ([String](/sql-reference/data-types/string)) — Content hash (lowercase hex) of the object; empty if not applicable. +- `token` ([String](/sql-reference/data-types/string)) — Incarnation token (`ETag`) involved; empty if not applicable. +- `round` ([UInt64](/sql-reference/data-types/int-uint)) — GC round (`0` if not applicable). +- `generation` ([UInt64](/sql-reference/data-types/int-uint)) — GC snapshot generation (`0` if not applicable). +- `at_version` ([UInt64](/sql-reference/data-types/int-uint)) — Manifest `shard_version` of the driving journal record (`0` if not applicable). +- `outcome` ([LowCardinality(String)](/sql-reference/data-types/lowcardinality)) — Decision outcome, e.g. `ok`, `adopt`, `resurrect`, `deleted`, `replaced`, `spared`, `absent`, `zeroed`, `skipped`. +- `reason` ([LowCardinality(String)](/sql-reference/data-types/lowcardinality)) — Human-readable rationale for the decision. Templated across rows, so it is `LowCardinality`. +- `thread_id` ([UInt64](/sql-reference/data-types/int-uint)) — OS thread that emitted the event. +- `query_id` ([String](/sql-reference/data-types/string)) — Query id for correlation with [`system.query_log`](/operations/system-tables/query_log); empty if not applicable. +- `detail` ([Map(LowCardinality(String), String)](/sql-reference/data-types/map)) — Structured event-specific facts, e.g. `condemn_round`, `superseded_token`, `code`, `site`. + +## Example {#example} + +```sql +SELECT + event_time_microseconds, + event_type, + disk_name, + ref_name, + object_kind, + outcome, + reason +FROM system.cas_log +ORDER BY event_time_microseconds DESC +LIMIT 10 +FORMAT Vertical; +``` + +## See Also {#see-also} + +- [`system.cas_gc_log`](/operations/system-tables/cas_gc_log) — per-round GC event log. +- [`system.cas_mounts`](/operations/system-tables/cas_mounts) — live per-`server_root_id` mount and GC-health state. +- [`system.query_log`](/operations/system-tables/query_log) — correlate via `query_id`. diff --git a/docs/en/operations/system-tables/cas_mounts.md b/docs/en/operations/system-tables/cas_mounts.md new file mode 100644 index 000000000000..5a9465db04af --- /dev/null +++ b/docs/en/operations/system-tables/cas_mounts.md @@ -0,0 +1,81 @@ +--- +description: 'System table containing the live mount and GC-health state of every server mounted onto a content-addressed (CAS) disk pool.' +sidebar_label: 'cas_mounts' +sidebar_position: 31 +slug: /operations/system-tables/cas_mounts +title: 'system.cas_mounts' +doc_type: 'reference' +--- + +## Description {#description} + +The `system.cas_mounts` table contains one row per mount slot discovered on every +content-addressed (CAS) disk configured on the node. A pool may be shared by several servers (or +several `server_root_id` mounts on the same server), and this table lists every mount visible in +the pool's backend at query time, not only the querying server's own mount — it exists for +incident-time diagnosis of leases, epochs, and GC leadership across a shared pool. + +The table is read directly from the CAS disk's backend on every query (there is no persisted log +behind it); a transient backend error on one disk is skipped and does not blind the rest of the +rows. + +## Columns {#columns} + +- `disk` ([String](/sql-reference/data-types/string)) — Name of the content-addressed disk. +- `server_root_id` ([String](/sql-reference/data-types/string)) — Server root id owning the mount slot. +- `server_uuid` ([UUID](/sql-reference/data-types/uuid)) — UUID of the server incarnation holding the lease. +- `hostname` ([String](/sql-reference/data-types/string)) — Hostname recorded in the lease body. +- `process_id` ([UInt64](/sql-reference/data-types/int-uint)) — Process id recorded in the lease body. +- `writer_epoch` ([UInt64](/sql-reference/data-types/int-uint)) — Fenced writer epoch of the incarnation. +- `renewal_sequence` ([UInt64](/sql-reference/data-types/int-uint)) — Lease renewal sequence number. +- `started_at` ([DateTime64(3)](/sql-reference/data-types/datetime64)) — Time when the lease started. +- `expires_at` ([DateTime64(3)](/sql-reference/data-types/datetime64)) — Time when the lease expires. +- `min_active_build_sequence` ([UInt64](/sql-reference/data-types/int-uint)) — Oldest in-flight build sequence (`UINT64_MAX` means the mount said farewell). +- `gc_fenced` ([UInt8](/sql-reference/data-types/int-uint)) — `1` if GC fenced this slot out (terminal). +- `state` ([String](/sql-reference/data-types/string)) — One of `live`, `expired`, `terminated`, `fenced`, `corrupt`. +- `is_leader` ([Nullable(UInt8)](/sql-reference/data-types/nullable)) — `1` if this server's GC scheduler currently holds this disk's leadership lease. +- `pending_reclaim` ([Nullable(Int64)](/sql-reference/data-types/nullable)) — Cumulative two-phase deletion backlog observed by this process's GC on this disk (condemned entries minus executed exact-token deletes). +- `last_success_age_seconds` ([Nullable(UInt64)](/sql-reference/data-types/nullable)) — Seconds since this disk's GC last led a round (`0` if it has never led or GC is not running here). +- `wedged_namespace_count` ([Nullable(UInt64)](/sql-reference/data-types/nullable)) — Ref-append lanes currently wedged on this disk (an uncertain `PUT` exhausted its retry budget). +- `lifecycle` ([String](/sql-reference/data-types/string)) — This server's content-addressed pool lifecycle for the disk (a non-gated snapshot, always populated so a not-live disk stays visible): one of `live`, `not_live`, `identity_lost`, `vanished`, `constructing` (never started), or `shutdown` (torn down). +- `lifecycle_reason` ([String](/sql-reference/data-types/string)) — The enum-clean sub-state word for a `vanished` disk: `replaced` or `forgotten`. Empty for every other lifecycle, so `lifecycle || '(' || lifecycle_reason || ')'` reads e.g. `vanished(forgotten)`. +- `lifecycle_detail` ([String](/sql-reference/data-types/string)) — The full typed reason text naming the actual cause when not live: the vanish diagnosis (a data root replaced by a foreign pool, or decommissioned by `SYSTEM CAS FORGET` at a given time) or the identity-loss message. Empty when live. +- `lifecycle_since` ([Nullable(DateTime)](/sql-reference/data-types/nullable)) — When this server entered the current non-live lifecycle state. `NULL` when live, or when the state has no backing pool to date from. + +`lifecycle`/`lifecycle_reason`/`lifecycle_detail`/`lifecycle_since` are the SQL surface for +diagnosing an identity-lost or forgotten disk without reading server logs — see the +`IdentityLost`/`VanishedReplaced`/`VanishedForgotten` states on the +[mount-slot behavioral model](/antalya/cas/architecture/mounts-and-leases#mount-state-machines) for +what each lifecycle value means, and [`SYSTEM CAS FORGET`](/antalya/cas/operations/debugging#sql-forget) +for the command that produces `vanished(forgotten)`. + +:::note Local-only GC-health columns +`is_leader`, `pending_reclaim`, `last_success_age_seconds`, and `wedged_namespace_count` are process-local +facts about *this* server's own GC scheduler. They are populated **only** on the row whose `server_root_id` matches +this server's own mount, and are `NULL` on every row describing another server's mount — stamping a local +health fact onto a peer's row would misread as "the peer is the GC leader" during an incident. To see the +peer's own view of these columns, query `system.cas_mounts` on that server. +::: + +## Example {#example} + +```sql +SELECT + disk, + server_root_id, + state, + writer_epoch, + is_leader, + pending_reclaim, + last_success_age_seconds +FROM system.cas_mounts +ORDER BY disk, server_root_id +FORMAT Vertical; +``` + +## See Also {#see-also} + +- [`system.cas_gc_log`](/operations/system-tables/cas_gc_log) — per-round GC event log. +- [`system.cas_log`](/operations/system-tables/cas_log) — per-decision event log for the CAS garbage collector and writer. +- [`SYSTEM CAS GC RUN`](/sql-reference/statements/system#system-cas-gc-run) — run one GC round synchronously. +- [`SYSTEM CAS DROP POOL MEMBER`](/sql-reference/statements/system#system-cas-drop-pool-member) — permanently decommission a dead pool member's `server_root_id`. diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 69b2b59cb0ae..ed3a2b97cf87 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -457,6 +457,151 @@ Wait until all asynchronously loading data parts of a table (outdated data parts SYSTEM WAIT LOADING PARTS [ON CLUSTER cluster_name] [db.]merge_tree_family_table_name ``` +### SYSTEM CAS GC RUN {#system-cas-gc-run} + +Runs one garbage-collection round of the content-addressed (CAS) MergeTree garbage collector synchronously and node-local: it reclaims content-addressed objects that are no longer referenced by any part. This is the on-demand counterpart of the background GC scheduler; it is useful for tests and diagnostics. + +```sql +SYSTEM CAS GC RUN [ON CLUSTER cluster_name] [disk_name] +``` + +When `disk_name` is given, the round runs on that content-addressed disk only; targeting a non-content-addressed disk raises an exception. When `disk_name` is omitted, one round runs on every content-addressed disk configured on the node; if none are configured, the command raises an exception. + +Each round is recorded in [`system.cas_gc_log`](/operations/system-tables/cas_gc_log) as a `Start` and a `Finish` row (with `trigger = 'Manual'`). + +The command returns one row per disk it ran on (multiple rows when `disk_name` is omitted), with columns `disk`, `acquired_lease`, `deferred`, `round`, `candidates_marked`, `objects_deleted`, `objects_absent`, `objects_replaced`, `objects_spared`, `manifests_deleted`, `entries_condemned`, `entries_graduated`, `entries_redeleted`, `fence_outs`, `anomalies`, `pending_candidates`, `pending_condemned`, and `pending_retired`, describing the outcome of that round. The `pending_*` columns are the retire pipeline's remaining backlog sizes read from the `gc/state` this round's own commit just published (not this round's own delta, unlike the columns before them) — `0` on a non-authoritative row (`acquired_lease = 0` or `deferred = 1`), same as every other counter. + +A manual run always executes, regardless of [`SYSTEM CAS GC STOP`](#system-cas-gc-stop-start): `STOP` pauses only the background scheduler on that disk. + +### SYSTEM CAS GC REBUILD {#system-cas-gc-rebuild} + +Disaster-recovery command for the content-addressed (CAS) MergeTree garbage collector. It rebuilds a +CAS disk's `gc/state` baseline from scratch, by re-discovering the whole ref universe and re-folding +manifest edges into a fresh generation. It writes only the GC plane (`gc/state` and the `gc/gen/*` +artifacts) and never touches ref shards, manifests, or blobs, and it never deletes anything itself — +but the rebuilt baseline drives every subsequent GC round's retire decisions, so this is a +**destructive disaster-recovery tool**, not something to run routinely: a rebuild performed against +a state that was not actually corrupted discards live bookkeeping, and an incorrect rebuild can make +a later round delete objects that are still referenced. + +```sql +SYSTEM CAS GC REBUILD [FORCE] [ON CLUSTER cluster_name] disk_name +``` + +Unlike `SYSTEM CAS GC RUN`, `disk_name` is **required**: the destructive +baseline rebuild must never fan out across every content-addressed disk on the node from a bare +command; targeting a non-content-addressed disk raises an exception. + +By default the command refuses to run when the disk's existing `gc/state` and every artifact it +references decode successfully and are present — a rebuild would needlessly discard healthy live +bookkeeping. Add `FORCE` to rebuild deliberately even though the existing state looks healthy. The +command also refuses (regardless of `FORCE`) when another GC leader currently holds the disk's +lease. In both refusal cases it raises an exception instead of returning a row. + +On success it returns one row with columns `disk`, `performed`, `round`, `generation`, `namespaces`, +`shards`, `committed_refs`, `live_precommits`, `unowned_alive_manifests`, `edges`, +`clamped_shards`, `virgin_by_enumeration`, and `adopted_seal_generation`, describing the freshly +rebuilt baseline. `virgin_by_enumeration = 1` means the rebuild found no fold seal at all and +carried no durable hold forward, concluding from enumeration alone that the pool never sealed a +baseline — on a pool that has ever completed a GC round this means the object listing lied. +`adopted_seal_generation` names which generation's fold seal the rebuild carried holds from; `0` +when it carried none. + +### SYSTEM CAS GC STOP / SYSTEM CAS GC START {#system-cas-gc-stop-start} + +Pause or resume the background GC scheduler on one content-addressed disk, without affecting reads +or writes on that disk. This is granular operator control of GC alone — for example to pause +reclamation during an incident — not a lifecycle transition; the disk stays fully usable throughout. + +```sql +SYSTEM CAS GC STOP [ON CLUSTER cluster_name] disk_name +SYSTEM CAS GC START [ON CLUSTER cluster_name] disk_name +``` + +`disk_name` is **required** for both — unlike `SYSTEM CAS GC RUN`, there is no fan-out form, since +each command targets exactly one disk's scheduler. + +`GC STOP` stops in place: the scheduler object is retained, so a later `GC START` resumes the *same* +instance, preserving its `gc_id` and lease-observation history. It is idempotent, and works even on +a disk that is not currently live (stopping GC on a sick disk is a legitimate operation). It does +not stop a manual [`SYSTEM CAS GC RUN`](#system-cas-gc-run) on the same disk. + +`GC START` re-enters that same scheduler instance rather than creating a new one; leadership is +**not** automatically restored — the scheduler re-acquires the durable `gc/state` lease through the +next round's normal acquisition, the same as any other contender. It is idempotent (a no-op on an +already-running scheduler), and refuses with a typed error on a decommissioned or uncertain pool, +since restarting GC there would only spin failing rounds. + +Neither command returns a result set. + +### SYSTEM CAS FSCK {#system-cas-fsck} + +Independently verifies content-addressed pool reachability against a **running, mounted** disk — the +scan re-validates every finding against a fresh authoritative read, so unlike the offline +`clickhouse-disks cas-fsck` tool it needs no quiesce and no read-only mount. + +```sql +SYSTEM CAS FSCK [ON CLUSTER cluster_name] disk_name +``` + +`disk_name` is **required**. The command returns one row with columns `disk`, `reachable`, +`dangling`, `unreachable`, `pending_gc`, `awaiting_gc`, `unaccounted`, `stale_edge`, +`corrupted_runs`, `chain_broken`, `unchecked`, `lifeless_keys`, `namespace_janitor_pending`, +`namespace_janitor_pending_bytes`, `namespace_janitor_pending_lives`, `ref_records_walked`, +`physical_bytes`, `referenced_logical_bytes`, `distinct_blobs`, and `total_blob_refs`. `dangling` is +the one column that means data loss; `unreachable`, `pending_gc`, and `awaiting_gc` are objects +still moving through the normal condemn/graduate/delete pipeline, not a problem on their own. This +is a summary-only scan; per-object detail requires the offline `clickhouse-disks cas-fsck --detail`. + +### SYSTEM CAS FORGET {#system-cas-forget} + +Node-local operator assertion that a content-addressed disk is permanently gone — the "fire marshal" +verb for a stuck disk (a transient or identity-lost pool, or an operator-asserted decommission). +Unlike the other `SYSTEM CAS` commands, it deliberately works on a disk that is **not** live, since +that is its whole purpose. + +```sql +SYSTEM CAS FORGET [ON CLUSTER cluster_name] disk_name +``` + +`disk_name` is **required**. It is an assertion, not a proof of erasure: the disk stays registered +and answers further store-class access with a typed error, and a server restart re-registers the +name. Returns no result set. This is different from +[`SYSTEM CAS DROP POOL MEMBER`](#system-cas-drop-pool-member), which permanently retires one pool +*member's* identity across the whole shared pool — `FORGET` only affects this node's own local view +of one disk. + +### SYSTEM CAS DROP POOL MEMBER {#system-cas-drop-pool-member} + +Permanently decommissions a dead member (`server_root_id`) of a content-addressed disk +pool. It claims the member's mount slot as an administrative writer — fencing that `server_root_id` from ever +writing again — then drops every table namespace the member owned, drains eligible manifest debris, +staging objects, and mountpoint objects belonging to it, and retires the mount slot once all drains +are confirmed. This is a **destructive, irreversible** operation: only run it once the `server_root_id` is +confirmed permanently dead, since it fences the member out even if it later comes back online, and +it deletes namespace and drain state that cannot be recovered. + +It is a writer operation, not GC: it emits ordinary ref-edge deltas rather than inventing GC +transitions, and it does not synchronously reclaim shared blob content — removing the ref edges only +makes the now-unreferenced blobs eligible for an ordinary GC round to reclaim later. + +```sql +SYSTEM CAS DROP POOL MEMBER 'server_root_id' FROM DISK 'disk_name' [ON CLUSTER cluster_name] +``` + +Both `server_root_id` and `disk_name` are required string literals (a `server_root_id` is an opaque server-root path +that may contain `/`, not a plain identifier, so it cannot be written unquoted). + +The operation is resumable: a rerun skips namespaces already marked `Removed` and reports them +separately from namespaces newly removed by this invocation. Per-object drain failures are recorded +as warnings and leave the slot in a terminated-but-not-fully-drained state that a later invocation +can resume from, rather than raising an exception. + +The command returns one row with columns `server_root_id`, `namespaces_removed`, `namespaces_already_removed`, +`committed_refs_removed`, `precommits_removed`, `manifest_debris_removed`, `staging_objects_removed`, +`mountpoint_objects_removed`, `slot_removed`, and `warnings`. A non-empty `warnings` means some +drain was not confirmed and the mount slot was left in place as a resume anchor. + ## Managing ReplicatedMergeTree Tables {#managing-replicatedmergetree-tables} ClickHouse can manage background replication related processes in [ReplicatedMergeTree](/engines/table-engines/mergetree-family/replication) tables.