Only analyze backed pages for compressibility

tcmalloc/internal/BUILD

@@ -515,7 +515,6 @@ cc_library(
         ":config",
         ":logging",
         ":page_size",
-        ":pageflags",
         ":range_tracker",
         ":util",
         "@com_google_absl//absl/status",
@@ -995,10 +994,15 @@ cc_library(
         "//tcmalloc:__subpackages__",
     ],
     deps = [
+        ":page_size",
+        ":residency",
         ":util",
+        "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/container:fixed_array",
         "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:status_macros",
         "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
     ],
 )
@@ -1010,8 +1014,11 @@ cc_test(
     tags = ["nompu64"],  # tcmalloc:google3-only
     deps = [
         ":compressibility",
+        ":page_size",
+        ":range_tracker",
+        ":residency",
         "@com_google_absl//absl/status:status_matchers",
-        "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
         "@com_google_googletest//:gtest_main",
     ],
@@ -1024,6 +1031,8 @@ cc_test(
     tags = ["nompu64"],  # tcmalloc:google3-only
     deps = [
         ":compressibility",
+        ":page_size",
+        ":residency",
         "@com_google_absl//absl/types:span",
         "@com_google_fuzztest//fuzztest",
         "@com_google_fuzztest//fuzztest:fuzztest_gtest_main",
@@ -1043,15 +1052,12 @@ cc_library(
         ":compressibility",
         ":logging",
         ":pageflags",
-        ":parameter_accessors",
         ":residency",
-        ":util",
         "//tcmalloc:malloc_extension",
         "//tcmalloc/internal:profile_cc_proto",
         "@com_google_absl//absl/base",
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/container:btree",
-        "@com_google_absl//absl/container:fixed_array",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/hash",
         "@com_google_absl//absl/status",
@@ -1494,7 +1500,9 @@ cc_library(
     deps = [
         ":config",
         ":logging",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/time",
+        "@com_google_absl//absl/types:span",
     ],
 )
 

tcmalloc/internal/compressibility.cc

@@ -17,50 +17,127 @@
 #include <algorithm>
 #include <cstddef>
 #include <cstdint>
+#include <vector>
 
+#include "absl/algorithm/container.h"
 #include "absl/status/status.h"
+#include "absl/status/status_macros.h"
 #include "absl/status/statusor.h"
+#include "absl/strings/string_view.h"
 #include "absl/types/span.h"
+#include "tcmalloc/internal/page_size.h"
+#include "tcmalloc/internal/residency.h"
 #include "tcmalloc/internal/util.h"
 
 namespace tcmalloc {
 namespace tcmalloc_internal {
 
+namespace {
+
+constexpr uintptr_t AlignDown(uintptr_t addr, size_t alignment) {
+  return addr & ~(alignment - 1);
+}
+
+constexpr uintptr_t AlignUp(uintptr_t addr, size_t alignment) {
+  return (addr + alignment - 1) & ~(alignment - 1);
+}
+
+// Copies up to `dst.size()` resident bytes from `data` into `dst`, skipping any
+// unbacked or swapped pages and concatenating the resident pages.
+// Returns the number of bytes copied.
+absl::StatusOr<size_t> CopyResidentPages(absl::Span<const char> data,
+                                         const Residency::Info& residency_info,
+                                         absl::Span<char> dst) {
+  const size_t hardware_page_size = GetPageSize();
+
+  const uintptr_t uaddr = reinterpret_cast<uintptr_t>(data.data());
+  const uintptr_t start_page_addr = AlignDown(uaddr, hardware_page_size);
+  const uintptr_t end_page_addr =
+      AlignUp(uaddr + data.size(), hardware_page_size);
+  const size_t num_pages =
+      (end_page_addr - start_page_addr) / hardware_page_size;
+  const size_t pages_to_scan =
+      std::min(num_pages, residency_info.page_is_resident.size());
+
+  std::vector<absl::string_view> remote_chunks;
+  size_t total_resident_bytes = 0;
+  size_t page_index = 0;
+
+  do {
+    // Find the next range of resident pages.
+    size_t start_page_index =
+        residency_info.page_is_resident.FindSet(page_index);
+    if (start_page_index >= pages_to_scan) {
+      break;
+    }
+    size_t end_page_index =
+        residency_info.page_is_resident.FindClear(start_page_index);
+    end_page_index = std::min(end_page_index, pages_to_scan);
+
+    const uintptr_t chunk_start = std::max(
+        uaddr, start_page_addr + start_page_index * hardware_page_size);
+    const uintptr_t chunk_end =
+        std::min(uaddr + data.size(),
+                 start_page_addr + end_page_index * hardware_page_size);
+    const size_t chunk_bytes = chunk_end - chunk_start;
+    const size_t to_copy =
+        std::min(chunk_bytes, dst.size() - total_resident_bytes);
+    remote_chunks.push_back(
+        absl::string_view(reinterpret_cast<const char*>(chunk_start), to_copy));
+    total_resident_bytes += to_copy;
+
+    page_index = end_page_index;
+  } while (page_index < pages_to_scan && total_resident_bytes < dst.size());
+
+  if (total_resident_bytes > 0) {
+    if (!SafeCopyMemory(remote_chunks, /*dst=*/dst.data())) {
+      return absl::InternalError("SafeCopyMemory failed");
+    }
+  }
+
+  return total_resident_bytes;
+}
+
+// Extrapolates total zero bytes in the allocation by combining zeroes measured
+// within the resident sample with known unbacked pages.
+size_t EstimateZeroBytes(size_t alloc_size,
+                         const Residency::Info& residency_info,
+                         absl::Span<const char> resident_sample) {
+  const size_t backed_bytes = std::min(
+      alloc_size, residency_info.bytes_resident + residency_info.bytes_swapped);
+  const size_t unbacked_bytes = alloc_size - backed_bytes;
+  if (resident_sample.empty()) {
+    return unbacked_bytes;
+  }
+
+  const size_t sample_zeroes = absl::c_count(resident_sample, '\0');
+  const double zero_ratio =
+      static_cast<double>(sample_zeroes) / resident_sample.size();
+  const size_t estimated_backed_zeroes =
+      static_cast<size_t>(zero_ratio * backed_bytes);
+
+  return unbacked_bytes + estimated_backed_zeroes;
+}
+
+}  // namespace
+
 CompressionAnalyzer::CompressionAnalyzer(size_t max_local_copy_size)
     : local_copy_(max_local_copy_size)
 {}
 
 absl::StatusOr<CompressionAnalyzer::Results> CompressionAnalyzer::Analyze(
-    absl::Span<const char> data) {
-  Results results;
-  bool still_in_trailing_zeroes = true;
-
-  // Walk backwards from the end of the data, copying chunks.
-  int64_t end_offset = data.size();
-  while (end_offset > 0) {
-    int64_t chunk_size =
-        std::min(end_offset, static_cast<int64_t>(local_copy_.size()));
-    if (!SafeCopyMemory(/*src=*/data.data() + end_offset - chunk_size,
-                        /*dst=*/local_copy_.data(), /*size=*/chunk_size)) {
-      return absl::InternalError("SafeCopyMemory failed");
-    }
-    auto chunk = absl::MakeConstSpan(local_copy_.data(), chunk_size);
-
-    // Count zero bytes and trailing zero bytes in chunk.
-    for (size_t i = chunk.size(); i > 0; --i) {
-      char c = chunk[i - 1];
-      if (c == 0) {
-        results.zero_bytes++;
-        if (still_in_trailing_zeroes) {
-          results.trailing_zero_bytes++;
-        }
-      } else {
-        still_in_trailing_zeroes = false;
-      }
-    }
+    absl::Span<const char> data, const Residency::Info& residency_info) {
+  // Sample resident pages up to local buffer capacity (2MB).
+  ASSIGN_OR_RETURN(
+      const size_t copied_bytes,
+      CopyResidentPages(data, residency_info, absl::MakeSpan(local_copy_)));
 
-    end_offset -= chunk_size;
-  }
+  const absl::Span<const char> resident_sample =
+      absl::MakeConstSpan(local_copy_).first(copied_bytes);
+
+  Results results;
+  results.zero_bytes =
+      EstimateZeroBytes(data.size(), residency_info, resident_sample);
 
   return results;
 }

tcmalloc/internal/compressibility.h

@@ -21,6 +21,7 @@
 #include "absl/container/fixed_array.h"
 #include "absl/status/statusor.h"
 #include "absl/types/span.h"
+#include "tcmalloc/internal/residency.h"
 
 namespace tcmalloc {
 namespace tcmalloc_internal {
@@ -34,10 +35,10 @@ class CompressionAnalyzer {
 
   struct Results {
     size_t zero_bytes = 0;
-    size_t trailing_zero_bytes = 0;
   };
 
-  absl::StatusOr<Results> Analyze(absl::Span<const char> data);
+  absl::StatusOr<Results> Analyze(absl::Span<const char> data,
+                                  const Residency::Info& residency_info);
 
  private:
   absl::FixedArray<char> local_copy_;

tcmalloc/internal/compressibility_fuzz.cc

@@ -12,30 +12,67 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <algorithm>
 #include <cstddef>
+#include <cstdint>
 #include <string>
+#include <utility>
 
 #include "gtest/gtest.h"
 #include "fuzztest/fuzztest.h"
 #include "absl/types/span.h"
 #include "tcmalloc/internal/compressibility.h"
+#include "tcmalloc/internal/page_size.h"
+#include "tcmalloc/internal/residency.h"
 
 namespace tcmalloc {
 namespace tcmalloc_internal {
 namespace {
 
-void FuzzAnalyze(std::string data, size_t max_local_copy_size) {
+void FuzzAnalyze(std::string data, size_t max_local_copy_size,
+                 std::vector<bool> page_resident_bits, size_t bytes_swapped) {
   CompressionAnalyzer analyzer(max_local_copy_size);
-  auto res = analyzer.Analyze(absl::MakeConstSpan(data));
+  const size_t page_size = GetPageSize();
+  const uintptr_t uaddr = reinterpret_cast<uintptr_t>(data.data());
+  const size_t start_page = uaddr / page_size;
+  const size_t end_page = (uaddr + data.size() + page_size - 1) / page_size;
+  const size_t num_pages = end_page - start_page;
+
+  if (page_resident_bits.size() < num_pages) {
+    page_resident_bits.resize(num_pages, false);
+  }
+
+  size_t bytes_resident = 0;
+  for (size_t i = 0; i < num_pages; ++i) {
+    if (!page_resident_bits[i]) continue;
+    const uintptr_t page_start_addr = (start_page + i) * page_size;
+    const uintptr_t page_end_addr = page_start_addr + page_size;
+    const uintptr_t chunk_start = std::max(uaddr, page_start_addr);
+    const uintptr_t chunk_end = std::min(uaddr + data.size(), page_end_addr);
+    if (chunk_end > chunk_start) {
+      bytes_resident += (chunk_end - chunk_start);
+    }
+  }
+
+  Residency::Info info;
+  info.bytes_resident = bytes_resident;
+  info.bytes_swapped = std::min(data.size() - bytes_resident, bytes_swapped);
+  for (size_t i = 0; i < std::min(num_pages, kMaxResidencyBits); ++i) {
+    if (page_resident_bits[i]) {
+      info.page_is_resident.SetBit(i);
+    }
+  }
+
+  auto res = analyzer.Analyze(absl::MakeConstSpan(data), info);
   if (res.ok()) {
     EXPECT_LE(res->zero_bytes, data.size());
-    EXPECT_LE(res->trailing_zero_bytes, data.size());
-    EXPECT_LE(res->trailing_zero_bytes, res->zero_bytes);
   }
 }
 
 FUZZ_TEST(CompressibilityFuzzTest, FuzzAnalyze)
-    .WithDomains(fuzztest::String(), fuzztest::InRange<size_t>(1, 1024));
+    .WithDomains(fuzztest::String(), fuzztest::InRange<size_t>(1, 1024),
+                 fuzztest::Arbitrary<std::vector<bool>>(),
+                 fuzztest::Arbitrary<size_t>());
 
 }  // namespace
 }  // namespace tcmalloc_internal