Slow restart file reading

entity.schema.json

@@ -1943,7 +1943,7 @@
       }
     },
     "adios2": {
-      "description": "ADIOS2 BP5 tuning, applied to both [output] and [checkpoint] writers",
+      "description": "ADIOS2 BP5 tuning, applied to both [output] and [checkpoint] writers; the read_* keys tune the checkpoint reader on restart",
       "type": "object",
       "additionalProperties": false,
       "properties": {
@@ -1982,6 +1982,39 @@
               "Scales with per-rank output volume; matches ADIOS2's default"
             ]
           }
+        },
+        "read_threads": {
+          "description": "Size of the BP5 reader thread pool used when resuming (BP5 Threads)",
+          "type": "integer",
+          "minimum": 0,
+          "default": 0,
+          "x-entity": {
+            "type": "uint",
+            "notes": [
+              "0 lets ADIOS2 size the pool itself. The best value depends on how many ranks share a node, so tune it per machine if restarts are slow"
+            ]
+          }
+        },
+        "read_open_timeout_secs": {
+          "description": "How long to wait for the checkpoint file to appear, in seconds (BP5 OpenTimeoutSecs)",
+          "type": "integer",
+          "minimum": 0,
+          "default": 600,
+          "x-entity": {
+            "type": "uint",
+            "notes": [
+              "Guards against a checkpoint directory that is slow to become visible on a shared filesystem"
+            ]
+          }
+        },
+        "read_poll_secs": {
+          "description": "Polling interval while waiting for the checkpoint file, in seconds (BP5 BeginStepPollingFrequencySecs)",
+          "type": "integer",
+          "minimum": 0,
+          "default": 1,
+          "x-entity": {
+            "type": "uint"
+          }
         }
       }
     },

input.default.toml

@@ -824,24 +824,44 @@
   #     @type: float
   #     @from: automatically determined during restart
 
-# ADIOS2 BP5 tuning, applied to both [output] and [checkpoint] writers
+# ADIOS2 BP5 tuning, applied to both [output] and [checkpoint] writers; the
+# read_* keys tune the checkpoint reader on restart
 [adios2]
   # Number of ADIOS2 aggregators per node
   #   @type: uint
   #   @default: 0
   #   @note: Set to either MPI ranks/node or NICs/node for best performance
   #          If set to 0, will use ADIOS2 default (one aggregator per node)
-  aggregators_per_node = 0
+  aggregators_per_node   = 0
   # Maximum shared-memory segment size per node, in bytes (BP5 MaxShmSize)
   #   @type: uint
   #   @default: 4294967296
   #   @note: Lower this on memory-constrained nodes; matches ADIOS2's default
-  max_shm_size         = 4294967296
+  max_shm_size           = 4294967296
   # Internal serialization buffer chunk size, in bytes (BP5 BufferChunkSize)
   #   @type: uint
   #   @default: 16777216
   #   @note: Scales with per-rank output volume; matches ADIOS2's default
-  buffer_chunk_size    = 16777216
+  buffer_chunk_size      = 16777216
+  # Size of the BP5 reader thread pool used when resuming (BP5 Threads)
+  #   @type: uint
+  #   @default: 0
+  #   @note: 0 lets ADIOS2 size the pool itself. The best value depends on how
+  #          many ranks share a node, so tune it per machine if restarts are
+  #          slow
+  read_threads           = 0
+  # How long to wait for the checkpoint file to appear, in seconds (BP5
+  # OpenTimeoutSecs)
+  #   @type: uint
+  #   @default: 600
+  #   @note: Guards against a checkpoint directory that is slow to become
+  #          visible on a shared filesystem
+  read_open_timeout_secs = 600
+  # Polling interval while waiting for the checkpoint file, in seconds (BP5
+  # BeginStepPollingFrequencySecs)
+  #   @type: uint
+  #   @default: 1
+  read_poll_secs         = 1
 
 # In-situ renderer. Renders scalar fields on the GPU and writes PNG images
 # directly to `<simname>/renders/` each cadence -- no field data is written to

src/framework/domain/checkpoint/resume.cpp

@@ -1,3 +1,4 @@
+#include "defaults.h"
 #include "enums.h"
 #include "global.h"
 
@@ -10,6 +11,7 @@
 #include "framework/parameters/parameters.h"
 #include "framework/specialization_registry.h"
 #include "output/utils/readers.h"
+#include "output/utils/tuning.h"
 
 #if defined(MPI_ENABLED)
   #include <mpi.h>
@@ -122,11 +124,17 @@ namespace ntt {
 
     adios2::IO io = ptr_adios->DeclareIO("Entity::CheckpointRead");
     io.SetEngine("BPFile");
-#if !defined(MPI_ENABLED)
+    out::ApplyBp5ReadTuning(
+      io,
+      "BPFile",
+      { params.template get<int>("adios2.read_threads",
+                                 defaults::adios2::read_threads),
+        params.template get<int>("adios2.read_open_timeout_secs",
+                                 defaults::adios2::read_open_timeout_secs),
+        params.template get<int>("adios2.read_poll_secs",
+                                 defaults::adios2::read_poll_secs) });
+
     adios2::Engine reader = io.Open(fname, adios2::Mode::Read);
-#else
-    adios2::Engine reader = io.Open(fname, adios2::Mode::Read, MPI_COMM_SELF);
-#endif
 
     reader.BeginStep();
 
@@ -140,33 +148,36 @@ namespace ntt {
                                  std::numeric_limits<real_t>::lowest());
     }
 
+    // Each subdomain_* variable is read whole, once. Per-domain reads would
+    // cost one filesystem round-trip per domain per rank, since consecutive
+    // entries live in different writers' blocks.
     bool needs_reconstruction = false;
-    for (unsigned int dom_idx { 0 }; dom_idx < g_ndomains; ++dom_idx) {
-      for (auto d { 0u }; d < M::Dim; ++d) {
-        real_t x_min, x_max;
-        out::ReadVariable<real_t>(io,
-                                  reader,
-                                  fmt::format("subdomain_x%d_min", d + 1),
-                                  x_min,
-                                  dom_idx);
-        out::ReadVariable<real_t>(io,
-                                  reader,
-                                  fmt::format("subdomain_x%d_max", d + 1),
-                                  x_max,
-                                  dom_idx);
-        saved_extents[dom_idx].emplace_back(x_min, x_max);
-        global_extent[d].first  = std::min(global_extent[d].first, x_min);
-        global_extent[d].second = std::max(global_extent[d].second, x_max);
-
-        ncells_t nx;
-        out::ReadVariable<ncells_t>(io,
-                                    reader,
-                                    fmt::format("subdomain_nx%d", d + 1),
-                                    nx,
-                                    dom_idx);
-        saved_ncells[dom_idx][d] = nx;
-
-        if (nx != subdomain_ptr(dom_idx)->mesh.n_active()[d]) {
+    for (auto d { 0u }; d < M::Dim; ++d) {
+      std::vector<real_t>   x_min, x_max;
+      std::vector<ncells_t> nx;
+      out::ReadVariableAll<real_t>(io,
+                                   reader,
+                                   fmt::format("subdomain_x%d_min", d + 1),
+                                   x_min,
+                                   g_ndomains);
+      out::ReadVariableAll<real_t>(io,
+                                   reader,
+                                   fmt::format("subdomain_x%d_max", d + 1),
+                                   x_max,
+                                   g_ndomains);
+      out::ReadVariableAll<ncells_t>(io,
+                                     reader,
+                                     fmt::format("subdomain_nx%d", d + 1),
+                                     nx,
+                                     g_ndomains);
+
+      for (unsigned int dom_idx { 0 }; dom_idx < g_ndomains; ++dom_idx) {
+        saved_extents[dom_idx].emplace_back(x_min[dom_idx], x_max[dom_idx]);
+        global_extent[d].first  = std::min(global_extent[d].first, x_min[dom_idx]);
+        global_extent[d].second = std::max(global_extent[d].second, x_max[dom_idx]);
+
+        saved_ncells[dom_idx][d] = nx[dom_idx];
+        if (nx[dom_idx] != subdomain_ptr(dom_idx)->mesh.n_active()[d]) {
           needs_reconstruction = true;
         }
       }

src/framework/parameters/parameters.cpp

@@ -216,6 +216,21 @@ namespace ntt {
                               "adios2",
                               "buffer_chunk_size",
                               defaults::adios2::buffer_chunk_size));
+    set("adios2.read_threads",
+        toml::find_or<int>(toml_data,
+                           "adios2",
+                           "read_threads",
+                           defaults::adios2::read_threads));
+    set("adios2.read_open_timeout_secs",
+        toml::find_or<int>(toml_data,
+                           "adios2",
+                           "read_open_timeout_secs",
+                           defaults::adios2::read_open_timeout_secs));
+    set("adios2.read_poll_secs",
+        toml::find_or<int>(toml_data,
+                           "adios2",
+                           "read_poll_secs",
+                           defaults::adios2::read_poll_secs));
 
     /* [diagnostics] -------------------------------------------------------- */
     set("diagnostics.interval",

src/global/defaults.h

@@ -102,6 +102,12 @@ namespace ntt::defaults {
     const int    aggregators_per_node = 0;
     const size_t max_shm_size         = 4294967296ull; // 4 GiB
     const size_t buffer_chunk_size    = 16777216ull;   // 16 MiB
+    // Checkpoint-read knobs. read_threads == 0 leaves ADIOS2 auto-sizing its
+    // reader thread pool; the best value depends on the ranks-per-node layout,
+    // so it is left to the input file rather than fixed here.
+    const int    read_threads           = 0;
+    const int    read_open_timeout_secs = 600;
+    const int    read_poll_secs         = 1;
   } // namespace adios2
 
   namespace gca {

src/output/utils/readers.cpp

@@ -11,6 +11,7 @@
 #include <cstddef>
 #include <string>
 #include <utility>
+#include <vector>
 
 namespace out {
 
@@ -31,6 +32,22 @@ namespace out {
     }
   }
 
+  template <typename T>
+  void ReadVariableAll(adios2::IO&        io,
+                       adios2::Engine&    reader,
+                       const std::string& quantity,
+                       std::vector<T>&    data,
+                       std::size_t        count) {
+    auto var = io.InquireVariable<T>(quantity);
+    if (var) {
+      data.resize(count);
+      var.SetSelection(adios2::Box<adios2::Dims>({ 0 }, { count }));
+      reader.Get(var, data.data(), adios2::Mode::Sync);
+    } else {
+      raise::Error(fmt::format("Variable: %s not found", quantity.c_str()), HERE);
+    }
+  }
+
   template <typename T>
   void Read1DArray(adios2::IO&        io,
                    adios2::Engine&    reader,
@@ -110,6 +127,11 @@ namespace out {
                              const std::string&,                               \
                              T&,                                               \
                              std::size_t);                                     \
+  template void ReadVariableAll<T>(adios2::IO&,                                \
+                                   adios2::Engine&,                            \
+                                   const std::string&,                         \
+                                   std::vector<T>&,                            \
+                                   std::size_t);                               \
   template void Read1DArray<T>(adios2::IO&,                                    \
                                adios2::Engine&,                                \
                                const std::string&,                             \

src/output/utils/readers.h

@@ -4,6 +4,7 @@
  * Defines generic reader functions.
  * @implements
  *   - out::ReadVariable<> -> void
+ *   - out::ReadVariableAll<> -> void
  *   - out::Read1DArray<> -> void
  *   - out::Read2DArray<> -> void
  *   - out::ReadNDField<> -> void
@@ -21,12 +22,26 @@
 #include <adios2.h>
 
 #include <string>
+#include <vector>
 
 namespace out {
 
   template <typename T>
   void ReadVariable(adios2::IO&, adios2::Engine&, const std::string&, T&, std::size_t);
 
+  /**
+   * @brief Read the first `count` elements of a per-domain variable at once.
+   * @note One selection instead of `count` single-element reads: each read
+   *       lands in a different writer's block, so per-element reads cost a
+   *       separate filesystem round-trip apiece.
+   */
+  template <typename T>
+  void ReadVariableAll(adios2::IO&,
+                       adios2::Engine&,
+                       const std::string&,
+                       std::vector<T>&,
+                       std::size_t);
+
   template <typename T>
   void Read1DArray(adios2::IO&,
                    adios2::Engine&,

src/output/utils/tuning.cpp

@@ -51,4 +51,21 @@ namespace out {
     io.SetParameter("OpenTimeoutSecs", "600");
   }
 
+  void ApplyBp5ReadTuning(adios2::IO&          io,
+                          const std::string&   engine,
+                          const Bp5ReadTuning& bp5) {
+    const auto eng = fmt::toLower(engine);
+    if (eng != "bpfile" && eng != "bp5") {
+      return;
+    }
+    // Tolerate a checkpoint directory that is slow to become visible on a
+    // shared filesystem instead of failing the restart outright.
+    io.SetParameter("OpenTimeoutSecs", std::to_string(bp5.open_timeout_secs));
+    io.SetParameter("BeginStepPollingFrequencySecs",
+                    std::to_string(bp5.poll_secs));
+    if (bp5.threads > 0) {
+      io.SetParameter("Threads", std::to_string(bp5.threads));
+    }
+  }
+
 } // namespace out

src/output/utils/tuning.h

@@ -4,8 +4,10 @@
  * for large-scale parallel filesystems.
  * @implements
  *   - out::Bp5Tuning
+ *   - out::Bp5ReadTuning
  *   - out::TotalAggregators -> int
  *   - out::ApplyBp5Tuning -> void
+ *   - out::ApplyBp5ReadTuning -> void
  * @cpp:
  *   - tuning.cpp
  * @namespaces:
@@ -37,10 +39,27 @@ namespace out {
   // aggregators_per_node <= 0, which leaves ADIOS2 on its built-in default.
   auto TotalAggregators(int aggregators_per_node) -> int;
 
+  // Checkpoint-read knobs from the [adios2] toml section. The aggregation and
+  // buffering parameters above are write-side only and are deliberately not
+  // reused here: ADIOS2 accepts unknown parameters silently, so setting them
+  // on a reader would look like tuning while doing nothing.
+  struct Bp5ReadTuning {
+    const int threads { ntt::defaults::adios2::read_threads };
+    const int open_timeout_secs { ntt::defaults::adios2::read_open_timeout_secs };
+    const int poll_secs { ntt::defaults::adios2::read_poll_secs };
+  };
+
   // Apply the [adios2] BP5 tuning to a freshly declared IO whose engine is
   // BPFile/BP5. A no-op for other engines.
   void ApplyBp5Tuning(adios2::IO&, const std::string& engine, const Bp5Tuning&);
 
+  // Apply the checkpoint-read knobs to a freshly declared reader IO whose
+  // engine is BPFile/BP5. A no-op for other engines. `threads <= 0` leaves
+  // ADIOS2 to size its own reader thread pool.
+  void ApplyBp5ReadTuning(adios2::IO&,
+                          const std::string& engine,
+                          const Bp5ReadTuning&);
+
 } // namespace out
 
 #endif // OUTPUT_UTILS_TUNING_H