apps/argos/Checkpointer.hpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // <Checkpointer.hpp> -*- C++ -*- | ||
| 2 | |||
| 3 | #pragma once | ||
| 4 | |||
| 5 | #include "simdb/apps/argos/Checkpoint.hpp" | ||
| 6 | #include "simdb/sqlite/DatabaseManager.hpp" | ||
| 7 | #include "simdb/utils/ValidValue.hpp" | ||
| 8 | |||
| 9 | #include <vector> | ||
| 10 | |||
| 11 | namespace simdb::argos { | ||
| 12 | |||
| 13 | namespace detail { | ||
| 14 | /// Counts occupied front-packed bins in a contig snapshot. | ||
| 15 | 24030 | inline uint16_t getContainerSize(const std::vector<std::vector<char>>& contig_bin_bytes) | |
| 16 | { | ||
| 17 | 24030 | uint64_t size = 0; | |
| 18 | 136368 | for (const auto& bin_bytes : contig_bin_bytes) | |
| 19 | { | ||
| 20 | 112338 | if (!bin_bytes.empty()) | |
| 21 | { | ||
| 22 | 112338 | ++size; | |
| 23 | } else | ||
| 24 | { | ||
| 25 | ✗ | break; | |
| 26 | } | ||
| 27 | } | ||
| 28 | 24030 | simdb_assert(size <= UINT16_MAX); | |
| 29 | 24030 | return size; | |
| 30 | } | ||
| 31 | |||
| 32 | /// Counts occupied bins in a sparse snapshot. | ||
| 33 | 24876 | inline uint16_t getContainerSize(const std::map<uint16_t, std::vector<char>>& sparse_bin_bytes) | |
| 34 | { | ||
| 35 | 24876 | uint64_t size = 0; | |
| 36 | 137670 | for (const auto& [_, bin_bytes] : sparse_bin_bytes) | |
| 37 | { | ||
| 38 | 112794 | if (!bin_bytes.empty()) | |
| 39 | { | ||
| 40 | 112794 | ++size; | |
| 41 | } | ||
| 42 | } | ||
| 43 | 24876 | simdb_assert(size <= UINT16_MAX); | |
| 44 | 24876 | return size; | |
| 45 | } | ||
| 46 | } // namespace detail | ||
| 47 | |||
| 48 | //! \class CollectableCheckpointer | ||
| 49 | class CollectableCheckpointer | ||
| 50 | { | ||
| 51 | public: | ||
| 52 | /// Destroys the checkpointer and its checkpoint chain. | ||
| 53 | 91 | virtual ~CollectableCheckpointer() = default; | |
| 54 | |||
| 55 | /// Appends a scalar snapshot checkpoint; \throws DBException on unsupported checkpointer types. | ||
| 56 | ✗ | virtual void createCheckpoint(uint64_t window_id, const std::vector<char>& scalar_bytes) | |
| 57 | { | ||
| 58 | (void)window_id; | ||
| 59 | (void)scalar_bytes; | ||
| 60 | ✗ | throw DBException("Not implemented"); | |
| 61 | } | ||
| 62 | |||
| 63 | /// Appends a contig container snapshot checkpoint; \throws DBException on unsupported checkpointer types. | ||
| 64 | ✗ | virtual void createCheckpoint(uint64_t window_id, const std::vector<std::vector<char>>& contig_bin_bytes) | |
| 65 | { | ||
| 66 | (void)window_id; | ||
| 67 | (void)contig_bin_bytes; | ||
| 68 | ✗ | throw DBException("Not implemented"); | |
| 69 | } | ||
| 70 | |||
| 71 | /// Appends a sparse container snapshot checkpoint; \throws DBException on unsupported checkpointer types. | ||
| 72 | ✗ | virtual void createCheckpoint(uint64_t window_id, const std::map<uint16_t, std::vector<char>>& sparse_bin_bytes) | |
| 73 | { | ||
| 74 | (void)window_id; | ||
| 75 | (void)sparse_bin_bytes; | ||
| 76 | ✗ | throw DBException("Not implemented"); | |
| 77 | } | ||
| 78 | |||
| 79 | /// Records a closed lifecycle event at \p window_id. | ||
| 80 | virtual void closeRecord(uint64_t window_id) = 0; | ||
| 81 | |||
| 82 | /// Encodes wire records for \p cid at \p sim_time, using the checkpoint for \p window_id when present. | ||
| 83 | virtual std::vector<std::unique_ptr<CollectedData>> encodeForPipeline(uint64_t window_id, uint64_t sim_time, | ||
| 84 | uint16_t cid) = 0; | ||
| 85 | |||
| 86 | /// Returns true if this collectable staged data in \p window_id. | ||
| 87 | virtual bool participatedInWindow(uint64_t window_id) const = 0; | ||
| 88 | |||
| 89 | /// Writes collectable-specific metadata after simulation teardown; default is a no-op. | ||
| 90 | ✗ | virtual void writeMetaOnPostTeardown(uint16_t cid, DatabaseManager*) { (void)cid; } | |
| 91 | |||
| 92 | protected: | ||
| 93 | /// Stores the heartbeat interval used to force periodic FULL snapshots. | ||
| 94 | 91 | explicit CollectableCheckpointer(size_t heartbeat) : | |
| 95 | 91 | heartbeat_(heartbeat) | |
| 96 | { | ||
| 97 | 91 | } | |
| 98 | |||
| 99 | /// Finds the latest checkpoint in \p tail at or before \p window_id. | ||
| 100 | template <typename CheckpointT> | ||
| 101 | 90005 | static CheckpointT* getAnchorForWindow_(uint64_t window_id, const std::shared_ptr<CheckpointT>& tail) | |
| 102 | { | ||
| 103 | 90005 | CheckpointT* anchor = tail.get(); | |
| 104 | 90005 | CheckpointT* best = nullptr; | |
| 105 | 221732 | while (anchor) | |
| 106 | { | ||
| 107 | 131727 | if (anchor->getWindowID() > window_id) | |
| 108 | { | ||
| 109 | ✗ | break; | |
| 110 | } | ||
| 111 | 131727 | best = anchor; | |
| 112 | 131727 | anchor = anchor->next(); | |
| 113 | } | ||
| 114 | 90005 | return best; | |
| 115 | } | ||
| 116 | |||
| 117 | /// Returns true if \p tail contains a checkpoint whose window equals \p window_id. | ||
| 118 | template <typename CheckpointT> | ||
| 119 | ✗ | static bool participatedInWindow_(uint64_t window_id, const std::shared_ptr<CheckpointT>& tail) | |
| 120 | { | ||
| 121 | ✗ | if (!tail) | |
| 122 | { | ||
| 123 | ✗ | return false; | |
| 124 | } | ||
| 125 | ✗ | auto* anchor = getAnchorForWindow_(window_id, tail); | |
| 126 | ✗ | return anchor && anchor->getWindowID() == window_id; | |
| 127 | } | ||
| 128 | |||
| 129 | /// Locates the shared_ptr in \p head that owns \p raw. | ||
| 130 | template <typename CheckpointT> | ||
| 131 | 41873 | static std::shared_ptr<CheckpointT> getSharedCheckpoint_(CheckpointT* raw, const std::shared_ptr<CheckpointT>& head) | |
| 132 | { | ||
| 133 | 41873 | std::shared_ptr<CheckpointT> sp = head; | |
| 134 | 41873 | while (sp) | |
| 135 | { | ||
| 136 | 41873 | if (sp.get() == raw) | |
| 137 | { | ||
| 138 | 41873 | return sp; | |
| 139 | } | ||
| 140 | ✗ | auto prev = sp->getPrevShared(); | |
| 141 | ✗ | if (!prev) | |
| 142 | { | ||
| 143 | ✗ | return nullptr; | |
| 144 | } | ||
| 145 | ✗ | sp = std::static_pointer_cast<CheckpointT>(prev); | |
| 146 | } | ||
| 147 | ✗ | return nullptr; | |
| 148 | 41873 | } | |
| 149 | |||
| 150 | /// Reads the action byte from an encoded wire record (after the CID prefix). | ||
| 151 | 41873 | static Action readEncodedAction_(const CollectedData& encoded) | |
| 152 | { | ||
| 153 | 41873 | return static_cast<Action>(encoded.getData().at(sizeof(uint16_t))); | |
| 154 | } | ||
| 155 | |||
| 156 | /// Wraps a single encoded record in a one-element vector. | ||
| 157 | 51582 | template <typename T> static std::vector<T> makeSingleElemVector_(T&& only) | |
| 158 | { | ||
| 159 | 51582 | std::vector<T> out; | |
| 160 | 51582 | out.emplace_back(std::move(only)); | |
| 161 | 51582 | return out; | |
| 162 | ✗ | } | |
| 163 | |||
| 164 | /// Returns true when the next wire must be a FULL snapshot (delta limit or heartbeat window). | ||
| 165 | 41873 | bool forceSnapshot_(uint64_t sim_time) const | |
| 166 | { | ||
| 167 | 41873 | return distance_to_snapshot_ + 1 >= heartbeat_ || shouldHeartbeatRefresh_(sim_time); | |
| 168 | } | ||
| 169 | |||
| 170 | /// Returns true when the last FULL wire is outside the current heartbeat window. | ||
| 171 | 48132 | bool shouldHeartbeatRefresh_(uint64_t sim_time) const | |
| 172 | { | ||
| 173 | 48132 | if (!last_snapshot_sim_time_.isValid()) | |
| 174 | { | ||
| 175 | 60 | return false; | |
| 176 | } | ||
| 177 | 48072 | if (sim_time <= last_snapshot_sim_time_.getValue()) | |
| 178 | { | ||
| 179 | ✗ | return false; | |
| 180 | } | ||
| 181 | 48072 | const uint64_t window_lo = sim_time >= heartbeat_ ? sim_time - heartbeat_ + 1 : 0; | |
| 182 | 48072 | return last_snapshot_sim_time_.getValue() < window_lo; | |
| 183 | } | ||
| 184 | |||
| 185 | /// Returns true when a CID that skipped this window still needs a heartbeat refresh on the wire. | ||
| 186 | 48132 | bool shouldRefreshAbsentCid_(uint64_t sim_time, bool tip_closed) const | |
| 187 | { | ||
| 188 | 48132 | if (shouldHeartbeatRefresh_(sim_time)) | |
| 189 | { | ||
| 190 | 48072 | return true; | |
| 191 | } | ||
| 192 | 60 | if (!tip_closed || !last_record_closed_sim_time_.isValid() || | |
| 193 | ✗ | sim_time <= last_record_closed_sim_time_.getValue()) | |
| 194 | { | ||
| 195 | 60 | return false; | |
| 196 | } | ||
| 197 | ✗ | const uint64_t window_lo = sim_time >= heartbeat_ ? sim_time - heartbeat_ + 1 : 0; | |
| 198 | ✗ | return last_record_closed_sim_time_.getValue() < window_lo; | |
| 199 | } | ||
| 200 | |||
| 201 | /// Returns true when a CLOSED wire should be preceded by a priming FULL snapshot. | ||
| 202 | 38363 | bool needsClosedPriming_(uint64_t sim_time) const | |
| 203 | { | ||
| 204 | 38363 | if (!last_snapshot_sim_time_.isValid()) | |
| 205 | { | ||
| 206 | ✗ | return true; | |
| 207 | } | ||
| 208 | 38363 | const uint64_t window_lo = sim_time >= heartbeat_ ? sim_time - heartbeat_ + 1 : 0; | |
| 209 | 38363 | return last_snapshot_sim_time_.getValue() < window_lo; | |
| 210 | } | ||
| 211 | |||
| 212 | /// Updates delta-chain and heartbeat bookkeeping after a wire is emitted. | ||
| 213 | 128308 | void recordCidSent_(Action action, uint64_t sim_time) | |
| 214 | { | ||
| 215 | 128308 | if (action == Action::FULL) | |
| 216 | { | ||
| 217 | 89945 | distance_to_snapshot_ = 0; | |
| 218 | } else | ||
| 219 | { | ||
| 220 | 38363 | ++distance_to_snapshot_; | |
| 221 | } | ||
| 222 | |||
| 223 | 128308 | if (action == Action::FULL) | |
| 224 | { | ||
| 225 | 89945 | last_snapshot_sim_time_ = sim_time; | |
| 226 | 38363 | } else if (action == Action::CLOSED) | |
| 227 | { | ||
| 228 | 38363 | last_record_closed_sim_time_ = sim_time; | |
| 229 | } | ||
| 230 | 128308 | } | |
| 231 | |||
| 232 | const size_t heartbeat_; | ||
| 233 | size_t distance_to_snapshot_ = 0; | ||
| 234 | ValidValue<uint64_t> last_snapshot_sim_time_; | ||
| 235 | ValidValue<uint64_t> last_record_closed_sim_time_; | ||
| 236 | }; | ||
| 237 | |||
| 238 | //! \class ScalarCheckpointer | ||
| 239 | class ScalarCheckpointer final : public CollectableCheckpointer | ||
| 240 | { | ||
| 241 | public: | ||
| 242 | /// Constructs a scalar checkpointer with the given heartbeat interval. | ||
| 243 | 31 | ScalarCheckpointer(size_t heartbeat) : | |
| 244 | 31 | CollectableCheckpointer(heartbeat) | |
| 245 | { | ||
| 246 | 31 | } | |
| 247 | |||
| 248 | /// Appends a scalar data checkpoint for \p window_id. | ||
| 249 | 13008 | void createCheckpoint(uint64_t window_id, const std::vector<char>& scalar_bytes) override | |
| 250 | { | ||
| 251 | 13008 | head_ = std::make_shared<ScalarCheckpoint>(head_, window_id, scalar_bytes); | |
| 252 | 13008 | if (!tail_) | |
| 253 | { | ||
| 254 | 31 | tail_ = head_; | |
| 255 | } | ||
| 256 | 13008 | } | |
| 257 | |||
| 258 | using CollectableCheckpointer::createCheckpoint; // un-hide the other two | ||
| 259 | |||
| 260 | /// Appends a scalar close lifecycle checkpoint for \p window_id. | ||
| 261 | 1091 | void closeRecord(uint64_t window_id) override | |
| 262 | { | ||
| 263 | 1091 | constexpr bool closed = true; | |
| 264 | 1091 | head_ = std::make_shared<ScalarCheckpoint>(head_, window_id, closed); | |
| 265 | 1091 | if (!tail_) | |
| 266 | { | ||
| 267 | ✗ | tail_ = head_; | |
| 268 | } | ||
| 269 | 1091 | } | |
| 270 | |||
| 271 | /// Returns true if this scalar collectable staged data in \p window_id. | ||
| 272 | ✗ | bool participatedInWindow(uint64_t window_id) const override { return participatedInWindow_(window_id, tail_); } | |
| 273 | |||
| 274 | /// Encodes scalar wire records for \p cid at \p sim_time. | ||
| 275 | 30005 | std::vector<std::unique_ptr<CollectedData>> encodeForPipeline(uint64_t window_id, uint64_t sim_time, | |
| 276 | uint16_t cid) override | ||
| 277 | { | ||
| 278 | 30005 | auto* anchor = getAnchorForWindow_(window_id, tail_); | |
| 279 | 30005 | if (anchor && anchor->getWindowID() == window_id) | |
| 280 | { | ||
| 281 | 14099 | return encodeForPipeline_(anchor, cid, forceSnapshot_(sim_time), sim_time); | |
| 282 | } | ||
| 283 | |||
| 284 | 15906 | auto* tip = head_.get(); | |
| 285 | 15906 | const bool tip_closed = tip && tip->isClosedEvent(); | |
| 286 | 15906 | if (shouldRefreshAbsentCid_(sim_time, tip_closed)) | |
| 287 | { | ||
| 288 | 15906 | if (tip_closed) | |
| 289 | { | ||
| 290 | 11007 | return emitRecordClosed_(cid, sim_time); | |
| 291 | } | ||
| 292 | |||
| 293 | 4899 | auto* latest = latestDataCheckpoint_(); | |
| 294 | 4899 | if (!latest) | |
| 295 | { | ||
| 296 | ✗ | return {}; | |
| 297 | } | ||
| 298 | |||
| 299 | 4899 | auto encoded = latest->encodeSnapshotForPipeline(cid); | |
| 300 | 4899 | recordCidSent_(Action::FULL, sim_time); | |
| 301 | 4899 | return makeSingleElemVector_(std::move(encoded)); | |
| 302 | 4899 | } | |
| 303 | |||
| 304 | ✗ | return {}; | |
| 305 | } | ||
| 306 | |||
| 307 | private: | ||
| 308 | /// Returns the newest scalar data checkpoint, skipping lifecycle-only nodes. | ||
| 309 | 16997 | ScalarCheckpoint* latestDataCheckpoint_() const | |
| 310 | { | ||
| 311 | 16997 | auto* checkpoint = head_.get(); | |
| 312 | 29095 | while (checkpoint && !checkpoint->isDataCheckpoint()) | |
| 313 | { | ||
| 314 | 12098 | checkpoint = checkpoint->prev(); | |
| 315 | } | ||
| 316 | 16997 | return checkpoint; | |
| 317 | } | ||
| 318 | |||
| 319 | /// Emits CLOSED, optionally priming with FULL when the heartbeat window requires it. | ||
| 320 | 12098 | std::vector<std::unique_ptr<CollectedData>> emitRecordClosed_(uint16_t cid, uint64_t sim_time) | |
| 321 | { | ||
| 322 | 12098 | std::vector<std::unique_ptr<CollectedData>> out; | |
| 323 | 12098 | if (needsClosedPriming_(sim_time)) | |
| 324 | { | ||
| 325 | 12098 | if (auto* latest = latestDataCheckpoint_()) | |
| 326 | { | ||
| 327 | 12098 | out.push_back(latest->encodeSnapshotForPipeline(cid)); | |
| 328 | 12098 | recordCidSent_(Action::FULL, sim_time); | |
| 329 | } | ||
| 330 | } | ||
| 331 | |||
| 332 | 12098 | auto encoded = std::make_unique<CollectedData>(cid); | |
| 333 | 12098 | encoded->getBuffer().append(Action::CLOSED); | |
| 334 | 12098 | recordCidSent_(Action::CLOSED, sim_time); | |
| 335 | 12098 | out.push_back(std::move(encoded)); | |
| 336 | 12098 | return out; | |
| 337 | 12098 | } | |
| 338 | |||
| 339 | /// Encodes the scalar checkpoint at \p anchor and trims the chain through that node. | ||
| 340 | 14099 | std::vector<std::unique_ptr<CollectedData>> encodeForPipeline_(ScalarCheckpoint* anchor, uint16_t cid, | |
| 341 | bool force_snapshot, uint64_t sim_time) | ||
| 342 | { | ||
| 343 | 14099 | auto encoded = anchor->encodeForPipeline(cid, force_snapshot); | |
| 344 | 14099 | if (!encoded) | |
| 345 | { | ||
| 346 | ✗ | return {}; | |
| 347 | } | ||
| 348 | |||
| 349 | 14099 | const auto action = readEncodedAction_(*encoded); | |
| 350 | 14099 | if (action == Action::CLOSED) | |
| 351 | { | ||
| 352 | 1091 | cleanupThrough_(anchor, action); | |
| 353 | 1091 | return emitRecordClosed_(cid, sim_time); | |
| 354 | } | ||
| 355 | |||
| 356 | 13008 | recordCidSent_(action, sim_time); | |
| 357 | 13008 | cleanupThrough_(anchor, action); | |
| 358 | 13008 | return makeSingleElemVector_(std::move(encoded)); | |
| 359 | 14099 | } | |
| 360 | |||
| 361 | /// Advances \p tail_ to \p anchor and detaches earlier nodes on FULL. | ||
| 362 | 14099 | void cleanupThrough_(ScalarCheckpoint* anchor, Action action) | |
| 363 | { | ||
| 364 | 14099 | auto sp = getSharedCheckpoint_(anchor, head_); | |
| 365 | 14099 | if (!sp) | |
| 366 | { | ||
| 367 | ✗ | return; | |
| 368 | } | ||
| 369 | 14099 | if (action == Action::FULL) | |
| 370 | { | ||
| 371 | 13008 | sp->detachPrev(); | |
| 372 | } | ||
| 373 | 14099 | tail_ = sp; | |
| 374 | 14099 | } | |
| 375 | |||
| 376 | std::shared_ptr<ScalarCheckpoint> head_; | ||
| 377 | std::shared_ptr<ScalarCheckpoint> tail_; | ||
| 378 | }; | ||
| 379 | |||
| 380 | //! \class ContigContainerCheckpointer | ||
| 381 | class ContigContainerCheckpointer final : public CollectableCheckpointer | ||
| 382 | { | ||
| 383 | public: | ||
| 384 | /// Constructs a contig checkpointer with heartbeat and fixed capacity. | ||
| 385 | 30 | ContigContainerCheckpointer(size_t heartbeat, size_t capacity) : | |
| 386 | CollectableCheckpointer(heartbeat), | ||
| 387 | 30 | capacity_(capacity) | |
| 388 | { | ||
| 389 | 30 | } | |
| 390 | |||
| 391 | /// Appends a contig container data checkpoint for \p window_id. | ||
| 392 | 12015 | void createCheckpoint(uint64_t window_id, const std::vector<std::vector<char>>& contig_bin_bytes) override | |
| 393 | { | ||
| 394 | 12015 | auto size = detail::getContainerSize(contig_bin_bytes); | |
| 395 | 12015 | simdb_assert(size <= capacity_); | |
| 396 | (void)capacity_; | ||
| 397 | 12015 | max_container_size_ = std::max(max_container_size_, size); | |
| 398 | |||
| 399 | 12015 | head_ = std::make_shared<ContigContainerCheckpoint>(head_, window_id, contig_bin_bytes); | |
| 400 | 12015 | if (!tail_) | |
| 401 | { | ||
| 402 | 30 | tail_ = head_; | |
| 403 | } | ||
| 404 | 12015 | } | |
| 405 | |||
| 406 | using CollectableCheckpointer::createCheckpoint; // un-hide the other two | ||
| 407 | |||
| 408 | /// Appends a contig close lifecycle checkpoint for \p window_id. | ||
| 409 | 1713 | void closeRecord(uint64_t window_id) override | |
| 410 | { | ||
| 411 | 1713 | constexpr bool closed = true; | |
| 412 | 1713 | head_ = std::make_shared<ContigContainerCheckpoint>(head_, window_id, closed); | |
| 413 | 1713 | if (!tail_) | |
| 414 | { | ||
| 415 | ✗ | tail_ = head_; | |
| 416 | } | ||
| 417 | 1713 | } | |
| 418 | |||
| 419 | /// Returns true if this contig collectable staged data in \p window_id. | ||
| 420 | ✗ | bool participatedInWindow(uint64_t window_id) const override { return participatedInWindow_(window_id, tail_); } | |
| 421 | |||
| 422 | /// Encodes contig container wire records for \p cid at \p sim_time. | ||
| 423 | 30000 | std::vector<std::unique_ptr<CollectedData>> encodeForPipeline(uint64_t window_id, uint64_t sim_time, | |
| 424 | uint16_t cid) override | ||
| 425 | { | ||
| 426 | 30000 | auto* anchor = getAnchorForWindow_(window_id, tail_); | |
| 427 | 30000 | if (anchor && anchor->getWindowID() == window_id) | |
| 428 | { | ||
| 429 | 13728 | return encodeForPipeline_(anchor, cid, forceSnapshot_(sim_time), sim_time); | |
| 430 | } | ||
| 431 | |||
| 432 | 16272 | auto* tip = head_.get(); | |
| 433 | 16272 | const bool tip_closed = tip && tip->isClosedEvent(); | |
| 434 | 16272 | if (shouldRefreshAbsentCid_(sim_time, tip_closed)) | |
| 435 | { | ||
| 436 | 16251 | if (tip_closed) | |
| 437 | { | ||
| 438 | 11682 | return emitRecordClosed_(cid, sim_time); | |
| 439 | } | ||
| 440 | |||
| 441 | 4569 | auto* latest = latestDataCheckpoint_(); | |
| 442 | 4569 | if (!latest) | |
| 443 | { | ||
| 444 | ✗ | return {}; | |
| 445 | } | ||
| 446 | |||
| 447 | 4569 | auto encoded = latest->encodeSnapshotForPipeline(cid); | |
| 448 | 4569 | recordCidSent_(Action::FULL, sim_time); | |
| 449 | 4569 | return makeSingleElemVector_(std::move(encoded)); | |
| 450 | 4569 | } | |
| 451 | |||
| 452 | 21 | return {}; | |
| 453 | } | ||
| 454 | |||
| 455 | /// Writes observed max queue occupancy to QueueMaxSizes. | ||
| 456 | ✗ | void writeMetaOnPostTeardown(uint16_t cid, DatabaseManager* db_mgr) override | |
| 457 | { | ||
| 458 | ✗ | db_mgr->INSERT(SQL_TABLE("QueueMaxSizes"), SQL_VALUES((int)cid, (int)max_container_size_)); | |
| 459 | ✗ | } | |
| 460 | |||
| 461 | private: | ||
| 462 | /// Returns the newest contig data checkpoint, skipping lifecycle-only nodes. | ||
| 463 | 17964 | ContigContainerCheckpoint* latestDataCheckpoint_() const | |
| 464 | { | ||
| 465 | 17964 | auto* checkpoint = head_.get(); | |
| 466 | 31359 | while (checkpoint && !checkpoint->isDataCheckpoint()) | |
| 467 | { | ||
| 468 | 13395 | checkpoint = checkpoint->prev(); | |
| 469 | } | ||
| 470 | 17964 | return checkpoint; | |
| 471 | } | ||
| 472 | |||
| 473 | /// Emits CLOSED, optionally priming with FULL when the heartbeat window requires it. | ||
| 474 | 13395 | std::vector<std::unique_ptr<CollectedData>> emitRecordClosed_(uint16_t cid, uint64_t sim_time) | |
| 475 | { | ||
| 476 | 13395 | std::vector<std::unique_ptr<CollectedData>> out; | |
| 477 | 13395 | if (needsClosedPriming_(sim_time)) | |
| 478 | { | ||
| 479 | 13395 | if (auto* latest = latestDataCheckpoint_()) | |
| 480 | { | ||
| 481 | 13395 | out.push_back(latest->encodeSnapshotForPipeline(cid)); | |
| 482 | 13395 | recordCidSent_(Action::FULL, sim_time); | |
| 483 | } | ||
| 484 | } | ||
| 485 | |||
| 486 | 13395 | auto encoded = std::make_unique<CollectedData>(cid); | |
| 487 | 13395 | encoded->getBuffer().append(Action::CLOSED); | |
| 488 | 13395 | recordCidSent_(Action::CLOSED, sim_time); | |
| 489 | 13395 | out.push_back(std::move(encoded)); | |
| 490 | 13395 | return out; | |
| 491 | 13395 | } | |
| 492 | |||
| 493 | /// Encodes the contig checkpoint at \p anchor and trims the chain through that node. | ||
| 494 | 13728 | std::vector<std::unique_ptr<CollectedData>> encodeForPipeline_(ContigContainerCheckpoint* anchor, uint16_t cid, | |
| 495 | bool force_snapshot, uint64_t sim_time) | ||
| 496 | { | ||
| 497 | 13728 | auto encoded = anchor->encodeForPipeline(cid, force_snapshot); | |
| 498 | 13728 | if (!encoded) | |
| 499 | { | ||
| 500 | ✗ | return {}; | |
| 501 | } | ||
| 502 | |||
| 503 | 13728 | const auto action = readEncodedAction_(*encoded); | |
| 504 | 13728 | if (action == Action::CLOSED) | |
| 505 | { | ||
| 506 | 1713 | cleanupThrough_(anchor, action); | |
| 507 | 1713 | return emitRecordClosed_(cid, sim_time); | |
| 508 | } | ||
| 509 | |||
| 510 | 12015 | recordCidSent_(action, sim_time); | |
| 511 | 12015 | cleanupThrough_(anchor, action); | |
| 512 | 12015 | return makeSingleElemVector_(std::move(encoded)); | |
| 513 | 13728 | } | |
| 514 | |||
| 515 | /// Advances \p tail_ to \p anchor and detaches earlier nodes on FULL. | ||
| 516 | 13728 | void cleanupThrough_(ContigContainerCheckpoint* anchor, Action action) | |
| 517 | { | ||
| 518 | 13728 | auto sp = getSharedCheckpoint_(anchor, head_); | |
| 519 | 13728 | if (!sp) | |
| 520 | { | ||
| 521 | ✗ | return; | |
| 522 | } | ||
| 523 | 13728 | if (action == Action::FULL) | |
| 524 | { | ||
| 525 | 12015 | sp->detachPrev(); | |
| 526 | } | ||
| 527 | 13728 | tail_ = sp; | |
| 528 | 13728 | } | |
| 529 | |||
| 530 | std::shared_ptr<ContigContainerCheckpoint> head_; | ||
| 531 | std::shared_ptr<ContigContainerCheckpoint> tail_; | ||
| 532 | uint16_t max_container_size_ = 0; | ||
| 533 | const uint16_t capacity_; | ||
| 534 | }; | ||
| 535 | |||
| 536 | //! \class SparseContainerCheckpointer | ||
| 537 | class SparseContainerCheckpointer final : public CollectableCheckpointer | ||
| 538 | { | ||
| 539 | public: | ||
| 540 | /// Constructs a sparse checkpointer with heartbeat and fixed capacity. | ||
| 541 | 30 | SparseContainerCheckpointer(size_t heartbeat, size_t capacity) : | |
| 542 | CollectableCheckpointer(heartbeat), | ||
| 543 | 30 | capacity_(capacity) | |
| 544 | { | ||
| 545 | 30 | } | |
| 546 | |||
| 547 | /// Appends a sparse container data checkpoint for \p window_id. | ||
| 548 | 12438 | void createCheckpoint(uint64_t window_id, const std::map<uint16_t, std::vector<char>>& sparse_bin_bytes) override | |
| 549 | { | ||
| 550 | 12438 | auto size = detail::getContainerSize(sparse_bin_bytes); | |
| 551 | 12438 | simdb_assert(size <= capacity_); | |
| 552 | (void)capacity_; | ||
| 553 | 12438 | max_container_size_ = std::max(max_container_size_, size); | |
| 554 | |||
| 555 | 12438 | head_ = std::make_shared<SparseContainerCheckpoint>(head_, window_id, sparse_bin_bytes); | |
| 556 | 12438 | if (!tail_) | |
| 557 | { | ||
| 558 | 30 | tail_ = head_; | |
| 559 | } | ||
| 560 | 12438 | } | |
| 561 | |||
| 562 | using CollectableCheckpointer::createCheckpoint; // un-hide the other two | ||
| 563 | |||
| 564 | /// Appends a sparse close lifecycle checkpoint for \p window_id. | ||
| 565 | 1608 | void closeRecord(uint64_t window_id) override | |
| 566 | { | ||
| 567 | 1608 | constexpr bool closed = true; | |
| 568 | 1608 | head_ = std::make_shared<SparseContainerCheckpoint>(head_, window_id, closed); | |
| 569 | 1608 | if (!tail_) | |
| 570 | { | ||
| 571 | ✗ | tail_ = head_; | |
| 572 | } | ||
| 573 | 1608 | } | |
| 574 | |||
| 575 | /// Returns true if this sparse collectable staged data in \p window_id. | ||
| 576 | ✗ | bool participatedInWindow(uint64_t window_id) const override { return participatedInWindow_(window_id, tail_); } | |
| 577 | |||
| 578 | /// Encodes sparse container wire records for \p cid at \p sim_time. | ||
| 579 | 30000 | std::vector<std::unique_ptr<CollectedData>> encodeForPipeline(uint64_t window_id, uint64_t sim_time, | |
| 580 | uint16_t cid) override | ||
| 581 | { | ||
| 582 | 30000 | auto* anchor = getAnchorForWindow_(window_id, tail_); | |
| 583 | 30000 | if (anchor && anchor->getWindowID() == window_id) | |
| 584 | { | ||
| 585 | 14046 | return encodeForPipeline_(anchor, cid, forceSnapshot_(sim_time), sim_time); | |
| 586 | } | ||
| 587 | |||
| 588 | 15954 | auto* tip = head_.get(); | |
| 589 | 15954 | const bool tip_closed = tip && tip->isClosedEvent(); | |
| 590 | 15954 | if (shouldRefreshAbsentCid_(sim_time, tip_closed)) | |
| 591 | { | ||
| 592 | 15915 | if (tip_closed) | |
| 593 | { | ||
| 594 | 11262 | return emitRecordClosed_(cid, sim_time); | |
| 595 | } | ||
| 596 | |||
| 597 | 4653 | auto* latest = latestDataCheckpoint_(); | |
| 598 | 4653 | if (!latest) | |
| 599 | { | ||
| 600 | ✗ | return {}; | |
| 601 | } | ||
| 602 | |||
| 603 | 4653 | auto encoded = latest->encodeSnapshotForPipeline(cid); | |
| 604 | 4653 | recordCidSent_(Action::FULL, sim_time); | |
| 605 | 4653 | return makeSingleElemVector_(std::move(encoded)); | |
| 606 | 4653 | } | |
| 607 | |||
| 608 | 39 | return {}; | |
| 609 | } | ||
| 610 | |||
| 611 | /// Writes observed max queue occupancy to QueueMaxSizes. | ||
| 612 | ✗ | void writeMetaOnPostTeardown(uint16_t cid, DatabaseManager* db_mgr) override | |
| 613 | { | ||
| 614 | ✗ | db_mgr->INSERT(SQL_TABLE("QueueMaxSizes"), SQL_VALUES((int)cid, (int)max_container_size_)); | |
| 615 | ✗ | } | |
| 616 | |||
| 617 | private: | ||
| 618 | /// Returns the newest sparse data checkpoint, skipping lifecycle-only nodes. | ||
| 619 | 17523 | SparseContainerCheckpoint* latestDataCheckpoint_() const | |
| 620 | { | ||
| 621 | 17523 | auto* checkpoint = head_.get(); | |
| 622 | 30393 | while (checkpoint && !checkpoint->isDataCheckpoint()) | |
| 623 | { | ||
| 624 | 12870 | checkpoint = checkpoint->prev(); | |
| 625 | } | ||
| 626 | 17523 | return checkpoint; | |
| 627 | } | ||
| 628 | |||
| 629 | /// Emits CLOSED, optionally priming with FULL when the heartbeat window requires it. | ||
| 630 | 12870 | std::vector<std::unique_ptr<CollectedData>> emitRecordClosed_(uint16_t cid, uint64_t sim_time) | |
| 631 | { | ||
| 632 | 12870 | std::vector<std::unique_ptr<CollectedData>> out; | |
| 633 | 12870 | if (needsClosedPriming_(sim_time)) | |
| 634 | { | ||
| 635 | 12870 | if (auto* latest = latestDataCheckpoint_()) | |
| 636 | { | ||
| 637 | 12870 | out.push_back(latest->encodeSnapshotForPipeline(cid)); | |
| 638 | 12870 | recordCidSent_(Action::FULL, sim_time); | |
| 639 | } | ||
| 640 | } | ||
| 641 | |||
| 642 | 12870 | auto encoded = std::make_unique<CollectedData>(cid); | |
| 643 | 12870 | encoded->getBuffer().append(Action::CLOSED); | |
| 644 | 12870 | recordCidSent_(Action::CLOSED, sim_time); | |
| 645 | 12870 | out.push_back(std::move(encoded)); | |
| 646 | 12870 | return out; | |
| 647 | 12870 | } | |
| 648 | |||
| 649 | /// Encodes the sparse checkpoint at \p anchor and trims the chain through that node. | ||
| 650 | 14046 | std::vector<std::unique_ptr<CollectedData>> encodeForPipeline_(SparseContainerCheckpoint* anchor, uint16_t cid, | |
| 651 | bool force_snapshot, uint64_t sim_time) | ||
| 652 | { | ||
| 653 | 14046 | auto encoded = anchor->encodeForPipeline(cid, force_snapshot); | |
| 654 | 14046 | if (!encoded) | |
| 655 | { | ||
| 656 | ✗ | return {}; | |
| 657 | } | ||
| 658 | |||
| 659 | 14046 | const auto action = readEncodedAction_(*encoded); | |
| 660 | 14046 | if (action == Action::CLOSED) | |
| 661 | { | ||
| 662 | 1608 | cleanupThrough_(anchor, action); | |
| 663 | 1608 | return emitRecordClosed_(cid, sim_time); | |
| 664 | } | ||
| 665 | |||
| 666 | 12438 | recordCidSent_(action, sim_time); | |
| 667 | 12438 | cleanupThrough_(anchor, action); | |
| 668 | 12438 | return makeSingleElemVector_(std::move(encoded)); | |
| 669 | 14046 | } | |
| 670 | |||
| 671 | /// Advances \p tail_ to \p anchor and detaches earlier nodes on FULL. | ||
| 672 | 14046 | void cleanupThrough_(SparseContainerCheckpoint* anchor, Action action) | |
| 673 | { | ||
| 674 | 14046 | auto sp = getSharedCheckpoint_(anchor, head_); | |
| 675 | 14046 | if (!sp) | |
| 676 | { | ||
| 677 | ✗ | return; | |
| 678 | } | ||
| 679 | 14046 | if (action == Action::FULL) | |
| 680 | { | ||
| 681 | 12438 | sp->detachPrev(); | |
| 682 | } | ||
| 683 | 14046 | tail_ = sp; | |
| 684 | 14046 | } | |
| 685 | |||
| 686 | std::shared_ptr<SparseContainerCheckpoint> head_; | ||
| 687 | std::shared_ptr<SparseContainerCheckpoint> tail_; | ||
| 688 | uint16_t max_container_size_ = 0; | ||
| 689 | const uint16_t capacity_; | ||
| 690 | }; | ||
| 691 | |||
| 692 | } // namespace simdb::argos | ||
| 693 |