From 68aef2912495d22ddc465d81294574c5e37b4401 Mon Sep 17 00:00:00 2001 From: Alejandro Canela Date: Fri, 17 Jul 2026 11:13:48 +0200 Subject: [PATCH 1/6] feat: First phase of hardening and recovering vtx file in case of a crash. Implement a durable_file class that syncs and flushes to hard disk, file_sink uses now the durable_file stead of a ffstream. Added checksum to each chunk to validate chunks separetly --- sdk/include/vtx/common/vtx_types.h | 5 +- .../vtx/writer/policies/sinks/durable_file.h | 112 ++++++++++++++++++ .../vtx/writer/policies/sinks/file_sink.h | 41 +++---- sdk/src/schemas/vtx_schema.fbs | 1 + sdk/src/schemas/vtx_schema.proto | 1 + .../formatters/flatbuffer_reader_policy.cpp | 15 +++ .../formatters/protobuff_reader_policy.cpp | 1 + .../formatters/flatbuffers_vtx_policy.cpp | 4 +- .../formatters/protobuff_vtx_policy.cpp | 1 + 9 files changed, 158 insertions(+), 23 deletions(-) create mode 100644 sdk/include/vtx/writer/policies/sinks/durable_file.h diff --git a/sdk/include/vtx/common/vtx_types.h b/sdk/include/vtx/common/vtx_types.h index a2ff5ee..128fa70 100644 --- a/sdk/include/vtx/common/vtx_types.h +++ b/sdk/include/vtx/common/vtx_types.h @@ -635,6 +635,7 @@ namespace VTX { uint64_t file_offset = 0; ///< Byte offset in the file where the chunk begins. uint32_t chunk_size_bytes = 0; ///< Size of the chunk in bytes. + uint64_t checksum = 0; ///< xxHash64 of the on-disk chunk payload (0 = not set). }; /** @@ -1223,12 +1224,14 @@ namespace VTX { , file_offset(0) , chunk_size_bytes(0) , start_frame(0) - , end_frame(0) {} + , end_frame(0) + , checksum(0) {} int32_t chunk_index; int64_t file_offset; uint32_t chunk_size_bytes; int32_t start_frame; int32_t end_frame; + uint64_t checksum; ///< xxHash64 of the on-disk chunk payload. }; } // namespace VTX diff --git a/sdk/include/vtx/writer/policies/sinks/durable_file.h b/sdk/include/vtx/writer/policies/sinks/durable_file.h new file mode 100644 index 0000000..a91749b --- /dev/null +++ b/sdk/include/vtx/writer/policies/sinks/durable_file.h @@ -0,0 +1,112 @@ +/** + * @file durable_file.h + * @brief Minimal binary file wrapper that can flush all the way to physical disk. + * + * @details std::ofstream cannot portably fsync (there is no standard way to reach + * the underlying descriptor/handle), which is required for crash/power-loss + * durability. DurableFile wraps a C stdio FILE* and exposes Sync() = fflush + + * fsync/_commit, plus the seek/tell primitives the sink needs. Sequential-write + * oriented; opened read/write+truncate so a repair pass can reuse the handle. + * + * @author Zenos Interactive + */ +#pragma once + +#include +#include +#include + +#ifdef _WIN32 +#include // _commit, _fileno +#else +#include // fsync, fileno +#endif + +namespace VTX { + + class DurableFile { + public: + DurableFile() = default; + ~DurableFile() { Close(); } + + DurableFile(const DurableFile&) = delete; + DurableFile& operator=(const DurableFile&) = delete; + + /// Open for binary read/write, truncating any existing file. Returns false on failure. + bool Open(const std::string& path) { + Close(); + fp_ = std::fopen(path.c_str(), "wb+"); + return fp_ != nullptr; + } + + bool IsOpen() const { return fp_ != nullptr; } + + void Write(const void* data, size_t size) { + if (fp_ && size > 0) { + std::fwrite(data, 1, size, fp_); + } + } + + void Write(const std::string& data) { Write(data.data(), data.size()); } + + /// Current write position (byte offset from the start of the file). + uint64_t Tell() { + if (!fp_) + return 0; +#ifdef _WIN32 + return static_cast(_ftelli64(fp_)); +#else + return static_cast(ftello(fp_)); +#endif + } + + void Seek(uint64_t offset) { + if (!fp_) + return; +#ifdef _WIN32 + _fseeki64(fp_, static_cast(offset), SEEK_SET); +#else + fseeko(fp_, static_cast(offset), SEEK_SET); +#endif + } + + void SeekEnd() { + if (!fp_) + return; +#ifdef _WIN32 + _fseeki64(fp_, 0, SEEK_END); +#else + fseeko(fp_, 0, SEEK_END); +#endif + } + + /// Push the user-space buffer to the OS (survives a process crash, not power loss). + void Flush() { + if (fp_) + std::fflush(fp_); + } + + /// Flush all the way to physical media (survives power loss). + void Sync() { + if (!fp_) + return; + std::fflush(fp_); +#ifdef _WIN32 + _commit(_fileno(fp_)); +#else + fsync(fileno(fp_)); +#endif + } + + void Close() { + if (fp_) { + std::fclose(fp_); + fp_ = nullptr; + } + } + + private: + std::FILE* fp_ = nullptr; + }; + +} // namespace VTX diff --git a/sdk/include/vtx/writer/policies/sinks/file_sink.h b/sdk/include/vtx/writer/policies/sinks/file_sink.h index 106c597..07cac15 100644 --- a/sdk/include/vtx/writer/policies/sinks/file_sink.h +++ b/sdk/include/vtx/writer/policies/sinks/file_sink.h @@ -1,12 +1,13 @@ #pragma once -#include #include #include #include #include #include +#include #include "vtx/common/vtx_types.h" #include "vtx/common/vtx_concepts.h" +#include "vtx/writer/policies/sinks/durable_file.h" namespace VTX { template @@ -22,12 +23,12 @@ namespace VTX { HeaderType header_config; bool b_use_compression = true; int8_t compression_level = 10; + bool durable_writes = true; ///< fsync each chunk to physical disk (crash/power-loss safe). }; explicit ChunkedFileSink(Config config) : config_(std::move(config)) { - file_.open(config_.filename, std::ios::binary | std::ios::out | std::ios::trunc); - if (!file_.is_open()) + if (!file_.Open(config_.filename)) throw std::runtime_error("VTX: Could not open " + config_.filename); } @@ -39,8 +40,10 @@ namespace VTX { std::string header_payload = SerializerPolicy::SerializeHeader(config_.header_config, schema); header_payload = CompressIfBeneficial(std::move(header_payload)); uint32_t final_size = static_cast(header_payload.size()); - file_.write(reinterpret_cast(&final_size), sizeof(final_size)); - file_.write(header_payload.data(), final_size); + file_.Write(&final_size, sizeof(final_size)); + file_.Write(header_payload.data(), final_size); + if (config_.durable_writes) + file_.Sync(); } void SaveChunk(std::vector>& frames, const std::vector& created_utc, @@ -48,21 +51,16 @@ namespace VTX { if (frames.empty()) return; - float chunk_start_time = 0.0f; - float chunk_end_time = 0.0f; - if (!created_utc.empty()) { - chunk_start_time = static_cast(created_utc.front()); - chunk_end_time = static_cast(created_utc.back()); - } - std::string payload = SerializerPolicy::SerializeChunk(frames, chunkIndex_, config_.b_use_compression); payload = CompressIfBeneficial(std::move(payload)); - uint64_t current_offset = file_.tellp(); + uint64_t current_offset = file_.Tell(); uint32_t final_size = static_cast(payload.size()); - file_.write(reinterpret_cast(&final_size), sizeof(final_size)); - file_.write(payload.data(), final_size); + file_.Write(&final_size, sizeof(final_size)); + file_.Write(payload.data(), final_size); + if (config_.durable_writes) + file_.Sync(); ChunkIndexData indexEntry; indexEntry.chunk_index = chunkIndex_++; @@ -70,23 +68,26 @@ namespace VTX { indexEntry.start_frame = start_frame; indexEntry.end_frame = total_frames - 1; indexEntry.chunk_size_bytes = final_size + sizeof(uint32_t); + indexEntry.checksum = XXH3_64bits(payload.data(), payload.size()); seek_table_.push_back(indexEntry); } void Close(const SessionFooter& footerData) { - if (!file_.is_open()) + if (!file_.IsOpen()) return; std::string footer_payload = SerializerPolicy::SerializeFooter(seek_table_, footerData); footer_payload = CompressIfBeneficial(std::move(footer_payload)); - file_.write(footer_payload.data(), footer_payload.size()); + file_.Write(footer_payload.data(), footer_payload.size()); uint32_t final_size = static_cast(footer_payload.size()); - file_.write(reinterpret_cast(&final_size), sizeof(final_size)); + file_.Write(&final_size, sizeof(final_size)); WriteBlob(SerializerPolicy::GetMagicBytes()); + if (config_.durable_writes) + file_.Sync(); } private: - void WriteBlob(const std::string& data) { file_.write(data.data(), data.size()); } + void WriteBlob(const std::string& data) { file_.Write(data.data(), data.size()); } std::string CompressIfBeneficial(std::string payload) { if (!config_.b_use_compression || payload.size() < 512) { @@ -112,7 +113,7 @@ namespace VTX { } Config config_; - std::ofstream file_; + DurableFile file_; int32_t chunkIndex_ = 0; std::vector seek_table_; //Generic tables, format agnostic }; diff --git a/sdk/src/schemas/vtx_schema.fbs b/sdk/src/schemas/vtx_schema.fbs index ebeb8c5..c558f93 100644 --- a/sdk/src/schemas/vtx_schema.fbs +++ b/sdk/src/schemas/vtx_schema.fbs @@ -198,6 +198,7 @@ table ChunkIndexEntry { end_frame: int; file_offset: ulong; chunk_size_bytes: uint; + checksum: ulong; // xxHash64 of the on-disk chunk payload (0 = not set) } table ReplayTimeData { diff --git a/sdk/src/schemas/vtx_schema.proto b/sdk/src/schemas/vtx_schema.proto index b95e3f8..9942731 100644 --- a/sdk/src/schemas/vtx_schema.proto +++ b/sdk/src/schemas/vtx_schema.proto @@ -236,6 +236,7 @@ message ChunkIndexEntry { uint64 file_offset = 4; // Byte offset where the chunk starts uint32 chunk_size_bytes = 5; // Size of the serialized chunk + uint64 checksum = 6; // xxHash64 of the on-disk chunk payload (0 = not set) } message ReplayTimeData{ diff --git a/sdk/src/vtx_reader/src/vtx/reader/formatters/flatbuffer_reader_policy.cpp b/sdk/src/vtx_reader/src/vtx/reader/formatters/flatbuffer_reader_policy.cpp index d1b6c4f..33b9086 100644 --- a/sdk/src/vtx_reader/src/vtx/reader/formatters/flatbuffer_reader_policy.cpp +++ b/sdk/src/vtx_reader/src/vtx/reader/formatters/flatbuffer_reader_policy.cpp @@ -1,5 +1,6 @@ #include "vtx/reader/policies/formatters/flatbuffer_reader_policy.h" +#include #include #include @@ -11,6 +12,12 @@ VTX::FlatBuffersReaderPolicy::HeaderType VTX::FlatBuffersReaderPolicy::ParseHeader(const std::string& buffer) { HeaderType header; + // Verify before touching offsets: a corrupt/truncated buffer must fail + // cleanly (throw), never dereference a bad offset (undefined behavior). + flatbuffers::Verifier verifier(reinterpret_cast(buffer.data()), buffer.size()); + if (!verifier.VerifyBuffer(nullptr)) { + throw std::runtime_error("VTX [FlatBuffers]: header failed verification (corrupt or truncated)"); + } auto* root = fbsvtx::GetFileHeader(buffer.data()); if (root) { root->UnPackTo(&header); @@ -24,6 +31,13 @@ std::string VTX::FlatBuffersReaderPolicy::GetMagicBytes() { VTX::FlatBuffersReaderPolicy::FooterType VTX::FlatBuffersReaderPolicy::ParseFooter(const std::string& buffer) { FooterType footer; + // Verify before touching offsets: a corrupt/truncated footer (e.g. a + // crash-truncated file whose tail bytes happen to pass the size gate) must + // fail cleanly instead of dereferencing garbage offsets (was a crash). + flatbuffers::Verifier verifier(reinterpret_cast(buffer.data()), buffer.size()); + if (!verifier.VerifyBuffer(nullptr)) { + throw std::runtime_error("VTX [FlatBuffers]: footer failed verification (corrupt or truncated)"); + } auto* root = flatbuffers::GetRoot(buffer.data()); if (root) { root->UnPackTo(&footer); @@ -96,6 +110,7 @@ void VTX::FlatBuffersReaderPolicy::PopulateIndexTable(const FooterType& footer, ce.end_frame = e->end_frame; ce.file_offset = e->file_offset; ce.chunk_size_bytes = e->chunk_size_bytes; + ce.checksum = e->checksum; chunk_index_table.push_back(ce); } diff --git a/sdk/src/vtx_reader/src/vtx/reader/formatters/protobuff_reader_policy.cpp b/sdk/src/vtx_reader/src/vtx/reader/formatters/protobuff_reader_policy.cpp index e575ce5..2473175 100644 --- a/sdk/src/vtx_reader/src/vtx/reader/formatters/protobuff_reader_policy.cpp +++ b/sdk/src/vtx_reader/src/vtx/reader/formatters/protobuff_reader_policy.cpp @@ -87,6 +87,7 @@ void VTX::ProtobufReaderPolicy::PopulateIndexTable(const FooterType& footer, nativeEntry.end_frame = protoEntry.end_frame(); nativeEntry.file_offset = protoEntry.file_offset(); nativeEntry.chunk_size_bytes = protoEntry.chunk_size_bytes(); + nativeEntry.checksum = protoEntry.checksum(); chunk_index_table.push_back(nativeEntry); } diff --git a/sdk/src/vtx_writer/src/vtx/writer/formatters/flatbuffers_vtx_policy.cpp b/sdk/src/vtx_writer/src/vtx/writer/formatters/flatbuffers_vtx_policy.cpp index 85fec1b..e0bab92 100644 --- a/sdk/src/vtx_writer/src/vtx/writer/formatters/flatbuffers_vtx_policy.cpp +++ b/sdk/src/vtx_writer/src/vtx/writer/formatters/flatbuffers_vtx_policy.cpp @@ -113,8 +113,8 @@ std::string VTX::FlatBuffersVtxPolicy::SerializeFooter(const std::vector(e.file_offset), - e.chunk_size_bytes)); + static_cast(e.file_offset), e.chunk_size_bytes, + e.checksum)); } auto index_vector = builder.CreateVector(index_offsets); diff --git a/sdk/src/vtx_writer/src/vtx/writer/formatters/protobuff_vtx_policy.cpp b/sdk/src/vtx_writer/src/vtx/writer/formatters/protobuff_vtx_policy.cpp index 0e44c6c..7a038eb 100644 --- a/sdk/src/vtx_writer/src/vtx/writer/formatters/protobuff_vtx_policy.cpp +++ b/sdk/src/vtx_writer/src/vtx/writer/formatters/protobuff_vtx_policy.cpp @@ -135,6 +135,7 @@ std::string VTX::ProtobufVtxPolicy::SerializeFooter(const std::vectorset_end_frame(entry.end_frame); proto_entry->set_file_offset(entry.file_offset); proto_entry->set_chunk_size_bytes(entry.chunk_size_bytes); + proto_entry->set_checksum(entry.checksum); } std::string payload; From 5aa3882e8b5aed95fd1f99a02154bef7a09d6c46 Mon Sep 17 00:00:00 2001 From: Alejandro Canela Date: Fri, 17 Jul 2026 11:40:30 +0200 Subject: [PATCH 2/6] feat : vtx crash recovery, create a .recovery file with only chunks, repairs the replay from the .recovery, it includes a test forcing a crash --- .../vtx/writer/core/vtx_replay_recovery.h | 38 ++++ .../vtx/writer/policies/sinks/durable_file.h | 7 + .../vtx/writer/policies/sinks/file_sink.h | 24 ++- .../writer/policies/sinks/recovery_journal.h | 170 ++++++++++++++++ .../vtx/writer/core/vtx_replay_recovery.cpp | 161 +++++++++++++++ tests/CMakeLists.txt | 1 + tests/writer/test_crash_recovery.cpp | 184 ++++++++++++++++++ 7 files changed, 584 insertions(+), 1 deletion(-) create mode 100644 sdk/include/vtx/writer/core/vtx_replay_recovery.h create mode 100644 sdk/include/vtx/writer/policies/sinks/recovery_journal.h create mode 100644 sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp create mode 100644 tests/writer/test_crash_recovery.cpp diff --git a/sdk/include/vtx/writer/core/vtx_replay_recovery.h b/sdk/include/vtx/writer/core/vtx_replay_recovery.h new file mode 100644 index 0000000..9baada0 --- /dev/null +++ b/sdk/include/vtx/writer/core/vtx_replay_recovery.h @@ -0,0 +1,38 @@ +/** + * @file vtx_replay_recovery.h + * @brief Repair a .vtx file whose writing crashed before the footer was written. + * + * @details When the writer's recovery journal is enabled, a sidecar ".recovery" + * records every committed chunk. If the process crashes before Close() writes the + * footer (and deletes the sidecar), RepairReplayFile() reconstructs a valid footer + * from the journal: it drops any torn tail chunk, verifies each surviving chunk's + * checksum, appends a synthesized footer, and removes the sidecar. The result is a + * normal .vtx that the standard reader opens with no special handling. + * + * @author Zenos Interactive + */ +#pragma once + +#include +#include + +namespace VTX { + + struct RepairResult { + bool was_clean = false; ///< No recovery journal present -> nothing to repair. + bool repaired = false; ///< A footer was reconstructed and written. + int32_t recovered_chunks = 0; ///< Chunks preserved in the recovered file. + int32_t recovered_frames = 0; ///< total_frames of the recovered file. + std::string error; ///< Non-empty on failure. + + bool ok() const { return error.empty(); } + }; + + /** + * @brief Recover a crashed .vtx using its ".recovery" journal. + * @param path Path to the (footerless) main .vtx file. + * @return Outcome. If no journal exists, was_clean is true and nothing is changed. + */ + RepairResult RepairReplayFile(const std::string& path); + +} // namespace VTX diff --git a/sdk/include/vtx/writer/policies/sinks/durable_file.h b/sdk/include/vtx/writer/policies/sinks/durable_file.h index a91749b..44adabc 100644 --- a/sdk/include/vtx/writer/policies/sinks/durable_file.h +++ b/sdk/include/vtx/writer/policies/sinks/durable_file.h @@ -39,6 +39,13 @@ namespace VTX { return fp_ != nullptr; } + /// Open an EXISTING file for binary read/write WITHOUT truncating (used by repair). + bool OpenExisting(const std::string& path) { + Close(); + fp_ = std::fopen(path.c_str(), "rb+"); + return fp_ != nullptr; + } + bool IsOpen() const { return fp_ != nullptr; } void Write(const void* data, size_t size) { diff --git a/sdk/include/vtx/writer/policies/sinks/file_sink.h b/sdk/include/vtx/writer/policies/sinks/file_sink.h index 07cac15..593c69e 100644 --- a/sdk/include/vtx/writer/policies/sinks/file_sink.h +++ b/sdk/include/vtx/writer/policies/sinks/file_sink.h @@ -2,12 +2,14 @@ #include #include #include +#include #include #include #include #include "vtx/common/vtx_types.h" #include "vtx/common/vtx_concepts.h" #include "vtx/writer/policies/sinks/durable_file.h" +#include "vtx/writer/policies/sinks/recovery_journal.h" namespace VTX { template @@ -23,7 +25,8 @@ namespace VTX { HeaderType header_config; bool b_use_compression = true; int8_t compression_level = 10; - bool durable_writes = true; ///< fsync each chunk to physical disk (crash/power-loss safe). + bool durable_writes = true; ///< fsync each chunk to physical disk (crash/power-loss safe). + bool enable_recovery_journal = true; ///< maintain a ".recovery" sidecar for crash recovery. }; explicit ChunkedFileSink(Config config) @@ -44,6 +47,12 @@ namespace VTX { file_.Write(header_payload.data(), final_size); if (config_.durable_writes) file_.Sync(); + + // Start the crash-recovery journal only once the header is durable. + if (config_.enable_recovery_journal) { + journal_.Open(RecoveryJournal::PathFor(config_.filename), SerializerPolicy::GetMagicBytes(), + config_.durable_writes); + } } void SaveChunk(std::vector>& frames, const std::vector& created_utc, @@ -70,6 +79,11 @@ namespace VTX { indexEntry.chunk_size_bytes = final_size + sizeof(uint32_t); indexEntry.checksum = XXH3_64bits(payload.data(), payload.size()); seek_table_.push_back(indexEntry); + + // Journal the committed chunk AFTER its bytes are durable on disk, so + // the journal never references a chunk that isn't there (data-before-journal). + if (journal_.IsOpen()) + journal_.AppendChunk(indexEntry); } void Close(const SessionFooter& footerData) { @@ -84,6 +98,13 @@ namespace VTX { WriteBlob(SerializerPolicy::GetMagicBytes()); if (config_.durable_writes) file_.Sync(); + + // Clean shutdown: the footer is durable, so the recovery journal is no + // longer needed. Its absence signals a clean file to the repair path. + if (journal_.IsOpen()) { + journal_.Close(); + std::remove(RecoveryJournal::PathFor(config_.filename).c_str()); + } } private: @@ -114,6 +135,7 @@ namespace VTX { Config config_; DurableFile file_; + RecoveryJournal journal_; int32_t chunkIndex_ = 0; std::vector seek_table_; //Generic tables, format agnostic }; diff --git a/sdk/include/vtx/writer/policies/sinks/recovery_journal.h b/sdk/include/vtx/writer/policies/sinks/recovery_journal.h new file mode 100644 index 0000000..e8df07f --- /dev/null +++ b/sdk/include/vtx/writer/policies/sinks/recovery_journal.h @@ -0,0 +1,170 @@ +/** + * @file recovery_journal.h + * @brief Write-ahead journal (sidecar ".recovery" file) of committed chunk index + * entries, so a crash before the footer is written can be recovered. + * + * @details Each record is fixed-size and self-validating (per-record xxHash64), so + * a torn last record from a crash mid-write is detected and dropped. Records are + * appended AFTER the corresponding chunk is durably on disk (data-before-journal + * ordering) so the journal never references a chunk that isn't there. On a clean + * Close() the main-file footer is written and the ".recovery" file is deleted; its + * presence at open time therefore signals an unclean shutdown. + * + * Byte layout (little-endian host): + * Header: "VTXR" | u32 version | 4-byte main-file magic ("VTXP"/"VTXF") + * Record: i32 chunk_index | i32 start_frame | i32 end_frame | u64 file_offset | + * u32 chunk_size_bytes | u64 chunk_checksum | u64 record_checksum + * + * @author Zenos Interactive + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "vtx/common/vtx_types.h" +#include "vtx/writer/policies/sinks/durable_file.h" + +namespace VTX { + + class RecoveryJournal { + public: + static constexpr uint32_t kVersion = 1; + static constexpr size_t kPayloadSize = 4 + 4 + 4 + 8 + 4 + 8; // 32 + static constexpr size_t kRecordSize = kPayloadSize + 8; // 40 (+ record checksum) + + /// The sidecar path for a given main output file. + static std::string PathFor(const std::string& main_file) { return main_file + ".recovery"; } + + /// Create/truncate the journal and write its header. @p format_magic is the + /// main file's 4-byte magic. Returns false on failure. + bool Open(const std::string& path, const std::string& format_magic, bool durable) { + durable_ = durable; + if (!file_.Open(path)) + return false; + char magic[4] = {'V', 'T', 'X', 'R'}; + file_.Write(magic, 4); + uint32_t version = kVersion; + file_.Write(&version, sizeof(version)); + char fmt[4] = {' ', ' ', ' ', ' '}; + for (size_t i = 0; i < 4 && i < format_magic.size(); ++i) + fmt[i] = format_magic[i]; + file_.Write(fmt, 4); + if (durable_) + file_.Sync(); + return true; + } + + bool IsOpen() const { return file_.IsOpen(); } + + /// Append one committed-chunk record. Call AFTER the chunk is durable on disk. + void AppendChunk(const ChunkIndexData& e) { + uint8_t rec[kRecordSize]; + size_t o = 0; + PackI32(rec, o, e.chunk_index); + PackI32(rec, o, e.start_frame); + PackI32(rec, o, e.end_frame); + PackU64(rec, o, static_cast(e.file_offset)); + PackU32(rec, o, e.chunk_size_bytes); + PackU64(rec, o, e.checksum); + uint64_t record_checksum = XXH3_64bits(rec, kPayloadSize); + PackU64(rec, o, record_checksum); + file_.Write(rec, kRecordSize); + if (durable_) + file_.Sync(); + } + + void Close() { file_.Close(); } + + // --- Read side (repair) --- + + struct Parsed { + bool has_journal = false; ///< The sidecar file existed. + bool header_valid = false; ///< Header magic/version parsed OK. + std::string format_magic; ///< Main-file magic recorded in the header. + std::vector chunks; ///< Valid committed-chunk records, in order. + }; + + /// Parse a journal file, returning all VALID records up to the first torn/invalid one. + static Parsed ReadValid(const std::string& path) { + Parsed out; + std::ifstream in(path, std::ios::binary); + if (!in.is_open()) + return out; // has_journal stays false + out.has_journal = true; + + char header[12]; + in.read(header, sizeof(header)); + if (in.gcount() != static_cast(sizeof(header))) + return out; // truncated header -> no usable records + if (std::memcmp(header, "VTXR", 4) != 0) + return out; + uint32_t version = 0; + std::memcpy(&version, header + 4, sizeof(version)); + if (version != kVersion) + return out; + out.format_magic.assign(header + 8, 4); + out.header_valid = true; + + uint8_t rec[kRecordSize]; + while (in.read(reinterpret_cast(rec), kRecordSize), + in.gcount() == static_cast(kRecordSize)) { + uint64_t stored = 0; + std::memcpy(&stored, rec + kPayloadSize, sizeof(stored)); + if (XXH3_64bits(rec, kPayloadSize) != stored) + break; // torn / corrupt record -> stop here + + ChunkIndexData e; + size_t o = 0; + e.chunk_index = UnpackI32(rec, o); + e.start_frame = UnpackI32(rec, o); + e.end_frame = UnpackI32(rec, o); + e.file_offset = static_cast(UnpackU64(rec, o)); + e.chunk_size_bytes = UnpackU32(rec, o); + e.checksum = UnpackU64(rec, o); + out.chunks.push_back(e); + } + return out; + } + + private: + static void PackI32(uint8_t* b, size_t& o, int32_t v) { + uint32_t u = static_cast(v); + PackU32(b, o, u); + } + static void PackU32(uint8_t* b, size_t& o, uint32_t v) { + b[o + 0] = static_cast(v); + b[o + 1] = static_cast(v >> 8); + b[o + 2] = static_cast(v >> 16); + b[o + 3] = static_cast(v >> 24); + o += 4; + } + static void PackU64(uint8_t* b, size_t& o, uint64_t v) { + for (int i = 0; i < 8; ++i) + b[o + i] = static_cast(v >> (8 * i)); + o += 8; + } + static int32_t UnpackI32(const uint8_t* b, size_t& o) { return static_cast(UnpackU32(b, o)); } + static uint32_t UnpackU32(const uint8_t* b, size_t& o) { + uint32_t v = static_cast(b[o]) | (static_cast(b[o + 1]) << 8) | + (static_cast(b[o + 2]) << 16) | (static_cast(b[o + 3]) << 24); + o += 4; + return v; + } + static uint64_t UnpackU64(const uint8_t* b, size_t& o) { + uint64_t v = 0; + for (int i = 0; i < 8; ++i) + v |= static_cast(b[o + i]) << (8 * i); + o += 8; + return v; + } + + DurableFile file_; + bool durable_ = true; + }; + +} // namespace VTX diff --git a/sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp b/sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp new file mode 100644 index 0000000..eeff5a1 --- /dev/null +++ b/sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp @@ -0,0 +1,161 @@ +#include "vtx/writer/core/vtx_replay_recovery.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "vtx/common/vtx_types.h" +#include "vtx/writer/policies/formatters/flatbuffers_vtx_policy.h" +#include "vtx/writer/policies/formatters/protobuff_vtx_policy.h" +#include "vtx/writer/policies/sinks/durable_file.h" +#include "vtx/writer/policies/sinks/recovery_journal.h" + +namespace VTX { + namespace { + + int64_t FileSizeOf(const std::string& path) { + std::error_code ec; + auto s = std::filesystem::file_size(path, ec); + return ec ? -1 : static_cast(s); + } + + // Byte offset where chunks begin (magic(4) + u32 header_size + header), or -1 if + // the header framing is not intact. + int64_t ReadHeaderEnd(std::ifstream& in, const std::string& expect_magic, int64_t file_size) { + in.clear(); + in.seekg(0); + char magic[4]; + in.read(magic, 4); + if (in.gcount() != 4 || std::string(magic, 4) != expect_magic) + return -1; + uint32_t header_size = 0; + in.read(reinterpret_cast(&header_size), sizeof(header_size)); + if (in.gcount() != static_cast(sizeof(header_size))) + return -1; + const int64_t header_end = 8 + static_cast(header_size); + if (header_size == 0 || header_end > file_size) + return -1; + return header_end; + } + + template + RepairResult RepairImpl(const std::string& path, const RecoveryJournal::Parsed& journal) { + RepairResult r; + const int64_t file_size = FileSizeOf(path); + if (file_size < 8) { + r.error = "main file too small or missing"; + return r; + } + + std::ifstream in(path, std::ios::binary); + if (!in.is_open()) { + r.error = "cannot open main file"; + return r; + } + const int64_t header_end = ReadHeaderEnd(in, Policy::GetMagicBytes(), file_size); + if (header_end < 0) { + r.error = "header framing not intact; cannot repair"; + return r; + } + + // Keep only chunks whose on-disk bytes are present and checksum-clean, in order. + std::vector good; + int64_t truncate_to = header_end; + for (const auto& c : journal.chunks) { + const int64_t off = c.file_offset; + const int64_t end = off + static_cast(c.chunk_size_bytes); + if (off < header_end || c.chunk_size_bytes <= sizeof(uint32_t) || end > file_size) + break; // torn tail / beyond EOF + + std::string bytes; + bytes.resize(c.chunk_size_bytes); + in.clear(); + in.seekg(off); + in.read(bytes.data(), c.chunk_size_bytes); + if (in.gcount() != static_cast(c.chunk_size_bytes)) + break; + + // The stored checksum covers the payload after the 4-byte length prefix. + const uint64_t hash = + XXH3_64bits(bytes.data() + sizeof(uint32_t), c.chunk_size_bytes - sizeof(uint32_t)); + if (c.checksum != 0 && hash != c.checksum) + break; // corrupt chunk -> stop, drop it and everything after + + good.push_back(c); + truncate_to = end; + } + in.close(); + + // Drop the torn tail (a partial chunk written before its journal record, or a + // stale footer from a crash between footer-write and journal-delete). + std::error_code ec; + std::filesystem::resize_file(path, static_cast(truncate_to), ec); + if (ec) { + r.error = "failed to truncate main file: " + ec.message(); + return r; + } + + // Synthesize and append a footer over the recovered chunk index. + SessionFooter footer_data; + footer_data.total_frames = good.empty() ? 0 : (good.back().end_frame + 1); + const std::string footer_payload = Policy::SerializeFooter(good, footer_data); + + DurableFile out; + if (!out.OpenExisting(path)) { + r.error = "cannot reopen main file to append footer"; + return r; + } + out.SeekEnd(); + out.Write(footer_payload.data(), footer_payload.size()); + const uint32_t footer_size = static_cast(footer_payload.size()); + out.Write(&footer_size, sizeof(footer_size)); + const std::string magic = Policy::GetMagicBytes(); + out.Write(magic.data(), magic.size()); + out.Sync(); + out.Close(); + + std::remove(RecoveryJournal::PathFor(path).c_str()); + + r.repaired = true; + r.recovered_chunks = static_cast(good.size()); + r.recovered_frames = footer_data.total_frames; + return r; + } + + } // namespace + + RepairResult RepairReplayFile(const std::string& path) { + RepairResult r; + + const std::string journal_path = RecoveryJournal::PathFor(path); + const RecoveryJournal::Parsed journal = RecoveryJournal::ReadValid(journal_path); + if (!journal.has_journal) { + r.was_clean = true; // no sidecar -> file was closed cleanly (or never journaled) + return r; + } + + // Format is decided by the main file's leading magic. + std::ifstream in(path, std::ios::binary); + if (!in.is_open()) { + r.error = "cannot open main file: " + path; + return r; + } + char magic[4] = {0, 0, 0, 0}; + in.read(magic, 4); + in.close(); + const std::string m(magic, 4); + + if (m == "VTXP") + return RepairImpl(path, journal); + if (m == "VTXF") + return RepairImpl(path, journal); + + r.error = "unrecognized magic in main file; cannot repair"; + return r; + } + +} // namespace VTX diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 68093dd..328ec0f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -76,6 +76,7 @@ add_executable(vtx_tests writer/test_schema_in_memory.cpp writer/test_frame_post_processor.cpp writer/test_frame_finalization.cpp + writer/test_crash_recovery.cpp writer/test_network_sink.cpp writer/test_pipe_source.cpp writer/test_websocket_source.cpp diff --git a/tests/writer/test_crash_recovery.cpp b/tests/writer/test_crash_recovery.cpp new file mode 100644 index 0000000..45e2770 --- /dev/null +++ b/tests/writer/test_crash_recovery.cpp @@ -0,0 +1,184 @@ +// Crash-recovery tests: a writer that dies before Stop() leaves a footerless .vtx +// plus a ".recovery" journal; RepairReplayFile() must reconstruct a valid, readable +// file from it, dropping any torn/corrupt tail. +// +// The crash state is reproduced faithfully without racing a real process death: +// write a valid file, read its seek table (the exact per-chunk index the sink +// journals, checksums included), truncate the footer off, and re-create the +// ".recovery" journal from those records. That is byte-for-byte what the sink +// leaves on disk after a mid-write crash (chunks + journal, no footer). + +#include + +#include +#include +#include +#include +#include +#include + +#include "vtx/common/vtx_types.h" +#include "vtx/reader/core/vtx_reader_facade.h" +#include "vtx/writer/core/vtx_replay_recovery.h" +#include "vtx/writer/core/vtx_writer_facade.h" +#include "vtx/writer/policies/sinks/recovery_journal.h" + +#include "util/test_fixtures.h" + +namespace { + + VTX::Frame BuildFrame(int i) { + VTX::Frame f; + auto& bucket = f.CreateBucket("entity"); + VTX::PropertyContainer pc; + pc.entity_type_id = 0; // Player + pc.string_properties = {"player_0", "Alpha"}; + pc.int32_properties = {1, i, 0}; // Team, Score(=i), Deaths + pc.float_properties = {100.0f, 50.0f}; + bucket.unique_ids.push_back("player_0"); + bucket.entities.push_back(std::move(pc)); + return f; + } + + // Writes `chunks` single-frame chunks to a valid .vtx (footer + no journal). + std::string WriteValidFile(const std::string& suffix, int chunks) { + VTX::WriterFacadeConfig cfg; + cfg.output_filepath = VtxTest::OutputPath("crash_" + suffix + ".vtx"); + cfg.schema_json_path = VtxTest::FixturePath("test_schema.json"); + cfg.replay_name = "CrashRecoveryTest"; + cfg.default_fps = 60.0f; + cfg.use_compression = true; + + auto writer = VTX::CreateFlatBuffersWriterFacade(cfg); + EXPECT_NE(writer, nullptr); + for (int i = 0; i < chunks; ++i) { + auto frame = BuildFrame(i); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + writer->RecordFrame(frame, t); + writer->Flush(); // one chunk per frame + } + writer->Stop(); + return cfg.output_filepath; + } + + // Reproduce the on-disk crash state: [header + chunks] with NO footer, plus a + // ".recovery" journal listing the chunks. `truncate_extra` shaves extra bytes + // off the last chunk to simulate a torn final write. + std::vector MakeCrashState(const std::string& path, int64_t truncate_extra = 0) { + std::vector records; + int64_t last_chunk_end = 0; + { + auto ctx = VTX::OpenReplayFile(path); + EXPECT_TRUE(ctx) << ctx.error; + for (const auto& e : ctx.reader->GetSeekTable()) { + VTX::ChunkIndexData d; + d.chunk_index = e.chunk_index; + d.start_frame = e.start_frame; + d.end_frame = e.end_frame; + d.file_offset = static_cast(e.file_offset); + d.chunk_size_bytes = e.chunk_size_bytes; + d.checksum = e.checksum; + records.push_back(d); + last_chunk_end = + std::max(last_chunk_end, static_cast(e.file_offset) + e.chunk_size_bytes); + } + } + // Truncate off the footer (and optionally part of the last chunk). + std::filesystem::resize_file(path, static_cast(last_chunk_end - truncate_extra)); + // Re-create the recovery journal exactly as the sink would have left it. + VTX::RecoveryJournal journal; + EXPECT_TRUE(journal.Open(VTX::RecoveryJournal::PathFor(path), "VTXF", true)); + for (const auto& d : records) + journal.AppendChunk(d); + journal.Close(); + return records; + } + +} // namespace + +// A crash after N committed chunks (footerless file + journal) recovers all N. +TEST(CrashRecovery, RecoversAllChunksAfterCrash) { + const std::string path = WriteValidFile("all", 3); + MakeCrashState(path); + + const std::string journal_path = VTX::RecoveryJournal::PathFor(path); + ASSERT_TRUE(std::filesystem::exists(journal_path)); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_TRUE(rr.repaired); + EXPECT_EQ(rr.recovered_chunks, 3); + EXPECT_EQ(rr.recovered_frames, 3); + EXPECT_FALSE(std::filesystem::exists(journal_path)); // deleted after repair + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + EXPECT_EQ(ctx.reader->GetTotalFrames(), 3); + const VTX::Frame* f = ctx.reader->GetFrameSync(2); + ASSERT_NE(f, nullptr); + ASSERT_EQ(f->GetBuckets().size(), 1u); + ASSERT_EQ(f->GetBuckets()[0].entities.size(), 1u); + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 2); // Score of frame 2 +} + +// A torn final chunk (bytes missing) is dropped; the earlier chunks are recovered. +TEST(CrashRecovery, DropsTornTailChunk) { + const std::string path = WriteValidFile("torn", 3); + const auto records = MakeCrashState(path, /*truncate_extra=*/0); + // Shave the last chunk so its recorded extent runs past EOF. + const int64_t mid_last = records.back().file_offset + 4; // just past the size prefix + std::filesystem::resize_file(path, static_cast(mid_last)); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_TRUE(rr.repaired); + EXPECT_EQ(rr.recovered_chunks, 2); // last chunk dropped + EXPECT_EQ(rr.recovered_frames, 2); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + EXPECT_EQ(ctx.reader->GetTotalFrames(), 2); +} + +// A chunk whose bytes were corrupted (checksum mismatch) stops recovery at it. +TEST(CrashRecovery, ChecksumDetectsCorruptChunk) { + const std::string path = WriteValidFile("corrupt", 3); + const auto records = MakeCrashState(path); + // Flip a byte inside chunk 1's payload (after its 4-byte size prefix). + { + std::fstream f(path, std::ios::binary | std::ios::in | std::ios::out); + ASSERT_TRUE(f.is_open()); + const std::streamoff pos = static_cast(records[1].file_offset + 5); + char c = 0; + f.seekg(pos); + f.read(&c, 1); + c = static_cast(c ^ 0xFF); + f.seekp(pos); + f.write(&c, 1); + f.flush(); + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_chunks, 1); // chunk 0 only; chunk 1 fails checksum -> stop + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + EXPECT_EQ(ctx.reader->GetTotalFrames(), 1); +} + +// A cleanly-closed file has no journal, so repair is a no-op and the file is intact. +TEST(CrashRecovery, CleanFileNeedsNoRepair) { + const std::string path = WriteValidFile("clean", 3); + ASSERT_FALSE(std::filesystem::exists(VTX::RecoveryJournal::PathFor(path))); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_TRUE(rr.was_clean); + EXPECT_FALSE(rr.repaired); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + EXPECT_EQ(ctx.reader->GetTotalFrames(), 3); +} From a5ac4644a6a3092417961dbdd350cd2e079f5043 Mon Sep 17 00:00:00 2001 From: Alejandro Canela Date: Fri, 17 Jul 2026 12:03:38 +0200 Subject: [PATCH 3/6] feat : recovery fixes --- .../vtx/writer/core/vtx_replay_recovery.h | 10 +-- .../vtx/writer/policies/sinks/file_sink.h | 10 ++- .../writer/policies/sinks/recovery_journal.h | 10 ++- .../vtx/writer/core/vtx_replay_recovery.cpp | 41 ++++++++++ tests/writer/test_crash_recovery.cpp | 81 +++++++++++++++++++ 5 files changed, 142 insertions(+), 10 deletions(-) diff --git a/sdk/include/vtx/writer/core/vtx_replay_recovery.h b/sdk/include/vtx/writer/core/vtx_replay_recovery.h index 9baada0..b13d988 100644 --- a/sdk/include/vtx/writer/core/vtx_replay_recovery.h +++ b/sdk/include/vtx/writer/core/vtx_replay_recovery.h @@ -19,11 +19,11 @@ namespace VTX { struct RepairResult { - bool was_clean = false; ///< No recovery journal present -> nothing to repair. - bool repaired = false; ///< A footer was reconstructed and written. - int32_t recovered_chunks = 0; ///< Chunks preserved in the recovered file. - int32_t recovered_frames = 0; ///< total_frames of the recovered file. - std::string error; ///< Non-empty on failure. + bool was_clean = false; ///< No recovery journal present -> nothing to repair. + bool repaired = false; ///< A footer was reconstructed and written. + int32_t recovered_chunks = 0; ///< Chunks preserved in the recovered file. + int32_t recovered_frames = 0; ///< total_frames of the recovered file. + std::string error; ///< Non-empty on failure. bool ok() const { return error.empty(); } }; diff --git a/sdk/include/vtx/writer/policies/sinks/file_sink.h b/sdk/include/vtx/writer/policies/sinks/file_sink.h index 593c69e..4ac0c44 100644 --- a/sdk/include/vtx/writer/policies/sinks/file_sink.h +++ b/sdk/include/vtx/writer/policies/sinks/file_sink.h @@ -25,8 +25,8 @@ namespace VTX { HeaderType header_config; bool b_use_compression = true; int8_t compression_level = 10; - bool durable_writes = true; ///< fsync each chunk to physical disk (crash/power-loss safe). - bool enable_recovery_journal = true; ///< maintain a ".recovery" sidecar for crash recovery. + bool durable_writes = true; ///< fsync each chunk to physical disk (crash/power-loss safe). + bool enable_recovery_journal = true; ///< maintain a ".recovery" sidecar for crash recovery. }; explicit ChunkedFileSink(Config config) @@ -47,6 +47,8 @@ namespace VTX { file_.Write(header_payload.data(), final_size); if (config_.durable_writes) file_.Sync(); + else + file_.Flush(); // process-crash safe (reaches the OS) even without fsync // Start the crash-recovery journal only once the header is durable. if (config_.enable_recovery_journal) { @@ -70,6 +72,8 @@ namespace VTX { file_.Write(payload.data(), final_size); if (config_.durable_writes) file_.Sync(); + else + file_.Flush(); // process-crash safe (reaches the OS) even without fsync ChunkIndexData indexEntry; indexEntry.chunk_index = chunkIndex_++; @@ -98,6 +102,8 @@ namespace VTX { WriteBlob(SerializerPolicy::GetMagicBytes()); if (config_.durable_writes) file_.Sync(); + else + file_.Flush(); // process-crash safe (reaches the OS) even without fsync // Clean shutdown: the footer is durable, so the recovery journal is no // longer needed. Its absence signals a clean file to the repair path. diff --git a/sdk/include/vtx/writer/policies/sinks/recovery_journal.h b/sdk/include/vtx/writer/policies/sinks/recovery_journal.h index e8df07f..ef18060 100644 --- a/sdk/include/vtx/writer/policies/sinks/recovery_journal.h +++ b/sdk/include/vtx/writer/policies/sinks/recovery_journal.h @@ -56,6 +56,8 @@ namespace VTX { file_.Write(fmt, 4); if (durable_) file_.Sync(); + else + file_.Flush(); // process-crash safe (reaches the OS) even without fsync return true; } @@ -76,6 +78,8 @@ namespace VTX { file_.Write(rec, kRecordSize); if (durable_) file_.Sync(); + else + file_.Flush(); // process-crash safe (reaches the OS) even without fsync } void Close() { file_.Close(); } @@ -83,9 +87,9 @@ namespace VTX { // --- Read side (repair) --- struct Parsed { - bool has_journal = false; ///< The sidecar file existed. - bool header_valid = false; ///< Header magic/version parsed OK. - std::string format_magic; ///< Main-file magic recorded in the header. + bool has_journal = false; ///< The sidecar file existed. + bool header_valid = false; ///< Header magic/version parsed OK. + std::string format_magic; ///< Main-file magic recorded in the header. std::vector chunks; ///< Valid committed-chunk records, in order. }; diff --git a/sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp b/sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp index eeff5a1..519c860 100644 --- a/sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp +++ b/sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,28 @@ namespace VTX { return ec ? -1 : static_cast(s); } + // True if the file already ends with a well-formed footer trailer + // [u32 footer_size][4-byte magic]. A cleanly-closed file ends this way; a + // crash-truncated (footerless) file almost never does. Used to detect a + // complete file with only a leftover journal, so we DON'T rewrite its footer. + bool HasValidTrailingFooter(const std::string& path, const std::string& magic, int64_t file_size) { + if (file_size < 8) + return false; + std::ifstream in(path, std::ios::binary); + if (!in.is_open()) + return false; + in.seekg(file_size - 8); + char buf[8]; + in.read(buf, 8); + if (in.gcount() != 8) + return false; + uint32_t footer_size = 0; + std::memcpy(&footer_size, buf, sizeof(footer_size)); // host-native, matches the sink + if (std::string(buf + 4, 4) != magic) + return false; + return footer_size > 0 && static_cast(footer_size) + 8 <= file_size; + } + // Byte offset where chunks begin (magic(4) + u32 header_size + header), or -1 if // the header framing is not intact. int64_t ReadHeaderEnd(std::ifstream& in, const std::string& expect_magic, int64_t file_size) { @@ -149,6 +172,24 @@ namespace VTX { in.close(); const std::string m(magic, 4); + const int64_t file_size = FileSizeOf(path); + + // A cleanly-finished file already ends with a valid footer; the journal is + // just leftover (crash between the footer fsync and the journal delete). + // Preserve the complete footer (incl. per-frame times) -- only drop the journal. + if (HasValidTrailingFooter(path, m, file_size)) { + std::remove(journal_path.c_str()); + r.was_clean = true; + return r; + } + + // Refuse a journal whose recorded format does not match this file (e.g. a + // stale sidecar left over a file that was replaced with the other format). + if (journal.header_valid && !journal.format_magic.empty() && journal.format_magic != m) { + r.error = "recovery journal format (" + journal.format_magic + ") does not match file (" + m + ")"; + return r; + } + if (m == "VTXP") return RepairImpl(path, journal); if (m == "VTXF") diff --git a/tests/writer/test_crash_recovery.cpp b/tests/writer/test_crash_recovery.cpp index 45e2770..beb3300 100644 --- a/tests/writer/test_crash_recovery.cpp +++ b/tests/writer/test_crash_recovery.cpp @@ -168,6 +168,87 @@ TEST(CrashRecovery, ChecksumDetectsCorruptChunk) { EXPECT_EQ(ctx.reader->GetTotalFrames(), 1); } +// A crash between the footer fsync and the journal delete leaves a COMPLETE file +// plus a leftover .recovery. Repair must detect the valid footer and preserve it +// (incl. timing) rather than truncating and rewriting a timing-less one. +TEST(CrashRecovery, PreservesValidFooterWhenJournalLeftover) { + const std::string path = WriteValidFile("leftover", 3); + + // Recreate the leftover journal WITHOUT truncating the (valid) footer. + { + std::vector records; + { + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + for (const auto& e : ctx.reader->GetSeekTable()) { + VTX::ChunkIndexData d; + d.chunk_index = e.chunk_index; + d.start_frame = e.start_frame; + d.end_frame = e.end_frame; + d.file_offset = static_cast(e.file_offset); + d.chunk_size_bytes = e.chunk_size_bytes; + d.checksum = e.checksum; + records.push_back(d); + } + } + VTX::RecoveryJournal journal; + ASSERT_TRUE(journal.Open(VTX::RecoveryJournal::PathFor(path), "VTXF", true)); + for (const auto& d : records) + journal.AppendChunk(d); + journal.Close(); + } + + const auto size_before = std::filesystem::file_size(path); + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_TRUE(rr.was_clean); // detected an already-complete file + EXPECT_FALSE(rr.repaired); // did NOT rewrite the footer + EXPECT_EQ(std::filesystem::file_size(path), size_before); // footer left untouched + EXPECT_FALSE(std::filesystem::exists(VTX::RecoveryJournal::PathFor(path))); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + EXPECT_EQ(ctx.reader->GetTotalFrames(), 3); +} + +// A journal whose recorded format magic disagrees with the main file (e.g. a stale +// sidecar left over a file replaced with the other format) must be refused, not +// applied (which would truncate a valid file). +TEST(CrashRecovery, RefusesMismatchedJournalFormat) { + const std::string path = WriteValidFile("mismatch", 2); + + std::vector records; + int64_t last_end = 0; + { + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + for (const auto& e : ctx.reader->GetSeekTable()) { + VTX::ChunkIndexData d; + d.chunk_index = e.chunk_index; + d.start_frame = e.start_frame; + d.end_frame = e.end_frame; + d.file_offset = static_cast(e.file_offset); + d.chunk_size_bytes = e.chunk_size_bytes; + d.checksum = e.checksum; + records.push_back(d); + last_end = std::max(last_end, static_cast(e.file_offset) + e.chunk_size_bytes); + } + } + std::filesystem::resize_file(path, static_cast(last_end)); // footerless + { + VTX::RecoveryJournal journal; + // Wrong format magic: the file is VTXF (FlatBuffers). + ASSERT_TRUE(journal.Open(VTX::RecoveryJournal::PathFor(path), "VTXP", true)); + for (const auto& d : records) + journal.AppendChunk(d); + journal.Close(); + } + + const auto rr = VTX::RepairReplayFile(path); + EXPECT_FALSE(rr.ok()); // refused: journal format does not match the file + EXPECT_FALSE(rr.repaired); +} + // A cleanly-closed file has no journal, so repair is a no-op and the file is intact. TEST(CrashRecovery, CleanFileNeedsNoRepair) { const std::string path = WriteValidFile("clean", 3); From 9811aac02acc869448cdefe8e55a57df4cfac11f Mon Sep 17 00:00:00 2001 From: Alejandro Canela Date: Fri, 17 Jul 2026 18:46:43 +0200 Subject: [PATCH 4/6] feat : recovery replay completed --- .gitignore | 1 + CHANGELOG.md | 7 + README.md | 1 + benchmarks/CMakeLists.txt | 4 + docs/SDK_API.md | 35 + .../vtx/writer/core/vtx_replay_recovery.h | 35 +- sdk/include/vtx/writer/core/writer.h | 12 + .../vtx/writer/policies/sinks/durable_file.h | 118 +- .../vtx/writer/policies/sinks/file_sink.h | 58 +- .../vtx/writer/policies/sinks/network_sink.h | 6 + .../writer/policies/sinks/recovery_journal.h | 491 +++- .../vtx/writer/core/vtx_replay_recovery.cpp | 174 +- tests/writer/test_crash_recovery.cpp | 2450 ++++++++++++++++- 13 files changed, 3121 insertions(+), 271 deletions(-) diff --git a/.gitignore b/.gitignore index 4b4d8b8..4b1582d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Build output build/ +build-asan/ build-bench/ dist/ out/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a2d2c5d..28a4246 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **reader/api**: **bucket names restored on read** -- bucket names never hit the wire (both formats serialize buckets positionally), so deserialized frames used to come back with an empty `bucket_map`. The reader now stamps `bucket_map` from the embedded schema's `"buckets"` array when chunks are deserialized, so by-name lookups (`Frame::GetBucket(name)` const) and `bucket_map` iteration work on read frames. Replays written without a `"buckets"` array keep the old positional-only behavior - **common/schema**: **array & map pre-sizing** -- extends the existing scalar pre-sizing (`type_max_indices`) so declared *array* and *map* fields are also materialized on load. `SchemaStruct` / `StructSchemaCache` gain `array_max_indices` (max Array-container index per `FieldType`) and `map_max_index` (count of Struct-valued map fields); `Helpers::ResizeContainerToMaxIndices` now pre-creates one empty subarray per declared array field in the matching `FlatArray` (via new `FlatArray::EnsureSubArrayCount`) and pre-sizes `map_properties`. A declared-but-unpopulated array/map field is now present-and-empty instead of absent, symmetric with scalars. Applies wherever scalar pre-sizing already ran (the four frame loaders + `schema_dynamic_loader`); the loader CRTP hook `GetTypeMaxIndices` became `GetStructSizing` (returns the `StructSchemaCache`) - **reader/api**: **declared-empty arrays restored on read** -- an array with no data is not serialized (nothing to store), so deserialized entities used to come back missing those subarrays even for schema-declared array fields (maps already round-trip their slot count). The reader now re-creates the declared-but-empty subarrays from the embedded schema (`Helpers::EnsureDeclaredArrays`, grow-only, recursing into nested structs / struct-array elements / map values) so a read frame mirrors the array layout of an ingest-loaded frame. Arrays that carry data round-trip unchanged (their `offsets` already encode empty subarrays); scalars and maps are untouched +- **writer/durability**: **crash recovery for the file sink** -- a recording that dies before `Stop()` (process crash or power loss, mid-chunk or mid-frame, before the body/footer are closed) can now be recovered up to the last recorded frame instead of losing the whole session. Three cooperating pieces, all file-sink only (the network sink accepts the per-frame hook as a no-op): + - **per-chunk integrity + durability** -- `ChunkedFileSink::Config` gains `durable_writes` (default **on**: `fsync`/`_commit` each chunk and journal record to physical media via the new `DurableFile` FILE* wrapper; set **off** to only `fflush` to the OS -- survives a process crash but not power loss) and `enable_recovery_journal` (default **on**). Every chunk carries an **xxHash64 `checksum`** of its on-disk payload, added to `ChunkIndexEntry` / `ChunkIndexData` and to both the FlatBuffers (`vtx_schema.fbs`) and Protobuf (`vtx_schema.proto`, field 6) footer schemas, so a torn or corrupt chunk is detectable on recovery. The reader parses and exposes it; FlatBuffers header/footer parsing gained a `flatbuffers::Verifier` guard so a truncated footer is rejected cleanly instead of reading out of bounds + - **recovery journal** -- a single write-ahead sidecar **`.recovery`** (`sdk/include/vtx/writer/policies/sinks/recovery_journal.h`) holds a typed, self-validating record stream (each record `[u8 type][u32 len][payload][u64 xxHash64]`, so a torn last record from a mid-write crash is detected and dropped): an `S` record (written once) carries the writer's timing parameters `{fps, is_increasing}`, `C` records commit a durable chunk's index entry, `T` records carry each committed frame's exact `{game_time, created_utc}`, and `F` records carry each in-flight (un-flushed) frame's times + serialized payload. The log is strictly **append-only** in the crash-critical path -- a frame appends an `F` record; a flush appends the chunk's `C` + `T` records -- so at every instant the file is a valid prefix and there is no window in which a fsync'd chunk is described by neither its `F` records nor its `C` record. Ordering is **data-before-journal**: a chunk is fsync'd into the `.vtx` first, then its `C`/`T` records are appended and fsync'd. A committed chunk's now-redundant `F` records (repair dedups them by frame index) are reclaimed by periodic **compaction** -- the log is rewritten (all `C`/`T` + only the still-pending `F`) into a temp file that **atomically replaces** the sidecar, so a crash during compaction leaves either the old or the new complete journal. The writer journals every frame as it is recorded (`ReplayWriter::TryRecordFrame` -> new `SinkPolicy::JournalFrame` hook; timing via `SinkPolicy::JournalTiming`), so a crash between flushes still recovers the pending batch. If the journal cannot be opened cleanly, journaling is disabled and the torn sidecar is removed (a half-written journal would otherwise block repair). On a clean `Close()` the footer is written and the `.recovery` is deleted -- its presence at open time signals an unclean shutdown + - **repair** -- **`VTX::RepairReplayFile(path)`** (`sdk/include/vtx/writer/core/vtx_replay_recovery.h`) reconstructs a valid, readable file from a footerless `.vtx` + its journal: it verifies each committed chunk's checksum (dropping a torn/corrupt tail), re-appends the in-flight frames as chunks, rebuilds the seek table and the **exact per-frame times**, and synthesizes the footer -- including the **derived time data** (`duration_seconds`, timeline **gaps**, game **segments**), reconstructed from the journaled timing (`S` record) with the same expressions `VTXGameTimes` uses, so a recovered footer matches a clean `Stop()` **exactly** (manual segment marks are the one thing not journaled). Recovery is deliberately **not automatic on open** -- the user-driven flow is `ReplayNeedsRecovery(path)` (cheap sidecar check) / `RecoveryJournalPath(path)` (locate the sidecar) then `RepairReplayFile(path)`. A file that already ends with a valid footer plus a leftover journal (a crash between the footer fsync and the journal delete) is detected and preserved untouched; a journal whose recorded format magic disagrees with the file, or whose header is unreadable, is refused rather than applied; a **recording still being written is refused** without touching it (on Windows the writer holds a deny-write handle, so repair's truncate fails cleanly). Returns a `RepairResult` (`was_clean` / `repaired` / `recovered_chunks` / `recovered_frames` / `error`). Both FlatBuffers and Protobuf files are supported +- **tests**: **`tests/writer/test_crash_recovery.cpp`** -- 65 cases covering the full crash-window matrix: recovery between chunks (single- and multi-frame), between frames (in-flight batch, and nothing-flushed-yet), a torn tail chunk, a partial chunk written before its journal record, a checksum-detected corrupt chunk, a torn pending-frame record, a **torn chunk-commit record** (the batch falls back to its still-present `F` records -- the append-only guarantee), a crash mid-footer-write, a leftover-journal-over-valid-footer, a **stale compaction temp**, a mismatched journal format, a clean file (no-op), a header-only crash (0-frame result), the lingering-`F`-record dedup guard, compaction reclaim, the recovery helpers, exact per-frame time preservation, a **live-recording repair refusal** (Windows), and the FlatBuffers + Protobuf repair paths. Crash states are fabricated byte-faithfully through the same `RecoveryJournal` API the sink uses, plus **end-to-end tests through the real writer** (a raw `ReplayWriter` dropped without `Stop()`) for both formats that require the recovered footer to match a cleanly-stopped control **exactly**: per-frame `game_time` + `created_utc`, `duration_seconds`, timeline gaps, game segments, and per-entity `content_hash` -- and require the recovered file to pass whole-replay `ValidateReplayFile` and cache-hostile out-of-order seeks across the committed-chunk/recovered-chunk boundary. Also covered: the sink's **full config matrix** (`durable_writes` x `b_use_compression`) with frames large enough that **zstd compression genuinely engages** (committed chunks and journaled `F` payloads take the compressed path), a **repair interrupted mid-run and re-run** (idempotent), a **second repair on an already-repaired file** (clean no-op, byte-identical), a **journal-opted-out crash** (repair reports clean and leaves the footerless file byte-identical; the reader rejects it gracefully), a **session-start crash** (torn `S` record + header-only `.vtx` -> valid 0-frame file), **torn `T` records** (frame times fall back to the still-present `F` records -- exact, not zeroed), a **0-byte journal** (refused, main file untouched), **compaction through the real sink** under crash (new `Config::journal_compact_threshold_bytes`, 0 = default), **map-container frames** recovered intact from `F` records, a **non-ASCII path** through the full sidecar flow, **hostile checksummed journal records** (a `C` offset inside the header region, a `C` size at or below its own length prefix, an `F` with an empty payload -- each dropped safely, degrading to a valid file), a **torn main-file header** alongside a valid journal (refused, bytes untouched), and a **decreasing-time recording** (`is_increasing = false` -- the other branch of the segment reconstruction) matching its clean control. Scale + pipeline coverage: a **stress run** (2030 multi-entity frames, 40 committed chunks + a 30-frame in-flight batch -- every timestamp exact, whole-replay validation clean), **rejected frames interleaved mid-recording** (timer rejections leave no trace and do not desync the journal's frame indexing), and a **post-processor recording** (journaled `F` records capture the frame as it would hit disk -- post-mutation -- proven by a pending frame that only ever existed in the journal). Lifecycle + hygiene: a **double-crash lifecycle** over the same path (crash -> repair -> re-record -> crash -> repair, second recovery reflects only the second session), a **stale sidecar under journaling opt-out** (a session with `enable_recovery_journal = false` removes any leftover `.recovery` at start so it cannot masquerade as recovery state), a **foreign `.recovery` file** (no `VTXR` magic -- never deleted by repair, even on the clean-file path), a **hostile out-of-range `T` record** (ignored by the bounds check), the **map crash-recovery path on Protobuf** as well as FlatBuffers, a **stale journal from a different recording** over a replaced file (per-chunk checksums reject the foreign chunks -- graceful degradation to a valid empty file), a **read-only crashed file** (repair refuses without consuming the journal; succeeds untouched once the file is writable), a **no-time-registry recording** (`EGameTimeType::None`, fully FPS-synthesized timeline recovered exactly), and **byte-budget chunk splitting** (`max_bytes`, the other `ThresholdChunkPolicy` branch) through crash + recovery. The maximal guarantee is pinned by **`BoundaryCrashRecoversByteIdenticalFile`** (FlatBuffers **and** Protobuf, plus a 120-frame large-footer variant that exercises the compressed-footer path): a crash exactly at a chunk boundary recovers to a file that is **bit-for-bit identical** to a clean `Stop()` with the same inputs -- repair passes the footer's time vectors exactly as `Stop()` does (present-but-empty rather than absent) and compresses the synthesized footer exactly as the sink's `Close()` would (the `S` record now also journals `{use_compression, compression_level}`; journal version 4). A journal with an **incompatible version field** is refused outright, main file untouched. On top of the hand-picked windows, a **brute-force sweep suite** (`CrashRecoverySweep`) proves the "any crash point" claim literally: the journal truncated at **every byte length**, the main `.vtx` truncated at **every byte length**, and **every journal byte flipped** one at a time -- thousands of repair runs, none may crash, and every claimed repair must open and agree with its own reported frame count and footer times. `ReadValid` now bounds each record's payload allocation by the journal's **actual file size** (a corrupt length field can no longer trigger a giant transient allocation), pinned by `HostileRecordLengthIsBoundedByFileSize`; the truncation sweep also runs over the **post-compaction journal layout**. Finally, `CrashRecoveryProcess` (Windows) performs a **genuine process kill**: vtx_tests spawns itself as a child in a hidden endless-writer mode and `TerminateProcess`es it mid-recording -- no destructors, no fclose, handles abandoned to the kernel -- then repairs and verifies frame content and exact footer times, in **both** durability modes (this is the honest validation of the flush-only claim that data handed to the OS survives a process death) -- plus a kill under an **aggressive compaction cadence** (a full close/rewrite/atomic-rename cycle per commit), so real process death lands amid compaction traffic and the atomic-rename guarantee holds, and a **varied-progress kill soak** (kills landing just after the first journaled frames, around the first flush, amid compaction cycles, and several chunks deep -- alternating durability modes). The recovered output was also smoke-checked through the end-user toolchain: `vtx_cli` opens a repaired file and serves `info` / `footer` / `frame` normally. `RecoveredFileTranscodesToCleanChunks` pins the **normalization workflow** for salvaged files (open -> drain frames with footer times -> re-record): the one-frame recovery chunks become proper chunks, `created_utc` round-trips exactly, and `game_time` drifts at most 1 tick (100 ns) per frame through the float-seconds register -- the documented transcode caveat. `BM_ReaderRecoveredTail` (benchmarks) measures the salvage cost that motivates it: a 200-frame one-frame-chunk tail reads sequentially ~3x slower than clean chunking +- **benchmarks**: new **`benchmarks/bench_crash_recovery.cpp`** (`BM_WriterDurabilityTier`) quantifies the writer hot-path cost of the recovery defaults across the three durability tiers (journal off / journal + flush-only / journal + fsync). Reference numbers on an NVMe dev machine, 200 small frames per iteration: **~19 us/frame** with no journal, **~30 us/frame** flush-only (process-crash safe -- effectively free), **~289 us/frame** with the default per-operation fsync (power-loss safe; ~1.7% of a 60 fps frame budget on NVMe -- on spinning disks prefer `durable_writes = false`). The byte-identity tests now compare from the end of the header on (the header embeds a second-granularity recording timestamp, so two separately recorded sessions may legitimately differ there; repair never rewrites the header) +- **docs**: **`docs/SDK_API.md`** "Writing Replays" gains a **"Crash recovery"** section -- the `.recovery` sidecar model, the user-driven `ReplayNeedsRecovery` / `RepairReplayFile` / `RecoveryJournalPath` flow with `RepairResult` semantics, the exact-times guarantee, the refusal cases, and the sink durability knob table (`durable_writes` / `enable_recovery_journal` / `journal_compact_threshold_bytes`). **`README.md`** feature list gains a "Crash-safe recording" bullet. `DurableFile` hardening: `Seek`/`SeekEnd` failures now latch `Good()` false (a write after an unnoticed seek failure would land at the wrong offset), and the dead `Truncate` primitive left over from the pre-append-only journal design was removed ### Changed diff --git a/README.md b/README.md index f9999a4..4466a22 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ VTX is an open binary format for real-time per-frame state data, plus a C++20 SD - **Random access by frame or timestamp.** Footer-indexed seek, not a linear scan. O(log n) lookup plus one chunk decompress. - **Both Protobuf and FlatBuffers.** SDK supports both backends out of the box. The file announces which in its magic bytes; readers auto-detect. - **Live streaming transports.** Ingest frames live from an OS pipe (Windows named pipe, POSIX FIFO, stdin) or a WebSocket connection, and emit the `.vtx` byte stream over a TCP socket instead of writing to disk. Same writer API behind each transport -- file or network, sink-agnostic. +- **Crash-safe recording.** A write-ahead `.recovery` sidecar (checksummed, append-only, fsync'd per operation by default) makes a recording that dies before `Stop()` recoverable up to the last recorded frame -- exact per-frame timestamps included. `RepairReplayFile()` reconstructs the footer; a hours-long session that crashes is no longer lost. - **Validation & structured diagnostics.** Validate a schema, entity, frame, or whole replay independently; strict recording rejects frames observably (bad game-time, unresolved entity type). Every failure is a structured `VtxError` -- code, severity, location, expected-vs-provided type -- so automation acts on data, not parsed log lines. - **Engine-independent C++20.** No engine dependency. Language bindings wherever Protobuf or FlatBuffers exist (Python, Go, Rust, Java, JS). - **Open.** Apache-2.0. Spec, reference reader, and tooling all in the repo. diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index f56d894..1d03045 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -53,6 +53,7 @@ endif() # static init. `benchmark::benchmark_main` provides the main() that calls # Initialize + RunSpecifiedBenchmarks + Shutdown. add_executable(vtx_benchmarks + bench_crash_recovery.cpp bench_reader.cpp bench_reader_ready.cpp bench_writer.cpp @@ -79,6 +80,9 @@ target_include_directories(vtx_benchmarks PRIVATE # vtx_common keeps protobuf / flatbuffers headers PRIVATE; benchmarks # that touch PropertyContainer internals need the same thirdparty paths. "${PROJECT_SOURCE_DIR}/thirdparty/protobuf/include" + # bench_crash_recovery.cpp instantiates the raw ReplayWriter/ChunkedFileSink, + # which needs the generated schema headers (same as tests/). + "${VTX_COMMON_GENERATED_DIR}" ) target_compile_definitions(vtx_benchmarks PRIVATE diff --git a/docs/SDK_API.md b/docs/SDK_API.md index 8177d96..e4873bc 100644 --- a/docs/SDK_API.md +++ b/docs/SDK_API.md @@ -255,6 +255,41 @@ writer->Stop(); // Write footer and close file If the facade is destroyed without an explicit `Stop()`, its destructor finalizes the replay as a best-effort fallback, so a dropped writer still yields a readable `.vtx`. +### Crash recovery + +A recording that dies before `Stop()` -- process crash or power loss, mid-chunk or mid-frame -- is **recoverable up to the last recorded frame**. While recording, the file sink maintains a write-ahead sidecar **`.vtx.recovery`**: every recorded frame is journaled before it joins the pending batch, and every flushed chunk is committed to the journal *after* its bytes are durable in the `.vtx` (data-before-journal ordering, append-only log, per-record checksums). On a clean `Stop()` the sidecar is deleted -- its presence at open time is the signal of an unclean shutdown. + +Recovery is **never automatic**; the flow is user-driven: + +```cpp +#include "vtx/writer/core/vtx_replay_recovery.h" + +if (VTX::ReplayNeedsRecovery(path)) { // cheap: does ".recovery" exist? + VTX::RepairResult r = VTX::RepairReplayFile(path); + if (r.ok()) { + // r.was_clean -> the file was already complete (leftover sidecar cleaned up) + // r.repaired -> a footer was reconstructed + // r.recovered_chunks / r.recovered_frames + } else { + // r.error -- the journal is left in place, so a retry (or a copy of the + // file made writable, freed disk space, ...) can still recover. + } +} +auto ctx = VTX::OpenReplayFile(path); // a repaired file opens like any other +``` + +`RepairReplayFile` verifies each committed chunk against its journaled xxHash64 checksum (dropping a torn or corrupt tail), re-appends the in-flight frames that only existed in the journal, and synthesizes the footer with the **exact per-frame times** -- `game_time`, `created_utc`, duration, timeline gaps and game segments all match what a clean `Stop()` would have written (at a chunk boundary the recovered file is byte-identical to a cleanly closed one). Both FlatBuffers and Protobuf files are supported; a journal whose format or version does not match the file is refused, a recording still being written is refused (the writer holds a deny-write handle on Windows), and a `.recovery` file that is not actually a journal is never deleted. `RecoveryJournalPath(path)` returns the sidecar's path if you need to locate or archive it. + +One property of salvaged files to know about: the in-flight frames that only existed in the journal are re-appended as **one-frame chunks**, so a recovery with a large pending batch reads sequentially ~3x slower than a cleanly chunked file (measured in `BM_ReaderRecoveredTail`). For hot-path use, transcode the salvaged file with the public API -- open it, drain every frame with its footer times into a fresh writer, `Stop()`. `created_utc` round-trips exactly; `game_time` re-enters through the float-seconds register and can lose 1 tick (100 ns) per frame. + +Durability knobs on the file sink (`ChunkedFileSink::Config`; facade users currently get the defaults): + +| Knob | Default | Meaning | +|------|---------|---------| +| `durable_writes` | `true` | fsync every chunk and journal record to physical media -- survives **power loss**. Set `false` to only flush to the OS: cheaper, still survives a **process crash**. | +| `enable_recovery_journal` | `true` | Maintain the `.recovery` sidecar. Opting out removes any stale sidecar at session start; a crash then leaves an unrecoverable (footerless) file. | +| `journal_compact_threshold_bytes` | `0` (64 MB) | How many superseded journal bytes accrue before the sidecar is compacted (rewritten via an atomic rename). | + ### One call: `WriteReplay` When you already have an `IFrameDataSource` and just want a finished `.vtx`, `WriteReplay` runs the whole pipeline in one call -- create the writer, initialize the source, drain every frame through `TryRecordFrame`, finalize, and report: diff --git a/sdk/include/vtx/writer/core/vtx_replay_recovery.h b/sdk/include/vtx/writer/core/vtx_replay_recovery.h index b13d988..ff8bda4 100644 --- a/sdk/include/vtx/writer/core/vtx_replay_recovery.h +++ b/sdk/include/vtx/writer/core/vtx_replay_recovery.h @@ -3,11 +3,21 @@ * @brief Repair a .vtx file whose writing crashed before the footer was written. * * @details When the writer's recovery journal is enabled, a sidecar ".recovery" - * records every committed chunk. If the process crashes before Close() writes the - * footer (and deletes the sidecar), RepairReplayFile() reconstructs a valid footer - * from the journal: it drops any torn tail chunk, verifies each surviving chunk's - * checksum, appends a synthesized footer, and removes the sidecar. The result is a - * normal .vtx that the standard reader opens with no special handling. + * records every committed chunk and every in-flight frame. If the process crashes + * before Close() writes the footer (and deletes the sidecar), RepairReplayFile() + * reconstructs a valid footer from the journal: it drops any torn tail chunk, verifies + * each surviving chunk's checksum, re-appends the in-flight frames, rebuilds the exact + * per-frame times, and removes the sidecar. The result is a normal .vtx that the + * standard reader opens with no special handling. + * + * Recovery is deliberately NOT automatic: opening a replay never repairs it behind the + * caller's back. The intended flow is user-driven -- + * + * if (VTX::ReplayNeedsRecovery(path)) { + * const auto r = VTX::RepairReplayFile(path); + * // inspect r (was_clean / repaired / recovered_frames / error) and decide + * } + * auto ctx = VTX::OpenReplayFile(path); * * @author Zenos Interactive */ @@ -28,6 +38,21 @@ namespace VTX { bool ok() const { return error.empty(); } }; + /** + * @brief The recovery sidecar path for a given main .vtx path (".recovery"). + * @details Lets callers locate / inspect / delete the sidecar without depending on + * the internal journal header. + */ + std::string RecoveryJournalPath(const std::string& path); + + /** + * @brief Cheap check for whether @p path was left by an unclean shutdown. + * @return true if the ".recovery" sidecar exists. A leftover sidecar over an + * already-complete file still returns true here; RepairReplayFile() makes the + * final determination (and reports was_clean if the footer was in fact intact). + */ + bool ReplayNeedsRecovery(const std::string& path); + /** * @brief Recover a crashed .vtx using its ".recovery" journal. * @param path Path to the (footerless) main .vtx file. diff --git a/sdk/include/vtx/writer/core/writer.h b/sdk/include/vtx/writer/core/writer.h index e3f8c4c..3e3830e 100644 --- a/sdk/include/vtx/writer/core/writer.h +++ b/sdk/include/vtx/writer/core/writer.h @@ -62,6 +62,9 @@ namespace VTX { } else { registry_.LoadFromJson(config.schema_json_path); } + // Journal the resolved timing before the session opens, so a crash-repair can + // reconstruct the footer's derived time data (duration/gaps/segments) exactly. + sink_.JournalTiming(config.default_fps, config.is_increasing); auto schema = Serializer::CreateSchema(registry_); sink_.OnSessionStart(schema); frame_accessor_.InitializeFromCache(registry_.GetPropertyCache()); @@ -137,6 +140,15 @@ namespace VTX { const int32_t assigned_index = total_frames_; total_frames_++; current_chunk_bytes_ += frame_size; + + // Journal the frame (payload + resolved times) before it joins the pending + // batch, so a crash before the next flush can still recover it. + const auto& game_times = timer_.GetGameTime(); + const auto& created_utc = timer_.GetCreatedUtc(); + const int64_t frame_game_time = game_times.empty() ? 0 : game_times.back(); + const int64_t frame_created_utc = created_utc.empty() ? 0 : created_utc.back(); + sink_.JournalFrame(*sink_frame, assigned_index, frame_game_time, frame_created_utc); + pending_frames_.push_back(std::move(sink_frame)); return RecordResult::MadeWritten(assigned_index); } diff --git a/sdk/include/vtx/writer/policies/sinks/durable_file.h b/sdk/include/vtx/writer/policies/sinks/durable_file.h index 44adabc..5800b91 100644 --- a/sdk/include/vtx/writer/policies/sinks/durable_file.h +++ b/sdk/include/vtx/writer/policies/sinks/durable_file.h @@ -17,7 +17,8 @@ #include #ifdef _WIN32 -#include // _commit, _fileno +#include // _commit, _fileno +#include // _SH_DENYWR #else #include // fsync, fileno #endif @@ -33,9 +34,16 @@ namespace VTX { DurableFile& operator=(const DurableFile&) = delete; /// Open for binary read/write, truncating any existing file. Returns false on failure. + /// On Windows the handle denies other WRITERS (readers stay allowed), so a stray + /// repair pass cannot truncate a recording that is still being written. bool Open(const std::string& path) { Close(); +#ifdef _WIN32 + fp_ = _fsopen(path.c_str(), "wb+", _SH_DENYWR); +#else fp_ = std::fopen(path.c_str(), "wb+"); +#endif + good_ = (fp_ != nullptr); return fp_ != nullptr; } @@ -43,66 +51,117 @@ namespace VTX { bool OpenExisting(const std::string& path) { Close(); fp_ = std::fopen(path.c_str(), "rb+"); + good_ = (fp_ != nullptr); return fp_ != nullptr; } bool IsOpen() const { return fp_ != nullptr; } - void Write(const void* data, size_t size) { - if (fp_ && size > 0) { - std::fwrite(data, 1, size, fp_); - } + /// True while no write/seek/tell/sync has failed since the last Open. A caller + /// about to make an irreversible decision on the strength of prior writes (e.g. + /// deleting the recovery journal after writing a footer) must check this first. + bool Good() const { return good_; } + + /// Returns false (and latches Good() false) on a short write -- e.g. a full disk. + bool Write(const void* data, size_t size) { + if (!fp_ || size == 0) + return fp_ != nullptr; + const size_t written = std::fwrite(data, 1, size, fp_); + if (written != size) + good_ = false; + return written == size; } - void Write(const std::string& data) { Write(data.data(), data.size()); } + bool Write(const std::string& data) { return Write(data.data(), data.size()); } - /// Current write position (byte offset from the start of the file). + /// Current write position (byte offset from the start of the file), or 0 on error + /// (which also latches Good() false, since a poisoned offset would corrupt the seek table). uint64_t Tell() { - if (!fp_) + if (!fp_) { + good_ = false; return 0; + } #ifdef _WIN32 - return static_cast(_ftelli64(fp_)); + const long long pos = _ftelli64(fp_); #else - return static_cast(ftello(fp_)); + const off_t pos = ftello(fp_); #endif + if (pos < 0) { + good_ = false; + return 0; + } + return static_cast(pos); } - void Seek(uint64_t offset) { - if (!fp_) - return; + /// Returns false (and latches Good() false) on a failed seek -- a write issued + /// after an unnoticed seek failure would land at the wrong offset. + bool Seek(uint64_t offset) { + if (!fp_) { + good_ = false; + return false; + } #ifdef _WIN32 - _fseeki64(fp_, static_cast(offset), SEEK_SET); + if (_fseeki64(fp_, static_cast(offset), SEEK_SET) != 0) { #else - fseeko(fp_, static_cast(offset), SEEK_SET); + if (fseeko(fp_, static_cast(offset), SEEK_SET) != 0) { #endif + good_ = false; + return false; + } + return true; } - void SeekEnd() { - if (!fp_) - return; + bool SeekEnd() { + if (!fp_) { + good_ = false; + return false; + } #ifdef _WIN32 - _fseeki64(fp_, 0, SEEK_END); + if (_fseeki64(fp_, 0, SEEK_END) != 0) { #else - fseeko(fp_, 0, SEEK_END); + if (fseeko(fp_, 0, SEEK_END) != 0) { #endif + good_ = false; + return false; + } + return true; } /// Push the user-space buffer to the OS (survives a process crash, not power loss). - void Flush() { - if (fp_) - std::fflush(fp_); + bool Flush() { + if (!fp_) { + good_ = false; + return false; + } + if (std::fflush(fp_) != 0) { + good_ = false; + return false; + } + return true; } /// Flush all the way to physical media (survives power loss). - void Sync() { - if (!fp_) - return; - std::fflush(fp_); + bool Sync() { + if (!fp_) { + good_ = false; + return false; + } + if (std::fflush(fp_) != 0) { + good_ = false; + return false; + } #ifdef _WIN32 - _commit(_fileno(fp_)); + if (_commit(_fileno(fp_)) != 0) { + good_ = false; + return false; + } #else - fsync(fileno(fp_)); + if (fsync(fileno(fp_)) != 0) { + good_ = false; + return false; + } #endif + return true; } void Close() { @@ -114,6 +173,7 @@ namespace VTX { private: std::FILE* fp_ = nullptr; + bool good_ = true; ///< Latches false on the first failed write/seek/tell/sync. }; } // namespace VTX diff --git a/sdk/include/vtx/writer/policies/sinks/file_sink.h b/sdk/include/vtx/writer/policies/sinks/file_sink.h index 4ac0c44..06084cf 100644 --- a/sdk/include/vtx/writer/policies/sinks/file_sink.h +++ b/sdk/include/vtx/writer/policies/sinks/file_sink.h @@ -27,6 +27,7 @@ namespace VTX { int8_t compression_level = 10; bool durable_writes = true; ///< fsync each chunk to physical disk (crash/power-loss safe). bool enable_recovery_journal = true; ///< maintain a ".recovery" sidecar for crash recovery. + uint64_t journal_compact_threshold_bytes = 0; ///< journal compaction trigger; 0 = journal default. }; explicit ChunkedFileSink(Config config) @@ -50,13 +51,37 @@ namespace VTX { else file_.Flush(); // process-crash safe (reaches the OS) even without fsync - // Start the crash-recovery journal only once the header is durable. + // Start the crash-recovery journal only once the header is durable. If it + // cannot be opened cleanly (or its own header write failed), disable it and + // remove the torn sidecar -- a half-written journal would later block repair, + // which is worse than recording without one. if (config_.enable_recovery_journal) { - journal_.Open(RecoveryJournal::PathFor(config_.filename), SerializerPolicy::GetMagicBytes(), - config_.durable_writes); + const std::string journal_path = RecoveryJournal::PathFor(config_.filename); + if (!journal_.Open(journal_path, SerializerPolicy::GetMagicBytes(), config_.durable_writes, + journal_fps_, journal_is_increasing_, config_.b_use_compression, + config_.compression_level)) { + journal_.Close(); + std::remove(journal_path.c_str()); + } else if (config_.journal_compact_threshold_bytes > 0) { + journal_.SetCompactThresholdBytes(config_.journal_compact_threshold_bytes); + } + } else { + // Journaling opted out: remove any stale sidecar a previous (crashed) + // session left for this filename, so it cannot masquerade as recovery + // state for THIS recording. + std::remove(RecoveryJournal::PathFor(config_.filename).c_str()); + std::remove(RecoveryJournal::CompactTempFor(RecoveryJournal::PathFor(config_.filename)).c_str()); } } + // Timing parameters the writer resolved from its config; journaled once ('S' + // record) so a repair can reconstruct the footer's derived time data (duration, + // gaps, segments) exactly. Call before OnSessionStart. + void JournalTiming(float fps, bool is_increasing) { + journal_fps_ = fps; + journal_is_increasing_ = is_increasing; + } + void SaveChunk(std::vector>& frames, const std::vector& created_utc, int32_t start_frame, int32_t total_frames) { if (frames.empty()) @@ -84,10 +109,26 @@ namespace VTX { indexEntry.checksum = XXH3_64bits(payload.data(), payload.size()); seek_table_.push_back(indexEntry); - // Journal the committed chunk AFTER its bytes are durable on disk, so - // the journal never references a chunk that isn't there (data-before-journal). - if (journal_.IsOpen()) - journal_.AppendChunk(indexEntry); + // Commit the chunk to the journal AFTER its bytes are durable on disk + // (data-before-journal): drop the now-redundant pending-frame tail and + // record the chunk (C) plus its frames' times (T). + if (journal_.IsOpen()) { + journal_.CommitChunk(indexEntry, batch_times_); + batch_times_.clear(); + } + } + + // Journal a single recorded frame BEFORE it is flushed as part of a chunk, so + // a crash mid-batch can still recover the in-flight frames (and their times). + void JournalFrame(const FrameType& frame, int32_t frame_index, int64_t game_time, int64_t created_utc) { + if (!journal_.IsOpen()) + return; + std::vector> one; + one.push_back(std::make_unique(frame)); + std::string payload = SerializerPolicy::SerializeChunk(one, /*chunk_idx*/ 0, config_.b_use_compression); + payload = CompressIfBeneficial(std::move(payload)); + journal_.AppendFrame(frame_index, game_time, created_utc, payload); + batch_times_.push_back({frame_index, game_time, created_utc}); } void Close(const SessionFooter& footerData) { @@ -142,6 +183,9 @@ namespace VTX { Config config_; DurableFile file_; RecoveryJournal journal_; + std::vector batch_times_; // times of the current un-flushed batch + float journal_fps_ = 0.0f; // writer timing, journaled in the 'S' record + bool journal_is_increasing_ = true; int32_t chunkIndex_ = 0; std::vector seek_table_; //Generic tables, format agnostic }; diff --git a/sdk/include/vtx/writer/policies/sinks/network_sink.h b/sdk/include/vtx/writer/policies/sinks/network_sink.h index 4458691..c0afd67 100644 --- a/sdk/include/vtx/writer/policies/sinks/network_sink.h +++ b/sdk/include/vtx/writer/policies/sinks/network_sink.h @@ -130,6 +130,12 @@ namespace VTX { seek_table_.push_back(entry); } + // Crash-recovery journaling is file-sink only (a socket has no local file to + // repair); accept the writer's journaling hooks as no-ops over the network. + void JournalFrame(const FrameType& /*frame*/, int32_t /*frame_index*/, int64_t /*game_time*/, + int64_t /*created_utc*/) {} + void JournalTiming(float /*fps*/, bool /*is_increasing*/) {} + void Close(const SessionFooter& footerData) { if (socket_ == kVtxInvalidSocket) return; diff --git a/sdk/include/vtx/writer/policies/sinks/recovery_journal.h b/sdk/include/vtx/writer/policies/sinks/recovery_journal.h index ef18060..1e4de87 100644 --- a/sdk/include/vtx/writer/policies/sinks/recovery_journal.h +++ b/sdk/include/vtx/writer/policies/sinks/recovery_journal.h @@ -1,19 +1,45 @@ /** * @file recovery_journal.h - * @brief Write-ahead journal (sidecar ".recovery" file) of committed chunk index - * entries, so a crash before the footer is written can be recovered. + * @brief Single write-ahead sidecar (".recovery") that makes a crashed recording + * recoverable up to the last frame. * - * @details Each record is fixed-size and self-validating (per-record xxHash64), so - * a torn last record from a crash mid-write is detected and dropped. Records are - * appended AFTER the corresponding chunk is durably on disk (data-before-journal - * ordering) so the journal never references a chunk that isn't there. On a clean - * Close() the main-file footer is written and the ".recovery" file is deleted; its - * presence at open time therefore signals an unclean shutdown. + * @details One file holds a typed record stream so a crash before the footer is + * written can be recovered without losing committed chunks, per-frame times, or the + * in-flight (un-flushed) frame batch. Every record is self-validating (per-record + * xxHash64), so a torn last record from a crash mid-write is detected and dropped. * - * Byte layout (little-endian host): - * Header: "VTXR" | u32 version | 4-byte main-file magic ("VTXP"/"VTXF") - * Record: i32 chunk_index | i32 start_frame | i32 end_frame | u64 file_offset | - * u32 chunk_size_bytes | u64 chunk_checksum | u64 record_checksum + * Record types (each: [u8 type][u32 payload_len][payload][u64 checksum]): + * 'S' session: written once after the header: {f32 fps, u8 is_increasing, + * u8 use_compression, i8 compression_level}. Lets the repair pass + * reconstruct the footer's derived time data (duration, timeline gaps, + * game segments) exactly as VTXGameTimes would have, and compress the + * synthesized footer exactly as the sink would have -- so a recovered + * file can be byte-identical to a cleanly closed one. + * 'C' chunk : a committed chunk's index entry (offset/size/frames/checksum). + * 'T' time : one committed frame's {index, game_time, created_utc}. + * 'F' frame : an un-flushed frame's {index, game_time, created_utc, chunk-payload + * bytes}. + * + * The log is strictly APPEND-ONLY in the crash-critical path: a frame appends an F + * record; a flush appends the chunk's C record plus its frames' T records. Nothing + * already written is ever moved or truncated in place, so at every instant the file + * on disk is a valid prefix of records -- there is no window in which a durable chunk + * is described by neither its F records nor its C record. Ordering is + * data-before-journal: a chunk is fsync'd into the .vtx first, then its C/T records + * are appended and fsync'd here. + * + * Once a chunk's C/T records are durable, that chunk's F records are redundant (the + * frames now live in the .vtx and are recoverable via the C record); repair dedups + * them by frame index. To keep the journal from growing to the size of the whole + * recording, those redundant F records are reclaimed by periodic COMPACTION: the log + * is rewritten (all C/T + only the still-pending F) into a temp file that atomically + * replaces the sidecar. A crash during compaction leaves either the old or the new + * file (rename is atomic) -- both are valid, complete journals. + * + * On a clean Close() the footer is written to the .vtx and the ".recovery" file is + * deleted, so its presence at open time signals an unclean shutdown. + * + * Byte order: little-endian host (documented; matches the rest of the format). * * @author Zenos Interactive */ @@ -21,8 +47,10 @@ #include #include +#include #include #include +#include #include #include @@ -33,53 +61,93 @@ namespace VTX { class RecoveryJournal { public: - static constexpr uint32_t kVersion = 1; - static constexpr size_t kPayloadSize = 4 + 4 + 4 + 8 + 4 + 8; // 32 - static constexpr size_t kRecordSize = kPayloadSize + 8; // 40 (+ record checksum) + static constexpr uint32_t kVersion = 4; + static constexpr size_t kHeaderSize = 12; // "VTXR" + u32 version + 4-byte format magic + + struct FrameTime { + int32_t index = 0; + int64_t game_time = 0; + int64_t created_utc = 0; + }; + struct PendingFrame { + int32_t index = 0; + int64_t game_time = 0; + int64_t created_utc = 0; + std::string payload; // serialized single-frame chunk payload (as stored in the .vtx) + }; - /// The sidecar path for a given main output file. static std::string PathFor(const std::string& main_file) { return main_file + ".recovery"; } + static std::string CompactTempFor(const std::string& journal_file) { return journal_file + ".compact"; } - /// Create/truncate the journal and write its header. @p format_magic is the - /// main file's 4-byte magic. Returns false on failure. - bool Open(const std::string& path, const std::string& format_magic, bool durable) { + // --- Write side (append-only) --- + + /// @param fps Writer's default FPS; lets repair reconstruct the + /// footer's timeline-gap data exactly (0 disables gap + /// detection, as in VTXGameTimes). + /// @param is_increasing Writer's game-time direction; lets repair reconstruct + /// the footer's game-segment data exactly. + /// @param use_compression Sink's compression setting; lets repair compress the + /// synthesized footer exactly as the sink would have. + /// @param compression_level Sink's zstd level, paired with @p use_compression. + bool Open(const std::string& path, const std::string& format_magic, bool durable, float fps = 0.0f, + bool is_increasing = true, bool use_compression = true, int8_t compression_level = 10) { durable_ = durable; - if (!file_.Open(path)) - return false; - char magic[4] = {'V', 'T', 'X', 'R'}; - file_.Write(magic, 4); - uint32_t version = kVersion; - file_.Write(&version, sizeof(version)); - char fmt[4] = {' ', ' ', ' ', ' '}; + path_ = path; + fps_ = fps; + is_increasing_ = is_increasing; + use_compression_ = use_compression; + compression_level_ = compression_level; + format_magic_.assign(4, ' '); for (size_t i = 0; i < 4 && i < format_magic.size(); ++i) - fmt[i] = format_magic[i]; - file_.Write(fmt, 4); - if (durable_) - file_.Sync(); - else - file_.Flush(); // process-crash safe (reaches the OS) even without fsync - return true; + format_magic_[i] = format_magic[i]; + + // Drop any temp left behind by a compaction interrupted in a previous run. + std::error_code ec; + std::filesystem::remove(CompactTempFor(path_), ec); + + if (!file_.Open(path_)) + return false; + WriteHeaderTo(file_, format_magic_); + WriteRecordTo(file_, 'S', BuildTimingPayload(fps_, is_increasing_, use_compression_, compression_level_)); + SyncOrFlush(); + pending_f_bytes_ = 0; + orphan_f_bytes_ = 0; + return file_.Good(); } bool IsOpen() const { return file_.IsOpen(); } - /// Append one committed-chunk record. Call AFTER the chunk is durable on disk. - void AppendChunk(const ChunkIndexData& e) { - uint8_t rec[kRecordSize]; - size_t o = 0; - PackI32(rec, o, e.chunk_index); - PackI32(rec, o, e.start_frame); - PackI32(rec, o, e.end_frame); - PackU64(rec, o, static_cast(e.file_offset)); - PackU32(rec, o, e.chunk_size_bytes); - PackU64(rec, o, e.checksum); - uint64_t record_checksum = XXH3_64bits(rec, kPayloadSize); - PackU64(rec, o, record_checksum); - file_.Write(rec, kRecordSize); - if (durable_) - file_.Sync(); - else - file_.Flush(); // process-crash safe (reaches the OS) even without fsync + // Compaction fires when reclaimable (superseded) F bytes exceed this. Exposed so + // tests can force compaction without writing hundreds of MB. + void SetCompactThresholdBytes(uint64_t bytes) { compact_threshold_ = bytes; } + + // Append an un-flushed frame (F record) at the tail. Call as each frame is recorded. + void AppendFrame(int32_t index, int64_t game_time, int64_t created_utc, const std::string& chunk_payload) { + // A frame whose record would exceed the read-side sanity bound could not be + // parsed back on repair; skip journaling it rather than write an unreadable + // record (the frame is still captured once its chunk is flushed to the .vtx). + if (chunk_payload.size() + kFramePrefixBytes > kMaxRecordPayload) + return; + const std::vector payload = BuildFramePayload(index, game_time, created_utc, chunk_payload); + pending_f_bytes_ += WriteRecordTo(file_, 'F', payload); + SyncOrFlush(); + } + + // Commit a flushed chunk: durably append the chunk (C) and its frames' times (T). + // Call AFTER the chunk is durable in the .vtx. Append-only -- the now-redundant F + // records for this batch are left in place (repair dedups them) and reclaimed later + // by compaction, so there is never a window in which the batch is unrecoverable. + void CommitChunk(const ChunkIndexData& chunk, const std::vector& batch_times) { + WriteRecordTo(file_, 'C', BuildChunkPayload(chunk)); + for (const auto& t : batch_times) + WriteRecordTo(file_, 'T', BuildTimePayload(t)); + SyncOrFlush(); + + // This batch's F records are now superseded by the durable C record. + orphan_f_bytes_ += pending_f_bytes_; + pending_f_bytes_ = 0; + if (orphan_f_bytes_ > compact_threshold_) + Compact(); } void Close() { file_.Close(); } @@ -87,25 +155,43 @@ namespace VTX { // --- Read side (repair) --- struct Parsed { - bool has_journal = false; ///< The sidecar file existed. - bool header_valid = false; ///< Header magic/version parsed OK. - std::string format_magic; ///< Main-file magic recorded in the header. - std::vector chunks; ///< Valid committed-chunk records, in order. + bool has_journal = false; + bool looks_like_journal = false; ///< File starts with "VTXR" (any version). + bool header_valid = false; + std::string format_magic; + bool has_timing = false; ///< An 'S' record was read. + float fps = 0.0f; ///< Writer FPS (0 = gap detection disabled). + bool is_increasing = true; ///< Writer game-time direction. + bool use_compression = true; ///< Sink compression setting at record time. + int8_t compression_level = 10; ///< Sink zstd level at record time. + std::vector chunks; + std::vector times; + std::vector frames; }; - /// Parse a journal file, returning all VALID records up to the first torn/invalid one. static Parsed ReadValid(const std::string& path) { Parsed out; std::ifstream in(path, std::ios::binary); if (!in.is_open()) - return out; // has_journal stays false + return out; out.has_journal = true; - char header[12]; + // Actual size on disk, used to bound per-record allocations: a corrupt + // length field must not trigger a huge transient allocation (up to + // kMaxRecordPayload) for a journal that is only a few KB long. + uint64_t file_size = 0; + { + std::error_code ec; + const auto s = std::filesystem::file_size(path, ec); + if (!ec) + file_size = static_cast(s); + } + + char header[kHeaderSize]; in.read(header, sizeof(header)); - if (in.gcount() != static_cast(sizeof(header))) - return out; // truncated header -> no usable records - if (std::memcmp(header, "VTXR", 4) != 0) + if (in.gcount() >= 4 && std::memcmp(header, "VTXR", 4) == 0) + out.looks_like_journal = true; + if (in.gcount() != static_cast(sizeof(header)) || !out.looks_like_journal) return out; uint32_t version = 0; std::memcpy(&version, header + 4, sizeof(version)); @@ -114,61 +200,276 @@ namespace VTX { out.format_magic.assign(header + 8, 4); out.header_valid = true; - uint8_t rec[kRecordSize]; - while (in.read(reinterpret_cast(rec), kRecordSize), - in.gcount() == static_cast(kRecordSize)) { + uint64_t pos = kHeaderSize; // bytes consumed so far + for (;;) { + char head[5]; + in.read(head, 5); // u8 type + u32 len + if (in.gcount() != 5) + break; + pos += 5; + const uint8_t type = static_cast(head[0]); + uint32_t len = 0; + std::memcpy(&len, head + 1, sizeof(len)); + if (len > kMaxRecordPayload) + break; // implausible -> torn + // The payload + trailing checksum cannot exceed what the file holds: + // reject before allocating, so a corrupt length cannot balloon memory. + // (Additive form -- no unsigned underflow if the size query failed.) + if (pos + static_cast(len) + 8 > file_size) + break; + pos += static_cast(len) + 8; + std::string payload(len, '\0'); + if (len > 0) { + in.read(payload.data(), len); + if (in.gcount() != static_cast(len)) + break; + } + char cbuf[8]; + in.read(cbuf, 8); + if (in.gcount() != 8) + break; uint64_t stored = 0; - std::memcpy(&stored, rec + kPayloadSize, sizeof(stored)); - if (XXH3_64bits(rec, kPayloadSize) != stored) - break; // torn / corrupt record -> stop here - - ChunkIndexData e; - size_t o = 0; - e.chunk_index = UnpackI32(rec, o); - e.start_frame = UnpackI32(rec, o); - e.end_frame = UnpackI32(rec, o); - e.file_offset = static_cast(UnpackU64(rec, o)); - e.chunk_size_bytes = UnpackU32(rec, o); - e.checksum = UnpackU64(rec, o); - out.chunks.push_back(e); + std::memcpy(&stored, cbuf, sizeof(stored)); + + // Checksum covers type + len + payload. + XXH3_state_t state; + XXH3_64bits_reset(&state); + XXH3_64bits_update(&state, head, 5); + if (len > 0) + XXH3_64bits_update(&state, payload.data(), len); + if (XXH3_64bits_digest(&state) != stored) + break; // torn / corrupt record -> stop + + if (type == 'S') { + if (payload.size() != 7) + break; + size_t o = 0; + const uint32_t fps_bits = ReadU32(payload, o); + std::memcpy(&out.fps, &fps_bits, sizeof(out.fps)); + out.is_increasing = payload[4] != 0; + out.use_compression = payload[5] != 0; + out.compression_level = static_cast(payload[6]); + out.has_timing = true; + } else if (type == 'C') { + if (payload.size() != 32) + break; + size_t o = 0; + ChunkIndexData e; + e.chunk_index = ReadI32(payload, o); + e.start_frame = ReadI32(payload, o); + e.end_frame = ReadI32(payload, o); + e.file_offset = static_cast(ReadU64(payload, o)); + e.chunk_size_bytes = ReadU32(payload, o); + e.checksum = ReadU64(payload, o); + out.chunks.push_back(e); + } else if (type == 'T') { + if (payload.size() != 20) + break; + size_t o = 0; + FrameTime t; + t.index = ReadI32(payload, o); + t.game_time = ReadI64(payload, o); + t.created_utc = ReadI64(payload, o); + out.times.push_back(t); + } else if (type == 'F') { + if (payload.size() < 20) + break; + size_t o = 0; + PendingFrame f; + f.index = ReadI32(payload, o); + f.game_time = ReadI64(payload, o); // carried for pending-frame time recovery + f.created_utc = ReadI64(payload, o); + f.payload = payload.substr(o); + out.frames.push_back(f); + } else { + break; // unknown type -> stop + } } return out; } private: - static void PackI32(uint8_t* b, size_t& o, int32_t v) { - uint32_t u = static_cast(v); - PackU32(b, o, u); - } - static void PackU32(uint8_t* b, size_t& o, uint32_t v) { - b[o + 0] = static_cast(v); - b[o + 1] = static_cast(v >> 8); - b[o + 2] = static_cast(v >> 16); - b[o + 3] = static_cast(v >> 24); - o += 4; + static constexpr uint32_t kMaxRecordPayload = 512u * 1024u * 1024u; // 512 MB sanity bound + static constexpr size_t kFramePrefixBytes = 20; // i32 index + i64 gt + i64 cu + // Reclaim superseded F records once this many bytes have accrued. Also bounds the + // transient memory a compaction reads back (superseded payloads are loaded then dropped). + static constexpr uint64_t kDefaultCompactThreshold = 64ull * 1024ull * 1024ull; + + void SyncOrFlush() { + if (durable_) + file_.Sync(); + else + file_.Flush(); + } + + // Rewrite the log keeping every C/T record and only the still-pending F records, + // into a temp file that atomically replaces the sidecar. Crash-safe: until the + // rename lands the original journal is intact; after it, the compacted one is. + void Compact() { + SyncOrFlush(); + file_.Close(); + + const Parsed p = ReadValid(path_); + if (!p.header_valid) { + // Unexpected -- reopen the original and keep appending rather than risk data. + file_.OpenExisting(path_); + file_.SeekEnd(); + return; + } + int32_t last_committed = -1; + for (const auto& c : p.chunks) + if (c.end_frame > last_committed) + last_committed = c.end_frame; + + const std::string tmp = CompactTempFor(path_); + std::error_code ec; + std::filesystem::remove(tmp, ec); + + DurableFile out; + uint64_t kept_pending = 0; + if (out.Open(tmp)) { + WriteHeaderTo(out, p.format_magic.empty() ? format_magic_ : p.format_magic); + WriteRecordTo(out, 'S', BuildTimingPayload(fps_, is_increasing_, use_compression_, compression_level_)); + for (const auto& c : p.chunks) + WriteRecordTo(out, 'C', BuildChunkPayload(c)); + for (const auto& t : p.times) + WriteRecordTo(out, 'T', BuildTimePayload(t)); + for (const auto& f : p.frames) { + if (f.index <= last_committed) + continue; // superseded -> drop + kept_pending += + WriteRecordTo(out, 'F', BuildFramePayload(f.index, f.game_time, f.created_utc, f.payload)); + } + out.Sync(); + const bool ok = out.Good(); + out.Close(); + if (ok) { + std::filesystem::rename(tmp, path_, ec); + if (ec) + std::filesystem::remove(tmp, ec); // rename failed -> keep the original + else { + orphan_f_bytes_ = 0; + pending_f_bytes_ = kept_pending; + } + } else { + std::filesystem::remove(tmp, ec); // temp write failed -> keep the original + } + } + + file_.OpenExisting(path_); + file_.SeekEnd(); + } + + static void WriteHeaderTo(DurableFile& f, const std::string& format_magic) { + char magic[4] = {'V', 'T', 'X', 'R'}; + f.Write(magic, 4); + uint32_t version = kVersion; + f.Write(&version, sizeof(version)); + char fmt[4] = {' ', ' ', ' ', ' '}; + for (size_t i = 0; i < 4 && i < format_magic.size(); ++i) + fmt[i] = format_magic[i]; + f.Write(fmt, 4); + } + + // Writes one framed record and returns the number of bytes it occupies on disk. + static uint64_t WriteRecordTo(DurableFile& f, char type, const std::vector& payload) { + uint8_t head[5]; + head[0] = static_cast(type); + const uint32_t len = static_cast(payload.size()); + std::memcpy(head + 1, &len, sizeof(len)); + XXH3_state_t state; + XXH3_64bits_reset(&state); + XXH3_64bits_update(&state, head, 5); + if (!payload.empty()) + XXH3_64bits_update(&state, payload.data(), payload.size()); + const uint64_t checksum = XXH3_64bits_digest(&state); + f.Write(head, 5); + if (!payload.empty()) + f.Write(payload.data(), payload.size()); + f.Write(&checksum, sizeof(checksum)); + return 5ull + payload.size() + 8ull; + } + + static std::vector BuildTimingPayload(float fps, bool is_increasing, bool use_compression, + int8_t compression_level) { + std::vector s; + uint32_t fps_bits = 0; + std::memcpy(&fps_bits, &fps, sizeof(fps_bits)); + AppendU32(s, fps_bits); + s.push_back(is_increasing ? 1 : 0); + s.push_back(use_compression ? 1 : 0); + s.push_back(static_cast(compression_level)); + return s; + } + + static std::vector BuildChunkPayload(const ChunkIndexData& chunk) { + std::vector c; + AppendI32(c, chunk.chunk_index); + AppendI32(c, chunk.start_frame); + AppendI32(c, chunk.end_frame); + AppendU64(c, static_cast(chunk.file_offset)); + AppendU32(c, chunk.chunk_size_bytes); + AppendU64(c, chunk.checksum); + return c; } - static void PackU64(uint8_t* b, size_t& o, uint64_t v) { + + static std::vector BuildTimePayload(const FrameTime& t) { + std::vector tp; + AppendI32(tp, t.index); + AppendI64(tp, t.game_time); + AppendI64(tp, t.created_utc); + return tp; + } + + static std::vector BuildFramePayload(int32_t index, int64_t game_time, int64_t created_utc, + const std::string& chunk_payload) { + std::vector payload; + AppendI32(payload, index); + AppendI64(payload, game_time); + AppendI64(payload, created_utc); + payload.insert(payload.end(), chunk_payload.begin(), chunk_payload.end()); + return payload; + } + + static void AppendU32(std::vector& b, uint32_t v) { + for (int i = 0; i < 4; ++i) + b.push_back(static_cast(v >> (8 * i))); + } + static void AppendI32(std::vector& b, int32_t v) { AppendU32(b, static_cast(v)); } + static void AppendU64(std::vector& b, uint64_t v) { for (int i = 0; i < 8; ++i) - b[o + i] = static_cast(v >> (8 * i)); - o += 8; + b.push_back(static_cast(v >> (8 * i))); } - static int32_t UnpackI32(const uint8_t* b, size_t& o) { return static_cast(UnpackU32(b, o)); } - static uint32_t UnpackU32(const uint8_t* b, size_t& o) { - uint32_t v = static_cast(b[o]) | (static_cast(b[o + 1]) << 8) | - (static_cast(b[o + 2]) << 16) | (static_cast(b[o + 3]) << 24); + static void AppendI64(std::vector& b, int64_t v) { AppendU64(b, static_cast(v)); } + + static uint32_t ReadU32(const std::string& b, size_t& o) { + uint32_t v = static_cast(b[o]) | (static_cast(static_cast(b[o + 1])) << 8) | + (static_cast(static_cast(b[o + 2])) << 16) | + (static_cast(static_cast(b[o + 3])) << 24); o += 4; return v; } - static uint64_t UnpackU64(const uint8_t* b, size_t& o) { + static int32_t ReadI32(const std::string& b, size_t& o) { return static_cast(ReadU32(b, o)); } + static uint64_t ReadU64(const std::string& b, size_t& o) { uint64_t v = 0; for (int i = 0; i < 8; ++i) - v |= static_cast(b[o + i]) << (8 * i); + v |= static_cast(static_cast(b[o + i])) << (8 * i); o += 8; return v; } + static int64_t ReadI64(const std::string& b, size_t& o) { return static_cast(ReadU64(b, o)); } DurableFile file_; bool durable_ = true; + std::string path_; + std::string format_magic_; + float fps_ = 0.0f; + bool is_increasing_ = true; + bool use_compression_ = true; + int8_t compression_level_ = 10; + uint64_t pending_f_bytes_ = 0; // F bytes for the current un-flushed batch + uint64_t orphan_f_bytes_ = 0; // superseded F bytes awaiting compaction + uint64_t compact_threshold_ = kDefaultCompactThreshold; // reclaim when orphan_f_bytes_ exceeds this }; } // namespace VTX diff --git a/sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp b/sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp index 519c860..9f4e98f 100644 --- a/sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp +++ b/sdk/src/vtx_writer/src/vtx/writer/core/vtx_replay_recovery.cpp @@ -1,5 +1,6 @@ #include "vtx/writer/core/vtx_replay_recovery.h" +#include #include #include #include @@ -8,6 +9,7 @@ #include #include #include +#include #include "vtx/common/vtx_types.h" #include "vtx/writer/policies/formatters/flatbuffers_vtx_policy.h" @@ -46,6 +48,22 @@ namespace VTX { return footer_size > 0 && static_cast(footer_size) + 8 <= file_size; } + // Mirror of ChunkedFileSink::CompressIfBeneficial, so a footer synthesized by + // repair is byte-identical to one written by a clean Close() under the same + // (journaled) compression settings. + std::string CompressIfBeneficial(std::string payload, bool use_compression, int8_t level) { + if (!use_compression || payload.size() < 512) + return payload; + const size_t max_size = ZSTD_compressBound(payload.size()); + std::string compressed(max_size, '\0'); + const size_t compressed_size = + ZSTD_compress(compressed.data(), max_size, payload.data(), payload.size(), level); + if (ZSTD_isError(compressed_size) || compressed_size >= payload.size()) + return payload; + compressed.resize(compressed_size); + return compressed; + } + // Byte offset where chunks begin (magic(4) + u32 header_size + header), or -1 if // the header framing is not intact. int64_t ReadHeaderEnd(std::ifstream& in, const std::string& expect_magic, int64_t file_size) { @@ -85,9 +103,10 @@ namespace VTX { return r; } - // Keep only chunks whose on-disk bytes are present and checksum-clean, in order. + // Keep only committed chunks whose on-disk bytes are present and checksum-clean, in order. std::vector good; int64_t truncate_to = header_end; + int32_t last_committed_frame = -1; for (const auto& c : journal.chunks) { const int64_t off = c.file_offset; const int64_t end = off + static_cast(c.chunk_size_bytes); @@ -110,6 +129,7 @@ namespace VTX { good.push_back(c); truncate_to = end; + last_committed_frame = c.end_frame; } in.close(); @@ -122,35 +142,144 @@ namespace VTX { return r; } - // Synthesize and append a footer over the recovered chunk index. - SessionFooter footer_data; - footer_data.total_frames = good.empty() ? 0 : (good.back().end_frame + 1); - const std::string footer_payload = Policy::SerializeFooter(good, footer_data); - DurableFile out; if (!out.OpenExisting(path)) { - r.error = "cannot reopen main file to append footer"; + r.error = "cannot reopen main file to append"; return r; } out.SeekEnd(); - out.Write(footer_payload.data(), footer_payload.size()); + + // Recover the in-flight (un-flushed) frames: append each pending frame (index + // beyond the last committed one) as its own chunk, in contiguous order. + std::vector pending; + for (const auto& f : journal.frames) + if (f.index > last_committed_frame) + pending.push_back(&f); + std::sort(pending.begin(), pending.end(), + [](const RecoveryJournal::PendingFrame* a, const RecoveryJournal::PendingFrame* b) { + return a->index < b->index; + }); + + int32_t last_frame = last_committed_frame; + int32_t expected = last_committed_frame + 1; + for (const auto* f : pending) { + if (f->index != expected || f->payload.empty()) + break; // gap / duplicate / empty -> stop at the first hole + const uint64_t offset = out.Tell(); + const uint32_t size = static_cast(f->payload.size()); + out.Write(&size, sizeof(size)); + out.Write(f->payload.data(), f->payload.size()); + + ChunkIndexData e; + e.chunk_index = good.empty() ? 0 : good.back().chunk_index + 1; + e.file_offset = static_cast(offset); + e.start_frame = f->index; + e.end_frame = f->index; + e.chunk_size_bytes = size + static_cast(sizeof(uint32_t)); + e.checksum = XXH3_64bits(f->payload.data(), f->payload.size()); + good.push_back(e); + + last_frame = f->index; + ++expected; + } + + const int32_t total_frames = last_frame + 1; + + // Reconstruct exact per-frame times from T records (committed) and F records (pending). + std::vector game_times(total_frames > 0 ? static_cast(total_frames) : 0, 0); + std::vector created_utc(total_frames > 0 ? static_cast(total_frames) : 0, 0); + auto place = [&](int32_t idx, int64_t gt, int64_t cu) { + if (idx >= 0 && idx < total_frames) { + game_times[static_cast(idx)] = gt; + created_utc[static_cast(idx)] = cu; + } + }; + for (const auto& t : journal.times) + place(t.index, t.game_time, t.created_utc); + for (const auto& f : journal.frames) + place(f.index, f.game_time, f.created_utc); + + // Reconstruct the footer's derived time data exactly as VTXGameTimes would have + // produced it (mirrors GetDuration / DetectGap / DetectGameSegment): duration + // from the created_utc span, timeline gaps from UTC deltas vs the FPS-derived + // threshold, and game segments from game-time direction reversals. Frame + // numbers are 1-based, as in VTXGameTimes::GetFrameNumber(). Manual segment + // marks are not journaled and so are not recovered. + SessionFooter footer_data; + footer_data.total_frames = total_frames; + std::vector gaps; + std::vector segments; + if (total_frames > 0) { + // GetDuration() returns float; keep the same float rounding before the + // footer's double field so a recovered footer matches a clean Stop() exactly. + const double duration = static_cast(created_utc.back() - created_utc.front()) / + static_cast(GameTime::TICKS_PER_SECOND); + footer_data.duration_seconds = static_cast(duration); + } + if (journal.has_timing && total_frames > 1) { + if (journal.fps > 0.0f) { + // Same expression as VTXGameTimes::SetFPS -> fps_inverse_. + const int64_t fps_inverse = static_cast((1.0f / journal.fps) * GameTime::TICKS_PER_SECOND); + const int64_t threshold = 3 * fps_inverse; + for (int32_t i = 1; i < total_frames; ++i) + if (created_utc[static_cast(i)] - created_utc[static_cast(i) - 1] > threshold) + gaps.push_back(i + 1); + } + for (int32_t i = 1; i < total_frames; ++i) { + const int64_t delta = game_times[static_cast(i)] - game_times[static_cast(i) - 1]; + if ((journal.is_increasing && delta < 0) || (!journal.is_increasing && delta > 0)) + segments.push_back(i + 1); + } + } + // Always pass the vectors -- even empty -- exactly as Stop() does, so a + // recovered footer serializes byte-identically to a clean shutdown's. + footer_data.game_times = &game_times; + footer_data.created_utc = &created_utc; + footer_data.gaps = &gaps; + footer_data.segments = &segments; + // Compress exactly as the sink's Close() would have (settings journaled in + // the 'S' record), so large recovered footers also match byte-for-byte. + const std::string footer_payload = CompressIfBeneficial(Policy::SerializeFooter(good, footer_data), + journal.use_compression, journal.compression_level); + + bool footer_ok = out.Write(footer_payload.data(), footer_payload.size()); const uint32_t footer_size = static_cast(footer_payload.size()); - out.Write(&footer_size, sizeof(footer_size)); + footer_ok = out.Write(&footer_size, sizeof(footer_size)) && footer_ok; const std::string magic = Policy::GetMagicBytes(); - out.Write(magic.data(), magic.size()); - out.Sync(); + footer_ok = out.Write(magic.data(), magic.size()) && footer_ok; + footer_ok = out.Sync() && footer_ok; + // Good() also catches any failure in the pending-frame append writes above. + const bool all_ok = footer_ok && out.Good(); out.Close(); - std::remove(RecoveryJournal::PathFor(path).c_str()); + if (!all_ok) { + // Leave the journal in place: a re-run (more disk free, or a CLI tool) is + // idempotent -- it re-truncates to the last committed chunk and retries. + r.error = "failed to write the recovered footer durably (disk full?); journal left intact"; + return r; + } + + const std::string journal_path = RecoveryJournal::PathFor(path); + std::remove(journal_path.c_str()); + std::remove(RecoveryJournal::CompactTempFor(journal_path).c_str()); r.repaired = true; r.recovered_chunks = static_cast(good.size()); - r.recovered_frames = footer_data.total_frames; + r.recovered_frames = total_frames; return r; } } // namespace + std::string RecoveryJournalPath(const std::string& path) { + return RecoveryJournal::PathFor(path); + } + + bool ReplayNeedsRecovery(const std::string& path) { + std::error_code ec; + return std::filesystem::exists(RecoveryJournal::PathFor(path), ec); + } + RepairResult RepairReplayFile(const std::string& path) { RepairResult r; @@ -178,14 +307,29 @@ namespace VTX { // just leftover (crash between the footer fsync and the journal delete). // Preserve the complete footer (incl. per-frame times) -- only drop the journal. if (HasValidTrailingFooter(path, m, file_size)) { - std::remove(journal_path.c_str()); + // Only delete the sidecar if it actually IS a journal ("VTXR" magic) -- an + // unrelated user file that merely shares the ".recovery" suffix must never + // be destroyed on the strength of a name collision. + if (journal.looks_like_journal) { + std::remove(journal_path.c_str()); + std::remove(RecoveryJournal::CompactTempFor(journal_path).c_str()); + } r.was_clean = true; return r; } + // A present-but-unparseable journal header (corrupt first bytes, or a version + // from an incompatible SDK) carries no chunk index. Repairing from it would + // truncate the main file down to just its header and destroy the body -- refuse + // instead, leaving the file untouched for manual recovery. + if (!journal.header_valid) { + r.error = "recovery journal header is unreadable or from an incompatible version; refusing to repair"; + return r; + } + // Refuse a journal whose recorded format does not match this file (e.g. a // stale sidecar left over a file that was replaced with the other format). - if (journal.header_valid && !journal.format_magic.empty() && journal.format_magic != m) { + if (!journal.format_magic.empty() && journal.format_magic != m) { r.error = "recovery journal format (" + journal.format_magic + ") does not match file (" + m + ")"; return r; } diff --git a/tests/writer/test_crash_recovery.cpp b/tests/writer/test_crash_recovery.cpp index beb3300..9000660 100644 --- a/tests/writer/test_crash_recovery.cpp +++ b/tests/writer/test_crash_recovery.cpp @@ -1,32 +1,59 @@ // Crash-recovery tests: a writer that dies before Stop() leaves a footerless .vtx // plus a ".recovery" journal; RepairReplayFile() must reconstruct a valid, readable -// file from it, dropping any torn/corrupt tail. +// file from it, recovering every durable chunk AND every in-flight (un-flushed) +// frame, while dropping any torn/corrupt tail. // -// The crash state is reproduced faithfully without racing a real process death: -// write a valid file, read its seek table (the exact per-chunk index the sink -// journals, checksums included), truncate the footer off, and re-create the -// ".recovery" journal from those records. That is byte-for-byte what the sink -// leaves on disk after a mid-write crash (chunks + journal, no footer). +// Crash states are fabricated byte-faithfully rather than by racing a real process +// death: a valid recording is written and captured (header bytes, real per-chunk +// payloads/offsets/checksums, per-frame times), then a footerless .vtx + a +// ".recovery" journal are rebuilt through the *same* RecoveryJournal API the sink +// uses. That is byte-for-byte what the sink leaves on disk after a mid-write crash, +// and it lets each crash window (between chunks, between frames, torn chunk, torn +// frame record, corrupt chunk, mid-footer, leftover footer) be reproduced exactly. #include #include +#include #include +#include #include #include +#include #include #include +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include // CreateProcessA / TerminateProcess for the real-kill tests +#endif + +#include "vtx_schema_generated.h" // complete fbsvtx/cppvtx types before the policy headers +#include "vtx_schema.pb.h" + #include "vtx/common/vtx_types.h" #include "vtx/reader/core/vtx_reader_facade.h" +#include "vtx/reader/core/vtx_replay_validation.h" +#include "vtx/writer/core/vtx_frame_post_processor.h" #include "vtx/writer/core/vtx_replay_recovery.h" #include "vtx/writer/core/vtx_writer_facade.h" +#include "vtx/writer/core/writer.h" +#include "vtx/writer/policies/formatters/flatbuffers_vtx_policy.h" +#include "vtx/writer/policies/formatters/protobuff_vtx_policy.h" +#include "vtx/writer/policies/sinks/file_sink.h" #include "vtx/writer/policies/sinks/recovery_journal.h" #include "util/test_fixtures.h" namespace { + enum class Fmt { Flat, Proto }; + VTX::Frame BuildFrame(int i) { VTX::Frame f; auto& bucket = f.CreateBucket("entity"); @@ -40,38 +67,56 @@ namespace { return f; } - // Writes `chunks` single-frame chunks to a valid .vtx (footer + no journal). - std::string WriteValidFile(const std::string& suffix, int chunks) { + // A captured valid recording: everything needed to rebuild an equivalent + // footerless .vtx and its recovery journal, byte-for-byte. + struct Recording { + std::string format_magic; // "VTXF" / "VTXP" + std::string header; // file bytes [0 .. first_chunk_offset) + std::vector chunks; // real seek table (offsets/sizes/checksums) + std::vector payloads; // per-chunk on-disk payload (len == chunk_size_bytes - 4) + std::vector game_times; // per-frame, from the footer + std::vector created_utc; // per-frame, from the footer + int total_frames = 0; + }; + + // Write a valid recording (footer + no journal) and capture it. `per_chunk` + // frames are flushed per chunk; use per_chunk==1 when the payloads must double as + // single-frame (F-record) payloads for pending-frame fabrication. + Recording CaptureRecording(const std::string& suffix, int frames, int per_chunk, Fmt fmt) { VTX::WriterFacadeConfig cfg; - cfg.output_filepath = VtxTest::OutputPath("crash_" + suffix + ".vtx"); + cfg.output_filepath = VtxTest::OutputPath("cap_" + suffix + ".vtx"); cfg.schema_json_path = VtxTest::FixturePath("test_schema.json"); cfg.replay_name = "CrashRecoveryTest"; cfg.default_fps = 60.0f; cfg.use_compression = true; - auto writer = VTX::CreateFlatBuffersWriterFacade(cfg); + auto writer = + (fmt == Fmt::Flat) ? VTX::CreateFlatBuffersWriterFacade(cfg) : VTX::CreateProtobufWriterFacade(cfg); EXPECT_NE(writer, nullptr); - for (int i = 0; i < chunks; ++i) { + int in_batch = 0; + for (int i = 0; i < frames; ++i) { auto frame = BuildFrame(i); VTX::GameTime::GameTimeRegister t; t.game_time = float(i) / 60.0f; writer->RecordFrame(frame, t); - writer->Flush(); // one chunk per frame + if (++in_batch >= per_chunk) { + writer->Flush(); + in_batch = 0; + } } + if (in_batch > 0) + writer->Flush(); writer->Stop(); - return cfg.output_filepath; - } + writer.reset(); // release the file handle before reading it back - // Reproduce the on-disk crash state: [header + chunks] with NO footer, plus a - // ".recovery" journal listing the chunks. `truncate_extra` shaves extra bytes - // off the last chunk to simulate a torn final write. - std::vector MakeCrashState(const std::string& path, int64_t truncate_extra = 0) { - std::vector records; - int64_t last_chunk_end = 0; + Recording rec; + rec.format_magic = (fmt == Fmt::Flat) ? "VTXF" : "VTXP"; { - auto ctx = VTX::OpenReplayFile(path); + auto ctx = VTX::OpenReplayFile(cfg.output_filepath); EXPECT_TRUE(ctx) << ctx.error; - for (const auto& e : ctx.reader->GetSeekTable()) { + ctx->WaitUntilReady(); + rec.total_frames = ctx->GetTotalFrames(); + for (const auto& e : ctx->GetSeekTable()) { VTX::ChunkIndexData d; d.chunk_index = e.chunk_index; d.start_frame = e.start_frame; @@ -79,170 +124,381 @@ namespace { d.file_offset = static_cast(e.file_offset); d.chunk_size_bytes = e.chunk_size_bytes; d.checksum = e.checksum; - records.push_back(d); - last_chunk_end = - std::max(last_chunk_end, static_cast(e.file_offset) + e.chunk_size_bytes); + rec.chunks.push_back(d); + } + const VTX::FileFooter footer = ctx->GetFooter(); + rec.game_times.assign(frames, 0); + rec.created_utc.assign(frames, 0); + for (int i = 0; i < frames; ++i) { + if (static_cast(i) < footer.times.game_time.size()) + rec.game_times[i] = static_cast(footer.times.game_time[i]); + if (static_cast(i) < footer.times.created_utc.size()) + rec.created_utc[i] = static_cast(footer.times.created_utc[i]); + } + } + + std::ifstream in(cfg.output_filepath, std::ios::binary); + const std::string all((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + EXPECT_FALSE(rec.chunks.empty()); + const size_t first_off = static_cast(rec.chunks.front().file_offset); + rec.header = all.substr(0, first_off); + for (const auto& c : rec.chunks) { + const size_t payload_off = static_cast(c.file_offset) + sizeof(uint32_t); + const size_t payload_len = c.chunk_size_bytes - sizeof(uint32_t); + EXPECT_LE(payload_off + payload_len, all.size()); + rec.payloads.push_back(all.substr(payload_off, payload_len)); + } + return rec; + } + + // Rebuild a footerless .vtx (header + committed chunks) plus the ".recovery" + // journal the sink would have left: C/T records for the committed chunks and F + // records for `pending_frames` un-flushed frames (drawn from the recording, whose + // payloads must be single-frame -> capture with per_chunk==1). `journal_magic` + // overrides the journal's recorded format for the mismatch test. + std::string FabricateCrash(const std::string& out_suffix, const Recording& rec, int committed_chunks, + int pending_frames, const std::string& journal_magic = "") { + const std::string magic = journal_magic.empty() ? rec.format_magic : journal_magic; + const std::string path = VtxTest::OutputPath("fab_" + out_suffix + ".vtx"); + + { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(rec.header.data(), static_cast(rec.header.size())); + for (int i = 0; i < committed_chunks; ++i) { + const uint32_t payload_size = rec.chunks[i].chunk_size_bytes - static_cast(sizeof(uint32_t)); + out.write(reinterpret_cast(&payload_size), sizeof(payload_size)); + out.write(rec.payloads[i].data(), static_cast(rec.payloads[i].size())); } } - // Truncate off the footer (and optionally part of the last chunk). - std::filesystem::resize_file(path, static_cast(last_chunk_end - truncate_extra)); - // Re-create the recovery journal exactly as the sink would have left it. + + const int last_committed_frame = committed_chunks > 0 ? rec.chunks[committed_chunks - 1].end_frame : -1; + VTX::RecoveryJournal journal; - EXPECT_TRUE(journal.Open(VTX::RecoveryJournal::PathFor(path), "VTXF", true)); - for (const auto& d : records) - journal.AppendChunk(d); + EXPECT_TRUE(journal.Open(VTX::RecoveryJournal::PathFor(path), magic, /*durable=*/true)); + int64_t offset = static_cast(rec.header.size()); + for (int i = 0; i < committed_chunks; ++i) { + VTX::ChunkIndexData cd; + cd.chunk_index = i; + cd.file_offset = offset; + cd.start_frame = rec.chunks[i].start_frame; + cd.end_frame = rec.chunks[i].end_frame; + cd.chunk_size_bytes = rec.chunks[i].chunk_size_bytes; + cd.checksum = rec.chunks[i].checksum; + + std::vector times; + for (int f = cd.start_frame; f <= cd.end_frame; ++f) + times.push_back({f, rec.game_times[f], rec.created_utc[f]}); + journal.CommitChunk(cd, times); + offset += rec.chunks[i].chunk_size_bytes; + } + for (int p = 0; p < pending_frames; ++p) { + const int idx = last_committed_frame + 1 + p; + journal.AppendFrame(idx, rec.game_times[idx], rec.created_utc[idx], rec.payloads[idx]); + } journal.Close(); - return records; + return path; + } + + // Byte offset just past the last committed chunk (== end of the footerless body). + int64_t BodyEnd(const Recording& rec, int committed_chunks) { + int64_t end = static_cast(rec.header.size()); + for (int i = 0; i < committed_chunks; ++i) + end += rec.chunks[i].chunk_size_bytes; + return end; + } + + // Build a footerless .vtx + journal the way the sink actually does: for each + // single-frame chunk, AppendFrame THEN CommitChunk (so each committed frame's F + // record is superseded by its C record), then leave `pending` trailing frames + // un-committed. `compact_threshold` drives when the append-only journal reclaims + // superseded F records. Requires a per_chunk==1 recording. + std::string BuildInterleaved(const std::string& suffix, const Recording& rec, int committed, int pending, + uint64_t compact_threshold) { + const std::string path = VtxTest::OutputPath("fab_" + suffix + ".vtx"); + { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(rec.header.data(), static_cast(rec.header.size())); + for (int i = 0; i < committed; ++i) { + const uint32_t payload_size = rec.chunks[i].chunk_size_bytes - static_cast(sizeof(uint32_t)); + out.write(reinterpret_cast(&payload_size), sizeof(payload_size)); + out.write(rec.payloads[i].data(), static_cast(rec.payloads[i].size())); + } + } + + VTX::RecoveryJournal j; + EXPECT_TRUE(j.Open(VTX::RecoveryJournal::PathFor(path), rec.format_magic, /*durable=*/true)); + j.SetCompactThresholdBytes(compact_threshold); + int64_t offset = static_cast(rec.header.size()); + for (int i = 0; i < committed; ++i) { + j.AppendFrame(i, rec.game_times[i], rec.created_utc[i], rec.payloads[i]); + VTX::ChunkIndexData cd; + cd.chunk_index = i; + cd.file_offset = offset; + cd.start_frame = i; + cd.end_frame = i; + cd.chunk_size_bytes = rec.chunks[i].chunk_size_bytes; + cd.checksum = rec.chunks[i].checksum; + j.CommitChunk(cd, {{i, rec.game_times[i], rec.created_utc[i]}}); + offset += rec.chunks[i].chunk_size_bytes; + } + for (int p = 0; p < pending; ++p) { + const int idx = committed + p; + j.AppendFrame(idx, rec.game_times[idx], rec.created_utc[idx], rec.payloads[idx]); + } + j.Close(); + return path; } } // namespace -// A crash after N committed chunks (footerless file + journal) recovers all N. -TEST(CrashRecovery, RecoversAllChunksAfterCrash) { - const std::string path = WriteValidFile("all", 3); - MakeCrashState(path); +// --- Between chunks: every committed chunk is recovered ----------------------- - const std::string journal_path = VTX::RecoveryJournal::PathFor(path); - ASSERT_TRUE(std::filesystem::exists(journal_path)); +TEST(CrashRecovery, RecoversAllCommittedChunks) { + const Recording rec = CaptureRecording("all", 5, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("all", rec, /*committed=*/5, /*pending=*/0); + ASSERT_TRUE(std::filesystem::exists(VTX::RecoveryJournal::PathFor(path))); const auto rr = VTX::RepairReplayFile(path); ASSERT_TRUE(rr.ok()) << rr.error; EXPECT_TRUE(rr.repaired); - EXPECT_EQ(rr.recovered_chunks, 3); - EXPECT_EQ(rr.recovered_frames, 3); - EXPECT_FALSE(std::filesystem::exists(journal_path)); // deleted after repair + EXPECT_EQ(rr.recovered_chunks, 5); + EXPECT_EQ(rr.recovered_frames, 5); + EXPECT_FALSE(std::filesystem::exists(VTX::RecoveryJournal::PathFor(path))); auto ctx = VTX::OpenReplayFile(path); ASSERT_TRUE(ctx) << ctx.error; - EXPECT_EQ(ctx.reader->GetTotalFrames(), 3); - const VTX::Frame* f = ctx.reader->GetFrameSync(2); + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 5); + const VTX::Frame* f = ctx->GetFrameSync(4); ASSERT_NE(f, nullptr); ASSERT_EQ(f->GetBuckets().size(), 1u); ASSERT_EQ(f->GetBuckets()[0].entities.size(), 1u); - EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 2); // Score of frame 2 + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 4); // Score of frame 4 +} + +// Multi-frame committed chunks (3 frames each) recover all their frames. +TEST(CrashRecovery, RecoversMultiFrameChunks) { + const Recording rec = CaptureRecording("multi", 6, /*per_chunk=*/3, Fmt::Flat); + ASSERT_EQ(rec.chunks.size(), 2u); + const std::string path = FabricateCrash("multi", rec, /*committed=*/2, /*pending=*/0); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_chunks, 2); + EXPECT_EQ(rr.recovered_frames, 6); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 6); + const VTX::Frame* f = ctx->GetFrameSync(5); + ASSERT_NE(f, nullptr); + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 5); +} + +// --- Between frames: in-flight (un-flushed) frames are recovered from F records -- + +TEST(CrashRecovery, RecoversPendingFramesAfterLastChunk) { + const Recording rec = CaptureRecording("pending", 6, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("pending", rec, /*committed=*/3, /*pending=*/3); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_TRUE(rr.repaired); + EXPECT_EQ(rr.recovered_frames, 6); // 3 committed + 3 pending + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 6); + const VTX::Frame* f = ctx->GetFrameSync(5); // a recovered pending frame + ASSERT_NE(f, nullptr); + ASSERT_EQ(f->GetBuckets()[0].entities.size(), 1u); + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 5); +} + +// A crash before ANY chunk was flushed: only F records exist -> recover them all. +TEST(CrashRecovery, RecoversOnlyPendingWhenNothingFlushed) { + const Recording rec = CaptureRecording("onlypending", 4, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("onlypending", rec, /*committed=*/0, /*pending=*/4); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 4); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 4); + const VTX::Frame* f = ctx->GetFrameSync(0); + ASSERT_NE(f, nullptr); + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 0); } -// A torn final chunk (bytes missing) is dropped; the earlier chunks are recovered. +// --- Torn / corrupt tails: dropped, everything before is kept ------------------ + +// A crash mid-write of the last chunk (its bytes run past EOF): drop it, keep the rest. TEST(CrashRecovery, DropsTornTailChunk) { - const std::string path = WriteValidFile("torn", 3); - const auto records = MakeCrashState(path, /*truncate_extra=*/0); - // Shave the last chunk so its recorded extent runs past EOF. - const int64_t mid_last = records.back().file_offset + 4; // just past the size prefix + const Recording rec = CaptureRecording("torn", 4, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("torn", rec, /*committed=*/4, /*pending=*/0); + + // Truncate into the middle of the last chunk so its recorded extent exceeds EOF. + const int64_t mid_last = rec.chunks[3].file_offset + 6; std::filesystem::resize_file(path, static_cast(mid_last)); const auto rr = VTX::RepairReplayFile(path); ASSERT_TRUE(rr.ok()) << rr.error; - EXPECT_TRUE(rr.repaired); - EXPECT_EQ(rr.recovered_chunks, 2); // last chunk dropped + EXPECT_EQ(rr.recovered_chunks, 3); // torn last chunk dropped + EXPECT_EQ(rr.recovered_frames, 3); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 3); +} + +// A partial chunk written to the body before its journal (C) record existed: the +// journal lists fewer chunks than the body holds -> the un-journaled tail is dropped. +TEST(CrashRecovery, DropsPartialChunkWrittenBeforeJournalRecord) { + const Recording rec = CaptureRecording("partial", 4, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("partial", rec, /*committed=*/2, /*pending=*/0); + + // Append a half-written chunk framing (size prefix + a few payload bytes) that no + // C record covers -- exactly what a crash mid-chunk-write leaves after the body. + { + std::ofstream out(path, std::ios::binary | std::ios::app); + const uint32_t bogus_size = 4096; + out.write(reinterpret_cast(&bogus_size), sizeof(bogus_size)); + const char junk[7] = {1, 2, 3, 4, 5, 6, 7}; + out.write(junk, sizeof(junk)); + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_chunks, 2); // partial tail dropped EXPECT_EQ(rr.recovered_frames, 2); auto ctx = VTX::OpenReplayFile(path); ASSERT_TRUE(ctx) << ctx.error; - EXPECT_EQ(ctx.reader->GetTotalFrames(), 2); + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 2); } -// A chunk whose bytes were corrupted (checksum mismatch) stops recovery at it. +// A chunk whose on-disk bytes were corrupted (checksum mismatch) stops recovery at it. TEST(CrashRecovery, ChecksumDetectsCorruptChunk) { - const std::string path = WriteValidFile("corrupt", 3); - const auto records = MakeCrashState(path); - // Flip a byte inside chunk 1's payload (after its 4-byte size prefix). + const Recording rec = CaptureRecording("corrupt", 4, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("corrupt", rec, /*committed=*/4, /*pending=*/0); + + // Flip a byte inside chunk 1's payload (past its 4-byte size prefix). { std::fstream f(path, std::ios::binary | std::ios::in | std::ios::out); ASSERT_TRUE(f.is_open()); - const std::streamoff pos = static_cast(records[1].file_offset + 5); + const std::streamoff pos = static_cast(rec.chunks[1].file_offset + 5); char c = 0; f.seekg(pos); f.read(&c, 1); c = static_cast(c ^ 0xFF); f.seekp(pos); f.write(&c, 1); - f.flush(); } const auto rr = VTX::RepairReplayFile(path); ASSERT_TRUE(rr.ok()) << rr.error; EXPECT_EQ(rr.recovered_chunks, 1); // chunk 0 only; chunk 1 fails checksum -> stop + EXPECT_EQ(rr.recovered_frames, 1); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 1); +} + +// A crash mid-write of a frame record: the torn last F record is detected (per-record +// checksum) and dropped; the committed chunks and the earlier pending frames survive. +TEST(CrashRecovery, DropsTornPendingFrameRecord) { + const Recording rec = CaptureRecording("tornframe", 6, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("tornframe", rec, /*committed=*/2, /*pending=*/3); + + // Shave a few bytes off the journal tail: the last F record loses its checksum and + // is rejected as torn, but the record before it remains intact. + const std::string journal_path = VTX::RecoveryJournal::PathFor(path); + const auto jsz = std::filesystem::file_size(journal_path); + std::filesystem::resize_file(journal_path, static_cast(jsz - 4)); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 4); // 2 committed + 2 intact pending (last F torn) + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 4); +} + +// --- Footer windows ----------------------------------------------------------- + +// A crash mid-footer-write leaves garbage after the last chunk and no valid trailer: +// repair truncates the garbage and rebuilds a valid footer from the journal. +TEST(CrashRecovery, RepairsCrashDuringFooterWrite) { + const Recording rec = CaptureRecording("footer", 3, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("footer", rec, /*committed=*/3, /*pending=*/0); + + // Simulate a footer write interrupted before the [u32 size][magic] trailer landed. + { + std::ofstream out(path, std::ios::binary | std::ios::app); + const std::string partial_footer(64, '\xAB'); + out.write(partial_footer.data(), static_cast(partial_footer.size())); + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_TRUE(rr.repaired); + EXPECT_EQ(rr.recovered_chunks, 3); + EXPECT_EQ(rr.recovered_frames, 3); auto ctx = VTX::OpenReplayFile(path); ASSERT_TRUE(ctx) << ctx.error; - EXPECT_EQ(ctx.reader->GetTotalFrames(), 1); + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 3); } -// A crash between the footer fsync and the journal delete leaves a COMPLETE file -// plus a leftover .recovery. Repair must detect the valid footer and preserve it -// (incl. timing) rather than truncating and rewriting a timing-less one. +// A crash between the footer fsync and the journal delete leaves a COMPLETE file plus +// a leftover .recovery. Repair must detect the valid footer and preserve it (incl. +// timing) rather than truncating and rewriting a timing-less one. TEST(CrashRecovery, PreservesValidFooterWhenJournalLeftover) { - const std::string path = WriteValidFile("leftover", 3); + const Recording rec = CaptureRecording("leftover", 3, /*per_chunk=*/1, Fmt::Flat); + const std::string path = VtxTest::OutputPath("cap_leftover.vtx"); // the valid file capture wrote + ASSERT_TRUE(std::filesystem::exists(path)); + ASSERT_FALSE(std::filesystem::exists(VTX::RecoveryJournal::PathFor(path))); - // Recreate the leftover journal WITHOUT truncating the (valid) footer. + // Recreate a leftover journal (header only is enough; the valid-footer detection + // fires before the journal contents are ever consulted). { - std::vector records; - { - auto ctx = VTX::OpenReplayFile(path); - ASSERT_TRUE(ctx) << ctx.error; - for (const auto& e : ctx.reader->GetSeekTable()) { - VTX::ChunkIndexData d; - d.chunk_index = e.chunk_index; - d.start_frame = e.start_frame; - d.end_frame = e.end_frame; - d.file_offset = static_cast(e.file_offset); - d.chunk_size_bytes = e.chunk_size_bytes; - d.checksum = e.checksum; - records.push_back(d); - } - } - VTX::RecoveryJournal journal; - ASSERT_TRUE(journal.Open(VTX::RecoveryJournal::PathFor(path), "VTXF", true)); - for (const auto& d : records) - journal.AppendChunk(d); - journal.Close(); + VTX::RecoveryJournal j; + ASSERT_TRUE(j.Open(VTX::RecoveryJournal::PathFor(path), rec.format_magic, /*durable=*/true)); + j.Close(); } const auto size_before = std::filesystem::file_size(path); const auto rr = VTX::RepairReplayFile(path); ASSERT_TRUE(rr.ok()) << rr.error; - EXPECT_TRUE(rr.was_clean); // detected an already-complete file - EXPECT_FALSE(rr.repaired); // did NOT rewrite the footer + EXPECT_TRUE(rr.was_clean); // detected an already-complete file + EXPECT_FALSE(rr.repaired); // did NOT rewrite the footer EXPECT_EQ(std::filesystem::file_size(path), size_before); // footer left untouched EXPECT_FALSE(std::filesystem::exists(VTX::RecoveryJournal::PathFor(path))); auto ctx = VTX::OpenReplayFile(path); ASSERT_TRUE(ctx) << ctx.error; - EXPECT_EQ(ctx.reader->GetTotalFrames(), 3); + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 3); } -// A journal whose recorded format magic disagrees with the main file (e.g. a stale -// sidecar left over a file replaced with the other format) must be refused, not -// applied (which would truncate a valid file). +// A journal whose recorded format disagrees with the main file (a stale sidecar left +// over a file replaced with the other format) must be refused, not applied. TEST(CrashRecovery, RefusesMismatchedJournalFormat) { - const std::string path = WriteValidFile("mismatch", 2); - - std::vector records; - int64_t last_end = 0; - { - auto ctx = VTX::OpenReplayFile(path); - ASSERT_TRUE(ctx) << ctx.error; - for (const auto& e : ctx.reader->GetSeekTable()) { - VTX::ChunkIndexData d; - d.chunk_index = e.chunk_index; - d.start_frame = e.start_frame; - d.end_frame = e.end_frame; - d.file_offset = static_cast(e.file_offset); - d.chunk_size_bytes = e.chunk_size_bytes; - d.checksum = e.checksum; - records.push_back(d); - last_end = std::max(last_end, static_cast(e.file_offset) + e.chunk_size_bytes); - } - } - std::filesystem::resize_file(path, static_cast(last_end)); // footerless - { - VTX::RecoveryJournal journal; - // Wrong format magic: the file is VTXF (FlatBuffers). - ASSERT_TRUE(journal.Open(VTX::RecoveryJournal::PathFor(path), "VTXP", true)); - for (const auto& d : records) - journal.AppendChunk(d); - journal.Close(); - } + const Recording rec = CaptureRecording("mismatch", 2, /*per_chunk=*/1, Fmt::Flat); + // File is VTXF; journal claims VTXP. + const std::string path = FabricateCrash("mismatch", rec, /*committed=*/2, /*pending=*/0, "VTXP"); const auto rr = VTX::RepairReplayFile(path); EXPECT_FALSE(rr.ok()); // refused: journal format does not match the file @@ -251,7 +507,9 @@ TEST(CrashRecovery, RefusesMismatchedJournalFormat) { // A cleanly-closed file has no journal, so repair is a no-op and the file is intact. TEST(CrashRecovery, CleanFileNeedsNoRepair) { - const std::string path = WriteValidFile("clean", 3); + const Recording rec = CaptureRecording("clean", 3, /*per_chunk=*/1, Fmt::Flat); + const std::string path = VtxTest::OutputPath("cap_clean.vtx"); + ASSERT_TRUE(std::filesystem::exists(path)); ASSERT_FALSE(std::filesystem::exists(VTX::RecoveryJournal::PathFor(path))); const auto rr = VTX::RepairReplayFile(path); @@ -261,5 +519,1957 @@ TEST(CrashRecovery, CleanFileNeedsNoRepair) { auto ctx = VTX::OpenReplayFile(path); ASSERT_TRUE(ctx) << ctx.error; - EXPECT_EQ(ctx.reader->GetTotalFrames(), 3); + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 3); + (void)rec; +} + +// A crash right after the header (before any chunk or frame is durable) leaves a +// header-only journal: repair produces a valid, openable 0-frame file. +TEST(CrashRecovery, RecoversHeaderOnlyCrashAsEmptyFile) { + const Recording rec = CaptureRecording("headeronly", 2, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("headeronly", rec, /*committed=*/0, /*pending=*/0); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 0); + EXPECT_FALSE(std::filesystem::exists(VTX::RecoveryJournal::PathFor(path))); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 0); +} + +// Defensive: if F records for already-committed frames linger in the journal (a crash +// window CommitChunk normally forecloses by truncating first), they are deduped by +// index and NOT re-appended -- the frame count stays correct. +TEST(CrashRecovery, IgnoresLingeringFRecordsForCommittedFrames) { + const Recording rec = CaptureRecording("dedup", 4, /*per_chunk=*/1, Fmt::Flat); + const std::string path = VtxTest::OutputPath("fab_dedup.vtx"); + + // Footerless body: 2 committed chunks (frames 0 and 1). + { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(rec.header.data(), static_cast(rec.header.size())); + for (int i = 0; i < 2; ++i) { + const uint32_t payload_size = rec.chunks[i].chunk_size_bytes - static_cast(sizeof(uint32_t)); + out.write(reinterpret_cast(&payload_size), sizeof(payload_size)); + out.write(rec.payloads[i].data(), static_cast(rec.payloads[i].size())); + } + } + + VTX::RecoveryJournal j; + ASSERT_TRUE(j.Open(VTX::RecoveryJournal::PathFor(path), rec.format_magic, /*durable=*/true)); + int64_t offset = static_cast(rec.header.size()); + for (int i = 0; i < 2; ++i) { + VTX::ChunkIndexData cd; + cd.chunk_index = i; + cd.file_offset = offset; + cd.start_frame = rec.chunks[i].start_frame; + cd.end_frame = rec.chunks[i].end_frame; + cd.chunk_size_bytes = rec.chunks[i].chunk_size_bytes; + cd.checksum = rec.chunks[i].checksum; + j.CommitChunk(cd, {{i, rec.game_times[i], rec.created_utc[i]}}); + offset += rec.chunks[i].chunk_size_bytes; + } + // Lingering F records for the already-committed frames 0 and 1 (must be ignored)... + j.AppendFrame(0, rec.game_times[0], rec.created_utc[0], rec.payloads[0]); + j.AppendFrame(1, rec.game_times[1], rec.created_utc[1], rec.payloads[1]); + // ...plus genuine pending frames 2 and 3. + j.AppendFrame(2, rec.game_times[2], rec.created_utc[2], rec.payloads[2]); + j.AppendFrame(3, rec.game_times[3], rec.created_utc[3], rec.payloads[3]); + j.Close(); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 4); // 0,1 committed + 2,3 pending; lingering F for 0,1 ignored + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 4); + const VTX::Frame* f = ctx->GetFrameSync(3); + ASSERT_NE(f, nullptr); + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 3); +} + +// Append-only journal + compaction: superseded F records are reclaimed (the journal +// stays smaller than an un-compacted one) while recovery is byte-identical. Threshold +// 1 forces a compaction (close -> rewrite temp -> atomic rename -> reopen) on every +// commit, stressing that path; recovery must still yield every frame. +TEST(CrashRecovery, CompactionReclaimsSupersededFrames) { + const Recording rec = CaptureRecording("compact", 8, /*per_chunk=*/1, Fmt::Flat); + const std::string compacted = BuildInterleaved("compact_on", rec, /*committed=*/6, /*pending=*/2, /*threshold=*/1); + const std::string uncompacted = + BuildInterleaved("compact_off", rec, /*committed=*/6, /*pending=*/2, /*threshold=*/~uint64_t(0)); + + const auto sz_on = std::filesystem::file_size(VTX::RecoveryJournal::PathFor(compacted)); + const auto sz_off = std::filesystem::file_size(VTX::RecoveryJournal::PathFor(uncompacted)); + EXPECT_LT(sz_on, sz_off); // compaction dropped the 6 superseded F payloads + + for (const std::string& path : {compacted, uncompacted}) { + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 8) << path; + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 8) << path; + const VTX::Frame* f = ctx->GetFrameSync(7); // a pending frame that survived compaction + ASSERT_NE(f, nullptr) << path; + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 7) << path; + } +} + +// The user-driven helpers locate the sidecar and detect an unclean shutdown; after a +// successful repair the sidecar is gone so a re-check reports clean. +TEST(CrashRecovery, HelpersDetectAndLocateSidecar) { + const Recording rec = CaptureRecording("helpers", 3, /*per_chunk=*/1, Fmt::Flat); + + const std::string clean = VtxTest::OutputPath("cap_helpers.vtx"); // capture left a clean file + EXPECT_EQ(VTX::RecoveryJournalPath(clean), clean + ".recovery"); + EXPECT_FALSE(VTX::ReplayNeedsRecovery(clean)); + + const std::string crashed = FabricateCrash("helpers", rec, /*committed=*/3, /*pending=*/0); + EXPECT_TRUE(VTX::ReplayNeedsRecovery(crashed)); + + const auto rr = VTX::RepairReplayFile(crashed); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_FALSE(VTX::ReplayNeedsRecovery(crashed)); // sidecar removed after repair +} + +// --- Exact-time recovery ------------------------------------------------------ + +// Per-frame times survive recovery exactly, across both the committed (T record) and +// the pending (F record) reconstruction paths. +TEST(CrashRecovery, RecoveredTimesMatchOriginal) { + const Recording rec = CaptureRecording("times", 6, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("times", rec, /*committed=*/3, /*pending=*/3); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 6); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + const VTX::FileFooter footer = ctx->GetFooter(); + ASSERT_EQ(footer.times.game_time.size(), 6u); + for (int i = 0; i < 6; ++i) + EXPECT_EQ(static_cast(footer.times.game_time[i]), rec.game_times[i]) << "game_time[" << i << "]"; + if (!footer.times.created_utc.empty()) { + ASSERT_EQ(footer.times.created_utc.size(), 6u); + for (int i = 0; i < 6; ++i) + EXPECT_EQ(static_cast(footer.times.created_utc[i]), rec.created_utc[i]) + << "created_utc[" << i << "]"; + } +} + +// --- End-to-end through the REAL writer ---------------------------------------- +// +// The facade auto-Stops in its destructor, but a raw ReplayWriter does not: dropping +// it without Stop() leaves exactly what a crash leaves (header + fsync'd chunks + +// journal, no footer). These tests drive the real writer/sink/journal path -- no +// fabrication -- and require the recovered file to match a cleanly-stopped control +// byte-for-byte in every footer time field. + +namespace { + + template + using RawWriterFor = VTX::ReplayWriter>; + + template + std::unique_ptr> MakeRawWriter(const std::string& path, int32_t chunk_max_frames, + bool durable = true, bool compression = true, + bool journal = true, uint64_t compact_threshold = 0, + const std::string& schema_name = "test_schema.json") { + typename RawWriterFor::Config cfg; + cfg.sink_config.filename = path; + cfg.sink_config.header_config.replay_name = "CrashRecoveryE2E"; + cfg.sink_config.durable_writes = durable; + cfg.sink_config.b_use_compression = compression; + cfg.sink_config.enable_recovery_journal = journal; + cfg.sink_config.journal_compact_threshold_bytes = compact_threshold; + cfg.schema_json_path = VtxTest::FixturePath(schema_name); + cfg.default_fps = 60.0f; + cfg.chunker_config.max_frames = chunk_max_frames; + return std::make_unique>(cfg); + } + + // Records 8 frames with explicit game_time + created_utc: a UTC jump at frame 3 + // (timeline gap) and a game-time reversal at frame 5 (game segment). Chunks of 3 -> + // crash state = 2 committed chunks + 2 in-flight frames. + template + void RecordEightFrames(Writer& writer) { + constexpr int64_t kBaseUtc = 1'000'000'000'000; + constexpr int64_t kStepUtc = 166'667; // ~1/60s in ticks + const float game_times[8] = {0.f, 1.f / 60, 2.f / 60, 3.f / 60, 4.f / 60, 2.f / 60 /*reversal*/, + 5.f / 60, 6.f / 60}; + int64_t utc = kBaseUtc; + for (int i = 0; i < 8; ++i) { + utc += (i == 3) ? 10'000'000 : kStepUtc; // 1s jump at frame 3 -> gap + auto frame = BuildFrame(i); + VTX::GameTime::GameTimeRegister t; + t.game_time = game_times[i]; + t.created_utc_time = utc; + const auto res = writer.TryRecordFrame(frame, t); + ASSERT_TRUE(res.written) << "frame " << i << ": " << res.error.message; + } + } + + // Full crash-vs-control comparison through the real writer for one policy. + template + void RunRealWriterCrashVsControl(const std::string& prefix) { + // Control: identical inputs, clean Stop(). + const std::string control_path = VtxTest::OutputPath(prefix + "_e2e_control.vtx"); + { + auto w = MakeRawWriter(control_path, 3); + RecordEightFrames(*w); + w->Stop(); + } + ASSERT_FALSE(VTX::ReplayNeedsRecovery(control_path)); + + // Crash: same inputs, writer dropped without Stop(). + const std::string crash_path = VtxTest::OutputPath(prefix + "_e2e_crash.vtx"); + { + auto w = MakeRawWriter(crash_path, 3); + RecordEightFrames(*w); + // dropped here -- no Stop(), no footer + } + ASSERT_TRUE(VTX::ReplayNeedsRecovery(crash_path)); + + const auto rr = VTX::RepairReplayFile(crash_path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_TRUE(rr.repaired); + EXPECT_EQ(rr.recovered_frames, 8); // 6 committed (2 chunks of 3) + 2 in-flight + + auto control = VTX::OpenReplayFile(control_path); + ASSERT_TRUE(control) << control.error; + control->WaitUntilReady(); + auto recovered = VTX::OpenReplayFile(crash_path); + ASSERT_TRUE(recovered) << recovered.error; + recovered->WaitUntilReady(); + + EXPECT_EQ(recovered->GetTotalFrames(), control->GetTotalFrames()); + + // Every frame's content survives -- compare the full per-entity content_hash + // against the control, not just one property. + for (int i = 0; i < 8; ++i) { + const VTX::Frame* rfme = recovered->GetFrameSync(i); + const VTX::Frame* cfme = control->GetFrameSync(i); + ASSERT_NE(rfme, nullptr) << "frame " << i; + ASSERT_NE(cfme, nullptr) << "frame " << i; + EXPECT_EQ(rfme->GetBuckets()[0].entities[0].int32_properties[1], i) << "frame " << i; + EXPECT_EQ(rfme->GetBuckets()[0].entities[0].content_hash, cfme->GetBuckets()[0].entities[0].content_hash) + << "content_hash mismatch at frame " << i; + } + + // Seeks across the committed-chunk / recovered-chunk boundary in cache-hostile + // order (frames 0-5 live in 3-frame chunks, 6-7 in repair-appended 1-frame + // chunks): every jump must land on the right frame. GetFrameSync blocks until + // the chunk is loaded, so this is deterministic (GetFrameRange is best-effort + // by design -- it only returns already-cached frames). + for (int i : {7, 0, 6, 2, 5, 3, 7, 1}) { + const VTX::Frame* f = recovered->GetFrameSync(i); + ASSERT_NE(f, nullptr) << "seek to frame " << i; + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], i) << "seek to frame " << i; + } + + // The recovered file passes whole-replay validation (schema + every frame). + const VTX::ValidationReport report = VTX::ValidateReplayFile(crash_path); + EXPECT_FALSE(report.HasErrors()) << report.ToString(); + + // Every footer time field matches the clean control exactly: per-frame + // timestamps, duration, timeline gaps, and game segments. + const VTX::FileFooter cf = control->GetFooter(); + const VTX::FileFooter rf = recovered->GetFooter(); + ASSERT_EQ(cf.times.game_time.size(), 8u); + EXPECT_EQ(rf.times.game_time, cf.times.game_time); + ASSERT_EQ(cf.times.created_utc.size(), 8u); + EXPECT_EQ(rf.times.created_utc, cf.times.created_utc); + EXPECT_FALSE(cf.times.gaps.empty()); // the UTC jump must register in the control... + EXPECT_EQ(rf.times.gaps, cf.times.gaps); + EXPECT_FALSE(cf.times.segments.empty()); // ...and so must the game-time reversal + EXPECT_EQ(rf.times.segments, cf.times.segments); + EXPECT_FLOAT_EQ(rf.duration_seconds, cf.duration_seconds); + EXPECT_GT(rf.duration_seconds, 0.0f); + } + +} // namespace + +TEST(CrashRecoveryE2E, RealWriterCrashMatchesCleanStopExactly) { + RunRealWriterCrashVsControl("fb"); +} + +TEST(CrashRecoveryE2E, RealWriterCrashMatchesCleanStopExactly_Protobuf) { + RunRealWriterCrashVsControl("pb"); +} + +namespace { + + // A frame big enough (~4KB) that its serialized chunk clears the 512-byte floor and + // zstd compression actually engages -- the small frames used elsewhere are always + // stored raw, which would leave the compressed-payload path through the journal's F + // records and the repair checksums untested. + VTX::Frame BuildBigFrame(int i, std::string& out_blob) { + out_blob.clear(); + out_blob.reserve(4096); + while (out_blob.size() < 4000) + out_blob += "payload-" + std::to_string(i) + "-"; + VTX::Frame f; + auto& bucket = f.CreateBucket("entity"); + VTX::PropertyContainer pc; + pc.entity_type_id = 0; // Player + pc.string_properties = {"player_0", out_blob}; + pc.int32_properties = {1, i, 0}; + pc.float_properties = {100.0f, 50.0f}; + bucket.unique_ids.push_back("player_0"); + bucket.entities.push_back(std::move(pc)); + return f; + } + +} // namespace + +// Crash + recovery across the sink's whole configuration matrix (durable_writes x +// b_use_compression), with frames large enough that compression genuinely engages: +// committed chunks AND journaled F payloads take the zstd path, and repair's checksum +// verification runs over compressed bytes. +TEST(CrashRecoveryE2E, ConfigMatrixWithCompressedPayloads) { + struct Cfg { + bool durable; + bool compression; + const char* tag; + }; + const Cfg configs[] = {{true, true, "d1c1"}, {true, false, "d1c0"}, {false, true, "d0c1"}, {false, false, "d0c0"}}; + + for (const auto& c : configs) { + SCOPED_TRACE(c.tag); + const std::string path = VtxTest::OutputPath(std::string("matrix_") + c.tag + ".vtx"); + std::vector blobs(5); + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/2, c.durable, c.compression); + for (int i = 0; i < 5; ++i) { + auto frame = BuildBigFrame(i, blobs[static_cast(i)]); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + const auto res = w->TryRecordFrame(frame, t); + ASSERT_TRUE(res.written) << res.error.message; + } + // dropped without Stop(): 2 committed chunks (0-1, 2-3) + frame 4 in flight + } + ASSERT_TRUE(VTX::ReplayNeedsRecovery(path)); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 5); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 5); + for (int i : {0, 3, 4}) { // a committed frame, a chunk-1 frame, the recovered pending frame + const VTX::Frame* f = ctx->GetFrameSync(i); + ASSERT_NE(f, nullptr) << "frame " << i; + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], i); + EXPECT_EQ(f->GetBuckets()[0].entities[0].string_properties[1], blobs[static_cast(i)]) + << "big payload mismatch at frame " << i; + } + } +} + +// A crash DURING a previous repair (file already truncated, one pending frame already +// re-appended, footer not yet written, journal still present) must be repairable by +// simply running the repair again -- it re-truncates to the last committed chunk and +// re-appends everything. +TEST(CrashRecoveryE2E, InterruptedRepairRerunsCleanly) { + const Recording rec = CaptureRecording("interrupted", 5, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("interrupted", rec, /*committed=*/3, /*pending=*/2); + + // Simulate the first repair dying mid-way: it had appended pending frame 3 as a + // chunk (no C record covers it) and was killed before writing the footer. + { + std::ofstream out(path, std::ios::binary | std::ios::app); + const uint32_t payload_size = rec.chunks[3].chunk_size_bytes - static_cast(sizeof(uint32_t)); + out.write(reinterpret_cast(&payload_size), sizeof(payload_size)); + out.write(rec.payloads[3].data(), static_cast(rec.payloads[3].size())); + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 5); // 3 committed + both pending, no duplicates + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 5); + const VTX::Frame* f = ctx->GetFrameSync(4); + ASSERT_NE(f, nullptr); + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 4); +} + +// Repair is terminal: a second invocation reports clean and does not modify the file. +TEST(CrashRecoveryE2E, SecondRepairIsCleanNoOp) { + const Recording rec = CaptureRecording("secondrep", 3, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("secondrep", rec, /*committed=*/3, /*pending=*/0); + + const auto first = VTX::RepairReplayFile(path); + ASSERT_TRUE(first.ok()) << first.error; + EXPECT_TRUE(first.repaired); + const auto size_after = std::filesystem::file_size(path); + + const auto second = VTX::RepairReplayFile(path); + ASSERT_TRUE(second.ok()) << second.error; + EXPECT_TRUE(second.was_clean); + EXPECT_FALSE(second.repaired); + EXPECT_EQ(std::filesystem::file_size(path), size_after); // untouched + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 3); +} + +// With the journal opted out, a crash leaves a footerless file and NO sidecar: repair +// must report clean and leave the bytes untouched (no journal -> nothing to apply), +// and the reader must reject the footerless file gracefully rather than crash. +TEST(CrashRecoveryE2E, JournalDisabledCrashIsLeftUntouched) { + const std::string path = VtxTest::OutputPath("nojournal_crash.vtx"); + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/2, /*durable=*/true, + /*compression=*/true, /*journal=*/false); + RecordEightFrames(*w); + // dropped without Stop() + } + ASSERT_FALSE(VTX::ReplayNeedsRecovery(path)); // opted out -> no sidecar + + const auto bytes_before = VtxTest::ReadAllBytes(path); + ASSERT_FALSE(bytes_before.empty()); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_TRUE(rr.was_clean); // no journal -> repair has nothing to do + EXPECT_FALSE(rr.repaired); + EXPECT_EQ(VtxTest::ReadAllBytes(path), bytes_before); // byte-identical + + // The footerless file is rejected cleanly by the reader (no crash, no reader). + auto ctx = VTX::OpenReplayFile(path); + EXPECT_FALSE(ctx.Loaded()); +} + +// Crash in the middle of CommitChunk: the chunk's bytes are fsync'd in the .vtx but +// its C record is torn. Append-only journaling means the batch's F records are still +// present, so the frames are recovered from them -- nothing durable is lost. +TEST(CrashRecoveryE2E, TornCommitRecordFallsBackToFrameRecords) { + const Recording rec = CaptureRecording("torncommit", 4, /*per_chunk=*/1, Fmt::Flat); + // 3 committed chunks + F record for frame 3 (its chunk not yet committed). + const std::string path = + BuildInterleaved("torncommit", rec, /*committed=*/3, /*pending=*/1, /*threshold=*/~uint64_t(0)); + + // Simulate the crash window: chunk 3's bytes reached the .vtx (data-before-journal)... + { + std::ofstream out(path, std::ios::binary | std::ios::app); + const uint32_t payload_size = rec.chunks[3].chunk_size_bytes - static_cast(sizeof(uint32_t)); + out.write(reinterpret_cast(&payload_size), sizeof(payload_size)); + out.write(rec.payloads[3].data(), static_cast(rec.payloads[3].size())); + } + // ...but its C record tore mid-write (header promises 32 payload bytes; EOF cuts it). + { + std::ofstream j(VTX::RecoveryJournal::PathFor(path), std::ios::binary | std::ios::app); + const char head[5] = {'C', 32, 0, 0, 0}; + j.write(head, sizeof(head)); + const char partial[12] = {3, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0}; + j.write(partial, sizeof(partial)); + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 4); // frame 3 recovered from its F record + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 4); + const VTX::Frame* f = ctx->GetFrameSync(3); + ASSERT_NE(f, nullptr); + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 3); +} + +// A crash during journal compaction can leave a stale ".recovery.compact" temp next +// to the (still intact) journal. Repair must ignore it, succeed from the journal, and +// clean both up. +TEST(CrashRecoveryE2E, StaleCompactTempIsIgnoredAndCleaned) { + const Recording rec = CaptureRecording("staletemp", 3, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("staletemp", rec, /*committed=*/3, /*pending=*/0); + + const std::string journal_path = VTX::RecoveryJournal::PathFor(path); + const std::string temp_path = VTX::RecoveryJournal::CompactTempFor(journal_path); + { + std::ofstream tmp(temp_path, std::ios::binary); + tmp << "half-written compaction temp"; + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 3); + EXPECT_FALSE(std::filesystem::exists(journal_path)); + EXPECT_FALSE(std::filesystem::exists(temp_path)); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 3); +} + +// A crash at the very instant the session opened: the .vtx holds only its header and +// the journal's 'S' record tore mid-write. Per design, "no chunks yet" recovers to a +// valid, openable 0-frame file (the journal header is intact, so repair proceeds with +// an empty record set). +TEST(CrashRecoveryE2E, SessionStartCrashTornTimingRecordYieldsEmptyFile) { + const Recording rec = CaptureRecording("sessionstart", 2, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("sessionstart", rec, /*committed=*/0, /*pending=*/0); + + // Cut into the 'S' record, 3 bytes past the journal header: [VTXR|ver|magic| S.. + const std::string journal_path = VTX::RecoveryJournal::PathFor(path); + std::filesystem::resize_file(journal_path, VTX::RecoveryJournal::kHeaderSize + 3); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 0); + EXPECT_FALSE(std::filesystem::exists(journal_path)); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 0); +} + +// A crash tearing the T records of a commit (C durable, T torn): the frame times must +// fall back to the still-present F records (append-only keeps them), so the recovered +// footer's timestamps stay exact -- not zeros. +TEST(CrashRecoveryE2E, TornTimeRecordsFallBackToFrameRecordTimes) { + const Recording rec = CaptureRecording("torntime", 2, /*per_chunk=*/1, Fmt::Flat); + // 1 committed chunk (frame 0): journal = [S][F0][C0][T0]. + const std::string path = BuildInterleaved("torntime", rec, /*committed=*/1, /*pending=*/0, + /*threshold=*/~uint64_t(0)); + + // Tear into T0 (the last record: [u8 'T'][u32 20][20-byte payload][u64 checksum]). + const std::string journal_path = VTX::RecoveryJournal::PathFor(path); + const auto jsz = std::filesystem::file_size(journal_path); + std::filesystem::resize_file(journal_path, jsz - 20); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 1); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 1); + const VTX::FileFooter footer = ctx->GetFooter(); + ASSERT_EQ(footer.times.game_time.size(), 1u); + // Exact time recovered from the F record despite the torn T record. + EXPECT_EQ(static_cast(footer.times.game_time[0]), rec.game_times[0]); + ASSERT_EQ(footer.times.created_utc.size(), 1u); + EXPECT_EQ(static_cast(footer.times.created_utc[0]), rec.created_utc[0]); +} + +// A 0-byte journal (crash between the sidecar's creation and its header write) is +// refused conservatively -- and the main file must not be touched. +TEST(CrashRecoveryE2E, EmptyJournalIsRefusedWithoutTouchingFile) { + const Recording rec = CaptureRecording("emptyjournal", 2, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("emptyjournal", rec, /*committed=*/2, /*pending=*/0); + + // Replace the journal with an empty file. + const std::string journal_path = VTX::RecoveryJournal::PathFor(path); + { std::ofstream truncate_it(journal_path, std::ios::binary | std::ios::trunc); } + ASSERT_EQ(std::filesystem::file_size(journal_path), 0u); + + const auto bytes_before = VtxTest::ReadAllBytes(path); + const auto rr = VTX::RepairReplayFile(path); + EXPECT_FALSE(rr.ok()); // unreadable journal header -> refused + EXPECT_FALSE(rr.repaired); + EXPECT_EQ(VtxTest::ReadAllBytes(path), bytes_before); // main file untouched +} + +// Compaction through the REAL sink (not just the journal API): with the sink's +// compaction threshold forced to 1 byte, every commit triggers a full compaction +// cycle (close -> rewrite -> atomic rename -> reopen) during recording; a crash after +// that must still recover every frame with exact times. +TEST(CrashRecoveryE2E, RealSinkCompactionCrashRecovery) { + const std::string path = VtxTest::OutputPath("sink_compact.vtx"); + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/2, /*durable=*/true, + /*compression=*/true, /*journal=*/true, + /*compact_threshold=*/1); + for (int i = 0; i < 7; ++i) { + auto frame = BuildFrame(i); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + const auto res = w->TryRecordFrame(frame, t); + ASSERT_TRUE(res.written) << res.error.message; + } + // dropped: 3 committed chunks (each triggering a compaction) + frame 6 in flight + } + ASSERT_TRUE(VTX::ReplayNeedsRecovery(path)); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 7); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 7); + for (int i : {0, 5, 6}) { + const VTX::Frame* f = ctx->GetFrameSync(i); + ASSERT_NE(f, nullptr) << "frame " << i; + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], i); + } + const VTX::FileFooter footer = ctx->GetFooter(); + ASSERT_EQ(footer.times.game_time.size(), 7u); // exact per-frame times survived compaction +} + +namespace { + + // Little-endian append helpers for hand-rolled journal records. + void PushI32(std::vector& b, int32_t v) { + for (int i = 0; i < 4; ++i) + b.push_back(static_cast(static_cast(v) >> (8 * i))); + } + void PushI64(std::vector& b, int64_t v) { + for (int i = 0; i < 8; ++i) + b.push_back(static_cast(static_cast(v) >> (8 * i))); + } + + // Append one well-formed (checksummed) record to an existing journal file -- + // used to inject hostile-but-valid records the write API would never produce. + void AppendRawJournalRecord(const std::string& journal_path, char type, const std::vector& payload) { + uint8_t head[5]; + head[0] = static_cast(type); + const uint32_t len = static_cast(payload.size()); + std::memcpy(head + 1, &len, sizeof(len)); + XXH3_state_t state; + XXH3_64bits_reset(&state); + XXH3_64bits_update(&state, head, 5); + if (!payload.empty()) + XXH3_64bits_update(&state, payload.data(), payload.size()); + const uint64_t checksum = XXH3_64bits_digest(&state); + std::ofstream j(journal_path, std::ios::binary | std::ios::app); + j.write(reinterpret_cast(head), 5); + if (!payload.empty()) + j.write(reinterpret_cast(payload.data()), static_cast(payload.size())); + j.write(reinterpret_cast(&checksum), sizeof(checksum)); + } + +} // namespace + +// Checksummed-but-hostile C records (an offset pointing inside the header; a size no +// bigger than its own length prefix) must be dropped by the repair guards, degrading +// to a valid 0-frame file rather than corrupting or crashing. +TEST(CrashRecoveryE2E, BogusJournalChunkRecordsAreDroppedSafely) { + const Recording rec = CaptureRecording("boguschunk", 2, /*per_chunk=*/1, Fmt::Flat); + + struct Bogus { + const char* tag; + int64_t offset; + uint32_t size; + }; + const Bogus cases[] = { + {"offset_inside_header", 4, 64}, // points into the .vtx header region + {"size_below_prefix", 100, 4}, // chunk cannot even hold its length prefix + }; + for (const auto& c : cases) { + SCOPED_TRACE(c.tag); + // Header-only body + a journal whose only C record is hostile. + const std::string path = FabricateCrash(std::string("bogus_") + c.tag, rec, /*committed=*/0, /*pending=*/0); + VTX::ChunkIndexData cd; + cd.chunk_index = 0; + cd.file_offset = c.offset; + cd.start_frame = 0; + cd.end_frame = 0; + cd.chunk_size_bytes = c.size; + cd.checksum = 0; + { + // Rebuild the journal with the hostile record (Open truncates the sidecar). + VTX::RecoveryJournal j; + ASSERT_TRUE(j.Open(VTX::RecoveryJournal::PathFor(path), rec.format_magic, /*durable=*/true)); + j.CommitChunk(cd, {{0, rec.game_times[0], rec.created_utc[0]}}); + j.Close(); + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_chunks, 0); // hostile record dropped + EXPECT_EQ(rr.recovered_frames, 0); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 0); // degraded to a valid empty file + } +} + +// An F record with an EMPTY frame payload (valid checksum, nothing to append) must +// stop the pending-frame walk at that hole without corrupting what came before. +TEST(CrashRecoveryE2E, EmptyPendingFramePayloadStopsPendingRecovery) { + const Recording rec = CaptureRecording("emptyf", 3, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("emptyf", rec, /*committed=*/1, /*pending=*/0); + + // Hand-append: F(frame 1, EMPTY payload), then a well-formed F(frame 2). The empty + // one is a hole, so frame 2 must NOT be applied either (indices would gap). + const std::string journal_path = VTX::RecoveryJournal::PathFor(path); + { + std::vector empty_f; + PushI32(empty_f, 1); + PushI64(empty_f, rec.game_times[1]); + PushI64(empty_f, rec.created_utc[1]); // no payload bytes after the 20-byte prefix + AppendRawJournalRecord(journal_path, 'F', empty_f); + + std::vector good_f; + PushI32(good_f, 2); + PushI64(good_f, rec.game_times[2]); + PushI64(good_f, rec.created_utc[2]); + good_f.insert(good_f.end(), rec.payloads[2].begin(), rec.payloads[2].end()); + AppendRawJournalRecord(journal_path, 'F', good_f); + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 1); // committed chunk only; the empty F is a hole + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 1); +} + +// A main file whose own header tore (smaller than magic + size prefix) with a valid +// journal alongside: repair must refuse gracefully and leave the bytes alone. +TEST(CrashRecoveryE2E, TornMainHeaderWithJournalIsRefused) { + const Recording rec = CaptureRecording("tornhdr", 2, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("tornhdr", rec, /*committed=*/1, /*pending=*/0); + std::filesystem::resize_file(path, 6); // magic + half the header_size field + + const auto bytes_before = VtxTest::ReadAllBytes(path); + const auto rr = VTX::RepairReplayFile(path); + EXPECT_FALSE(rr.ok()); // header framing not intact -> refused + EXPECT_FALSE(rr.repaired); + EXPECT_EQ(VtxTest::ReadAllBytes(path), bytes_before); // untouched +} + +namespace { + + // Records 8 frames with DECREASING game time (is_increasing = false) and one + // increase at frame 5 -> a game segment in decreasing mode. Only game_time is + // supplied (UTC is faked by the timer), matching how a rewind-style recording + // would drive the writer. + template + void RecordEightDecreasingFrames(Writer& writer) { + const float game_times[8] = {8.f / 60, 7.f / 60, 6.f / 60, 5.f / 60, 4.f / 60, 6.f / 60 /*reversal*/, + 3.f / 60, 2.f / 60}; + for (int i = 0; i < 8; ++i) { + auto frame = BuildFrame(i); + VTX::GameTime::GameTimeRegister t; + t.game_time = game_times[i]; + const auto res = writer.TryRecordFrame(frame, t); + ASSERT_TRUE(res.written) << "frame " << i << ": " << res.error.message; + } + } + +} // namespace + +// DECREASING-time recordings (is_increasing = false) exercise the other branch of the +// segment reconstruction: after a crash, the recovered game times, game segments and +// duration must match a cleanly-stopped control. (Absolute created_utc values are +// faked from the wall clock and differ run-to-run, so only their count is compared.) +TEST(CrashRecoveryE2E, DecreasingTimeCrashMatchesCleanStop) { + auto make_writer = [](const std::string& path) { + using Policy = VTX::FlatBuffersVtxPolicy; + typename RawWriterFor::Config cfg; + cfg.sink_config.filename = path; + cfg.sink_config.header_config.replay_name = "CrashRecoveryE2E"; + cfg.schema_json_path = VtxTest::FixturePath("test_schema.json"); + cfg.default_fps = 60.0f; + cfg.is_increasing = false; + cfg.chunker_config.max_frames = 3; + return std::make_unique>(cfg); + }; + + const std::string control_path = VtxTest::OutputPath("dec_control.vtx"); + { + auto w = make_writer(control_path); + RecordEightDecreasingFrames(*w); + w->Stop(); + } + const std::string crash_path = VtxTest::OutputPath("dec_crash.vtx"); + { + auto w = make_writer(crash_path); + RecordEightDecreasingFrames(*w); + // dropped without Stop() + } + + const auto rr = VTX::RepairReplayFile(crash_path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 8); + + auto control = VTX::OpenReplayFile(control_path); + ASSERT_TRUE(control) << control.error; + control->WaitUntilReady(); + auto recovered = VTX::OpenReplayFile(crash_path); + ASSERT_TRUE(recovered) << recovered.error; + recovered->WaitUntilReady(); + + const VTX::FileFooter cf = control->GetFooter(); + const VTX::FileFooter rf = recovered->GetFooter(); + ASSERT_EQ(cf.times.game_time.size(), 8u); + EXPECT_EQ(rf.times.game_time, cf.times.game_time); + EXPECT_EQ(rf.times.created_utc.size(), cf.times.created_utc.size()); + EXPECT_FALSE(cf.times.segments.empty()); // the mid-run increase registers in decreasing mode + EXPECT_EQ(rf.times.segments, cf.times.segments); + EXPECT_FLOAT_EQ(rf.duration_seconds, cf.duration_seconds); +} + +namespace { + + // A Player carrying a Map field (AmmoByWeapon: weapon -> AmmoEntry), per + // test_schema_map.json. Mirrors the differ's map fixture. + VTX::PropertyContainer MakeMapPlayer(int i) { + VTX::PropertyContainer pc; + pc.entity_type_id = 0; // Player + pc.string_properties = {"map_player", "name"}; + pc.int32_properties = {1, i, 0}; + pc.float_properties = {100.0f, 50.0f}; + pc.vector_properties = {VTX::Vector {}, VTX::Vector {}}; + pc.quat_properties = {VTX::Quat {}}; + pc.bool_properties = {true}; + + VTX::MapContainer ammo; + ammo.keys.push_back("Rifle"); + VTX::PropertyContainer rifle; + rifle.entity_type_id = 5; // AmmoEntry + rifle.string_properties = {"Rifle"}; + rifle.int32_properties = {30 - i, 90}; // clip drains per frame + rifle.content_hash = VTX::Helpers::CalculateContainerHash(rifle); + ammo.values.push_back(std::move(rifle)); + pc.map_properties.push_back(std::move(ammo)); + + pc.content_hash = VTX::Helpers::CalculateContainerHash(pc); + return pc; + } + +} // namespace + +namespace { + + // Frames with MAP containers survive crash + recovery intact -- including a frame + // that only ever existed as a journaled F record (the pending one). + template + void RunMapCrashRecovery(const std::string& prefix) { + const std::string path = VtxTest::OutputPath(prefix + "_map_crash.vtx"); + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/2, /*durable=*/true, + /*compression=*/true, /*journal=*/true, + /*compact_threshold=*/0, "test_schema_map.json"); + for (int i = 0; i < 3; ++i) { + VTX::Frame frame; + auto& b = frame.CreateBucket("entity"); + b.unique_ids.push_back("map_player"); + b.entities.push_back(MakeMapPlayer(i)); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + const auto res = w->TryRecordFrame(frame, t); + ASSERT_TRUE(res.written) << res.error.message; + } + // dropped: 1 committed chunk (frames 0-1) + frame 2 in flight + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 3); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 3); + for (int i : {0, 2}) { // a committed frame and the F-record-recovered pending frame + const VTX::Frame* f = ctx->GetFrameSync(i); + ASSERT_NE(f, nullptr) << "frame " << i; + const auto& entity = f->GetBuckets()[0].entities[0]; + ASSERT_FALSE(entity.map_properties.empty()) << "frame " << i; + const auto& ammo = entity.map_properties[0]; + ASSERT_EQ(ammo.keys.size(), 1u) << "frame " << i; + EXPECT_EQ(ammo.keys[0], "Rifle"); + ASSERT_EQ(ammo.values.size(), 1u); + EXPECT_EQ(ammo.values[0].int32_properties[0], 30 - i) << "frame " << i; + } + } + +} // namespace + +TEST(CrashRecoveryE2E, MapFramesSurviveCrashRecovery) { + RunMapCrashRecovery("fb"); +} + +TEST(CrashRecoveryE2E, MapFramesSurviveCrashRecovery_Protobuf) { + RunMapCrashRecovery("pb"); +} + +// A crashed session leaves a sidecar; a LATER session to the same path that OPTS OUT +// of journaling must clear it at session start -- otherwise the stale journal would +// masquerade as recovery state for the new recording. +TEST(CrashRecoveryE2E, StaleSidecarRemovedWhenJournalingOptedOut) { + const std::string path = VtxTest::OutputPath("stale_sidecar.vtx"); + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/2); + for (int i = 0; i < 3; ++i) { + auto frame = BuildFrame(i); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + ASSERT_TRUE(w->TryRecordFrame(frame, t).written); + } + // session 1 crashes -> sidecar left behind + } + ASSERT_TRUE(VTX::ReplayNeedsRecovery(path)); + + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/2, /*durable=*/true, + /*compression=*/true, /*journal=*/false); + // The opted-out session must have removed the stale sidecar at start. + EXPECT_FALSE(VTX::ReplayNeedsRecovery(path)); + auto frame = BuildFrame(0); + VTX::GameTime::GameTimeRegister t; + t.game_time = 0.0f; + ASSERT_TRUE(w->TryRecordFrame(frame, t).written); + // session 2 crashes too -- journaling was off, so no sidecar may appear + } + EXPECT_FALSE(VTX::ReplayNeedsRecovery(path)); // no stale recovery signal +} + +// A user file that merely shares the ".recovery" suffix (not a journal -- no "VTXR" +// magic) must never be deleted by repair, even on the clean-file path that normally +// cleans the sidecar up. +TEST(CrashRecoveryE2E, ForeignRecoveryFileIsNeverDeleted) { + const Recording rec = CaptureRecording("foreign", 2, /*per_chunk=*/1, Fmt::Flat); + const std::string path = VtxTest::OutputPath("cap_foreign.vtx"); // valid, cleanly closed + ASSERT_TRUE(std::filesystem::exists(path)); + + const std::string foreign_path = VTX::RecoveryJournalPath(path); + const std::string foreign_content = "user notes -- definitely not a VTXR journal"; + { + std::ofstream f(foreign_path, std::ios::binary); + f << foreign_content; + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_TRUE(rr.was_clean); + ASSERT_TRUE(std::filesystem::exists(foreign_path)); // preserved, not cleaned up + { + std::ifstream f(foreign_path, std::ios::binary); + std::string readback((std::istreambuf_iterator(f)), std::istreambuf_iterator()); + EXPECT_EQ(readback, foreign_content); + } + (void)rec; +} + +// A checksummed T record whose frame index is out of range (beyond any recovered +// frame) must be ignored by the bounds check, leaving the valid times untouched. +TEST(CrashRecoveryE2E, HostileTimeRecordIndexOutOfRangeIsIgnored) { + const Recording rec = CaptureRecording("hostilet", 2, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("hostilet", rec, /*committed=*/2, /*pending=*/0); + + std::vector hostile_t; + PushI32(hostile_t, 9999); // far beyond total_frames + PushI64(hostile_t, 123456789); + PushI64(hostile_t, 987654321); + AppendRawJournalRecord(VTX::RecoveryJournal::PathFor(path), 'T', hostile_t); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 2); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + const VTX::FileFooter footer = ctx->GetFooter(); + ASSERT_EQ(footer.times.game_time.size(), 2u); // not resized by the hostile index + for (int i = 0; i < 2; ++i) + EXPECT_EQ(static_cast(footer.times.game_time[static_cast(i)]), rec.game_times[i]); +} + +// A stale journal from a DIFFERENT recording (same format, e.g. the file was replaced +// out-of-band) must degrade gracefully: the per-chunk checksums reject the foreign +// chunks and repair yields a valid (empty) file instead of corrupt data or a crash. +TEST(CrashRecoveryE2E, StaleJournalFromDifferentRecordingDegradesGracefully) { + const Recording rec_a = CaptureRecording("stalex_a", 2, /*per_chunk=*/1, Fmt::Flat); + // Same shape, different content (scores 50/51 vs 0/1) -> same offsets, different bytes. + Recording rec_b; + { + VTX::WriterFacadeConfig cfg; + cfg.output_filepath = VtxTest::OutputPath("cap_stalex_b.vtx"); + cfg.schema_json_path = VtxTest::FixturePath("test_schema.json"); + cfg.replay_name = "CrashRecoveryTest"; + auto writer = VTX::CreateFlatBuffersWriterFacade(cfg); + ASSERT_NE(writer, nullptr); + for (int i = 0; i < 2; ++i) { + auto frame = BuildFrame(50 + i); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + writer->RecordFrame(frame, t); + writer->Flush(); + } + writer->Stop(); + } + + // Crash state whose BODY is from recording B... + const std::string path = VtxTest::OutputPath("fab_stalex.vtx"); + { + const std::string b_path = VtxTest::OutputPath("cap_stalex_b.vtx"); + std::ifstream in(b_path, std::ios::binary); + std::string all((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + // keep header + everything up to the footer region intact enough: reuse A's + // chunk extents (identical layout) to compute the footerless length + int64_t body_end = static_cast(rec_a.header.size()); + for (const auto& c : rec_a.chunks) + body_end += c.chunk_size_bytes; + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(all.data(), std::min(static_cast(all.size()), body_end)); + } + // ...but whose journal carries recording A's chunk records (stale metadata). + { + VTX::RecoveryJournal j; + ASSERT_TRUE(j.Open(VTX::RecoveryJournal::PathFor(path), "VTXF", /*durable=*/true)); + for (size_t i = 0; i < rec_a.chunks.size(); ++i) + j.CommitChunk(rec_a.chunks[i], {{static_cast(i), rec_a.game_times[i], rec_a.created_utc[i]}}); + j.Close(); + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_chunks, 0); // foreign chunks rejected by checksum + EXPECT_EQ(rr.recovered_frames, 0); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; // degraded to a valid, openable empty file + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 0); +} + +// A crashed file that sits READ-ONLY (archived, restored from backup) must make +// repair fail gracefully -- and once the file is writable again, the SAME journal +// must still drive a full recovery (the refusal destroyed nothing). +TEST(CrashRecoveryE2E, ReadOnlyFileRepairFailsGracefullyThenSucceeds) { + const Recording rec = CaptureRecording("readonly", 3, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("readonly", rec, /*committed=*/3, /*pending=*/0); + + // MSVC's std::filesystem maps the Windows READONLY attribute to the write bits + // COLLECTIVELY -- all three must be removed for the file to become read-only. + const auto all_write = std::filesystem::perms::owner_write | std::filesystem::perms::group_write | + std::filesystem::perms::others_write; + std::filesystem::permissions(path, all_write, std::filesystem::perm_options::remove); + const auto rr_denied = VTX::RepairReplayFile(path); + // Restore write access BEFORE asserting so a failure can't leave a read-only + // artifact behind for later runs. + std::filesystem::permissions(path, all_write, std::filesystem::perm_options::add); + + EXPECT_FALSE(rr_denied.ok()); // cannot truncate/append a read-only file + EXPECT_FALSE(rr_denied.repaired); + EXPECT_TRUE(VTX::ReplayNeedsRecovery(path)); // journal preserved for a retry + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 3); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 3); +} + +// A recording that never supplies ANY time registry (EGameTimeType::None -- both +// game_time and created_utc synthesized from FPS) crashes and recovers with the +// exact synthesized timeline: game times in fps_inverse steps from 0, monotonic +// created_utc, and a coherent duration. +TEST(CrashRecoveryE2E, NoTimeRegistryCrashRecovery) { + const std::string path = VtxTest::OutputPath("notime_crash.vtx"); + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/2); + for (int i = 0; i < 5; ++i) { + auto frame = BuildFrame(i); + VTX::GameTime::GameTimeRegister t; // both fields nullopt -> fully faked times + ASSERT_TRUE(w->TryRecordFrame(frame, t).written); + } + // dropped: 2 committed chunks + 1 in-flight frame + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 5); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 5); + + const VTX::FileFooter footer = ctx->GetFooter(); + ASSERT_EQ(footer.times.game_time.size(), 5u); + ASSERT_EQ(footer.times.created_utc.size(), 5u); + // FakeBothTimesFromFPS: game_time starts at 0 and advances fps_inverse per frame. + const int64_t fps_inverse = + static_cast((1.0f / 60.0f) * static_cast(VTX::GameTime::TICKS_PER_SECOND)); + for (int i = 0; i < 5; ++i) + EXPECT_EQ(static_cast(footer.times.game_time[static_cast(i)]), i * fps_inverse) + << "game_time[" << i << "]"; + for (int i = 1; i < 5; ++i) + EXPECT_EQ(footer.times.created_utc[static_cast(i)] - + footer.times.created_utc[static_cast(i) - 1], + static_cast(fps_inverse)) + << "created_utc step at " << i; + EXPECT_NEAR(footer.duration_seconds, 4.0f / 60.0f, 1e-4f); +} + +// Chunk splitting driven by the BYTE budget (max_bytes) rather than the frame count +// -- the other branch of ThresholdChunkPolicy -- through crash + recovery. +TEST(CrashRecoveryE2E, ByteBudgetChunkingCrashRecovery) { + const std::string path = VtxTest::OutputPath("bytebudget_crash.vtx"); + std::vector blobs(7); + { + using Policy = VTX::FlatBuffersVtxPolicy; + typename RawWriterFor::Config cfg; + cfg.sink_config.filename = path; + cfg.sink_config.header_config.replay_name = "CrashRecoveryE2E"; + cfg.schema_json_path = VtxTest::FixturePath("test_schema.json"); + cfg.default_fps = 60.0f; + cfg.chunker_config.max_frames = 1000; // never reached + cfg.chunker_config.max_bytes = 10'000; // ~2 big frames per chunk + auto w = std::make_unique>(cfg); + for (int i = 0; i < 7; ++i) { + auto frame = BuildBigFrame(i, blobs[static_cast(i)]); // ~4KB each + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + ASSERT_TRUE(w->TryRecordFrame(frame, t).written); + } + // dropped without Stop() + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 7); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 7); + EXPECT_GE(ctx->GetSeekTable().size(), 3u); // byte budget actually split the chunks + for (int i : {0, 3, 6}) { + const VTX::Frame* f = ctx->GetFrameSync(i); + ASSERT_NE(f, nullptr) << "frame " << i; + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], i); + EXPECT_EQ(f->GetBuckets()[0].entities[0].string_properties[1], blobs[static_cast(i)]) + << "payload mismatch at frame " << i; + } +} + +namespace { + + // The maximal assertion: a crash EXACTLY at a chunk boundary (all frames flushed, + // nothing in flight) must recover to a file that is BYTE-IDENTICAL to one produced + // by a clean Stop() with the same inputs -- same chunks, same seek table, same + // footer (times, gaps, segments, duration, compression), same trailer. + template + void RunBoundaryByteIdentity(const std::string& prefix, int frames, int32_t chunk_max) { + auto record_all = [frames](RawWriterFor& w) { + constexpr int64_t kBaseUtc = 4'000'000'000'000; + for (int i = 0; i < frames; ++i) { + auto frame = BuildFrame(i); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + t.created_utc_time = kBaseUtc + (i + 1) * 166'667; + ASSERT_TRUE(w.TryRecordFrame(frame, t).written); + } + w.Flush(); // land the trailing batch -> crash sits exactly on a chunk boundary + }; + + const std::string control_path = VtxTest::OutputPath(prefix + "_ident_control.vtx"); + { + auto w = MakeRawWriter(control_path, chunk_max); + record_all(*w); + w->Stop(); + } + const std::string crash_path = VtxTest::OutputPath(prefix + "_ident_crash.vtx"); + { + auto w = MakeRawWriter(crash_path, chunk_max); + record_all(*w); + // dropped without Stop(): every frame is in a committed chunk, none pending + } + + const auto rr = VTX::RepairReplayFile(crash_path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, frames); + + const auto control_bytes = VtxTest::ReadAllBytes(control_path); + const auto recovered_bytes = VtxTest::ReadAllBytes(crash_path); + ASSERT_FALSE(control_bytes.empty()); + ASSERT_GE(control_bytes.size(), 8u); + ASSERT_GE(recovered_bytes.size(), 8u); + + // The header embeds a second-granularity recording timestamp, so two separately + // recorded sessions may legitimately differ there (repair never touches the + // header -- the recovered file keeps its own, which IS the guarantee). Compare + // everything from the end of the header on: chunks, seek table, footer, trailer. + auto header_end = [](const std::vector& bytes) { + uint32_t header_size = 0; + std::memcpy(&header_size, bytes.data() + 4, sizeof(header_size)); + return static_cast(8) + header_size; + }; + const size_t control_body = header_end(control_bytes); + ASSERT_EQ(header_end(recovered_bytes), control_body); // same header structure/size + ASSERT_LT(control_body, control_bytes.size()); + + EXPECT_EQ(recovered_bytes.size(), control_bytes.size()); + const bool body_identical = + recovered_bytes.size() == control_bytes.size() && + std::equal(control_bytes.begin() + static_cast(control_body), control_bytes.end(), + recovered_bytes.begin() + static_cast(control_body)); + EXPECT_TRUE(body_identical); // chunks + footer + trailer bit-for-bit equal + } + +} // namespace + +TEST(CrashRecoveryE2E, BoundaryCrashRecoversByteIdenticalFile) { + RunBoundaryByteIdentity("fb", /*frames=*/4, /*chunk_max=*/2); +} + +TEST(CrashRecoveryE2E, BoundaryCrashRecoversByteIdenticalFile_Protobuf) { + RunBoundaryByteIdentity("pb", /*frames=*/4, /*chunk_max=*/2); +} + +// Same identity with a LARGE footer (120 frames -> multi-KB time vectors): the +// control's footer goes through zstd in Close(), and the repair must compress its +// synthesized footer identically (settings journaled in the 'S' record). +TEST(CrashRecoveryE2E, LargeFooterBoundaryCrashIsByteIdentical) { + RunBoundaryByteIdentity("fb_big", /*frames=*/120, /*chunk_max=*/30); +} + +namespace { + + // Shared harness for the brute-force sweeps: a canonical crash state (2 committed + // chunks + 2 pending frames) captured as pristine bytes, restored before every + // mutation so each iteration starts from the identical on-disk state. + struct SweepState { + std::string path; + std::string journal_path; + std::vector main_bytes; + std::vector journal_bytes; + }; + + SweepState MakeSweepState(const std::string& tag) { + const Recording rec = CaptureRecording("sweep_" + tag, 4, /*per_chunk=*/1, Fmt::Flat); + SweepState s; + s.path = FabricateCrash("sweep_" + tag, rec, /*committed=*/2, /*pending=*/2); + s.journal_path = VTX::RecoveryJournal::PathFor(s.path); + s.main_bytes = VtxTest::ReadAllBytes(s.path); + s.journal_bytes = VtxTest::ReadAllBytes(s.journal_path); + EXPECT_FALSE(s.main_bytes.empty()); + EXPECT_FALSE(s.journal_bytes.empty()); + return s; + } + + void WriteBytes(const std::string& path, const std::byte* data, size_t len) { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(reinterpret_cast(data), static_cast(len)); + } + + // Invariants every mutation must uphold: repair never crashes; when it claims a + // repair, the file opens and agrees with the reported frame count (with per-frame + // footer times to match); when it refuses, a valid journal restored over the same + // main file can still drive a full recovery later (checked by the caller's final + // pristine-restore pass). + void CheckRepairInvariants(const std::string& path, size_t iteration, bool open_check) { + const auto rr = VTX::RepairReplayFile(path); + if (!rr.ok()) + return; // refusal is a legal outcome; the harness restores state next round + EXPECT_GE(rr.recovered_frames, 0) << "iteration " << iteration; + EXPECT_LE(rr.recovered_frames, 4) << "iteration " << iteration; + if (!open_check) + return; + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << "iteration " << iteration << ": repaired file failed to open"; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), rr.recovered_frames) << "iteration " << iteration; + const VTX::FileFooter footer = ctx->GetFooter(); + EXPECT_EQ(footer.times.game_time.size(), static_cast(rr.recovered_frames)) << "iteration " << iteration; + } + +} // namespace + +// Brute force over EVERY possible crash point in the journal: truncate it at every +// byte length from 0 to full size and repair. No length may crash, corrupt, or +// produce a file that disagrees with the reported recovery. +TEST(CrashRecoverySweep, JournalTruncationEveryByte) { + const SweepState s = MakeSweepState("jtrunc"); + for (size_t cut = 0; cut <= s.journal_bytes.size(); ++cut) { + WriteBytes(s.path, s.main_bytes.data(), s.main_bytes.size()); + WriteBytes(s.journal_path, s.journal_bytes.data(), cut); + // Full open verification sampled; repair-level invariants checked every time. + CheckRepairInvariants(s.path, cut, /*open_check=*/(cut % 7 == 0) || cut == s.journal_bytes.size()); + if (::testing::Test::HasFatalFailure()) + return; + } + // Final sanity: the pristine state still recovers completely. + WriteBytes(s.path, s.main_bytes.data(), s.main_bytes.size()); + WriteBytes(s.journal_path, s.journal_bytes.data(), s.journal_bytes.size()); + const auto rr = VTX::RepairReplayFile(s.path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 4); +} + +// Brute force over EVERY possible crash point in the MAIN file: truncate the .vtx at +// every byte length (with the full journal alongside) and repair. +TEST(CrashRecoverySweep, MainFileTruncationEveryByte) { + const SweepState s = MakeSweepState("mtrunc"); + for (size_t cut = 0; cut <= s.main_bytes.size(); ++cut) { + WriteBytes(s.path, s.main_bytes.data(), cut); + WriteBytes(s.journal_path, s.journal_bytes.data(), s.journal_bytes.size()); + CheckRepairInvariants(s.path, cut, /*open_check=*/(cut % 7 == 0) || cut == s.main_bytes.size()); + if (::testing::Test::HasFatalFailure()) + return; + } +} + +// Brute force over single-byte corruption: flip every byte of the journal, one at a +// time, and repair. The per-record checksums must contain the damage -- never a +// crash, never a repaired file that disagrees with its own report. +TEST(CrashRecoverySweep, JournalBitFlipEveryByte) { + const SweepState s = MakeSweepState("jflip"); + std::vector mutated = s.journal_bytes; + for (size_t i = 0; i < s.journal_bytes.size(); ++i) { + mutated[i] = s.journal_bytes[i] ^ std::byte {0xFF}; + WriteBytes(s.path, s.main_bytes.data(), s.main_bytes.size()); + WriteBytes(s.journal_path, mutated.data(), mutated.size()); + CheckRepairInvariants(s.path, i, /*open_check=*/(i % 7 == 0)); + if (::testing::Test::HasFatalFailure()) + return; + mutated[i] = s.journal_bytes[i]; // restore for the next flip + } +} + +// Truncation sweep over a POST-COMPACTION journal shape (the round-11 sweeps use the +// uncompacted layout): every byte length of a compacted journal must repair without +// crashing or self-contradiction. +TEST(CrashRecoverySweep, CompactedJournalTruncationSweep) { + const Recording rec = CaptureRecording("sweep_compact", 4, /*per_chunk=*/1, Fmt::Flat); + const std::string path = BuildInterleaved("sweep_compact", rec, /*committed=*/2, /*pending=*/2, /*threshold=*/1); + const std::string journal_path = VTX::RecoveryJournal::PathFor(path); + const auto main_bytes = VtxTest::ReadAllBytes(path); + const auto journal_bytes = VtxTest::ReadAllBytes(journal_path); + ASSERT_FALSE(journal_bytes.empty()); + + for (size_t cut = 0; cut <= journal_bytes.size(); cut += 2) { + WriteBytes(path, main_bytes.data(), main_bytes.size()); + WriteBytes(journal_path, journal_bytes.data(), cut); + CheckRepairInvariants(path, cut, /*open_check=*/(cut % 8 == 0)); + if (::testing::Test::HasFatalFailure()) + return; + } + // Pristine compacted journal still recovers everything. + WriteBytes(path, main_bytes.data(), main_bytes.size()); + WriteBytes(journal_path, journal_bytes.data(), journal_bytes.size()); + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 4); +} + +#ifdef _WIN32 + +// --- REAL process kill -------------------------------------------------------- +// +// Everything above simulates crashes in-process (dropping the writer still closes +// the FILE* handles). These tests spawn vtx_tests.exe AS A CHILD in a hidden +// writer-loop mode and TerminateProcess() it mid-recording: no destructors, no +// fclose, handles abandoned to the kernel -- a genuine process death. This is also +// the only honest way to validate the flush-only (durable_writes=false) claim that +// data pushed to the OS survives a process crash. + +// Child mode: an endless writer loop. Skipped unless spawned with the env var set. +TEST(CrashRecoveryProcess, ChildWriterLoop) { + const char* path = std::getenv("VTX_CHILD_WRITE_PATH"); + if (!path) + GTEST_SKIP() << "child-mode body; only runs when spawned by the KilledMidRecording tests"; + const char* durable_env = std::getenv("VTX_CHILD_DURABLE"); + const bool durable = !durable_env || std::string(durable_env) != "0"; + const char* compact_env = std::getenv("VTX_CHILD_COMPACT_THRESHOLD"); + const uint64_t compact_threshold = compact_env ? std::strtoull(compact_env, nullptr, 10) : 0; + + auto w = MakeRawWriter(path, /*chunk_max_frames=*/10, durable, + /*compression=*/true, /*journal=*/true, compact_threshold); + for (int i = 0;; ++i) { // terminated externally + auto frame = BuildFrame(i); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + (void)w->TryRecordFrame(frame, t); + } +} + +namespace { + + void RunKilledProcessTest(bool durable, const char* tag, uint64_t compact_threshold = 0, + uint64_t progress_threshold = 20'000) { + const std::string path = VtxTest::OutputPath(std::string("killed_") + tag + ".vtx"); + const std::string journal_path = VTX::RecoveryJournalPath(path); + std::filesystem::remove(path); + std::filesystem::remove(journal_path); + + char exe[MAX_PATH] = {}; + ASSERT_GT(GetModuleFileNameA(nullptr, exe, MAX_PATH), 0u); + SetEnvironmentVariableA("VTX_CHILD_WRITE_PATH", path.c_str()); + SetEnvironmentVariableA("VTX_CHILD_DURABLE", durable ? "1" : "0"); + if (compact_threshold > 0) + SetEnvironmentVariableA("VTX_CHILD_COMPACT_THRESHOLD", std::to_string(compact_threshold).c_str()); + std::string cmd = std::string("\"") + exe + "\" --gtest_filter=CrashRecoveryProcess.ChildWriterLoop"; + STARTUPINFOA si {}; + si.cb = sizeof(si); + PROCESS_INFORMATION pi {}; + const BOOL created = + CreateProcessA(nullptr, cmd.data(), nullptr, nullptr, FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &si, &pi); + SetEnvironmentVariableA("VTX_CHILD_WRITE_PATH", nullptr); + SetEnvironmentVariableA("VTX_CHILD_DURABLE", nullptr); + SetEnvironmentVariableA("VTX_CHILD_COMPACT_THRESHOLD", nullptr); + ASSERT_TRUE(created); + + // Let the child journal a healthy amount of frames, then KILL it cold. (With an + // aggressive compaction threshold the journal shrinks on every commit, so gate + // on the MAIN file's growth instead -- it only ever grows.) + const std::string& progress_file = (compact_threshold > 0) ? path : journal_path; + uint64_t journal_size = 0; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(20); + while (std::chrono::steady_clock::now() < deadline) { + std::error_code ec; + const auto s = std::filesystem::file_size(progress_file, ec); + if (!ec && s > progress_threshold) { + journal_size = static_cast(s); + break; + } + Sleep(5); + } + TerminateProcess(pi.hProcess, 1); + WaitForSingleObject(pi.hProcess, 10'000); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + ASSERT_GT(journal_size, 0u) << "child process never made journaling progress"; + + // The kernel released the child's handles; recover and verify integrity. + ASSERT_TRUE(VTX::ReplayNeedsRecovery(path)); + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_GT(rr.recovered_frames, 0); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + ASSERT_EQ(ctx->GetTotalFrames(), rr.recovered_frames); + if (rr.recovered_frames > 0) { + const VTX::Frame* first = ctx->GetFrameSync(0); + ASSERT_NE(first, nullptr); + EXPECT_EQ(first->GetBuckets()[0].entities[0].int32_properties[1], 0); + const int last = rr.recovered_frames - 1; + const VTX::Frame* last_frame = ctx->GetFrameSync(last); + ASSERT_NE(last_frame, nullptr); + EXPECT_EQ(last_frame->GetBuckets()[0].entities[0].int32_properties[1], last); + } + const VTX::FileFooter footer = ctx->GetFooter(); + EXPECT_EQ(footer.times.game_time.size(), static_cast(rr.recovered_frames)); + EXPECT_EQ(footer.times.created_utc.size(), static_cast(rr.recovered_frames)); + } + +} // namespace + +// fsync-per-operation mode: even a cold kill loses nothing that was recorded. +TEST(CrashRecoveryProcess, KilledMidRecordingDurable) { + RunKilledProcessTest(/*durable=*/true, "durable"); +} + +// Flush-only mode: data handed to the OS (fflush) must survive a PROCESS death (the +// documented durability tier below power-loss safety). +TEST(CrashRecoveryProcess, KilledMidRecordingFlushOnly) { + RunKilledProcessTest(/*durable=*/false, "flushonly"); +} + +// Aggressive compaction (threshold 1 -> a full close/rewrite/atomic-rename cycle on +// EVERY commit): a real kill lands mid-compaction traffic sooner or later, and the +// atomic-rename design must leave either the old or the new journal -- always +// recoverable. +TEST(CrashRecoveryProcess, KilledMidRecordingWhileCompacting) { + RunKilledProcessTest(/*durable=*/true, "compacting", /*compact_threshold=*/1); +} + +// Soak: kill the child at a SPREAD of progress points -- just after the first frames, +// mid-first-chunk, after a few commits, deep into the session -- alternating durability +// and compaction cadence. Every kill timing must recover to a self-consistent file. +TEST(CrashRecoveryProcess, KilledAtVariedProgressPointsSoak) { + struct Round { + bool durable; + uint64_t compact_threshold; + uint64_t progress_threshold; + const char* tag; + }; + const Round rounds[] = { + {true, 0, 800, "soak_early"}, // only a few F records journaled + {false, 0, 3'000, "soak_firstchunk"}, // around the first flush + {true, 1, 8'000, "soak_compact"}, // amid compaction cycles + {false, 0, 25'000, "soak_deep"}, // several chunks in + }; + for (const auto& r : rounds) { + SCOPED_TRACE(r.tag); + RunKilledProcessTest(r.durable, r.tag, r.compact_threshold, r.progress_threshold); + if (::testing::Test::HasFatalFailure()) + return; + } +} + +#endif // _WIN32 + +// A record whose length field claims far more than the journal holds must be +// rejected up front (bounded by the file's real size -- no giant transient +// allocation), recovering everything before it. +TEST(CrashRecoverySweep, HostileRecordLengthIsBoundedByFileSize) { + const Recording rec = CaptureRecording("hostilelen", 2, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("hostilelen", rec, /*committed=*/2, /*pending=*/0); + + // Append a record header claiming a 400MB payload the 1-KB journal cannot hold. + { + std::ofstream j(VTX::RecoveryJournal::PathFor(path), std::ios::binary | std::ios::app); + const char type = 'F'; + const uint32_t huge_len = 400u * 1024u * 1024u; + j.write(&type, 1); + j.write(reinterpret_cast(&huge_len), sizeof(huge_len)); + const char few[16] = {}; + j.write(few, sizeof(few)); + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 2); // hostile tail rejected, committed chunks intact + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 2); +} + +// A journal from an incompatible version (here: a patched version field) must be +// refused outright -- never applied -- leaving the main file untouched. +TEST(CrashRecoveryE2E, JournalVersionMismatchIsRefused) { + const Recording rec = CaptureRecording("verjournal", 2, /*per_chunk=*/1, Fmt::Flat); + const std::string path = FabricateCrash("verjournal", rec, /*committed=*/2, /*pending=*/0); + + // Patch the journal's u32 version field (bytes 4..7 after "VTXR") to 99. + { + std::fstream j(VTX::RecoveryJournal::PathFor(path), std::ios::binary | std::ios::in | std::ios::out); + ASSERT_TRUE(j.is_open()); + const uint32_t bogus_version = 99; + j.seekp(4); + j.write(reinterpret_cast(&bogus_version), sizeof(bogus_version)); + } + + const auto bytes_before = VtxTest::ReadAllBytes(path); + const auto rr = VTX::RepairReplayFile(path); + EXPECT_FALSE(rr.ok()); // incompatible journal -> refused + EXPECT_FALSE(rr.repaired); + EXPECT_EQ(VtxTest::ReadAllBytes(path), bytes_before); // main file untouched +} + +// The documented normalization workflow for salvaged files: a recovered file whose +// tail is many one-frame chunks (slower to read) can be transcoded into a first-class +// file with proper chunking using only the public API -- open it, drain the frames +// with their footer times, and re-record. created_utc (int64) round-trips exactly; +// game_time re-enters through the float-seconds register, whose truncating +// ticks->float->ticks conversion can lose 1 tick (100 ns) per frame -- the documented +// precision caveat of the transcode path. +TEST(CrashRecoveryE2E, RecoveredFileTranscodesToCleanChunks) { + // A crash with EVERYTHING in flight -> repair yields 50 one-frame chunks. + const std::string salvaged = VtxTest::OutputPath("transcode_salvaged.vtx"); + { + auto w = MakeRawWriter(salvaged, /*chunk_max_frames=*/100); + constexpr int64_t kBaseUtc = 5'000'000'000'000; + for (int i = 0; i < 50; ++i) { + auto frame = BuildFrame(i); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + t.created_utc_time = kBaseUtc + (i + 1) * 166'667; + ASSERT_TRUE(w->TryRecordFrame(frame, t).written); + } + // dropped without Stop(): nothing committed, 50 pending F records + } + const auto rr = VTX::RepairReplayFile(salvaged); + ASSERT_TRUE(rr.ok()) << rr.error; + ASSERT_EQ(rr.recovered_frames, 50); + + auto salvage = VTX::OpenReplayFile(salvaged); + ASSERT_TRUE(salvage) << salvage.error; + salvage->WaitUntilReady(); + ASSERT_EQ(salvage->GetSeekTable().size(), 50u); // one-frame chunks, as repaired + + // Transcode: drain into a fresh writer with normal chunking. + const std::string clean = VtxTest::OutputPath("transcode_clean.vtx"); + { + VTX::WriterFacadeConfig cfg; + cfg.output_filepath = clean; + cfg.schema_json_path = VtxTest::FixturePath("test_schema.json"); + cfg.replay_name = "TranscodedSalvage"; + cfg.chunk_max_frames = 100; + auto writer = VTX::CreateFlatBuffersWriterFacade(cfg); + ASSERT_NE(writer, nullptr); + + const VTX::FileFooter footer = salvage->GetFooter(); + for (int i = 0; i < 50; ++i) { + VTX::Frame frame; + ASSERT_TRUE(salvage->GetFrame(i, frame) || (salvage->GetFrameSync(i) && salvage->GetFrame(i, frame))); + VTX::GameTime::GameTimeRegister t; + t.game_time = static_cast(static_cast(footer.times.game_time[static_cast(i)]) / + VTX::GameTime::TICKS_PER_SECOND); + t.created_utc_time = static_cast(footer.times.created_utc[static_cast(i)]); + const auto res = writer->TryRecordFrame(frame, t); + ASSERT_TRUE(res.written) << "frame " << i << ": " << res.error.message; + } + writer->Stop(); + } + + auto normalized = VTX::OpenReplayFile(clean); + ASSERT_TRUE(normalized) << normalized.error; + normalized->WaitUntilReady(); + EXPECT_EQ(normalized->GetTotalFrames(), 50); + EXPECT_EQ(normalized->GetSeekTable().size(), 1u); // back to one proper 50-frame chunk + + const VTX::FileFooter sf = salvage->GetFooter(); + const VTX::FileFooter nf = normalized->GetFooter(); + EXPECT_EQ(nf.times.created_utc, sf.times.created_utc); // int64 register -> exact round-trip + ASSERT_EQ(nf.times.game_time.size(), sf.times.game_time.size()); + for (size_t i = 0; i < sf.times.game_time.size(); ++i) { + const int64_t drift = static_cast(nf.times.game_time[i]) - static_cast(sf.times.game_time[i]); + // Float-seconds register: truncating ticks->float->ticks loses at most 1 tick. + EXPECT_LE(std::abs(drift), 1) << "game_time drift beyond 1 tick at frame " << i; + } + for (int i : {0, 25, 49}) { + const VTX::Frame* f = normalized->GetFrameSync(i); + ASSERT_NE(f, nullptr) << "frame " << i; + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], i); + } +} + +// Full lifecycle twice over the same path: crash -> repair -> re-record -> crash -> +// repair. The second recovery must reflect only the second session. +TEST(CrashRecoveryE2E, DoubleCrashLifecycleOnSamePath) { + const std::string path = VtxTest::OutputPath("double_crash.vtx"); + + // Session 1: 3 frames, crash, repair. + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/2); + for (int i = 0; i < 3; ++i) { + auto frame = BuildFrame(i); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + ASSERT_TRUE(w->TryRecordFrame(frame, t).written); + } + } + { + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 3); + } + + // Session 2: same path, 5 frames with distinct scores, crash, repair. + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/2); + for (int i = 0; i < 5; ++i) { + auto frame = BuildFrame(100 + i); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + ASSERT_TRUE(w->TryRecordFrame(frame, t).written); + } + } + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 5); + EXPECT_FALSE(VTX::ReplayNeedsRecovery(path)); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 5); + for (int i = 0; i < 5; ++i) { + const VTX::Frame* f = ctx->GetFrameSync(i); + ASSERT_NE(f, nullptr); + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 100 + i); // session-2 content only + } +} + +// The ".recovery" sidecar naming and repair I/O must work wherever the main file +// does -- including non-ASCII paths. +TEST(CrashRecoveryE2E, NonAsciiPathCrashRecovery) { + // "reproducción_dañada.vtx" (UTF-8 bytes split so \x escapes don't swallow hex digits) + const std::string path = VtxTest::OutputPath("reproducci\xC3\xB3" + "n_da\xC3\xB1" + "ada.vtx"); + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/2); + for (int i = 0; i < 3; ++i) { + auto frame = BuildFrame(i); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + const auto res = w->TryRecordFrame(frame, t); + ASSERT_TRUE(res.written) << res.error.message; + } + // dropped: 1 committed chunk + 1 in-flight frame + } + ASSERT_TRUE(VTX::ReplayNeedsRecovery(path)); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 3); + EXPECT_FALSE(VTX::ReplayNeedsRecovery(path)); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 3); +} + +// Scale check for the motivating scenario (a long recording): thousands of frames +// with multiple entities each, dozens of committed chunks, and a LARGE in-flight +// batch at crash time. Everything must come back -- committed and pending -- with +// exact timestamps, and the recovered file must pass whole-replay validation. +TEST(CrashRecoveryE2E, StressManyChunksAndLargePendingBatch) { + constexpr int kFrames = 2030; // 40 committed chunks of 50 + 30 in-flight frames + constexpr int kEntities = 5; + constexpr int64_t kBaseUtc = 2'000'000'000'000; + constexpr int64_t kStepUtc = 166'667; + + const std::string path = VtxTest::OutputPath("stress_crash.vtx"); + { + // Flush-only durability keeps the test fast; the journal/recovery logic path + // is identical to fsync mode. + auto w = MakeRawWriter(path, /*chunk_max_frames=*/50, /*durable=*/false); + for (int i = 0; i < kFrames; ++i) { + VTX::Frame frame; + auto& bucket = frame.CreateBucket("entity"); + for (int k = 0; k < kEntities; ++k) { + VTX::PropertyContainer pc; + pc.entity_type_id = 0; + pc.string_properties = {"p" + std::to_string(k), "Alpha"}; + pc.int32_properties = {1, i * 10 + k, 0}; + pc.float_properties = {100.0f, 50.0f}; + bucket.unique_ids.push_back("p" + std::to_string(k)); + bucket.entities.push_back(std::move(pc)); + } + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + t.created_utc_time = kBaseUtc + (i + 1) * kStepUtc; + const auto res = w->TryRecordFrame(frame, t); + ASSERT_TRUE(res.written) << "frame " << i << ": " << res.error.message; + } + // dropped without Stop() + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, kFrames); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), kFrames); + + // Spot frames across the file: first, mid-chunk, last committed, deep in the + // recovered pending batch -- with every entity intact. + for (int i : {0, 999, 1999, 2029}) { + const VTX::Frame* f = ctx->GetFrameSync(i); + ASSERT_NE(f, nullptr) << "frame " << i; + ASSERT_EQ(f->GetBuckets()[0].entities.size(), static_cast(kEntities)) << "frame " << i; + for (int k = 0; k < kEntities; ++k) + EXPECT_EQ(f->GetBuckets()[0].entities[static_cast(k)].int32_properties[1], i * 10 + k) + << "frame " << i << " entity " << k; + } + + // Exact timestamps for every frame, committed (T records) and pending (F records). + const VTX::FileFooter footer = ctx->GetFooter(); + ASSERT_EQ(footer.times.created_utc.size(), static_cast(kFrames)); + ASSERT_EQ(footer.times.game_time.size(), static_cast(kFrames)); + for (int i = 0; i < kFrames; ++i) + ASSERT_EQ(static_cast(footer.times.created_utc[static_cast(i)]), kBaseUtc + (i + 1) * kStepUtc) + << "created_utc[" << i << "]"; + for (int i = 1; i < kFrames; ++i) + ASSERT_LT(footer.times.game_time[static_cast(i) - 1], footer.times.game_time[static_cast(i)]) + << "game_time monotonicity at " << i; + + const VTX::ValidationReport report = VTX::ValidateReplayFile(path); + EXPECT_FALSE(report.HasErrors()) << report.ToString(); +} + +// Frames REJECTED mid-recording (here: a stale created_utc the timer refuses) must +// not desync the journal's frame indexing -- a realistic occurrence over a long +// session. The accepted frames recover completely, with their exact times. +TEST(CrashRecoveryE2E, RejectedFramesDoNotDesyncJournal) { + constexpr int64_t kBaseUtc = 3'000'000'000'000; + const int64_t utcs[5] = {kBaseUtc + 1000, kBaseUtc + 2000, kBaseUtc + 3000, kBaseUtc + 4000, kBaseUtc + 5000}; + + const std::string path = VtxTest::OutputPath("rejected_crash.vtx"); + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/3); + int accepted = 0; + for (int step = 0; step < 6; ++step) { + const bool poison = (step == 3); // 4th record attempt reuses the last UTC + auto frame = BuildFrame(accepted); + VTX::GameTime::GameTimeRegister t; + t.game_time = float(accepted) / 60.0f; + t.created_utc_time = poison ? utcs[accepted - 1] : utcs[accepted]; + const auto res = w->TryRecordFrame(frame, t); + if (poison) { + ASSERT_FALSE(res.written); // duplicate UTC -> timer rejects, writer rolls back + } else { + ASSERT_TRUE(res.written) << res.error.message; + ++accepted; + } + } + ASSERT_EQ(accepted, 5); + // dropped without Stop(): 1 committed chunk (0-2) + 2 in-flight frames + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 5); // the rejected attempt left no trace + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 5); + const VTX::FileFooter footer = ctx->GetFooter(); + ASSERT_EQ(footer.times.created_utc.size(), 5u); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(static_cast(footer.times.created_utc[static_cast(i)]), utcs[i]) << "utc " << i; + const VTX::Frame* f = ctx->GetFrameSync(i); + ASSERT_NE(f, nullptr); + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], i); + } +} + +namespace { + + // Stamps every Player's Health with a marker so a recovered frame can prove it + // was journaled AFTER post-processing (i.e., the journal holds what would have + // been written to disk, not the raw input). + class HealthStamper : public VTX::IFramePostProcessor { + public: + void Init(const VTX::FramePostProcessorInitContext& ctx) override { + health_key_ = ctx.frame_accessor->Get("Player", "Health"); + } + void Process(VTX::FrameMutationView& view, const VTX::FramePostProcessContext&) override { + if (!health_key_.IsValid()) + return; + auto bucket = view.GetBucket("entity"); + for (auto entity : bucket) + entity.Set(health_key_, 777.0f); + } + void Clear() override {} + + private: + VTX::PropertyKey health_key_ {-1}; + }; + +} // namespace + +// The journal must capture the frame AS IT WOULD HIT DISK -- i.e., after the writer's +// post-processor ran. A pending frame that only ever existed as an F record must come +// back with the post-processed values, identical to its committed siblings. +TEST(CrashRecoveryE2E, PostProcessedFramesAreJournaledPostMutation) { + const std::string path = VtxTest::OutputPath("postproc_crash.vtx"); + { + auto w = MakeRawWriter(path, /*chunk_max_frames=*/3); + w->SetPostProcessor(std::make_shared()); + for (int i = 0; i < 4; ++i) { + auto frame = BuildFrame(i); // input Health = 100.0f + VTX::GameTime::GameTimeRegister t; + t.game_time = float(i) / 60.0f; + const auto res = w->TryRecordFrame(frame, t); + ASSERT_TRUE(res.written) << res.error.message; + } + // dropped without Stop(): chunk (0-2) committed + frame 3 in flight + } + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 4); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + for (int i : {0, 3}) { // a committed frame and the F-record-only pending frame + const VTX::Frame* f = ctx->GetFrameSync(i); + ASSERT_NE(f, nullptr) << "frame " << i; + EXPECT_FLOAT_EQ(f->GetBuckets()[0].entities[0].float_properties[0], 777.0f) + << "frame " << i << " missing the post-processed value"; + } +} + +#ifdef _WIN32 +// Repairing a recording that is STILL BEING WRITTEN must be refused without touching +// the file (the writer holds a deny-write handle, so repair's truncate fails), and the +// live writer must be able to finish cleanly afterwards. +TEST(CrashRecoveryE2E, LiveRecordingRepairIsRefused) { + const std::string path = VtxTest::OutputPath("e2e_live.vtx"); + auto w = MakeRawWriter(path, 3); + RecordEightFrames(*w); // 2 chunks committed, journal present, writer still open + + ASSERT_TRUE(VTX::ReplayNeedsRecovery(path)); // sidecar exists while recording + const auto rr = VTX::RepairReplayFile(path); + EXPECT_FALSE(rr.ok()); // refused: file is held by the live writer + EXPECT_FALSE(rr.repaired); + + w->Stop(); // the recording finishes untouched + w.reset(); + ASSERT_FALSE(VTX::ReplayNeedsRecovery(path)); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 8); +} +#endif + +// --- Protobuf format ---------------------------------------------------------- + +TEST(CrashRecovery, RecoversAllCommittedChunks_Protobuf) { + const Recording rec = CaptureRecording("pb_all", 4, /*per_chunk=*/1, Fmt::Proto); + const std::string path = FabricateCrash("pb_all", rec, /*committed=*/4, /*pending=*/0); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_chunks, 4); + EXPECT_EQ(rr.recovered_frames, 4); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 4); + const VTX::Frame* f = ctx->GetFrameSync(3); + ASSERT_NE(f, nullptr); + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 3); +} + +TEST(CrashRecovery, RecoversPendingFrames_Protobuf) { + const Recording rec = CaptureRecording("pb_pending", 4, /*per_chunk=*/1, Fmt::Proto); + const std::string path = FabricateCrash("pb_pending", rec, /*committed=*/2, /*pending=*/2); + + const auto rr = VTX::RepairReplayFile(path); + ASSERT_TRUE(rr.ok()) << rr.error; + EXPECT_EQ(rr.recovered_frames, 4); + + auto ctx = VTX::OpenReplayFile(path); + ASSERT_TRUE(ctx) << ctx.error; + ctx->WaitUntilReady(); + EXPECT_EQ(ctx->GetTotalFrames(), 4); + const VTX::Frame* f = ctx->GetFrameSync(3); + ASSERT_NE(f, nullptr); + EXPECT_EQ(f->GetBuckets()[0].entities[0].int32_properties[1], 3); } From d2aa421c18cc4ad908dd6528452027e3b46ea9e1 Mon Sep 17 00:00:00 2001 From: Alejandro Canela Date: Mon, 20 Jul 2026 11:58:09 +0200 Subject: [PATCH 5/6] test: crash test recovery --- tests/sanitizer_suppressions/asan.supp | 11 ++++ tests/writer/test_crash_recovery.cpp | 74 +++++++++++++++----------- 2 files changed, 55 insertions(+), 30 deletions(-) diff --git a/tests/sanitizer_suppressions/asan.supp b/tests/sanitizer_suppressions/asan.supp index 9f5163b..b0a2496 100644 --- a/tests/sanitizer_suppressions/asan.supp +++ b/tests/sanitizer_suppressions/asan.supp @@ -8,3 +8,14 @@ # Example (uncomment when needed): # # Known leak in libprotobuf's global init, deliberate design. # # leak:google::protobuf::internal::OnShutdown +# +# Known third-party finding (2026-07, full suite under MSVC /fsanitize=address): +# new-delete-type-mismatch inside google::protobuf::DynamicMessage's scalar +# deleting dtor (allocated 56B, deleted as 32B). DynamicMessage by design +# over-allocates its block and stores field data inline after the object, so +# sized delete disagrees with the allocation. Purely protobuf-internal +# (surfaces via VtxDiff::ProtobufDifferFacade::DiffRawFrames destroying a +# unique_ptr -- correct usage on our side). This check cannot be +# suppressed from this file; disable it for ASan runs with: +# ASAN_OPTIONS=new_delete_type_mismatch=0 +# With that flag, the full 419-test suite runs clean under ASan. diff --git a/tests/writer/test_crash_recovery.cpp b/tests/writer/test_crash_recovery.cpp index 9000660..1b57934 100644 --- a/tests/writer/test_crash_recovery.cpp +++ b/tests/writer/test_crash_recovery.cpp @@ -1677,6 +1677,14 @@ namespace { // nothing in flight) must recover to a file that is BYTE-IDENTICAL to one produced // by a clean Stop() with the same inputs -- same chunks, same seek table, same // footer (times, gaps, segments, duration, compression), same trailer. + // + // The file header embeds a SECOND-granularity recording timestamp and is + // zstd-compressed, so a control/crash pair recorded across a second boundary gets + // different header bytes -- and possibly a different compressed header SIZE, which + // shifts every absolute offset in the seek table (a legitimate difference; repair + // never rewrites the header). Full-file identity is therefore only meaningful for + // a SAME-SECOND pair: re-record the pair (bounded retries, each takes well under a + // second) until both headers are byte-identical, then demand total equality. template void RunBoundaryByteIdentity(const std::string& prefix, int frames, int32_t chunk_max) { auto record_all = [frames](RawWriterFor& w) { @@ -1690,19 +1698,45 @@ namespace { } w.Flush(); // land the trailing batch -> crash sits exactly on a chunk boundary }; + auto header_region = [](const std::vector& bytes) { + if (bytes.size() < 8) + return std::vector(); + uint32_t header_size = 0; + std::memcpy(&header_size, bytes.data() + 4, sizeof(header_size)); + const size_t end = static_cast(8) + header_size; + if (end > bytes.size()) + return std::vector(); + return std::vector(bytes.begin(), bytes.begin() + static_cast(end)); + }; const std::string control_path = VtxTest::OutputPath(prefix + "_ident_control.vtx"); - { - auto w = MakeRawWriter(control_path, chunk_max); - record_all(*w); - w->Stop(); - } const std::string crash_path = VtxTest::OutputPath(prefix + "_ident_crash.vtx"); - { - auto w = MakeRawWriter(crash_path, chunk_max); - record_all(*w); - // dropped without Stop(): every frame is in a committed chunk, none pending + + // Flush-only durability: fsync vs fflush changes NO file bytes (and the flag is + // not journaled), but it makes each recording take milliseconds instead of + // seconds -- essential so a same-second pair is reachable even on slow CI + // runners, where fsync-per-frame recordings span more than a second each. + bool same_second_pair = false; + for (int attempt = 0; attempt < 8 && !same_second_pair; ++attempt) { + { + auto w = MakeRawWriter(control_path, chunk_max, /*durable=*/false); + record_all(*w); + if (::testing::Test::HasFatalFailure()) + return; + w->Stop(); + } + { + auto w = MakeRawWriter(crash_path, chunk_max, /*durable=*/false); + record_all(*w); + if (::testing::Test::HasFatalFailure()) + return; + // dropped without Stop(): every frame is in a committed chunk, none pending + } + const auto control_header = header_region(VtxTest::ReadAllBytes(control_path)); + const auto crash_header = header_region(VtxTest::ReadAllBytes(crash_path)); + same_second_pair = !control_header.empty() && control_header == crash_header; } + ASSERT_TRUE(same_second_pair) << "could not record a same-second control/crash pair in 8 attempts"; const auto rr = VTX::RepairReplayFile(crash_path); ASSERT_TRUE(rr.ok()) << rr.error; @@ -1711,28 +1745,8 @@ namespace { const auto control_bytes = VtxTest::ReadAllBytes(control_path); const auto recovered_bytes = VtxTest::ReadAllBytes(crash_path); ASSERT_FALSE(control_bytes.empty()); - ASSERT_GE(control_bytes.size(), 8u); - ASSERT_GE(recovered_bytes.size(), 8u); - - // The header embeds a second-granularity recording timestamp, so two separately - // recorded sessions may legitimately differ there (repair never touches the - // header -- the recovered file keeps its own, which IS the guarantee). Compare - // everything from the end of the header on: chunks, seek table, footer, trailer. - auto header_end = [](const std::vector& bytes) { - uint32_t header_size = 0; - std::memcpy(&header_size, bytes.data() + 4, sizeof(header_size)); - return static_cast(8) + header_size; - }; - const size_t control_body = header_end(control_bytes); - ASSERT_EQ(header_end(recovered_bytes), control_body); // same header structure/size - ASSERT_LT(control_body, control_bytes.size()); - EXPECT_EQ(recovered_bytes.size(), control_bytes.size()); - const bool body_identical = - recovered_bytes.size() == control_bytes.size() && - std::equal(control_bytes.begin() + static_cast(control_body), control_bytes.end(), - recovered_bytes.begin() + static_cast(control_body)); - EXPECT_TRUE(body_identical); // chunks + footer + trailer bit-for-bit equal + EXPECT_EQ(recovered_bytes, control_bytes); // bit-for-bit equal to the clean file } } // namespace From bf259e4908a8787310bc7eed5f284958d6278ef7 Mon Sep 17 00:00:00 2001 From: Alejandro Canela Date: Mon, 20 Jul 2026 11:58:09 +0200 Subject: [PATCH 6/6] test: crash test recovery --- CHANGELOG.md | 2 +- tests/reader/test_reader_api.cpp | 8 ++- tests/writer/test_crash_recovery.cpp | 74 +++++++++++++++++----------- 3 files changed, 52 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28a4246..70316b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **per-chunk integrity + durability** -- `ChunkedFileSink::Config` gains `durable_writes` (default **on**: `fsync`/`_commit` each chunk and journal record to physical media via the new `DurableFile` FILE* wrapper; set **off** to only `fflush` to the OS -- survives a process crash but not power loss) and `enable_recovery_journal` (default **on**). Every chunk carries an **xxHash64 `checksum`** of its on-disk payload, added to `ChunkIndexEntry` / `ChunkIndexData` and to both the FlatBuffers (`vtx_schema.fbs`) and Protobuf (`vtx_schema.proto`, field 6) footer schemas, so a torn or corrupt chunk is detectable on recovery. The reader parses and exposes it; FlatBuffers header/footer parsing gained a `flatbuffers::Verifier` guard so a truncated footer is rejected cleanly instead of reading out of bounds - **recovery journal** -- a single write-ahead sidecar **`.recovery`** (`sdk/include/vtx/writer/policies/sinks/recovery_journal.h`) holds a typed, self-validating record stream (each record `[u8 type][u32 len][payload][u64 xxHash64]`, so a torn last record from a mid-write crash is detected and dropped): an `S` record (written once) carries the writer's timing parameters `{fps, is_increasing}`, `C` records commit a durable chunk's index entry, `T` records carry each committed frame's exact `{game_time, created_utc}`, and `F` records carry each in-flight (un-flushed) frame's times + serialized payload. The log is strictly **append-only** in the crash-critical path -- a frame appends an `F` record; a flush appends the chunk's `C` + `T` records -- so at every instant the file is a valid prefix and there is no window in which a fsync'd chunk is described by neither its `F` records nor its `C` record. Ordering is **data-before-journal**: a chunk is fsync'd into the `.vtx` first, then its `C`/`T` records are appended and fsync'd. A committed chunk's now-redundant `F` records (repair dedups them by frame index) are reclaimed by periodic **compaction** -- the log is rewritten (all `C`/`T` + only the still-pending `F`) into a temp file that **atomically replaces** the sidecar, so a crash during compaction leaves either the old or the new complete journal. The writer journals every frame as it is recorded (`ReplayWriter::TryRecordFrame` -> new `SinkPolicy::JournalFrame` hook; timing via `SinkPolicy::JournalTiming`), so a crash between flushes still recovers the pending batch. If the journal cannot be opened cleanly, journaling is disabled and the torn sidecar is removed (a half-written journal would otherwise block repair). On a clean `Close()` the footer is written and the `.recovery` is deleted -- its presence at open time signals an unclean shutdown - **repair** -- **`VTX::RepairReplayFile(path)`** (`sdk/include/vtx/writer/core/vtx_replay_recovery.h`) reconstructs a valid, readable file from a footerless `.vtx` + its journal: it verifies each committed chunk's checksum (dropping a torn/corrupt tail), re-appends the in-flight frames as chunks, rebuilds the seek table and the **exact per-frame times**, and synthesizes the footer -- including the **derived time data** (`duration_seconds`, timeline **gaps**, game **segments**), reconstructed from the journaled timing (`S` record) with the same expressions `VTXGameTimes` uses, so a recovered footer matches a clean `Stop()` **exactly** (manual segment marks are the one thing not journaled). Recovery is deliberately **not automatic on open** -- the user-driven flow is `ReplayNeedsRecovery(path)` (cheap sidecar check) / `RecoveryJournalPath(path)` (locate the sidecar) then `RepairReplayFile(path)`. A file that already ends with a valid footer plus a leftover journal (a crash between the footer fsync and the journal delete) is detected and preserved untouched; a journal whose recorded format magic disagrees with the file, or whose header is unreadable, is refused rather than applied; a **recording still being written is refused** without touching it (on Windows the writer holds a deny-write handle, so repair's truncate fails cleanly). Returns a `RepairResult` (`was_clean` / `repaired` / `recovered_chunks` / `recovered_frames` / `error`). Both FlatBuffers and Protobuf files are supported -- **tests**: **`tests/writer/test_crash_recovery.cpp`** -- 65 cases covering the full crash-window matrix: recovery between chunks (single- and multi-frame), between frames (in-flight batch, and nothing-flushed-yet), a torn tail chunk, a partial chunk written before its journal record, a checksum-detected corrupt chunk, a torn pending-frame record, a **torn chunk-commit record** (the batch falls back to its still-present `F` records -- the append-only guarantee), a crash mid-footer-write, a leftover-journal-over-valid-footer, a **stale compaction temp**, a mismatched journal format, a clean file (no-op), a header-only crash (0-frame result), the lingering-`F`-record dedup guard, compaction reclaim, the recovery helpers, exact per-frame time preservation, a **live-recording repair refusal** (Windows), and the FlatBuffers + Protobuf repair paths. Crash states are fabricated byte-faithfully through the same `RecoveryJournal` API the sink uses, plus **end-to-end tests through the real writer** (a raw `ReplayWriter` dropped without `Stop()`) for both formats that require the recovered footer to match a cleanly-stopped control **exactly**: per-frame `game_time` + `created_utc`, `duration_seconds`, timeline gaps, game segments, and per-entity `content_hash` -- and require the recovered file to pass whole-replay `ValidateReplayFile` and cache-hostile out-of-order seeks across the committed-chunk/recovered-chunk boundary. Also covered: the sink's **full config matrix** (`durable_writes` x `b_use_compression`) with frames large enough that **zstd compression genuinely engages** (committed chunks and journaled `F` payloads take the compressed path), a **repair interrupted mid-run and re-run** (idempotent), a **second repair on an already-repaired file** (clean no-op, byte-identical), a **journal-opted-out crash** (repair reports clean and leaves the footerless file byte-identical; the reader rejects it gracefully), a **session-start crash** (torn `S` record + header-only `.vtx` -> valid 0-frame file), **torn `T` records** (frame times fall back to the still-present `F` records -- exact, not zeroed), a **0-byte journal** (refused, main file untouched), **compaction through the real sink** under crash (new `Config::journal_compact_threshold_bytes`, 0 = default), **map-container frames** recovered intact from `F` records, a **non-ASCII path** through the full sidecar flow, **hostile checksummed journal records** (a `C` offset inside the header region, a `C` size at or below its own length prefix, an `F` with an empty payload -- each dropped safely, degrading to a valid file), a **torn main-file header** alongside a valid journal (refused, bytes untouched), and a **decreasing-time recording** (`is_increasing = false` -- the other branch of the segment reconstruction) matching its clean control. Scale + pipeline coverage: a **stress run** (2030 multi-entity frames, 40 committed chunks + a 30-frame in-flight batch -- every timestamp exact, whole-replay validation clean), **rejected frames interleaved mid-recording** (timer rejections leave no trace and do not desync the journal's frame indexing), and a **post-processor recording** (journaled `F` records capture the frame as it would hit disk -- post-mutation -- proven by a pending frame that only ever existed in the journal). Lifecycle + hygiene: a **double-crash lifecycle** over the same path (crash -> repair -> re-record -> crash -> repair, second recovery reflects only the second session), a **stale sidecar under journaling opt-out** (a session with `enable_recovery_journal = false` removes any leftover `.recovery` at start so it cannot masquerade as recovery state), a **foreign `.recovery` file** (no `VTXR` magic -- never deleted by repair, even on the clean-file path), a **hostile out-of-range `T` record** (ignored by the bounds check), the **map crash-recovery path on Protobuf** as well as FlatBuffers, a **stale journal from a different recording** over a replaced file (per-chunk checksums reject the foreign chunks -- graceful degradation to a valid empty file), a **read-only crashed file** (repair refuses without consuming the journal; succeeds untouched once the file is writable), a **no-time-registry recording** (`EGameTimeType::None`, fully FPS-synthesized timeline recovered exactly), and **byte-budget chunk splitting** (`max_bytes`, the other `ThresholdChunkPolicy` branch) through crash + recovery. The maximal guarantee is pinned by **`BoundaryCrashRecoversByteIdenticalFile`** (FlatBuffers **and** Protobuf, plus a 120-frame large-footer variant that exercises the compressed-footer path): a crash exactly at a chunk boundary recovers to a file that is **bit-for-bit identical** to a clean `Stop()` with the same inputs -- repair passes the footer's time vectors exactly as `Stop()` does (present-but-empty rather than absent) and compresses the synthesized footer exactly as the sink's `Close()` would (the `S` record now also journals `{use_compression, compression_level}`; journal version 4). A journal with an **incompatible version field** is refused outright, main file untouched. On top of the hand-picked windows, a **brute-force sweep suite** (`CrashRecoverySweep`) proves the "any crash point" claim literally: the journal truncated at **every byte length**, the main `.vtx` truncated at **every byte length**, and **every journal byte flipped** one at a time -- thousands of repair runs, none may crash, and every claimed repair must open and agree with its own reported frame count and footer times. `ReadValid` now bounds each record's payload allocation by the journal's **actual file size** (a corrupt length field can no longer trigger a giant transient allocation), pinned by `HostileRecordLengthIsBoundedByFileSize`; the truncation sweep also runs over the **post-compaction journal layout**. Finally, `CrashRecoveryProcess` (Windows) performs a **genuine process kill**: vtx_tests spawns itself as a child in a hidden endless-writer mode and `TerminateProcess`es it mid-recording -- no destructors, no fclose, handles abandoned to the kernel -- then repairs and verifies frame content and exact footer times, in **both** durability modes (this is the honest validation of the flush-only claim that data handed to the OS survives a process death) -- plus a kill under an **aggressive compaction cadence** (a full close/rewrite/atomic-rename cycle per commit), so real process death lands amid compaction traffic and the atomic-rename guarantee holds, and a **varied-progress kill soak** (kills landing just after the first journaled frames, around the first flush, amid compaction cycles, and several chunks deep -- alternating durability modes). The recovered output was also smoke-checked through the end-user toolchain: `vtx_cli` opens a repaired file and serves `info` / `footer` / `frame` normally. `RecoveredFileTranscodesToCleanChunks` pins the **normalization workflow** for salvaged files (open -> drain frames with footer times -> re-record): the one-frame recovery chunks become proper chunks, `created_utc` round-trips exactly, and `game_time` drifts at most 1 tick (100 ns) per frame through the float-seconds register -- the documented transcode caveat. `BM_ReaderRecoveredTail` (benchmarks) measures the salvage cost that motivates it: a 200-frame one-frame-chunk tail reads sequentially ~3x slower than clean chunking +- **tests**: **`tests/writer/test_crash_recovery.cpp`** -- 65 cases covering the full crash-window matrix: recovery between chunks (single- and multi-frame), between frames (in-flight batch, and nothing-flushed-yet), a torn tail chunk, a partial chunk written before its journal record, a checksum-detected corrupt chunk, a torn pending-frame record, a **torn chunk-commit record** (the batch falls back to its still-present `F` records -- the append-only guarantee), a crash mid-footer-write, a leftover-journal-over-valid-footer, a **stale compaction temp**, a mismatched journal format, a clean file (no-op), a header-only crash (0-frame result), the lingering-`F`-record dedup guard, compaction reclaim, the recovery helpers, exact per-frame time preservation, a **live-recording repair refusal** (Windows), and the FlatBuffers + Protobuf repair paths. Crash states are fabricated byte-faithfully through the same `RecoveryJournal` API the sink uses, plus **end-to-end tests through the real writer** (a raw `ReplayWriter` dropped without `Stop()`) for both formats that require the recovered footer to match a cleanly-stopped control **exactly**: per-frame `game_time` + `created_utc`, `duration_seconds`, timeline gaps, game segments, and per-entity `content_hash` -- and require the recovered file to pass whole-replay `ValidateReplayFile` and cache-hostile out-of-order seeks across the committed-chunk/recovered-chunk boundary. Also covered: the sink's **full config matrix** (`durable_writes` x `b_use_compression`) with frames large enough that **zstd compression genuinely engages** (committed chunks and journaled `F` payloads take the compressed path), a **repair interrupted mid-run and re-run** (idempotent), a **second repair on an already-repaired file** (clean no-op, byte-identical), a **journal-opted-out crash** (repair reports clean and leaves the footerless file byte-identical; the reader rejects it gracefully), a **session-start crash** (torn `S` record + header-only `.vtx` -> valid 0-frame file), **torn `T` records** (frame times fall back to the still-present `F` records -- exact, not zeroed), a **0-byte journal** (refused, main file untouched), **compaction through the real sink** under crash (new `Config::journal_compact_threshold_bytes`, 0 = default), **map-container frames** recovered intact from `F` records, a **non-ASCII path** through the full sidecar flow, **hostile checksummed journal records** (a `C` offset inside the header region, a `C` size at or below its own length prefix, an `F` with an empty payload -- each dropped safely, degrading to a valid file), a **torn main-file header** alongside a valid journal (refused, bytes untouched), and a **decreasing-time recording** (`is_increasing = false` -- the other branch of the segment reconstruction) matching its clean control. Scale + pipeline coverage: a **stress run** (2030 multi-entity frames, 40 committed chunks + a 30-frame in-flight batch -- every timestamp exact, whole-replay validation clean), **rejected frames interleaved mid-recording** (timer rejections leave no trace and do not desync the journal's frame indexing), and a **post-processor recording** (journaled `F` records capture the frame as it would hit disk -- post-mutation -- proven by a pending frame that only ever existed in the journal). Lifecycle + hygiene: a **double-crash lifecycle** over the same path (crash -> repair -> re-record -> crash -> repair, second recovery reflects only the second session), a **stale sidecar under journaling opt-out** (a session with `enable_recovery_journal = false` removes any leftover `.recovery` at start so it cannot masquerade as recovery state), a **foreign `.recovery` file** (no `VTXR` magic -- never deleted by repair, even on the clean-file path), a **hostile out-of-range `T` record** (ignored by the bounds check), the **map crash-recovery path on Protobuf** as well as FlatBuffers, a **stale journal from a different recording** over a replaced file (per-chunk checksums reject the foreign chunks -- graceful degradation to a valid empty file), a **read-only crashed file** (repair refuses without consuming the journal; succeeds untouched once the file is writable), a **no-time-registry recording** (`EGameTimeType::None`, fully FPS-synthesized timeline recovered exactly), and **byte-budget chunk splitting** (`max_bytes`, the other `ThresholdChunkPolicy` branch) through crash + recovery. The maximal guarantee is pinned by **`BoundaryCrashRecoversByteIdenticalFile`** (FlatBuffers **and** Protobuf, plus a 120-frame large-footer variant that exercises the compressed-footer path): a crash exactly at a chunk boundary recovers to a file that is **bit-for-bit identical** to a clean `Stop()` with the same inputs -- repair passes the footer's time vectors exactly as `Stop()` does (present-but-empty rather than absent) and compresses the synthesized footer exactly as the sink's `Close()` would (the `S` record now also journals `{use_compression, compression_level}`; journal version 4). A journal with an **incompatible version field** is refused outright, main file untouched. On top of the hand-picked windows, a **brute-force sweep suite** (`CrashRecoverySweep`) proves the "any crash point" claim literally: the journal truncated at **every byte length**, the main `.vtx` truncated at **every byte length**, and **every journal byte flipped** one at a time -- thousands of repair runs, none may crash, and every claimed repair must open and agree with its own reported frame count and footer times. `ReadValid` now bounds each record's payload allocation by the journal's **actual file size** (a corrupt length field can no longer trigger a giant transient allocation), pinned by `HostileRecordLengthIsBoundedByFileSize`; the truncation sweep also runs over the **post-compaction journal layout**. Finally, `CrashRecoveryProcess` (Windows) performs a **genuine process kill**: vtx_tests spawns itself as a child in a hidden endless-writer mode and `TerminateProcess`es it mid-recording -- no destructors, no fclose, handles abandoned to the kernel -- then repairs and verifies frame content and exact footer times, in **both** durability modes (this is the honest validation of the flush-only claim that data handed to the OS survives a process death) -- plus a kill under an **aggressive compaction cadence** (a full close/rewrite/atomic-rename cycle per commit), so real process death lands amid compaction traffic and the atomic-rename guarantee holds, and a **varied-progress kill soak** (kills landing just after the first journaled frames, around the first flush, amid compaction cycles, and several chunks deep -- alternating durability modes). The recovered output was also smoke-checked through the end-user toolchain: `vtx_cli` opens a repaired file and serves `info` / `footer` / `frame` normally. The whole suite (including the brute-force sweeps and real process kills) also ran clean under **AddressSanitizer** (MSVC `/fsanitize=address`) -- zero memory errors in the SDK; the one report is a known protobuf-internal `DynamicMessage` sized-delete artifact, documented in `tests/sanitizer_suppressions/asan.supp` (`ASAN_OPTIONS=new_delete_type_mismatch=0`). `RecoveredFileTranscodesToCleanChunks` pins the **normalization workflow** for salvaged files (open -> drain frames with footer times -> re-record): the one-frame recovery chunks become proper chunks, `created_utc` round-trips exactly, and `game_time` drifts at most 1 tick (100 ns) per frame through the float-seconds register -- the documented transcode caveat. `BM_ReaderRecoveredTail` (benchmarks) measures the salvage cost that motivates it: a 200-frame one-frame-chunk tail reads sequentially ~3x slower than clean chunking - **benchmarks**: new **`benchmarks/bench_crash_recovery.cpp`** (`BM_WriterDurabilityTier`) quantifies the writer hot-path cost of the recovery defaults across the three durability tiers (journal off / journal + flush-only / journal + fsync). Reference numbers on an NVMe dev machine, 200 small frames per iteration: **~19 us/frame** with no journal, **~30 us/frame** flush-only (process-crash safe -- effectively free), **~289 us/frame** with the default per-operation fsync (power-loss safe; ~1.7% of a 60 fps frame budget on NVMe -- on spinning disks prefer `durable_writes = false`). The byte-identity tests now compare from the end of the header on (the header embeds a second-granularity recording timestamp, so two separately recorded sessions may legitimately differ there; repair never rewrites the header) - **docs**: **`docs/SDK_API.md`** "Writing Replays" gains a **"Crash recovery"** section -- the `.recovery` sidecar model, the user-driven `ReplayNeedsRecovery` / `RepairReplayFile` / `RecoveryJournalPath` flow with `RepairResult` semantics, the exact-times guarantee, the refusal cases, and the sink durability knob table (`durable_writes` / `enable_recovery_journal` / `journal_compact_threshold_bytes`). **`README.md`** feature list gains a "Crash-safe recording" bullet. `DurableFile` hardening: `Seek`/`SeekEnd` failures now latch `Good()` false (a write after an unnoticed seek failure would land at the wrong offset), and the dead `Truncate` primitive left over from the pre-append-only journal design was removed diff --git a/tests/reader/test_reader_api.cpp b/tests/reader/test_reader_api.cpp index 11a201e..32c376a 100644 --- a/tests/reader/test_reader_api.cpp +++ b/tests/reader/test_reader_api.cpp @@ -161,8 +161,14 @@ TEST(ReaderApiFlatBuffers, CacheWindowZeroEvictsPreviousChunks) { auto ctx = VTX::OpenReplayFile(path); ASSERT_TRUE(ctx) << ctx.error; + // OpenReplayFile eagerly warms chunk 0 on a background thread; wait for it so + // the cache assertions below are deterministic. (Asserting a cold cache here + // raced that warm-up -- it only passed when the assert beat the loader thread, + // which slow/instrumented runners lose.) + ASSERT_TRUE(ctx.reader->WaitUntilReady()); + EXPECT_EQ(ctx.reader->GetChunkFrameCountSafe(0), 1); // warmed by open + ctx.reader->SetCacheWindow(0, 0); - EXPECT_EQ(ctx.reader->GetChunkFrameCountSafe(0), 0); ASSERT_NE(ctx.reader->GetFrameSync(0), nullptr); EXPECT_EQ(ctx.reader->GetChunkFrameCountSafe(0), 1); diff --git a/tests/writer/test_crash_recovery.cpp b/tests/writer/test_crash_recovery.cpp index 9000660..1b57934 100644 --- a/tests/writer/test_crash_recovery.cpp +++ b/tests/writer/test_crash_recovery.cpp @@ -1677,6 +1677,14 @@ namespace { // nothing in flight) must recover to a file that is BYTE-IDENTICAL to one produced // by a clean Stop() with the same inputs -- same chunks, same seek table, same // footer (times, gaps, segments, duration, compression), same trailer. + // + // The file header embeds a SECOND-granularity recording timestamp and is + // zstd-compressed, so a control/crash pair recorded across a second boundary gets + // different header bytes -- and possibly a different compressed header SIZE, which + // shifts every absolute offset in the seek table (a legitimate difference; repair + // never rewrites the header). Full-file identity is therefore only meaningful for + // a SAME-SECOND pair: re-record the pair (bounded retries, each takes well under a + // second) until both headers are byte-identical, then demand total equality. template void RunBoundaryByteIdentity(const std::string& prefix, int frames, int32_t chunk_max) { auto record_all = [frames](RawWriterFor& w) { @@ -1690,19 +1698,45 @@ namespace { } w.Flush(); // land the trailing batch -> crash sits exactly on a chunk boundary }; + auto header_region = [](const std::vector& bytes) { + if (bytes.size() < 8) + return std::vector(); + uint32_t header_size = 0; + std::memcpy(&header_size, bytes.data() + 4, sizeof(header_size)); + const size_t end = static_cast(8) + header_size; + if (end > bytes.size()) + return std::vector(); + return std::vector(bytes.begin(), bytes.begin() + static_cast(end)); + }; const std::string control_path = VtxTest::OutputPath(prefix + "_ident_control.vtx"); - { - auto w = MakeRawWriter(control_path, chunk_max); - record_all(*w); - w->Stop(); - } const std::string crash_path = VtxTest::OutputPath(prefix + "_ident_crash.vtx"); - { - auto w = MakeRawWriter(crash_path, chunk_max); - record_all(*w); - // dropped without Stop(): every frame is in a committed chunk, none pending + + // Flush-only durability: fsync vs fflush changes NO file bytes (and the flag is + // not journaled), but it makes each recording take milliseconds instead of + // seconds -- essential so a same-second pair is reachable even on slow CI + // runners, where fsync-per-frame recordings span more than a second each. + bool same_second_pair = false; + for (int attempt = 0; attempt < 8 && !same_second_pair; ++attempt) { + { + auto w = MakeRawWriter(control_path, chunk_max, /*durable=*/false); + record_all(*w); + if (::testing::Test::HasFatalFailure()) + return; + w->Stop(); + } + { + auto w = MakeRawWriter(crash_path, chunk_max, /*durable=*/false); + record_all(*w); + if (::testing::Test::HasFatalFailure()) + return; + // dropped without Stop(): every frame is in a committed chunk, none pending + } + const auto control_header = header_region(VtxTest::ReadAllBytes(control_path)); + const auto crash_header = header_region(VtxTest::ReadAllBytes(crash_path)); + same_second_pair = !control_header.empty() && control_header == crash_header; } + ASSERT_TRUE(same_second_pair) << "could not record a same-second control/crash pair in 8 attempts"; const auto rr = VTX::RepairReplayFile(crash_path); ASSERT_TRUE(rr.ok()) << rr.error; @@ -1711,28 +1745,8 @@ namespace { const auto control_bytes = VtxTest::ReadAllBytes(control_path); const auto recovered_bytes = VtxTest::ReadAllBytes(crash_path); ASSERT_FALSE(control_bytes.empty()); - ASSERT_GE(control_bytes.size(), 8u); - ASSERT_GE(recovered_bytes.size(), 8u); - - // The header embeds a second-granularity recording timestamp, so two separately - // recorded sessions may legitimately differ there (repair never touches the - // header -- the recovered file keeps its own, which IS the guarantee). Compare - // everything from the end of the header on: chunks, seek table, footer, trailer. - auto header_end = [](const std::vector& bytes) { - uint32_t header_size = 0; - std::memcpy(&header_size, bytes.data() + 4, sizeof(header_size)); - return static_cast(8) + header_size; - }; - const size_t control_body = header_end(control_bytes); - ASSERT_EQ(header_end(recovered_bytes), control_body); // same header structure/size - ASSERT_LT(control_body, control_bytes.size()); - EXPECT_EQ(recovered_bytes.size(), control_bytes.size()); - const bool body_identical = - recovered_bytes.size() == control_bytes.size() && - std::equal(control_bytes.begin() + static_cast(control_body), control_bytes.end(), - recovered_bytes.begin() + static_cast(control_body)); - EXPECT_TRUE(body_identical); // chunks + footer + trailer bit-for-bit equal + EXPECT_EQ(recovered_bytes, control_bytes); // bit-for-bit equal to the clean file } } // namespace