From c3451e92483918dae5ad04a35e568db25e5e8e9d Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 08:31:26 +0800 Subject: [PATCH 01/15] Segment the feed journal and serve it to many runners Journal: the normalized log is now a sequence of segments under journal/, each with a hashed checkpoint of the verified state before its first message. A full segment is sealed after a commit (successor checkpoint, then the empty successor, then the cursor that names it), and sealed segments expire by retained bytes (--replay-bytes) and venue time (--replay-age-seconds), never the open segment or one holding the protected window (open minute and closed-minute proofs). Resume verifies the retained journal from its oldest checkpoint, removes the residue of an interrupted seal or expiry, and refuses an expired --output-from with recovery guidance. The 256 MiB terminal budget (--max-log-bytes) is gone. Group commit, persist-before-publish and message boundaries are kept. PINEFORGE_FEED_CRASH_AT is a test hook that kills the process at a named durable step. serve: the same producer publishes to WebSocket clients over CivetWeb 1.16 (MIT, fetched by release archive and pinned by SHA-256, HTTP and WebSocket core only, no TLS, CGI, Lua, WebDAV, SSI or file serving, behind PINEFORGE_FEED_SERVE). GET /v1/status, GET /v1/snapshot (complete prefix from index 0, at most 4 MiB) and ws://HOST/v1/stream?epoch=E&from=I with cursor negotiation in the HTTP upgrade (400/409/410/416/503). Each client reads the journal to the head, then drains a bounded queue (--client-queue-bytes); an overflowing client is closed with 1008 "slow consumer" and never blocks the producer or other clients. An unsolicited PONG after 5 s without a message keeps the runner's 15 s idle deadline. Loopback by default; another interface needs --allow-remote-listen. REST catch-up: a long outage no longer ends in exit 22: when the reader's queue is full it drops the connection, waits for the session to drain and reconnects, and REST heals the rest after the overlap check. Binance REST is paced by the venue's X-MBX-USED-WEIGHT-1M count up to 90% of the one-minute ceiling instead of a fixed half-rate floor. A quiet Binance stream is proven alive with a LIST_SUBSCRIPTIONS probe instead of a reconnect every 75 s (--silence-seconds). Follow-ups: the OKX multiplier test waits on answered instrument reads instead of a pause and also runs in tick mode; the USD-M quiet start searches hour windows up to the newest venue time (48 h at most) and has its own test; Venue::predecessor_if_ready is pure virtual; Bybit's 403 stop names a regional block and a ten-minute backoff. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/CMakeLists.txt | 30 ++ native/feed/NOTICE | 11 + native/feed/src/binance.cpp | 58 ++- native/feed/src/binance.hpp | 5 + native/feed/src/bybit.cpp | 4 +- native/feed/src/bybit.hpp | 1 + native/feed/src/core.cpp | 16 + native/feed/src/core.hpp | 15 +- native/feed/src/main.cpp | 42 +- native/feed/src/okx.cpp | 2 +- native/feed/src/serve.cpp | 466 +++++++++++++++++++ native/feed/src/serve.hpp | 34 ++ native/feed/src/session.cpp | 65 ++- native/feed/src/session.hpp | 2 + native/feed/src/state.cpp | 370 ++++++++++++--- native/feed/src/state.hpp | 37 +- native/feed/src/transport.cpp | 100 +++- native/feed/src/transport.hpp | 8 +- native/feed/src/venue.hpp | 17 +- native/feed/tests/mock_venue.py | 799 ++++++++++++++++++++++++++++++-- native/feed/tests/public_e2e.py | 82 +++- native/feed/tests/serve_e2e.py | 263 +++++++++++ native/feed/tests/unit.cpp | 205 +++++++- 23 files changed, 2419 insertions(+), 213 deletions(-) create mode 100644 native/feed/src/serve.cpp create mode 100644 native/feed/src/serve.hpp create mode 100644 native/feed/tests/serve_e2e.py diff --git a/native/feed/CMakeLists.txt b/native/feed/CMakeLists.txt index da33174..3f462dd 100644 --- a/native/feed/CMakeLists.txt +++ b/native/feed/CMakeLists.txt @@ -55,10 +55,40 @@ target_link_libraries(pineforge-feed PRIVATE pineforge_feed_core) target_compile_options(pineforge-feed PRIVATE -Wall -Wextra -Wpedantic -Werror) install(TARGETS pineforge-feed RUNTIME DESTINATION bin) +# serve: the only code path that links an HTTP/WebSocket server. CivetWeb 1.16 (MIT) is fetched by release +# archive and pinned by SHA-256; only its core is compiled: no TLS, CGI, Lua, Duktape, WebDAV, SSI or files. +option(PINEFORGE_FEED_SERVE "Build the serve subcommand with the pinned CivetWeb release" ON) +if(PINEFORGE_FEED_SERVE) + enable_language(C) + include(FetchContent) + set(PINEFORGE_FEED_CIVETWEB_ARCHIVE "https://github.com/civetweb/civetweb/archive/refs/tags/v1.16.tar.gz" + CACHE STRING "CivetWeb 1.16 release archive, a URL or a local path; its SHA-256 is pinned") + FetchContent_Declare(civetweb URL "${PINEFORGE_FEED_CIVETWEB_ARCHIVE}" + URL_HASH SHA256=f0e471c1bf4e7804a6cfb41ea9d13e7d623b2bcc7bc1e2a4dd54951a24d60285 + DOWNLOAD_EXTRACT_TIMESTAMP TRUE SOURCE_SUBDIR do-not-add-civetweb-cmake) + FetchContent_MakeAvailable(civetweb) + add_library(pineforge_feed_civetweb STATIC "${civetweb_SOURCE_DIR}/src/civetweb.c") + target_include_directories(pineforge_feed_civetweb SYSTEM PUBLIC "${civetweb_SOURCE_DIR}/include") + target_compile_definitions(pineforge_feed_civetweb PRIVATE NO_SSL NO_CGI NO_FILES NO_CACHING + PUBLIC USE_WEBSOCKET) + set_target_properties(pineforge_feed_civetweb PROPERTIES C_STANDARD 99) + target_compile_options(pineforge_feed_civetweb PRIVATE -w) + target_link_libraries(pineforge_feed_civetweb PUBLIC Threads::Threads ${CMAKE_DL_LIBS}) + add_library(pineforge_feed_serve STATIC src/serve.cpp) + target_link_libraries(pineforge_feed_serve PUBLIC pineforge_feed_core PRIVATE pineforge_feed_civetweb) + target_compile_options(pineforge_feed_serve PRIVATE -Wall -Wextra -Wpedantic -Werror) + target_link_libraries(pineforge-feed PRIVATE pineforge_feed_serve) + target_compile_definitions(pineforge-feed PRIVATE PINEFORGE_FEED_SERVE=1) +endif() + include(CTest) if(BUILD_TESTING) add_executable(feed-unit tests/unit.cpp) target_link_libraries(feed-unit PRIVATE pineforge_feed_core) + if(PINEFORGE_FEED_SERVE) + target_link_libraries(feed-unit PRIVATE pineforge_feed_serve) + target_compile_definitions(feed-unit PRIVATE PINEFORGE_FEED_SERVE=1) + endif() target_compile_options(feed-unit PRIVATE -Wall -Wextra -Wpedantic -Werror -UNDEBUG) add_test(NAME feed-unit COMMAND feed-unit) find_package(Python3 REQUIRED COMPONENTS Interpreter) diff --git a/native/feed/NOTICE b/native/feed/NOTICE index 362639a..c7513bc 100644 --- a/native/feed/NOTICE +++ b/native/feed/NOTICE @@ -18,6 +18,17 @@ Linked dependencies retain their upstream terms: OpenSSL 3.x: Apache License, Version 2.0, https://openssl.org Preserve upstream licenses/notices when distributing bundled libraries. +The serve subcommand (build option PINEFORGE_FEED_SERVE, on by default) compiles CivetWeb: + CivetWeb 1.16, MIT License. Copyright (c) 2013-2021 The CivetWeb developers; + Copyright (c) 2004-2013 Sergey Lyubka; Copyright (c) 2013 No Face Press, LLC (Thomas Davis); + Copyright (c) 2013 F-Secure Corporation. https://github.com/civetweb/civetweb + Fetched at configure time from the release archive + https://github.com/civetweb/civetweb/archive/refs/tags/v1.16.tar.gz + Archive SHA-256: f0e471c1bf4e7804a6cfb41ea9d13e7d623b2bcc7bc1e2a4dd54951a24d60285 + Only src/civetweb.c is compiled, with NO_SSL, NO_CGI, NO_FILES and NO_CACHING; Lua and Duktape + are not enabled. Its MIT license text (LICENSE.md in the archive) must accompany binary + distributions that include serve. + CI builds libcurl 8.14.1 from curl-8.14.1.tar.xz. Archive SHA-256: f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd Source: https://curl.se/download/curl-8.14.1.tar.xz diff --git a/native/feed/src/binance.cpp b/native/feed/src/binance.cpp index f9e5f60..72c7d17 100644 --- a/native/feed/src/binance.cpp +++ b/native/feed/src/binance.cpp @@ -74,6 +74,12 @@ Trade usdm_aggregate(const Json& value) { Frame binance_frame(const std::string& message) { try { const auto envelope = parse_json(message); + // The reply to the liveness probe: {"result":[subscribed streams],"id":N}. Without them all, reconnect. + if (const auto* result = envelope.find("result")) { + if (!envelope.find("id")) throw Error(23, "unexpected public stream reply"); + if (result->kind != Json::Kind::Array || result->items.empty()) return Frame::Stale; + return Frame::Control; + } const auto* wrapped = envelope.find("data"); const auto& data = wrapped ? *wrapped : envelope; const auto* event = data.find("e"); @@ -83,7 +89,8 @@ Frame binance_frame(const std::string& message) { Connection BinanceSpot::connection() const { const auto symbol = lower(config_.symbol); - return {"/stream?streams=" + (config_.mode == "ticks" ? symbol + "@trade/" + symbol + "@kline_1m" : symbol + "@kline_1m"), {}, {}, &binance_frame}; + return {"/stream?streams=" + (config_.mode == "ticks" ? symbol + "@trade/" + symbol + "@kline_1m" : symbol + "@kline_1m"), {}, {}, + &binance_frame, "{\"method\":\"LIST_SUBSCRIPTIONS\",\"id\":1}"}; } std::vector BinanceSpot::decode(const std::string& message) const { try { @@ -176,7 +183,7 @@ unsigned int usdm_kline_weight(std::size_t limit) { return limit < 100 ? 1 : lim Connection usdm_connection(const Config& config) { const auto symbol = lower(config.symbol); return {"/market/stream?streams=" + (config.mode == "agg-ticks" ? symbol + "@aggTrade/" + symbol + "@kline_1m" : symbol + "@kline_1m"), - {}, {}, &binance_frame}; + {}, {}, &binance_frame, "{\"method\":\"LIST_SUBSCRIPTIONS\",\"id\":1}"}; } BinanceUsdm::BinanceUsdm(const Config& config) : config_(config), http_(config, usdm_rest()) { // An unknown symbol is refused before any stream is opened (HTTP 400, code -1121). @@ -199,10 +206,16 @@ std::vector BinanceUsdm::decode(const std::string& message) const { if (wrapped && !(aggregates && type == "aggTrade" && config_.mode == "agg-ticks") && !(klines && type == "kline")) throw Error(23, "unsupported USD-M stream event"); if (data.at("s").text() != config_.symbol) throw Error(23, "public stream symbol changed"); - if (type == "aggTrade") return {{VenueEvent::Kind::Trade, usdm_aggregate(data), {}}}; + if (type == "aggTrade") { + const auto trade = usdm_aggregate(data); + newest_ = std::max(newest_, trade.ts); + return {{VenueEvent::Kind::Trade, trade, {}}}; + } const auto& row = data.at("k"); if (row.at("s").text() != config_.symbol) throw Error(23, "kline symbol changed"); - return {{VenueEvent::Kind::Kline, {}, binance_kline(row, true)}}; + const auto kline = binance_kline(row, true); + newest_ = std::max(newest_, kline.bar.ts + 59999); + return {{VenueEvent::Kind::Kline, {}, kline}}; } if (type == "serverShutdown") return {}; log("warn", "unknown_stream_event", Json::object({{"stream", Json::string(stream)}, {"type", Json::string(type)}})); @@ -224,31 +237,38 @@ std::vector BinanceUsdm::history(std::uint64_t from, std::size_t limit) { } return trades; } -namespace { -// The first aggregate at or after the start, searched over the next hour (the venue's longest time window), -// so a quiet start minute still has a fence; 0 when none is visible yet. -std::uint64_t first_aggregate(HttpClient& http, const std::string& symbol, std::int64_t minute) { - const auto response = http.get("/fapi/v1/aggTrades?symbol=" + symbol + "&startTime=" + std::to_string(minute) + - "&endTime=" + std::to_string(minute + 3599999) + "&limit=1", 20); - const auto& rows = array(response, 1); - if (rows.empty()) return 0; - const auto first = usdm_aggregate(rows.front()); - if (first.ts < minute || first.ts >= minute + 3600000) throw Error(20, "initial aggregate fence is unavailable or ambiguous"); - return first.id; -} +// The first aggregate at or after the start, searched hour window by hour window (the venue's longest time +// window) up to the newest venue time seen on the WebSocket, so a quiet start of any length within the 48-hour +// history finds its fence; 0 when none is visible yet. A window that ended two minutes before that venue time +// is settled (REST trails the WebSocket by seconds) and is not read again. +std::uint64_t BinanceUsdm::first_aggregate(std::int64_t minute) { + for (auto window = std::max(minute, quiet_until_);; window += 3600000) { + if (window - minute >= retention_ms()) throw Error(20, "no aggregate within the venue's 48-hour history after the start minute"); + const auto response = http_.get("/fapi/v1/aggTrades?symbol=" + config_.symbol + "&startTime=" + std::to_string(window) + + "&endTime=" + std::to_string(window + 3599999) + "&limit=1", 20); + const auto& rows = array(response, 1); + if (!rows.empty()) { + const auto first = usdm_aggregate(rows.front()); + if (first.ts < window || first.ts >= window + 3600000) throw Error(20, "initial aggregate fence is unavailable or ambiguous"); + return first.id; + } + if (window + 3600000 + 120000 <= newest_) quiet_until_ = window + 3600000; + if (window + 3600000 > newest_) return 0; + } } std::uint64_t BinanceUsdm::first_trade_id(std::int64_t minute) { for (unsigned int attempt = 0; attempt < 3; ++attempt) { if (attempt) pause_for(std::chrono::milliseconds(500 * attempt)); - if (const auto first = first_aggregate(http_, config_.symbol, minute)) return first; + if (const auto first = first_aggregate(minute)) return first; } throw Error(20, "no aggregate after the start minute is available yet"); } std::optional BinanceUsdm::predecessor_if_ready(std::int64_t minute) { - const auto first = first_aggregate(http_, config_.symbol, minute); + const auto first = first_aggregate(minute); if (!first) return std::nullopt; + if (first <= 1) throw Error(20, "raw predecessor cannot be proven"); const auto previous = history(first - 1, 1); - if (first <= 1 || previous.empty() || previous.front().id != first - 1) throw Error(20, "raw predecessor is unavailable"); + if (previous.empty() || previous.front().id != first - 1) throw Error(20, "raw predecessor is unavailable"); return previous.front(); } std::vector BinanceUsdm::klines(std::int64_t start, std::int64_t end) { diff --git a/native/feed/src/binance.hpp b/native/feed/src/binance.hpp index aed398e..a4cd5d3 100644 --- a/native/feed/src/binance.hpp +++ b/native/feed/src/binance.hpp @@ -22,6 +22,8 @@ class BinanceSpot final : public Venue { std::vector decode(const std::string& message) const override; std::vector history(std::uint64_t from, std::size_t limit = 1000) override; std::uint64_t first_trade_id(std::int64_t minute) override; + // Binance spot ticks close minutes on the kline ID range, never on a next-print fence. + std::optional predecessor_if_ready(std::int64_t) override { throw Error(23, "Binance spot has no next-print fence"); } std::vector klines(std::int64_t start, std::int64_t end) override; std::string kline_source() const override { return "/api/v3/klines"; } }; @@ -31,6 +33,9 @@ class BinanceSpot final : public Venue { class BinanceUsdm final : public Venue { Config config_; HttpClient http_; + mutable std::int64_t newest_ = -1; // newest venue time decoded from the WebSocket (main thread only) + std::int64_t quiet_until_ = -1; // [start, quiet_until_) is settled and holds no aggregate + std::uint64_t first_aggregate(std::int64_t minute); public: explicit BinanceUsdm(const Config& config); Connection connection() const override { return usdm_connection(config_); } diff --git a/native/feed/src/bybit.cpp b/native/feed/src/bybit.cpp index 4cded5f..8a63e09 100644 --- a/native/feed/src/bybit.cpp +++ b/native/feed/src/bybit.cpp @@ -23,7 +23,7 @@ RestPolicy bybit_rest() { policy.throttled = [](const std::string& body) { return body.find("\"retCode\":10006") != std::string::npos; }; // Bybit answers HTTP 403 for its roughly ten-minute IP rate ban: a restart after the ban recovers. policy.rejected = [](long status, const std::string&) { - if (status == 403) throw Error(20, "Bybit IP rate ban (HTTP 403); restart after the venue's ban window"); + if (status == 403) throw Error(20, "Bybit IP rate ban or regional block (HTTP 403); restart with a backoff above ten minutes"); }; // Server timeout and server error: the venue asks to retry. policy.transient = [](long, const std::string& body) { @@ -106,7 +106,7 @@ Json Bybit::result(const std::string& path) { } Connection Bybit::connection() const { return {"/v5/public/" + category(config_), {"{\"op\":\"subscribe\",\"args\":[\"kline.1." + config_.symbol + "\"]}"}, - "{\"op\":\"ping\"}", &bybit_frame}; + "{\"op\":\"ping\"}", &bybit_frame, {}}; } std::vector Bybit::decode(const std::string& message) const { try { diff --git a/native/feed/src/bybit.hpp b/native/feed/src/bybit.hpp index fbb1be4..8ebf78c 100644 --- a/native/feed/src/bybit.hpp +++ b/native/feed/src/bybit.hpp @@ -22,6 +22,7 @@ class Bybit final : public Venue { Connection connection() const override; std::vector decode(const std::string& message) const override; std::vector history(std::uint64_t, std::size_t) override { throw Error(23, "Bybit has no pageable trade history"); } + std::optional predecessor_if_ready(std::int64_t) override { throw Error(23, "Bybit has no pageable trade history"); } std::vector klines(std::int64_t start, std::int64_t end) override; std::string kline_source() const override { return "/v5/market/kline"; } }; diff --git a/native/feed/src/core.cpp b/native/feed/src/core.cpp index 666df49..7355859 100644 --- a/native/feed/src/core.cpp +++ b/native/feed/src/core.cpp @@ -9,6 +9,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -163,6 +166,19 @@ void output_line(std::string_view line) { } } +void crash_point(const char* name) { + static const std::string target = [] { + const char* value = std::getenv("PINEFORGE_FEED_CRASH_AT"); + return std::string(value ? value : ""); + }(); + if (target.empty()) return; + static std::atomic passes{0}; + const auto colon = target.find(':'); + if (target.compare(0, colon, name) != 0 || std::strlen(name) != (colon == std::string::npos ? target.size() : colon)) return; + const auto wanted = colon == std::string::npos ? 1UL : std::strtoul(target.c_str() + colon + 1, nullptr, 10); + if (++passes == wanted) ::raise(SIGKILL); +} + void atomic_file(const std::string& path, std::string_view bytes) { const auto temporary = path + ".tmp"; const int descriptor = ::open(temporary.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600); diff --git a/native/feed/src/core.hpp b/native/feed/src/core.hpp index c35d229..9d9bf7e 100644 --- a/native/feed/src/core.hpp +++ b/native/feed/src/core.hpp @@ -27,6 +27,9 @@ std::int64_t timestamp(const std::string& token); void write_all(int descriptor, std::string_view bytes); void output_line(std::string_view line); void atomic_file(const std::string& path, std::string_view bytes); +// Test hook for crash-safety probes: PINEFORGE_FEED_CRASH_AT=NAME[:N] kills the process with SIGKILL at the +// N-th (default first) pass through the named durable step. Unset, it does nothing. +void crash_point(const char* name); class Decimal { std::string coefficient_; @@ -89,11 +92,21 @@ struct Config { std::int64_t start = -1, end = -1; bool resume = false, allow_insecure = false; std::uint64_t output_from = 0, max_messages = 0; - std::uint64_t max_log_bytes = 256 * 1024 * 1024; + // Journal segments seal at segment_bytes; sealed segments expire beyond replay_bytes retained bytes or, + // when replay_age_ms is set, beyond that much venue time. + std::uint64_t segment_bytes = 16 * 1024 * 1024; + std::uint64_t replay_bytes = 256 * 1024 * 1024; + std::int64_t replay_age_ms = 0; std::uint64_t max_replay_seconds = 60; std::size_t max_queue_bytes = 16 * 1024 * 1024; std::uint64_t reconnect_seconds = 23 * 3600 + 55 * 60; std::uint64_t keepalive_seconds = 20; + std::uint64_t silence_seconds = 75; // data silence before a probe (or, without one, a reconnect) + // serve: the same producer publishes to WebSocket clients instead of stdout. + bool serve = false, allow_remote_listen = false; + std::string listen = "127.0.0.1:8787"; + std::size_t client_queue_bytes = 1024 * 1024; + std::size_t max_clients = 64; }; // Raw ticks and USD-M aggregate prints share the tick/time journal grammar. inline bool tick_mode(const std::string& mode) { return mode == "ticks" || mode == "agg-ticks"; } diff --git a/native/feed/src/main.cpp b/native/feed/src/main.cpp index 0bb6c1b..9ef1c9d 100644 --- a/native/feed/src/main.cpp +++ b/native/feed/src/main.cpp @@ -1,6 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 #include "session.hpp" #include "transport.hpp" +#if PINEFORGE_FEED_SERVE +#include "serve.hpp" +#endif #include #include #include @@ -26,6 +29,9 @@ struct StdoutFlags { void help() { std::cout << "pineforge-feed warmup --venue VENUE --market MARKET --symbol SYMBOL --start UTC|MS --end UTC|MS --output FILE\n" "pineforge-feed run --venue VENUE --market MARKET --symbol SYMBOL --mode MODE --state-dir DIR [--start UTC|MS | --resume] [--output-from INDEX]\n" + "pineforge-feed serve --venue VENUE --market MARKET --symbol SYMBOL --mode MODE --state-dir DIR [--start UTC|MS | --resume]\n" + " [--listen HOST:PORT (default 127.0.0.1:8787)] [--allow-remote-listen] [--client-queue-bytes N] [--max-clients N]\n" + " GET /v1/status, GET /v1/snapshot (complete prefix from index 0, at most 4 MiB), ws://HOST:PORT/v1/stream?epoch=E&from=I\n" "Venues and modes:\n" " binance spot BTCUSDT bars | ticks\n" " binance usdm BTCUSDT bars | agg-ticks (aggregate prints, not raw trades; bars can\n" @@ -33,7 +39,8 @@ void help() { " okx spot BTC-USDT bars | ticks\n" " okx swap BTC-USDT-SWAP bars | ticks (linear swaps; base quantities)\n" " bybit spot|linear BTCUSDT bars\n" - "Limits: --max-messages N --max-log-bytes N --max-replay-seconds N --max-queue-bytes N --reconnect-seconds N --keepalive-seconds N\n" + "Journal: --segment-bytes N --replay-bytes N --replay-age-seconds N\n" + "Limits: --max-messages N --max-replay-seconds N --max-queue-bytes N --reconnect-seconds N --keepalive-seconds N --silence-seconds N\n" "Testing/public origins: --rest-url ORIGIN --ws-url ORIGIN [--allow-insecure-http (loopback only)]\n"; } // Venue, market and mode gates are checked before any network access. @@ -88,7 +95,7 @@ int run(int argc, char** argv) { if (argc == 2 && std::string(argv[1]) == "--version") { check_runtime_curl(); std::cout << "pineforge-feed 0.1.0\n"; return 0; } if (argc < 2) throw Error(23, "choose warmup or run; see --help"); const std::string command = argv[1]; - if (command != "run" && command != "warmup") throw Error(23, "only warmup and run are implemented"); + if (command != "run" && command != "warmup" && command != "serve") throw Error(23, "choose warmup, run or serve; see --help"); Config config; std::string output; std::set seen; @@ -97,6 +104,7 @@ int run(int argc, char** argv) { if (!seen.insert(key).second) throw Error(23, "duplicate CLI option: " + key); if (key == "--resume") { config.resume = true; continue; } if (key == "--allow-insecure-http") { config.allow_insecure = true; continue; } + if (key == "--allow-remote-listen") { config.allow_remote_listen = true; continue; } if (index + 1 >= argc) throw Error(23, "CLI option needs a value: " + key); const std::string value = argv[++index]; if (key == "--venue") config.venue = value; @@ -109,21 +117,37 @@ int run(int argc, char** argv) { else if (key == "--output") output = value; else if (key == "--output-from") config.output_from = unsigned_value(value); else if (key == "--max-messages") config.max_messages = unsigned_value(value); - else if (key == "--max-log-bytes") config.max_log_bytes = unsigned_value(value); + else if (key == "--segment-bytes") config.segment_bytes = unsigned_value(value); + else if (key == "--replay-bytes") config.replay_bytes = unsigned_value(value); + else if (key == "--replay-age-seconds") { + const auto seconds = unsigned_value(value); + if (seconds > 10ULL * 366 * 86400) throw Error(23, "--replay-age-seconds is at most ten years"); + config.replay_age_ms = static_cast(seconds) * 1000; + } + else if (key == "--listen") config.listen = value; + else if (key == "--client-queue-bytes") config.client_queue_bytes = unsigned_value(value); + else if (key == "--max-clients") config.max_clients = unsigned_value(value); else if (key == "--max-replay-seconds") config.max_replay_seconds = unsigned_value(value); else if (key == "--max-queue-bytes") config.max_queue_bytes = unsigned_value(value); else if (key == "--reconnect-seconds") config.reconnect_seconds = unsigned_value(value); else if (key == "--keepalive-seconds") config.keepalive_seconds = unsigned_value(value); + else if (key == "--silence-seconds") config.silence_seconds = unsigned_value(value); else if (key == "--rest-url") config.rest_url = value; else if (key == "--ws-url") config.ws_url = value; else throw Error(23, "unknown CLI option: " + key); } gate(config, command == "warmup", seen.count("--rest-url") != 0, seen.count("--ws-url") != 0); - if (!config.max_log_bytes || !config.max_queue_bytes || !config.max_replay_seconds || config.max_replay_seconds > 3600 || + if (!config.max_queue_bytes || !config.max_replay_seconds || config.max_replay_seconds > 3600 || !config.reconnect_seconds || config.reconnect_seconds > 86100) throw Error(23, "budgets must be positive; reconnect must precede the 24-hour connection limit"); + if (config.segment_bytes < 4096 || config.segment_bytes > (1ULL << 30) || config.replay_bytes / 2 < config.segment_bytes) + throw Error(23, "--segment-bytes must be 4096..1 GiB and --replay-bytes at least twice --segment-bytes"); + // A client queue holds at least one atomic message (at most 4096 bytes). + if (config.client_queue_bytes < 4096 || config.client_queue_bytes > (1ULL << 30) || !config.max_clients || config.max_clients > 1024) + throw Error(23, "--client-queue-bytes must be 4096..1 GiB and --max-clients 1..1024"); // OKX closes a connection after 30 s without traffic; Bybit recommends a ping every 20 s. if (!config.keepalive_seconds || config.keepalive_seconds > 25) throw Error(23, "--keepalive-seconds must be 1..25"); + if (!config.silence_seconds || config.silence_seconds > 75) throw Error(23, "--silence-seconds must be 1..75"); check_runtime_curl(); validate_origin(config.rest_url, false, config.allow_insecure); validate_origin(config.ws_url, true, config.allow_insecure); @@ -131,7 +155,17 @@ int run(int argc, char** argv) { if (output.empty() || config.resume || !config.state_dir.empty() || seen.count("--output-from") || seen.count("--mode")) throw Error(23, "warmup requires --output and does not accept run-state options"); warmup(config, output); + } else if (command == "serve") { + if (config.state_dir.empty() || seen.count("--end") || !output.empty() || seen.count("--output-from")) + throw Error(23, "serve requires --state-dir; clients choose their own cursor (from=I), so --output-from is refused"); +#if PINEFORGE_FEED_SERVE + serve_feed(config); +#else + throw Error(23, "this build has no serve subcommand (configured with PINEFORGE_FEED_SERVE=OFF)"); +#endif } else { + if (seen.count("--listen") || config.allow_remote_listen || seen.count("--client-queue-bytes") || seen.count("--max-clients")) + throw Error(23, "--listen, --allow-remote-listen, --client-queue-bytes and --max-clients are serve options"); if (config.state_dir.empty() || seen.count("--end") || !output.empty() || (!config.resume && seen.count("--output-from"))) throw Error(23, "run requires --state-dir; --output-from requires --resume"); const auto flags = ::fcntl(STDOUT_FILENO, F_GETFL); diff --git a/native/feed/src/okx.cpp b/native/feed/src/okx.cpp index 4c3a379..1af908e 100644 --- a/native/feed/src/okx.cpp +++ b/native/feed/src/okx.cpp @@ -119,7 +119,7 @@ Connection Okx::connection() const { const auto subscribe = "{\"op\":\"subscribe\",\"args\":[" + argument("candle1m") + (config_.mode == "ticks" ? "," + argument("trades-all") : std::string()) + "]}"; // Business endpoint: trades-all and candles live there. The text keepalive must precede 30 s idle. - return {"/ws/v5/business", {subscribe}, "ping", &okx_frame}; + return {"/ws/v5/business", {subscribe}, "ping", &okx_frame, {}}; } std::vector Okx::decode(const std::string& message) const { try { diff --git a/native/feed/src/serve.cpp b/native/feed/src/serve.cpp new file mode 100644 index 0000000..fae789f --- /dev/null +++ b/native/feed/src/serve.cpp @@ -0,0 +1,466 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "serve.hpp" +#include "session.hpp" +#include "state.hpp" +#include "transport.hpp" +#include "civetweb.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +namespace { +constexpr std::size_t snapshot_limit = 4 * 1024 * 1024; // the runner's HTTP snapshot bound +constexpr auto keepalive = std::chrono::seconds(5); // the runner's WebSocket idle deadline is 15 s + +Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } + +struct Client { + std::uint64_t next = 0; // the next message index this client receives + std::deque> queue; + std::size_t queue_bytes = 0; + bool live = false, slow = false; +}; + +// Producer-side fan-out state. The producer appends released lines; each client's worker thread reads the +// journal from its cursor until it reaches the released head, then drains its own bounded queue. +struct Hub { + Config config; + std::string epoch, journal, source; + std::mutex mutex; + std::condition_variable changed; + std::uint64_t published = 0, first_retained = 0; + std::string status; // the durable cursor's public fields + std::set clients; + std::size_t connected = 0, serving = 0; + bool stopping = false; + + void publish(const std::string& line) { + const auto shared = std::make_shared(line); + std::lock_guard guard(mutex); + ++published; + for (auto* client : clients) { + if (!client->live || client->slow) continue; + // A client that cannot keep up is cut off; it never slows the producer or another client. + if (shared->size() > config.client_queue_bytes - std::min(client->queue_bytes, config.client_queue_bytes)) { + client->slow = true; + continue; + } + client->queue_bytes += shared->size(); + client->queue.push_back(shared); + } + changed.notify_all(); + } + void commit(const State& state) { + const auto& durable = state.durable(); + const auto document = Json::object({ + {"schema", Json::string("pineforge-feed-status/v1")}, {"epoch", Json::string(durable.epoch)}, + {"source", parse_json(source)}, + {"retained", Json::object({{"first_index", number(state.first_retained())}, {"next_index", number(durable.message_index)}})}, + {"last", Json::object({{"seq", number(durable.seq)}, {"verified_cut", Json::number(std::to_string(durable.cut))}, + {"last_tick_ts", Json::number(std::to_string(durable.last_tick_ts))}, + {"last_bar", Json::number(std::to_string(durable.last_bar))}})}}); + std::lock_guard guard(mutex); + first_retained = state.first_retained(); + status = document.dump(); + } + // Clients get their close frame (1001) before the server stops its connections. + void stop() { + std::unique_lock guard(mutex); + stopping = true; + changed.notify_all(); + changed.wait_for(guard, std::chrono::seconds(3), [this] { return serving == 0; }); + } +}; + +void respond(mg_connection* connection, int status, const char* type, const std::string& body) { + mg_printf(connection, "HTTP/1.1 %d %s\r\nContent-Type: %s\r\nCache-Control: no-store\r\nContent-Length: %zu\r\n" + "Connection: close\r\n\r\n", status, mg_get_response_code_text(connection, status), type, body.size()); + if (!body.empty()) mg_write(connection, body.data(), body.size()); +} +// Cursor negotiation fails in HTTP, before any WebSocket message, with the reason in the body. +int refuse(mg_connection* connection, int status, const std::string& reason) { + log("warn", "client_refused", Json::object({{"status", Json::number(std::to_string(status))}, {"reason", Json::string(reason)}})); + const auto body = Json::object({{"error", Json::string(reason)}}).dump() + '\n'; + respond(connection, status, "application/json", body); + return status; +} +bool get_only(mg_connection* connection) { + const auto* request = mg_get_request_info(connection); + if (std::string(request->request_method) == "GET") return true; + refuse(connection, 405, "only GET is served"); + return false; +} + +int on_status(mg_connection* connection, void* data) { + if (!get_only(connection)) return 405; + auto& hub = *static_cast(data); + std::string body; + { + std::lock_guard guard(hub.mutex); + auto document = parse_json(hub.status); + document.members["clients"] = number(hub.connected); + document.members["limits"] = Json::object({{"replay_bytes", number(hub.config.replay_bytes)}, + {"replay_age_seconds", number(static_cast(hub.config.replay_age_ms / 1000))}, + {"client_queue_bytes", number(hub.config.client_queue_bytes)}, {"max_clients", number(hub.config.max_clients)}}); + body = document.dump() + '\n'; + } + respond(connection, 200, "application/json", body); + return 200; +} + +// The complete committed prefix from index 0, the only HTTP shape the current runner accepts. +int on_snapshot(mg_connection* connection, void* data) { + if (!get_only(connection)) return 405; + auto& hub = *static_cast(data); + std::uint64_t published = 0, first = 0; + { + std::lock_guard guard(hub.mutex); + published = hub.published; + first = hub.first_retained; + } + const std::string expired = "the snapshot prefix from index 0 has expired from the retained journal; use /v1/stream"; + if (first) return refuse(connection, 410, expired); + try { + const auto body = snapshot_prefix(hub.journal, published, snapshot_limit); + if (!body) return refuse(connection, 413, "the committed prefix exceeds the runner's 4 MiB HTTP snapshot bound; use /v1/stream"); + respond(connection, 200, "application/x-ndjson", *body); + return 200; + } catch (const Error&) { return refuse(connection, 410, expired); } +} + +bool query_value(const std::string& query, const char* name, std::string& value) { + char buffer[128]; + const auto length = mg_get_var(query.c_str(), query.size(), name, buffer, sizeof(buffer)); + if (length < 0) return false; + value.assign(buffer, static_cast(length)); + return true; +} + +int on_connect(const mg_connection* constant, void* data) { + auto* connection = const_cast(constant); + auto& hub = *static_cast(data); + const auto* request = mg_get_request_info(connection); + const std::string query = request->query_string ? request->query_string : ""; + std::string epoch, from; + std::uint64_t index = 0; + if (!query_value(query, "epoch", epoch) || !query_value(query, "from", from) || from.empty() || + from.find_first_not_of("0123456789") != std::string::npos || + std::from_chars(from.data(), from.data() + from.size(), index).ec != std::errc{}) + return refuse(connection, 400, "use /v1/stream?epoch=E&from=I with the stream epoch and a message index") != 0; + int status = 0; + std::string reason; + { + std::lock_guard guard(hub.mutex); + if (hub.stopping) status = 503, reason = "the producer is stopping"; + else if (epoch != hub.epoch) status = 409, reason = "wrong epoch: this stream's epoch is " + hub.epoch; + else if (index > hub.published) + status = 416, reason = "unknown cursor: index " + from + " is beyond the committed log (next index " + + std::to_string(hub.published) + ")"; + else if (index < hub.first_retained) + status = 410, reason = "cursor expired: index " + from + " is older than the retained journal (first retained index " + + std::to_string(hub.first_retained) + "); rebuild the consumer from a warmup inside the retained window"; + else if (hub.connected >= hub.config.max_clients) status = 503, reason = "client limit reached (--max-clients)"; + else { + auto* client = new Client; + client->next = index; + hub.clients.insert(client); + ++hub.connected; + mg_set_user_connection_data(constant, client); + } + } + if (status) return refuse(connection, status, reason) != 0; + log("info", "client_connected", Json::object({{"from", number(index)}})); + return 0; +} + +bool send_close(mg_connection* connection, unsigned short code, const std::string& reason) { + std::string payload; + payload += static_cast(code >> 8); + payload += static_cast(code & 0xff); + payload += reason.substr(0, 120); + log(code == 1000 || code == 1001 ? "info" : "warn", "client_closed", + Json::object({{"code", Json::number(std::to_string(code))}, {"reason", Json::string(reason)}})); + return mg_websocket_write(connection, MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE, payload.data(), payload.size()) > 0; +} + +// One client, on its CivetWeb worker thread: the retained journal from its cursor, then live messages from +// its bounded queue, one WebSocket text message per normalized message, and an unsolicited PONG after 5 s +// without a message so a quiet stream never trips the runner's idle deadline. +void on_ready(mg_connection* connection, void* data) { + auto& hub = *static_cast(data); + auto* client = static_cast(mg_get_user_connection_data(connection)); + if (!client) return; + { + std::lock_guard guard(hub.mutex); + ++hub.serving; + } + struct Done { + Hub& hub; + ~Done() { + std::lock_guard guard(hub.mutex); + --hub.serving; + hub.changed.notify_all(); + } + } done{hub}; + auto last_send = std::chrono::steady_clock::now(); + // A write that cannot complete within the request timeout leaves a partial frame: the connection is dead. + const auto send = [&](const std::string& bytes, int opcode) { + const auto written = mg_websocket_write(connection, opcode, bytes.data(), bytes.size()); + if (written <= 0 || (!bytes.empty() && static_cast(written) != bytes.size())) { + log("warn", "client_write_failed", Json::object({{"next", number(client->next)}})); + return false; + } + last_send = std::chrono::steady_clock::now(); + return true; + }; + JournalReader reader(hub.journal); + if (!reader.seek(client->next)) { + send_close(connection, 4410, "cursor expired: older than the retained journal"); + return; + } + for (;;) { + std::uint64_t head = 0; + { + std::lock_guard guard(hub.mutex); + if (hub.stopping) { send_close(connection, 1001, "producer stopping"); return; } + head = hub.published; + if (client->next == head) { client->live = true; break; } + } + while (client->next < head) { + auto line = reader.next(); + if (!line) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + if (std::chrono::steady_clock::now() - last_send > std::chrono::seconds(30)) { + send_close(connection, 1011, "journal segment unavailable"); + return; + } + continue; + } + if (!send(*line, MG_WEBSOCKET_OPCODE_TEXT)) return; + ++client->next; + } + } + for (;;) { + std::shared_ptr line; + { + std::unique_lock guard(hub.mutex); + hub.changed.wait_until(guard, last_send + keepalive, [&] { return hub.stopping || client->slow || !client->queue.empty(); }); + // A stopping producer still delivers what it already released, then says goodbye. + if (hub.stopping && client->queue.empty() && !client->slow) { + guard.unlock(); + send_close(connection, 1001, "producer stopping"); + return; + } + if (client->slow) { + guard.unlock(); + send_close(connection, 1008, "slow consumer: queued messages exceeded --client-queue-bytes " + + std::to_string(hub.config.client_queue_bytes)); + return; + } + if (!client->queue.empty()) { + line = std::move(client->queue.front()); + client->queue.pop_front(); + client->queue_bytes -= line->size(); + } + } + if (line) { + if (!send(*line, MG_WEBSOCKET_OPCODE_TEXT)) return; + ++client->next; + } else if (std::chrono::steady_clock::now() - last_send >= keepalive && !send("", MG_WEBSOCKET_OPCODE_PONG)) return; + } +} +int on_data(mg_connection*, int, char*, std::size_t, void*) { return 1; } +void on_close(const mg_connection*, void*) {} + +// Every connection end, handshake failures included, releases its client. +void on_connection_close(const mg_connection* connection) { + auto* hub = static_cast(mg_get_user_data(mg_get_context(connection))); + auto* client = static_cast(mg_get_user_connection_data(connection)); + if (!hub || !client) return; + { + std::lock_guard guard(hub->mutex); + hub->clients.erase(client); + --hub->connected; + } + log("info", "client_disconnected", Json::object({{"next", number(client->next)}})); + delete client; + mg_set_user_connection_data(connection, nullptr); +} +int on_log(const mg_connection*, const char* message) { + log("warn", "http_server", Json::object({{"message", Json::string(std::string(message).substr(0, 512))}})); + return 1; +} + +class Server { + mg_context* context_ = nullptr; +public: + Server(Hub& hub, const std::string& address) { + mg_init_library(0); + const auto threads = std::to_string(hub.config.max_clients + 4); + const char* options[] = {"listening_ports", address.c_str(), "num_threads", threads.c_str(), + "request_timeout_ms", "30000", "websocket_timeout_ms", "10000", + "enable_websocket_ping_pong", "yes", "enable_keep_alive", "no", nullptr}; + mg_callbacks callbacks{}; + callbacks.log_message = &on_log; + callbacks.connection_close = &on_connection_close; + context_ = mg_start(&callbacks, &hub, options); + if (!context_) { + mg_exit_library(); + throw Error(23, "cannot listen on " + address); + } + mg_set_request_handler(context_, "/v1/status$", &on_status, &hub); + mg_set_request_handler(context_, "/v1/snapshot$", &on_snapshot, &hub); + mg_set_websocket_handler(context_, "/v1/stream$", &on_connect, &on_ready, &on_data, &on_close, &hub); + mg_server_port ports[4]{}; + const auto count = mg_get_server_ports(context_, 4, ports); + log("info", "serve_listening", Json::object({{"address", Json::string(address)}, + {"port", Json::number(std::to_string(count > 0 ? ports[0].port : 0))}})); + } + ~Server() { + mg_stop(context_); + mg_exit_library(); + } + Server(const Server&) = delete; + Server& operator=(const Server&) = delete; +}; +} + +JournalReader::~JournalReader() { if (descriptor_ >= 0) ::close(descriptor_); } + +bool JournalReader::open(std::uint64_t base) { + const auto path = directory_ + "/" + segment_name(base) + ".jsonl"; + const int descriptor = ::open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (descriptor < 0) return false; + if (descriptor_ >= 0) ::close(descriptor_); + descriptor_ = descriptor; + buffer_.clear(); + offset_ = 0; + index_ = base; + return true; +} + +bool JournalReader::seek(std::uint64_t index) { + // Retention may expire the chosen segment between the listing and the open: list once more. + for (int attempt = 0; attempt < 2; ++attempt) { + std::uint64_t chosen = 0; + bool found = false; + std::error_code failure; + for (const auto& entry : std::filesystem::directory_iterator(directory_, failure)) { + const auto name = entry.path().filename().string(); + if (name.size() != 26 || name.compare(20, 6, ".jsonl") != 0) continue; + std::uint64_t base = 0; + if (std::from_chars(name.data(), name.data() + 20, base).ec != std::errc{}) continue; + if (base <= index && (!found || base > chosen)) { chosen = base; found = true; } + } + if (failure || !found) return false; + if (!open(chosen)) continue; + while (index_ < index) { + auto skipped = next(); + if (!skipped) return false; + } + return true; + } + return false; +} + +std::optional JournalReader::next() { + for (;;) { + const auto newline = buffer_.find('\n', offset_); + if (newline != std::string::npos) { + std::string line = buffer_.substr(offset_, newline - offset_); + offset_ = newline + 1; + if (offset_ > 65536) { buffer_.erase(0, offset_); offset_ = 0; } + ++index_; + return line; + } + char chunk[65536]; + const auto count = ::read(descriptor_, chunk, sizeof(chunk)); + if (count < 0 && errno == EINTR) continue; + if (count > 0) { buffer_.append(chunk, static_cast(count)); continue; } + // End of this segment: a sealed segment ends on a line boundary, and its successor starts here. + if (offset_ == buffer_.size() && open(index_)) continue; + return std::nullopt; + } +} + +std::optional snapshot_prefix(const std::string& journal, std::uint64_t published, std::size_t limit) { + JournalReader reader(journal); + if (!reader.seek(0)) throw Error(22, "the snapshot prefix has expired"); + std::string body; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (reader.index() < published) { + auto line = reader.next(); + if (!line) { + if (std::chrono::steady_clock::now() >= deadline) throw Error(22, "a snapshot segment expired while it was read"); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + if (line->size() + 1 > limit - body.size()) return std::nullopt; + body += *line; + body += '\n'; + } + return body; +} + +std::string listen_address(const std::string& listen, bool allow_remote) { + const auto colon = listen.rfind(':'); + if (colon == std::string::npos || colon + 1 == listen.size()) throw Error(23, "--listen takes HOST:PORT"); + auto host = listen.substr(0, colon); + const auto port = listen.substr(colon + 1); + unsigned int value = 0; + if (port.find_first_not_of("0123456789") != std::string::npos || + std::from_chars(port.data(), port.data() + port.size(), value).ec != std::errc{} || value > 65535) + throw Error(23, "--listen port must be 0..65535"); + if (host == "localhost") host = "127.0.0.1"; + in_addr parsed{}; + if (::inet_pton(AF_INET, host.c_str(), &parsed) != 1) throw Error(23, "--listen host must be an IPv4 address or localhost"); + const bool loopback = (ntohl(parsed.s_addr) >> 24) == 127; + if (!loopback && !allow_remote) + throw Error(23, "--listen on a non-loopback address needs --allow-remote-listen; serve has no TLS or authentication"); + return host + ":" + std::to_string(value); +} + +void serve_feed(Config config) { + const auto address = listen_address(config.listen, config.allow_remote_listen); + // Instrument metadata fixes the quantity units before the cursor binds them. + const auto venue = make_venue(config); + config.qty_multiplier = venue->qty_multiplier(); + config.serve = true; + State state(config); + Hub hub; + hub.config = config; + hub.epoch = state.durable().epoch; + hub.journal = config.state_dir + "/journal"; + hub.published = state.durable().message_index; + hub.source = Json::object({{"venue", Json::string(config.venue)}, {"market", Json::string(config.market)}, + {"symbol", Json::string(config.symbol)}, {"mode", Json::string(config.mode)}, {"rest_origin", Json::string(config.rest_url)}, + {"ws_origin", Json::string(config.ws_url)}, {"start", Json::number(std::to_string(state.durable().start))}, + {"qty_multiplier", Json::string(config.qty_multiplier)}}).dump(); + hub.commit(state); + Server server(hub, address); + try { + const auto session = make_session(state, *venue, [&](const std::string& line) { hub.publish(line); }); + stream(config, *venue, *session, [&] { hub.commit(state); }); + } catch (const Stopped&) { + hub.stop(); + log("info", "stopped", state.status()); + return; + } catch (...) { + hub.stop(); + log("error", "verified_cursor_retained", state.status()); + throw; + } +} +} diff --git a/native/feed/src/serve.hpp b/native/feed/src/serve.hpp new file mode 100644 index 0000000..c1e429f --- /dev/null +++ b/native/feed/src/serve.hpp @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "core.hpp" +#include + +namespace pineforge::feed { +// Reads committed journal lines by message index straight from the segment files, independently of the +// producer's State: a client catching up, and the snapshot, read here while the producer appends. +class JournalReader { + std::string directory_; + int descriptor_ = -1; + std::string buffer_; + std::size_t offset_ = 0; + std::uint64_t index_ = 0; + bool open(std::uint64_t base); +public: + explicit JournalReader(std::string directory) : directory_(std::move(directory)) {} + ~JournalReader(); + JournalReader(const JournalReader&) = delete; + JournalReader& operator=(const JournalReader&) = delete; + // Positions before message `index`; false when no retained segment holds it (expired). + bool seek(std::uint64_t index); + // The next complete line, or nullopt while it is not on disk yet. + std::optional next(); + std::uint64_t index() const { return index_; } +}; + +// The committed prefix [0, published) as JSONL, or nullopt when it exceeds `limit` bytes; throws 22 when index 0 +// has expired or a segment disappears while it is read. +std::optional snapshot_prefix(const std::string& journal, std::uint64_t published, std::size_t limit); +// The listening address: loopback unless the operator explicitly allows another interface. +std::string listen_address(const std::string& listen, bool allow_remote); +void serve_feed(Config config); +} diff --git a/native/feed/src/session.cpp b/native/feed/src/session.cpp index 415b76e..c74663c 100644 --- a/native/feed/src/session.cpp +++ b/native/feed/src/session.cpp @@ -89,7 +89,8 @@ void Session::ingest(const VenueEvent& event) { } void Session::replay() { if (!replayed_) { - if (state_.config().resume) state_.visit(state_.config().output_from, output_); + // serve publishes only new messages; its clients read the retained journal by index themselves. + if (state_.config().resume && !state_.config().serve) state_.visit(state_.config().output_from, output_); replayed_ = true; } log("info", "overlap_verified", state_.status()); @@ -97,7 +98,7 @@ void Session::replay() { FeedSession::FeedSession(State& state, Venue& venue, std::function output) : Session(state, venue, std::move(output)) { - if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { + if (state_.config().mode == "ticks") state_.visit(state_.index_for_minute(state_.cursor().cut), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() == "tick") { const auto trade = normalized_trade(event); @@ -125,7 +126,7 @@ void FeedSession::connected() { [&](const auto& rows) { return same_bars(rows, proofs); }); if (fetched.size() != proofs.size()) throw Error(20, "reconnect closed-bar overlap is unavailable"); std::map verified; - if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { + if (state_.config().mode == "ticks") state_.visit(state_.index_for_minute(proofs.front().ts), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() != "tick") return; const auto trade = normalized_trade(event); @@ -158,6 +159,10 @@ void FeedSession::put(const Trade& trade) { trade.validate(); if (trade.ts < state_.cursor().start) return; if (state_.cursor().seq && trade.id <= state_.cursor().seq) { + if (state_.expired_seq(trade.id)) { + log("warn", "expired_duplicate_unverified", Json::object({{"seq", Json::number(std::to_string(trade.id))}})); + return; + } const auto previous = state_.trade(trade.id); if (!previous || !(*previous == trade)) throw Error(21, "duplicate raw trade conflicts with the immutable prefix"); return; @@ -247,6 +252,11 @@ void FeedSession::closed(const Kline& kline) { const auto minute = kline.bar.ts; watermark_ = std::max(watermark_, minute + 60000); if (minute < state_.cursor().start) return; + if (minute < state_.cursor().cut && state_.expired_minute(minute)) { + // Retention dropped this minute: an old duplicate can no longer be compared with what was emitted. + log("warn", "expired_duplicate_unverified", Json::object({{"minute", Json::number(std::to_string(minute))}})); + return; + } if (minute < state_.cursor().cut) { const auto old = state_.bar(minute); if (old) { @@ -254,7 +264,7 @@ void FeedSession::closed(const Kline& kline) { } if (state_.config().mode == "ticks") { Aggregate reconstructed; - state_.visit(0, [&](const std::string& line) { + state_.visit(state_.index_for_minute(minute), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() == "tick") { const auto trade = normalized_trade(event); @@ -313,7 +323,7 @@ void reconcile_fenced(const Aggregate& prints, const Kline& candle) { FenceSession::FenceSession(State& state, Venue& venue, std::function output) : Session(state, venue, std::move(output)) { - state_.visit(0, [&](const std::string& line) { + state_.visit(state_.index_for_minute(state_.cursor().cut), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() == "tick") { const auto trade = normalized_trade(event); @@ -343,6 +353,10 @@ void FenceSession::put(const Trade& trade, bool healed) { throw Error(23, "venue matched time regressed: a print before the proven predecessor is at or after the start minute"); } if (state_.cursor().seq && trade.id <= state_.cursor().seq) { + if (state_.expired_seq(trade.id)) { + log("warn", "expired_duplicate_unverified", Json::object({{"seq", Json::number(std::to_string(trade.id))}})); + return; + } const auto previous = state_.trade(trade.id); if (!previous || !(*previous == trade)) throw Error(21, "duplicate print conflicts with the immutable prefix"); return; @@ -453,6 +467,10 @@ void FenceSession::candle(const Kline& kline) { const auto minute = kline.bar.ts; watermark_ = std::max(watermark_, minute + 60000); if (minute < state_.cursor().start) return; + if (minute < state_.cursor().cut && state_.expired_minute(minute)) { + log("warn", "expired_duplicate_unverified", Json::object({{"minute", Json::number(std::to_string(minute))}})); + return; + } if (minute < state_.cursor().cut) { for (const auto& proof : state_.cursor().proofs) if (proof.ts == minute) { @@ -462,7 +480,7 @@ void FenceSession::candle(const Kline& kline) { // Older than the retained proofs: rebuild that minute from the journal, where the candle is a sum. if (!venue_.candle_is_print_sum()) return; Aggregate rebuilt; - state_.visit(0, [&](const std::string& line) { + state_.visit(state_.index_for_minute(minute), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() != "tick") return; const auto trade = normalized_trade(event); @@ -537,26 +555,35 @@ std::unique_ptr make_session(State& state, Venue& venue, std::function< return std::make_unique(state, venue, std::move(output)); } +void stream(const Config& config, Venue& venue, Session& session, const std::function& committed) { + WebSocketPump pump(config, venue.connection()); + for (;;) { + auto message = pump.take(); + try { + // Group commit: verify every source message already queued (bounded), then persist once. + for (std::size_t taken = 1;; ++taken) { + if (message.connected) session.connected(); + else for (const auto& event : venue.decode(message.text)) session.stage(event); + if (taken >= 1024 || !pump.try_take(message)) break; + } + } catch (const Error&) { + session.salvage(); + if (committed) committed(); + throw; + } + session.publish(); + if (committed) committed(); + } +} + void run_feed(Config config) { // Instrument metadata fixes the quantity units before the cursor binds them. const auto venue = make_venue(config); config.qty_multiplier = venue->qty_multiplier(); State state(config); try { - WebSocketPump pump(config, venue->connection()); const auto session = make_session(state, *venue, &output_line); - for (;;) { - auto message = pump.take(); - try { - // Group commit: verify every source message already queued (bounded), then persist once. - for (std::size_t taken = 1;; ++taken) { - if (message.connected) session->connected(); - else for (const auto& event : venue->decode(message.text)) session->stage(event); - if (taken >= 1024 || !pump.try_take(message)) break; - } - } catch (const Error&) { session->salvage(); throw; } - session->publish(); - } + stream(config, *venue, *session, {}); } catch (const Stopped&) { log("info", "stopped", state.status()); } catch (...) { log("error", "verified_cursor_retained", state.status()); throw; } } diff --git a/native/feed/src/session.hpp b/native/feed/src/session.hpp index 3833ae0..e110de0 100644 --- a/native/feed/src/session.hpp +++ b/native/feed/src/session.hpp @@ -85,6 +85,8 @@ std::vector settled_prints(Venue& venue, const Config& config, const std: void reconcile_fenced(const Aggregate& prints, const Kline& candle); std::unique_ptr make_session(State& state, Venue& venue, std::function output); +// The source loop shared by run and serve: group-commit every queued source message, publish, then report. +void stream(const Config& config, Venue& venue, Session& session, const std::function& committed); void run_feed(Config config); void warmup(const Config& config, const std::string& output); } diff --git a/native/feed/src/state.cpp b/native/feed/src/state.cpp index 571d1b4..e5303d6 100644 --- a/native/feed/src/state.cpp +++ b/native/feed/src/state.cpp @@ -1,11 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 #include "state.hpp" #include +#include #include +#include #include #include #include #include +#include #include #include #include @@ -17,6 +20,11 @@ void Descriptor::reset(int value) { if (value_ >= 0) ::close(value_); value_ = value; } +std::string segment_name(std::uint64_t base) { + char name[32]; + std::snprintf(name, sizeof(name), "%020llu", static_cast(base)); + return name; +} namespace { Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } Json signed_number(std::int64_t value) { return Json::number(std::to_string(value)); } @@ -46,16 +54,69 @@ void advance(Cursor& cursor, const Json& event, const std::string& mode) { cursor.cut = cut; } else throw Error(23, "journal contains an unsupported normalized message"); } +void sync_directory(const std::string& path) { + const int directory = ::open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (directory < 0) throw Error(22, "cannot open journal directory for sync"); + const auto status = ::fsync(directory); + ::close(directory); + if (status != 0) throw Error(22, "cannot sync journal directory"); +} +std::string small_file(const std::string& path) { + if (!std::filesystem::exists(path) || std::filesystem::file_size(path) > 65536) throw Error(22, "missing or oversized durable state file"); + std::ifstream input(path, std::ios::binary); + return std::string((std::istreambuf_iterator(input)), {}); +} +// Every complete line of the first `limit` bytes of `path`; a line cut by `limit` or the end of the file +// is a broken message partition. +void read_lines(const std::string& path, std::uint64_t limit, std::chrono::steady_clock::time_point deadline, + const std::function& visitor) { + std::ifstream input(path, std::ios::binary); + if (!input) throw Error(22, "cannot read retained normalized journal segment"); + std::uint64_t bytes = 0; + std::string line; + while (bytes < limit) { + if (stopping) throw Stopped{}; + if (std::chrono::steady_clock::now() >= deadline) throw Error(22, "retained prefix replay time budget exhausted"); + if (!std::getline(input, line) || input.eof() || line.size() > 4095 || bytes + line.size() + 1 > limit) + throw Error(21, "truncated or oversized verified message"); + bytes += line.size() + 1; + visitor(line); + } +} +Json checkpoint_document(const Cursor& cursor) { + return Json::object({ + {"schema", Json::string("pineforge-feed-checkpoint/v1")}, {"epoch", Json::string(cursor.epoch)}, + {"message_index", number(cursor.message_index)}, {"log_bytes", number(cursor.log_bytes)}, + {"prefix_hash", Json::string(cursor.prefix_hash)}, {"overlap_hash", Json::string(cursor.overlap_hash)}, + {"start", signed_number(cursor.start)}, {"verified_cut", signed_number(cursor.cut)}, + {"emitted_seq", number(cursor.seq)}, {"last_tick_ts", signed_number(cursor.last_tick_ts)}, + {"last_bar", signed_number(cursor.last_bar)}, + {"predecessor", cursor.predecessor ? parse_json(cursor.predecessor->wire()) : Json{}} + }); +} +// The verified fields a segment boundary must reproduce exactly. +bool same_state(const Cursor& left, const Cursor& right) { + return left.message_index == right.message_index && left.log_bytes == right.log_bytes && + left.prefix_hash == right.prefix_hash && left.overlap_hash == right.overlap_hash && left.cut == right.cut && + left.seq == right.seq && left.last_tick_ts == right.last_tick_ts && left.last_bar == right.last_bar; +} +bool parse_base(const std::string& digits, std::uint64_t& base) { + if (digits.size() != 20 || digits.find_first_not_of("0123456789") != std::string::npos) return false; + base = std::stoull(digits); + return true; +} } -std::string State::log_path() const { return config_.state_dir + "/events.jsonl"; } +std::string State::journal_dir() const { return config_.state_dir + "/journal"; } +std::string State::segment_path(std::uint64_t base) const { return journal_dir() + "/" + segment_name(base) + ".jsonl"; } +std::string State::checkpoint_path(std::uint64_t base) const { return journal_dir() + "/" + segment_name(base) + ".checkpoint.json"; } Json State::serialize(const Cursor& cursor) const { Json proofs; proofs.kind = Json::Kind::Array; for (const auto& bar : cursor.proofs) proofs.items.push_back(parse_json(bar.wire())); return Json::object({ - {"schema", Json::string("pineforge-feed-cursor/v1")}, {"epoch", Json::string(cursor.epoch)}, + {"schema", Json::string("pineforge-feed-cursor/v2")}, {"epoch", Json::string(cursor.epoch)}, {"venue", Json::string(config_.venue)}, {"market", Json::string(config_.market)}, {"symbol", Json::string(config_.symbol)}, {"mode", Json::string(config_.mode)}, {"rest_origin", Json::string(config_.rest_url)}, {"ws_origin", Json::string(config_.ws_url)}, @@ -65,8 +126,8 @@ Json State::serialize(const Cursor& cursor) const { {"venue_id", tick_mode(config_.mode) ? number(cursor.seq) : signed_number(cursor.last_bar)}, {"emitted_seq", number(cursor.seq)}, {"last_tick_ts", signed_number(cursor.last_tick_ts)}, {"last_bar", signed_number(cursor.last_bar)}, {"message_index", number(cursor.message_index)}, - {"log_bytes", number(cursor.log_bytes)}, {"prefix_hash", Json::string(cursor.prefix_hash)}, - {"overlap_hash", Json::string(cursor.overlap_hash)}, + {"log_bytes", number(cursor.log_bytes)}, {"segment", number(cursor.segment)}, + {"prefix_hash", Json::string(cursor.prefix_hash)}, {"overlap_hash", Json::string(cursor.overlap_hash)}, {"predecessor", cursor.predecessor ? parse_json(cursor.predecessor->wire()) : Json{}}, {"closed_proofs", std::move(proofs)}, {"partition", Json::string("one-event-per-message")} }); @@ -78,6 +139,38 @@ void State::persist(const Cursor& cursor) const { {"sha256", Json::string(sha256(document.dump()))}}).dump() + '\n'); } +void State::write_checkpoint(const Cursor& cursor) const { + const auto document = checkpoint_document(cursor); + atomic_file(checkpoint_path(cursor.message_index), Json::object({{"checkpoint", document}, + {"sha256", Json::string(sha256(document.dump()))}}).dump() + '\n'); +} + +Cursor State::read_checkpoint(std::uint64_t base) const { + try { + const auto envelope = parse_json(small_file(checkpoint_path(base))); + const auto& document = envelope.at("checkpoint"); + if (envelope.at("sha256").text() != sha256(document.dump())) throw Error(21, "journal checkpoint digest mismatch"); + if (document.at("schema").text() != "pineforge-feed-checkpoint/v1") throw Error(23, "unsupported journal checkpoint schema"); + Cursor checkpoint; + checkpoint.epoch = document.at("epoch").text(); + checkpoint.message_index = document.at("message_index").integer(); + checkpoint.log_bytes = document.at("log_bytes").integer(); + checkpoint.prefix_hash = document.at("prefix_hash").text(); + checkpoint.overlap_hash = document.at("overlap_hash").text(); + checkpoint.start = document.at("start").integer(); + checkpoint.cut = document.at("verified_cut").integer(); + checkpoint.seq = document.at("emitted_seq").integer(); + checkpoint.last_tick_ts = document.at("last_tick_ts").integer(); + checkpoint.last_bar = document.at("last_bar").integer(); + if (document.at("predecessor").kind != Json::Kind::Null) checkpoint.predecessor = normalized_trade(document.at("predecessor")); + if (checkpoint.message_index != base || !digest_token(checkpoint.prefix_hash) || !digest_token(checkpoint.overlap_hash)) + throw Error(21, "journal checkpoint does not describe its segment"); + checkpoint.segment = base; + return checkpoint; + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(21, "invalid journal checkpoint"); } +} + State::State(const Config& config) : config_(config) { std::error_code failure; std::filesystem::create_directories(config_.state_dir, failure); @@ -87,39 +180,55 @@ State::State(const Config& config) : config_(config) { throw Error(22, "state directory has another producer or cannot be locked"); if (config_.resume) { recover(); - journal_.reset(::open(log_path().c_str(), O_WRONLY | O_APPEND | O_CLOEXEC)); + journal_.reset(::open(segment_path(cursor_.segment).c_str(), O_WRONLY | O_APPEND | O_CLOEXEC)); if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot sync rolled-back journal tail"); - } else { - if (config_.start < 0 || config_.start % 60000) throw Error(23, "new streams require a minute-aligned --start"); - // An empty journal without a cursor is an initialization that crashed before its first cursor - // write: nothing was committed or published, so a fresh start may take the directory over. - if (std::filesystem::exists(config_.state_dir + "/cursor.json") || - (std::filesystem::exists(log_path()) && std::filesystem::file_size(log_path()) != 0)) - throw Error(23, "state already exists; use --resume or an explicitly new state directory"); - cursor_.epoch = random_epoch(); - cursor_.start = cursor_.cut = config_.start; - cursor_.prefix_hash = sha256(""); - cursor_.overlap_hash = sha256(""); - journal_.reset(::open(log_path().c_str(), O_WRONLY | O_APPEND | O_CREAT | O_TRUNC | O_CLOEXEC, 0600)); - if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot initialize durable journal"); - persist(cursor_); - durable_ = cursor_; - } + } else initialize(); if (journal_.get() < 0) throw Error(22, "cannot open durable journal"); if (config_.output_from > cursor_.message_index) throw Error(22, "output cursor exceeds the retained verified prefix"); + if (config_.resume && !config_.serve && config_.output_from < first_retained()) + throw Error(22, "output cursor " + std::to_string(config_.output_from) + " has expired: messages before index " + + std::to_string(first_retained()) + " left the retained journal (--replay-bytes, --replay-age-seconds). A consumer " + "behind that index cannot resume from this stream: rebuild it from a warmup that ends inside the retained window, " + "or keep a longer retention"); +} + +void State::initialize() { + if (config_.start < 0 || config_.start % 60000) throw Error(23, "new streams require a minute-aligned --start"); + if (std::filesystem::exists(config_.state_dir + "/events.jsonl")) + throw Error(23, "this state directory holds an unsegmented journal from an earlier release; start a new state directory"); + if (std::filesystem::exists(config_.state_dir + "/cursor.json")) + throw Error(23, "state already exists; use --resume or an explicitly new state directory"); + std::error_code failure; + std::filesystem::create_directories(journal_dir(), failure); + if (failure) throw Error(22, "cannot create journal directory"); + // A journal without a cursor is an initialization that crashed before its first cursor write: nothing + // was committed or published, so a fresh start may take the directory over if no segment holds a byte. + std::vector leftovers; + for (const auto& entry : std::filesystem::directory_iterator(journal_dir())) { + if (entry.path().extension() == ".jsonl" && entry.file_size() != 0) + throw Error(23, "state already exists; use --resume or an explicitly new state directory"); + leftovers.push_back(entry.path()); + } + for (const auto& path : leftovers) std::filesystem::remove(path); + cursor_.epoch = random_epoch(); + cursor_.start = cursor_.cut = config_.start; + cursor_.prefix_hash = sha256(""); + cursor_.overlap_hash = sha256(""); + write_checkpoint(cursor_); + journal_.reset(::open(segment_path(0).c_str(), O_WRONLY | O_APPEND | O_CREAT | O_EXCL | O_CLOEXEC, 0600)); + if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot initialize durable journal"); + sync_directory(journal_dir()); + persist(cursor_); + durable_ = cursor_; + segments_ = {{0, 0, 0, cursor_.start}}; } void State::recover() { try { - const auto path = config_.state_dir + "/cursor.json"; - if (!std::filesystem::exists(path) || std::filesystem::file_size(path) > 65536) - throw Error(22, "missing or oversized durable cursor"); - std::ifstream input(path); - const std::string bytes((std::istreambuf_iterator(input)), {}); - const auto envelope = parse_json(bytes); + const auto envelope = parse_json(small_file(config_.state_dir + "/cursor.json")); const auto& document = envelope.at("cursor"); if (envelope.at("sha256").text() != sha256(document.dump())) throw Error(21, "cursor digest mismatch"); - if (document.at("schema").text() != "pineforge-feed-cursor/v1" || document.at("partition").text() != "one-event-per-message") + if (document.at("schema").text() != "pineforge-feed-cursor/v2" || document.at("partition").text() != "one-event-per-message") throw Error(23, "unsupported cursor schema or message partition"); for (const auto& item : {std::make_pair("venue", config_.venue), {"market", config_.market}, {"symbol", config_.symbol}, {"mode", config_.mode}, {"rest_origin", config_.rest_url}, {"ws_origin", config_.ws_url}}) @@ -128,9 +237,7 @@ void State::recover() { if (units.at("price").text() != "quote/base" || units.at("qty").text() != "base" || units.at("ts").text() != "unix-ms") throw Error(21, "resume units changed"); // Contract quantities were converted with this instrument multiplier; a change would mix units. - const auto* multiplier = document.find("qty_multiplier"); - if ((multiplier ? multiplier->text() : std::string("1")) != config_.qty_multiplier) - throw Error(21, "resume contract quantity multiplier changed"); + if (document.at("qty_multiplier").text() != config_.qty_multiplier) throw Error(21, "resume contract quantity multiplier changed"); cursor_.epoch = document.at("epoch").text(); cursor_.prefix_hash = document.at("prefix_hash").text(); cursor_.overlap_hash = document.at("overlap_hash").text(); @@ -146,38 +253,78 @@ void State::recover() { throw Error(21, "venue ID does not match the emitted cursor"); cursor_.message_index = document.at("message_index").integer(); cursor_.log_bytes = document.at("log_bytes").integer(); + cursor_.segment = document.at("segment").integer(); if (cursor_.start < 0 || cursor_.start % 60000 || cursor_.cut < cursor_.start || cursor_.cut % 60000 || - cursor_.log_bytes > config_.max_log_bytes || (config_.start >= 0 && config_.start != cursor_.start)) + cursor_.segment > cursor_.message_index || (config_.start >= 0 && config_.start != cursor_.start)) throw Error(21, "invalid or changed resume cut"); if (document.at("predecessor").kind != Json::Kind::Null) cursor_.predecessor = normalized_trade(document.at("predecessor")); const auto& proofs = document.at("closed_proofs"); if (proofs.kind != Json::Kind::Array || proofs.items.size() > 32) throw Error(21, "invalid closed-minute proof window"); for (const auto& proof : proofs.items) cursor_.proofs.push_back(normalized_bar(proof)); - if (!std::filesystem::exists(log_path()) || std::filesystem::file_size(log_path()) < cursor_.log_bytes) - throw Error(21, "durable journal is shorter than its verified cursor"); - durable_ = cursor_; - Cursor reconstructed; - reconstructed.start = reconstructed.cut = cursor_.start; - reconstructed.predecessor = cursor_.predecessor; - reconstructed.prefix_hash = reconstructed.overlap_hash = sha256(""); - visit(0, [&](const std::string& line) { - const auto event = parse_json(line); - advance(reconstructed, event, config_.mode); - reconstructed.prefix_hash = sha256(reconstructed.prefix_hash + line + '\n'); - reconstructed.overlap_hash = sha256(line + '\n'); - reconstructed.log_bytes += line.size() + 1; - ++reconstructed.message_index; - remember(event); - }); - if (reconstructed.prefix_hash != cursor_.prefix_hash || reconstructed.overlap_hash != cursor_.overlap_hash || - reconstructed.log_bytes != cursor_.log_bytes || reconstructed.message_index != cursor_.message_index || - reconstructed.seq != cursor_.seq || reconstructed.cut != cursor_.cut || reconstructed.last_bar != cursor_.last_bar || - reconstructed.last_tick_ts != cursor_.last_tick_ts) - throw Error(21, "durable journal prefix does not match its cursor"); - if (::truncate(log_path().c_str(), static_cast(cursor_.log_bytes)) != 0) - throw Error(22, "cannot roll back an uncommitted journal tail"); + // Segments after the cursor's open one are an interrupted rotation: never committed, so removed. A + // checkpoint without its segment is an interrupted expiry. A segment without its checkpoint never occurs. + std::map> files; + for (const auto& entry : std::filesystem::directory_iterator(journal_dir())) { + const auto name = entry.path().filename().string(); + std::uint64_t base = 0; + if (name.size() > 4 && name.compare(name.size() - 4, 4, ".tmp") == 0) { std::filesystem::remove(entry.path()); continue; } + if (name.size() == 26 && name.compare(20, 6, ".jsonl") == 0 && parse_base(name.substr(0, 20), base)) files[base].first = true; + else if (name.size() == 36 && name.compare(20, 16, ".checkpoint.json") == 0 && parse_base(name.substr(0, 20), base)) files[base].second = true; + else throw Error(21, "unexpected file in the journal directory: " + name); + } + std::vector retained; + bool removed = false; + for (const auto& [base, kinds] : files) { + if (base > cursor_.segment) { + std::filesystem::remove(segment_path(base)); + std::filesystem::remove(checkpoint_path(base)); + log("warn", "uncommitted_rotation_removed", Json::object({{"segment", number(base)}})); + removed = true; + } else if (!kinds.first) { + if (base == cursor_.segment) throw Error(21, "the open journal segment is missing"); + std::filesystem::remove(checkpoint_path(base)); + removed = true; + } else if (!kinds.second) throw Error(21, "a journal segment has no checkpoint"); + else retained.push_back(base); + } + if (removed) sync_directory(journal_dir()); + if (retained.empty() || retained.back() != cursor_.segment) throw Error(21, "the open journal segment is missing"); + Cursor reconstructed = read_checkpoint(retained.front()); + if (reconstructed.epoch != cursor_.epoch || reconstructed.start != cursor_.start) + throw Error(21, "journal checkpoint belongs to another stream"); + // The start predecessor is anchored once, before the first message, and never changes. + if (!reconstructed.predecessor) reconstructed.predecessor = cursor_.predecessor; + else if (!cursor_.predecessor || !(*reconstructed.predecessor == *cursor_.predecessor)) + throw Error(21, "journal checkpoint predecessor does not match the cursor"); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(config_.max_replay_seconds); + for (std::size_t index = 0; index < retained.size(); ++index) { + const auto base = retained[index]; + if (index && !same_state(reconstructed, read_checkpoint(base))) + throw Error(21, "durable journal segment boundary does not match its checkpoint"); + segments_.push_back({base, reconstructed.log_bytes, reconstructed.seq, reconstructed.cut}); + const auto path = segment_path(base); + const auto size = std::filesystem::file_size(path); + const bool open = index + 1 == retained.size(); + if (open && cursor_.log_bytes < reconstructed.log_bytes) throw Error(21, "cursor precedes its open journal segment"); + const auto limit = open ? cursor_.log_bytes - reconstructed.log_bytes : size; + if (size < limit) throw Error(21, "durable journal is shorter than its verified cursor"); + read_lines(path, limit, deadline, [&](const std::string& line) { + const auto event = parse_json(line); + advance(reconstructed, event, config_.mode); + reconstructed.prefix_hash = sha256(reconstructed.prefix_hash + line + '\n'); + reconstructed.overlap_hash = sha256(line + '\n'); + reconstructed.log_bytes += line.size() + 1; + ++reconstructed.message_index; + remember(event); + }); + if (open && ::truncate(path.c_str(), static_cast(limit)) != 0) + throw Error(22, "cannot roll back an uncommitted journal tail"); + } + if (!same_state(reconstructed, cursor_)) throw Error(21, "durable journal prefix does not match its cursor"); config_.start = cursor_.start; + durable_ = cursor_; } catch (const Error&) { throw; } + catch (const Stopped&) { throw; } catch (const std::exception&) { throw Error(21, "invalid durable cursor or journal"); } } @@ -212,9 +359,7 @@ void State::stage(const std::string& line, const std::optional& proof) { next.proofs.push_back(*proof); if (next.proofs.size() > 32) next.proofs.pop_front(); } - if (next.message_index == std::numeric_limits::max() || - line.size() + 1 > config_.max_log_bytes - std::min(cursor_.log_bytes, config_.max_log_bytes)) - throw Error(22, "retained replay budget exhausted"); + if (next.message_index == std::numeric_limits::max()) throw Error(22, "message index exhausted"); next.log_bytes += line.size() + 1; ++next.message_index; next.prefix_hash = sha256(cursor_.prefix_hash + line + '\n'); @@ -232,31 +377,102 @@ std::vector State::flush() { if (unusable_) throw Error(22, "durable journal is unusable after a failed commit"); unusable_ = true; write_all(journal_.get(), staged_bytes_); + crash_point("commit-append"); if (::fsync(journal_.get()) != 0) throw Error(22, "cannot sync normalized messages"); persist(cursor_); + crash_point("commit-cursor"); unusable_ = false; durable_ = cursor_; staged_bytes_.clear(); - return std::exchange(staged_, {}); + auto lines = std::exchange(staged_, {}); + if (durable_.log_bytes - segments_.back().bytes >= config_.segment_bytes) rotate(); + retain(); + return lines; +} + +// Seal the open segment: its successor's checkpoint first, then the empty successor, then the cursor that +// names it. A crash before the cursor leaves files recovery removes; the old segment stays open. +void State::rotate() { + const auto base = durable_.message_index; + if (!staged_.empty() || base == segments_.back().base) return; + write_checkpoint(durable_); + crash_point("rotate-checkpoint"); + Descriptor next(::open(segment_path(base).c_str(), O_WRONLY | O_APPEND | O_CREAT | O_EXCL | O_CLOEXEC, 0600)); + if (next.get() < 0 || ::fsync(next.get()) != 0) throw Error(22, "cannot create the next journal segment"); + sync_directory(journal_dir()); + crash_point("rotate-segment"); + auto rotated = durable_; + rotated.segment = base; + persist(rotated); + crash_point("rotate-cursor"); + durable_.segment = cursor_.segment = base; + journal_.reset(next.release()); + segments_.push_back({base, durable_.log_bytes, durable_.seq, durable_.cut}); + log("info", "journal_segment_sealed", Json::object({{"next_segment", number(base)}, {"segments", number(segments_.size())}})); +} + +// Expire the oldest sealed segments over the byte or venue-time budget. The open segment and every segment +// that can hold a message of the protected window (the open minute and the closed-minute proofs) stay. +void State::retain() { + const auto protected_start = durable_.proofs.empty() ? durable_.cut : std::min(durable_.proofs.front().ts, durable_.cut); + while (segments_.size() > 1) { + const auto oldest = segments_[0]; + const auto& next = segments_[1]; + if (next.cut >= protected_start) break; + const bool bytes = durable_.log_bytes - oldest.bytes > config_.replay_bytes; + const bool age = config_.replay_age_ms && durable_.cut - next.cut > config_.replay_age_ms; + if (!bytes && !age) break; + if (::unlink(segment_path(oldest.base).c_str()) != 0) throw Error(22, "cannot expire a journal segment"); + crash_point("retain-segment"); + if (::unlink(checkpoint_path(oldest.base).c_str()) != 0 && errno != ENOENT) throw Error(22, "cannot expire a journal checkpoint"); + sync_directory(journal_dir()); + segments_.erase(segments_.begin()); + log("info", "journal_segment_expired", Json::object({{"segment", number(oldest.base)}, + {"first_retained", number(segments_.front().base)}, {"reason", Json::string(bytes ? "bytes" : "age")}})); + } + const bool over = durable_.log_bytes - segments_.front().bytes > config_.replay_bytes; + if (over && !over_budget_) + log("warn", "replay_budget_held_by_protected_window", Json::object({{"retained_bytes", + number(durable_.log_bytes - segments_.front().bytes)}, {"replay_bytes", number(config_.replay_bytes)}})); + over_budget_ = over; +} + +std::size_t State::segment_for(std::uint64_t index) const { + const auto found = std::upper_bound(segments_.begin(), segments_.end(), index, + [](std::uint64_t value, const Segment& segment) { return value < segment.base; }); + return found == segments_.begin() ? 0 : static_cast(found - segments_.begin() - 1); +} + +bool State::expired_minute(std::int64_t minute) const { + const auto cut = segments_.front().cut; + return first_retained() && (minute < cut || (tick_mode(config_.mode) && minute == cut)); +} + +std::uint64_t State::index_for_minute(std::int64_t minute) const { + std::size_t chosen = 0; + for (std::size_t index = 0; index < segments_.size(); ++index) + if (segments_[index].cut < minute) chosen = index; + return segments_[chosen].base; } void State::visit(std::uint64_t from, const std::function& visitor) const { if (from > cursor_.message_index) throw Error(22, "replay cursor is unavailable"); - std::ifstream input(log_path(), std::ios::binary); - if (!input) throw Error(22, "cannot read retained normalized log"); - std::uint64_t index = 0, bytes = 0; + if (from < first_retained()) throw Error(22, "replay cursor has expired from the retained journal"); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(config_.max_replay_seconds); - std::string line; - while (bytes < durable_.log_bytes) { - if (stopping) throw Stopped{}; - if (std::chrono::steady_clock::now() >= deadline) throw Error(22, "retained prefix replay time budget exhausted"); - if (!std::getline(input, line) || line.size() > 4095 || bytes + line.size() + 1 > durable_.log_bytes) - throw Error(21, "truncated or oversized verified message"); - bytes += line.size() + 1; - if (index >= from) visitor(line); - ++index; + std::uint64_t index = durable_.message_index; + for (auto position = segment_for(from); position < segments_.size() && from < durable_.message_index; ++position) { + const auto& segment = segments_[position]; + const bool last = position + 1 == segments_.size(); + const auto end_bytes = last ? durable_.log_bytes : segments_[position + 1].bytes; + const auto end_index = last ? durable_.message_index : segments_[position + 1].base; + index = segment.base; + read_lines(segment_path(segment.base), end_bytes - segment.bytes, deadline, [&](const std::string& line) { + if (index >= from) visitor(line); + ++index; + }); + if (index != end_index) throw Error(21, "verified message partition changed"); } - if (bytes != durable_.log_bytes || index != durable_.message_index) throw Error(21, "verified message partition changed"); + index = durable_.message_index; for (const auto& staged : staged_) { if (index >= from) visitor(staged); ++index; @@ -265,8 +481,12 @@ void State::visit(std::uint64_t from, const std::function State::trade(std::uint64_t id) const { for (const auto& trade : recent_trades_) if (trade.id == id) return trade; + std::size_t chosen = 0; + for (std::size_t index = 0; index < segments_.size(); ++index) + if (segments_[index].seq < id) chosen = index; std::optional found; - visit(0, [&](const std::string& line) { + visit(segments_[chosen].base, [&](const std::string& line) { + if (found) return; const auto event = parse_json(line); if (event.at("type").text() == "tick" && event.at("seq").integer() == id) found = normalized_trade(event); }); @@ -276,7 +496,8 @@ std::optional State::bar(std::int64_t ts) const { for (const auto& bar : recent_bars_) if (bar.ts == ts) return bar; for (const auto& bar : cursor_.proofs) if (bar.ts == ts) return bar; std::optional found; - visit(0, [&](const std::string& line) { + visit(index_for_minute(ts), [&](const std::string& line) { + if (found) return; const auto event = parse_json(line); if (event.at("type").text() == "bar" && event.at("bar").at("ts_open").integer() == ts) found = normalized_bar(event); }); @@ -284,6 +505,7 @@ std::optional State::bar(std::int64_t ts) const { } Json State::status() const { return Json::object({{"epoch", Json::string(durable_.epoch)}, {"message_index", number(durable_.message_index)}, + {"first_retained", number(first_retained())}, {"segment", number(durable_.segment)}, {"emitted_seq", number(durable_.seq)}, {"verified_cut", signed_number(durable_.cut)}, {"prefix_hash", Json::string(durable_.prefix_hash)}, {"cursor", Json::string(config_.state_dir + "/cursor.json")}}); } diff --git a/native/feed/src/state.hpp b/native/feed/src/state.hpp index a7ff0e3..096227d 100644 --- a/native/feed/src/state.hpp +++ b/native/feed/src/state.hpp @@ -15,6 +15,7 @@ class Descriptor { Descriptor(const Descriptor&) = delete; Descriptor& operator=(const Descriptor&) = delete; int get() const { return value_; } + int release() { const auto value = value_; value_ = -1; return value; } void reset(int value); }; @@ -24,30 +25,60 @@ struct Cursor { std::int64_t start = 0, cut = 0, last_tick_ts = -1, last_bar = -1; std::optional predecessor; std::deque proofs; + std::uint64_t segment = 0; // base message index of the open journal segment }; +// One retained journal segment: messages [base, next segment's base), its checkpoint holding the verified +// state before its first message. `bytes` and `cut` and `seq` are that state's absolute log bytes, verified +// cut and emitted sequence. +struct Segment { + std::uint64_t base = 0, bytes = 0, seq = 0; + std::int64_t cut = 0; +}; +std::string segment_name(std::uint64_t base); + // Messages are staged (verified, cursor advanced in memory), then group-committed: one journal append and -// fsync, one cursor replacement, and only then returned for publication. +// fsync, one cursor replacement, and only then returned for publication. The journal is a sequence of +// segments under `journal/`; a full segment is sealed and a new one opened, and sealed segments expire by +// bytes and by venue time, except those holding the protected window (the open minute and the closed-minute +// proofs the reconnect overlap re-verifies). class State { Config config_; Cursor durable_, cursor_; Descriptor lock_, journal_; + std::vector segments_; std::deque recent_trades_; std::deque recent_bars_; std::vector staged_; std::string staged_bytes_; - bool unusable_ = false; - std::string log_path() const; + bool unusable_ = false, over_budget_ = false; + std::string journal_dir() const; + std::string segment_path(std::uint64_t base) const; + std::string checkpoint_path(std::uint64_t base) const; Json serialize(const Cursor& cursor) const; void persist(const Cursor& cursor) const; + void write_checkpoint(const Cursor& cursor) const; + Cursor read_checkpoint(std::uint64_t base) const; + void initialize(); void recover(); void remember(const Json& event); + void rotate(); + void retain(); + std::size_t segment_for(std::uint64_t index) const; public: explicit State(const Config& config); const Cursor& cursor() const { return cursor_; } const Cursor& durable() const { return durable_; } const Config& config() const { return config_; } std::size_t staged() const { return staged_.size(); } + const std::vector& segments() const { return segments_; } + std::uint64_t first_retained() const { return segments_.front().base; } + // Whether retention has dropped (part of) this minute or this print, so an old duplicate can no longer be + // compared with what was emitted. + bool expired_minute(std::int64_t minute) const; + bool expired_seq(std::uint64_t id) const { return first_retained() && id <= segments_.front().seq; } + // The first retained message index from which every message of `minute` (and later) is visited. + std::uint64_t index_for_minute(std::int64_t minute) const; void anchor(const Trade& predecessor); void stage(const std::string& line, const std::optional& proof = std::nullopt); std::vector flush(); diff --git a/native/feed/src/transport.cpp b/native/feed/src/transport.cpp index 7a1997d..324c056 100644 --- a/native/feed/src/transport.cpp +++ b/native/feed/src/transport.cpp @@ -126,6 +126,14 @@ std::uint64_t retry_after_seconds(const std::string& value, std::int64_t now) { return when > now ? static_cast(when - now) : 0; } +std::uint64_t quota_wait_ms(std::uint64_t used, std::uint64_t used_window, unsigned int weight, std::uint64_t limit, std::int64_t now_ms) { + if (!limit || now_ms < 0) return 0; + const auto window = static_cast(now_ms) / 60000; + const auto spent = window == used_window ? used : 0; + if (spent + weight <= limit * 9 / 10) return 0; + return (window + 1) * 60000 + 500 - static_cast(now_ms); +} + void check_runtime_curl() { static GlobalCurl initialization; (void)initialization; @@ -192,17 +200,19 @@ HttpClient::HttpClient(const Config& config, RestPolicy policy) : config_(config interval == "HOUR" ? 3600000ULL : interval == "DAY" ? 86400000ULL : 0ULL; if (!unit || !multiple || multiple > 10000 || !ceiling || ceiling > 1000000000) throw Error(23, "unsupported public rate-limit interval or ceiling"); - // Spend at most half of each published ceiling (about 50% headroom). - const auto pace = (unit * multiple * 2 + ceiling - 1) / ceiling; + // The one-minute weight ceiling is spent by the venue's own count (X-MBX-USED-WEIGHT-1M, shared by + // every client on this IP); any other ceiling is paced evenly across its interval. + const auto pace = (unit * multiple + ceiling - 1) / ceiling; if (type == "REQUEST_WEIGHT") { weighted = true; - milliseconds_per_weight_ = std::max(milliseconds_per_weight_, pace); if (interval == "MINUTE" && multiple == 1) weight_limit_ = ceiling; + else milliseconds_per_weight_ = std::max(milliseconds_per_weight_, pace); } else milliseconds_per_request_ = std::max(milliseconds_per_request_, pace); } if (!weighted) throw Error(23, "public request-weight ceiling is unavailable"); - log("info", "rest_limits_verified", Json::object({{"milliseconds_per_weight", Json::number(std::to_string(milliseconds_per_weight_))}, - {"weight_limit_1m", Json::number(std::to_string(weight_limit_))}})); + log("info", "rest_limits_verified", Json::object({{"weight_limit_1m", Json::number(std::to_string(weight_limit_))}, + {"milliseconds_per_weight", Json::number(std::to_string(milliseconds_per_weight_))}, + {"milliseconds_per_request", Json::number(std::to_string(milliseconds_per_request_))}})); } catch (const Error&) { throw; } catch (const std::exception&) { throw Error(23, "invalid public exchange rate-limit metadata"); } } @@ -218,6 +228,13 @@ std::string HttpClient::body(const std::string& path, unsigned int weight, std:: if (!limit || limit > 4 * 1024 * 1024) throw Error(22, "REST body bound exceeds 4 MiB"); for (unsigned int attempt = 0; attempt < 4; ++attempt) { pause_until(next_request_); + const auto wait = quota_wait_ms(used_weight_, used_window_, weight, weight_limit_, std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + if (wait && !config_.allow_insecure) { + log("warn", "rest_quota_wait", Json::object({{"used_weight_1m", Json::number(std::to_string(used_weight_))}, + {"weight_limit_1m", Json::number(std::to_string(weight_limit_))}, {"wait_ms", Json::number(std::to_string(wait))}})); + pause_for(std::chrono::milliseconds(wait)); + } Progress state; auto handle = handle_for(config_.rest_url + path, false, config_.allow_insecure, &state); Response response; @@ -257,12 +274,11 @@ std::string HttpClient::body(const std::string& path, unsigned int weight, std:: continue; } if (status != 200) throw Error(23, "public market-data request rejected (HTTP " + std::to_string(status) + ")"); - if (response.used_weight) log("info", "rest_weight", Json::object({{"used_weight_1m", Json::number(std::to_string(response.used_weight))}})); - // The venue's used-weight header covers every client on this IP: past half the ceiling, wait for the window. - if (weight_limit_ && response.used_weight >= weight_limit_ / 2) { - const auto elapsed = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count() % 60000; - next_request_ = std::max(next_request_, std::chrono::steady_clock::now() + std::chrono::milliseconds(60500 - elapsed)); - log("warn", "rest_shared_quota_pause"); + if (response.used_weight) { + used_weight_ = response.used_weight; + used_window_ = static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()) / 60000; + log("info", "rest_weight", Json::object({{"used_weight_1m", Json::number(std::to_string(response.used_weight))}})); } return std::move(response.body); } @@ -283,13 +299,26 @@ WebSocketPump::~WebSocketPump() { ready_.notify_all(); if (worker_.joinable()) worker_.join(); } -void WebSocketPump::push(SourceMessage message) { +// False when the message does not fit the queue now; one message larger than the whole budget stops 22. +bool WebSocketPump::push(SourceMessage message) { std::lock_guard guard(mutex_); const auto bytes = message.text.size() + 32; - if (bytes > config_.max_queue_bytes - std::min(queue_bytes_, config_.max_queue_bytes)) throw Error(22, "WebSocket healing buffer overrun"); + if (bytes > config_.max_queue_bytes) throw Error(22, "one source message exceeds --max-queue-bytes"); + if (bytes > config_.max_queue_bytes - std::min(queue_bytes_, config_.max_queue_bytes)) return false; queue_bytes_ += bytes; queue_.push_back(std::move(message)); ready_.notify_one(); + return true; +} +void WebSocketPump::wait_for_room(std::size_t bytes) { + for (;;) { + { + std::lock_guard guard(mutex_); + if (queue_bytes_ + bytes <= config_.max_queue_bytes) return; + } + if (stopped()) return; + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } } bool WebSocketPump::try_take(SourceMessage& message) { std::lock_guard guard(mutex_); @@ -344,20 +373,33 @@ void WebSocketPump::run() { curl_easy_getinfo(handle.get(), CURLINFO_ACTIVESOCKET, &socket); // The marker precedes every message of this connection: the session verifies its REST // overlap while the subscribed data is buffered behind it. - push({true, {}}); + wait_for_room(32); + if (stopped() || !push({true, {}})) return; bool subscribed = true; for (const auto& request : connection_.subscribe) subscribed = subscribed && send_text(handle.get(), socket, request); log(subscribed ? "info" : "warn", subscribed ? "websocket_connected" : "websocket_subscribe_failed"); const auto birth = std::chrono::steady_clock::now(); - auto last_data = birth, message_birth = birth, last_ping = birth; + auto last_data = birth, message_birth = birth, last_ping = birth, probe_sent = birth; std::string message; - bool assembling = false, heard = false; + bool assembling = false, heard = false, probing = false, backpressure = false; std::uint64_t frame_offset = 0; while (subscribed && !stopped()) { const auto now = std::chrono::steady_clock::now(); - // Keepalive replies are control messages: they cannot hide a silent data stream. - if (now - birth >= std::chrono::seconds(config_.reconnect_seconds) || now - last_data > std::chrono::seconds(75) || - (assembling && now - message_birth > std::chrono::seconds(30))) break; + if (now - birth >= std::chrono::seconds(config_.reconnect_seconds) || (assembling && now - message_birth > std::chrono::seconds(30))) break; + // Keepalive replies are control messages: they cannot hide a silent data stream. A venue with a + // probe proves the connection and its subscriptions instead of paying a reconnect; completeness + // never rests on the connection (chain IDs, watermarks and the reconnect overlap prove it). + if (now - last_data > std::chrono::seconds(config_.silence_seconds)) { + if (connection_.probe.empty()) break; + if (!probing) { + if (!send_text(handle.get(), socket, connection_.probe)) break; + probing = true; + probe_sent = now; + } else if (now - probe_sent > std::chrono::seconds(10)) { + log("warn", "source_probe_unanswered"); + break; + } + } if (!connection_.ping.empty() && now - last_ping >= std::chrono::seconds(config_.keepalive_seconds)) { if (!send_text(handle.get(), socket, connection_.ping)) break; last_ping = now; @@ -387,12 +429,28 @@ void WebSocketPump::run() { assembling = false; // A connection counts as established only once the venue answers on it. if (!heard) { heard = true; failures = 0; } - if (kind == Frame::Control) { message.clear(); continue; } + if (kind == Frame::Stale) { log("warn", "source_subscription_lost"); break; } + if (kind == Frame::Control) { + // A probe reply restarts the silence clock: the connection and subscriptions are alive. + if (probing) { probing = false; last_data = now; log("info", "source_probe_answered"); } + message.clear(); + continue; + } last_data = now; - push({false, std::move(message)}); + probing = false; + // A full queue (the session is busy healing a long outage) drops this connection instead of + // stopping: everything from here on is healed from REST after the reconnect's overlap check. + if (!push({false, std::move(message)})) { backpressure = true; break; } message.clear(); if (kind == Frame::Retire) { log("info", "source_retiring_reconnect"); break; } } + if (backpressure && !stopped()) { + log("warn", "websocket_backpressure_reconnect", Json::object({{"max_queue_bytes", Json::number(std::to_string(config_.max_queue_bytes))}})); + handle.reset(); + wait_for_room(config_.max_queue_bytes - config_.max_queue_bytes / 4); + next_attempt = std::chrono::steady_clock::now(); + continue; + } if (!stopped()) { log("warn", "websocket_reconnect"); std::this_thread::sleep_for(std::chrono::milliseconds(250)); diff --git a/native/feed/src/transport.hpp b/native/feed/src/transport.hpp index d4b41c2..bb37eaa 100644 --- a/native/feed/src/transport.hpp +++ b/native/feed/src/transport.hpp @@ -15,6 +15,10 @@ void check_runtime_curl(); void validate_origin(const std::string& origin, bool websocket, bool allow_insecure); void pause_for(std::chrono::milliseconds duration); std::uint64_t retry_after_seconds(const std::string& value, std::int64_t now); +// Milliseconds to wait before a request of `weight`: none while the venue-reported use of the current +// one-minute window (`used`, reported in window `used_window`) leaves room under 90% of `limit`; otherwise +// until the next window opens (500 ms margin). +std::uint64_t quota_wait_ms(std::uint64_t used, std::uint64_t used_window, unsigned int weight, std::uint64_t limit, std::int64_t now_ms); // A venue's public REST contract: allowlisted paths and how requests are paced. struct RestPolicy { @@ -38,6 +42,7 @@ class HttpClient { RestPolicy policy_; std::chrono::steady_clock::time_point next_request_{}; std::uint64_t weight_limit_ = 0, milliseconds_per_weight_ = 0, milliseconds_per_request_ = 0; + std::uint64_t used_weight_ = 0, used_window_ = 0; // the venue's X-MBX-USED-WEIGHT-1M and its minute public: HttpClient(const Config& config, RestPolicy policy); std::string body(const std::string& path, unsigned int weight, std::size_t limit = 1024 * 1024); @@ -59,7 +64,8 @@ class WebSocketPump { std::exception_ptr failure_; std::thread worker_; bool stopped() const; - void push(SourceMessage message); + bool push(SourceMessage message); + void wait_for_room(std::size_t bytes); void run(); public: WebSocketPump(const Config& config, Connection connection); diff --git a/native/feed/src/venue.hpp b/native/feed/src/venue.hpp index f1a25ff..07dea21 100644 --- a/native/feed/src/venue.hpp +++ b/native/feed/src/venue.hpp @@ -13,14 +13,17 @@ struct VenueEvent { Kline kline; }; // How the WebSocket reader treats one complete text message. Control messages (subscription -// acknowledgements, keepalive replies) prove liveness but are not queued; Retire queues the message, -// then reconnects. -enum class Frame { Data, Control, Retire }; +// acknowledgements, keepalive and probe replies) prove liveness but are not queued; Retire queues the +// message, then reconnects; Stale (a probe reply without the subscriptions) reconnects without queuing. +enum class Frame { Data, Control, Retire, Stale }; struct Connection { std::string path; // appended to the WebSocket origin std::vector subscribe; // text frames sent after every handshake std::string ping; // venue text keepalive; empty when curl's PONG suffices std::function classify; // runs on the reader thread: no shared state + // A request whose Control reply proves the connection and its subscriptions are alive (Binance + // LIST_SUBSCRIPTIONS): a quiet stream is probed instead of reconnected. Empty: silence reconnects. + std::string probe; }; // How a tick minute is proven complete before its `time` event. // KlineIdRange: the closed kline names the minute's raw-ID range and count (Binance spot). @@ -46,11 +49,9 @@ class Venue { return previous.front(); } // The same predecessor, but only once the venue holds a print at or after `minute`: nullopt while it - // holds none, without waiting or stopping (a quiet start). Fence venues override it. - virtual std::optional predecessor_if_ready(std::int64_t minute) { - try { return predecessor(minute); } - catch (const Error& failure) { if (failure.code == 20) return std::nullopt; throw; } - } + // holds none, without waiting or stopping (a quiet start). Every venue states its own answer: a default + // that mapped stops to "not ready" would wait silently on an unavailable predecessor. + virtual std::optional predecessor_if_ready(std::int64_t minute) = 0; // Re-checks instrument metadata that fixes the stream's units; a change stops 21. Called on connect. virtual void reverify() {} // Contiguous closed one-minute candles in [start, end), ascending. diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py index 4705376..b192000 100644 --- a/native/feed/tests/mock_venue.py +++ b/native/feed/tests/mock_venue.py @@ -3,6 +3,7 @@ injection.""" import base64 +import contextlib import copy import email.utils import fcntl @@ -12,6 +13,7 @@ import os import pathlib import signal +import socket import struct import subprocess import sys @@ -19,7 +21,9 @@ import threading import time import unittest +import urllib.error import urllib.parse +import urllib.request from types import SimpleNamespace from public_e2e import Soak, action_key, proven_actions @@ -267,6 +271,10 @@ def __init__(self, scripts, venue="binance"): self.exchange_info_bytes = 0 self.pings = [] self.okx_time_lookup_skip = 0 + self.instrument_reads = 0 + self.history_delay = 0.0 + self.probes = [] + self.stale_subscriptions = False self.refuse_once = None self.usdm_unknown_symbol = False self.connections = 0 @@ -300,6 +308,8 @@ def log_message(self, *arguments): def do_GET(self): parsed = urllib.parse.urlsplit(self.path) query = urllib.parse.parse_qs(parsed.query) + if parsed.path == "/api/v3/historicalTrades" and self.server.history_delay: + time.sleep(self.server.history_delay) with self.server.lock: self.server.requests.append(parsed.path) self.server.queries.append((parsed.path, query)) @@ -508,7 +518,11 @@ def rest_okx(self, path, query): server = self.server if path == "/api/v5/public/instruments": match = query["instId"][0] == server.instrument["instId"] - return 200, {"code": "0", "msg": "", "data": [server.instrument] if match else []} + body = {"code": "0", "msg": "", "data": [dict(server.instrument)] if match else []} + # Counted once the answer is built (under the server lock): scripts can wait on the + # feed's reads. + server.instrument_reads += 1 + return 200, body if path == "/api/v5/market/history-trades": limit = int(query["limit"][0]) if query.get("type") == ["2"]: @@ -619,26 +633,58 @@ def websocket(self, path, query): if self.server.venue in ("okx", "bybit"): self.subscribed(script) return - self.connection.settimeout(3) + lock, closed, ponged = threading.Lock(), threading.Event(), threading.Event() + + def send(payload, opcode=1, final=True): + with lock: + frame(self.connection, payload, opcode, final) + + def read(): + # Client frames while the script runs: PONG replies, and liveness probes (answered). + self.connection.settimeout(0.1) + while not closed.is_set(): + try: + opcode, payload = receive(self.connection) + except TimeoutError: + continue + except (EOFError, ConnectionResetError, OSError): + break + if opcode == 8: + break + if opcode == 10: + self.server.pongs.append(payload) + ponged.set() + elif opcode == 1: + request = json.loads(payload) + with self.server.lock: + self.server.probes.append(request) + streams = ( + [] if self.server.stale_subscriptions else query["streams"][0].split("/") + ) + send({"result": streams, "id": request["id"]}) + closed.set() + + reader = threading.Thread(target=read, daemon=True) + reader.start() for action in script: if isinstance(action, dict): - frame(self.connection, action) + send(action) elif action[0] == "pause": time.sleep(action[1]) elif action[0] == "ping": - frame(self.connection, action[1], opcode=9) - opcode, payload = receive(self.connection) - assert opcode == 10 and payload == action[1] - self.server.pongs.append(payload) + ponged.clear() + send(action[1], opcode=9) + assert ponged.wait(3) and self.server.pongs[-1] == action[1] elif action[0] == "fragment": payload = json.dumps(action[1], separators=(",", ":")).encode() middle = len(payload) // 2 - frame(self.connection, payload[:middle], final=False) - frame(self.connection, payload[middle:], opcode=0) + with lock: + frame(self.connection, payload[:middle], final=False) + frame(self.connection, payload[middle:], opcode=0) elif action[0] == "binary": - frame(self.connection, action[1], opcode=2) + send(action[1], opcode=2) elif action[0] == "raw": - frame(self.connection, action[1]) + send(action[1]) elif action[0] == "revise_trade": with self.server.lock: row = list(self.server.trades[action[1]]) @@ -649,6 +695,9 @@ def websocket(self, path, query): row = list(self.server.bars[action[1]]) row[4] = "0.70000000" self.server.bars[action[1]] = tuple(row) + elif action[0] == "add_aggregate": + with self.server.lock: + self.server.aggregates[action[1]] = action[2] elif action[0] == "lag_rest": # The REST row trails the WebSocket close, then catches up. with self.server.lock: @@ -661,17 +710,12 @@ def restore(minute=action[1], row=original): threading.Timer(action[2], restore).start() elif action[0] == "close": - frame(self.connection, b"", opcode=8) + send(b"", opcode=8) + closed.set() + reader.join(timeout=5) self.close_connection = True return - self.connection.settimeout(0.1) - while True: - try: - receive(self.connection) - except TimeoutError: - continue - except (EOFError, ConnectionResetError): - break + reader.join() self.close_connection = True def subscribed(self, script): @@ -740,6 +784,16 @@ def read(): elif action[0] == "set_instrument": with self.server.lock: self.server.instrument = dict(self.server.instrument, **action[1]) + elif action[0] == "wait_instrument_reads": + deadline = time.monotonic() + 15 + while True: + with self.server.lock: + if self.server.instrument_reads >= action[1]: + break + if time.monotonic() > deadline: + self.server.errors.append(f"instrument reads never reached {action[1]}") + break + time.sleep(0.005) elif action[0] == "okx_unconfirm": # REST trails the WebSocket confirm of this minute for a moment. with self.server.lock: @@ -772,6 +826,158 @@ def sequence(output): return [event["seq"] if event["type"] == "tick" else ("time", event["ts"]) for event in output] +def journal_lines(state): + """The retained normalized journal: segment files in message-index order.""" + lines = [] + for path in sorted((pathlib.Path(state) / "journal").glob("*.jsonl")): + lines += path.read_text().splitlines() + return lines + + +def minute_bars(count): + """Synthetic contiguous one-minute bars from 120000, in the mock's BARS tuple shape.""" + return { + 120000 + index * 60000: ( + "10.00000000", + "10.50000000", + "9.50000000", + "10.25000000", + f"{index % 7 + 1}.00000000", + 0, + 0, + 1, + ) + for index in range(count) + } + + +def bar_message(minute, bars): + opening, high, low, close, volume, _first, _last, count = bars[minute] + row = { + "t": minute, + "T": minute + 59999, + "s": "TESTUSDT", + "i": "1m", + "f": 1, + "L": 1, + "o": opening, + "h": high, + "l": low, + "c": close, + "v": volume, + "n": count, + "x": True, + } + return { + "stream": "testusdt@kline_1m", + "data": {"e": "kline", "E": minute + 60000, "s": "TESTUSDT", "k": row}, + } + + +def bar_line(minute, bars): + opening, high, low, close, volume = bars[minute][:5] + return ( + f'{{"type":"bar","bar":{{"ts_open":{minute},"o":{opening},"h":{high},"l":{low},' + f'"c":{close},"v":{volume}}}}}' + ) + + +class StreamClient: + """A minimal RFC 6455 client for the serve endpoint: masked frames out, PING answered.""" + + def __init__(self, port, path, receive_buffer=None, timeout=15): + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if receive_buffer: + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, receive_buffer) + self.socket.settimeout(timeout) + self.socket.connect(("127.0.0.1", port)) + key = base64.b64encode(os.urandom(16)).decode() + self.socket.sendall( + ( + f"GET {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nUpgrade: websocket\r\n" + f"Connection: Upgrade\r\nSec-WebSocket-Key: {key}\r\n" + "Sec-WebSocket-Version: 13\r\n\r\n" + ).encode() + ) + self.buffer = b"" + while b"\r\n\r\n" not in self.buffer: + self.fill() + head, self.buffer = self.buffer.split(b"\r\n\r\n", 1) + self.status = int(head.split(b" ")[1]) + self.body = b"" + if self.status != 101: + length = 0 + for line in head.split(b"\r\n")[1:]: + name, _, value = line.partition(b":") + if name.strip().lower() == b"content-length": + length = int(value) + while len(self.buffer) < length: + self.fill() + self.body = self.buffer[:length] + + def fill(self): + chunk = self.socket.recv(65536) + if not chunk: + raise EOFError + self.buffer += chunk + + def exact(self, size): + while len(self.buffer) < size: + self.fill() + result, self.buffer = self.buffer[:size], self.buffer[size:] + return result + + def send(self, opcode, payload=b""): + mask = os.urandom(4) + header = bytes([128 | opcode, 128 | len(payload)]) + self.socket.sendall(header + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))) + + def message(self): + """("text", str), ("pong", b""), or ("close", code, reason).""" + while True: + first, second = self.exact(2) + size = second & 127 + if size == 126: + size = struct.unpack("!H", self.exact(2))[0] + elif size == 127: + size = struct.unpack("!Q", self.exact(8))[0] + payload = self.exact(size) + opcode = first & 15 + if opcode == 1: + return ("text", payload.decode()) + if opcode == 10: + return ("pong", payload) + if opcode == 9: + self.send(10, payload) + continue + if opcode == 8: + code = struct.unpack("!H", payload[:2])[0] if len(payload) >= 2 else None + return ("close", code, payload[2:].decode()) + raise AssertionError(f"unexpected opcode {opcode}") + + def texts(self, count): + result = [] + while len(result) < count: + kind, *rest = self.message() + if kind == "text": + result.append(rest[0]) + elif kind == "close": + raise AssertionError(f"closed early: {rest}") + return result + + def close(self): + with contextlib.suppress(OSError): + self.socket.close() + + +def http_get(port, path): + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=10) as response: + return response.status, response.read() + except urllib.error.HTTPError as failure: + return failure.code, failure.read() + + class FeedMockTests(unittest.TestCase): def test_public_qualification_fails_on_any_action_difference(self): for ticks_equal, expected in ((False, 1), (True, 0)): @@ -877,10 +1083,8 @@ def run_feed(self, server, mode="bars", extra=(), expected=0, resume=False): cursor = self.directory / "state" / "cursor.json" if cursor.exists(): saved = json.loads(cursor.read_text())["cursor"] - self.assertEqual( - saved["message_index"], - len((cursor.parent / "events.jsonl").read_text().splitlines()), - ) + first = int(sorted((cursor.parent / "journal").glob("*.jsonl"))[0].name[:20]) + self.assertEqual(saved["message_index"], first + len(journal_lines(cursor.parent))) return output, result def test_bars_hole_is_healed_behind_closed_watermark(self): @@ -1247,12 +1451,6 @@ def test_429_retry_after_preserves_prefix(self): self.assertIn("rest_rate_limited", result.stderr) def test_replay_and_queue_budgets_stop_22(self): - server = self.server([[candle(120000)]]) - output, _ = self.run_feed(server, extra=["--max-log-bytes", "4"], expected=22) - self.assertEqual(output, []) - import shutil - - shutil.rmtree(self.directory / "state") server = self.server([[trade(100)]]) output, _ = self.run_feed(server, "ticks", ["--max-queue-bytes", "64"], expected=22) self.assertEqual(output, []) @@ -1284,7 +1482,7 @@ def test_sigterm_then_resume_tail_has_no_gap_or_duplicate(self): def test_changed_committed_journal_is_refused(self): server = self.server([[candle(120000)], [candle(180000)]]) self.run_feed(server, extra=["--max-messages", "1"]) - journal = self.directory / "state" / "events.jsonl" + journal = sorted((self.directory / "state" / "journal").glob("*.jsonl"))[0] journal.write_text(journal.read_text().replace("10.10000000", "10.20000000")) output, _ = self.run_feed(server, resume=True, expected=21) self.assertEqual(output, []) @@ -1301,6 +1499,66 @@ def history_pages(self, server): if path == "/api/v5/market/history-trades" and query.get("type") == ["1"] ] + def test_long_outage_drops_the_connection_and_heals_instead_of_stopping_22(self): + # The session heals a long hole slowly (REST pages take time) while the venue keeps + # sending: the reader's bounded queue fills, so the connection is dropped and everything + # after it is healed by ID + # from REST once the reconnect's overlap check passes. Exit 22 is gone; the tape is exact. + count = 2400 + trades = {99: TRADES[99]} + for offset in range(count): + trades[100 + offset] = (120001 + offset * 20, "10.00000000", "0.00000001") + + def push(identifier): + matched, price, quantity = trades[identifier] + data = { + "e": "trade", + "E": matched + 1, + "s": "TESTUSDT", + "t": identifier, + "p": price, + "q": quantity, + "T": matched, + "m": False, + } + return {"stream": "testusdt@trade", "data": data} + + # The first connection floods from print 1300 while 100..1299 heal; later ones carry + # the venue's head. + flood = [push(identifier) for identifier in range(1300, 100 + count)] + head = [push(identifier) for identifier in range(100 + count - 50, 100 + count)] + server = self.server([flood, head]) + server.trades = trades + server.history_delay = 0.4 + output, result = self.run_feed( + server, "ticks", ["--max-messages", str(count), "--max-queue-bytes", "16384"] + ) + self.assertEqual([event["seq"] for event in output], list(range(100, 100 + count))) + self.assertIn("websocket_backpressure_reconnect", result.stderr) + self.assertGreaterEqual(server.connections, 2) + + def test_quiet_binance_stream_is_probed_not_reconnected(self): + server = self.server([[candle(120000), ("pause", 3.5), candle(180000)]]) + output, result = self.run_feed( + server, extra=["--max-messages", "2", "--silence-seconds", "1"] + ) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000, 180000]) + self.assertEqual(server.connections, 1) + self.assertGreaterEqual(len(server.probes), 2) + self.assertEqual(server.probes[0], {"method": "LIST_SUBSCRIPTIONS", "id": 1}) + self.assertIn("source_probe_answered", result.stderr) + # A probe reply without the subscriptions is a lost subscription: reconnect with the + # overlap check. + self.reset_state() + server = self.server([[candle(120000), ("pause", 2.0)], [candle(180000)]]) + server.stale_subscriptions = True + output, result = self.run_feed( + server, extra=["--max-messages", "2", "--silence-seconds", "1"] + ) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000, 180000]) + self.assertGreaterEqual(server.connections, 2) + self.assertIn("source_subscription_lost", result.stderr) + def test_okx_swap_bars_confirm_flag_and_contract_units(self): server = self.server([[okx_candle(120000, "0"), okx_candle(180000)]], "okx") output, result = self.run_feed(server, extra=["--max-messages", "2"]) @@ -1524,6 +1782,104 @@ def test_okx_quiet_start_waits_for_the_first_print(self): ) self.assertNotIn('"code":20', result.stderr) + def test_usdm_quiet_start_waits_for_the_first_aggregate(self): + # No aggregate at or after --start for longer than the hour the venue searches at once: + # the feed waits, + # anchors nothing, and then proves the predecessor by searching forward hour by hour. + late = 120000 + 61 * 60000 + quiet = { + minute: ( + "10.10000000", + "10.10000000", + "10.10000000", + "10.10000000", + "0.00000000", + 0, + 0, + 0, + ) + for minute in range(120000, late, 60000) + } + quiet[late] = ( + "12.00000000", + "12.00000000", + "12.00000000", + "12.00000000", + "0.40000000", + 0, + 0, + 1, + ) + row = { + "T": late + 1, + "p": "12.00000000", + "q": "0.40000000", + "nq": "0.40000000", + "f": 1010, + "l": 1012, + } + nxt = { + "T": late + 60001, + "p": "12.00000000", + "q": "0.10000000", + "nq": "0.10000000", + "f": 1013, + "l": 1013, + } + + def closing(minute): + opening, high, low, close, volume = quiet[minute][:5] + return candle( + 120000, + t=minute, + T=minute + 59999, + o=opening, + h=high, + l=low, + c=close, + v=volume, + f=-1, + L=-1, + n=0, + ) + + server = self.server( + [ + [ + closing(120000), + closing(180000), + # Longer than the venue's predecessor retries: a blind anchor would stop 20. + ("pause", 3.0), + ( + "add_aggregate", + 500, + (row["T"], row["p"], row["q"], row["nq"], row["f"], row["l"]), + ), + ( + "add_aggregate", + 501, + (nxt["T"], nxt["p"], nxt["q"], nxt["nq"], nxt["f"], nxt["l"]), + ), + aggregate(500, a=500, **row), + closing(late), + aggregate(501, a=501, **nxt), + ] + ], + "usdm", + ) + server.aggregates = {499: AGGREGATES[499]} + server.bars = quiet + output, result = self.run_feed(server, "agg-ticks", ["--max-messages", "63"]) + minutes = [("time", minute + 60000) for minute in range(120000, late, 60000)] + self.assertEqual(sequence(output), [*minutes, 500, ("time", late + 60000)]) + self.assertNotIn('"code":20', result.stderr) + windows = [ + int(query["startTime"][0]) + for path, query in server.queries + if path == "/fapi/v1/aggTrades" and "startTime" in query + ] + self.assertIn(120000 + 3600000, windows) + def test_bybit_http_403_is_a_retryable_rate_ban(self): server = self.server([[bybit_push(bybit_row(120000, True))]], "bybit") server.fail_status = 403 @@ -1534,17 +1890,37 @@ def test_bybit_http_403_is_a_retryable_rate_ban(self): def test_okx_changed_contract_multiplier_stops_21_on_reconnect(self): notice = {"event": "notice", "code": "64008", "msg": "upgrade", "connId": "mock"} - server = self.server( - [ - # The pause lets the first connection's instrument check and its bar finish first. - [okx_candle(120000), ("pause", 1.0), ("set_instrument", {"ctVal": "0.1"}), notice], - [okx_candle(180000)], - ], - "okx", - ) - output, result = self.run_feed(server, expected=21) - self.assertEqual([event["bar"]["ts_open"] for event in output], [120000]) - self.assertIn("contract multiplier changed", result.stderr) + for mode, first, emitted in ( + ("bars", okx_candle(120000), [120000]), + ("ticks", okx_trades(100, 101), [100, 101]), + ): + with self.subTest(mode=mode): + self.reset_state() + server = self.server( + [ + # Two instrument reads answered (startup, then the first connection's + # re-check) + # before the instrument changes: no timing assumption. + [ + ("wait_instrument_reads", 2), + first, + ("set_instrument", {"ctVal": "0.1"}), + notice, + ], + [okx_candle(180000)], + ], + "okx", + ) + output, result = self.run_feed(server, mode, expected=21) + self.assertEqual( + [ + event["bar"]["ts_open"] if mode == "bars" else event["seq"] + for event in output + ], + emitted, + ) + self.assertIn("contract multiplier changed", result.stderr) + self.assertEqual(server.instrument_reads, 3) def test_okx_websocket_and_rest_lexemes_differ_but_overlap_values_match(self): notice = {"event": "notice", "code": "64008", "msg": "upgrade", "connId": "mock"} @@ -1896,6 +2272,345 @@ def test_coinbase_is_deferred(self): self.assertIn("Coinbase is deferred", result.stderr) self.assertFalse((self.directory / "state").exists()) + def serve(self, server, mode="bars", extra=(), resume=False, environment=None): + command = self.command(server, mode, [*extra, "--listen", "127.0.0.1:0"], resume) + command[1] = "serve" + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=dict(os.environ, **(environment or {})), + ) + records, ready = [], threading.Event() + + def read(): + for line in process.stderr: + records.append(json.loads(line)) + if records[-1]["event"] == "serve_listening": + ready.set() + ready.set() + + threading.Thread(target=read, daemon=True).start() + + def cleanup(): + if process.poll() is None: + process.kill() + process.wait() + + self.addCleanup(cleanup) + self.assertTrue(ready.wait(15)) + listening = [record for record in records if record["event"] == "serve_listening"] + self.assertTrue(listening, records) + return process, listening[0]["port"], records + + def status(self, port): + code, body = http_get(port, "/v1/status") + self.assertEqual(code, 200, body) + return json.loads(body) + + def wait_status(self, port, predicate, timeout=20): + deadline = time.monotonic() + timeout + status = None + while time.monotonic() < deadline: + status = self.status(port) + if predicate(status): + return status + time.sleep(0.05) + self.fail(f"status never satisfied the condition: {status}") + + def stop(self, process, expected=0): + process.send_signal(signal.SIGTERM) + self.assertEqual(process.wait(timeout=30), expected) + + def test_serve_streams_from_cursor_with_status_snapshot_and_keepalive(self): + server = self.server([[candle(240000)]]) + process, port, records = self.serve(server) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] == 3) + journal = journal_lines(self.directory / "state") + self.assertEqual(len(journal), 3) + self.assertEqual(status["retained"]["first_index"], 0) + self.assertEqual(status["source"]["venue"], "binance") + self.assertEqual(status["source"]["mode"], "bars") + self.assertEqual(status["last"]["verified_cut"], 300000) + self.assertEqual(status["clients"], 0) + code, body = http_get(port, "/v1/snapshot") + self.assertEqual((code, body.decode().splitlines()), (200, journal)) + epoch = status["epoch"] + client = StreamClient(port, f"/v1/stream?epoch={epoch}&from=0") + self.assertEqual(client.status, 101) + self.assertEqual(client.texts(3), journal) + # A quiet stream: an unsolicited PONG within 5 s keeps the runner's 15 s idle deadline + # alive. + started = time.monotonic() + self.assertEqual(client.message()[0], "pong") + self.assertLess(time.monotonic() - started, 6.5) + self.assertEqual(client.message()[0], "pong") + self.assertLess(time.monotonic() - started, 12) + tail = StreamClient(port, f"/v1/stream?epoch={epoch}&from=2") + self.assertEqual(tail.texts(1), journal[2:]) + self.assertEqual(self.status(port)["clients"], 2) + for path, expected in ( + (f"/v1/stream?epoch={'0' * 64}&from=0", 409), + (f"/v1/stream?epoch={epoch}&from=4", 416), + (f"/v1/stream?epoch={epoch}", 400), + (f"/v1/stream?epoch={epoch}&from=-1", 400), + ): + with self.subTest(path=path): + refused = StreamClient(port, path) + self.assertEqual(refused.status, expected, refused.body) + self.assertIn("error", json.loads(refused.body)) + refused.close() + self.stop(process) + while True: + kind, *rest = client.message() + if kind == "close": + self.assertEqual(rest[0], 1001) + break + self.assertTrue(any(record["event"] == "client_refused" for record in records)) + + def test_serve_expired_cursor_and_snapshot_fail_explicitly(self): + server = self.server([[]]) + server.bars = minute_bars(300) + script = [] + for index in range(300): + script += [bar_message(120000 + index * 60000, server.bars), ("pause", 0.002)] + server.scripts = [script] + process, port, _ = self.serve( + server, extra=["--segment-bytes", "4096", "--replay-bytes", "8192"] + ) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] == 300) + first = status["retained"]["first_index"] + self.assertGreater(first, 0) + expired = StreamClient(port, f"/v1/stream?epoch={status['epoch']}&from={first - 1}") + self.assertEqual(expired.status, 410) + self.assertIn("cursor expired", json.loads(expired.body)["error"]) + code, body = http_get(port, "/v1/snapshot") + self.assertEqual(code, 410) + self.assertIn("expired", json.loads(body)["error"]) + client = StreamClient(port, f"/v1/stream?epoch={status['epoch']}&from={first}") + expected = [bar_line(120000 + index * 60000, server.bars) for index in range(first, 300)] + self.assertEqual(client.texts(300 - first), expected) + self.assertEqual(journal_lines(self.directory / "state")[-len(expected) :], expected) + self.stop(process) + + def test_serve_resume_after_producer_sigkill_reconnects_at_the_cursor(self): + server = self.server([[]]) + server.bars = minute_bars(20) + server.scripts = [ + [bar_message(120000 + 9 * 60000, server.bars)], + [bar_message(120000 + 19 * 60000, server.bars)], + ] + expected = [bar_line(120000 + index * 60000, server.bars) for index in range(20)] + process, port, _ = self.serve(server) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] == 10) + epoch = status["epoch"] + client = StreamClient(port, f"/v1/stream?epoch={epoch}&from=0") + received = client.texts(6) + client.close() + process.send_signal(signal.SIGKILL) + process.wait(timeout=10) + process, port, _ = self.serve(server, resume=True) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] == 20) + self.assertEqual(status["epoch"], epoch) + resumed = StreamClient(port, f"/v1/stream?epoch={epoch}&from={len(received)}") + received += resumed.texts(20 - len(received)) + self.assertEqual(received, expected) + self.assertEqual(journal_lines(self.directory / "state"), expected) + self.stop(process) + + def test_serve_slow_client_is_closed_with_a_reason_and_never_blocks_others(self): + # A burst well beyond the kernel's socket buffers plus the client queue: a client that + # stops reading overflows its own queue and is closed with a reason; the producer and a + # reading client never wait. + server = self.server([[]]) + count = 70000 + server.bars = minute_bars(count) + last = 120000 + (count - 1) * 60000 + server.scripts = [ + [bar_message(120000, server.bars), ("pause", 2.0), bar_message(last, server.bars)] + ] + process, port, _ = self.serve(server, extra=["--client-queue-bytes", "1048576"]) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] >= 1) + path = f"/v1/stream?epoch={status['epoch']}&from=0" + stuck = StreamClient(port, path, receive_buffer=4096, timeout=60) + self.assertEqual(stuck.texts(1), [bar_line(120000, server.bars)]) + received = [] + + def consume(): + stream = StreamClient(port, path, timeout=60) + received.extend(stream.texts(count)) + stream.close() + + reader = threading.Thread(target=consume, daemon=True) + reader.start() + self.wait_status(port, lambda status: status["retained"]["next_index"] == count, timeout=60) + reader.join(timeout=60) + self.assertEqual(len(received), count) + self.assertEqual(received[-1], bar_line(last, server.bars)) + lines, closed = 1, None + while closed is None: + kind, *rest = stuck.message() + if kind == "text": + self.assertEqual(rest[0], bar_line(120000 + lines * 60000, server.bars)) + lines += 1 + elif kind == "close": + closed = rest + self.assertEqual(closed[0], 1008) + self.assertIn("slow consumer", closed[1]) + self.assertLess(lines, count) + self.assertIsNone(process.poll()) + self.stop(process) + + def test_serve_fanout_stress_clients_connect_and_disconnect_during_rotation(self): + server = self.server([[]]) + count = 1500 + server.bars = minute_bars(count) + script = [bar_message(120000, server.bars), ("pause", 1.0)] + for index in range(1, count): + script += [bar_message(120000 + index * 60000, server.bars), ("pause", 0.002)] + server.scripts = [script] + expected = [bar_line(120000 + index * 60000, server.bars) for index in range(count)] + process, port, _ = self.serve( + server, + extra=[ + "--segment-bytes", + "4096", + "--replay-bytes", + "16384", + "--client-queue-bytes", + "8192", + ], + ) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] >= 1) + epoch, outcomes, failures = status["epoch"], [], [] + finished = threading.Event() + + def client(seed): + import random + + chooser = random.Random(seed) + while not finished.is_set(): + try: + current = self.status(port)["retained"] + start = chooser.randint(current["first_index"], current["next_index"]) + slow = chooser.random() < 0.2 + stream = StreamClient( + port, + f"/v1/stream?epoch={epoch}&from={start}", + receive_buffer=4096 if slow else None, + ) + if stream.status == 410: + outcomes.append("expired-at-handshake") + stream.close() + continue + if stream.status != 101: + failures.append(("status", stream.status, stream.body)) + return + index, wanted = start, chooser.randint(1, 120) + while index < start + wanted and index < count: + kind, *rest = stream.message() + if kind == "pong": + continue + if kind == "close": + if rest[0] not in (1008, 4410, 1001): + failures.append(("close", rest)) + outcomes.append(f"closed-{rest[0]}") + break + if rest[0] != expected[index]: + failures.append(("line", index, rest[0][:80])) + return + index += 1 + if slow: + time.sleep(0.01) + else: + outcomes.append("read") + stream.close() + except (OSError, EOFError) as failure: + outcomes.append(f"transport-{type(failure).__name__}") + + threads = [threading.Thread(target=client, args=(seed,), daemon=True) for seed in range(12)] + for thread in threads: + thread.start() + self.wait_status(port, lambda status: status["retained"]["next_index"] == count, timeout=60) + finished.set() + for thread in threads: + thread.join(timeout=30) + self.assertEqual(failures, []) + self.assertGreater(outcomes.count("read"), 10) + last = self.status(port) + self.assertGreater(last["retained"]["first_index"], 0) + self.assertEqual( + journal_lines(self.directory / "state"), expected[last["retained"]["first_index"] :] + ) + self.stop(process) + + def test_serve_listens_on_loopback_unless_remote_is_explicit(self): + server = self.server([[]]) + command = self.command(server, extra=["--listen", "0.0.0.0:0"]) + command[1] = "serve" + result = subprocess.run(command, capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 23) + self.assertIn("--allow-remote-listen", result.stderr) + self.assertEqual(server.requests, []) + result = subprocess.run( + self.command(server, extra=["--listen", "127.0.0.1:0"]), + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 23) + self.assertIn("serve options", result.stderr) + + def test_rotation_and_expiry_crash_points_resume_with_an_exact_tape(self): + count = 200 + bars = minute_bars(count) + script = [] + for index in range(count): + script += [bar_message(120000 + index * 60000, bars), ("pause", 0.004)] + expected = [bar_line(120000 + index * 60000, bars) for index in range(count)] + journal = ["--segment-bytes", "4096", "--replay-bytes", "8192"] + for point in ( + "commit-append", + "commit-cursor", + "rotate-checkpoint", + "rotate-segment", + "rotate-cursor", + "retain-segment", + ): + with self.subTest(point=point): + self.reset_state() + server = self.server([script]) + server.bars = bars + crashed = subprocess.run( + self.command(server, extra=journal), + capture_output=True, + text=True, + timeout=30, + env=dict(os.environ, PINEFORGE_FEED_CRASH_AT=point + ":3"), + ) + self.assertEqual(crashed.returncode, -signal.SIGKILL, crashed.stderr[-500:]) + received = crashed.stdout.splitlines() + saved = json.loads((self.directory / "state" / "cursor.json").read_text())["cursor"] + self.assertGreaterEqual(saved["message_index"], len(received)) + remaining = count - saved["message_index"] + self.assertGreater(remaining, 0) + output, result = self.run_feed( + server, + extra=[ + *journal, + *("--output-from", str(len(received)), "--max-messages", str(remaining)), + ], + resume=True, + ) + self.assertEqual(len(output), count - len(received)) + received += result.stdout.splitlines() + self.assertEqual(received, expected) + if point == "retain-segment": + self.assertIn("expired_duplicate_unverified", result.stderr) + retained = journal_lines(self.directory / "state") + self.assertEqual(retained, expected[count - len(retained) :]) + def warmup(self, server): port = server.server_address[1] output = self.directory / "warmup.csv" diff --git a/native/feed/tests/public_e2e.py b/native/feed/tests/public_e2e.py index 94ac685..6edcdac 100644 --- a/native/feed/tests/public_e2e.py +++ b/native/feed/tests/public_e2e.py @@ -28,6 +28,14 @@ def rows(path, exact=False): ] +def journal_rows(state, exact=False): + """The feed's retained durable journal: segment files in message-index order.""" + result = [] + for path in sorted((pathlib.Path(state) / "journal").glob("*.jsonl")): + result += rows(path, exact) + return result + + SCRIPT_TF_MS = 60000 # the runner and batch both run --script-tf 1 @@ -387,7 +395,7 @@ def validate(self, mode, duration, cursor, minutes): json.loads(line, parse_float=str, parse_int=str) for line in source if line.strip() ] assert len(tape) == cursor, "capture/runner cursor mismatch" - durable = rows(self.directory / (mode + "-state/events.jsonl"), True) + durable = journal_rows(self.directory / (mode + "-state"), True) assert tape == durable, "restart changed normalized prefix or message boundaries" self.receipt(f"PASS {mode} restart without gap or duplicate messages={cursor}") if mode == "bars": @@ -546,6 +554,9 @@ def compare_minutes(): ) if mode != "bars" and not self.venue.candle_is_print_sum: self.kline_batch(mode, normalized, actual_actions, len(differing)) + exported = None + if mode != "bars" and getattr(self.options, "export", False): + exported = self.export_check(mode, normalized, actual_actions) if mode != "bars": name = mode + "-replay" completed = subprocess.run( @@ -568,6 +579,7 @@ def compare_minutes(): ] self.receipt(f"PASS {mode} same-print runner replay actions equal actions={len(raw)}") return { + "export": exported, "duration_seconds": duration, "messages": cursor, "minutes": len(normalized), @@ -576,6 +588,71 @@ def compare_minutes(): "batch_actions_equal": equal, } + def export_check(self, mode, normalized, actual_actions): + """`pineforge-feed export` over the proven window, from venue REST: its prints-built bars + must equal the bars the runner built from the same prints (the tape's minutes), value for + value, and the batch over the exported CSV must equal the runner's actions.""" + end = normalized[-1]["ts_open"] + 60000 + output = self.directory / (mode + "-export.csv") + command = [ + self.options.feed, + "export", + "--venue", + self.venue.venue, + "--market", + self.venue.market, + "--symbol", + self.venue.symbol, + "--mode", + mode, + "--start", + str(self.cut), + "--end", + str(end), + "--output", + str(output), + ] + with open(self.directory / (mode + "-export.stderr"), "w") as error: + subprocess.run(command, stderr=error, check=True, timeout=600) + with open(output) as source: + lines = source.read().splitlines() + assert lines[0] == "timestamp,open,high,low,close,volume" + exported = [ + dict( + zip( + ("ts_open", "o", "h", "l", "c", "v"), + [int(row[0]), *(Decimal(v) for v in row[1:])], + strict=True, + ) + ) + for row in (line.split(",") for line in lines[1:]) + ] + assert exported == normalized, "exported bars differ from the runner's prints-built bars" + manifest = json.loads((self.directory / (mode + "-export.csv.manifest.json")).read_text()) + combined = self.directory / (mode + "-export-combined.csv") + with open(combined, "w") as target: + target.write((self.directory / "warmup.csv").read_text()) + target.write("\n".join(lines[1:]) + "\n") + actions = self.directory / (mode + "-export-batch-actions.jsonl") + subprocess.run( + [self.options.batch_probe, self.options.observed_strategy, str(combined), str(actions)], + check=True, + ) + expected = [ + action_key(record, mode) + for record in rows(actions) + if record["origin_input_index"] >= 200 + ] + equal = expected == actual_actions + self.receipt( + f"{'PASS' if equal else 'FAIL'} {mode} export prints-built bars equal the runner's " + f"tick-built bars minutes={len(exported)} prints={manifest['prints']} " + f"quiet_minutes={manifest['quiet_minutes']} " + f"batch_over_export_actions={len(expected)} runner={len(actual_actions)}" + ) + assert equal, "the batch over the exported bars differs from the runner" + return {"minutes": len(exported), "prints": manifest["prints"], "actions": len(expected)} + def kline_batch(self, mode, normalized, actual_actions, kline_differs): """Information, not a gate: the same strategy over Binance klines instead of the bars built from the prints. Aggregates can straddle a minute edge, so fills there can differ.""" @@ -683,4 +760,7 @@ def run(self): parser.add_argument("--bar-minutes", type=int, default=46) parser.add_argument("--tick-minutes", type=int, default=21) parser.add_argument("--restart-seconds", type=int, default=180) + parser.add_argument( + "--export", action="store_true", help="also check `pineforge-feed export` over tick windows" + ) raise SystemExit(Soak(parser.parse_args()).run()) diff --git a/native/feed/tests/serve_e2e.py b/native/feed/tests/serve_e2e.py new file mode 100644 index 0000000..1132aa6 --- /dev/null +++ b/native/feed/tests/serve_e2e.py @@ -0,0 +1,263 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Explicit public-data soak through `serve`: a producer, runners on --feed-url, cursor resumes.""" + +import argparse +import json +import os +import signal +import sqlite3 +import subprocess +import time +import traceback +import urllib.request +from decimal import Decimal + +from public_e2e import Soak, action_key, journal_rows, proven_actions, rows + + +def ledger_inputs(path): + with sqlite3.connect(path) as ledger: + return [ + row[0] + for row in ledger.execute("SELECT canonical_json FROM inputs ORDER BY input_index") + ] + + +class ServeSoak(Soak): + def __init__(self, options): + options.modes = options.mode + options.bar_minutes = options.tick_minutes = options.minutes + super().__init__(options) + self.mode = options.mode + self.state = self.directory / (self.mode + "-state") + self.producer = None + self.port = None + self.epoch = None + self.generation = 0 + self.runners = {} + + def start_producer(self, resume): + command = [ + self.options.feed, + "serve", + "--venue", + self.venue.venue, + "--market", + self.venue.market, + "--symbol", + self.venue.symbol, + "--mode", + self.mode, + "--state-dir", + str(self.state), + "--listen", + "127.0.0.1:0", + ] + command += ["--resume"] if resume else ["--start", str(self.cut)] + error = self.directory / f"producer-{self.generation}.stderr" + with open(error, "w") as log: + self.producer = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=log) + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + for line in error.read_text().splitlines(): + record = json.loads(line) + if record["event"] == "serve_listening": + self.port = record["port"] + status = self.status() + assert self.epoch in (None, status["epoch"]), ( + "the resumed producer changed the epoch" + ) + self.epoch = status["epoch"] + self.generation += 1 + return status + assert self.producer.poll() is None, f"producer exited {self.producer.returncode}" + time.sleep(0.2) + raise AssertionError("producer never listened") + + def status(self): + with urllib.request.urlopen( + f"http://127.0.0.1:{self.port}/v1/status", timeout=10 + ) as response: + return json.loads(response.read()) + + def start_runner(self, name, cursor): + state = self.runners.setdefault(name, {"generations": 0, "exits": []}) + generation = state["generations"] + url = f"ws://127.0.0.1:{self.port}/v1/stream?epoch={self.epoch}&from={cursor}" + command = self.runner_command( + self.mode, name, self.directory / (name + ".sqlite3"), "-", cursor + ) + index = command.index("--feed") + command[index : index + 2] = ["--feed-url", url] + prefix = self.directory / f"{name}-runner-{generation}" + with open(f"{prefix}.stdout", "w") as output, open(f"{prefix}.stderr", "w") as error: + state["process"] = subprocess.Popen( + command, stdin=subprocess.DEVNULL, stdout=output, stderr=error + ) + state["generations"] += 1 + return url + + def wait_runner(self, name, timeout=60): + state = self.runners[name] + code = state["process"].wait(timeout=timeout) + state["exits"].append(code) + return code, len(ledger_inputs(self.directory / (name + ".sqlite3"))) + + def run(self): + try: + self.warmup() + started = time.monotonic() + self.start_producer(False) + names = [ + f"{self.mode}" if index == 0 else f"{self.mode}-r{index + 1}" + for index in range(self.options.runners) + ] + for name in names: + self.start_runner(name, 0) + self.receipt( + f"START serve {self.mode} public {self.venue.venue} {self.venue.market} " + f"{self.venue.symbol} cut={self.cut} runners={len(names)} " + f"minimum_minutes={self.options.minutes}" + ) + events = [] + if len(names) > 1: + events.append((self.options.runner_restart_seconds, "runner")) + if self.options.producer_kill_seconds: + events.append((self.options.producer_kill_seconds, "producer")) + for moment, kind in sorted(events): + while time.monotonic() - started < moment: + assert self.producer.poll() is None, "producer stopped early" + for name in names: + assert self.runners[name]["process"].poll() is None, ( + f"runner {name} stopped early" + ) + time.sleep(1) + if kind == "runner": + name = names[-1] + self.runners[name]["process"].send_signal(signal.SIGTERM) + code, cursor = self.wait_runner(name) + assert code == 130, f"runner {name} exit={code}" + self.start_runner(name, cursor) + self.receipt(f"RESTART runner {name} --from-input={cursor} from={cursor}") + else: + self.producer.send_signal(signal.SIGKILL) + assert self.producer.wait(timeout=20) == -signal.SIGKILL + cursors = {} + for name in names: + code, cursors[name] = self.wait_runner(name) + assert code == 1, f"runner {name} exit={code} after the producer was killed" + status = self.start_producer(True) + for name in names: + self.start_runner(name, cursors[name]) + self.receipt( + "RESUME producer --resume epoch-unchanged next_index=" + f"{status['retained']['next_index']} " + + " ".join( + f"{name}:--from-input={cursor}" for name, cursor in cursors.items() + ) + ) + while time.monotonic() - started < self.options.minutes * 60: + assert self.producer.poll() is None, "producer stopped early" + for name in names: + assert self.runners[name]["process"].poll() is None, ( + f"runner {name} stopped early" + ) + time.sleep(5) + final = self.status() + self.producer.send_signal(signal.SIGTERM) + assert self.producer.wait(timeout=30) == 0, "producer did not stop cleanly" + cursors = {name: self.wait_runner(name)[1] for name in names} + duration = time.monotonic() - started + journal = journal_rows(self.state, True) + assert final["retained"]["first_index"] == 0 + assert len(journal) >= final["retained"]["next_index"] + for name in names: + ledger = [ + json.loads(text, parse_float=Decimal) + for text in ledger_inputs(self.directory / (name + ".sqlite3")) + ] + assert ledger == journal[: len(ledger)], ( + f"runner {name} ledger differs from the journal prefix" + ) + assert len(ledger) == len(journal), ( + f"runner {name} committed {len(ledger)} of {len(journal)}" + ) + self.receipt( + f"PASS serve {self.mode} runner {name} ledger equals the journal " + f"messages={len(ledger)} generations={self.runners[name]['generations']} " + "(no loss, no duplicate)" + ) + idle = [ + path.name + for path in self.directory.glob("*-runner-*.stderr") + if "idle or message timeout" in path.read_text() + ] + assert not idle, f"runner idle deadline tripped: {idle}" + self.receipt( + f"PASS serve {self.mode} quiet periods never tripped the runner idle deadline " + f"runner_generations={sum(state['generations'] for state in self.runners.values())}" + ) + # The journal is the tape: validate it like the stdout path (REST, batch, replay). + tape = self.directory / (self.mode + "-feed.jsonl") + with open(tape, "w") as output: + for path in sorted((self.state / "journal").glob("*.jsonl")): + output.write(path.read_text()) + result = self.validate(self.mode, duration, len(journal), self.options.minutes) + batch = [ + action_key(record, self.mode) + for record in rows(self.directory / (self.mode + "-batch-actions.jsonl")) + if record["origin_input_index"] >= 200 + ] + window = 200 + result["minutes"] + for name in names[1:]: + actual, trailing = proven_actions( + self.receiver.payloads.get(name, []), self.mode, window + ) + equal = actual == batch + result.setdefault("runners", {})[name] = { + "batch_actions_equal": equal, + "actions": len(actual), + } + result["batch_actions_equal"] = result["batch_actions_equal"] and equal + self.receipt( + f"{'PASS' if equal else 'FAIL'} serve {self.mode} runner {name} actions equal " + f"batch batch={len(batch)} runner={len(actual)} " + f"after_last_proven_minute={trailing}" + ) + self.results[self.mode] = result + except Exception: + failure = traceback.format_exc() + (self.directory / "serve-failure.txt").write_text(failure) + self.receipt(f"FAIL serve {self.mode} qualification; see serve-failure.txt") + self.results[self.mode] = {"error": failure} + finally: + if self.producer and self.producer.poll() is None: + self.producer.kill() + for state in self.runners.values(): + if state.get("process") and state["process"].poll() is None: + state["process"].kill() + self.receiver.shutdown() + (self.directory / "summary.json").write_text( + json.dumps(self.results, indent=2, default=str) + "\n" + ) + failed = any( + "error" in result or not result.get("batch_actions_equal") + for result in self.results.values() + ) + return 1 if failed else 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + for argument in ("feed", "runner", "strategy", "observed-strategy", "batch-probe", "directory"): + parser.add_argument("--" + argument, required=True) + parser.add_argument("--venue", default="binance") + parser.add_argument("--market", default="spot") + parser.add_argument("--symbol", default="BTCUSDT") + parser.add_argument("--mode", default="bars") + parser.add_argument("--minutes", type=int, default=17) + parser.add_argument("--runners", type=int, default=2) + parser.add_argument("--runner-restart-seconds", type=int, default=240) + parser.add_argument("--producer-kill-seconds", type=int, default=480) + os.environ.setdefault("PYTHONUNBUFFERED", "1") + raise SystemExit(ServeSoak(parser.parse_args()).run()) diff --git a/native/feed/tests/unit.cpp b/native/feed/tests/unit.cpp index 9ee537f..cb1c26c 100644 --- a/native/feed/tests/unit.cpp +++ b/native/feed/tests/unit.cpp @@ -3,6 +3,10 @@ #include "bybit.hpp" #include "okx.hpp" #include "session.hpp" +#if PINEFORGE_FEED_SERVE +#include "serve.hpp" +#endif +#include #include #include #include @@ -55,7 +59,7 @@ struct SyntheticVenue final : Venue { {180000, {{180000, "12.00000000", "12.00000000", "11.00000000", "11.00000000", "0.50000000"}, 103, 104, 2, true}}, {240000, {{240000, "12.00000000", "12.00000000", "12.00000000", "12.00000000", "0.10000000"}, 105, 105, 1, true}} }; - Connection connection() const override { return {"/stream?streams=test@trade/test@kline_1m", {}, {}, &binance_frame}; } + Connection connection() const override { return {"/stream?streams=test@trade/test@kline_1m", {}, {}, &binance_frame, {}}; } std::vector decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } std::string kline_source() const override { return "/synthetic"; } std::size_t pages = 0; @@ -71,6 +75,7 @@ struct SyntheticVenue final : Venue { return result; } std::uint64_t first_trade_id(std::int64_t minute) override { return static_cast(bars.at(minute).first); } + std::optional predecessor_if_ready(std::int64_t) override { throw Error(23, "kline-range venue"); } // REST rows carry no x=true and no f..L, exactly like the venue's klines endpoint. std::vector klines(std::int64_t start, std::int64_t end) override { std::vector result; @@ -110,7 +115,7 @@ struct FenceVenue final : Venue { std::size_t pages = 0, candle_requests = 0; // REST lag: the next `empty_reads` history reads return nothing, the next `short_reads` drop their last row. std::size_t empty_reads = 0, short_reads = 0; - Connection connection() const override { return {"/synthetic", {}, {}, &binance_frame}; } + Connection connection() const override { return {"/synthetic", {}, {}, &binance_frame, {}}; } std::vector decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } std::vector history(std::uint64_t from, std::size_t limit) override { ++pages; @@ -147,6 +152,30 @@ struct FenceVenue final : Venue { VenueEvent tick(std::uint64_t id) const { return {VenueEvent::Kind::Trade, trades.at(id), {}}; } VenueEvent close(std::int64_t minute) const { return {VenueEvent::Kind::Kline, {}, candles.at(minute)}; } }; +std::string segment_file(const Temporary& temporary, std::uint64_t base) { + return temporary.path + "/journal/" + segment_name(base) + ".jsonl"; +} +std::string checkpoint_file(const Temporary& temporary, std::uint64_t base) { + return temporary.path + "/journal/" + segment_name(base) + ".checkpoint.json"; +} +// Contiguous synthetic minutes from `first`, each staged with its proof and committed in batches of `batch`. +void commit_bars(State& state, std::int64_t first, std::size_t count, std::size_t batch = 7) { + for (std::size_t index = 0; index < count; ++index) { + const Bar bar{first + static_cast(index) * 60000, "10.00000000", "10.50000000", "9.50000000", "10.25000000", "1.00000000"}; + state.stage(bar.wire(), bar); + if ((index + 1) % batch == 0) state.flush(); + } + state.flush(); +} +std::vector journal_bases(const Temporary& temporary, const char* suffix) { + std::vector bases; + for (const auto& entry : std::filesystem::directory_iterator(temporary.path + "/journal")) { + const auto name = entry.path().filename().string(); + if (name.size() > 20 && name.substr(20) == suffix) bases.push_back(std::stoull(name.substr(0, 20))); + } + std::sort(bases.begin(), bases.end()); + return bases; +} std::vector kinds(const std::vector& lines) { std::vector result; for (const auto& line : lines) { @@ -271,14 +300,14 @@ int main() { second.resume = true; expect(22, [&] { State locked(second); }); } - { std::ofstream tail(temporary.path + "/events.jsonl", std::ios::app); tail << "uncommitted-torn-tail"; } + { std::ofstream tail(segment_file(temporary, 0), std::ios::app); tail << "uncommitted-torn-tail"; } options.resume = true; options.start = -1; options.output_from = 2; { State state(options); assert(state.cursor().epoch == epoch); - assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == state.cursor().log_bytes); + assert(std::filesystem::file_size(segment_file(temporary, 0)) == state.cursor().log_bytes); std::vector output; FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); session.connected(); @@ -289,7 +318,7 @@ int main() { assert(parse_json(output.back()).at("type").text() == "time"); } { - std::fstream corrupt(temporary.path + "/events.jsonl", std::ios::in | std::ios::out); + std::fstream corrupt(segment_file(temporary, 0), std::ios::in | std::ios::out); corrupt.seekp(0); corrupt << 'X'; } @@ -309,15 +338,105 @@ int main() { passed("historical_ticks_require_closed_watermark_next_id_fence_and_rest_reconciliation"); } { + // Segments seal at --segment-bytes; sealed segments beyond --replay-bytes expire oldest first, but never + // the open segment or one holding a protected minute (the 32 closed-minute proofs and the open minute). Temporary temporary; auto options = config(temporary, "bars"); - options.max_log_bytes = 4; - State state(options); - SyntheticVenue venue; - FeedSession session(state, venue, [](const auto&) {}); - expect(22, [&] { session.ingest(venue.close(120000)); }); - assert(state.cursor().message_index == 0); - passed("bounded_replay_budget_stops_before_commit"); + options.segment_bytes = 4096; + options.replay_bytes = 8192; + std::string epoch, prefix; + std::uint64_t first = 0; + { + State state(options); + commit_bars(state, 120000, 400); + epoch = state.durable().epoch; + prefix = state.durable().prefix_hash; + first = state.first_retained(); + assert(state.durable().message_index == 400 && first > 0 && state.segments().size() >= 2); + assert(journal_bases(temporary, ".jsonl") == journal_bases(temporary, ".checkpoint.json")); + assert(journal_bases(temporary, ".jsonl").front() == first && journal_bases(temporary, ".jsonl").back() == state.durable().segment); + // The retained bytes stay within the budget plus the open segment. + assert(state.durable().log_bytes - state.segments().front().bytes <= options.replay_bytes + options.segment_bytes); + // The protected window: every proof minute is still readable. + std::vector minutes; + state.visit(state.first_retained(), [&](const std::string& line) { minutes.push_back(normalized_bar(parse_json(line)).ts); }); + assert(minutes.front() <= state.durable().proofs.front().ts && minutes.back() == 120000 + 399 * 60000); + for (std::size_t index = 1; index < minutes.size(); ++index) assert(minutes[index] == minutes[index - 1] + 60000); + expect(22, [&] { state.visit(0, [](const auto&) {}); }); + assert(state.bar(120000 + 399 * 60000) && state.bar(state.durable().proofs.front().ts)); + } + options.resume = true; + options.start = -1; + expect(22, [&] { State replay_from_origin(options); }); + options.output_from = first; + { + State resumed(options); + assert(resumed.durable().epoch == epoch && resumed.durable().prefix_hash == prefix && resumed.first_retained() == first); + assert(resumed.recent_bars().size() == 32 && resumed.durable().message_index == 400); + commit_bars(resumed, 120000 + 400 * 60000, 3); + } + options.output_from = first - 1; + expect(22, [&] { State expired(options); }); + options.output_from = first; + { State exact(options); assert(exact.durable().message_index == 403); } + // Retention by venue time: with an age limit, segments older than it expire once outside the protected window. + Temporary aged; + auto timed = config(aged, "bars"); + timed.segment_bytes = 4096; + timed.replay_bytes = 1ULL << 30; + timed.replay_age_ms = 60 * 60000; + State state(timed); + commit_bars(state, 120000, 300); + assert(state.first_retained() > 0); + assert(state.durable().cut - state.segments().front().cut <= timed.replay_age_ms + 40 * 60000); + passed("journal_segments_rotate_expire_by_bytes_and_age_and_resume"); + } + { + // Crash residue of every rotation and expiry step is recognised: files after the cursor's open segment + // (an interrupted rotation) and a checkpoint without its segment (an interrupted expiry) are removed; a + // segment without its checkpoint, or a changed checkpoint, stops 21. + Temporary temporary; + auto options = config(temporary, "bars"); + options.segment_bytes = 4096; + options.replay_bytes = 8192; + std::uint64_t open = 0, first = 0, expired = 0; + std::string prefix; + { + State state(options); + commit_bars(state, 120000, 300); + open = state.durable().segment; + first = state.first_retained(); + prefix = state.durable().prefix_hash; + } + expired = first - 1; + std::filesystem::copy_file(checkpoint_file(temporary, first), checkpoint_file(temporary, expired)); + std::filesystem::copy_file(checkpoint_file(temporary, open), checkpoint_file(temporary, open + 1000)); + { std::ofstream residue(segment_file(temporary, open + 1000)); } + std::filesystem::copy_file(checkpoint_file(temporary, open), checkpoint_file(temporary, open + 2000)); + options.resume = true; + options.start = -1; + options.output_from = first; + { + State resumed(options); + assert(resumed.durable().prefix_hash == prefix && resumed.durable().segment == open && resumed.first_retained() == first); + assert(!std::filesystem::exists(checkpoint_file(temporary, expired)) && !std::filesystem::exists(segment_file(temporary, open + 1000))); + assert(!std::filesystem::exists(checkpoint_file(temporary, open + 1000)) && !std::filesystem::exists(checkpoint_file(temporary, open + 2000))); + commit_bars(resumed, 120000 + 300 * 60000, 2); + } + const auto saved = checkpoint_file(temporary, open) + ".saved"; + std::filesystem::rename(checkpoint_file(temporary, open), saved); + expect(21, [&] { State missing(options); }); + std::filesystem::rename(saved, checkpoint_file(temporary, open)); + { + std::string text; + { std::ifstream input(checkpoint_file(temporary, open)); text.assign(std::istreambuf_iterator(input), {}); } + const auto position = text.find("\"verified_cut\":"); + text[position + 15] = text[position + 15] == '1' ? '2' : '1'; + std::ofstream output(checkpoint_file(temporary, open), std::ios::trunc); + output << text; + } + expect(21, [&] { State changed(options); }); + passed("journal_rotation_and_expiry_crash_residue_recovers_or_stops_21"); } { Temporary temporary; @@ -353,7 +472,7 @@ int main() { options.resume = true; State recovered(options); assert(recovered.cursor().message_index == 0); - assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == 0); + assert(std::filesystem::file_size(segment_file(temporary, 0)) == 0); passed("failed_cursor_write_rolls_back_uncommitted_message_on_resume"); } { @@ -439,22 +558,28 @@ int main() { const Bar bar{120000, "10.10000000", "11.20000000", "9.90000000", "9.90000000", "0.60000000"}; state.stage(bar.wire(), bar); assert(state.staged() == 1 && state.cursor().message_index == 1 && state.durable().message_index == 0); - assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == 0); + assert(std::filesystem::file_size(segment_file(temporary, 0)) == 0); std::size_t seen = 0; state.visit(0, [&](const auto&) { ++seen; }); assert(seen == 1 && state.bar(120000)); const auto lines = state.flush(); assert(lines.size() == 1 && state.staged() == 0 && state.durable().message_index == 1); - assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == lines.front().size() + 1); + assert(std::filesystem::file_size(segment_file(temporary, 0)) == lines.front().size() + 1); passed("group_commit_stages_then_persists_before_publication"); } { Temporary temporary; - { std::ofstream empty(temporary.path + "/events.jsonl"); } + std::filesystem::create_directories(temporary.path + "/journal"); + { std::ofstream empty(segment_file(temporary, 0)); } + { std::ofstream checkpoint(checkpoint_file(temporary, 0)); checkpoint << "{}"; } { State state(config(temporary, "bars")); assert(state.durable().message_index == 0); } Temporary second; - { std::ofstream used(second.path + "/events.jsonl"); used << "x\n"; } + std::filesystem::create_directories(second.path + "/journal"); + { std::ofstream used(segment_file(second, 0)); used << "x\n"; } expect(23, [&] { State state(config(second, "bars")); }); + Temporary legacy; + { std::ofstream old(legacy.path + "/events.jsonl"); } + expect(23, [&] { State state(config(legacy, "bars")); }); passed("crashed_initialization_without_cursor_is_reinitialized"); } @@ -886,6 +1011,52 @@ int main() { expect(23, [&] { make_session(state, venue, [](const auto&) {}); }); passed("aggregate_mode_needs_a_next_print_fence_venue"); } +#if PINEFORGE_FEED_SERVE + { + // The serve reader follows the journal across segments by message index, as the producer appends. + Temporary temporary; + auto options = config(temporary, "bars"); + options.segment_bytes = 4096; + options.replay_bytes = 1ULL << 30; + State state(options); + commit_bars(state, 120000, 120); + JournalReader reader(temporary.path + "/journal"); + assert(reader.seek(37) && reader.index() == 37); + std::vector minutes; + while (reader.index() < 120) minutes.push_back(normalized_bar(parse_json(*reader.next())).ts); + assert(minutes.size() == 83 && minutes.front() == 120000 + 37 * 60000 && minutes.back() == 120000 + 119 * 60000); + assert(!reader.next()); + commit_bars(state, 120000 + 120 * 60000, 60); + assert(normalized_bar(parse_json(*reader.next())).ts == 120000 + 120 * 60000); + assert(state.segments().size() > 2); + std::filesystem::remove(segment_file(temporary, 0)); + JournalReader expired(temporary.path + "/journal"); + assert(!expired.seek(3) && expired.seek(state.segments()[1].base)); + expect(22, [&] { snapshot_prefix(temporary.path + "/journal", 3, 4096); }); // index 0 expired above + Temporary small; + auto bounded = config(small, "bars"); + State fresh(bounded); + commit_bars(fresh, 120000, 40); + const auto full = snapshot_prefix(small.path + "/journal", 40, 1ULL << 22); + assert(full && std::count(full->begin(), full->end(), '\n') == 40); + assert(!snapshot_prefix(small.path + "/journal", 40, full->size() - 1) && snapshot_prefix(small.path + "/journal", 40, full->size())); + assert(listen_address("localhost:0", false) == "127.0.0.1:0" && listen_address("127.0.0.2:8787", false) == "127.0.0.2:8787"); + expect(23, [] { listen_address("0.0.0.0:8787", false); }); + assert(listen_address("0.0.0.0:8787", true) == "0.0.0.0:8787"); + for (const auto* invalid : {"8787", "127.0.0.1:", "127.0.0.1:70000", "[::1]:8787", "example.com:80"}) + expect(23, [&] { listen_address(invalid, true); }); + passed("serve_journal_reader_follows_segments_and_listen_is_loopback_by_default"); + } +#endif + // The one-minute weight ceiling is spent by the venue's own count, up to 90%, then the next window. + assert(quota_wait_ms(2000, 100, 20, 2400, 100 * 60000 + 1000) == 0); + assert(quota_wait_ms(2150, 100, 20, 2400, 100 * 60000 + 1000) == 59500); + assert(quota_wait_ms(2150, 99, 20, 2400, 100 * 60000 + 1000) == 0); + assert(quota_wait_ms(0, 0, 20, 0, 1000) == 0); + assert(binance_frame(R"({"result":["testusdt@kline_1m"],"id":1})") == Frame::Control); + assert(binance_frame(R"({"result":[],"id":1})") == Frame::Stale); + expect(23, [] { binance_frame(R"({"result":[]})"); }); + passed("rest_weight_quota_and_liveness_probe_replies"); assert(retry_after_seconds("120", 0) == 120); assert(retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT", 1445412475) == 5); assert(retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT", 1445412490) == 0); From 25844befe995a990641f961405fc1fb9af6084d6 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 08:32:15 +0800 Subject: [PATCH 02/15] Export prints-built one-minute bars for tick-mode backtests pineforge-feed export writes the bars a tick-mode runner builds from the same prints, in the warmup CSV format: per minute the first, highest, lowest and last price and the exact decimal volume sum, and a quiet minute that repeats the previous close with zero volume. The window is proven by a contiguous ID chain from the last print before --start to the first print at or after --end; anything less stops with 20 and writes nothing. Sources: venue REST within its history (USD-M aggTrades by fromId within 48 h, Binance spot historicalTrades, OKX history-trades), or a local copy of the Binance USD-M daily aggTrades archive as published: the zip is inflated in-process (no new dependency) and CRC-checked, or the extracted CSV is read, streamed either way; --checksum verifies the published CHECKSUM file's hash and name. A manifest records the source, window, fence prints, counts and the CSV's SHA-256. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/CMakeLists.txt | 8 +- native/feed/src/export.cpp | 599 ++++++++++++++++++++++++++++++ native/feed/src/export.hpp | 65 ++++ native/feed/src/main.cpp | 20 +- native/feed/tests/export_unit.cpp | 443 ++++++++++++++++++++++ native/feed/tests/mock_export.py | 340 +++++++++++++++++ 6 files changed, 1472 insertions(+), 3 deletions(-) create mode 100644 native/feed/src/export.cpp create mode 100644 native/feed/src/export.hpp create mode 100644 native/feed/tests/export_unit.cpp create mode 100644 native/feed/tests/mock_export.py diff --git a/native/feed/CMakeLists.txt b/native/feed/CMakeLists.txt index 3f462dd..110b8bd 100644 --- a/native/feed/CMakeLists.txt +++ b/native/feed/CMakeLists.txt @@ -46,7 +46,7 @@ if(PINEFORGE_FEED_SANITIZER) endif() add_library(pineforge_feed_core STATIC src/core.cpp src/state.cpp src/transport.cpp src/binance.cpp src/okx.cpp - src/bybit.cpp src/session.cpp) + src/bybit.cpp src/session.cpp src/export.cpp) target_include_directories(pineforge_feed_core PUBLIC src vendor) target_link_libraries(pineforge_feed_core PUBLIC CURL::libcurl OpenSSL::Crypto Threads::Threads) target_compile_options(pineforge_feed_core PRIVATE -Wall -Wextra -Wpedantic -Werror) @@ -91,7 +91,13 @@ if(BUILD_TESTING) endif() target_compile_options(feed-unit PRIVATE -Wall -Wextra -Wpedantic -Werror -UNDEBUG) add_test(NAME feed-unit COMMAND feed-unit) + add_executable(feed-export-unit tests/export_unit.cpp) + target_link_libraries(feed-export-unit PRIVATE pineforge_feed_core) + target_compile_options(feed-export-unit PRIVATE -Wall -Wextra -Wpedantic -Werror -UNDEBUG) + add_test(NAME feed-export-unit COMMAND feed-export-unit) find_package(Python3 REQUIRED COMPONENTS Interpreter) + add_test(NAME feed-mock-export COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/mock_export.py $) + set_tests_properties(feed-mock-export PROPERTIES TIMEOUT 300) add_test(NAME feed-mock COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/mock_venue.py $) set_tests_properties(feed-mock PROPERTIES TIMEOUT 420) endif() diff --git a/native/feed/src/export.cpp b/native/feed/src/export.cpp new file mode 100644 index 0000000..0dd66e1 --- /dev/null +++ b/native/feed/src/export.cpp @@ -0,0 +1,599 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "export.hpp" +#include "venue.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +namespace { +constexpr std::size_t csv_limit = 64 * 1024 * 1024; +constexpr std::string_view archive_header = "agg_trade_id,price,quantity,first_trade_id,last_trade_id,transact_time,is_buyer_maker"; +constexpr const char* bar_rule = "a print belongs to minute floor(ts/60000)*60000; open=first price, high=max, low=min, close=last, " + "volume=exact sum of quantities; a minute without a print has open=high=low=close=the previous close (the " + "predecessor's price before the first print) and volume 0"; +Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } +Json signed_number(std::int64_t value) { return Json::number(std::to_string(value)); } +std::string base_name(const std::string& path) { return std::filesystem::path(path).filename().string(); } +std::int64_t now_ms() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); +} +template Integer digits(std::string_view text) { + Integer result{}; + const auto parsed = std::from_chars(text.data(), text.data() + text.size(), result); + if (text.empty() || text.find_first_not_of("0123456789") != std::string_view::npos || parsed.ec != std::errc{} || + parsed.ptr != text.data() + text.size()) + throw Error(23, "archive row has an invalid unsigned integer"); + return result; +} +std::uint32_t little(const std::string& bytes, std::size_t offset, std::size_t width) { + if (offset + width > bytes.size()) throw Error(21, "zip structure is truncated"); + std::uint32_t value = 0; + for (std::size_t index = width; index > 0; --index) value = (value << 8) | static_cast(bytes[offset + index - 1]); + return value; +} +void read_at(std::ifstream& file, std::uint64_t offset, std::string& bytes) { + file.clear(); + file.seekg(static_cast(offset)); + file.read(bytes.data(), static_cast(bytes.size())); + if (static_cast(file.gcount()) != bytes.size()) throw Error(22, "cannot read the archive"); +} + +// Huffman lookup by the next 15 input bits (deflate sends codes least-significant bit first): symbol << 4 | length, +// 0 for a code the table does not assign. +using Table = std::vector; +void build(Table& table, const unsigned char* lengths, std::size_t count) { + std::array counts{}, next{}; + for (std::size_t symbol = 0; symbol < count; ++symbol) ++counts[lengths[symbol]]; + counts[0] = 0; + int left = 1; + for (unsigned int length = 1; length < 16; ++length) { + left = (left << 1) - static_cast(counts[length]); + if (left < 0) throw Error(21, "deflate Huffman code is over-subscribed"); + } + for (unsigned int length = 1, code = 0; length < 16; ++length) { + code = (code + counts[length - 1]) << 1; + next[length] = code; + } + table.assign(1U << 15, 0); + for (std::size_t symbol = 0; symbol < count; ++symbol) { + const unsigned int length = lengths[symbol]; + if (!length) continue; + unsigned int code = next[length]++, reversed = 0; + for (unsigned int bit = 0; bit < length; ++bit, code >>= 1) reversed = (reversed << 1) | (code & 1); + for (auto index = reversed; index < table.size(); index += 1U << length) + table[index] = (static_cast(symbol) << 4) | length; + } +} + +class Inflater { + const std::function& source_; + const std::function& sink_; + std::vector input_ = std::vector(1 << 16); + std::size_t position_ = 0, available_ = 0; + bool exhausted_ = false; + std::uint64_t bits_ = 0; + unsigned int count_ = 0, padding_ = 0; // padding_: zero bytes appended beyond the end of the input + std::string window_; // output not yet handed on, plus the last 32 KiB for back-references + static constexpr std::size_t flush_at = 1 << 20; + + bool byte(unsigned char& value) { + if (position_ == available_) { + if (exhausted_) return false; + available_ = source_(input_.data(), input_.size()); + position_ = 0; + if (!available_) { exhausted_ = true; return false; } + } + value = static_cast(input_[position_++]); + return true; + } + void need(unsigned int wanted) { + while (count_ < wanted) { + unsigned char value = 0; + if (!byte(value)) ++padding_; + bits_ |= static_cast(value) << count_; + count_ += 8; + } + } + void drop(unsigned int used) { + bits_ >>= used; + count_ -= used; + if (count_ < 8 * padding_) throw Error(21, "deflate stream is truncated"); + } + std::uint32_t bits(unsigned int wanted) { + if (!wanted) return 0; + need(wanted); + const auto value = static_cast(bits_ & ((1ULL << wanted) - 1)); + drop(wanted); + return value; + } + unsigned int decode(const Table& table) { + need(15); + const auto entry = table[bits_ & 0x7FFF]; + if (!(entry & 15)) throw Error(21, "deflate stream has an unassigned Huffman code"); + drop(entry & 15); + return entry >> 4; + } + void emit(std::size_t keep) { + keep = std::min(keep, window_.size()); + if (window_.size() > keep) sink_(std::string_view(window_.data(), window_.size() - keep)); + window_.erase(0, window_.size() - keep); + } + void put(char value) { + window_.push_back(value); + if (window_.size() >= flush_at) emit(32768); + } + void stored() { + drop(count_ % 8); + const auto length = bits(16), complement = bits(16); + if (length != (~complement & 0xFFFF)) throw Error(21, "deflate stored block length is corrupt"); + for (std::uint32_t index = 0; index < length; ++index) put(static_cast(bits(8))); + } + void codes(const Table& literal, const Table& distance) { + static constexpr std::uint16_t base[29] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, + 67, 83, 99, 115, 131, 163, 195, 227, 258}; + static constexpr std::uint8_t extra[29] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0}; + static constexpr std::uint16_t offset[30] = {1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, + 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577}; + static constexpr std::uint8_t spread[30] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, + 11, 11, 12, 12, 13, 13}; + for (;;) { + const auto symbol = decode(literal); + if (symbol < 256) { put(static_cast(symbol)); continue; } + if (symbol == 256) return; + if (symbol > 285) throw Error(21, "deflate stream has an invalid length code"); + const auto length = base[symbol - 257] + bits(extra[symbol - 257]); + const auto code = decode(distance); + if (code > 29) throw Error(21, "deflate stream has an invalid distance code"); + const auto back = offset[code] + bits(spread[code]); + if (back > window_.size()) throw Error(21, "deflate distance reaches before the start of the stream"); + for (std::uint32_t copied = 0; copied < length; ++copied) window_.push_back(window_[window_.size() - back]); + if (window_.size() >= flush_at) emit(32768); + } + } + void dynamic(Table& literal, Table& distance, Table& lengths_code) { + static constexpr unsigned char order[19] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + const auto literals = bits(5) + 257, distances = bits(5) + 1, header = bits(4) + 4; + if (literals > 286 || distances > 30) throw Error(21, "deflate dynamic block header is corrupt"); + unsigned char lengths[320] = {}; + for (unsigned int index = 0; index < header; ++index) lengths[order[index]] = static_cast(bits(3)); + build(lengths_code, lengths, 19); + std::fill(std::begin(lengths), std::end(lengths), 0); + for (unsigned int index = 0; index < literals + distances;) { + const auto symbol = decode(lengths_code); + if (symbol < 16) { lengths[index++] = static_cast(symbol); continue; } + unsigned char value = 0; + unsigned int repeat = 0; + if (symbol == 16) { + if (!index) throw Error(21, "deflate code lengths repeat before the first length"); + value = lengths[index - 1]; + repeat = 3 + bits(2); + } else repeat = symbol == 17 ? 3 + bits(3) : 11 + bits(7); + if (index + repeat > literals + distances) throw Error(21, "deflate code lengths overrun the header counts"); + while (repeat--) lengths[index++] = value; + } + if (!lengths[256]) throw Error(21, "deflate dynamic block has no end-of-block code"); + build(literal, lengths, literals); + build(distance, lengths + literals, distances); + } +public: + Inflater(const std::function& source, const std::function& sink) + : source_(source), sink_(sink) { window_.reserve(flush_at + 512); } + void run() { + static const auto fixed = [] { + unsigned char lengths[288 + 30]; + std::fill(lengths, lengths + 144, 8); + std::fill(lengths + 144, lengths + 256, 9); + std::fill(lengths + 256, lengths + 280, 7); + std::fill(lengths + 280, lengths + 288, 8); + std::fill(lengths + 288, lengths + 318, 5); + std::pair tables; + build(tables.first, lengths, 288); + build(tables.second, lengths + 288, 30); + return tables; + }(); + Table literal, distance, lengths_code; + for (bool last = false; !last;) { + last = bits(1) != 0; + const auto type = bits(2); + if (type == 0) stored(); + else if (type == 1) codes(fixed.first, fixed.second); + else if (type == 2) { + dynamic(literal, distance, lengths_code); + codes(literal, distance); + } else throw Error(21, "deflate stream has an invalid block type"); + } + unsigned char spare = 0; + if (count_ - 8 * padding_ >= 8 || byte(spare)) throw Error(21, "bytes follow the final deflate block"); + emit(0); + } +}; +} + +ChainBars::ChainBars(std::int64_t start, std::int64_t end) : start_(start), end_(end), next_(start), + csv_("timestamp,open,high,low,close,volume\n") { + if (start < 0 || end <= start || start % 60000 || end % 60000 || end > std::numeric_limits::max() - 60000) + throw Error(23, "export requires a nonempty minute-aligned exclusive range"); + if ((end - start) / 60000 > 100000) throw Error(22, "export range exceeds 100000 minutes"); +} +void ChainBars::anchor(const Trade& predecessor) { + predecessor.validate(); + if (predecessor_) throw Error(23, "the print chain is already anchored"); + if (predecessor.ts >= start_) throw Error(20, "the predecessor print is not strictly before the start"); + predecessor_ = predecessor; + last_id_ = predecessor.id; + last_ts_ = predecessor.ts; + close_ = predecessor.price; +} +bool ChainBars::add(const Trade& trade) { + if (!predecessor_ || fence_) throw Error(23, "a print chain takes its predecessor first and nothing after its fence"); + trade.validate(); + if (last_id_ == std::numeric_limits::max() || trade.id != last_id_ + 1) + throw Error(20, "the print chain has a hole after ID " + std::to_string(last_id_) + " (next print is " + std::to_string(trade.id) + ")"); + if (trade.ts < last_ts_) throw Error(23, "print time goes backwards along the ID chain at ID " + std::to_string(trade.id)); + if (trade.ts < start_) throw Error(20, "a print before the start follows the predecessor: the predecessor is not the last one"); + last_id_ = trade.id; + last_ts_ = trade.ts; + if (trade.ts >= end_) { + flush(); + carry(end_); + fence_ = trade; + return true; + } + const auto minute = trade.ts - trade.ts % 60000; + if (minute != minute_) { + flush(); + carry(minute); + minute_ = minute; + } + const Decimal price(trade.price); + if (!count_) { + open_ = high_ = low_ = trade.price; + high_value_ = low_value_ = price; + volume_ = Decimal("0"); + } else { + if (price.compare(high_value_) > 0) { high_ = trade.price; high_value_ = price; } + if (price.compare(low_value_) < 0) { low_ = trade.price; low_value_ = price; } + } + close_ = trade.price; + volume_ = volume_.add(Decimal(trade.qty)); + ++count_; + ++prints_; + return false; +} +void ChainBars::flush() { + if (!count_) return; + row(minute_, open_, high_, low_, close_, volume_.str()); + count_ = 0; + next_ = minute_ + 60000; +} +void ChainBars::carry(std::int64_t until) { + for (; next_ < until; next_ += 60000, ++quiet_) row(next_, close_, close_, close_, close_, "0"); +} +void ChainBars::row(std::int64_t minute, const std::string& open, const std::string& high, const std::string& low, + const std::string& close, const std::string& volume) { + csv_ += std::to_string(minute) + ',' + open + ',' + high + ',' + low + ',' + close + ',' + volume + '\n'; + ++bars_; + if (csv_.size() > csv_limit) throw Error(22, "export output exceeds 64 MiB"); +} +const std::string& ChainBars::csv() const { + if (!fence_) throw Error(20, "the window's fence (the first print at or after its end) is not proven"); + return csv_; +} +const Trade& ChainBars::predecessor() const { + if (!predecessor_) throw Error(20, "the window's predecessor print is not proven"); + return *predecessor_; +} +const Trade& ChainBars::fence() const { + if (!fence_) throw Error(20, "the window's fence (the first print at or after its end) is not proven"); + return *fence_; +} + +std::optional archive_row(std::string_view line, bool first_line) { + if (!line.empty() && line.back() == '\r') line.remove_suffix(1); + if (first_line && !line.empty() && (line.front() < '0' || line.front() > '9')) { + if (line != archive_header) throw Error(23, "unexpected archive header; expected " + std::string(archive_header)); + return std::nullopt; + } + std::array fields; + std::size_t count = 0; + for (std::size_t begin = 0;;) { + const auto comma = line.find(',', begin); + if (count == fields.size()) throw Error(23, "archive row must have seven fields"); + fields[count++] = line.substr(begin, comma == std::string_view::npos ? std::string_view::npos : comma - begin); + if (comma == std::string_view::npos) break; + begin = comma + 1; + } + if (count != fields.size()) throw Error(23, "archive row must have seven fields"); + const auto id = digits(fields[0]), first = digits(fields[3]), last = digits(fields[4]); + auto time = digits(fields[5]); + if (!first || last < first) throw Error(23, "invalid aggregate raw-trade range"); + // Binance moved its archives' transact_time to microseconds; no millisecond time reaches 10^14 before 5138. + if (time > 100000000000000LL) time /= 1000; + if (fields[6] != "true" && fields[6] != "false" && fields[6] != "True" && fields[6] != "False") + throw Error(23, "archive is_buyer_maker must be true or false"); + Trade trade{id, time, std::string(fields[1]), std::string(fields[2])}; + trade.validate(); + return trade; +} + +std::uint32_t crc32(std::uint32_t crc, std::string_view bytes) { + static const auto table = [] { + std::array values{}; + for (std::uint32_t index = 0; index < 256; ++index) { + auto value = index; + for (int bit = 0; bit < 8; ++bit) value = (value & 1) ? 0xEDB88320U ^ (value >> 1) : value >> 1; + values[index] = value; + } + return values; + }(); + crc = ~crc; + for (const auto letter : bytes) crc = table[(crc ^ static_cast(letter)) & 0xFF] ^ (crc >> 8); + return ~crc; +} + +void inflate(const std::function& source, const std::function& sink) { + Inflater(source, sink).run(); +} + +std::string unzip(const std::string& path, const std::function& sink) { + std::ifstream file(path, std::ios::binary); + if (!file) throw Error(23, "cannot open the archive " + path); + file.seekg(0, std::ios::end); + const auto end = file.tellg(); + if (end < 0) throw Error(22, "cannot size the archive"); + const auto size = static_cast(end); + // The end-of-central-directory record: 22 bytes, then a comment of at most 65535 bytes. + std::string tail(static_cast(std::min(size, 22 + 65535)), '\0'); + read_at(file, size - tail.size(), tail); + std::size_t record = std::string::npos; + for (auto index = tail.size() >= 22 ? tail.size() - 21 : 0; index-- > 0;) + if (little(tail, index, 4) == 0x06054b50 && index + 22 + little(tail, index + 20, 2) == tail.size()) { record = index; break; } + if (record == std::string::npos) throw Error(23, "the archive is not a zip file (no end-of-central-directory record)"); + if (little(tail, record + 4, 2) || little(tail, record + 6, 2) || little(tail, record + 8, 2) != 1 || little(tail, record + 10, 2) != 1) + throw Error(23, "the zip archive must be a single-disk archive holding exactly one entry"); + const std::uint64_t directory_size = little(tail, record + 12, 4), directory_offset = little(tail, record + 16, 4); + if (directory_size == 0xFFFFFFFF || directory_offset == 0xFFFFFFFF) + throw Error(23, "ZIP64 archives are not supported; pass the extracted .csv"); + if (directory_offset + directory_size > size - tail.size() + record) throw Error(21, "zip central directory is out of bounds"); + std::string directory(static_cast(directory_size), '\0'); + read_at(file, directory_offset, directory); + if (little(directory, 0, 4) != 0x02014b50) throw Error(21, "zip central directory entry is corrupt"); + const auto flags = little(directory, 8, 2), method = little(directory, 10, 2), crc = little(directory, 16, 4); + const std::uint64_t compressed = little(directory, 20, 4), uncompressed = little(directory, 24, 4), local = little(directory, 42, 4); + const auto name_size = little(directory, 28, 2); + if (46 + name_size + little(directory, 30, 2) + little(directory, 32, 2) != directory.size()) + throw Error(21, "zip central directory entry does not fill the directory"); + const auto name = directory.substr(46, name_size); + if (flags & 1) throw Error(23, "encrypted zip entries are not supported"); + if (method != 0 && method != 8) throw Error(23, "the zip entry must be stored or deflated"); + if (compressed == 0xFFFFFFFF || uncompressed == 0xFFFFFFFF || local == 0xFFFFFFFF) + throw Error(23, "ZIP64 archives are not supported; pass the extracted .csv"); + if (local + 30 > directory_offset) throw Error(21, "zip local header is out of bounds"); + std::string header(30, '\0'); + read_at(file, local, header); + if (little(header, 0, 4) != 0x04034b50 || little(header, 8, 2) != method) throw Error(21, "zip local header is corrupt"); + const auto data = local + 30 + little(header, 26, 2) + little(header, 28, 2); + if (data + compressed > directory_offset) throw Error(21, "zip entry data is out of bounds"); + if (method == 0 && compressed != uncompressed) throw Error(21, "stored zip entry sizes disagree"); + file.clear(); + file.seekg(static_cast(data)); + std::uint64_t remaining = compressed, produced = 0; + std::uint32_t check = 0; + const std::function source = [&](char* buffer, std::size_t limit) -> std::size_t { + const auto wanted = static_cast(std::min(remaining, limit)); + if (!wanted) return 0; + file.read(buffer, static_cast(wanted)); + if (static_cast(file.gcount()) != wanted) throw Error(22, "cannot read the zip entry"); + remaining -= wanted; + return wanted; + }; + const std::function output = [&](std::string_view bytes) { + produced += bytes.size(); + if (produced > uncompressed) throw Error(21, "zip entry inflates beyond its recorded size"); + check = crc32(check, bytes); + sink(bytes); + }; + if (method == 8) inflate(source, output); + else { + std::vector buffer(1 << 20); + while (const auto got = source(buffer.data(), buffer.size())) output(std::string_view(buffer.data(), got)); + } + if (produced != uncompressed || check != crc) throw Error(21, "zip entry CRC-32 or size mismatch: the archive is corrupt"); + return name; +} + +std::string file_sha256(const std::string& path) { + std::ifstream file(path, std::ios::binary); + if (!file) throw Error(23, "cannot open " + path); + struct Digest { + EVP_MD_CTX* context = EVP_MD_CTX_new(); + ~Digest() { EVP_MD_CTX_free(context); } + } digest; + if (!digest.context || EVP_DigestInit_ex(digest.context, EVP_sha256(), nullptr) != 1) throw Error(23, "SHA-256 failed"); + std::vector buffer(1 << 20); + while (file) { + file.read(buffer.data(), static_cast(buffer.size())); + const auto got = static_cast(file.gcount()); + if (got && EVP_DigestUpdate(digest.context, buffer.data(), got) != 1) throw Error(23, "SHA-256 failed"); + } + if (file.bad()) throw Error(22, "cannot read " + path); + unsigned char value[EVP_MAX_MD_SIZE]; + unsigned int size = 0; + if (EVP_DigestFinal_ex(digest.context, value, &size) != 1 || size != 32) throw Error(23, "SHA-256 failed"); + constexpr char hex[] = "0123456789abcdef"; + std::string result; + for (unsigned int index = 0; index < size; ++index) { + result += hex[value[index] >> 4]; + result += hex[value[index] & 15]; + } + return result; +} + +void verify_checksum(const std::string& checksum, const std::string& archive, const std::string& digest) { + std::ifstream file(checksum, std::ios::binary); + if (!file) throw Error(23, "cannot open the checksum file " + checksum); + std::string text(4097, '\0'); + file.read(text.data(), static_cast(text.size())); + text.resize(static_cast(file.gcount())); + if (text.size() > 4096) throw Error(23, "the checksum file exceeds 4096 bytes"); + while (!text.empty() && (text.back() == '\n' || text.back() == '\r')) text.pop_back(); + if (text.size() < 66 || text.find('\n') != std::string::npos || text[64] != ' ' || + text.find_first_not_of("0123456789abcdefABCDEF") != 64) + throw Error(23, "the checksum file must hold one line ` `"); + auto hash = text.substr(0, 64); + std::transform(hash.begin(), hash.end(), hash.begin(), [](unsigned char letter) { return static_cast(std::tolower(letter)); }); + const auto name = text.substr(text[65] == ' ' || text[65] == '*' ? 66 : 65); + if (name != base_name(archive)) throw Error(21, "the checksum file names " + name + ", not the archive " + base_name(archive)); + if (hash != digest) throw Error(21, "the archive's SHA-256 does not match its checksum file"); +} + +ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end) { + ChainBars chain(start, end); + std::optional candidate; + std::exception_ptr failure; + std::string pending; + bool first = true; + const auto line = [&](std::string_view text) { + const auto row = archive_row(text, first); + first = false; + if (!row) return; + if (!chain.anchored()) { + if (row->ts < start) { candidate = *row; return; } + if (!candidate) throw Error(20, "the archive holds no print before the start: the window's predecessor is not inside it"); + chain.anchor(*candidate); + } + chain.add(*row); + }; + // Rows stop counting at the fence or the first failure; a zip still inflates to its end so that its CRC-32 is + // checked first: a corrupt archive stops 21, not with whatever its damaged rows would have caused. + const std::function bytes = [&](std::string_view chunk) { + if (failure || chain.fenced()) return; + try { + while (!chunk.empty() && !chain.fenced()) { + const auto newline = chunk.find('\n'); + if (newline == std::string_view::npos) { + if (pending.size() + chunk.size() > 4096) throw Error(23, "archive line exceeds 4096 bytes"); + pending.append(chunk); + return; + } + if (pending.empty()) line(chunk.substr(0, newline)); + else { + pending.append(chunk.substr(0, newline)); + line(pending); + pending.clear(); + } + chunk.remove_prefix(newline + 1); + } + } catch (const Error&) { failure = std::current_exception(); } + }; + const auto name = base_name(archive); + const bool zipped = name.size() > 4 && (name.compare(name.size() - 4, 4, ".zip") == 0 || name.compare(name.size() - 4, 4, ".ZIP") == 0); + if (zipped) { + const auto entry = unzip(archive, bytes); + log("info", "export_archive_entry_verified", Json::object({{"entry", Json::string(entry)}})); + } else { + std::ifstream file(archive, std::ios::binary); + if (!file) throw Error(23, "cannot open the archive " + archive); + std::vector buffer(1 << 20); + while (file && !failure && !chain.fenced()) { + file.read(buffer.data(), static_cast(buffer.size())); + bytes(std::string_view(buffer.data(), static_cast(file.gcount()))); + } + if (file.bad()) throw Error(22, "cannot read the archive " + archive); + } + if (!failure && !chain.fenced() && !pending.empty()) { + try { line(pending); } catch (const Error&) { failure = std::current_exception(); } + } + if (failure) std::rethrow_exception(failure); + if (!chain.anchored()) throw Error(20, "the archive holds no print at or after the start: the window is not inside it"); + if (!chain.fenced()) throw Error(20, "the archive ends before the window's fence (the first print at or after the end)"); + return chain; +} + +namespace { +ChainBars rest_bars(const Config& config, std::string& multiplier) { + ChainBars chain(config.start, config.end); + const auto now = now_ms(); + if (config.end > now) throw Error(20, "the window has not closed yet: no print at or after its end can exist"); + const auto venue = make_venue(config); + multiplier = venue->qty_multiplier(); + const auto window = venue->retention_ms(); + if (window && now - config.start > window) + throw Error(20, "the export start is beyond the venue's REST print history window (" + std::to_string(window / 3600000) + + " h); use --archive for older windows"); + chain.anchor(venue->predecessor(config.start)); + log("info", "export_anchored", Json::object({{"predecessor_id", number(chain.predecessor().id)}, + {"predecessor_ts", signed_number(chain.predecessor().ts)}})); + // OKX history-trades pages hold 100 prints; Binance pages hold 1000. + const std::size_t limit = config.venue == "okx" ? 100 : 1000; + for (std::uint64_t pages = 1; !chain.fenced(); ++pages) { + const auto page = venue->history(chain.next_id(), limit); + if (page.empty()) throw Error(20, "no print at or after the window end is available yet: its fence cannot be proven"); + for (const auto& trade : page) + if (chain.add(trade)) break; + if (pages % 20 == 0 || chain.fenced()) + log("info", "export_progress", Json::object({{"pages", number(pages)}, {"prints", number(chain.prints())}, + {"next_id", number(chain.next_id())}, {"ts", signed_number(chain.last_ts())}})); + } + return chain; +} +} + +void export_bars(const Config& config, const ExportOptions& options) { + if (!tick_mode(config.mode)) throw Error(23, "export builds bars from prints: use --mode ticks or agg-ticks"); + const auto manifest_path = options.output + ".manifest.json"; + const auto refuse_existing = [&] { + if (options.output.empty() || std::filesystem::exists(options.output) || std::filesystem::exists(manifest_path)) + throw Error(23, "export output already exists (or is empty)"); + }; + refuse_existing(); + (void)ChainBars(config.start, config.end); // the range and its budget are checked before any network or file access + std::string multiplier = "1"; + Json source; + std::optional chain; + if (!options.archive.empty()) { + if (config.venue != "binance" || config.market != "usdm" || config.mode != "agg-ticks") + throw Error(23, "--archive reads the Binance USD-M daily aggTrades archive: use --venue binance --market usdm --mode agg-ticks"); + const auto digest = file_sha256(options.archive); + const bool verified = !options.checksum.empty(); + const auto file = Json::object({{"file", Json::string(base_name(options.archive))}, {"sha256", Json::string(digest)}}); + if (verified) verify_checksum(options.checksum, options.archive, digest); + else log("warn", "archive_not_checksum_verified", file); + if (base_name(options.archive).rfind(config.symbol + "-aggTrades-", 0) != 0) log("warn", "archive_name_not_symbol", file); + chain = archive_bars(options.archive, config.start, config.end); + source = Json::object({{"kind", Json::string("archive")}, {"file", Json::string(base_name(options.archive))}, + {"sha256", Json::string(digest)}, {"checksum_verified", Json::boolean(verified)}}); + } else { + chain = rest_bars(config, multiplier); + source = Json::object({{"kind", Json::string("rest")}, {"origin", Json::string(config.rest_url)}}); + } + const auto& csv = chain->csv(); + const auto& predecessor = chain->predecessor(); + const auto& fence = chain->fence(); + const bool any = chain->prints() != 0; + const auto manifest = Json::object({{"schema", Json::string("pineforge-feed-export/v1")}, + {"venue", Json::string(config.venue)}, {"market", Json::string(config.market)}, {"symbol", Json::string(config.symbol)}, + {"mode", Json::string(config.mode)}, {"source", source}, {"interval", Json::string("1m")}, + {"qty_multiplier", Json::string(multiplier)}, + {"start", signed_number(config.start)}, {"end_exclusive", signed_number(config.end)}, + {"predecessor", Json::object({{"id", number(predecessor.id)}, {"ts", signed_number(predecessor.ts)}, + {"price", Json::string(predecessor.price)}})}, + {"fence", Json::object({{"id", number(fence.id)}, {"ts", signed_number(fence.ts)}})}, + {"first_id", any ? number(predecessor.id + 1) : Json{}}, {"last_id", any ? number(fence.id - 1) : Json{}}, + {"prints", number(chain->prints())}, {"bars", number(chain->bars())}, {"quiet_minutes", number(chain->quiet_minutes())}, + {"rule", Json::string(bar_rule)}, {"sha256", Json::string(sha256(csv))}}); + refuse_existing(); + atomic_file(options.output, csv); + atomic_file(manifest_path, manifest.dump() + '\n'); + log("info", "export_written", manifest); +} +} diff --git a/native/feed/src/export.hpp b/native/feed/src/export.hpp new file mode 100644 index 0000000..3c15c05 --- /dev/null +++ b/native/feed/src/export.hpp @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "core.hpp" +#include +#include + +namespace pineforge::feed { +struct ExportOptions { + std::string output; // the warmup-format CSV to write; refused if it exists + std::string archive; // optional: a venue daily aggregate-trade archive (.zip or extracted .csv) + std::string checksum; // optional: the archive's published .CHECKSUM file +}; +// Prints-built one-minute bars over [config.start, config.end), by the runner's tick-built bar rule, from venue +// REST within its retention or from a local daily archive. +void export_bars(const Config& config, const ExportOptions& options); + +// The runner's tick-built bar rule over [start, end) on a proven print chain: anchor() takes the predecessor (the +// last print strictly before start), add() every later print in ID order until it returns true at the fence (the +// first print at or after end). A minute is first/max/min/last price and the exact volume sum; a minute without a +// print repeats the previous close with volume 0. A hole stops 20, a time regression along the chain 23. +class ChainBars { + std::int64_t start_, end_, minute_ = -1, next_, last_ts_ = -1; + std::uint64_t last_id_ = 0, count_ = 0, prints_ = 0, bars_ = 0, quiet_ = 0; + std::optional predecessor_, fence_; + std::string open_, high_, low_, close_, csv_; + Decimal high_value_{"0"}, low_value_{"0"}, volume_{"0"}; + void flush(); + void carry(std::int64_t until); + void row(std::int64_t minute, const std::string& open, const std::string& high, const std::string& low, + const std::string& close, const std::string& volume); +public: + ChainBars(std::int64_t start, std::int64_t end); + void anchor(const Trade& predecessor); + bool add(const Trade& trade); + bool anchored() const { return predecessor_.has_value(); } + bool fenced() const { return fence_.has_value(); } + std::uint64_t next_id() const { return last_id_ + 1; } + std::int64_t last_ts() const { return last_ts_; } + // The warmup CSV (header plus one row per minute); 20 until the fence is in hand. + const std::string& csv() const; + const Trade& predecessor() const; + const Trade& fence() const; + std::uint64_t prints() const { return prints_; } + std::uint64_t bars() const { return bars_; } + std::uint64_t quiet_minutes() const { return quiet_; } +}; +// One daily-archive line, columns agg_trade_id,price,quantity,first_trade_id,last_trade_id,transact_time, +// is_buyer_maker: nullopt for the header, which only the first line may be. A transact_time above 10^14 is +// microseconds and becomes floor(us / 1000) milliseconds. +std::optional archive_row(std::string_view line, bool first_line); +// CRC-32 (ISO-HDLC, as zip uses) of `bytes` continued from `crc`; start from 0. +std::uint32_t crc32(std::uint32_t crc, std::string_view bytes); +// RFC 1951 raw deflate: pulls input from `source` (0 at its end), pushes output to `sink`. Corrupt or truncated +// input, or bytes after the final block, stop 21. +void inflate(const std::function& source, const std::function& sink); +// Streams the only entry of a stored or deflated zip to `sink`, then checks its CRC-32 and size (21). Returns +// the entry name. +std::string unzip(const std::string& path, const std::function& sink); +std::string file_sha256(const std::string& path); +// A published ` ` checksum must name the archive's base name and carry its SHA-256 (21). +void verify_checksum(const std::string& checksum, const std::string& archive, const std::string& digest); +// The window's chain from a daily aggregate-trade archive (.zip, else an extracted .csv), streamed: only the +// predecessor candidate, the window's bars and the fence are kept. +ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end); +} diff --git a/native/feed/src/main.cpp b/native/feed/src/main.cpp index 9ef1c9d..7347507 100644 --- a/native/feed/src/main.cpp +++ b/native/feed/src/main.cpp @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +#include "export.hpp" #include "session.hpp" #include "transport.hpp" #if PINEFORGE_FEED_SERVE @@ -28,6 +29,8 @@ struct StdoutFlags { }; void help() { std::cout << "pineforge-feed warmup --venue VENUE --market MARKET --symbol SYMBOL --start UTC|MS --end UTC|MS --output FILE\n" + "pineforge-feed export --venue VENUE --market MARKET --symbol SYMBOL --mode ticks|agg-ticks --start UTC|MS --end UTC|MS --output FILE\n" + " [--archive DAILY.zip|DAILY.csv [--checksum DAILY.zip.CHECKSUM]] (prints-built 1m bars)\n" "pineforge-feed run --venue VENUE --market MARKET --symbol SYMBOL --mode MODE --state-dir DIR [--start UTC|MS | --resume] [--output-from INDEX]\n" "pineforge-feed serve --venue VENUE --market MARKET --symbol SYMBOL --mode MODE --state-dir DIR [--start UTC|MS | --resume]\n" " [--listen HOST:PORT (default 127.0.0.1:8787)] [--allow-remote-listen] [--client-queue-bytes N] [--max-clients N]\n" @@ -95,9 +98,11 @@ int run(int argc, char** argv) { if (argc == 2 && std::string(argv[1]) == "--version") { check_runtime_curl(); std::cout << "pineforge-feed 0.1.0\n"; return 0; } if (argc < 2) throw Error(23, "choose warmup or run; see --help"); const std::string command = argv[1]; - if (command != "run" && command != "warmup" && command != "serve") throw Error(23, "choose warmup, run or serve; see --help"); + if (command != "run" && command != "warmup" && command != "serve" && command != "export") + throw Error(23, "choose warmup, export, run or serve; see --help"); Config config; std::string output; + ExportOptions exported; std::set seen; for (int index = 2; index < argc; ++index) { const std::string key = argv[index]; @@ -115,6 +120,8 @@ int run(int argc, char** argv) { else if (key == "--start") config.start = timestamp(value); else if (key == "--end") config.end = timestamp(value); else if (key == "--output") output = value; + else if (key == "--archive") exported.archive = value; + else if (key == "--checksum") exported.checksum = value; else if (key == "--output-from") config.output_from = unsigned_value(value); else if (key == "--max-messages") config.max_messages = unsigned_value(value); else if (key == "--segment-bytes") config.segment_bytes = unsigned_value(value); @@ -137,6 +144,8 @@ int run(int argc, char** argv) { else throw Error(23, "unknown CLI option: " + key); } gate(config, command == "warmup", seen.count("--rest-url") != 0, seen.count("--ws-url") != 0); + if (command != "export" && (!exported.archive.empty() || !exported.checksum.empty())) + throw Error(23, "--archive and --checksum are export options"); if (!config.max_queue_bytes || !config.max_replay_seconds || config.max_replay_seconds > 3600 || !config.reconnect_seconds || config.reconnect_seconds > 86100) throw Error(23, "budgets must be positive; reconnect must precede the 24-hour connection limit"); @@ -151,7 +160,14 @@ int run(int argc, char** argv) { check_runtime_curl(); validate_origin(config.rest_url, false, config.allow_insecure); validate_origin(config.ws_url, true, config.allow_insecure); - if (command == "warmup") { + if (command == "export") { + if (output.empty() || config.resume || !config.state_dir.empty() || seen.count("--output-from") || !tick_mode(config.mode) || + config.start < 0 || config.end < 0) + throw Error(23, "export takes --mode ticks|agg-ticks, --start, --end and --output (plus --archive [--checksum])"); + if (!exported.checksum.empty() && exported.archive.empty()) throw Error(23, "--checksum verifies an --archive"); + exported.output = output; + export_bars(config, exported); + } else if (command == "warmup") { if (output.empty() || config.resume || !config.state_dir.empty() || seen.count("--output-from") || seen.count("--mode")) throw Error(23, "warmup requires --output and does not accept run-state options"); warmup(config, output); diff --git a/native/feed/tests/export_unit.cpp b/native/feed/tests/export_unit.cpp new file mode 100644 index 0000000..73fd16d --- /dev/null +++ b/native/feed/tests/export_unit.cpp @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "export.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge::feed; +namespace { +// Made offline with Python's zipfile and zlib from the synthetic rows below: a deflated (dynamic-Huffman) and a +// stored daily archive, and one raw deflate stream of generated() in a fixed, a stored and a dynamic block. +// 362 bytes +const unsigned char deflated_zip[] = { + 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x41, 0x5d, 0xdf, 0xdf, 0x4a, 0xd9, + 0xc6, 0x00, 0x00, 0x00, 0x99, 0x01, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x54, 0x45, 0x53, 0x54, 0x55, 0x53, + 0x44, 0x54, 0x2d, 0x61, 0x67, 0x67, 0x54, 0x72, 0x61, 0x64, 0x65, 0x73, 0x2d, 0x31, 0x39, 0x37, 0x30, 0x2d, + 0x30, 0x31, 0x2d, 0x30, 0x31, 0x2e, 0x63, 0x73, 0x76, 0x4d, 0x8e, 0xd1, 0x0e, 0x82, 0x30, 0x0c, 0x45, 0xdf, + 0xfd, 0x96, 0x86, 0xb4, 0x63, 0x03, 0xfb, 0x35, 0x64, 0xea, 0x20, 0x8b, 0x4a, 0x74, 0x8c, 0x07, 0xff, 0xde, + 0x76, 0x30, 0xe5, 0x61, 0x77, 0x6d, 0xef, 0xc9, 0x6d, 0xfd, 0x34, 0x0d, 0x39, 0xf9, 0x5b, 0x18, 0xe2, 0x0d, + 0x5e, 0x29, 0x5e, 0x03, 0xbc, 0x57, 0x3f, 0xe7, 0x98, 0x3f, 0x30, 0xc6, 0xb4, 0xe4, 0xbf, 0xfb, 0xf0, 0xc7, + 0x4e, 0x8a, 0x79, 0xf1, 0x57, 0x99, 0xc4, 0x67, 0x80, 0xb8, 0x0c, 0x97, 0xf5, 0x13, 0xd2, 0xf0, 0xf4, 0xf7, + 0x90, 0x4e, 0x96, 0xcf, 0x40, 0xd8, 0x20, 0x02, 0x36, 0x0e, 0x98, 0x3b, 0x79, 0x3d, 0x10, 0x31, 0xca, 0x28, + 0xa7, 0x35, 0x08, 0xc1, 0x4a, 0x90, 0x12, 0x24, 0xee, 0x59, 0x1e, 0x2b, 0xa1, 0x5f, 0x21, 0x9c, 0xa0, 0x7f, + 0x82, 0xb0, 0xb4, 0x68, 0x81, 0x8c, 0x94, 0x04, 0xa3, 0x7f, 0x2c, 0x0a, 0x89, 0x45, 0x8d, 0x51, 0xc8, 0xa8, + 0xef, 0x76, 0x11, 0xc8, 0x61, 0x0d, 0x32, 0xc0, 0x0d, 0x2b, 0xd2, 0xaa, 0xdb, 0xa9, 0xc8, 0x7d, 0x7d, 0xd9, + 0x55, 0x73, 0xc4, 0x32, 0xdb, 0xc1, 0x56, 0x7d, 0xde, 0xc4, 0xd8, 0xb2, 0x6c, 0xcf, 0xb1, 0xba, 0x0b, 0xeb, + 0x41, 0x84, 0x9b, 0x28, 0x23, 0xbb, 0x6a, 0x90, 0xfb, 0x05, 0x15, 0x68, 0x97, 0x16, 0x8f, 0x41, 0x9d, 0x32, + 0x54, 0x8f, 0xa6, 0x5d, 0x94, 0xf9, 0x05, 0x7d, 0x01, 0x50, 0x4b, 0x01, 0x02, 0x14, 0x03, 0x14, 0x00, 0x00, + 0x00, 0x08, 0x00, 0x00, 0x00, 0x41, 0x5d, 0xdf, 0xdf, 0x4a, 0xd9, 0xc6, 0x00, 0x00, 0x00, 0x99, 0x01, 0x00, + 0x00, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x54, 0x45, 0x53, 0x54, 0x55, 0x53, 0x44, 0x54, 0x2d, 0x61, 0x67, 0x67, 0x54, 0x72, 0x61, 0x64, 0x65, + 0x73, 0x2d, 0x31, 0x39, 0x37, 0x30, 0x2d, 0x30, 0x31, 0x2d, 0x30, 0x31, 0x2e, 0x63, 0x73, 0x76, 0x50, 0x4b, + 0x05, 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x05, 0x01, 0x00, 0x00, + 0x00, 0x00, +}; +// 487 bytes +const unsigned char stored_zip[] = { + 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x5d, 0x42, 0x8a, 0xb2, 0xcf, + 0x43, 0x01, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x54, 0x45, 0x53, 0x54, 0x55, 0x53, + 0x44, 0x54, 0x2d, 0x61, 0x67, 0x67, 0x54, 0x72, 0x61, 0x64, 0x65, 0x73, 0x2d, 0x31, 0x39, 0x37, 0x30, 0x2d, + 0x30, 0x31, 0x2d, 0x30, 0x31, 0x2e, 0x63, 0x73, 0x76, 0x34, 0x39, 0x38, 0x2c, 0x31, 0x30, 0x2e, 0x30, 0x30, + 0x2c, 0x30, 0x2e, 0x35, 0x2c, 0x39, 0x39, 0x36, 0x2c, 0x39, 0x39, 0x37, 0x2c, 0x31, 0x31, 0x39, 0x30, 0x30, + 0x30, 0x2c, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x34, 0x39, 0x39, 0x2c, 0x31, 0x30, 0x2e, 0x31, 0x30, 0x2c, 0x30, + 0x2e, 0x31, 0x2c, 0x39, 0x39, 0x38, 0x2c, 0x39, 0x39, 0x39, 0x2c, 0x31, 0x31, 0x39, 0x39, 0x39, 0x39, 0x2c, + 0x74, 0x72, 0x75, 0x65, 0x0a, 0x35, 0x30, 0x30, 0x2c, 0x31, 0x30, 0x2e, 0x31, 0x30, 0x2c, 0x30, 0x2e, 0x31, + 0x2c, 0x31, 0x30, 0x30, 0x30, 0x2c, 0x31, 0x30, 0x30, 0x34, 0x2c, 0x31, 0x32, 0x30, 0x30, 0x30, 0x31, 0x2c, + 0x66, 0x61, 0x6c, 0x73, 0x65, 0x0a, 0x35, 0x30, 0x31, 0x2c, 0x31, 0x31, 0x2e, 0x32, 0x30, 0x2c, 0x30, 0x2e, + 0x32, 0x2c, 0x31, 0x30, 0x30, 0x35, 0x2c, 0x31, 0x30, 0x30, 0x35, 0x2c, 0x31, 0x32, 0x30, 0x30, 0x35, 0x30, + 0x2c, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x35, 0x30, 0x32, 0x2c, 0x39, 0x2e, 0x39, 0x30, 0x2c, 0x30, 0x2e, 0x33, + 0x2c, 0x31, 0x30, 0x30, 0x36, 0x2c, 0x31, 0x30, 0x30, 0x38, 0x2c, 0x31, 0x37, 0x39, 0x39, 0x39, 0x39, 0x2c, + 0x66, 0x61, 0x6c, 0x73, 0x65, 0x0a, 0x35, 0x30, 0x33, 0x2c, 0x31, 0x32, 0x2e, 0x30, 0x30, 0x2c, 0x30, 0x2e, + 0x34, 0x2c, 0x31, 0x30, 0x30, 0x39, 0x2c, 0x31, 0x30, 0x30, 0x39, 0x2c, 0x32, 0x34, 0x30, 0x30, 0x30, 0x31, + 0x2c, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x35, 0x30, 0x34, 0x2c, 0x31, 0x31, 0x2e, 0x30, 0x30, 0x2c, 0x30, 0x2e, + 0x31, 0x2c, 0x31, 0x30, 0x31, 0x30, 0x2c, 0x31, 0x30, 0x31, 0x30, 0x2c, 0x32, 0x34, 0x30, 0x30, 0x35, 0x30, + 0x2c, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x0a, 0x35, 0x30, 0x35, 0x2c, 0x31, 0x32, 0x2e, 0x30, 0x30, 0x2c, 0x30, + 0x2e, 0x31, 0x2c, 0x31, 0x30, 0x31, 0x31, 0x2c, 0x31, 0x30, 0x31, 0x31, 0x2c, 0x33, 0x30, 0x30, 0x30, 0x30, + 0x31, 0x2c, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x35, 0x30, 0x36, 0x2c, 0x31, 0x32, 0x2e, 0x31, 0x30, 0x2c, 0x30, + 0x2e, 0x32, 0x2c, 0x31, 0x30, 0x31, 0x32, 0x2c, 0x31, 0x30, 0x31, 0x32, 0x2c, 0x33, 0x30, 0x30, 0x30, 0x35, + 0x30, 0x2c, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x0a, 0x50, 0x4b, 0x01, 0x02, 0x14, 0x03, 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x41, 0x5d, 0x42, 0x8a, 0xb2, 0xcf, 0x43, 0x01, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, + 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x54, 0x45, 0x53, 0x54, 0x55, 0x53, 0x44, 0x54, 0x2d, 0x61, 0x67, 0x67, 0x54, 0x72, 0x61, 0x64, 0x65, 0x73, + 0x2d, 0x31, 0x39, 0x37, 0x30, 0x2d, 0x30, 0x31, 0x2d, 0x30, 0x31, 0x2e, 0x63, 0x73, 0x76, 0x50, 0x4b, 0x05, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x82, 0x01, 0x00, 0x00, 0x00, + 0x00, +}; +// 925 bytes +const unsigned char mixed_deflate[] = { + 0x32, 0x34, 0x30, 0x30, 0xd0, 0x31, 0x34, 0xd0, 0x03, 0x92, 0x20, 0xc2, 0x50, 0xc7, 0x14, 0x24, 0x60, 0x0a, + 0x62, 0x19, 0x1a, 0x19, 0x80, 0xd8, 0x69, 0x89, 0x39, 0xc5, 0xa9, 0x5c, 0x86, 0x60, 0x11, 0x73, 0x3d, 0x63, + 0x73, 0xb0, 0x3a, 0x23, 0x90, 0x12, 0x30, 0x61, 0x0c, 0x54, 0x67, 0x08, 0xa4, 0x75, 0x4a, 0x8a, 0x4a, 0xc1, + 0xca, 0x8c, 0x74, 0x0c, 0x0d, 0xf5, 0xcc, 0x4d, 0xc0, 0xca, 0x8c, 0x41, 0x2a, 0x4c, 0x40, 0x84, 0x29, 0x50, + 0x99, 0x31, 0x8a, 0x71, 0x40, 0x8d, 0x16, 0x7a, 0x86, 0x86, 0x60, 0x75, 0x60, 0x25, 0x66, 0x20, 0xc2, 0x1c, + 0xa8, 0xce, 0x04, 0xd9, 0x38, 0x13, 0xa0, 0x80, 0x9e, 0x89, 0x05, 0x58, 0x99, 0x29, 0x48, 0x85, 0x05, 0x88, + 0xb0, 0x04, 0x8a, 0x9a, 0xa1, 0x18, 0x07, 0xb4, 0xc0, 0x52, 0xcf, 0xc2, 0x14, 0xac, 0x0e, 0x64, 0x92, 0x21, + 0xc8, 0x17, 0x86, 0x20, 0x5f, 0x98, 0x23, 0x1b, 0x67, 0xa6, 0x63, 0x68, 0xac, 0x67, 0x64, 0x04, 0x56, 0x66, + 0x0e, 0x52, 0x01, 0xf2, 0x84, 0x21, 0xc8, 0x13, 0x96, 0x28, 0xc6, 0x99, 0xeb, 0x18, 0x19, 0xe8, 0x99, 0x5a, + 0x82, 0xd5, 0x81, 0x6c, 0x34, 0x04, 0x39, 0xd1, 0x10, 0x68, 0x89, 0xb1, 0x01, 0xb2, 0x71, 0x16, 0x3a, 0x86, + 0x26, 0x7a, 0x96, 0x66, 0x60, 0x65, 0x96, 0x20, 0x15, 0x60, 0xab, 0x81, 0x9e, 0x30, 0x36, 0x42, 0x31, 0xce, + 0x52, 0xc7, 0xc8, 0x50, 0xcf, 0xd8, 0x18, 0x1e, 0xc6, 0x86, 0x60, 0x33, 0x81, 0xbe, 0x30, 0x36, 0x46, 0x32, + 0x0e, 0xe8, 0x66, 0x43, 0x53, 0x3d, 0x73, 0x03, 0x78, 0x10, 0x1b, 0x81, 0x3c, 0x61, 0x04, 0xf4, 0x84, 0xb1, + 0x29, 0xb2, 0x71, 0x40, 0x6f, 0x19, 0x19, 0xe9, 0x19, 0x98, 0xc3, 0xc3, 0xd8, 0x08, 0xac, 0x18, 0xe8, 0x0b, + 0x63, 0x33, 0x64, 0xe3, 0x80, 0x51, 0x61, 0xa6, 0x67, 0x62, 0x02, 0x0f, 0x62, 0x23, 0x30, 0x01, 0xf2, 0x84, + 0x05, 0x8a, 0x71, 0xc6, 0xa0, 0x14, 0x60, 0x61, 0x08, 0x0f, 0x63, 0x23, 0x90, 0x2f, 0x8c, 0x40, 0xbe, 0xb0, + 0x44, 0x36, 0xce, 0x04, 0x94, 0x00, 0x0c, 0x2d, 0xe0, 0x41, 0x6c, 0x04, 0xf2, 0x84, 0x11, 0xd0, 0x13, 0x26, + 0x86, 0x28, 0xc6, 0x99, 0x82, 0x52, 0x80, 0xa9, 0x29, 0x3c, 0x8c, 0x8d, 0x41, 0xbe, 0x30, 0x06, 0xfa, 0xc2, + 0xc4, 0x08, 0xd9, 0x38, 0x33, 0x50, 0x02, 0xb0, 0x34, 0x82, 0x07, 0xb1, 0x31, 0xc8, 0x13, 0xc0, 0x30, 0x32, + 0x34, 0x31, 0x41, 0x31, 0x0e, 0x94, 0x24, 0xf4, 0x8c, 0x2c, 0xe1, 0x61, 0x6c, 0x0c, 0xf2, 0x85, 0x31, 0xd0, + 0x12, 0x13, 0x53, 0x64, 0xe3, 0x2c, 0x40, 0x09, 0xc0, 0xcc, 0x0c, 0x1e, 0xc4, 0xc6, 0x20, 0x27, 0x02, 0x93, + 0xab, 0xa1, 0x89, 0x39, 0x8a, 0x71, 0xa0, 0x40, 0xd7, 0x33, 0x30, 0x86, 0x87, 0xb1, 0x31, 0xd8, 0x6e, 0x90, + 0x2f, 0x2c, 0xe0, 0xc6, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x9e, 0x00, 0x61, 0xff, 0x31, 0x30, 0x32, + 0x30, 0x2c, 0x32, 0x30, 0x2e, 0x34, 0x30, 0x2c, 0x30, 0x2e, 0x30, 0x30, 0x33, 0x2c, 0x35, 0x30, 0x34, 0x30, + 0x2c, 0x35, 0x30, 0x34, 0x31, 0x2c, 0x31, 0x35, 0x30, 0x30, 0x30, 0x30, 0x2c, 0x66, 0x61, 0x6c, 0x73, 0x65, + 0x0a, 0x31, 0x30, 0x32, 0x31, 0x2c, 0x31, 0x34, 0x2e, 0x37, 0x37, 0x2c, 0x30, 0x2e, 0x30, 0x30, 0x34, 0x2c, + 0x35, 0x30, 0x34, 0x32, 0x2c, 0x35, 0x30, 0x34, 0x33, 0x2c, 0x31, 0x35, 0x31, 0x35, 0x30, 0x30, 0x2c, 0x74, + 0x72, 0x75, 0x65, 0x0a, 0x31, 0x30, 0x32, 0x32, 0x2c, 0x32, 0x31, 0x2e, 0x31, 0x34, 0x2c, 0x30, 0x2e, 0x30, + 0x30, 0x35, 0x2c, 0x35, 0x30, 0x34, 0x34, 0x2c, 0x35, 0x30, 0x34, 0x35, 0x2c, 0x31, 0x35, 0x33, 0x30, 0x30, + 0x30, 0x2c, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x0a, 0x31, 0x30, 0x32, 0x33, 0x2c, 0x31, 0x35, 0x2e, 0x35, 0x31, + 0x2c, 0x30, 0x2e, 0x30, 0x30, 0x36, 0x2c, 0x35, 0x30, 0x34, 0x36, 0x2c, 0x35, 0x30, 0x34, 0x37, 0x2c, 0x31, + 0x35, 0x34, 0x35, 0x30, 0x30, 0x2c, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x00, 0x00, 0x00, 0xff, 0xff, 0x55, 0x93, + 0x39, 0x92, 0x60, 0x21, 0x0c, 0x43, 0xf3, 0x3e, 0x0b, 0x45, 0xd9, 0xc6, 0x78, 0x39, 0x4e, 0x07, 0x3d, 0xd1, + 0x44, 0xb3, 0xdc, 0xbf, 0x2d, 0x07, 0x34, 0x24, 0x0e, 0x7e, 0xe9, 0x0b, 0x3f, 0x21, 0x98, 0x44, 0x87, 0xc8, + 0x8c, 0x18, 0x34, 0x89, 0x7c, 0x6c, 0xd2, 0xc0, 0xc8, 0xc1, 0xdb, 0x88, 0x68, 0xfc, 0xfa, 0xfc, 0xfd, 0xf7, + 0xeb, 0x83, 0x49, 0xf6, 0x60, 0x9b, 0x35, 0xa1, 0x83, 0x64, 0x13, 0x06, 0x97, 0xce, 0x77, 0xe9, 0xfe, 0xfd, + 0xf9, 0xdf, 0x32, 0x1b, 0x4c, 0xd3, 0xa4, 0x65, 0x09, 0x85, 0x60, 0xac, 0x92, 0xe5, 0x63, 0xe7, 0x83, 0x7d, + 0x66, 0xb6, 0x8e, 0x21, 0x51, 0x0c, 0x1c, 0x42, 0xb7, 0x5d, 0x0c, 0xe6, 0xb9, 0xac, 0x65, 0xed, 0x64, 0x18, + 0xf5, 0xb3, 0xc9, 0x63, 0x57, 0xfb, 0xc6, 0xf4, 0xd5, 0xba, 0x05, 0x49, 0xaf, 0x58, 0x5f, 0x6d, 0x5d, 0x76, + 0x8b, 0x06, 0xcb, 0x64, 0x6a, 0x19, 0x0e, 0x34, 0x40, 0x58, 0x41, 0xd8, 0xbe, 0xed, 0x56, 0x7d, 0xc9, 0xa9, + 0xde, 0xba, 0x0d, 0x09, 0xce, 0xb6, 0xa2, 0x30, 0xbb, 0xed, 0x64, 0xf0, 0x9a, 0xa1, 0x2d, 0xc3, 0x62, 0xd6, + 0x9e, 0x80, 0x88, 0xc7, 0x6e, 0x0d, 0xa1, 0x29, 0x7c, 0x32, 0xb6, 0x16, 0x83, 0x22, 0x6f, 0x3b, 0x1d, 0xac, + 0x73, 0xc7, 0x89, 0xd8, 0x7a, 0x14, 0x84, 0xf3, 0x63, 0xb7, 0x87, 0xf0, 0xcc, 0x7d, 0x32, 0x76, 0x50, 0x78, + 0xed, 0xec, 0x72, 0xdb, 0xd5, 0x55, 0xec, 0xb9, 0xe4, 0x44, 0xec, 0x80, 0xa8, 0x8c, 0xd8, 0xf5, 0xb1, 0x73, + 0x34, 0xc0, 0xf2, 0x64, 0xec, 0xa0, 0xf0, 0xa2, 0xf0, 0x7d, 0xdb, 0x05, 0x0a, 0x40, 0x76, 0x22, 0x76, 0x40, + 0x38, 0xee, 0xd1, 0x1f, 0xbb, 0x44, 0x03, 0x74, 0x9d, 0x8c, 0x1d, 0x14, 0x0e, 0x8a, 0xb8, 0xec, 0x94, 0x50, + 0x80, 0xa0, 0x13, 0x71, 0x00, 0x22, 0x0a, 0x02, 0xd1, 0xfd, 0xd8, 0x29, 0xa3, 0x01, 0xec, 0x27, 0xe3, 0xc0, + 0x8a, 0x51, 0x14, 0xc1, 0xb7, 0x9d, 0xa0, 0x00, 0x5b, 0x4f, 0xc4, 0x81, 0xa3, 0xa3, 0x20, 0x62, 0x3d, 0x76, + 0x0b, 0x0d, 0x48, 0x3e, 0x19, 0x47, 0x7b, 0x16, 0x45, 0xe8, 0x6d, 0xa7, 0x28, 0x80, 0xc4, 0x89, 0x38, 0x5a, + 0x8b, 0x96, 0x3d, 0xaf, 0x42, 0x37, 0x1a, 0x60, 0xfb, 0x64, 0x9c, 0xa0, 0x48, 0x50, 0xdc, 0xaf, 0x42, 0x0d, + 0x05, 0x20, 0x39, 0x11, 0x67, 0x0f, 0x40, 0x3c, 0xaf, 0xa2, 0x1a, 0x57, 0x0d, 0x58, 0x79, 0x32, 0x4e, 0x50, + 0xd4, 0x4d, 0x73, 0xfe, 0xbc, 0x8a, 0x6f, +}; + +struct Temporary { + std::string path; + Temporary() { + char pattern[] = "/tmp/pineforge-feed-export-XXXXXX"; + const auto* result = ::mkdtemp(pattern); + assert(result); + path = result; + } + ~Temporary() { std::filesystem::remove_all(path); } +}; +template void expect(int code, Function operation) { + bool caught = false; + try { operation(); } + catch (const Error& failure) { assert(failure.code == code); caught = true; } + assert(caught); +} +void passed(const char* name) { std::cout << "PASS " << name << '\n'; } +template std::string bytes(const unsigned char (&data)[N]) { return std::string(reinterpret_cast(data), N); } +void put_file(const std::string& path, const std::string& content) { + std::ofstream file(path, std::ios::binary); + file.write(content.data(), static_cast(content.size())); + assert(file); +} +std::string inflated(const std::string& compressed, std::size_t chunk = 4096, std::size_t* calls = nullptr) { + std::size_t offset = 0; + std::string result; + inflate([&](char* buffer, std::size_t limit) { + const auto count = std::min({chunk, limit, compressed.size() - offset}); + std::memcpy(buffer, compressed.data() + offset, count); + offset += count; + return count; + }, [&](std::string_view output) { + result.append(output); + if (calls) ++*calls; + }); + return result; +} +std::string generated() { + std::string text; + for (int index = 0; index < 48; ++index) { + char line[128]; + std::snprintf(line, sizeof(line), "%d,%d.%02d,0.%03d,%d,%d,%d,%s\n", 1000 + index, 10 + index * 7 % 13, index * 37 % 100, + index % 9 + 1, 5000 + 2 * index, 5001 + 2 * index, 120000 + 1500 * index, index % 2 ? "true" : "false"); + text += line; + } + return text; +} +// Deflate bits go least-significant first; Huffman codes are sent from their most significant bit. +struct Bits { + std::string bytes; + std::uint64_t buffer = 0; + unsigned int count = 0; + void put(std::uint32_t value, unsigned int width) { + buffer |= static_cast(value) << count; + for (count += width; count >= 8; count -= 8, buffer >>= 8) bytes += static_cast(buffer & 0xFF); + } + void huffman(std::uint32_t code, unsigned int width) { + std::uint32_t reversed = 0; + for (unsigned int bit = 0; bit < width; ++bit) reversed |= ((code >> bit) & 1U) << (width - 1 - bit); + put(reversed, width); + } + std::string finish() { + if (count) bytes += static_cast(buffer & 0xFF); + count = 0; + return bytes; + } +}; +const std::string bar_header = "timestamp,open,high,low,close,volume\n"; +const std::string archive_header = "agg_trade_id,price,quantity,first_trade_id,last_trade_id,transact_time,is_buyer_maker\n"; +const std::string rows = + "498,10.00,0.5,996,997,119000,true\n" + "499,10.10,0.1,998,999,119999,true\n" + "500,10.10,0.1,1000,1004,120001,false\n" + "501,11.20,0.2,1005,1005,120050,true\n" + "502,9.90,0.3,1006,1008,179999,false\n" + "503,12.00,0.4,1009,1009,240001,true\n" + "504,11.00,0.1,1010,1010,240050,false\n" + "505,12.00,0.1,1011,1011,300001,true\n" + "506,12.10,0.2,1012,1012,300050,false\n"; +// [120000, 300000): predecessor 499, prints 500..504, a quiet minute, fence 505. +const std::string window_bars = bar_header + + "120000,10.10,11.20,9.90,9.90,0.6\n" + "180000,9.90,9.90,9.90,9.90,0\n" + "240000,12.00,12.00,11.00,11.00,0.5\n"; +std::string replaced(std::string text, const std::string& from, const std::string& to) { + const auto at = text.find(from); + assert(at != std::string::npos); + return text.replace(at, from.size(), to); +} +} + +int main() { + { + ChainBars chain(120000, 300000); + chain.anchor({499, 119999, "10.10", "0.1"}); + assert(!chain.add({500, 120001, "10.10", "0.1"}) && !chain.add({501, 120050, "11.20", "0.2"}) && + !chain.add({502, 179999, "9.90", "0.3"}) && !chain.add({503, 240001, "12.00", "0.4"}) && + !chain.add({504, 240050, "11.00", "0.1"})); + assert(chain.add({505, 300001, "12.00", "0.1"})); + assert(chain.csv() == window_bars && chain.prints() == 5 && chain.bars() == 3 && chain.quiet_minutes() == 1); + assert(chain.predecessor().id == 499 && chain.fence().id == 505 && chain.fence().ts == 300001); + expect(23, [&] { chain.add({506, 300050, "12.10", "0.2"}); }); + } + { + // The first minute is quiet: it carries the predecessor's price; a fence exactly at the end closes the window. + ChainBars chain(180000, 420000); + chain.anchor({502, 179999, "9.90", "0.3"}); + assert(!chain.add({503, 240001, "12.00", "0.4"})); + assert(chain.add({504, 420000, "11.00", "0.1"})); + assert(chain.csv() == bar_header + "180000,9.90,9.90,9.90,9.90,0\n240000,12.00,12.00,12.00,12.00,0.4\n" + "300000,12.00,12.00,12.00,12.00,0\n360000,12.00,12.00,12.00,12.00,0\n"); + assert(chain.quiet_minutes() == 3 && chain.prints() == 1 && chain.bars() == 4); + } + { + // A quiet window: the predecessor's ID + 1 is already the fence. + ChainBars chain(120000, 240000); + chain.anchor({7, 0, "100.5", "1"}); + assert(chain.add({8, 250000, "101", "1"})); + assert(chain.csv() == bar_header + "120000,100.5,100.5,100.5,100.5,0\n180000,100.5,100.5,100.5,100.5,0\n"); + assert(chain.prints() == 0 && chain.quiet_minutes() == 2); + } + { + ChainBars chain(60000, 120000); + chain.anchor({1, 0, "5", "1"}); + chain.add({2, 60000, "5", "0.1"}); + chain.add({3, 119999, "5", "0.2"}); + assert(chain.add({4, 120000, "5", "1"})); + assert(chain.csv() == bar_header + "60000,5,5,5,5,0.3\n"); + } + { + // Price tokens stay verbatim (an equal value keeps the first lexeme); volume is the exact canonical sum. + ChainBars chain(60000, 180000); + chain.anchor({1, 59999, "1.0", "1"}); + chain.add({2, 60000, "2.50", "0.1"}); + chain.add({3, 60001, "2.5", "0.2"}); + chain.add({4, 60002, "0.70", "9999999999.99999999"}); + chain.add({5, 60003, "0.7", "0.00000001"}); + chain.add({6, 60004, "1.25", "0.000000000000000000000000000001"}); + for (std::uint64_t id = 7; id < 17; ++id) chain.add({id, 120000 + static_cast(id), "3.10", "0.1"}); + assert(chain.add({17, 180000, "3", "1"})); + assert(chain.csv() == bar_header + "60000,2.50,2.50,0.70,1.25,10000000000.300000000000000000000000000001\n" + "120000,3.10,3.10,3.10,3.10,1\n"); + } + { + ChainBars chain(120000, 300000); + expect(23, [&] { chain.add({500, 120001, "1", "1"}); }); + expect(20, [&] { chain.anchor({499, 120000, "1", "1"}); }); + chain.anchor({499, 119999, "10.10", "0.1"}); + expect(23, [&] { chain.anchor({498, 119000, "1", "1"}); }); + chain.add({500, 120001, "10.10", "0.1"}); + expect(20, [&] { chain.add({502, 120002, "10", "1"}); }); + expect(20, [&] { chain.add({500, 120002, "10", "1"}); }); + expect(20, [&] { (void)chain.csv(); }); + expect(20, [&] { (void)chain.fence(); }); + expect(23, [&] { chain.add({501, 120000, "10", "1"}); }); + expect(23, [&] { chain.add({501, 120002, "10", "0"}); }); + } + { + ChainBars chain(120000, 300000); + chain.anchor({499, 119999, "10", "1"}); + expect(20, [&] { chain.add({500, 119999, "10", "1"}); }); + } + expect(23, [] { ChainBars chain(120001, 300000); (void)chain; }); + expect(23, [] { ChainBars chain(120000, 120000); (void)chain; }); + expect(22, [] { ChainBars chain(0, 100001LL * 60000); (void)chain; }); + { ChainBars chain(0, 100000LL * 60000); (void)chain; } + passed("tick_built_bar_rule_and_chain_proof"); + + assert(!archive_row(archive_header.substr(0, archive_header.size() - 1), true)); + expect(23, [] { (void)archive_row(archive_header.substr(0, archive_header.size() - 1), false); }); + expect(23, [] { (void)archive_row("agg_trade_id,price", true); }); + const auto row = archive_row("500,10.10,0.1,1000,1004,120001,false\r", false); + assert(row && row->id == 500 && row->ts == 120001 && row->price == "10.10" && row->qty == "0.1"); + assert(archive_row("7,10.1,0.5,1,1,1759276800123456,True", true)->ts == 1759276800123); + assert(archive_row("7,10.1,0.5,1,1,1759276800123,false", false)->ts == 1759276800123); + for (const auto* invalid : {"7,10.1,0.5,1,1,120001", "7,10.1,0.5,1,1,120001,true,x", "7,10.1,0,1,1,120001,true", "7,-1,0.5,1,1,120001,true", + "0,10.1,0.5,1,1,120001,true", "7,10.1,0.5,2,1,120001,true", "7,10.1,0.5,0,1,120001,true", + "7,10.1,0.5,1,1,12e5,true", "7,10.1,0.5,1,1,-5,true", "7,10.1,0.5,1,1,120001,yes", "7,1e1,0.5,1,1,120001,true", ""}) + expect(23, [&] { (void)archive_row(invalid, false); }); + + Temporary temporary; + const auto with_header = temporary.path + "/TESTUSDT-aggTrades-1970-01-01.csv"; + put_file(with_header, archive_header + rows); + assert(archive_bars(with_header, 120000, 300000).csv() == window_bars); + const auto headerless = temporary.path + "/headerless.csv"; + put_file(headerless, rows); + { + const auto chain = archive_bars(headerless, 120000, 300000); + assert(chain.csv() == window_bars && chain.predecessor().id == 499 && chain.predecessor().price == "10.10"); + assert(chain.fence().id == 505 && chain.prints() == 5 && chain.quiet_minutes() == 1); + } + assert(archive_bars(headerless, 180000, 300000).csv() == bar_header + "180000,9.90,9.90,9.90,9.90,0\n240000,12.00,12.00,11.00,11.00,0.5\n"); + const auto crlf = temporary.path + "/crlf.csv"; + put_file(crlf, replaced(replaced(archive_header + rows, "\n", "\r\n"), "499,10.10,0.1,998,999,119999,true\n", "499,10.10,0.1,998,999,119999,true\r\n")); + assert(archive_bars(crlf, 120000, 300000).csv() == window_bars); + const auto unterminated = temporary.path + "/unterminated.csv"; + put_file(unterminated, rows.substr(0, rows.find("505,")) + "505,12.00,0.1,1011,1011,300001,true"); + assert(archive_bars(unterminated, 120000, 300000).csv() == window_bars); + expect(20, [&] { (void)archive_bars(headerless, 60000, 180000); }); + expect(20, [&] { (void)archive_bars(headerless, 240000, 360000); }); + const auto empty = temporary.path + "/empty.csv"; + put_file(empty, archive_header); + expect(20, [&] { (void)archive_bars(empty, 120000, 300000); }); + const auto hole = temporary.path + "/hole.csv"; + put_file(hole, replaced(rows, "502,9.90,0.3,1006,1008,179999,false\n", "")); + expect(20, [&] { (void)archive_bars(hole, 120000, 300000); }); + const auto regression = temporary.path + "/regression.csv"; + put_file(regression, replaced(rows, "240001", "170000")); + expect(23, [&] { (void)archive_bars(regression, 120000, 300000); }); + const auto malformed = temporary.path + "/malformed.csv"; + put_file(malformed, replaced(rows, "0.3,", "0.3;")); + expect(23, [&] { (void)archive_bars(malformed, 120000, 300000); }); + expect(23, [&] { (void)archive_bars(temporary.path + "/missing.csv", 120000, 300000); }); + passed("archive_csv_rows_with_and_without_header"); + + assert(crc32(0, "123456789") == 0xCBF43926U && crc32(crc32(0, "1234"), "56789") == 0xCBF43926U); + const auto text = generated(); + assert(text.size() == 1896 && crc32(0, text) == 0x52AFB242U); + const auto mixed = bytes(mixed_deflate); + for (const std::size_t chunk : {1, 7, 4096}) assert(inflated(mixed, chunk) == text); + expect(21, [&] { (void)inflated(mixed.substr(0, mixed.size() - 1)); }); + expect(21, [&] { (void)inflated(mixed.substr(0, 200)); }); + expect(21, [&] { (void)inflated(mixed + std::string(1, '\0')); }); + expect(21, [] { (void)inflated(std::string(1, '\x07')); }); + expect(21, [] { (void)inflated(std::string("\x01\x05\x00\x00\x00", 5)); }); + expect(21, [] { (void)inflated(std::string()); }); + { + Bits far; // a back-reference before any output + far.put(1, 1); + far.put(1, 2); + far.huffman(1, 7); + far.huffman(0, 5); + far.huffman(0, 7); + expect(21, [&] { (void)inflated(far.finish()); }); + } + { + // Fixed-Huffman literals, then 5000 copies of 258 bytes at distance 9: output crosses the 1 MiB hand-off. + Bits big; + big.put(1, 1); + big.put(1, 2); + const std::string seed = "pineforge"; + for (const char letter : seed) big.huffman(0x30 + static_cast(letter), 8); + for (int index = 0; index < 5000; ++index) { + big.huffman(0xC5, 8); + big.huffman(6, 5); + big.put(0, 2); + } + big.huffman(0, 7); + std::string expected; + while (expected.size() < 9 + 5000 * 258) expected += seed; + expected.resize(9 + 5000 * 258); + std::size_t calls = 0; + assert(inflated(big.finish(), 4096, &calls) == expected && calls >= 2); + } + passed("raw_inflate_fixed_stored_dynamic_blocks"); + + const auto zip = temporary.path + "/TESTUSDT-aggTrades-1970-01-01.zip"; + put_file(zip, bytes(deflated_zip)); + std::string entry; + assert(unzip(zip, [&](std::string_view chunk) { entry.append(chunk); }) == "TESTUSDT-aggTrades-1970-01-01.csv"); + assert(entry == archive_header + rows); + assert(archive_bars(zip, 120000, 300000).csv() == window_bars); + const auto stored = temporary.path + "/stored.zip"; + put_file(stored, bytes(stored_zip)); + entry.clear(); + (void)unzip(stored, [&](std::string_view chunk) { entry.append(chunk); }); + assert(entry == rows && archive_bars(stored, 120000, 300000).csv() == window_bars); + const auto damaged = temporary.path + "/damaged.zip"; + auto altered = bytes(deflated_zip); + altered[altered.find("PK\x01\x02") + 16] ^= 1; + put_file(damaged, altered); + expect(21, [&] { (void)unzip(damaged, [](std::string_view) {}); }); + expect(21, [&] { (void)archive_bars(damaged, 120000, 300000); }); + altered = bytes(stored_zip); + altered[30 + 33 + 5] = 'x'; // a row that no longer parses: the CRC-32 mismatch (21) is reported first + put_file(damaged, altered); + expect(21, [&] { (void)archive_bars(damaged, 120000, 300000); }); + altered = bytes(deflated_zip); + altered[30 + 33 + 40] ^= 0x10; + put_file(damaged, altered); + expect(21, [&] { (void)archive_bars(damaged, 120000, 300000); }); + altered = bytes(stored_zip); + altered[altered.find("PK\x01\x02") + 24] ^= 1; // recorded size disagrees + put_file(damaged, altered); + expect(21, [&] { (void)archive_bars(damaged, 120000, 300000); }); + const auto plain = temporary.path + "/plain.zip"; + put_file(plain, rows); + expect(23, [&] { (void)archive_bars(plain, 120000, 300000); }); + passed("zip_deflated_and_stored_entries_crc_checked"); + + const auto digest = file_sha256(zip); + assert(digest == sha256(bytes(deflated_zip)) && file_sha256(headerless) == sha256(rows)); + expect(23, [&] { (void)file_sha256(temporary.path + "/missing.zip"); }); + const auto checksum = temporary.path + "/TESTUSDT-aggTrades-1970-01-01.zip.CHECKSUM"; + put_file(checksum, digest + " TESTUSDT-aggTrades-1970-01-01.zip\n"); + verify_checksum(checksum, zip, digest); + auto upper = digest; + std::transform(upper.begin(), upper.end(), upper.begin(), [](unsigned char letter) { return static_cast(std::toupper(letter)); }); + put_file(checksum, upper + " *TESTUSDT-aggTrades-1970-01-01.zip"); + verify_checksum(checksum, zip, digest); + auto wrong = digest; + wrong[0] = wrong[0] == '0' ? '1' : '0'; + put_file(checksum, wrong + " TESTUSDT-aggTrades-1970-01-01.zip\n"); + expect(21, [&] { verify_checksum(checksum, zip, digest); }); + put_file(checksum, digest + " TESTUSDT-aggTrades-1970-01-02.zip\n"); + expect(21, [&] { verify_checksum(checksum, zip, digest); }); + for (const auto* invalid : {"", "not a checksum\n", "abc TESTUSDT-aggTrades-1970-01-01.zip\n"}) { + put_file(checksum, invalid); + expect(23, [&] { verify_checksum(checksum, zip, digest); }); + } + put_file(checksum, digest + " TESTUSDT-aggTrades-1970-01-01.zip\n" + digest + " other.zip\n"); + expect(23, [&] { verify_checksum(checksum, zip, digest); }); + expect(23, [&] { verify_checksum(temporary.path + "/missing.CHECKSUM", zip, digest); }); + passed("archive_checksum_hash_and_name"); + std::cout << "PASS feed export unit suite\n"; +} diff --git a/native/feed/tests/mock_export.py b/native/feed/tests/mock_export.py new file mode 100644 index 0000000..9aaea17 --- /dev/null +++ b/native/feed/tests/mock_export.py @@ -0,0 +1,340 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Synthetic export scenarios: the real binary against the mock USD-M venue and daily archives.""" + +import hashlib +import importlib +import json +import pathlib +import subprocess +import sys +import tempfile +import threading +import time +import unittest +import zipfile +from decimal import Decimal, getcontext + +# mock_venue reads the binary from sys.argv[1] at import time. +sys.argv = [sys.argv[0], sys.argv[1]] +mock_venue = importlib.import_module("mock_venue") +BINARY = mock_venue.BINARY +getcontext().prec = 200 +SYMBOL = "TESTUSDT" +HEADER = "agg_trade_id,price,quantity,first_trade_id,last_trade_id,transact_time,is_buyer_maker" + + +def recent_minute(minutes_ago): + return (int(time.time() * 1000) // 60000 - minutes_ago) * 60000 + + +def canonical(value): + text = format(value, "f") + if "." in text: + text = text.rstrip("0").rstrip(".") + return text + + +def expected_bars(prints, start, end): + """The tick-built bar rule, independently: prints are (id, ts, price, qty) in ID order.""" + close = [p for p in prints if p[1] < start][-1][2] + rows = ["timestamp,open,high,low,close,volume"] + for minute in range(start, end, 60000): + inside = [p for p in prints if minute <= p[1] < minute + 60000] + if not inside: + rows.append(f"{minute},{close},{close},{close},{close},0") + continue + high = max(inside, key=lambda p: Decimal(p[2]))[2] + low = min(inside, key=lambda p: Decimal(p[2]))[2] + volume = sum((Decimal(p[3]) for p in inside), Decimal(0)) + close = inside[-1][2] + rows.append(f"{minute},{inside[0][2]},{high},{low},{close},{canonical(volume)}") + return "\n".join(rows) + "\n" + + +def synthetic(first_id, start, minutes, per_minute, quiet=()): + """Prints over `minutes` minutes from `start`, none in the `quiet` minute offsets.""" + prints = [] + identifier = first_id + for offset in range(minutes): + if offset in quiet: + continue + for index in range(per_minute): + ts = start + offset * 60000 + index * (59999 // max(per_minute - 1, 1)) + price = f"{100 + (identifier * 37) % 50}.{(identifier * 13) % 100:02d}" + quantity = f"0.{(identifier * 104729) % 99999999 + 1:08d}" + prints.append((identifier, ts, price, quantity)) + identifier += 1 + return prints + + +class ExportMockTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="pineforge-feed-export-") + self.addCleanup(self.temporary.cleanup) + self.directory = pathlib.Path(self.temporary.name) + self.output = self.directory / "bars.csv" + + def server(self, prints): + server = mock_venue.MockServer([[]], "usdm") + server.aggregates = { + identifier: ( + ts, + price, + quantity, + quantity, + 1000 + 2 * identifier, + 1001 + 2 * identifier, + ) + for identifier, ts, price, quantity in prints + } + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return server + + def export(self, start, end, server=None, extra=(), expected=0, venue=("binance", "usdm")): + port = server.server_address[1] if server else 9 + arguments = [ + BINARY, + "export", + "--venue", + venue[0], + "--market", + venue[1], + "--symbol", + SYMBOL, + ] + arguments += ["--mode", "agg-ticks" if venue[1] == "usdm" else "ticks"] + arguments += ["--start", str(start), "--end", str(end), "--output", str(self.output)] + arguments += [ + "--rest-url", + f"http://127.0.0.1:{port}", + "--ws-url", + f"ws://127.0.0.1:{port}", + ] + arguments += ["--allow-insecure-http", *extra] + result = subprocess.run(arguments, capture_output=True, text=True, timeout=40) + self.assertEqual(result.returncode, expected, result.stderr) + if expected: + self.assertFalse(self.output.exists()) + self.assertFalse(pathlib.Path(str(self.output) + ".manifest.json").exists()) + return result + + def manifest(self): + manifest = json.loads(pathlib.Path(str(self.output) + ".manifest.json").read_text()) + self.assertEqual(manifest["sha256"], hashlib.sha256(self.output.read_bytes()).hexdigest()) + self.assertEqual(manifest["schema"], "pineforge-feed-export/v1") + self.assertIn("floor(ts/60000)", manifest["rule"]) + return manifest + + def test_rest_bars_follow_the_rule_with_quiet_minute_and_carry(self): + start = recent_minute(30) + prints = [ + (499, start - 1, "10.10000000", "0.10000000"), + (500, start, "10.10000000", "0.10000000"), + (501, start + 50, "11.20000000", "0.20000000"), + (502, start + 59999, "9.90000000", "0.30000000"), + (503, start + 120001, "12.00000000", "0.40000000"), + (504, start + 120050, "11.00000000", "0.10000000"), + (505, start + 180000, "11.50000000", "0.00100000"), + (506, start + 240001, "12.00000000", "0.10000000"), + (507, start + 240050, "12.10000000", "0.20000000"), + ] + server = self.server(prints) + self.export(start, start + 240000, server) + text = self.output.read_text() + self.assertEqual(text, expected_bars(prints, start, start + 240000)) + self.assertIn(f"\n{start + 60000},9.90000000,9.90000000,9.90000000,9.90000000,0\n", text) + self.assertIn(f"\n{start},10.10000000,11.20000000,9.90000000,9.90000000,0.6\n", text) + manifest = self.manifest() + self.assertEqual( + {key: manifest[key] for key in ("venue", "market", "symbol", "mode")}, + {"venue": "binance", "market": "usdm", "symbol": SYMBOL, "mode": "agg-ticks"}, + ) + port = server.server_address[1] + self.assertEqual(manifest["source"], {"kind": "rest", "origin": f"http://127.0.0.1:{port}"}) + self.assertEqual((manifest["start"], manifest["end_exclusive"]), (start, start + 240000)) + self.assertEqual( + manifest["predecessor"], {"id": 499, "ts": start - 1, "price": "10.10000000"} + ) + self.assertEqual(manifest["fence"], {"id": 506, "ts": start + 240001}) + self.assertEqual((manifest["first_id"], manifest["last_id"]), (500, 505)) + self.assertEqual( + (manifest["prints"], manifest["bars"], manifest["quiet_minutes"]), (6, 4, 1) + ) + + def test_rest_pages_by_from_id_and_sums_volume_exactly(self): + start = recent_minute(40) + prints = synthetic(1000, start - 60000, 6, 700, quiet=(2,)) + server = self.server(prints) + self.export(start, start + 240000, server) + self.assertEqual(self.output.read_text(), expected_bars(prints, start, start + 240000)) + pages = [q for path, q in server.queries if path == "/fapi/v1/aggTrades" and "fromId" in q] + self.assertGreaterEqual(len([q for q in pages if q["limit"] == ["1000"]]), 3) + manifest = self.manifest() + self.assertEqual((manifest["prints"], manifest["quiet_minutes"]), (2100, 1)) + self.assertEqual(manifest["fence"]["id"], manifest["last_id"] + 1) + + def test_missing_fence_stops_20_and_writes_nothing(self): + start = recent_minute(20) + prints = [p for p in synthetic(1, start - 60000, 4, 5) if p[1] < start + 120000] + server = self.server(prints) + result = self.export(start, start + 180000, server, expected=20) + self.assertIn("fence cannot be proven", result.stderr) + + def test_hole_stops_20_and_time_regression_stops_23(self): + start = recent_minute(20) + prints = synthetic(1, start - 60000, 5, 5) + server = self.server([p for p in prints if p[0] != 9]) + result = self.export(start, start + 180000, server, expected=20) + self.assertIn("nonunit aggregate ID", result.stderr) + # ID 12 is dated before ID 11 (the first print of its minute). + regressed = [(i, start + 50000 if i == 12 else ts, p, q) for i, ts, p, q in prints] + server = self.server(regressed) + result = self.export(start, start + 180000, server, expected=23) + self.assertIn("backwards", result.stderr) + + def test_start_beyond_retention_or_open_window_stops_20(self): + start = recent_minute(49 * 60) + server = self.server(synthetic(1, start - 60000, 3, 3)) + result = self.export(start, start + 60000, server, expected=20) + self.assertIn("REST print history window", result.stderr) + self.assertNotIn("/fapi/v1/aggTrades", server.requests) + result = self.export(recent_minute(1), recent_minute(-5), server, expected=20) + self.assertIn("not closed yet", result.stderr) + + def test_existing_output_is_refused(self): + self.output.write_text("keep\n") + start = recent_minute(10) + arguments = [BINARY, "export", "--venue", "binance", "--market", "usdm", "--symbol", SYMBOL] + arguments += ["--mode", "agg-ticks", "--start", str(start), "--end", str(start + 60000)] + arguments += ["--output", str(self.output), "--rest-url", "http://127.0.0.1:9"] + arguments += ["--ws-url", "ws://127.0.0.1:9", "--allow-insecure-http"] + result = subprocess.run(arguments, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 23, result.stderr) + self.assertEqual(self.output.read_text(), "keep\n") + + def archive_prints(self): + start = recent_minute(24 * 60) + prints = synthetic(70000, start - 120000, 9, 400, quiet=(4,)) + return start, prints + + def write_archive(self, name, prints, microseconds=False, header=True): + lines = [HEADER] if header else [] + for identifier, ts, price, quantity in prints: + when = ts * 1000 + 456 if microseconds else ts + raw = 2 * identifier + maker = "true" if identifier % 3 else "false" + lines.append(f"{identifier},{price},{quantity},{raw},{raw + 1},{when},{maker}") + text = "\n".join(lines) + "\n" + path = self.directory / name + if name.endswith(".zip"): + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(name[:-4] + ".csv", text) + else: + path.write_text(text) + return path + + def checksum(self, archive, digest=None, name=None): + digest = digest or hashlib.sha256(archive.read_bytes()).hexdigest() + path = self.directory / (archive.name + ".CHECKSUM") + path.write_text(f"{digest} {name or archive.name}\n") + return path + + def test_archive_csv_and_zip_match_the_rule(self): + start, prints = self.archive_prints() + end = start + 300000 + expected = expected_bars(prints, start, end) + csv_archive = self.write_archive( + f"{SYMBOL}-aggTrades-day.csv", prints, microseconds=True, header=False + ) + checksum = self.checksum(csv_archive) + self.export(start, end, extra=["--archive", str(csv_archive), "--checksum", str(checksum)]) + self.assertEqual(self.output.read_text(), expected) + manifest = self.manifest() + self.assertEqual(manifest["source"]["checksum_verified"], True) + self.output.unlink() + pathlib.Path(str(self.output) + ".manifest.json").unlink() + zipped = self.write_archive(f"{SYMBOL}-aggTrades-day.zip", prints) + checksum = self.checksum(zipped) + self.export(start, end, extra=["--archive", str(zipped), "--checksum", str(checksum)]) + self.assertEqual(self.output.read_text(), expected) + manifest = self.manifest() + digest = hashlib.sha256(zipped.read_bytes()).hexdigest() + self.assertEqual( + manifest["source"], + {"kind": "archive", "file": zipped.name, "sha256": digest, "checksum_verified": True}, + ) + predecessor = [p for p in prints if p[1] < start][-1] + fence = next(p for p in prints if p[1] >= end) + self.assertEqual( + manifest["predecessor"], + {"id": predecessor[0], "ts": predecessor[1], "price": predecessor[2]}, + ) + self.assertEqual(manifest["fence"], {"id": fence[0], "ts": fence[1]}) + self.assertEqual(manifest["quiet_minutes"], 1) + + def test_archive_without_checksum_warns(self): + start, prints = self.archive_prints() + archive = self.write_archive(f"{SYMBOL}-aggTrades-day.zip", prints) + result = self.export(start, start + 120000, extra=["--archive", str(archive)]) + self.assertIn("archive_not_checksum_verified", result.stderr) + self.assertEqual(self.manifest()["source"]["checksum_verified"], False) + + def test_archive_checksum_mismatch_wrong_name_and_corruption_stop_21(self): + start, prints = self.archive_prints() + archive = self.write_archive(f"{SYMBOL}-aggTrades-day.zip", prints) + tampered = self.checksum(archive, digest="0" * 64) + result = self.export( + start, + start + 120000, + extra=["--archive", str(archive), "--checksum", str(tampered)], + expected=21, + ) + self.assertIn("does not match", result.stderr) + renamed = self.checksum(archive, name=f"{SYMBOL}-aggTrades-other.zip") + result = self.export( + start, + start + 120000, + extra=["--archive", str(archive), "--checksum", str(renamed)], + expected=21, + ) + self.assertIn("not the archive", result.stderr) + csv_archive = self.write_archive(f"{SYMBOL}-aggTrades-day.csv", prints) + tampered = self.checksum(csv_archive, digest="f" * 64) + result = self.export( + start, + start + 120000, + extra=["--archive", str(csv_archive), "--checksum", str(tampered)], + expected=21, + ) + self.assertIn("does not match", result.stderr) + data = bytearray(archive.read_bytes()) + data[len(data) // 2] ^= 0x55 + archive.write_bytes(bytes(data)) + result = self.export(start, start + 120000, extra=["--archive", str(archive)], expected=21) + self.assertRegex(result.stderr, "deflate|CRC-32|recorded size") + self.assertNotIn("export_written", result.stderr) + + def test_archive_outside_window_and_unsupported_venue(self): + start, prints = self.archive_prints() + archive = self.write_archive(f"{SYMBOL}-aggTrades-day.csv", prints) + self.export(start - 600000, start, extra=["--archive", str(archive)], expected=20) + self.export(start + 360000, start + 900000, extra=["--archive", str(archive)], expected=20) + result = self.export( + start, + start + 60000, + extra=["--archive", str(archive)], + expected=23, + venue=("binance", "spot"), + ) + self.assertIn("--venue binance --market usdm --mode agg-ticks", result.stderr) + + +if __name__ == "__main__": + suite = unittest.defaultTestLoader.loadTestsFromTestCase(ExportMockTests) + result = unittest.TextTestRunner(verbosity=2).run(suite) + if result.wasSuccessful(): + print(f"PASS mock export ({result.testsRun} scenarios)", flush=True) + sys.exit(0 if result.wasSuccessful() else 1) From cd28aa2fb27a08abf8d286ae217bc55cdac44101 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 08:34:59 +0800 Subject: [PATCH 03/15] Document serve, the segmented journal, export and REST pacing README: the serve endpoints, cursor negotiation, resume with from=S+N and --from-input N, client bounds, the keepalive PONG and the TLS-by-reverse- proxy rule; the segmented journal with retention by bytes and venue time, crash safety at every seal and expiry step and the expired-cursor stop; export of prints-built bars from REST or the daily archive; header-based Binance pacing, the quiet-stream probe and reader backpressure; and the follow-up wording for OKX re-checks, Bybit 403 and quiet starts. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/README.md | 319 +++++++++++++++++++++++++++++++++--------- 1 file changed, 249 insertions(+), 70 deletions(-) diff --git a/native/feed/README.md b/native/feed/README.md index 80ea19a..8b6934e 100644 --- a/native/feed/README.md +++ b/native/feed/README.md @@ -3,8 +3,9 @@ An independent Apache-2.0 C++17 public-market-data adapter. It produces the native `pineforge-live` runner's JSONL protocol; it does not load strategies, place orders, access accounts, consume fills, or accept API keys. It is not part -of the Python package or wheel. It has `warmup` and `run`; there is no `serve` -subcommand yet. +of the Python package or wheel. It has `warmup` (venue candles), `export` +(bars built from prints), `run` (JSONL on stdout) and `serve` (the same durable +stream to many runners over a local WebSocket). ## Venues and modes @@ -33,7 +34,13 @@ Requirements: CMake >= 3.20, a C++17 compiler, OpenSSL Crypto, threads, and feature check, and every running binary checks the actually loaded libcurl. Cross-compilation without an executable feature check is not qualified. `vendor/json.hpp` is SHA-256 pinned; attribution and dependency pins are in -`NOTICE`. No engine linkage or additional C++ dependency is required. +`NOTICE`. No engine linkage is required. `serve` compiles CivetWeb 1.16 (MIT), +fetched at configure time from its release archive and pinned by SHA-256; only +the serve code path links it, and `-DPINEFORGE_FEED_SERVE=OFF` builds a binary +without it (whose `serve` stops with 23). Only CivetWeb's HTTP/WebSocket core is +compiled: no TLS, CGI, Lua, Duktape, WebDAV, SSI or file serving. For an offline +build, point `-DPINEFORGE_FEED_CIVETWEB_ARCHIVE=/path/to/civetweb-1.16.tar.gz` at +a local copy of the same archive; its hash is still checked. ```sh cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ @@ -68,7 +75,9 @@ build-feed/pineforge-feed run --venue binance --market usdm --symbol BTCUSDT \ ``` Each venue's warmup takes the same flags and writes its own venue's candles: -use a warmup from the venue and market that the live feed reads. +use a warmup from the venue and market that the live feed reads. A tick-mode +deployment (raw ticks or `agg-ticks`) builds its bars from prints, so its +matching backtest history comes from `export` (below), not from candles. Times are minute-aligned Unix milliseconds or strict UTC RFC3339 seconds. Warmup fetches contiguous REST 1m klines only behind an observed `x=true` WS @@ -79,6 +88,53 @@ cut, confirmation watermark, row count, quantity multiplier, and the CSV SHA-256. It refuses to overwrite either output. Keep both files together; an interrupted two-file publication without its manifest is not a qualified warmup. +## Prints-built history: `export` + +```sh +build-feed/pineforge-feed export --venue binance --market usdm --symbol BTCUSDT \ + --mode agg-ticks --start 2026-10-03T00:00:00Z --end 2026-10-03T03:20:00Z --output agg-bars.csv +# or from a local copy of the venue's public daily aggregate-trade archive: +build-feed/pineforge-feed export --venue binance --market usdm --symbol BTCUSDT \ + --mode agg-ticks --start 2026-10-02T00:10:00Z --end 2026-10-02T23:50:00Z --output agg-bars.csv \ + --archive BTCUSDT-aggTrades-2026-10-02.zip --checksum BTCUSDT-aggTrades-2026-10-02.zip.CHECKSUM +``` + +In tick modes the runner builds its 1m bars from the prints, so a backtest that +must equal the forward run needs bars built the same way, not venue candles (see +"Binance USD-M" for how far `agg-ticks` bars and klines can differ). `export` +writes exactly those bars in the warmup CSV format, by the runner's tick-built +bar rule: a print belongs to minute `floor(ts / 60000) * 60000`; open is the first +price, high the maximum, low the minimum, close the last price, and volume the +exact decimal sum of the quantities; a minute without a print repeats the +previous close (for the first minute, the last print before `--start`) with +volume `0`, as the runner's carry-forward bar does. Price tokens are the venue's +own; the volume is the canonical exact sum. + +Completeness is proven, never assumed: the prints must form a contiguous ID chain +from the last print strictly before `--start` through the fence, the first print +at or after `--end`. A hole, a missing fence (the window has not closed yet), or +a start outside the venue's REST history (USD-M 48 hours) stops with 20 and +writes nothing; time running backwards along the chain stops with 23. From REST +the chain is paged by ID (Binance 1000, OKX 100 per page); a USD-M start needs an +aggregate within the hour after it. + +`--archive` reads the Binance USD-M daily `aggTrades` archive as published (the +`.zip`, inflated in-process and CRC-checked, or the extracted `.csv`, with or +without its header line), streamed rather than loaded. A `transact_time` above +10^14 is microseconds and becomes `floor(us / 1000)` milliseconds. `--checksum` +takes the published ` ` file and must match the archive's +SHA-256 and base name (21 otherwise); without it the export logs +`archive_not_checksum_verified`. The window, its predecessor and its fence must +all lie inside the one archive (else 20). Archives are never fetched +automatically. + +An existing output or manifest is refused. The CSV and then +`.manifest.json` are written atomically; the manifest records the source +(REST origin, or archive name, SHA-256 and whether the checksum was verified), +the window, the predecessor and fence prints, the first and last IDs, the print, +bar and quiet-minute counts, the rule, and the CSV's SHA-256. Limits: 100000 +minutes and 64 MiB of CSV (22 beyond). + ## Protocol and proof policy - Bars are `{"type":"bar","bar":{"ts_open":...,"o":...,"h":...,"l":...,"c":...,"v":...}}`. @@ -154,9 +210,12 @@ and `n` count raw trades, not aggregates), so a minute `M` closes with millisecond need not come back in ID order); USD-M takes the first aggregate at or after the start within the next hour and the one before it. Nothing is anchored, and no minute closes, until both sides are visible. A - quiet start waits for the first print (one REST read per closed minute); - once a WebSocket print exists, REST must show it within about 15 seconds, - or the feed stops with 20; + quiet start waits for the first print (each newly closed minute costs one + look-ahead: OKX a time lookup plus an ID page, USD-M one aggregate search + per unsettled hour window, searched hour by hour up to the newest venue time + and at most 48 hours past the start, after which it stops with 20); once a + WebSocket print exists, REST must show it within about 15 seconds, or the + feed stops with 20; 2. the fence: the next print in that chain has a matched time at or after `M + 60000`, so no further print of `M` can exist (matched time never goes backwards along the chain; if it does, the feed stops with 23); @@ -190,8 +249,10 @@ push on it, or any print carrying `count`, stops with 23. The feed sends text `ping` every `--keepalive-seconds` (default 20, at most 25: OKX closes a connection after 30 seconds without traffic) and treats `pong` as control, never data; a `notice` event (service upgrade) reconnects with overlap. Every -connection re-reads the instrument: a changed `ctVal x ctMult` stops with 21, -since quantities already emitted were converted with the old one. +connection re-reads the instrument and re-runs every instrument check: a changed +`ctVal x ctMult` stops with 21, since quantities already emitted were converted +with the old one, and an instrument that is no longer `live` (a suspension) +stops with 23. ### Bybit @@ -201,8 +262,9 @@ forming one; only the confirmed row is a bar. REST klines arrive newest first and a range wider than `limit` keeps only its newest rows, so each request asks for exactly the rows it can hold and the reversed rows must be contiguous. The feed sends `{"op":"ping"}` every `--keepalive-seconds`. Bybit answers HTTP 403 -for its IP rate ban (about ten minutes); the feed stops with 20 there, so a -supervised restart retries after the ban. +for its IP rate ban (about ten minutes) and for a restricted region; the feed +stops with 20 there, so a supervised restart retries; give that restart a backoff +above ten minutes, since a request inside the ban window extends it. ### Binance USD-M @@ -235,7 +297,7 @@ below that minute's kline open: the kline opened at the price of the fill that the aggregate dated in the previous minute, so a backtest on klines fills one tick away. A backtest on the bars built from the same prints equals the runner. The matching backtest for an `agg-ticks` deployment is therefore a backtest on -prints-built 1m bars; a later feed release adds the export of those bars. +prints-built 1m bars, which `export` writes from REST or the daily archive. The subscribed data streams are strict: another event type on them, a malformed trade or kline, or an unknown shape stops with 23 (on every venue). An event outside @@ -245,9 +307,21 @@ reconnects. Curl answers server PING with matching-payload PONG; OKX and Bybit get their text keepalives. Keepalive replies are control messages: they never count as -data, so a silent data stream still reconnects. Reconnect occurs on -`serverShutdown` or an OKX `notice`, transport loss/75 seconds without data, or -at 23h55 (before the 24-hour Binance limit). The reader buffers during REST healing. Every new +data. After `--silence-seconds` (default 75) without data, OKX and Bybit +reconnect; Binance instead sends a `LIST_SUBSCRIPTIONS` probe, and a reply that +lists the subscriptions proves the connection and its streams alive, so a quiet +symbol no longer pays a reconnect and its REST overlap every 75 seconds (an +empty list, or no reply within 10 seconds, reconnects). This is safe because +completeness never rests on the connection: every print is chained by ID, every +minute behind a confirmed watermark, and every reconnect re-verifies its overlap. +Reconnect also occurs on `serverShutdown` or an OKX `notice`, transport loss, or +at 23h55 (before the 24-hour Binance limit). The reader buffers during REST +healing; if a long outage keeps the session healing until the reader's bounded +queue (`--max-queue-bytes`) is full, the reader drops its connection instead of +stopping, waits until the queue has drained to a quarter, and reconnects: what +the venue sent meanwhile is healed by ID or behind the watermark from REST after +the reconnect's overlap check, exactly as after a restart. Only a single message +larger than the whole queue still stops with 22. Every new connection re-fetches up to 1000 raw prints and 32 closed-bar proofs, requires exact value/ID/time overlap, then continues the same stream epoch and sequence. A venue's REST row for the minute that just closed can trail its WebSocket @@ -264,12 +338,13 @@ REST routes are allowlisted per venue: Binance spot `exchangeInfo`, `market/kline`, `market/instruments-info` (one request per 100 ms, OKX `50011` and Bybit `10006` rate-limit bodies wait and retry like `429`; OKX `50001`, `50004`, `50013`, `50026` and Bybit `10000`, `10016` mean "try again" and are -retried with backoff like HTTP 5xx, within the same four attempts). Current public weight/raw-request limits are read -from `exchangeInfo`, and requests are spaced to spend at most half of each -published ceiling (about 50% headroom; 20 ms per weight at 6000 per minute, so -500 ms per 1000-print page). The venue's `X-MBX-USED-WEIGHT-1M` header counts -every client on the IP: once it shows half of the 1-minute ceiling, the feed -waits for the next window. `429` honours `Retry-After` as delta-seconds or an +retried with backoff like HTTP 5xx, within the same four attempts). Binance's +public weight and raw-request limits are read from `exchangeInfo`. The one-minute +weight ceiling is spent by the venue's own count: every response's +`X-MBX-USED-WEIGHT-1M` header (which counts every client on the IP) is kept, and +a request whose weight would take the current window past 90% of the ceiling +waits for the next window (`rest_quota_wait`); otherwise requests are not spaced. +Other published ceilings are paced evenly across their interval. `429` honours `Retry-After` as delta-seconds or an IMF-fixdate HTTP-date; a wait above 24 hours stops with 22 and an unparseable value with 23. A ban/access rejection is not retried as anonymous trading access. TLS verification is always enabled; redirects, URL credentials, and @@ -278,20 +353,47 @@ non-origin overrides are refused. Explicit insecure overrides are ## Durability and restart -One producer owns a state directory via `flock`. `events.jsonl` is append-only. -Messages are group-committed: every event made ready by the source messages -already queued (up to 1024 source messages, or 4096 events) is appended in one -write and fsynced once, then `cursor.json` is atomically replaced once and its -directory fsynced, and only **then** are those lines written to stdout, one -event per line as before. The cursor binds source origins, symbol/mode/units, -random stream epoch, initial/verified cut, venue ID, emitted sequence, message -index, byte count, predecessor, closed-minute proofs, SHA-256 prefix chain, -last-message hash, and fixed message partition. Its canonical document is -itself hashed. Resume verifies the retained committed prefix and fsyncs -truncation of an uncommitted crash tail. - -A pipe write is not a consumer acknowledgment. By default `--resume` replays the -whole verified origin prefix. If the runner has committed N messages: +One producer owns a state directory via `flock`. The normalized log is a +segmented journal under `journal/`: each segment file `.jsonl` holds +consecutive messages and is append-only, and beside it a hashed checkpoint +`.checkpoint.json` records the verified state before its first +message (prefix hash, last-message hash, message index and byte offset, cut, +sequence, last tick time, last bar, predecessor). Messages are group-committed: +every event made ready by the source messages already queued (up to 1024 source +messages, or 4096 events) is appended to the open segment in one write and +fsynced once, then `cursor.json` is atomically replaced once and its directory +fsynced, and only **then** are those lines published (stdout for `run`, clients +for `serve`), one event per line. The cursor binds source origins, +symbol/mode/units, random stream epoch, initial/verified cut, venue ID, emitted +sequence, message index, byte count, the open segment, predecessor, +closed-minute proofs, SHA-256 prefix chain, last-message hash, and fixed message +partition. Its canonical document is itself hashed. + +After a commit leaves the open segment at or above `--segment-bytes` (default +16 MiB), it is sealed: the successor's checkpoint is written atomically, then the +empty successor is created and synced, and only then does the cursor name it. A +crash between those steps leaves files that resume removes, and the old segment +stays open. Sealed segments then expire oldest first while the retained journal +exceeds `--replay-bytes` (default 256 MiB, at least twice `--segment-bytes`) or, +with `--replay-age-seconds`, while a segment's last minute is older than that in +venue time; a segment file goes before its checkpoint, so a crash in between +leaves an orphan checkpoint that resume removes. The open segment and every +segment that can hold a message of the protected window (the open minute and the +32 closed-minute proofs the reconnect overlap re-verifies) never expire; when +they alone exceed the budget the feed keeps them and logs +`replay_budget_held_by_protected_window`. Resume verifies the retained journal +from its oldest checkpoint, checks every segment boundary against its +checkpoint and the end against the cursor, and fsyncs truncation of an +uncommitted crash tail. Every crash point of a commit, a seal and an expiry is +exercised by the test suite (`PINEFORGE_FEED_CRASH_AT`, a test hook that kills +the process at a named step) and by random kills. + +A duplicate print or candle older than the retained journal can no longer be +compared with what was emitted: it is logged as `expired_duplicate_unverified` +and ignored. Inside the retained journal every duplicate is still checked. + +A pipe write is not a consumer acknowledgment. If the runner has committed N +messages: ```sh build-feed/pineforge-feed run --venue binance --market spot --symbol BTCUSDT \ @@ -302,12 +404,17 @@ build-feed/pineforge-feed run --venue binance --market spot --symbol BTCUSDT \ ``` Replace N with the runner's actual committed count; never use the producer's -possibly-ahead count. Bind warmup identity and stream epoch in orchestration. -An unavailable cursor or changed identity is refused. SIGTERM/SIGINT stops -intake, discards staged messages that were never committed or published, and -exits 0. A directory holding only an empty `events.jsonl` (a crash before the -first cursor write) is taken over by a fresh `--start`; other existing state -needs `--resume`. +possibly-ahead count. `--resume` without `--output-from` replays from index 0. +Bind warmup identity and stream epoch in orchestration. An unavailable cursor or +changed identity is refused. A cursor older than the retained journal stops with +22 and says how to recover: a consumer behind the first retained index cannot +resume from this stream; rebuild it from a warmup that ends inside the retained +window, or run with a longer retention. SIGTERM/SIGINT stops intake, discards +staged messages that were never committed or published, and exits 0. A +directory whose journal holds only empty segments and no cursor (a crash before +the first cursor write) is taken over by a fresh `--start`; other existing state +needs `--resume`. A state directory from a release before the segmented journal +(`events.jsonl`) is refused; start a new one. Stdout is nonblocking during `run` for a five-second stop drain, and its original flags are restored on exit (a SIGKILL cannot restore them); a consumer @@ -321,17 +428,18 @@ waits at most 2 seconds for queued records. |---|---| | 0 | Requested clean stop / successful command | | 20 | Unhealable history gap, ambiguous fence, exhausted source retries | -| 21 | Changed overlap, conflicting duplicate, revised bar, changed cursor/prefix | -| 22 | Replay/disk/queue/allocation budget or durable I/O failure | +| 21 | Changed overlap, conflicting duplicate, revised bar, changed cursor/prefix/checkpoint | +| 22 | Expired output cursor, queue/allocation budget or durable I/O failure | | 23 | Unsupported input/source, permanent access, protocol/decimal/dependency failure, matched-time regression | -Default replay disk budget: 256 MiB (`--max-log-bytes`); each prefix traversal: -60 seconds (`--max-replay-seconds`, maximum 3600); source and unconfirmed-print -queues: 16 MiB each (`--max-queue-bytes`); WS text: 1 MiB; REST body: 1 MiB. -Warmup: 100000 rows / 16 MiB. Exhaustion is a cursor-preserving stop, not silent -compaction or dropped events. Raising limits is an explicit operator choice. -`--max-messages` permits bounded qualification. `--reconnect-seconds` can shorten -the 23h55 rotation for synthetic tests, never extend it beyond that limit. +Retained journal: `--segment-bytes` (16 MiB), `--replay-bytes` (256 MiB), +`--replay-age-seconds` (off). Each prefix traversal: 60 seconds +(`--max-replay-seconds`, maximum 3600); source and unconfirmed-print queues: +16 MiB each (`--max-queue-bytes`); WS text: 1 MiB; REST body: 1 MiB. Warmup and +export: 100000 rows / 16 MiB and 64 MiB. Raising limits is an explicit operator +choice. `--max-messages` permits bounded qualification. `--reconnect-seconds` can +shorten the 23h55 rotation for synthetic tests, never extend it beyond that +limit, and `--silence-seconds` shortens the data-silence threshold. ### Throughput @@ -344,32 +452,31 @@ mock venue, wall time including startup) ran 30,000 prints at 5,811, 11,212 and of a recent 89-day sample averaged 655 prints/s, about one ninth of the slowest run. -### Run-time horizon per mode +### Retention per mode -`--max-log-bytes` is a hard stop (22): this release neither rotates nor -checkpoints the journal. Journal segmentation and a checkpoint (prefix hash, -cut, sequence, current-minute aggregate, the 32 proofs) are planned together -with bounded replay windows. Until then the horizon is the budget divided by -the journal's growth: +The journal no longer has a terminal budget: it keeps the newest +`--replay-bytes` and expires older segments. The retained replay window is the +budget divided by the journal's growth: -| Mode | Bytes per message | Messages | Horizon at the default 256 MiB | +| Mode | Bytes per message | Messages | Window at the default 256 MiB | |---|---|---|---| | bars | about 138 | 1 per minute | about 3.7 years | | ticks, BTCUSDT median day | about 92 | 2.92 M per day | about 24 hours | | ticks, BTCUSDT busiest day | about 92 | 8.93 M per day | about 8 hours | Message sizes come from a public BTCUSDT capture; the daily print counts are -REST kline trade counts over 89 days. - -Raising the budget lengthens every whole-journal scan: at startup, on every -tick-mode reconnect, on a duplicate closed kline and on an old duplicate print, -each bounded by `--max-replay-seconds`. +REST kline trade counts over 89 days. A consumer must resume inside that window. +Startup verifies the whole retained window, and the journal scans for the open +minute, a duplicate closed candle or an old duplicate print start at the +segment that holds the minute or sequence, each bounded by +`--max-replay-seconds`. ### Tick mode needs a liquid symbol -75 seconds without any data message forces a reconnect, and each reconnect -costs an overlap check (on Binance spot a weight-27 `historicalTrades` page and -`klines` call). On Binance spot a minute without a print cannot be fenced, so +On OKX and Bybit 75 seconds without any data message forces a reconnect, and +each reconnect costs an overlap check; Binance proves a quiet connection with a +probe instead (see the reconnect paragraph above). On Binance spot a minute +without a print cannot be fenced, so tick mode stops with 20 there, and again on every resume. OKX closes a quiet minute on the next-print fence and its zero-volume candle, USD-M on the fence alone; the runner then sees a `time` event with no print in that minute. Use bars mode for @@ -385,7 +492,72 @@ verified cursor retained. Run the feed under a supervisor (a systemd unit with backoff, as `--resume --output-from N` with the runner's committed count. Exits 21 and 23 need an operator (changed history, protocol or access failure): an automatic restart repeats the stop. 22 needs a larger budget, more -disk, or a consumer that reads stdout again. +disk, a consumer that reads stdout again, or (an expired cursor) a consumer +rebuilt inside the retained window. + +## Serve: one producer, many runners + +```sh +build-feed/pineforge-feed serve --venue binance --market spot --symbol BTCUSDT \ + --mode bars --state-dir feed-state --start 2026-10-03T03:20:00Z --listen 127.0.0.1:8787 +curl -s http://127.0.0.1:8787/v1/status # epoch, source, retained range, last seq/time, clients +pineforge-live run --strategy strategy.so --warmup warmup.csv --mode bars --script-tf 1 \ + --feed-url 'ws://127.0.0.1:8787/v1/stream?epoch=E&from=0' --allow-insecure-http \ + --ledger runner.sqlite3 --symbol BINANCE:BTCUSDT --webhook-url https://receiver.example/actions +``` + +`serve` is `run` with its stdout replaced by a local HTTP/WebSocket server: one +producer per venue, market, symbol and mode owns the same durable journal (same +proofs, group commit, persist-before-publish, `--resume`, retention) and any +number of runners read it, instead of each opening its own venue connection. +Every WebSocket text message is exactly one journal message, so message +boundaries, and therefore the runner's `--from-input` count, are the journal's. + +- `GET /v1/status`: `epoch`, `source` (venue, market, symbol, mode, origins, + start, quantity multiplier), `retained` (`first_index`, `next_index`), `last` + (sequence, verified cut, last tick time, last bar), `clients` and the limits. +- `GET /v1/snapshot`: the complete committed prefix from index 0 as JSONL, the + only HTTP shape the runner accepts (it starts every snapshot at index 0 and + refuses more than 4 MiB). Once that prefix exceeds 4 MiB the endpoint answers + 413, and once index 0 has expired 410, with the reason in the body: use the + WebSocket stream. +- `ws://HOST:PORT/v1/stream?epoch=E&from=I`: every message from index `I` on, + first from the journal, then live. Cursor negotiation is HTTP, during the + upgrade, never an event in the runner's input: a malformed request answers + 400, a wrong epoch 409, a cursor beyond the committed log 416, an expired + cursor 410, a full server 503, each with a JSON reason. The runner reports any + of them as a failed handshake. + +Resume: bind each runner to the stream's epoch and the index `S` of its first +message (0 when its warmup ends at the producer's start). With N messages in its +ledger, a restarted runner connects with `from=S+N` and `--from-input N` and +receives exactly the remaining messages. A producer restart (`serve --resume`) +keeps the epoch, so runners reconnect the same way; the runner stops when the +producer goes away and is restarted with its own committed count. + +Bounds: each client reads the journal until it reaches the head and then drains +its own queue of released messages, at most `--client-queue-bytes` (default +1 MiB, which must hold one burst: a group commit releases up to 4096 messages at +once); `--max-clients` (default 64). A client whose queue overflows (it reads +slower than the stream, beyond the kernel's socket buffers) is closed with code +1008 and the reason `slow consumer`; a client whose cursor expires while it +catches up is closed with 4410. Neither ever blocks the producer or another +client. A client that stops reading entirely is dropped when its blocked write +times out (30 seconds). On SIGTERM the producer drains each live client's queue +and closes every client with 1001 (a client still catching up from the journal +is closed at once and resumes from its own count). + +Quiet streams: the runner's WebSocket idle deadline is 15 seconds and libcurl +answers PINGs without telling it, so `serve` sends an unsolicited PONG to a +client after 5 seconds without a message; a bar stream (one message a minute) +stays connected. + +Listening: `--listen` defaults to `127.0.0.1:8787`; port 0 picks a free port, +logged as `serve_listening`. **`serve` has no TLS and no authentication.** A +non-loopback address needs `--allow-remote-listen`; expose it only behind a +reverse proxy that terminates TLS and authenticates clients (for example nginx +or Caddy proxying `wss://` to the loopback listener), and never on a public +interface directly. ## Qualification and next adapters @@ -413,7 +585,14 @@ feed dependency. `src/venue.hpp` is the adapter boundary: a venue supplies its WebSocket connection (path, subscription frames, keepalive, frame classifier), decoding, REST history and candles, its tick proof and its print history window. -`src/session.cpp` holds the venue-neutral sessions. Coinbase remains deferred. -`serve` will add approved MIT CivetWeb only to its server target, bounded client -replay/queues, and cursor-bound fan-out; no server library is linked yet. The -engine executable remains `pineforge-live`. +`src/session.cpp` holds the venue-neutral sessions, `src/state.cpp` the +segmented journal, `src/serve.cpp` the fan-out and `src/export.cpp` the +prints-built history. Coinbase remains deferred. The engine executable remains +`pineforge-live`. + +`tests/serve_e2e.py` qualifies `serve` against the public venue and the real +runner: one producer, runners on `--feed-url`, a runner restart with +`--from-input`, a producer SIGKILL and `--resume`, each runner's ledger equal to +the journal, no idle-deadline stop, and the same REST and batch checks. +`public_e2e.py --export` also checks `export` over a tick window: its bars must +equal the bars built from the tape's prints, and the batch over them the runner. From 8f65e3c5332a95de167bdc976b227e3249ba1d23 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 08:37:10 +0800 Subject: [PATCH 04/15] Build CivetWeb with clang in the native sanitizer jobs The sanitizer jobs set only the C++ compiler; serve also compiles CivetWeb's C source, which must be instrumented by the same toolchain. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/native-feed.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/native-feed.yml b/.github/workflows/native-feed.yml index 1c7aeda..9e41b81 100644 --- a/.github/workflows/native-feed.yml +++ b/.github/workflows/native-feed.yml @@ -55,7 +55,7 @@ jobs: run: | export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_CXX_COMPILER=clang++-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ + -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ -DPINEFORGE_FEED_SANITIZER=address,undefined cmake --build build-feed -j4 ctest --test-dir build-feed --output-on-failure @@ -68,13 +68,13 @@ jobs: - uses: actions/checkout@v7 - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev clang-18 - run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" - - name: Threaded receive, healing and termination + - name: Threaded receive, healing, termination and serve fan-out env: TSAN_OPTIONS: halt_on_error=1 run: | export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_CXX_COMPILER=clang++-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ + -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ -DPINEFORGE_FEED_SANITIZER=thread cmake --build build-feed -j4 ctest --test-dir build-feed --output-on-failure From 89c5aef075ef499237306e7dc283bd832a06025e Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 08:46:49 +0800 Subject: [PATCH 05/15] Build CivetWeb with a locked stop flag; suppress its stop CAS read ThreadSanitizer found CivetWeb 1.16's stop flag written by mg_stop while its master thread reads it. STOP_FLAG_NEEDS_LOCK makes those accesses atomic; the one remaining report is STOP_FLAG_ASSIGN's plain read of the expected value inside its compare-and-swap loop, which tests/tsan.supp suppresses by that function name only. The TSan CTest rows use it. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/CMakeLists.txt | 6 +++++- native/feed/tests/tsan.supp | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 native/feed/tests/tsan.supp diff --git a/native/feed/CMakeLists.txt b/native/feed/CMakeLists.txt index 110b8bd..3f7629e 100644 --- a/native/feed/CMakeLists.txt +++ b/native/feed/CMakeLists.txt @@ -69,7 +69,7 @@ if(PINEFORGE_FEED_SERVE) FetchContent_MakeAvailable(civetweb) add_library(pineforge_feed_civetweb STATIC "${civetweb_SOURCE_DIR}/src/civetweb.c") target_include_directories(pineforge_feed_civetweb SYSTEM PUBLIC "${civetweb_SOURCE_DIR}/include") - target_compile_definitions(pineforge_feed_civetweb PRIVATE NO_SSL NO_CGI NO_FILES NO_CACHING + target_compile_definitions(pineforge_feed_civetweb PRIVATE NO_SSL NO_CGI NO_FILES NO_CACHING STOP_FLAG_NEEDS_LOCK PUBLIC USE_WEBSOCKET) set_target_properties(pineforge_feed_civetweb PROPERTIES C_STANDARD 99) target_compile_options(pineforge_feed_civetweb PRIVATE -w) @@ -100,4 +100,8 @@ if(BUILD_TESTING) set_tests_properties(feed-mock-export PROPERTIES TIMEOUT 300) add_test(NAME feed-mock COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/mock_venue.py $) set_tests_properties(feed-mock PROPERTIES TIMEOUT 420) + if(PINEFORGE_FEED_SANITIZER STREQUAL "thread") + set_tests_properties(feed-unit feed-export-unit feed-mock feed-mock-export PROPERTIES ENVIRONMENT + "TSAN_OPTIONS=halt_on_error=1 suppressions=${CMAKE_CURRENT_SOURCE_DIR}/tests/tsan.supp") + endif() endif() diff --git a/native/feed/tests/tsan.supp b/native/feed/tests/tsan.supp new file mode 100644 index 0000000..518c6cb --- /dev/null +++ b/native/feed/tests/tsan.supp @@ -0,0 +1,5 @@ +# ThreadSanitizer suppressions for the feed's TSan suite; nothing in the feed's own code is suppressed. +# CivetWeb 1.16 (built with STOP_FLAG_NEEDS_LOCK): STOP_FLAG_ASSIGN passes the flag's current value to its +# compare-and-swap loop with a plain read, which races benignly with the atomic reads of mg_stop and the +# master thread during shutdown. +race:STOP_FLAG_ASSIGN From 84812cc585c22b34add54cc1f027f8ceda93c370 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 08:56:06 +0800 Subject: [PATCH 06/15] Let a USD-M export find its start after a quiet first hour The USD-M start lookup searches hour windows forward only up to the newest venue time it has seen, which export (no WebSocket) never set: a window whose first aggregate came more than an hour after --start stopped 20. Venue::horizon lets export pass the clock, bounded by the 48-hour history. README throughput adds the segmented-journal and serve bench figures. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/README.md | 14 +++++++++----- native/feed/src/binance.hpp | 2 ++ native/feed/src/export.cpp | 1 + native/feed/src/venue.hpp | 3 +++ native/feed/tests/mock_export.py | 17 +++++++++++++++++ 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/native/feed/README.md b/native/feed/README.md index 8b6934e..d45c793 100644 --- a/native/feed/README.md +++ b/native/feed/README.md @@ -115,8 +115,9 @@ from the last print strictly before `--start` through the fence, the first print at or after `--end`. A hole, a missing fence (the window has not closed yet), or a start outside the venue's REST history (USD-M 48 hours) stops with 20 and writes nothing; time running backwards along the chain stops with 23. From REST -the chain is paged by ID (Binance 1000, OKX 100 per page); a USD-M start needs an -aggregate within the hour after it. +the chain is paged by ID (Binance 1000, OKX 100 per page); a USD-M start is found +by searching hour windows forward from `--start` up to the present (within the +48-hour history), so a quiet first hour is not a stop. `--archive` reads the Binance USD-M daily `aggTrades` archive as published (the `.zip`, inflated in-process and CRC-checked, or the extracted `.csv`, with or @@ -448,9 +449,12 @@ Linux test machine (ext4 root), committing each message separately (three fsyncs per print) sustained about 120 messages/s. With group commit, the loopback bench (synthetic contiguous prints through the real binary and the mock venue, wall time including startup) ran 30,000 prints at 5,811, 11,212 and -10,842 messages/s in three runs on the same machine. The busiest BTCUSDT minute -of a recent 89-day sample averaged 655 prints/s, about one ninth of the slowest -run. +10,842 messages/s in three runs on the same machine under load; with the +segmented journal, on an idle machine, 30,000 prints ran at about 42,000 +messages/s, and through `serve` one client received a 30,000-print burst at about +19,000 messages/s (eight clients at once, about 17,000 each). The busiest BTCUSDT +minute of a recent 89-day sample averaged 655 prints/s, about one ninth of the +slowest run. ### Retention per mode diff --git a/native/feed/src/binance.hpp b/native/feed/src/binance.hpp index a4cd5d3..303632d 100644 --- a/native/feed/src/binance.hpp +++ b/native/feed/src/binance.hpp @@ -2,6 +2,7 @@ #pragma once #include "transport.hpp" #include "venue.hpp" +#include namespace pineforge::feed { Trade binance_trade(const Json& trade, bool websocket); @@ -43,6 +44,7 @@ class BinanceUsdm final : public Venue { std::vector history(std::uint64_t from, std::size_t limit = 1000) override; std::uint64_t first_trade_id(std::int64_t minute) override; std::optional predecessor_if_ready(std::int64_t minute) override; + void horizon(std::int64_t ts) override { newest_ = std::max(newest_, ts); } std::vector klines(std::int64_t start, std::int64_t end) override; TickProof tick_proof() const override { return TickProof::NextPrintFence; } std::int64_t retention_ms() const override { return 48LL * 3600000; } diff --git a/native/feed/src/export.cpp b/native/feed/src/export.cpp index 0dd66e1..5f719fa 100644 --- a/native/feed/src/export.cpp +++ b/native/feed/src/export.cpp @@ -525,6 +525,7 @@ ChainBars rest_bars(const Config& config, std::string& multiplier) { const auto now = now_ms(); if (config.end > now) throw Error(20, "the window has not closed yet: no print at or after its end can exist"); const auto venue = make_venue(config); + venue->horizon(now); multiplier = venue->qty_multiplier(); const auto window = venue->retention_ms(); if (window && now - config.start > window) diff --git a/native/feed/src/venue.hpp b/native/feed/src/venue.hpp index 07dea21..3720377 100644 --- a/native/feed/src/venue.hpp +++ b/native/feed/src/venue.hpp @@ -54,6 +54,9 @@ class Venue { virtual std::optional predecessor_if_ready(std::int64_t minute) = 0; // Re-checks instrument metadata that fixes the stream's units; a change stops 21. Called on connect. virtual void reverify() {} + // The newest venue time the caller knows of without a WebSocket (export passes the clock): bounds a + // start lookup that searches forward through time windows (USD-M). + virtual void horizon(std::int64_t) {} // Contiguous closed one-minute candles in [start, end), ascending. virtual std::vector klines(std::int64_t start, std::int64_t end) = 0; virtual TickProof tick_proof() const { return TickProof::KlineIdRange; } diff --git a/native/feed/tests/mock_export.py b/native/feed/tests/mock_export.py index 9aaea17..011f232 100644 --- a/native/feed/tests/mock_export.py +++ b/native/feed/tests/mock_export.py @@ -164,6 +164,23 @@ def test_rest_bars_follow_the_rule_with_quiet_minute_and_carry(self): (manifest["prints"], manifest["bars"], manifest["quiet_minutes"]), (6, 4, 1) ) + def test_rest_start_survives_a_quiet_first_hour(self): + # The first aggregate after --start lies in the second hour window: the start lookup searches + # forward up to the clock instead of stopping 20 after one window. + start = recent_minute(90) + prints = [(499, start - 1, "10.10000000", "0.10000000")] + prints += synthetic(500, start + 70 * 60000, 3, 4) + server = self.server(prints) + self.export(start, start + 72 * 60000, server) + self.assertEqual(self.output.read_text(), expected_bars(prints, start, start + 72 * 60000)) + self.assertEqual(self.manifest()["quiet_minutes"], 70) + windows = [ + int(query["startTime"][0]) + for path, query in server.queries + if path == "/fapi/v1/aggTrades" and "startTime" in query + ] + self.assertIn(start + 3600000, windows) + def test_rest_pages_by_from_id_and_sums_volume_exactly(self): start = recent_minute(40) prints = synthetic(1000, start - 60000, 6, 700, quiet=(2,)) From 144d718537479b2eb6e8ac405295d45be7ae3007 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 09:16:50 +0800 Subject: [PATCH 07/15] Wrap a long comment in the export mock test Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/tests/mock_export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/native/feed/tests/mock_export.py b/native/feed/tests/mock_export.py index 011f232..0ac3f23 100644 --- a/native/feed/tests/mock_export.py +++ b/native/feed/tests/mock_export.py @@ -165,8 +165,8 @@ def test_rest_bars_follow_the_rule_with_quiet_minute_and_carry(self): ) def test_rest_start_survives_a_quiet_first_hour(self): - # The first aggregate after --start lies in the second hour window: the start lookup searches - # forward up to the clock instead of stopping 20 after one window. + # The first aggregate after --start lies in the second hour window: the start lookup + # searches forward up to the clock instead of stopping 20 after one window. start = recent_minute(90) prints = [(499, start - 1, "10.10000000", "0.10000000")] prints += synthetic(500, start + 70 * 60000, 3, 4) From a5c28351a9ce3f82851d708d28ee977c9babd7b5 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 09:52:32 +0800 Subject: [PATCH 08/15] Harden serve clients, retention and recovery after review serve: a client holds a journal segment only while it catches up (one at the head opens none), so retained segments are not pinned by live clients; serve raises its open-file limit and refuses to start below 2 x --max-clients + 64; a client whose segment expires while it catches up is closed with 4410 at once instead of after a silent 30 s; the stopping check no longer sends a close frame under the hub mutex and is read per line while catching up; an EMFILE is an I/O error, never an expired cursor; no exception unwinds into CivetWeb callbacks. Journal: in tick modes the segment holding the newest print never expires, so a quiet fence-venue stream can still re-read its recent prints on resume; the predecessor overlap is checked only before the first print. Recovery removes only real crash leftovers (the empty successor of an interrupted seal, the oldest checkpoint of an interrupted expiry), stops 21 on anything else, truncates the open segment only after the journal verified, and maps filesystem errors to 22. Expiry erases its entry and syncs the directory between the two unlinks; a failed seal or expiry poisons the journal. Lookups stop at the first match, tick sessions compare a duplicate candle with the proofs only, and a retained duplicate is checked before the expiry rule applies. Binance probe: a reply must list every stream of the connection; an error reply or a partial list reconnects instead of being decoded as data (23). REST weight windows are keyed to the request's send time. A --max-queue-bytes below one connection marker stops 22 instead of waiting forever. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/README.md | 30 ++++--- native/feed/src/binance.cpp | 31 +++++-- native/feed/src/binance.hpp | 4 +- native/feed/src/serve.cpp | 146 +++++++++++++++++++++++--------- native/feed/src/serve.hpp | 4 +- native/feed/src/session.cpp | 17 ++-- native/feed/src/state.cpp | 103 ++++++++++++++-------- native/feed/src/state.hpp | 6 +- native/feed/src/transport.cpp | 10 ++- native/feed/tests/mock_venue.py | 34 +++++++- native/feed/tests/unit.cpp | 61 ++++++++++--- 11 files changed, 329 insertions(+), 117 deletions(-) diff --git a/native/feed/README.md b/native/feed/README.md index d45c793..a242bff 100644 --- a/native/feed/README.md +++ b/native/feed/README.md @@ -310,9 +310,10 @@ Curl answers server PING with matching-payload PONG; OKX and Bybit get their text keepalives. Keepalive replies are control messages: they never count as data. After `--silence-seconds` (default 75) without data, OKX and Bybit reconnect; Binance instead sends a `LIST_SUBSCRIPTIONS` probe, and a reply that -lists the subscriptions proves the connection and its streams alive, so a quiet -symbol no longer pays a reconnect and its REST overlap every 75 seconds (an -empty list, or no reply within 10 seconds, reconnects). This is safe because +lists every stream of the connection proves the connection and its streams +alive, so a quiet symbol no longer pays a reconnect and its REST overlap every +75 seconds (an error reply, a list missing a stream, or no reply within 10 +seconds reconnects; it is never taken for market data). This is safe because completeness never rests on the connection: every print is chained by ID, every minute behind a confirmed watermark, and every reconnect re-verifies its overlap. Reconnect also occurs on `serverShutdown` or an OKX `notice`, transport loss, or @@ -380,12 +381,18 @@ with `--replay-age-seconds`, while a segment's last minute is older than that in venue time; a segment file goes before its checkpoint, so a crash in between leaves an orphan checkpoint that resume removes. The open segment and every segment that can hold a message of the protected window (the open minute and the -32 closed-minute proofs the reconnect overlap re-verifies) never expire; when -they alone exceed the budget the feed keeps them and logs -`replay_budget_held_by_protected_window`. Resume verifies the retained journal +32 closed-minute proofs the reconnect overlap re-verifies) never expire, nor, in +tick modes, the segment holding the newest print (a reconnect re-reads recent +prints from the journal, and a quiet next-print-fence stream keeps closing +minutes without any); when they alone exceed the budget the feed keeps them +and logs `replay_budget_held_by_protected_window`. Resume verifies the retained journal from its oldest checkpoint, checks every segment boundary against its -checkpoint and the end against the cursor, and fsyncs truncation of an -uncommitted crash tail. Every crash point of a commit, a seal and an expiry is +checkpoint and the end against the cursor, and only then fsyncs truncation of an +uncommitted crash tail. It removes only what a crash can leave (the empty +successor of an interrupted seal, the oldest checkpoint of an interrupted +expiry); any other unexpected segment or checkpoint stops with 21 and is left in +place. Resume reads the whole retained window within `--max-replay-seconds`, so +a very large `--replay-bytes` needs a larger budget there. Every crash point of a commit, a seal and an expiry is exercised by the test suite (`PINEFORGE_FEED_CRASH_AT`, a test hook that kills the process at a named step) and by random kills. @@ -545,8 +552,11 @@ its own queue of released messages, at most `--client-queue-bytes` (default once); `--max-clients` (default 64). A client whose queue overflows (it reads slower than the stream, beyond the kernel's socket buffers) is closed with code 1008 and the reason `slow consumer`; a client whose cursor expires while it -catches up is closed with 4410. Neither ever blocks the producer or another -client. A client that stops reading entirely is dropped when its blocked write +catches up is closed with 4410 as soon as it reaches the expired segment. +Neither ever blocks the producer or another client. A client holds a journal +segment open only while it catches up. At start `serve` raises its open-file +limit to the hard limit and refuses to start (23) when that is below +`2 x --max-clients + 64`. A client that stops reading entirely is dropped when its blocked write times out (30 seconds). On SIGTERM the producer drains each live client's queue and closes every client with 1001 (a client still catching up from the journal is closed at once and resumes from its own count). diff --git a/native/feed/src/binance.cpp b/native/feed/src/binance.cpp index 72c7d17..6b33e2c 100644 --- a/native/feed/src/binance.cpp +++ b/native/feed/src/binance.cpp @@ -71,15 +71,21 @@ Trade usdm_aggregate(const Json& value) { trade.validate(); return trade; } -Frame binance_frame(const std::string& message) { +Frame binance_frame(const std::string& message, const std::vector& streams) { try { const auto envelope = parse_json(message); // The reply to the liveness probe: {"result":[subscribed streams],"id":N}. Without them all, reconnect. - if (const auto* result = envelope.find("result")) { - if (!envelope.find("id")) throw Error(23, "unexpected public stream reply"); - if (result->kind != Json::Kind::Array || result->items.empty()) return Frame::Stale; + if (envelope.find("id") && !envelope.find("stream") && !envelope.find("e")) { + const auto* result = envelope.find("result"); + if (!result || result->kind != Json::Kind::Array || result->items.empty()) return Frame::Stale; + for (const auto& stream : streams) { + bool listed = false; + for (const auto& item : result->items) listed = listed || (item.kind == Json::Kind::String && item.text() == stream); + if (!listed) return Frame::Stale; + } return Frame::Control; } + if (envelope.find("result")) throw Error(23, "unexpected public stream reply"); const auto* wrapped = envelope.find("data"); const auto& data = wrapped ? *wrapped : envelope; const auto* event = data.find("e"); @@ -87,10 +93,19 @@ Frame binance_frame(const std::string& message) { } catch (const std::exception&) { throw Error(23, "invalid public WebSocket JSON"); } } +namespace { +// A combined-stream connection whose probe reply must list every one of its streams. +Connection combined(const std::string& prefix, const std::vector& streams) { + std::string path = prefix; + for (const auto& stream : streams) path += (path.size() == prefix.size() ? "" : "/") + stream; + return {path, {}, {}, [streams](const std::string& message) { return binance_frame(message, streams); }, + "{\"method\":\"LIST_SUBSCRIPTIONS\",\"id\":1}"}; +} +} Connection BinanceSpot::connection() const { const auto symbol = lower(config_.symbol); - return {"/stream?streams=" + (config_.mode == "ticks" ? symbol + "@trade/" + symbol + "@kline_1m" : symbol + "@kline_1m"), {}, {}, - &binance_frame, "{\"method\":\"LIST_SUBSCRIPTIONS\",\"id\":1}"}; + if (config_.mode == "ticks") return combined("/stream?streams=", {symbol + "@trade", symbol + "@kline_1m"}); + return combined("/stream?streams=", {symbol + "@kline_1m"}); } std::vector BinanceSpot::decode(const std::string& message) const { try { @@ -182,8 +197,8 @@ RestPolicy usdm_rest() { unsigned int usdm_kline_weight(std::size_t limit) { return limit < 100 ? 1 : limit < 500 ? 2 : limit <= 1000 ? 5 : 10; } Connection usdm_connection(const Config& config) { const auto symbol = lower(config.symbol); - return {"/market/stream?streams=" + (config.mode == "agg-ticks" ? symbol + "@aggTrade/" + symbol + "@kline_1m" : symbol + "@kline_1m"), - {}, {}, &binance_frame, "{\"method\":\"LIST_SUBSCRIPTIONS\",\"id\":1}"}; + if (config.mode == "agg-ticks") return combined("/market/stream?streams=", {symbol + "@aggTrade", symbol + "@kline_1m"}); + return combined("/market/stream?streams=", {symbol + "@kline_1m"}); } BinanceUsdm::BinanceUsdm(const Config& config) : config_(config), http_(config, usdm_rest()) { // An unknown symbol is refused before any stream is opened (HTTP 400, code -1121). diff --git a/native/feed/src/binance.hpp b/native/feed/src/binance.hpp index 303632d..7ecfa32 100644 --- a/native/feed/src/binance.hpp +++ b/native/feed/src/binance.hpp @@ -10,7 +10,9 @@ Kline binance_kline(const Json& row, bool websocket); // One USD-M aggregate print: `a` is the sequence, `q` the quantity. Its raw `f..l` range is validated, // never expanded into imaginary raw trades. Trade usdm_aggregate(const Json& row); -Frame binance_frame(const std::string& message); +// A probe reply (a top-level `id`) proves the connection alive only if its `result` lists every stream in +// `streams`; an error or a partial list is a lost subscription (Stale), never market data. +Frame binance_frame(const std::string& message, const std::vector& streams = {}); Connection usdm_connection(const Config& config); unsigned int usdm_kline_weight(std::size_t limit); diff --git a/native/feed/src/serve.cpp b/native/feed/src/serve.cpp index fae789f..2bff8db 100644 --- a/native/feed/src/serve.cpp +++ b/native/feed/src/serve.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include namespace pineforge::feed { @@ -45,6 +46,7 @@ struct Hub { std::set clients; std::size_t connected = 0, serving = 0; bool stopping = false; + std::atomic halting{false}; // stopping, readable without the mutex by clients catching up void publish(const std::string& line) { const auto shared = std::make_shared(line); @@ -79,6 +81,7 @@ struct Hub { void stop() { std::unique_lock guard(mutex); stopping = true; + halting = true; changed.notify_all(); changed.wait_for(guard, std::chrono::seconds(3), [this] { return serving == 0; }); } @@ -103,7 +106,7 @@ bool get_only(mg_connection* connection) { return false; } -int on_status(mg_connection* connection, void* data) { +int status_page(mg_connection* connection, void* data) { if (!get_only(connection)) return 405; auto& hub = *static_cast(data); std::string body; @@ -121,7 +124,7 @@ int on_status(mg_connection* connection, void* data) { } // The complete committed prefix from index 0, the only HTTP shape the current runner accepts. -int on_snapshot(mg_connection* connection, void* data) { +int snapshot_page(mg_connection* connection, void* data) { if (!get_only(connection)) return 405; auto& hub = *static_cast(data); std::uint64_t published = 0, first = 0; @@ -137,7 +140,7 @@ int on_snapshot(mg_connection* connection, void* data) { if (!body) return refuse(connection, 413, "the committed prefix exceeds the runner's 4 MiB HTTP snapshot bound; use /v1/stream"); respond(connection, 200, "application/x-ndjson", *body); return 200; - } catch (const Error&) { return refuse(connection, 410, expired); } + } catch (const Error& failure) { return failure.code == 20 ? refuse(connection, 410, expired) : refuse(connection, 500, failure.what()); } } bool query_value(const std::string& query, const char* name, std::string& value) { @@ -148,7 +151,7 @@ bool query_value(const std::string& query, const char* name, std::string& value) return true; } -int on_connect(const mg_connection* constant, void* data) { +int admit(const mg_connection* constant, void* data) { auto* connection = const_cast(constant); auto& hub = *static_cast(data); const auto* request = mg_get_request_info(connection); @@ -185,6 +188,20 @@ int on_connect(const mg_connection* constant, void* data) { return 0; } +// No exception may unwind into CivetWeb's C code: a failing request answers 500, a failing upgrade is refused. +int on_status(mg_connection* connection, void* data) { + try { return status_page(connection, data); } + catch (...) { return refuse(connection, 500, "internal error"); } +} +int on_snapshot(mg_connection* connection, void* data) { + try { return snapshot_page(connection, data); } + catch (...) { return refuse(connection, 500, "internal error"); } +} +int on_connect(const mg_connection* connection, void* data) { + try { return admit(connection, data); } + catch (...) { return 1; } +} + bool send_close(mg_connection* connection, unsigned short code, const std::string& reason) { std::string payload; payload += static_cast(code >> 8); @@ -195,25 +212,7 @@ bool send_close(mg_connection* connection, unsigned short code, const std::strin return mg_websocket_write(connection, MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE, payload.data(), payload.size()) > 0; } -// One client, on its CivetWeb worker thread: the retained journal from its cursor, then live messages from -// its bounded queue, one WebSocket text message per normalized message, and an unsolicited PONG after 5 s -// without a message so a quiet stream never trips the runner's idle deadline. -void on_ready(mg_connection* connection, void* data) { - auto& hub = *static_cast(data); - auto* client = static_cast(mg_get_user_connection_data(connection)); - if (!client) return; - { - std::lock_guard guard(hub.mutex); - ++hub.serving; - } - struct Done { - Hub& hub; - ~Done() { - std::lock_guard guard(hub.mutex); - --hub.serving; - hub.changed.notify_all(); - } - } done{hub}; +void serve_client(mg_connection* connection, Hub& hub, Client* client) { auto last_send = std::chrono::steady_clock::now(); // A write that cannot complete within the request timeout leaves a partial frame: the connection is dead. const auto send = [&](const std::string& bytes, int opcode) { @@ -225,33 +224,55 @@ void on_ready(mg_connection* connection, void* data) { last_send = std::chrono::steady_clock::now(); return true; }; - JournalReader reader(hub.journal); - if (!reader.seek(client->next)) { - send_close(connection, 4410, "cursor expired: older than the retained journal"); - return; - } + const auto expired = [&] { + std::lock_guard guard(hub.mutex); + return client->next < hub.first_retained; + }; + // The journal is read only while catching up: a client at the head holds no segment open. + std::optional reader; for (;;) { std::uint64_t head = 0; + bool stopping = false; { std::lock_guard guard(hub.mutex); - if (hub.stopping) { send_close(connection, 1001, "producer stopping"); return; } + stopping = hub.stopping; head = hub.published; - if (client->next == head) { client->live = true; break; } + if (!stopping && client->next == head) client->live = true; + } + if (stopping) { send_close(connection, 1001, "producer stopping"); return; } + if (client->live) break; + if (!reader) { + reader.emplace(hub.journal); + if (!reader->seek(client->next)) { + send_close(connection, 4410, "cursor expired: older than the retained journal"); + return; + } } while (client->next < head) { - auto line = reader.next(); + if (hub.halting) { send_close(connection, 1001, "producer stopping"); return; } + auto line = reader->next(); if (!line) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (std::chrono::steady_clock::now() - last_send > std::chrono::seconds(30)) { + // A released line is on disk: missing means its segment expired (or the reader cannot open it). + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!line && std::chrono::steady_clock::now() < deadline && !hub.halting) { + if (expired()) { + send_close(connection, 4410, "cursor expired while catching up: the journal moved past it"); + return; + } + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + line = reader->next(); + } + if (!line) { + if (hub.halting) continue; send_close(connection, 1011, "journal segment unavailable"); return; } - continue; } if (!send(*line, MG_WEBSOCKET_OPCODE_TEXT)) return; ++client->next; } } + reader.reset(); for (;;) { std::shared_ptr line; { @@ -281,6 +302,32 @@ void on_ready(mg_connection* connection, void* data) { } else if (std::chrono::steady_clock::now() - last_send >= keepalive && !send("", MG_WEBSOCKET_OPCODE_PONG)) return; } } + +// One client, on its CivetWeb worker thread: the retained journal from its cursor, then live messages from +// its bounded queue, one WebSocket text message per normalized message, and an unsolicited PONG after 5 s +// without a message so a quiet stream never trips the runner's idle deadline. Nothing may unwind into +// CivetWeb's C code. +void on_ready(mg_connection* connection, void* data) { + auto& hub = *static_cast(data); + auto* client = static_cast(mg_get_user_connection_data(connection)); + if (!client) return; + { + std::lock_guard guard(hub.mutex); + ++hub.serving; + } + struct Done { + Hub& hub; + ~Done() { + std::lock_guard guard(hub.mutex); + --hub.serving; + hub.changed.notify_all(); + } + } done{hub}; + try { serve_client(connection, hub, client); } + catch (...) { + try { send_close(connection, 1011, "internal error"); } catch (...) {} + } +} int on_data(mg_connection*, int, char*, std::size_t, void*) { return 1; } void on_close(const mg_connection*, void*) {} @@ -294,9 +341,10 @@ void on_connection_close(const mg_connection* connection) { hub->clients.erase(client); --hub->connected; } - log("info", "client_disconnected", Json::object({{"next", number(client->next)}})); + const auto next = client->next; delete client; mg_set_user_connection_data(connection, nullptr); + try { log("info", "client_disconnected", Json::object({{"next", number(next)}})); } catch (...) {} } int on_log(const mg_connection*, const char* message) { log("warn", "http_server", Json::object({{"message", Json::string(std::string(message).substr(0, 512))}})); @@ -339,10 +387,13 @@ class Server { JournalReader::~JournalReader() { if (descriptor_ >= 0) ::close(descriptor_); } +// False only when the segment does not exist (expired, or not yet sealed into existence); any other failure, +// such as running out of descriptors, is an I/O error and never passes for an expired cursor. bool JournalReader::open(std::uint64_t base) { const auto path = directory_ + "/" + segment_name(base) + ".jsonl"; const int descriptor = ::open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (descriptor < 0) return false; + if (descriptor < 0 && errno == ENOENT) return false; + if (descriptor < 0) throw Error(22, "cannot open a journal segment for a client"); if (descriptor_ >= 0) ::close(descriptor_); descriptor_ = descriptor; buffer_.clear(); @@ -357,14 +408,16 @@ bool JournalReader::seek(std::uint64_t index) { std::uint64_t chosen = 0; bool found = false; std::error_code failure; - for (const auto& entry : std::filesystem::directory_iterator(directory_, failure)) { - const auto name = entry.path().filename().string(); + std::filesystem::directory_iterator entries(directory_, failure), end; + for (; !failure && entries != end; entries.increment(failure)) { + const auto name = entries->path().filename().string(); if (name.size() != 26 || name.compare(20, 6, ".jsonl") != 0) continue; std::uint64_t base = 0; if (std::from_chars(name.data(), name.data() + 20, base).ec != std::errc{}) continue; if (base <= index && (!found || base > chosen)) { chosen = base; found = true; } } - if (failure || !found) return false; + if (failure) throw Error(22, "cannot list the journal for a client"); + if (!found) return false; if (!open(chosen)) continue; while (index_ < index) { auto skipped = next(); @@ -397,13 +450,13 @@ std::optional JournalReader::next() { std::optional snapshot_prefix(const std::string& journal, std::uint64_t published, std::size_t limit) { JournalReader reader(journal); - if (!reader.seek(0)) throw Error(22, "the snapshot prefix has expired"); + if (!reader.seek(0)) throw Error(20, "the snapshot prefix has expired"); std::string body; const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); while (reader.index() < published) { auto line = reader.next(); if (!line) { - if (std::chrono::steady_clock::now() >= deadline) throw Error(22, "a snapshot segment expired while it was read"); + if (std::chrono::steady_clock::now() >= deadline) throw Error(20, "a snapshot segment expired while it was read"); std::this_thread::sleep_for(std::chrono::milliseconds(1)); continue; } @@ -434,6 +487,17 @@ std::string listen_address(const std::string& listen, bool allow_remote) { void serve_feed(Config config) { const auto address = listen_address(config.listen, config.allow_remote_listen); + // Each client holds a socket (and a journal segment while it catches up); the producer needs its own. + rlimit files{}; + if (::getrlimit(RLIMIT_NOFILE, &files) == 0 && files.rlim_cur < files.rlim_max) { + files.rlim_cur = files.rlim_max; + ::setrlimit(RLIMIT_NOFILE, &files); + ::getrlimit(RLIMIT_NOFILE, &files); + } + const auto needed = static_cast(2 * config.max_clients + 64); + if (files.rlim_cur != RLIM_INFINITY && files.rlim_cur < needed) + throw Error(23, "the open-file limit (" + std::to_string(files.rlim_cur) + ") is below " + std::to_string(needed) + + " for --max-clients " + std::to_string(config.max_clients) + "; raise it or lower --max-clients"); // Instrument metadata fixes the quantity units before the cursor binds them. const auto venue = make_venue(config); config.qty_multiplier = venue->qty_multiplier(); diff --git a/native/feed/src/serve.hpp b/native/feed/src/serve.hpp index c1e429f..e31856a 100644 --- a/native/feed/src/serve.hpp +++ b/native/feed/src/serve.hpp @@ -25,8 +25,8 @@ class JournalReader { std::uint64_t index() const { return index_; } }; -// The committed prefix [0, published) as JSONL, or nullopt when it exceeds `limit` bytes; throws 22 when index 0 -// has expired or a segment disappears while it is read. +// The committed prefix [0, published) as JSONL, or nullopt when it exceeds `limit` bytes; throws 20 when index 0 +// has expired or a segment disappears while it is read, 22 on an I/O error. std::optional snapshot_prefix(const std::string& journal, std::uint64_t published, std::size_t limit); // The listening address: loopback unless the operator explicitly allows another interface. std::string listen_address(const std::string& listen, bool allow_remote); diff --git a/native/feed/src/session.cpp b/native/feed/src/session.cpp index c74663c..2402bc0 100644 --- a/native/feed/src/session.cpp +++ b/native/feed/src/session.cpp @@ -116,7 +116,7 @@ void FeedSession::connected() { if (fetched.size() != trades.size()) throw Error(20, "reconnect raw overlap is unavailable"); for (std::size_t index = 0; index < trades.size(); ++index) if (!(trades[index] == fetched[index])) throw Error(21, "reconnect raw overlap changed"); - } else if (state_.cursor().predecessor) { + } else if (!state_.cursor().seq && state_.cursor().predecessor) { const auto fetched = venue_.history(state_.cursor().predecessor->id, 1); if (!(fetched.front() == *state_.cursor().predecessor)) throw Error(21, "initial predecessor overlap changed"); } @@ -159,11 +159,12 @@ void FeedSession::put(const Trade& trade) { trade.validate(); if (trade.ts < state_.cursor().start) return; if (state_.cursor().seq && trade.id <= state_.cursor().seq) { - if (state_.expired_seq(trade.id)) { + auto previous = state_.recent_trade(trade.id); + if (!previous && state_.expired_seq(trade.id)) { log("warn", "expired_duplicate_unverified", Json::object({{"seq", Json::number(std::to_string(trade.id))}})); return; } - const auto previous = state_.trade(trade.id); + if (!previous) previous = state_.trade(trade.id); if (!previous || !(*previous == trade)) throw Error(21, "duplicate raw trade conflicts with the immutable prefix"); return; } @@ -258,7 +259,8 @@ void FeedSession::closed(const Kline& kline) { return; } if (minute < state_.cursor().cut) { - const auto old = state_.bar(minute); + // Tick journals hold no bar lines: only the closed-minute proofs can hold this candle. + const auto old = state_.config().mode == "bars" ? state_.bar(minute) : state_.proof(minute); if (old) { if (!(*old == kline.bar)) throw Error(21, "already-emitted closed bar was revised"); } @@ -353,11 +355,12 @@ void FenceSession::put(const Trade& trade, bool healed) { throw Error(23, "venue matched time regressed: a print before the proven predecessor is at or after the start minute"); } if (state_.cursor().seq && trade.id <= state_.cursor().seq) { - if (state_.expired_seq(trade.id)) { + auto previous = state_.recent_trade(trade.id); + if (!previous && state_.expired_seq(trade.id)) { log("warn", "expired_duplicate_unverified", Json::object({{"seq", Json::number(std::to_string(trade.id))}})); return; } - const auto previous = state_.trade(trade.id); + if (!previous) previous = state_.trade(trade.id); if (!previous || !(*previous == trade)) throw Error(21, "duplicate print conflicts with the immutable prefix"); return; } @@ -530,7 +533,7 @@ void FenceSession::connected() { if (fetched.size() != trades.size()) throw Error(20, "reconnect print overlap is unavailable"); for (std::size_t index = 0; index < trades.size(); ++index) if (!(trades[index] == fetched[index])) throw Error(21, "reconnect print overlap changed"); - } else if (state_.cursor().predecessor) { + } else if (!state_.cursor().seq && state_.cursor().predecessor) { const auto fetched = venue_.history(state_.cursor().predecessor->id, 1); if (fetched.empty()) throw Error(20, "initial predecessor overlap is unavailable"); if (!(fetched.front() == *state_.cursor().predecessor)) throw Error(21, "initial predecessor overlap changed"); diff --git a/native/feed/src/state.cpp b/native/feed/src/state.cpp index e5303d6..2dd0372 100644 --- a/native/feed/src/state.cpp +++ b/native/feed/src/state.cpp @@ -220,7 +220,7 @@ void State::initialize() { sync_directory(journal_dir()); persist(cursor_); durable_ = cursor_; - segments_ = {{0, 0, 0, cursor_.start}}; + segments_ = {{0, 0, 0, cursor_.start, -1}}; } void State::recover() { @@ -267,6 +267,7 @@ void State::recover() { for (const auto& entry : std::filesystem::directory_iterator(journal_dir())) { const auto name = entry.path().filename().string(); std::uint64_t base = 0; + if (!name.empty() && name.front() == '.') continue; if (name.size() > 4 && name.compare(name.size() - 4, 4, ".tmp") == 0) { std::filesystem::remove(entry.path()); continue; } if (name.size() == 26 && name.compare(20, 6, ".jsonl") == 0 && parse_base(name.substr(0, 20), base)) files[base].first = true; else if (name.size() == 36 && name.compare(20, 16, ".checkpoint.json") == 0 && parse_base(name.substr(0, 20), base)) files[base].second = true; @@ -274,14 +275,21 @@ void State::recover() { } std::vector retained; bool removed = false; + std::uint64_t lowest = cursor_.segment; + for (const auto& [base, kinds] : files) + if (kinds.first) lowest = std::min(lowest, base); for (const auto& [base, kinds] : files) { if (base > cursor_.segment) { + // Only the successor an interrupted seal creates: at the cursor's index, and still empty. + if (base != cursor_.message_index || (kinds.first && std::filesystem::file_size(segment_path(base)) != 0)) + throw Error(21, "a journal segment after the cursor's open segment holds data the cursor does not cover"); std::filesystem::remove(segment_path(base)); std::filesystem::remove(checkpoint_path(base)); log("warn", "uncommitted_rotation_removed", Json::object({{"segment", number(base)}})); removed = true; } else if (!kinds.first) { - if (base == cursor_.segment) throw Error(21, "the open journal segment is missing"); + // An interrupted expiry leaves the oldest checkpoint without its segment; anything else is a hole. + if (base >= lowest) throw Error(21, "the retained journal has a hole: a checkpoint without its segment"); std::filesystem::remove(checkpoint_path(base)); removed = true; } else if (!kinds.second) throw Error(21, "a journal segment has no checkpoint"); @@ -301,7 +309,7 @@ void State::recover() { const auto base = retained[index]; if (index && !same_state(reconstructed, read_checkpoint(base))) throw Error(21, "durable journal segment boundary does not match its checkpoint"); - segments_.push_back({base, reconstructed.log_bytes, reconstructed.seq, reconstructed.cut}); + segments_.push_back({base, reconstructed.log_bytes, reconstructed.seq, reconstructed.cut, reconstructed.last_tick_ts}); const auto path = segment_path(base); const auto size = std::filesystem::file_size(path); const bool open = index + 1 == retained.size(); @@ -317,14 +325,16 @@ void State::recover() { ++reconstructed.message_index; remember(event); }); - if (open && ::truncate(path.c_str(), static_cast(limit)) != 0) - throw Error(22, "cannot roll back an uncommitted journal tail"); } if (!same_state(reconstructed, cursor_)) throw Error(21, "durable journal prefix does not match its cursor"); + // Only a verified journal is changed: the open segment's uncommitted tail goes last. + if (::truncate(segment_path(cursor_.segment).c_str(), static_cast(cursor_.log_bytes - segments_.back().bytes)) != 0) + throw Error(22, "cannot roll back an uncommitted journal tail"); config_.start = cursor_.start; durable_ = cursor_; } catch (const Error&) { throw; } catch (const Stopped&) { throw; } + catch (const std::filesystem::filesystem_error&) { throw Error(22, "journal I/O failed during recovery"); } catch (const std::exception&) { throw Error(21, "invalid durable cursor or journal"); } } @@ -385,8 +395,14 @@ std::vector State::flush() { durable_ = cursor_; staged_bytes_.clear(); auto lines = std::exchange(staged_, {}); - if (durable_.log_bytes - segments_.back().bytes >= config_.segment_bytes) rotate(); - retain(); + // The lines are committed; a failed seal or expiry leaves files only resume may reconcile. + try { + if (durable_.log_bytes - segments_.back().bytes >= config_.segment_bytes) rotate(); + retain(); + } catch (...) { + unusable_ = true; + throw; + } return lines; } @@ -407,7 +423,7 @@ void State::rotate() { crash_point("rotate-cursor"); durable_.segment = cursor_.segment = base; journal_.reset(next.release()); - segments_.push_back({base, durable_.log_bytes, durable_.seq, durable_.cut}); + segments_.push_back({base, durable_.log_bytes, durable_.seq, durable_.cut, durable_.last_tick_ts}); log("info", "journal_segment_sealed", Json::object({{"next_segment", number(base)}, {"segments", number(segments_.size())}})); } @@ -419,14 +435,17 @@ void State::retain() { const auto oldest = segments_[0]; const auto& next = segments_[1]; if (next.cut >= protected_start) break; + // The segment holding the newest print stays: a reconnect re-reads the recent prints from the journal. + if (tick_mode(config_.mode) && next.seq == durable_.seq && oldest.seq < durable_.seq) break; const bool bytes = durable_.log_bytes - oldest.bytes > config_.replay_bytes; const bool age = config_.replay_age_ms && durable_.cut - next.cut > config_.replay_age_ms; if (!bytes && !age) break; if (::unlink(segment_path(oldest.base).c_str()) != 0) throw Error(22, "cannot expire a journal segment"); + segments_.erase(segments_.begin()); + sync_directory(journal_dir()); crash_point("retain-segment"); if (::unlink(checkpoint_path(oldest.base).c_str()) != 0 && errno != ENOENT) throw Error(22, "cannot expire a journal checkpoint"); sync_directory(journal_dir()); - segments_.erase(segments_.begin()); log("info", "journal_segment_expired", Json::object({{"segment", number(oldest.base)}, {"first_retained", number(segments_.front().base)}, {"reason", Json::string(bytes ? "bytes" : "age")}})); } @@ -443,9 +462,12 @@ std::size_t State::segment_for(std::uint64_t index) const { return found == segments_.begin() ? 0 : static_cast(found - segments_.begin() - 1); } +// Ticks of the first retained segment's open minute can sit in the expired segment before it: that minute +// is expired only when its checkpoint shows a tick of it. bool State::expired_minute(std::int64_t minute) const { - const auto cut = segments_.front().cut; - return first_retained() && (minute < cut || (tick_mode(config_.mode) && minute == cut)); + const auto& first = segments_.front(); + return first_retained() && + (minute < first.cut || (tick_mode(config_.mode) && minute == first.cut && first.last_tick_ts >= first.cut)); } std::uint64_t State::index_for_minute(std::int64_t minute) const { @@ -456,50 +478,65 @@ std::uint64_t State::index_for_minute(std::int64_t minute) const { } void State::visit(std::uint64_t from, const std::function& visitor) const { + scan(from, [&](const std::string& line) { visitor(line); return true; }); +} + +void State::scan(std::uint64_t from, const std::function& visitor) const { if (from > cursor_.message_index) throw Error(22, "replay cursor is unavailable"); if (from < first_retained()) throw Error(22, "replay cursor has expired from the retained journal"); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(config_.max_replay_seconds); + struct Enough {}; std::uint64_t index = durable_.message_index; - for (auto position = segment_for(from); position < segments_.size() && from < durable_.message_index; ++position) { - const auto& segment = segments_[position]; - const bool last = position + 1 == segments_.size(); - const auto end_bytes = last ? durable_.log_bytes : segments_[position + 1].bytes; - const auto end_index = last ? durable_.message_index : segments_[position + 1].base; - index = segment.base; - read_lines(segment_path(segment.base), end_bytes - segment.bytes, deadline, [&](const std::string& line) { - if (index >= from) visitor(line); + try { + for (auto position = segment_for(from); position < segments_.size() && from < durable_.message_index; ++position) { + const auto& segment = segments_[position]; + const bool last = position + 1 == segments_.size(); + const auto end_bytes = last ? durable_.log_bytes : segments_[position + 1].bytes; + const auto end_index = last ? durable_.message_index : segments_[position + 1].base; + index = segment.base; + read_lines(segment_path(segment.base), end_bytes - segment.bytes, deadline, [&](const std::string& line) { + if (index >= from && !visitor(line)) throw Enough{}; + ++index; + }); + if (index != end_index) throw Error(21, "verified message partition changed"); + } + index = durable_.message_index; + for (const auto& staged : staged_) { + if (index >= from && !visitor(staged)) return; ++index; - }); - if (index != end_index) throw Error(21, "verified message partition changed"); - } - index = durable_.message_index; - for (const auto& staged : staged_) { - if (index >= from) visitor(staged); - ++index; - } + } + } catch (const Enough&) {} } -std::optional State::trade(std::uint64_t id) const { +std::optional State::recent_trade(std::uint64_t id) const { for (const auto& trade : recent_trades_) if (trade.id == id) return trade; + return std::nullopt; +} +std::optional State::trade(std::uint64_t id) const { + if (const auto recent = recent_trade(id)) return recent; std::size_t chosen = 0; for (std::size_t index = 0; index < segments_.size(); ++index) if (segments_[index].seq < id) chosen = index; std::optional found; - visit(segments_[chosen].base, [&](const std::string& line) { - if (found) return; + scan(segments_[chosen].base, [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() == "tick" && event.at("seq").integer() == id) found = normalized_trade(event); + return !found; }); return found; } +std::optional State::proof(std::int64_t ts) const { + for (const auto& bar : cursor_.proofs) if (bar.ts == ts) return bar; + return std::nullopt; +} std::optional State::bar(std::int64_t ts) const { for (const auto& bar : recent_bars_) if (bar.ts == ts) return bar; - for (const auto& bar : cursor_.proofs) if (bar.ts == ts) return bar; + if (const auto proven = proof(ts)) return proven; std::optional found; - visit(index_for_minute(ts), [&](const std::string& line) { - if (found) return; + scan(index_for_minute(ts), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() == "bar" && event.at("bar").at("ts_open").integer() == ts) found = normalized_bar(event); + return !found; }); return found; } diff --git a/native/feed/src/state.hpp b/native/feed/src/state.hpp index 096227d..65fc385 100644 --- a/native/feed/src/state.hpp +++ b/native/feed/src/state.hpp @@ -33,7 +33,7 @@ struct Cursor { // cut and emitted sequence. struct Segment { std::uint64_t base = 0, bytes = 0, seq = 0; - std::int64_t cut = 0; + std::int64_t cut = 0, last_tick_ts = -1; }; std::string segment_name(std::uint64_t base); @@ -65,6 +65,8 @@ class State { void rotate(); void retain(); std::size_t segment_for(std::uint64_t index) const; + // visit() that stops as soon as `visitor` returns false. + void scan(std::uint64_t from, const std::function& visitor) const; public: explicit State(const Config& config); const Cursor& cursor() const { return cursor_; } @@ -84,7 +86,9 @@ class State { std::vector flush(); void visit(std::uint64_t from, const std::function& visitor) const; std::optional trade(std::uint64_t id) const; + std::optional recent_trade(std::uint64_t id) const; std::optional bar(std::int64_t ts) const; + std::optional proof(std::int64_t ts) const; const std::deque& recent_trades() const { return recent_trades_; } const std::deque& recent_bars() const { return recent_bars_; } Json status() const; diff --git a/native/feed/src/transport.cpp b/native/feed/src/transport.cpp index 324c056..a0fd3c3 100644 --- a/native/feed/src/transport.cpp +++ b/native/feed/src/transport.cpp @@ -237,6 +237,9 @@ std::string HttpClient::body(const std::string& path, unsigned int weight, std:: } Progress state; auto handle = handle_for(config_.rest_url + path, false, config_.allow_insecure, &state); + // The venue counts a request in the window it was sent in, not the one its response arrives in. + const auto sent_window = static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()) / 60000; Response response; response.limit = limit; option(handle.get(), CURLOPT_WRITEFUNCTION, &body_callback); @@ -276,8 +279,7 @@ std::string HttpClient::body(const std::string& path, unsigned int weight, std:: if (status != 200) throw Error(23, "public market-data request rejected (HTTP " + std::to_string(status) + ")"); if (response.used_weight) { used_weight_ = response.used_weight; - used_window_ = static_cast(std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count()) / 60000; + used_window_ = sent_window; log("info", "rest_weight", Json::object({{"used_weight_1m", Json::number(std::to_string(response.used_weight))}})); } return std::move(response.body); @@ -311,6 +313,7 @@ bool WebSocketPump::push(SourceMessage message) { return true; } void WebSocketPump::wait_for_room(std::size_t bytes) { + if (bytes > config_.max_queue_bytes) throw Error(22, "--max-queue-bytes is too small for one source message"); for (;;) { { std::lock_guard guard(mutex_); @@ -374,7 +377,8 @@ void WebSocketPump::run() { // The marker precedes every message of this connection: the session verifies its REST // overlap while the subscribed data is buffered behind it. wait_for_room(32); - if (stopped() || !push({true, {}})) return; + if (stopped()) return; + if (!push({true, {}})) throw Error(22, "WebSocket queue cannot take the connection marker"); bool subscribed = true; for (const auto& request : connection_.subscribe) subscribed = subscribed && send_text(handle.get(), socket, request); log(subscribed ? "info" : "warn", subscribed ? "websocket_connected" : "websocket_subscribe_failed"); diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py index b192000..b58aa6e 100644 --- a/native/feed/tests/mock_venue.py +++ b/native/feed/tests/mock_venue.py @@ -275,6 +275,7 @@ def __init__(self, scripts, venue="binance"): self.history_delay = 0.0 self.probes = [] self.stale_subscriptions = False + self.probe_reply = "full" self.refuse_once = None self.usdm_unknown_symbol = False self.connections = 0 @@ -661,7 +662,12 @@ def read(): streams = ( [] if self.server.stale_subscriptions else query["streams"][0].split("/") ) - send({"result": streams, "id": request["id"]}) + if self.server.probe_reply == "error": + send({"code": 2, "msg": "synthetic refusal", "id": request["id"]}) + elif self.server.probe_reply == "partial": + send({"result": streams[-1:], "id": request["id"]}) + else: + send({"result": streams, "id": request["id"]}) closed.set() reader = threading.Thread(target=read, daemon=True) @@ -1451,6 +1457,12 @@ def test_429_retry_after_preserves_prefix(self): self.assertIn("rest_rate_limited", result.stderr) def test_replay_and_queue_budgets_stop_22(self): + # A queue that cannot hold even a connection marker stops at once, never waits forever. + server = self.server([[candle(120000)]]) + output, result = self.run_feed(server, extra=["--max-queue-bytes", "16"], expected=22) + self.assertEqual(output, []) + self.assertIn("too small", result.stderr) + self.reset_state() server = self.server([[trade(100)]]) output, _ = self.run_feed(server, "ticks", ["--max-queue-bytes", "64"], expected=22) self.assertEqual(output, []) @@ -1558,6 +1570,22 @@ def test_quiet_binance_stream_is_probed_not_reconnected(self): self.assertEqual([event["bar"]["ts_open"] for event in output], [120000, 180000]) self.assertGreaterEqual(server.connections, 2) self.assertIn("source_subscription_lost", result.stderr) + # An error reply, or a list missing one of the connection's streams, also reconnects + # (never 23). + for reply, mode, script in ( + ("error", "bars", [[candle(120000), ("pause", 2.0)], [candle(180000)]]), + ("partial", "ticks", [[trade(100), ("pause", 2.0)], [trade(101), trade(102)]]), + ): + with self.subTest(reply=reply): + self.reset_state() + server = self.server(script) + server.probe_reply = reply + output, result = self.run_feed( + server, mode, ["--max-messages", "2", "--silence-seconds", "1"] + ) + self.assertEqual(len(output), 2) + self.assertGreaterEqual(server.connections, 2) + self.assertIn("source_subscription_lost", result.stderr) def test_okx_swap_bars_confirm_flag_and_contract_units(self): server = self.server([[okx_candle(120000, "0"), okx_candle(180000)]], "okx") @@ -2526,6 +2554,10 @@ def client(seed): else: outcomes.append("read") stream.close() + except TimeoutError: + # A silent server (a catch-up client stuck on an expired segment) is a failure. + failures.append(("timeout", start)) + return except (OSError, EOFError) as failure: outcomes.append(f"transport-{type(failure).__name__}") diff --git a/native/feed/tests/unit.cpp b/native/feed/tests/unit.cpp index cb1c26c..ce70fa4 100644 --- a/native/feed/tests/unit.cpp +++ b/native/feed/tests/unit.cpp @@ -59,7 +59,7 @@ struct SyntheticVenue final : Venue { {180000, {{180000, "12.00000000", "12.00000000", "11.00000000", "11.00000000", "0.50000000"}, 103, 104, 2, true}}, {240000, {{240000, "12.00000000", "12.00000000", "12.00000000", "12.00000000", "0.10000000"}, 105, 105, 1, true}} }; - Connection connection() const override { return {"/stream?streams=test@trade/test@kline_1m", {}, {}, &binance_frame, {}}; } + Connection connection() const override { return {"/stream?streams=test@trade/test@kline_1m", {}, {}, [](const std::string& message) { return binance_frame(message); }, {}}; } std::vector decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } std::string kline_source() const override { return "/synthetic"; } std::size_t pages = 0; @@ -115,7 +115,7 @@ struct FenceVenue final : Venue { std::size_t pages = 0, candle_requests = 0; // REST lag: the next `empty_reads` history reads return nothing, the next `short_reads` drop their last row. std::size_t empty_reads = 0, short_reads = 0; - Connection connection() const override { return {"/synthetic", {}, {}, &binance_frame, {}}; } + Connection connection() const override { return {"/synthetic", {}, {}, [](const std::string& message) { return binance_frame(message); }, {}}; } std::vector decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } std::vector history(std::uint64_t from, std::size_t limit) override { ++pages; @@ -399,30 +399,42 @@ int main() { auto options = config(temporary, "bars"); options.segment_bytes = 4096; options.replay_bytes = 8192; - std::uint64_t open = 0, first = 0, expired = 0; + std::uint64_t open = 0, first = 0, expired = 0, next = 0; std::string prefix; { State state(options); commit_bars(state, 120000, 300); open = state.durable().segment; first = state.first_retained(); + next = state.durable().message_index; prefix = state.durable().prefix_hash; } + // An interrupted seal: the successor's checkpoint and its empty segment at the cursor's next index. expired = first - 1; std::filesystem::copy_file(checkpoint_file(temporary, first), checkpoint_file(temporary, expired)); - std::filesystem::copy_file(checkpoint_file(temporary, open), checkpoint_file(temporary, open + 1000)); - { std::ofstream residue(segment_file(temporary, open + 1000)); } - std::filesystem::copy_file(checkpoint_file(temporary, open), checkpoint_file(temporary, open + 2000)); + std::filesystem::copy_file(checkpoint_file(temporary, open), checkpoint_file(temporary, next)); + { std::ofstream residue(segment_file(temporary, next)); } options.resume = true; options.start = -1; options.output_from = first; { State resumed(options); assert(resumed.durable().prefix_hash == prefix && resumed.durable().segment == open && resumed.first_retained() == first); - assert(!std::filesystem::exists(checkpoint_file(temporary, expired)) && !std::filesystem::exists(segment_file(temporary, open + 1000))); - assert(!std::filesystem::exists(checkpoint_file(temporary, open + 1000)) && !std::filesystem::exists(checkpoint_file(temporary, open + 2000))); - commit_bars(resumed, 120000 + 300 * 60000, 2); + assert(!std::filesystem::exists(checkpoint_file(temporary, expired)) && !std::filesystem::exists(segment_file(temporary, next))); + assert(!std::filesystem::exists(checkpoint_file(temporary, next))); } + // Anything else is not a crash leftover: a segment after the cursor that is not the empty successor, + // or a checkpoint without its segment inside the retained range, stops 21 and is left in place. + std::filesystem::copy_file(checkpoint_file(temporary, open), checkpoint_file(temporary, next + 1000)); + { std::ofstream stray(segment_file(temporary, next + 1000)); stray << "{}\n"; } + expect(21, [&] { State stray(options); }); + assert(std::filesystem::exists(segment_file(temporary, next + 1000))); + std::filesystem::remove(segment_file(temporary, next + 1000)); + std::filesystem::remove(checkpoint_file(temporary, next + 1000)); + std::filesystem::copy_file(checkpoint_file(temporary, open), checkpoint_file(temporary, first + 1)); + expect(21, [&] { State hole(options); }); + std::filesystem::remove(checkpoint_file(temporary, first + 1)); + { State resumed(options); commit_bars(resumed, 120000 + 300 * 60000, 2); } const auto saved = checkpoint_file(temporary, open) + ".saved"; std::filesystem::rename(checkpoint_file(temporary, open), saved); expect(21, [&] { State missing(options); }); @@ -438,6 +450,35 @@ int main() { expect(21, [&] { State changed(options); }); passed("journal_rotation_and_expiry_crash_residue_recovers_or_stops_21"); } + { + // A quiet fence-venue tick stream: time events keep closing minutes after the last print, so age and + // byte retention would expire every segment holding a print; the newest print's segment stays, and a + // resume still has the recent prints its reconnect overlap re-reads. + Temporary temporary; + auto options = config(temporary, "ticks"); + options.segment_bytes = 4096; + options.replay_bytes = 8192; + options.replay_age_ms = 5 * 60000; + { + State state(options); + state.anchor({99, 119999, "10.10000000", "0.10000000"}); + state.stage(Trade{100, 120001, "10.10000000", "0.10000000"}.wire()); + state.flush(); + for (std::int64_t minute = 1; minute <= 400; ++minute) { + const Bar proof{120000 + (minute - 1) * 60000, "10.10000000", "10.10000000", "10.10000000", "10.10000000", + minute == 1 ? "0.10000000" : "0"}; + state.stage("{\"type\":\"time\",\"ts\":" + std::to_string(120000 + minute * 60000) + "}", proof); + if (minute % 5 == 0) state.flush(); + } + state.flush(); + assert(state.segments().size() > 2 && state.first_retained() == 0 && state.recent_trades().size() == 1); + } + options.resume = true; + options.start = -1; + State resumed(options); + assert(resumed.recent_trades().size() == 1 && resumed.recent_trades().front().id == 100 && resumed.durable().seq == 100); + passed("retention_keeps_the_segment_of_the_newest_print"); + } { Temporary temporary; State state(config(temporary, "ticks")); @@ -1032,7 +1073,7 @@ int main() { std::filesystem::remove(segment_file(temporary, 0)); JournalReader expired(temporary.path + "/journal"); assert(!expired.seek(3) && expired.seek(state.segments()[1].base)); - expect(22, [&] { snapshot_prefix(temporary.path + "/journal", 3, 4096); }); // index 0 expired above + expect(20, [&] { snapshot_prefix(temporary.path + "/journal", 3, 4096); }); // index 0 expired above Temporary small; auto bounded = config(small, "bars"); State fresh(bounded); From 74efac9c7b368ddbb8702e7389188668952c13b1 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 11:24:47 +0800 Subject: [PATCH 09/15] Keep serve workers for status and connects; close on client frames CivetWeb runs one worker per connection and every streaming client keeps its worker, so incomplete requests could take the few spare workers and block status and new runner connects for 30 s. Run max-clients + 16 workers and drop a request that is not complete within 10 s, which also bounds a blocked write to a dead client. A client never sends messages: any frame it sends after serve has closed it now ends the connection instead of being read on. The snapshot fails at once when a segment expires mid-read instead of polling, and --listen accepts the IPv6 loopback ([::1]:PORT; CivetWeb built with USE_IPV6). A client cut off for overflowing its queue is logged. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/CMakeLists.txt | 2 +- native/feed/src/serve.cpp | 39 ++++++++++------ native/feed/tests/mock_venue.py | 80 ++++++++++++++++++++++++++++++--- native/feed/tests/unit.cpp | 7 ++- 4 files changed, 105 insertions(+), 23 deletions(-) diff --git a/native/feed/CMakeLists.txt b/native/feed/CMakeLists.txt index 3f7629e..5587311 100644 --- a/native/feed/CMakeLists.txt +++ b/native/feed/CMakeLists.txt @@ -70,7 +70,7 @@ if(PINEFORGE_FEED_SERVE) add_library(pineforge_feed_civetweb STATIC "${civetweb_SOURCE_DIR}/src/civetweb.c") target_include_directories(pineforge_feed_civetweb SYSTEM PUBLIC "${civetweb_SOURCE_DIR}/include") target_compile_definitions(pineforge_feed_civetweb PRIVATE NO_SSL NO_CGI NO_FILES NO_CACHING STOP_FLAG_NEEDS_LOCK - PUBLIC USE_WEBSOCKET) + USE_IPV6 PUBLIC USE_WEBSOCKET) set_target_properties(pineforge_feed_civetweb PROPERTIES C_STANDARD 99) target_compile_options(pineforge_feed_civetweb PRIVATE -w) target_link_libraries(pineforge_feed_civetweb PUBLIC Threads::Threads ${CMAKE_DL_LIBS}) diff --git a/native/feed/src/serve.cpp b/native/feed/src/serve.cpp index 2bff8db..ec1ed7e 100644 --- a/native/feed/src/serve.cpp +++ b/native/feed/src/serve.cpp @@ -57,6 +57,7 @@ struct Hub { // A client that cannot keep up is cut off; it never slows the producer or another client. if (shared->size() > config.client_queue_bytes - std::min(client->queue_bytes, config.client_queue_bytes)) { client->slow = true; + log("warn", "client_slow", Json::object({{"next", number(client->next)}, {"queued_bytes", number(client->queue_bytes)}})); continue; } client->queue_bytes += shared->size(); @@ -328,7 +329,9 @@ void on_ready(mg_connection* connection, void* data) { try { send_close(connection, 1011, "internal error"); } catch (...) {} } } -int on_data(mg_connection*, int, char*, std::size_t, void*) { return 1; } +// The protocol has no client messages: any data frame (CivetWeb reads frames only after the ready handler has +// returned, i.e. after serve's close) ends the connection instead of being read on. +int on_data(mg_connection*, int, char*, std::size_t, void*) { return 0; } void on_close(const mg_connection*, void*) {} // Every connection end, handshake failures included, releases its client. @@ -356,9 +359,12 @@ class Server { public: Server(Hub& hub, const std::string& address) { mg_init_library(0); - const auto threads = std::to_string(hub.config.max_clients + 4); + // CivetWeb is thread-per-connection and every streaming client holds its worker. 16 spare workers serve + // status, snapshots and new upgrades; a request that is not complete within 10 s (a half-open + // connection) is dropped, which also bounds a blocked write to a dead client. + const auto threads = std::to_string(hub.config.max_clients + 16); const char* options[] = {"listening_ports", address.c_str(), "num_threads", threads.c_str(), - "request_timeout_ms", "30000", "websocket_timeout_ms", "10000", + "request_timeout_ms", "10000", "websocket_timeout_ms", "10000", "enable_websocket_ping_pong", "yes", "enable_keep_alive", "no", nullptr}; mg_callbacks callbacks{}; callbacks.log_message = &on_log; @@ -452,14 +458,10 @@ std::optional snapshot_prefix(const std::string& journal, std::uint JournalReader reader(journal); if (!reader.seek(0)) throw Error(20, "the snapshot prefix has expired"); std::string body; - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); while (reader.index() < published) { + // Every published line is on disk before it is published: a missing one means its segment expired. auto line = reader.next(); - if (!line) { - if (std::chrono::steady_clock::now() >= deadline) throw Error(20, "a snapshot segment expired while it was read"); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - continue; - } + if (!line) throw Error(20, "a snapshot segment expired while it was read"); if (line->size() + 1 > limit - body.size()) return std::nullopt; body += *line; body += '\n'; @@ -469,17 +471,26 @@ std::optional snapshot_prefix(const std::string& journal, std::uint std::string listen_address(const std::string& listen, bool allow_remote) { const auto colon = listen.rfind(':'); - if (colon == std::string::npos || colon + 1 == listen.size()) throw Error(23, "--listen takes HOST:PORT"); + if (colon == std::string::npos || colon + 1 == listen.size()) throw Error(23, "--listen takes HOST:PORT or [IPV6]:PORT"); auto host = listen.substr(0, colon); const auto port = listen.substr(colon + 1); unsigned int value = 0; if (port.find_first_not_of("0123456789") != std::string::npos || std::from_chars(port.data(), port.data() + port.size(), value).ec != std::errc{} || value > 65535) throw Error(23, "--listen port must be 0..65535"); - if (host == "localhost") host = "127.0.0.1"; - in_addr parsed{}; - if (::inet_pton(AF_INET, host.c_str(), &parsed) != 1) throw Error(23, "--listen host must be an IPv4 address or localhost"); - const bool loopback = (ntohl(parsed.s_addr) >> 24) == 127; + bool loopback = false; + if (host.size() > 2 && host.front() == '[' && host.back() == ']') { + in6_addr parsed{}; + if (::inet_pton(AF_INET6, host.substr(1, host.size() - 2).c_str(), &parsed) != 1) + throw Error(23, "--listen host must be an IPv4 address, [IPV6] or localhost"); + loopback = IN6_IS_ADDR_LOOPBACK(&parsed); + } else { + if (host == "localhost") host = "127.0.0.1"; + in_addr parsed{}; + if (::inet_pton(AF_INET, host.c_str(), &parsed) != 1) + throw Error(23, "--listen host must be an IPv4 address, [IPV6] or localhost"); + loopback = (ntohl(parsed.s_addr) >> 24) == 127; + } if (!loopback && !allow_remote) throw Error(23, "--listen on a non-loopback address needs --allow-remote-listen; serve has no TLS or authentication"); return host + ":" + std::to_string(value); diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py index b58aa6e..f78f36d 100644 --- a/native/feed/tests/mock_venue.py +++ b/native/feed/tests/mock_venue.py @@ -2300,8 +2300,10 @@ def test_coinbase_is_deferred(self): self.assertIn("Coinbase is deferred", result.stderr) self.assertFalse((self.directory / "state").exists()) - def serve(self, server, mode="bars", extra=(), resume=False, environment=None): - command = self.command(server, mode, [*extra, "--listen", "127.0.0.1:0"], resume) + def serve( + self, server, mode="bars", extra=(), resume=False, environment=None, listen="127.0.0.1:0" + ): + command = self.command(server, mode, [*extra, "--listen", listen], resume) command[1] = "serve" process = subprocess.Popen( command, @@ -2458,7 +2460,7 @@ def test_serve_slow_client_is_closed_with_a_reason_and_never_blocks_others(self) server.scripts = [ [bar_message(120000, server.bars), ("pause", 2.0), bar_message(last, server.bars)] ] - process, port, _ = self.serve(server, extra=["--client-queue-bytes", "1048576"]) + process, port, records = self.serve(server, extra=["--client-queue-bytes", "1048576"]) status = self.wait_status(port, lambda status: status["retained"]["next_index"] >= 1) path = f"/v1/stream?epoch={status['epoch']}&from=0" stuck = StreamClient(port, path, receive_buffer=4096, timeout=60) @@ -2472,10 +2474,11 @@ def consume(): reader = threading.Thread(target=consume, daemon=True) reader.start() - self.wait_status(port, lambda status: status["retained"]["next_index"] == count, timeout=60) - reader.join(timeout=60) - self.assertEqual(len(received), count) - self.assertEqual(received[-1], bar_line(last, server.bars)) + # The stuck client reads again once it is cut off, well inside the 10 s write bound. + deadline = time.monotonic() + 60 + while not any(record["event"] == "client_slow" for record in records): + self.assertLess(time.monotonic(), deadline, "the stuck client was never cut off") + time.sleep(0.01) lines, closed = 1, None while closed is None: kind, *rest = stuck.message() @@ -2487,6 +2490,16 @@ def consume(): self.assertEqual(closed[0], 1008) self.assertIn("slow consumer", closed[1]) self.assertLess(lines, count) + # After serve's close, a client frame ends the connection instead of being read on. + stuck.socket.settimeout(5) + stuck.send(1, b"after close") + with self.assertRaises((EOFError, ConnectionResetError)): + while True: + stuck.message() + stuck.close() + reader.join(timeout=60) + self.assertEqual(len(received), count) + self.assertEqual(received[-1], bar_line(last, server.bars)) self.assertIsNone(process.poll()) self.stop(process) @@ -2577,6 +2590,59 @@ def client(seed): ) self.stop(process) + def test_serve_half_open_requests_never_starve_status_or_a_runner(self): + # Eight connections that never finish their request on a listener for four clients: + # status and a runner's upgrade still get a worker at once, and each incomplete request + # is dropped within the 10 s request timeout. + server = self.server([[candle(240000)]]) + process, port, _ = self.serve(server, extra=["--max-clients", "4"]) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] == 3) + stalled = [] + for _ in range(8): + connection = socket.create_connection(("127.0.0.1", port), timeout=20) + self.addCleanup(connection.close) + connection.sendall(b"GET /v1/status HTTP/1.1\r\nHost: x\r\nX-Slow: ") + stalled.append(connection) + time.sleep(0.5) + started = time.monotonic() + self.assertEqual(self.status(port)["epoch"], status["epoch"]) + runner = StreamClient(port, f"/v1/stream?epoch={status['epoch']}&from=0", timeout=5) + self.assertEqual(runner.status, 101) + self.assertEqual(len(runner.texts(3)), 3) + self.assertLess(time.monotonic() - started, 5) + runner.close() + for connection in stalled: + connection.settimeout(max(started + 15 - time.monotonic(), 0.1)) + with contextlib.suppress(ConnectionResetError): + while connection.recv(4096): + pass + self.assertLess(time.monotonic() - started, 15) + self.assertIsNone(process.poll()) + self.stop(process) + + def test_serve_accepts_the_ipv6_loopback(self): + try: + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as probe: + probe.bind(("::1", 0)) + except OSError: + self.skipTest("this host has no IPv6 loopback") + server = self.server([[candle(240000)]]) + process, port, _ = self.serve(server, listen="[::1]:0") + deadline = time.monotonic() + 20 + while True: + with urllib.request.urlopen(f"http://[::1]:{port}/v1/status", timeout=10) as response: + body = json.loads(response.read()) + if body["retained"]["next_index"] == 3: + break + self.assertLess(time.monotonic(), deadline) + time.sleep(0.05) + self.stop(process) + command = self.command(server, extra=["--listen", "[::]:0"]) + command[1] = "serve" + result = subprocess.run(command, capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 23) + self.assertIn("--allow-remote-listen", result.stderr) + def test_serve_listens_on_loopback_unless_remote_is_explicit(self): server = self.server([[]]) command = self.command(server, extra=["--listen", "0.0.0.0:0"]) diff --git a/native/feed/tests/unit.cpp b/native/feed/tests/unit.cpp index ce70fa4..6a5226b 100644 --- a/native/feed/tests/unit.cpp +++ b/native/feed/tests/unit.cpp @@ -1084,7 +1084,12 @@ int main() { assert(listen_address("localhost:0", false) == "127.0.0.1:0" && listen_address("127.0.0.2:8787", false) == "127.0.0.2:8787"); expect(23, [] { listen_address("0.0.0.0:8787", false); }); assert(listen_address("0.0.0.0:8787", true) == "0.0.0.0:8787"); - for (const auto* invalid : {"8787", "127.0.0.1:", "127.0.0.1:70000", "[::1]:8787", "example.com:80"}) + // The IPv6 loopback is loopback; any other IPv6 address needs --allow-remote-listen. + assert(listen_address("[::1]:8787", false) == "[::1]:8787"); + expect(23, [] { listen_address("[::]:8787", false); }); + assert(listen_address("[::]:8787", true) == "[::]:8787"); + for (const auto* invalid : {"8787", "127.0.0.1:", "127.0.0.1:70000", "[::1", "[]:80", "[127.0.0.1]:80", "::1:8787", + "example.com:80"}) expect(23, [&] { listen_address(invalid, true); }); passed("serve_journal_reader_follows_segments_and_listen_is_loopback_by_default"); } From cf48ef55f9824d0422e659b37ea5905a11cc870f Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 11:24:47 +0800 Subject: [PATCH 10/15] Carry the warmup close into a quiet first export minute The runner's carry-forward bar after warmup repeats the last warmup close, not the last print before the start. export now takes --warmup (the runner's warmup CSV, ending at the minute before --start) and a quiet first minute carries its close; without it a quiet first minute stops 20 instead of guessing. The manifest records warmup_close. An archive window whose predecessor or fence lies in a neighbouring daily file stops 20 and names that file. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/src/export.cpp | 76 +++++++++++++++++++++++++++---- native/feed/src/export.hpp | 14 ++++-- native/feed/src/main.cpp | 11 +++-- native/feed/tests/export_unit.cpp | 22 +++++++-- native/feed/tests/mock_export.py | 72 +++++++++++++++++++++++++---- 5 files changed, 167 insertions(+), 28 deletions(-) diff --git a/native/feed/src/export.cpp b/native/feed/src/export.cpp index 5f719fa..8240990 100644 --- a/native/feed/src/export.cpp +++ b/native/feed/src/export.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -19,7 +20,7 @@ constexpr std::size_t csv_limit = 64 * 1024 * 1024; constexpr std::string_view archive_header = "agg_trade_id,price,quantity,first_trade_id,last_trade_id,transact_time,is_buyer_maker"; constexpr const char* bar_rule = "a print belongs to minute floor(ts/60000)*60000; open=first price, high=max, low=min, close=last, " "volume=exact sum of quantities; a minute without a print has open=high=low=close=the previous close (the " - "predecessor's price before the first print) and volume 0"; + "runner's last warmup close before the first print) and volume 0"; Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } Json signed_number(std::int64_t value) { return Json::number(std::to_string(value)); } std::string base_name(const std::string& path) { return std::filesystem::path(path).filename().string(); } @@ -224,6 +225,11 @@ ChainBars::ChainBars(std::int64_t start, std::int64_t end) : start_(start), end_ throw Error(23, "export requires a nonempty minute-aligned exclusive range"); if ((end - start) / 60000 > 100000) throw Error(22, "export range exceeds 100000 minutes"); } +void ChainBars::seed(const std::string& close) { + if (predecessor_) throw Error(23, "the warmup close seeds the chain before its predecessor"); + (void)Decimal(close); + seed_ = close; +} void ChainBars::anchor(const Trade& predecessor) { predecessor.validate(); if (predecessor_) throw Error(23, "the print chain is already anchored"); @@ -231,7 +237,7 @@ void ChainBars::anchor(const Trade& predecessor) { predecessor_ = predecessor; last_id_ = predecessor.id; last_ts_ = predecessor.ts; - close_ = predecessor.price; + close_ = seed_; } bool ChainBars::add(const Trade& trade) { if (!predecessor_ || fence_) throw Error(23, "a print chain takes its predecessor first and nothing after its fence"); @@ -276,6 +282,9 @@ void ChainBars::flush() { next_ = minute_ + 60000; } void ChainBars::carry(std::int64_t until) { + if (next_ < until && close_.empty()) + throw Error(20, "the window's first minute has no print: the runner carries its last warmup close there; " + "pass that warmup CSV with --warmup"); for (; next_ < until; next_ += 60000, ++quiet_) row(next_, close_, close_, close_, close_, "0"); } void ChainBars::row(std::int64_t minute, const std::string& open, const std::string& high, const std::string& low, @@ -456,9 +465,50 @@ void verify_checksum(const std::string& checksum, const std::string& archive, co if (hash != digest) throw Error(21, "the archive's SHA-256 does not match its checksum file"); } -ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end) { +namespace { +// The venue's daily file that holds `ts`, named like the given one (-aggTrades-YYYY-MM-DD.zip). +std::string daily_name(const std::string& archive, std::int64_t ts) { + const auto seconds = static_cast(ts / 1000); + std::tm day{}; + char date[16]; + if (!::gmtime_r(&seconds, &day) || !std::strftime(date, sizeof(date), "%Y-%m-%d", &day)) return "the neighbouring daily archive"; + const auto name = base_name(archive); + const std::string marker = "-aggTrades-"; + const auto at = name.find(marker); + if (at == std::string::npos || name.size() < at + marker.size() + 10) return std::string("the daily archive of ") + date; + return name.substr(0, at + marker.size()) + date + name.substr(at + marker.size() + 10); +} +} + +std::string warmup_close(const std::string& warmup, std::int64_t start) { + std::ifstream file(warmup, std::ios::binary); + if (!file) throw Error(23, "cannot open the warmup CSV " + warmup); + std::string line, last; + if (!std::getline(file, line) || line != "timestamp,open,high,low,close,volume") + throw Error(23, "the warmup CSV must start with timestamp,open,high,low,close,volume"); + while (std::getline(file, line)) + if (!line.empty()) last = line; + if (file.bad()) throw Error(22, "cannot read the warmup CSV " + warmup); + std::vector fields; + for (std::size_t from = 0;;) { + const auto comma = last.find(',', from); + fields.push_back(last.substr(from, comma == std::string::npos ? std::string::npos : comma - from)); + if (comma == std::string::npos) break; + from = comma + 1; + } + if (last.empty() || fields.size() != 6) throw Error(23, "the warmup CSV's last row must have six fields"); + if (fields[0] != std::to_string(start - 60000)) + throw Error(20, "the warmup CSV must end at the minute before the start (" + std::to_string(start - 60000) + + "); its last row is " + fields[0]); + (void)Decimal(fields[4]); + return fields[4]; +} + +ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end, const std::string& seed) { ChainBars chain(start, end); + if (!seed.empty()) chain.seed(seed); std::optional candidate; + std::int64_t last_row = -1; std::exception_ptr failure; std::string pending; bool first = true; @@ -466,9 +516,13 @@ ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int6 const auto row = archive_row(text, first); first = false; if (!row) return; + last_row = row->ts; if (!chain.anchored()) { if (row->ts < start) { candidate = *row; return; } - if (!candidate) throw Error(20, "the archive holds no print before the start: the window's predecessor is not inside it"); + if (!candidate) + throw Error(20, "the archive holds no print before the start: the window's predecessor is in " + + daily_name(archive, start - 1) + ", and export reads one daily file; start after the " + "day's first print or export from REST within its retention"); chain.anchor(*candidate); } chain.add(*row); @@ -515,13 +569,17 @@ ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int6 } if (failure) std::rethrow_exception(failure); if (!chain.anchored()) throw Error(20, "the archive holds no print at or after the start: the window is not inside it"); - if (!chain.fenced()) throw Error(20, "the archive ends before the window's fence (the first print at or after the end)"); + if (!chain.fenced()) + throw Error(20, "the archive ends before the window's fence (the first print at or after the end), which is in " + + daily_name(archive, (std::max(last_row, end - 1) / 86400000 + 1) * 86400000) + + ", and export reads one daily file; end the window before the day's last print"); return chain; } namespace { -ChainBars rest_bars(const Config& config, std::string& multiplier) { +ChainBars rest_bars(const Config& config, std::string& multiplier, const std::string& seed) { ChainBars chain(config.start, config.end); + if (!seed.empty()) chain.seed(seed); const auto now = now_ms(); if (config.end > now) throw Error(20, "the window has not closed yet: no print at or after its end can exist"); const auto venue = make_venue(config); @@ -561,6 +619,7 @@ void export_bars(const Config& config, const ExportOptions& options) { std::string multiplier = "1"; Json source; std::optional chain; + const auto seed = options.warmup.empty() ? std::string() : warmup_close(options.warmup, config.start); if (!options.archive.empty()) { if (config.venue != "binance" || config.market != "usdm" || config.mode != "agg-ticks") throw Error(23, "--archive reads the Binance USD-M daily aggTrades archive: use --venue binance --market usdm --mode agg-ticks"); @@ -570,11 +629,11 @@ void export_bars(const Config& config, const ExportOptions& options) { if (verified) verify_checksum(options.checksum, options.archive, digest); else log("warn", "archive_not_checksum_verified", file); if (base_name(options.archive).rfind(config.symbol + "-aggTrades-", 0) != 0) log("warn", "archive_name_not_symbol", file); - chain = archive_bars(options.archive, config.start, config.end); + chain = archive_bars(options.archive, config.start, config.end, seed); source = Json::object({{"kind", Json::string("archive")}, {"file", Json::string(base_name(options.archive))}, {"sha256", Json::string(digest)}, {"checksum_verified", Json::boolean(verified)}}); } else { - chain = rest_bars(config, multiplier); + chain = rest_bars(config, multiplier, seed); source = Json::object({{"kind", Json::string("rest")}, {"origin", Json::string(config.rest_url)}}); } const auto& csv = chain->csv(); @@ -589,6 +648,7 @@ void export_bars(const Config& config, const ExportOptions& options) { {"predecessor", Json::object({{"id", number(predecessor.id)}, {"ts", signed_number(predecessor.ts)}, {"price", Json::string(predecessor.price)}})}, {"fence", Json::object({{"id", number(fence.id)}, {"ts", signed_number(fence.ts)}})}, + {"warmup_close", seed.empty() ? Json{} : Json::string(seed)}, {"first_id", any ? number(predecessor.id + 1) : Json{}}, {"last_id", any ? number(fence.id - 1) : Json{}}, {"prints", number(chain->prints())}, {"bars", number(chain->bars())}, {"quiet_minutes", number(chain->quiet_minutes())}, {"rule", Json::string(bar_rule)}, {"sha256", Json::string(sha256(csv))}}); diff --git a/native/feed/src/export.hpp b/native/feed/src/export.hpp index 3c15c05..e56082b 100644 --- a/native/feed/src/export.hpp +++ b/native/feed/src/export.hpp @@ -9,6 +9,7 @@ struct ExportOptions { std::string output; // the warmup-format CSV to write; refused if it exists std::string archive; // optional: a venue daily aggregate-trade archive (.zip or extracted .csv) std::string checksum; // optional: the archive's published .CHECKSUM file + std::string warmup; // optional: the runner's warmup CSV, ending at the minute before the start }; // Prints-built one-minute bars over [config.start, config.end), by the runner's tick-built bar rule, from venue // REST within its retention or from a local daily archive. @@ -17,12 +18,14 @@ void export_bars(const Config& config, const ExportOptions& options); // The runner's tick-built bar rule over [start, end) on a proven print chain: anchor() takes the predecessor (the // last print strictly before start), add() every later print in ID order until it returns true at the fence (the // first print at or after end). A minute is first/max/min/last price and the exact volume sum; a minute without a -// print repeats the previous close with volume 0. A hole stops 20, a time regression along the chain 23. +// print repeats the previous close with volume 0. Before the window's first print that close is the runner's last +// warmup close (seed()), as the runner carries it; without a seed a quiet first minute stops 20. A hole stops 20, +// a time regression along the chain 23. class ChainBars { std::int64_t start_, end_, minute_ = -1, next_, last_ts_ = -1; std::uint64_t last_id_ = 0, count_ = 0, prints_ = 0, bars_ = 0, quiet_ = 0; std::optional predecessor_, fence_; - std::string open_, high_, low_, close_, csv_; + std::string open_, high_, low_, close_, csv_, seed_; Decimal high_value_{"0"}, low_value_{"0"}, volume_{"0"}; void flush(); void carry(std::int64_t until); @@ -30,6 +33,8 @@ class ChainBars { const std::string& close, const std::string& volume); public: ChainBars(std::int64_t start, std::int64_t end); + // The close a quiet first minute carries: the last warmup bar's close (before anchor()). + void seed(const std::string& close); void anchor(const Trade& predecessor); bool add(const Trade& trade); bool anchored() const { return predecessor_.has_value(); } @@ -43,6 +48,7 @@ class ChainBars { std::uint64_t prints() const { return prints_; } std::uint64_t bars() const { return bars_; } std::uint64_t quiet_minutes() const { return quiet_; } + const std::string& seeded_close() const { return seed_; } }; // One daily-archive line, columns agg_trade_id,price,quantity,first_trade_id,last_trade_id,transact_time, // is_buyer_maker: nullopt for the header, which only the first line may be. A transact_time above 10^14 is @@ -61,5 +67,7 @@ std::string file_sha256(const std::string& path); void verify_checksum(const std::string& checksum, const std::string& archive, const std::string& digest); // The window's chain from a daily aggregate-trade archive (.zip, else an extracted .csv), streamed: only the // predecessor candidate, the window's bars and the fence are kept. -ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end); +ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end, const std::string& seed = {}); +// The close of the warmup CSV's last row, which must be the minute before start (20 otherwise; 23 if malformed). +std::string warmup_close(const std::string& warmup, std::int64_t start); } diff --git a/native/feed/src/main.cpp b/native/feed/src/main.cpp index 7347507..7f58e91 100644 --- a/native/feed/src/main.cpp +++ b/native/feed/src/main.cpp @@ -30,10 +30,10 @@ struct StdoutFlags { void help() { std::cout << "pineforge-feed warmup --venue VENUE --market MARKET --symbol SYMBOL --start UTC|MS --end UTC|MS --output FILE\n" "pineforge-feed export --venue VENUE --market MARKET --symbol SYMBOL --mode ticks|agg-ticks --start UTC|MS --end UTC|MS --output FILE\n" - " [--archive DAILY.zip|DAILY.csv [--checksum DAILY.zip.CHECKSUM]] (prints-built 1m bars)\n" + " [--warmup RUNNER-WARMUP.csv] [--archive DAILY.zip|DAILY.csv [--checksum DAILY.zip.CHECKSUM]] (prints-built 1m bars)\n" "pineforge-feed run --venue VENUE --market MARKET --symbol SYMBOL --mode MODE --state-dir DIR [--start UTC|MS | --resume] [--output-from INDEX]\n" "pineforge-feed serve --venue VENUE --market MARKET --symbol SYMBOL --mode MODE --state-dir DIR [--start UTC|MS | --resume]\n" - " [--listen HOST:PORT (default 127.0.0.1:8787)] [--allow-remote-listen] [--client-queue-bytes N] [--max-clients N]\n" + " [--listen HOST:PORT|[IPV6]:PORT (default 127.0.0.1:8787)] [--allow-remote-listen] [--client-queue-bytes N] [--max-clients N]\n" " GET /v1/status, GET /v1/snapshot (complete prefix from index 0, at most 4 MiB), ws://HOST:PORT/v1/stream?epoch=E&from=I\n" "Venues and modes:\n" " binance spot BTCUSDT bars | ticks\n" @@ -122,6 +122,7 @@ int run(int argc, char** argv) { else if (key == "--output") output = value; else if (key == "--archive") exported.archive = value; else if (key == "--checksum") exported.checksum = value; + else if (key == "--warmup") exported.warmup = value; else if (key == "--output-from") config.output_from = unsigned_value(value); else if (key == "--max-messages") config.max_messages = unsigned_value(value); else if (key == "--segment-bytes") config.segment_bytes = unsigned_value(value); @@ -144,8 +145,8 @@ int run(int argc, char** argv) { else throw Error(23, "unknown CLI option: " + key); } gate(config, command == "warmup", seen.count("--rest-url") != 0, seen.count("--ws-url") != 0); - if (command != "export" && (!exported.archive.empty() || !exported.checksum.empty())) - throw Error(23, "--archive and --checksum are export options"); + if (command != "export" && (!exported.archive.empty() || !exported.checksum.empty() || !exported.warmup.empty())) + throw Error(23, "--archive, --checksum and --warmup are export options"); if (!config.max_queue_bytes || !config.max_replay_seconds || config.max_replay_seconds > 3600 || !config.reconnect_seconds || config.reconnect_seconds > 86100) throw Error(23, "budgets must be positive; reconnect must precede the 24-hour connection limit"); @@ -163,7 +164,7 @@ int run(int argc, char** argv) { if (command == "export") { if (output.empty() || config.resume || !config.state_dir.empty() || seen.count("--output-from") || !tick_mode(config.mode) || config.start < 0 || config.end < 0) - throw Error(23, "export takes --mode ticks|agg-ticks, --start, --end and --output (plus --archive [--checksum])"); + throw Error(23, "export takes --mode ticks|agg-ticks, --start, --end and --output (plus --warmup, --archive [--checksum])"); if (!exported.checksum.empty() && exported.archive.empty()) throw Error(23, "--checksum verifies an --archive"); exported.output = output; export_bars(config, exported); diff --git a/native/feed/tests/export_unit.cpp b/native/feed/tests/export_unit.cpp index 73fd16d..65eaabc 100644 --- a/native/feed/tests/export_unit.cpp +++ b/native/feed/tests/export_unit.cpp @@ -229,18 +229,30 @@ int main() { expect(23, [&] { chain.add({506, 300050, "12.10", "0.2"}); }); } { - // The first minute is quiet: it carries the predecessor's price; a fence exactly at the end closes the window. + // The first minute is quiet: it carries the warmup close, not the predecessor's price, exactly as the + // runner carries its last warmup bar; a later quiet minute carries the last print. A fence exactly at the + // end closes the window. ChainBars chain(180000, 420000); + chain.seed("9.95"); chain.anchor({502, 179999, "9.90", "0.3"}); assert(!chain.add({503, 240001, "12.00", "0.4"})); assert(chain.add({504, 420000, "11.00", "0.1"})); - assert(chain.csv() == bar_header + "180000,9.90,9.90,9.90,9.90,0\n240000,12.00,12.00,12.00,12.00,0.4\n" + assert(chain.csv() == bar_header + "180000,9.95,9.95,9.95,9.95,0\n240000,12.00,12.00,12.00,12.00,0.4\n" "300000,12.00,12.00,12.00,12.00,0\n360000,12.00,12.00,12.00,12.00,0\n"); - assert(chain.quiet_minutes() == 3 && chain.prints() == 1 && chain.bars() == 4); + assert(chain.quiet_minutes() == 3 && chain.prints() == 1 && chain.bars() == 4 && chain.seeded_close() == "9.95"); + } + { + // Without the warmup close a quiet first minute is not guessed from the predecessor. + ChainBars chain(180000, 420000); + chain.anchor({502, 179999, "9.90", "0.3"}); + expect(20, [&] { chain.add({503, 240001, "12.00", "0.4"}); }); + ChainBars late(180000, 240000); + expect(23, [&] { late.seed("x"); }); } { // A quiet window: the predecessor's ID + 1 is already the fence. ChainBars chain(120000, 240000); + chain.seed("100.5"); chain.anchor({7, 0, "100.5", "1"}); assert(chain.add({8, 250000, "101", "1"})); assert(chain.csv() == bar_header + "120000,100.5,100.5,100.5,100.5,0\n180000,100.5,100.5,100.5,100.5,0\n"); @@ -316,7 +328,9 @@ int main() { assert(chain.csv() == window_bars && chain.predecessor().id == 499 && chain.predecessor().price == "10.10"); assert(chain.fence().id == 505 && chain.prints() == 5 && chain.quiet_minutes() == 1); } - assert(archive_bars(headerless, 180000, 300000).csv() == bar_header + "180000,9.90,9.90,9.90,9.90,0\n240000,12.00,12.00,11.00,11.00,0.5\n"); + // A quiet first minute carries the warmup close it is given, and stops 20 without one. + assert(archive_bars(headerless, 180000, 300000, "9.95").csv() == bar_header + "180000,9.95,9.95,9.95,9.95,0\n240000,12.00,12.00,11.00,11.00,0.5\n"); + expect(20, [&] { (void)archive_bars(headerless, 180000, 300000); }); const auto crlf = temporary.path + "/crlf.csv"; put_file(crlf, replaced(replaced(archive_header + rows, "\n", "\r\n"), "499,10.10,0.1,998,999,119999,true\n", "499,10.10,0.1,998,999,119999,true\r\n")); assert(archive_bars(crlf, 120000, 300000).csv() == window_bars); diff --git a/native/feed/tests/mock_export.py b/native/feed/tests/mock_export.py index 0ac3f23..782462d 100644 --- a/native/feed/tests/mock_export.py +++ b/native/feed/tests/mock_export.py @@ -12,6 +12,7 @@ import time import unittest import zipfile +from datetime import UTC, datetime from decimal import Decimal, getcontext # mock_venue reads the binary from sys.argv[1] at import time. @@ -34,9 +35,10 @@ def canonical(value): return text -def expected_bars(prints, start, end): - """The tick-built bar rule, independently: prints are (id, ts, price, qty) in ID order.""" - close = [p for p in prints if p[1] < start][-1][2] +def expected_bars(prints, start, end, warmup_close=None): + """The tick-built bar rule, independently: prints are (id, ts, price, qty) in ID order. A + quiet first minute carries the runner's last warmup close.""" + close = warmup_close rows = ["timestamp,open,high,low,close,volume"] for minute in range(start, end, 60000): inside = [p for p in prints if minute <= p[1] < minute + 60000] @@ -164,16 +166,33 @@ def test_rest_bars_follow_the_rule_with_quiet_minute_and_carry(self): (manifest["prints"], manifest["bars"], manifest["quiet_minutes"]), (6, 4, 1) ) + def warmup(self, start, close): + path = self.directory / "warmup.csv" + path.write_text( + "timestamp,open,high,low,close,volume\n" + f"{start - 120000},9,11,8,10,1\n{start - 60000},10,10.5,9.5,{close},2\n" + ) + return path + def test_rest_start_survives_a_quiet_first_hour(self): # The first aggregate after --start lies in the second hour window: the start lookup - # searches forward up to the clock instead of stopping 20 after one window. + # searches forward up to the clock instead of stopping 20 after one window. The quiet + # first minutes carry the warmup close (10.2), not the predecessor's 10.10000000, as the + # runner carries its last warmup bar; the quiet later minute carries the last print. start = recent_minute(90) + end = start + 73 * 60000 prints = [(499, start - 1, "10.10000000", "0.10000000")] - prints += synthetic(500, start + 70 * 60000, 3, 4) + prints += synthetic(500, start + 70 * 60000, 4, 4, quiet=(1,)) server = self.server(prints) - self.export(start, start + 72 * 60000, server) - self.assertEqual(self.output.read_text(), expected_bars(prints, start, start + 72 * 60000)) - self.assertEqual(self.manifest()["quiet_minutes"], 70) + warmup = self.warmup(start, "10.2") + self.export(start, end, server, extra=["--warmup", str(warmup)]) + text = self.output.read_text() + self.assertEqual(text, expected_bars(prints, start, end, warmup_close="10.2")) + self.assertIn(f"\n{start},10.2,10.2,10.2,10.2,0\n", text) + last = [p for p in prints if p[1] < start + 71 * 60000][-1][2] + self.assertIn(f"\n{start + 71 * 60000},{last},{last},{last},{last},0\n", text) + manifest = self.manifest() + self.assertEqual((manifest["quiet_minutes"], manifest["warmup_close"]), (71, "10.2")) windows = [ int(query["startTime"][0]) for path, query in server.queries @@ -181,6 +200,27 @@ def test_rest_start_survives_a_quiet_first_hour(self): ] self.assertIn(start + 3600000, windows) + def test_quiet_first_minute_without_the_warmup_close_stops(self): + start = recent_minute(30) + prints = [(499, start - 1, "10.10000000", "0.10000000")] + prints += synthetic(500, start + 60000, 2, 3) + server = self.server(prints) + result = self.export(start, start + 120000, server, expected=20) + self.assertIn("--warmup", result.stderr) + warmup = self.warmup(start + 60000, "10.2") + result = self.export( + start, start + 120000, server, extra=["--warmup", str(warmup)], expected=20 + ) + self.assertIn("minute before the start", result.stderr) + warmup.write_text("timestamp,open,high,low,close,volume\n" + f"{start - 60000},1,1,1,x,1\n") + self.export(start, start + 120000, server, extra=["--warmup", str(warmup)], expected=23) + # A first minute with prints never reads the warmup close. + self.export(start + 60000, start + 120000, server) + self.assertEqual( + self.output.read_text(), expected_bars(prints, start + 60000, start + 120000) + ) + self.assertIsNone(self.manifest()["warmup_close"]) + def test_rest_pages_by_from_id_and_sums_volume_exactly(self): start = recent_minute(40) prints = synthetic(1000, start - 60000, 6, 700, quiet=(2,)) @@ -334,6 +374,22 @@ def test_archive_checksum_mismatch_wrong_name_and_corruption_stop_21(self): self.assertRegex(result.stderr, "deflate|CRC-32|recorded size") self.assertNotIn("export_written", result.stderr) + def test_archive_day_boundaries_name_the_missing_daily_file(self): + # A window from 00:00 UTC has its predecessor in the previous day's file, and one to 24:00 + # has its fence in the next day's: export reads one daily file and says which one is needed. + day = int(datetime(2026, 10, 2, tzinfo=UTC).timestamp() * 1000) + prints = synthetic(1000, day, 3, 4) + synthetic(1012, day + 86400000 - 180000, 3, 4) + archive = self.write_archive(f"{SYMBOL}-aggTrades-2026-10-02.csv", prints) + result = self.export(day, day + 120000, extra=["--archive", str(archive)], expected=20) + self.assertIn(f"{SYMBOL}-aggTrades-2026-10-01.csv", result.stderr) + result = self.export( + day + 86400000 - 120000, + day + 86400000, + extra=["--archive", str(archive)], + expected=20, + ) + self.assertIn(f"{SYMBOL}-aggTrades-2026-10-03.csv", result.stderr) + def test_archive_outside_window_and_unsupported_venue(self): start, prints = self.archive_prints() archive = self.write_archive(f"{SYMBOL}-aggTrades-day.csv", prints) From 30115576a6c566184c8778e8e5eff41c5f721a70 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 11:24:47 +0800 Subject: [PATCH 11/15] Compare the live export with the runner's own bars The export gate compared with bars the harness built from the tape. It now replays the runner's committed ledger inputs through the observed build of the strategy, requires the runner's recorded state hash after every input, and reads the bar the strategy saw at each time event through the engine observer. Open, high, low and close must be equal; volume may differ by one ulp, since without a qty_step the runner sums tick volume in compensated doubles. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/tests/public_e2e.py | 141 +++++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 2 deletions(-) diff --git a/native/feed/tests/public_e2e.py b/native/feed/tests/public_e2e.py index 6edcdac..28ab113 100644 --- a/native/feed/tests/public_e2e.py +++ b/native/feed/tests/public_e2e.py @@ -2,13 +2,16 @@ """Explicit public-data soak; captures stay in the operator-selected directory.""" import argparse +import ctypes import hashlib import http.server import itertools import json +import math import os import pathlib import signal +import sqlite3 import struct import subprocess import threading @@ -39,6 +42,118 @@ def journal_rows(state, exact=False): SCRIPT_TF_MS = 60000 # the runner and batch both run --script-tf 1 +class StrategyBar(ctypes.Structure): + _fields_ = [ + ("open", ctypes.c_double), + ("high", ctypes.c_double), + ("low", ctypes.c_double), + ("close", ctypes.c_double), + ("volume", ctypes.c_double), + ("timestamp", ctypes.c_int64), + ] + + +class StrategyTick(ctypes.Structure): + _fields_ = [ + ("timestamp", ctypes.c_int64), + ("sequence", ctypes.c_uint64), + ("price", ctypes.c_double), + ("quantity", ctypes.c_double), + ] + + +def runner_bars(strategy, warmup, ledger, symbol): + """The runner's own tick-built bars. Its committed ledger inputs are replayed in order through + the observed build of the same strategy; every input must reach the state hash the runner + recorded for it, so this is the runner's computation, not a re-implementation. The bar the + strategy saw at each time event is read back through the engine observer + (`equivalence_source_bar`). Returns {bar open time: (o, h, l, c, v) as the runner's doubles}.""" + library = ctypes.CDLL(str(strategy)) + signatures = { + "strategy_create": ([ctypes.c_char_p], ctypes.c_void_p), + "strategy_free": ([ctypes.c_void_p], None), + "strategy_set_syminfo_timezone": ([ctypes.c_void_p, ctypes.c_char_p], None), + "strategy_set_chart_timezone": ([ctypes.c_void_p, ctypes.c_char_p], None), + "strategy_set_syminfo_session": ([ctypes.c_void_p, ctypes.c_char_p], None), + "strategy_set_syminfo_string": ( + [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p], + ctypes.c_int, + ), + "strategy_stream_begin": ( + [ + ctypes.c_void_p, + ctypes.POINTER(StrategyBar), + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_char_p, + ], + ctypes.c_int, + ), + "strategy_stream_push_ticks": ( + [ctypes.c_void_p, ctypes.POINTER(StrategyTick), ctypes.c_int], + ctypes.c_int, + ), + "strategy_stream_advance_time": ([ctypes.c_void_p, ctypes.c_int64], ctypes.c_int), + "strategy_stream_state_hash": ([ctypes.c_void_p], ctypes.c_uint64), + "strategy_stream_order_actions_clear": ([ctypes.c_void_p], None), + "equivalence_source_bar": ( + [ctypes.c_void_p, ctypes.c_int64, ctypes.POINTER(StrategyBar)], + ctypes.c_int, + ), + } + for name, (arguments, result) in signatures.items(): + getattr(library, name).argtypes = arguments + getattr(library, name).restype = result + with open(warmup) as source: + lines = source.read().splitlines() + assert lines[0] == "timestamp,open,high,low,close,volume" + history = [] + for line in lines[1:]: + values = line.split(",") + history.append(StrategyBar(*(float(value) for value in values[1:]), int(values[0]))) + array = (StrategyBar * len(history))(*history) + with sqlite3.connect(f"file:{ledger}?mode=ro", uri=True) as connection: + inputs = connection.execute( + "SELECT input_index, canonical_json, state_hash FROM inputs ORDER BY input_index" + ).fetchall() + assert [row[0] for row in inputs] == list(range(len(inputs))), "ledger inputs have a gap" + handle = library.strategy_create(None) + assert handle, "strategy_create failed" + bars = {} + try: + # The runner's defaults for a run without symbol metadata: UTC, 24x7, its --symbol label. + library.strategy_set_syminfo_timezone(handle, b"UTC") + library.strategy_set_chart_timezone(handle, b"UTC") + library.strategy_set_syminfo_session(handle, b"24x7") + assert library.strategy_set_syminfo_string(handle, b"tickerid", symbol.encode()) == 0 + ticker = symbol.split(":", 1)[-1] + assert library.strategy_set_syminfo_string(handle, b"ticker", ticker.encode()) == 0 + assert library.strategy_stream_begin(handle, array, len(history), b"1", b"1") == 0 + library.strategy_stream_order_actions_clear(handle) + for index, text, recorded in inputs: + event = json.loads(text, parse_float=Decimal) + if event["type"] == "tick": + tick = StrategyTick( + event["ts"], event["seq"], float(event["price"]), float(event["qty"]) + ) + assert library.strategy_stream_push_ticks(handle, ctypes.byref(tick), 1) == 0 + else: + assert event["type"] == "time", event + assert library.strategy_stream_advance_time(handle, event["ts"]) == 0 + library.strategy_stream_order_actions_clear(handle) + state = str(library.strategy_stream_state_hash(handle)) + assert state == recorded, f"replay left the runner's state at input {index}" + if event["type"] == "time": + bar = StrategyBar() + assert library.equivalence_source_bar(handle, 60000, ctypes.byref(bar)) == 0 + assert str(library.strategy_stream_state_hash(handle)) == recorded + values = (bar.open, bar.high, bar.low, bar.close, bar.volume) + assert bars.setdefault(bar.timestamp, values) == values + finally: + library.strategy_free(handle) + return bars + + def action_key(record, mode): """Comparable action fields. Ticks use the predeclared R-B2 mapping: an action timestamp becomes floor(ts / script_tf) * script_tf on both sides, because a tick fill carries its @@ -611,6 +726,8 @@ def export_check(self, mode, normalized, actual_actions): str(end), "--output", str(output), + "--warmup", + str(self.directory / "warmup.csv"), ] with open(self.directory / (mode + "-export.stderr"), "w") as error: subprocess.run(command, stderr=error, check=True, timeout=600) @@ -627,7 +744,25 @@ def export_check(self, mode, normalized, actual_actions): ) for row in (line.split(",") for line in lines[1:]) ] - assert exported == normalized, "exported bars differ from the runner's prints-built bars" + assert exported == normalized, "exported bars differ from the tape's prints-built bars" + # The runner's own bars, value for value: the batch reads the exported CSV as doubles. The + # export's volume is the exact decimal sum; without a quantity grid (no qty_step) the + # runner sums volume in compensated doubles, which can land one ulp away. + volume_ulp = 0 + built = runner_bars( + self.options.observed_strategy, + self.directory / "warmup.csv", + self.directory / (mode + ".sqlite3"), + self.venue.label, + ) + for bar in exported: + runner = built.get(bar["ts_open"]) + assert runner is not None, f"the runner built no bar at {bar['ts_open']}" + values = tuple(float(bar[key]) for key in ("o", "h", "l", "c", "v")) + assert values[:4] == runner[:4], f"export {bar} differs from the runner's bar {runner}" + if values[4] != runner[4]: + assert abs(values[4] - runner[4]) <= math.ulp(values[4]), (bar, runner) + volume_ulp += 1 manifest = json.loads((self.directory / (mode + "-export.csv.manifest.json")).read_text()) combined = self.directory / (mode + "-export-combined.csv") with open(combined, "w") as target: @@ -646,7 +781,9 @@ def export_check(self, mode, normalized, actual_actions): equal = expected == actual_actions self.receipt( f"{'PASS' if equal else 'FAIL'} {mode} export prints-built bars equal the runner's " - f"tick-built bars minutes={len(exported)} prints={manifest['prints']} " + f"tick-built bars (state-hash-proven replay, read via the observer) " + f"minutes={len(exported)} ohlc_exact={len(exported)} volume_one_ulp={volume_ulp} " + f"prints={manifest['prints']} " f"quiet_minutes={manifest['quiet_minutes']} " f"batch_over_export_actions={len(expected)} runner={len(actual_actions)}" ) From 2b22949a4fd5fa38e88fd6c71674cff2d80e6139 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 11:24:47 +0800 Subject: [PATCH 12/15] Document serve capacity, export boundaries and per-process pacing Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/README.md | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/native/feed/README.md b/native/feed/README.md index a242bff..7c2bb16 100644 --- a/native/feed/README.md +++ b/native/feed/README.md @@ -92,7 +92,8 @@ publication without its manifest is not a qualified warmup. ```sh build-feed/pineforge-feed export --venue binance --market usdm --symbol BTCUSDT \ - --mode agg-ticks --start 2026-10-03T00:00:00Z --end 2026-10-03T03:20:00Z --output agg-bars.csv + --mode agg-ticks --start 2026-10-03T00:00:00Z --end 2026-10-03T03:20:00Z --output agg-bars.csv \ + --warmup warmup.csv # or from a local copy of the venue's public daily aggregate-trade archive: build-feed/pineforge-feed export --venue binance --market usdm --symbol BTCUSDT \ --mode agg-ticks --start 2026-10-02T00:10:00Z --end 2026-10-02T23:50:00Z --output agg-bars.csv \ @@ -106,9 +107,14 @@ writes exactly those bars in the warmup CSV format, by the runner's tick-built bar rule: a print belongs to minute `floor(ts / 60000) * 60000`; open is the first price, high the maximum, low the minimum, close the last price, and volume the exact decimal sum of the quantities; a minute without a print repeats the -previous close (for the first minute, the last print before `--start`) with -volume `0`, as the runner's carry-forward bar does. Price tokens are the venue's -own; the volume is the canonical exact sum. +previous close with volume `0`, as the runner's carry-forward bar does. Before +the window's first print that close is the runner's last warmup close, not the +last print before `--start`: `--warmup` takes the runner's warmup CSV, whose last +row must be the minute before `--start` (20 otherwise), and the manifest records +it as `warmup_close`. Without `--warmup` a quiet first minute stops with 20 +rather than guess. Price tokens are the venue's own; the volume is the canonical +exact sum. (Without a `qty_step` the runner sums tick volume in compensated +doubles, which can land one ulp from the exact sum the export writes.) Completeness is proven, never assumed: the prints must form a contiguous ID chain from the last print strictly before `--start` through the fence, the first print @@ -126,7 +132,11 @@ without its header line), streamed rather than loaded. A `transact_time` above takes the published ` ` file and must match the archive's SHA-256 and base name (21 otherwise); without it the export logs `archive_not_checksum_verified`. The window, its predecessor and its fence must -all lie inside the one archive (else 20). Archives are never fetched +all lie inside the one archive (else 20, naming the daily file that holds the +missing print). So a window starting at 00:00 UTC, whose predecessor is the +previous day's last print, or one ending at 24:00, whose fence is the next day's +first print, cannot come from the one archive: start or end the window inside +the day, or export it from REST within its retention. Archives are never fetched automatically. An existing output or manifest is refused. The CSV and then @@ -346,6 +356,11 @@ weight ceiling is spent by the venue's own count: every response's `X-MBX-USED-WEIGHT-1M` header (which counts every client on the IP) is kept, and a request whose weight would take the current window past 90% of the ceiling waits for the next window (`rest_quota_wait`); otherwise requests are not spaced. +The 90% share is per process: each producer sees the count only as of its own +last response, so several producers on one IP (one per symbol) can together cross +the ceiling between their requests. A `429` waits; a `418` ban stops every +producer on that IP with 22. Run one Binance REST client per IP for large +catch-ups, or stagger the producers. Other published ceilings are paced evenly across their interval. `429` honours `Retry-After` as delta-seconds or an IMF-fixdate HTTP-date; a wait above 24 hours stops with 22 and an unparseable value with 23. A ban/access rejection is not retried as anonymous trading @@ -557,7 +572,17 @@ Neither ever blocks the producer or another client. A client holds a journal segment open only while it catches up. At start `serve` raises its open-file limit to the hard limit and refuses to start (23) when that is below `2 x --max-clients + 64`. A client that stops reading entirely is dropped when its blocked write -times out (30 seconds). On SIGTERM the producer drains each live client's queue +times out (10 seconds). A client never sends a message: any frame it sends after +`serve` has closed it ends the connection at once. + +Capacity: the HTTP server runs one worker thread per connection, and each +streaming client holds one for its lifetime. `serve` runs `--max-clients + 16` +workers, so status, snapshot and new runner connects keep spare workers, and it +drops any request not complete within 10 seconds (a half-open connection). +CivetWeb has no separate limit for incomplete requests, and buffers a whole +client frame before `serve` sees it (its own ceiling is 2 GiB), so anything +beyond same-host runners must go through the reverse proxy below, which absorbs +incomplete requests and caps frame sizes. On SIGTERM the producer drains each live client's queue and closes every client with 1001 (a client still catching up from the journal is closed at once and resumes from its own count). @@ -566,8 +591,8 @@ answers PINGs without telling it, so `serve` sends an unsolicited PONG to a client after 5 seconds without a message; a bar stream (one message a minute) stays connected. -Listening: `--listen` defaults to `127.0.0.1:8787`; port 0 picks a free port, -logged as `serve_listening`. **`serve` has no TLS and no authentication.** A +Listening: `--listen` defaults to `127.0.0.1:8787`; `[::1]:PORT` listens on the +IPv6 loopback; port 0 picks a free port, logged as `serve_listening`. **`serve` has no TLS and no authentication.** A non-loopback address needs `--allow-remote-listen`; expose it only behind a reverse proxy that terminates TLS and authenticates clients (for example nginx or Caddy proxying `wss://` to the loopback listener), and never on a public From 6e63d30ad826a5198f0dae9dd1212f2e02949313 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 11:26:24 +0800 Subject: [PATCH 13/15] Build CivetWeb with reentrant time functions Without REENTRANT_TIME, CivetWeb formats the Date header with gmtime(), whose static result two workers overwrite when they answer at once, as the request timeouts of several incomplete requests do (found by TSan in the half-open scenario). The mock's serve reader now keeps non-JSON stderr lines, so a sanitizer report shows in the failure. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/CMakeLists.txt | 2 +- native/feed/tests/mock_venue.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/native/feed/CMakeLists.txt b/native/feed/CMakeLists.txt index 5587311..48754db 100644 --- a/native/feed/CMakeLists.txt +++ b/native/feed/CMakeLists.txt @@ -70,7 +70,7 @@ if(PINEFORGE_FEED_SERVE) add_library(pineforge_feed_civetweb STATIC "${civetweb_SOURCE_DIR}/src/civetweb.c") target_include_directories(pineforge_feed_civetweb SYSTEM PUBLIC "${civetweb_SOURCE_DIR}/include") target_compile_definitions(pineforge_feed_civetweb PRIVATE NO_SSL NO_CGI NO_FILES NO_CACHING STOP_FLAG_NEEDS_LOCK - USE_IPV6 PUBLIC USE_WEBSOCKET) + USE_IPV6 REENTRANT_TIME PUBLIC USE_WEBSOCKET) set_target_properties(pineforge_feed_civetweb PROPERTIES C_STANDARD 99) target_compile_options(pineforge_feed_civetweb PRIVATE -w) target_link_libraries(pineforge_feed_civetweb PUBLIC Threads::Threads ${CMAKE_DL_LIBS}) diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py index f78f36d..160ab23 100644 --- a/native/feed/tests/mock_venue.py +++ b/native/feed/tests/mock_venue.py @@ -2316,7 +2316,11 @@ def serve( def read(): for line in process.stderr: - records.append(json.loads(line)) + try: + records.append(json.loads(line)) + except ValueError: + # A sanitizer report, kept so that a failure shows it. + records.append({"event": "raw", "line": line}) if records[-1]["event"] == "serve_listening": ready.set() ready.set() From b3a72671e262be644d27770ae6dd823b17a95932 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 11:35:07 +0800 Subject: [PATCH 14/15] Name the neighbouring daily file from the archive's own rows The missing predecessor is in the day before the archive's first row, and the missing fence in the day after its last row. Deriving them from the window named the wrong file when the archive's first print came after the window start, or when the window reached past the day. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/src/export.cpp | 7 ++++--- native/feed/tests/mock_export.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/native/feed/src/export.cpp b/native/feed/src/export.cpp index 8240990..9d2dda8 100644 --- a/native/feed/src/export.cpp +++ b/native/feed/src/export.cpp @@ -519,10 +519,11 @@ ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int6 last_row = row->ts; if (!chain.anchored()) { if (row->ts < start) { candidate = *row; return; } + // This is the archive's first row: the predecessor is in the previous day's file. if (!candidate) throw Error(20, "the archive holds no print before the start: the window's predecessor is in " + - daily_name(archive, start - 1) + ", and export reads one daily file; start after the " - "day's first print or export from REST within its retention"); + daily_name(archive, row->ts - row->ts % 86400000 - 1) + ", and export reads one daily " + "file; start after the day's first print or export from REST within its retention"); chain.anchor(*candidate); } chain.add(*row); @@ -571,7 +572,7 @@ ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int6 if (!chain.anchored()) throw Error(20, "the archive holds no print at or after the start: the window is not inside it"); if (!chain.fenced()) throw Error(20, "the archive ends before the window's fence (the first print at or after the end), which is in " + - daily_name(archive, (std::max(last_row, end - 1) / 86400000 + 1) * 86400000) + + daily_name(archive, (last_row / 86400000 + 1) * 86400000) + ", and export reads one daily file; end the window before the day's last print"); return chain; } diff --git a/native/feed/tests/mock_export.py b/native/feed/tests/mock_export.py index 782462d..dc3767c 100644 --- a/native/feed/tests/mock_export.py +++ b/native/feed/tests/mock_export.py @@ -389,6 +389,17 @@ def test_archive_day_boundaries_name_the_missing_daily_file(self): expected=20, ) self.assertIn(f"{SYMBOL}-aggTrades-2026-10-03.csv", result.stderr) + # A window reaching past the day, or starting before its first print, names the same files. + result = self.export( + day + 86400000 - 120000, + day + 86400000 + 600000, + extra=["--archive", str(archive)], + expected=20, + ) + self.assertIn(f"{SYMBOL}-aggTrades-2026-10-03.csv", result.stderr) + late = self.write_archive(f"{SYMBOL}-aggTrades-2026-10-02.zip", prints[5:]) + result = self.export(day + 60000, day + 120000, extra=["--archive", str(late)], expected=20) + self.assertIn(f"{SYMBOL}-aggTrades-2026-10-01.zip", result.stderr) def test_archive_outside_window_and_unsupported_venue(self): start, prints = self.archive_prints() From 371fb22a20e9727b45eea206b5a2ee3082554bfd Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 11:56:36 +0800 Subject: [PATCH 15/15] Export the runner's own tick volume, bit for bit The export wrote the exact decimal sum of the quantities, while the runner builds bar volume with the engine's tick-volume rule, so a bar could differ from the runner's by one ulp. Port that rule operation for operation: with a decimal quantity step 1/10^k the exact sum of grid units, rounded once; otherwise, or after an off-grid quantity or an overflow, the compensated binary64 sum in print order. Quantities are read with std::stod as the runner reads the feed, and the volume is written as the shortest round-trip decimal. export takes --qty-step to match a runner deployed with --syminfo qty_step=V; without it the rule is the runner's default. The manifest records qty_step and volume_rule. The unit suite runs the engine's own tick-volume vectors against the port. The live export gate now requires every field to equal the runner's double, with no tolerance, and the harness passes --qty-step to the runner, the export, the replay and the batch probe. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/README.md | 27 ++++++-- native/feed/src/export.cpp | 111 +++++++++++++++++++++++++++--- native/feed/src/export.hpp | 31 ++++++++- native/feed/src/main.cpp | 9 ++- native/feed/tests/batch_probe.cpp | 6 +- native/feed/tests/export_unit.cpp | 56 ++++++++++++++- native/feed/tests/mock_export.py | 94 +++++++++++++++++++++++-- native/feed/tests/public_e2e.py | 62 ++++++++++------- 8 files changed, 346 insertions(+), 50 deletions(-) diff --git a/native/feed/README.md b/native/feed/README.md index 7c2bb16..65242dd 100644 --- a/native/feed/README.md +++ b/native/feed/README.md @@ -93,7 +93,7 @@ publication without its manifest is not a qualified warmup. ```sh build-feed/pineforge-feed export --venue binance --market usdm --symbol BTCUSDT \ --mode agg-ticks --start 2026-10-03T00:00:00Z --end 2026-10-03T03:20:00Z --output agg-bars.csv \ - --warmup warmup.csv + --warmup warmup.csv # add --qty-step V when the runner runs with --syminfo qty_step=V # or from a local copy of the venue's public daily aggregate-trade archive: build-feed/pineforge-feed export --venue binance --market usdm --symbol BTCUSDT \ --mode agg-ticks --start 2026-10-02T00:10:00Z --end 2026-10-02T23:50:00Z --output agg-bars.csv \ @@ -106,15 +106,32 @@ must equal the forward run needs bars built the same way, not venue candles (see writes exactly those bars in the warmup CSV format, by the runner's tick-built bar rule: a print belongs to minute `floor(ts / 60000) * 60000`; open is the first price, high the maximum, low the minimum, close the last price, and volume the -exact decimal sum of the quantities; a minute without a print repeats the +runner's own tick volume (below); a minute without a print repeats the previous close with volume `0`, as the runner's carry-forward bar does. Before the window's first print that close is the runner's last warmup close, not the last print before `--start`: `--warmup` takes the runner's warmup CSV, whose last row must be the minute before `--start` (20 otherwise), and the manifest records it as `warmup_close`. Without `--warmup` a quiet first minute stops with 20 -rather than guess. Price tokens are the venue's own; the volume is the canonical -exact sum. (Without a `qty_step` the runner sums tick volume in compensated -doubles, which can land one ulp from the exact sum the export writes.) +rather than guess. Price tokens are the venue's own. + +Volume is the double the runner builds, bit for bit, written as the shortest +decimal that reads back as that double. The runner reads each quantity from +the feed's JSON number (`std::stod`) and its rule depends on its symbol's +quantity step (the engine's tick-volume rule): + +- **No quantity step (the default, and the runner's default):** the runner adds + the quantities in print order as a compensated (Neumaier) binary64 sum, so + `0.1 + 0.2 + 0.05` gives `0.35000000000000003`. Use this when the runner runs + without `--syminfo qty_step=...`. +- **`--qty-step V`:** use the same value the runner gets as + `--syminfo qty_step=V`. A decimal step `1/10^k` (`0.001`, `0.1`, `1`) makes the + volume the exact sum of grid units, rounded once to the nearest double; any + other step (for example `0.0004`), a quantity off the grid, or overflow falls + back to the compensated sum, exactly as the runner does. + +The manifest records `qty_step` and `volume_rule` (`exact-decimal-grid` or +`compensated-binary64`). An export made for one setting does not match a runner +deployed with the other. Completeness is proven, never assumed: the prints must form a contiguous ID chain from the last print strictly before `--start` through the fence, the first print diff --git a/native/feed/src/export.cpp b/native/feed/src/export.cpp index 9d2dda8..1df61fa 100644 --- a/native/feed/src/export.cpp +++ b/native/feed/src/export.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -19,8 +20,9 @@ namespace { constexpr std::size_t csv_limit = 64 * 1024 * 1024; constexpr std::string_view archive_header = "agg_trade_id,price,quantity,first_trade_id,last_trade_id,transact_time,is_buyer_maker"; constexpr const char* bar_rule = "a print belongs to minute floor(ts/60000)*60000; open=first price, high=max, low=min, close=last, " - "volume=exact sum of quantities; a minute without a print has open=high=low=close=the previous close (the " - "runner's last warmup close before the first print) and volume 0"; + "volume=the runner's tick volume (exact on a 1/10^k qty_step grid, else the compensated binary64 sum in print " + "order) as the shortest round-trip decimal; a minute without a print has open=high=low=close=the previous close " + "(the runner's last warmup close before the first print) and volume 0"; Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } Json signed_number(std::int64_t value) { return Json::number(std::to_string(value)); } std::string base_name(const std::string& path) { return std::filesystem::path(path).filename().string(); } @@ -225,6 +227,84 @@ ChainBars::ChainBars(std::int64_t start, std::int64_t end) : start_(start), end_ throw Error(23, "export requires a nonempty minute-aligned exclusive range"); if ((end - start) / 60000 > 100000) throw Error(22, "export range exceeds 100000 minutes"); } +void RunnerVolume::reset(const std::optional& step) { + *this = RunnerVolume{}; + if (!step) return; + double scale = 1.0; + for (int places = 0; places <= 22; ++places, scale *= 10.0) { + if (*step != 1.0 / scale) continue; + divisor_ = scale; + places_ = places; + exact_ = true; + return; + } +} +double RunnerVolume::decimal(std::int64_t units) const { + if (units <= (std::int64_t{1} << 53)) return static_cast(units) / divisor_; + // Above 2^53 the quotient units / 10^places is rounded once, to nearest even, in integer arithmetic. + std::uint64_t numerator = static_cast(units), denominator = 1; + for (int places = 0; places < places_; ++places) denominator *= 5; + int exponent = -places_; + while (denominator <= numerator / 2) { + denominator *= 2; + ++exponent; + } + while (numerator < denominator) { + numerator *= 2; + --exponent; + } + std::uint64_t remainder = numerator - denominator, significand = 1; + for (int bit = 0; bit < 52; ++bit) { + remainder *= 2; + significand *= 2; + if (remainder >= denominator) { + remainder -= denominator; + ++significand; + } + } + const std::uint64_t complement = denominator - remainder; + if (remainder > complement || (remainder == complement && (significand & 1))) ++significand; + return std::ldexp(static_cast(significand), exponent - 52); +} +void RunnerVolume::add(double quantity) { + const double next = sum_ + quantity; + if (std::isfinite(next)) compensation_ += sum_ >= quantity ? (sum_ - next) + quantity : (quantity - next) + sum_; + else compensation_ = 0.0; + sum_ = next; + if (!exact_) return; + const double rounded = std::round(quantity * divisor_); + if (!std::isfinite(rounded) || rounded >= std::ldexp(1.0, 63)) { + exact_ = false; + return; + } + const auto units = static_cast(rounded); + if (decimal(units) != quantity || units > std::numeric_limits::max() - units_) { + exact_ = false; + return; + } + units_ += units; +} +double RunnerVolume::value() const { return exact_ ? decimal(units_) : sum_ + compensation_; } + +double runner_double(const std::string& token) { + std::size_t used = 0; + double value = 0; + try { value = std::stod(token, &used); } catch (const std::exception&) { used = 0; } + if (token.empty() || used != token.size() || !std::isfinite(value)) throw Error(23, "not a finite number: " + token); + return value; +} +std::string shortest_decimal(double value) { + char buffer[400]; + const auto written = std::to_chars(buffer, buffer + sizeof(buffer), value, std::chars_format::fixed); + if (written.ec != std::errc{}) throw Error(23, "cannot write a volume"); + return std::string(buffer, written.ptr); +} + +void ChainBars::quantity_grid(double step) { + if (predecessor_) throw Error(23, "the quantity grid is set before the chain's predecessor"); + if (!std::isfinite(step) || step <= 0) throw Error(23, "--qty-step must be a positive number"); + step_ = step; +} void ChainBars::seed(const std::string& close) { if (predecessor_) throw Error(23, "the warmup close seeds the chain before its predecessor"); (void)Decimal(close); @@ -264,20 +344,20 @@ bool ChainBars::add(const Trade& trade) { if (!count_) { open_ = high_ = low_ = trade.price; high_value_ = low_value_ = price; - volume_ = Decimal("0"); + volume_.reset(step_); } else { if (price.compare(high_value_) > 0) { high_ = trade.price; high_value_ = price; } if (price.compare(low_value_) < 0) { low_ = trade.price; low_value_ = price; } } close_ = trade.price; - volume_ = volume_.add(Decimal(trade.qty)); + volume_.add(runner_double(trade.qty)); ++count_; ++prints_; return false; } void ChainBars::flush() { if (!count_) return; - row(minute_, open_, high_, low_, close_, volume_.str()); + row(minute_, open_, high_, low_, close_, shortest_decimal(volume_.value())); count_ = 0; next_ = minute_ + 60000; } @@ -504,9 +584,11 @@ std::string warmup_close(const std::string& warmup, std::int64_t start) { return fields[4]; } -ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end, const std::string& seed) { +ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end, const std::string& seed, + const std::optional& step) { ChainBars chain(start, end); if (!seed.empty()) chain.seed(seed); + if (step) chain.quantity_grid(*step); std::optional candidate; std::int64_t last_row = -1; std::exception_ptr failure; @@ -578,9 +660,11 @@ ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int6 } namespace { -ChainBars rest_bars(const Config& config, std::string& multiplier, const std::string& seed) { +ChainBars rest_bars(const Config& config, std::string& multiplier, const std::string& seed, + const std::optional& step) { ChainBars chain(config.start, config.end); if (!seed.empty()) chain.seed(seed); + if (step) chain.quantity_grid(*step); const auto now = now_ms(); if (config.end > now) throw Error(20, "the window has not closed yet: no print at or after its end can exist"); const auto venue = make_venue(config); @@ -621,6 +705,13 @@ void export_bars(const Config& config, const ExportOptions& options) { Json source; std::optional chain; const auto seed = options.warmup.empty() ? std::string() : warmup_close(options.warmup, config.start); + std::optional step; + if (!options.qty_step.empty()) { + step = runner_double(options.qty_step); + if (*step <= 0) throw Error(23, "--qty-step must be a positive number"); + } + RunnerVolume grid; + grid.reset(step); if (!options.archive.empty()) { if (config.venue != "binance" || config.market != "usdm" || config.mode != "agg-ticks") throw Error(23, "--archive reads the Binance USD-M daily aggTrades archive: use --venue binance --market usdm --mode agg-ticks"); @@ -630,11 +721,11 @@ void export_bars(const Config& config, const ExportOptions& options) { if (verified) verify_checksum(options.checksum, options.archive, digest); else log("warn", "archive_not_checksum_verified", file); if (base_name(options.archive).rfind(config.symbol + "-aggTrades-", 0) != 0) log("warn", "archive_name_not_symbol", file); - chain = archive_bars(options.archive, config.start, config.end, seed); + chain = archive_bars(options.archive, config.start, config.end, seed, step); source = Json::object({{"kind", Json::string("archive")}, {"file", Json::string(base_name(options.archive))}, {"sha256", Json::string(digest)}, {"checksum_verified", Json::boolean(verified)}}); } else { - chain = rest_bars(config, multiplier, seed); + chain = rest_bars(config, multiplier, seed, step); source = Json::object({{"kind", Json::string("rest")}, {"origin", Json::string(config.rest_url)}}); } const auto& csv = chain->csv(); @@ -650,6 +741,8 @@ void export_bars(const Config& config, const ExportOptions& options) { {"price", Json::string(predecessor.price)}})}, {"fence", Json::object({{"id", number(fence.id)}, {"ts", signed_number(fence.ts)}})}, {"warmup_close", seed.empty() ? Json{} : Json::string(seed)}, + {"qty_step", step ? Json::string(options.qty_step) : Json{}}, + {"volume_rule", Json::string(grid.exact() ? "exact-decimal-grid" : "compensated-binary64")}, {"first_id", any ? number(predecessor.id + 1) : Json{}}, {"last_id", any ? number(fence.id - 1) : Json{}}, {"prints", number(chain->prints())}, {"bars", number(chain->bars())}, {"quiet_minutes", number(chain->quiet_minutes())}, {"rule", Json::string(bar_rule)}, {"sha256", Json::string(sha256(csv))}}); diff --git a/native/feed/src/export.hpp b/native/feed/src/export.hpp index e56082b..ca6d13d 100644 --- a/native/feed/src/export.hpp +++ b/native/feed/src/export.hpp @@ -10,7 +10,29 @@ struct ExportOptions { std::string archive; // optional: a venue daily aggregate-trade archive (.zip or extracted .csv) std::string checksum; // optional: the archive's published .CHECKSUM file std::string warmup; // optional: the runner's warmup CSV, ending at the minute before the start + std::string qty_step; // optional: the runner's syminfo qty_step, as given to it (--syminfo qty_step=...) }; +// The runner's tick-built bar volume, operation for operation as the engine's NativeExecutionConsumer::TickVolume +// (engine #319): with a quantity grid that is a decimal step 1/10^k, the exact sum of grid units, correctly rounded +// to binary64; otherwise (no step, another step, a quantity off the grid, or overflow) the compensated (Neumaier) +// binary64 sum in print order. Quantities are the doubles the runner reads (runner_double). +class RunnerVolume { + std::int64_t units_ = 0; + double sum_ = 0.0, compensation_ = 0.0, divisor_ = 1.0; + int places_ = 0; + bool exact_ = false; +public: + // Grid units as binary64: units / 10^k, rounded once to nearest even. + double decimal(std::int64_t units) const; + void reset(const std::optional& step); + void add(double quantity); + double value() const; + bool exact() const { return exact_; } +}; +// A JSON number token as the runner converts it (std::stod); 23 unless the whole token is a finite number. +double runner_double(const std::string& token); +// The shortest fixed-notation decimal that reads back as the same binary64. +std::string shortest_decimal(double value); // Prints-built one-minute bars over [config.start, config.end), by the runner's tick-built bar rule, from venue // REST within its retention or from a local daily archive. void export_bars(const Config& config, const ExportOptions& options); @@ -26,7 +48,9 @@ class ChainBars { std::uint64_t last_id_ = 0, count_ = 0, prints_ = 0, bars_ = 0, quiet_ = 0; std::optional predecessor_, fence_; std::string open_, high_, low_, close_, csv_, seed_; - Decimal high_value_{"0"}, low_value_{"0"}, volume_{"0"}; + Decimal high_value_{"0"}, low_value_{"0"}; + RunnerVolume volume_; + std::optional step_; void flush(); void carry(std::int64_t until); void row(std::int64_t minute, const std::string& open, const std::string& high, const std::string& low, @@ -35,6 +59,8 @@ class ChainBars { ChainBars(std::int64_t start, std::int64_t end); // The close a quiet first minute carries: the last warmup bar's close (before anchor()). void seed(const std::string& close); + // The runner's quantity grid (its qty_step), which decides its volume rule (before anchor()). + void quantity_grid(double step); void anchor(const Trade& predecessor); bool add(const Trade& trade); bool anchored() const { return predecessor_.has_value(); } @@ -67,7 +93,8 @@ std::string file_sha256(const std::string& path); void verify_checksum(const std::string& checksum, const std::string& archive, const std::string& digest); // The window's chain from a daily aggregate-trade archive (.zip, else an extracted .csv), streamed: only the // predecessor candidate, the window's bars and the fence are kept. -ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end, const std::string& seed = {}); +ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end, const std::string& seed = {}, + const std::optional& step = std::nullopt); // The close of the warmup CSV's last row, which must be the minute before start (20 otherwise; 23 if malformed). std::string warmup_close(const std::string& warmup, std::int64_t start); } diff --git a/native/feed/src/main.cpp b/native/feed/src/main.cpp index 7f58e91..72c7e0c 100644 --- a/native/feed/src/main.cpp +++ b/native/feed/src/main.cpp @@ -30,7 +30,8 @@ struct StdoutFlags { void help() { std::cout << "pineforge-feed warmup --venue VENUE --market MARKET --symbol SYMBOL --start UTC|MS --end UTC|MS --output FILE\n" "pineforge-feed export --venue VENUE --market MARKET --symbol SYMBOL --mode ticks|agg-ticks --start UTC|MS --end UTC|MS --output FILE\n" - " [--warmup RUNNER-WARMUP.csv] [--archive DAILY.zip|DAILY.csv [--checksum DAILY.zip.CHECKSUM]] (prints-built 1m bars)\n" + " [--warmup RUNNER-WARMUP.csv] [--qty-step RUNNER-QTY-STEP] [--archive DAILY.zip|DAILY.csv [--checksum DAILY.zip.CHECKSUM]]\n" + " (prints-built 1m bars; volume by the runner's rule for its qty_step, none by default)\n" "pineforge-feed run --venue VENUE --market MARKET --symbol SYMBOL --mode MODE --state-dir DIR [--start UTC|MS | --resume] [--output-from INDEX]\n" "pineforge-feed serve --venue VENUE --market MARKET --symbol SYMBOL --mode MODE --state-dir DIR [--start UTC|MS | --resume]\n" " [--listen HOST:PORT|[IPV6]:PORT (default 127.0.0.1:8787)] [--allow-remote-listen] [--client-queue-bytes N] [--max-clients N]\n" @@ -123,6 +124,7 @@ int run(int argc, char** argv) { else if (key == "--archive") exported.archive = value; else if (key == "--checksum") exported.checksum = value; else if (key == "--warmup") exported.warmup = value; + else if (key == "--qty-step") exported.qty_step = value; else if (key == "--output-from") config.output_from = unsigned_value(value); else if (key == "--max-messages") config.max_messages = unsigned_value(value); else if (key == "--segment-bytes") config.segment_bytes = unsigned_value(value); @@ -145,8 +147,9 @@ int run(int argc, char** argv) { else throw Error(23, "unknown CLI option: " + key); } gate(config, command == "warmup", seen.count("--rest-url") != 0, seen.count("--ws-url") != 0); - if (command != "export" && (!exported.archive.empty() || !exported.checksum.empty() || !exported.warmup.empty())) - throw Error(23, "--archive, --checksum and --warmup are export options"); + if (command != "export" && + (!exported.archive.empty() || !exported.checksum.empty() || !exported.warmup.empty() || !exported.qty_step.empty())) + throw Error(23, "--archive, --checksum, --warmup and --qty-step are export options"); if (!config.max_queue_bytes || !config.max_replay_seconds || config.max_replay_seconds > 3600 || !config.reconnect_seconds || config.reconnect_seconds > 86100) throw Error(23, "budgets must be positive; reconnect must precede the 24-hour connection limit"); diff --git a/native/feed/tests/batch_probe.cpp b/native/feed/tests/batch_probe.cpp index 6dfee52..03b6b4e 100644 --- a/native/feed/tests/batch_probe.cpp +++ b/native/feed/tests/batch_probe.cpp @@ -16,7 +16,8 @@ template Function load(void* library, const char* name) { int main(int argc, char** argv) { try { - if (argc != 4) throw std::runtime_error("usage: batch-probe observed-strategy.so bars.csv actions.jsonl"); + if (argc != 4 && argc != 5) + throw std::runtime_error("usage: batch-probe observed-strategy.so bars.csv actions.jsonl [qty_step]"); std::ifstream input(argv[2]); if (!input) throw std::runtime_error("cannot open batch input"); std::string line; @@ -45,6 +46,9 @@ int main(int argc, char** argv) { auto error = load(library, "strategy_get_last_error"); auto strategy = create(nullptr); if (!strategy || retain(strategy)) throw std::runtime_error("cannot retain batch execution receipts"); + // The runner's --syminfo qty_step=V, read the same way (std::stod). + if (argc == 5) load(library, "strategy_set_syminfo_metadata")( + strategy, "qty_step", std::stod(argv[4])); pf_report_t report{}; run(strategy, bars.data(), static_cast(bars.size()), "1", "1", 0, 4, PF_MAGNIFIER_ENDPOINTS, &report); const auto* failure = error(strategy); diff --git a/native/feed/tests/export_unit.cpp b/native/feed/tests/export_unit.cpp index 65eaabc..b2357e0 100644 --- a/native/feed/tests/export_unit.cpp +++ b/native/feed/tests/export_unit.cpp @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -264,10 +266,58 @@ int main() { chain.add({2, 60000, "5", "0.1"}); chain.add({3, 119999, "5", "0.2"}); assert(chain.add({4, 120000, "5", "1"})); - assert(chain.csv() == bar_header + "60000,5,5,5,5,0.3\n"); + // Without a qty_step the runner's volume is the compensated binary64 sum, written shortest. + assert(chain.csv() == bar_header + "60000,5,5,5,5,0.30000000000000004\n"); + ChainBars grid(60000, 120000); + grid.quantity_grid(0.1); + grid.anchor({1, 0, "5", "1"}); + grid.add({2, 60000, "5", "0.1"}); + grid.add({3, 119999, "5", "0.2"}); + assert(grid.add({4, 120000, "5", "1"})); + assert(grid.csv() == bar_header + "60000,5,5,5,5,0.3\n"); + ChainBars late(60000, 120000); + late.anchor({1, 0, "5", "1"}); + expect(23, [&] { late.quantity_grid(0.1); }); + ChainBars zero(60000, 120000); + expect(23, [&] { zero.quantity_grid(0); }); } { - // Price tokens stay verbatim (an equal value keeps the first lexeme); volume is the exact canonical sum. + // The engine's tick-volume vectors (#319, tests/test_native_tick_volume.cpp). + const auto volume = [](std::optional step, std::initializer_list quantities) { + RunnerVolume sum; + sum.reset(step); + for (const double quantity : quantities) sum.add(quantity); + return std::make_pair(sum.value(), sum.exact()); + }; + assert(volume(0.1, {0.1, 0.2}) == std::make_pair(0.3, true)); + assert(volume(0.1, {0.1, 0.2, 0.05}) == std::make_pair(std::strtod("0.35000000000000003", nullptr), false)); + assert(volume(0.1, {std::nextafter(0.3, 1.0)}) == std::make_pair(std::nextafter(0.3, 1.0), false)); + assert(volume(0.001, {4e15, 4e15, 4e15, 1, 1}) == std::make_pair(12000000000000002.0, false)); + assert(volume(1.0, {std::ldexp(1.0, 63), 1024, 1024}) == std::make_pair(std::ldexp(1.0, 63) + 2048, false)); + assert(volume(std::nullopt, {0.1, 0.2, 0.05}).first == std::strtod("0.35000000000000003", nullptr)); + assert(volume(0.0004, {0.1, 0.2}) == std::make_pair(0.30000000000000004, false)); + std::uint64_t seed = 84729531; + double divisor = 1.0; + for (int places = 0; places <= 22; ++places, divisor *= 10.0) { + RunnerVolume grid; + grid.reset(1.0 / divisor); + assert(grid.exact()); + for (int sample = 0; sample < 128; ++sample) { + seed = seed * 6364136223846793005ULL + 1442695040888963407ULL; + const auto units = static_cast(seed & 0x7fffffffffffffffULL); + const auto decimal = std::to_string(units) + "e-" + std::to_string(places); + assert(grid.decimal(units) == std::strtod(decimal.c_str(), nullptr)); + } + } + assert(runner_double("0.003") == 0.003 && runner_double("84797.50") == 84797.5); + expect(23, [] { runner_double("1x"); }); + expect(23, [] { runner_double(""); }); + assert(shortest_decimal(0.30000000000000004) == "0.30000000000000004" && shortest_decimal(1e-05) == "0.00001" && + shortest_decimal(3.0) == "3" && shortest_decimal(12000000000000002.0) == "12000000000000002"); + passed("runner_volume_matches_the_engine_tick_volume_vectors"); + } + { + // Price tokens stay verbatim (an equal value keeps the first lexeme); volume is the runner's sum. ChainBars chain(60000, 180000); chain.anchor({1, 59999, "1.0", "1"}); chain.add({2, 60000, "2.50", "0.1"}); @@ -277,7 +327,7 @@ int main() { chain.add({6, 60004, "1.25", "0.000000000000000000000000000001"}); for (std::uint64_t id = 7; id < 17; ++id) chain.add({id, 120000 + static_cast(id), "3.10", "0.1"}); assert(chain.add({17, 180000, "3", "1"})); - assert(chain.csv() == bar_header + "60000,2.50,2.50,0.70,1.25,10000000000.300000000000000000000000000001\n" + assert(chain.csv() == bar_header + "60000,2.50,2.50,0.70,1.25,10000000000.3\n" "120000,3.10,3.10,3.10,3.10,1\n"); } { diff --git a/native/feed/tests/mock_export.py b/native/feed/tests/mock_export.py index dc3767c..d518e81 100644 --- a/native/feed/tests/mock_export.py +++ b/native/feed/tests/mock_export.py @@ -4,6 +4,7 @@ import hashlib import importlib import json +import math import pathlib import subprocess import sys @@ -14,6 +15,7 @@ import zipfile from datetime import UTC, datetime from decimal import Decimal, getcontext +from fractions import Fraction # mock_venue reads the binary from sys.argv[1] at import time. sys.argv = [sys.argv[0], sys.argv[1]] @@ -35,7 +37,51 @@ def canonical(value): return text -def expected_bars(prints, start, end, warmup_close=None): +def runner_volume(quantities, step=None): + """The runner's tick-built volume, independently (engine TickVolume): on a 1/10^k step the + exact grid units, correctly rounded; else, or once a quantity is off it, the compensated + binary64 sum in print order. Quantities are JSON number tokens, read as correctly rounded + doubles (the runner's std::stod).""" + divisor, places, exact = 1.0, 0, False + if step is not None: + scale = 1.0 + for k in range(23): + if float(step) == 1.0 / scale: + divisor, places, exact = scale, k, True + break + scale *= 10.0 + + def decimal(units): + return units / divisor if units <= 2**53 else float(Fraction(units, 10**places)) + + units, total, compensation = 0, 0.0, 0.0 + for token in quantities: + quantity = float(token) + following = total + quantity + if math.isfinite(following): + if total >= quantity: + compensation += (total - following) + quantity + else: + compensation += (quantity - following) + total + else: + compensation = 0.0 + total = following + if not exact: + continue + scaled = quantity * divisor + if not math.isfinite(scaled): + exact = False + continue + rounded = math.floor(scaled) + (1 if scaled - math.floor(scaled) >= 0.5 else 0) + if rounded >= 2**63 or decimal(rounded) != quantity or rounded > 2**63 - 1 - units: + exact = False + continue + units += rounded + value = decimal(units) if exact else total + compensation + return canonical(Decimal(repr(value))) + + +def expected_bars(prints, start, end, warmup_close=None, step=None): """The tick-built bar rule, independently: prints are (id, ts, price, qty) in ID order. A quiet first minute carries the runner's last warmup close.""" close = warmup_close @@ -47,9 +93,9 @@ def expected_bars(prints, start, end, warmup_close=None): continue high = max(inside, key=lambda p: Decimal(p[2]))[2] low = min(inside, key=lambda p: Decimal(p[2]))[2] - volume = sum((Decimal(p[3]) for p in inside), Decimal(0)) + volume = runner_volume([p[3] for p in inside], step) close = inside[-1][2] - rows.append(f"{minute},{inside[0][2]},{high},{low},{close},{canonical(volume)}") + rows.append(f"{minute},{inside[0][2]},{high},{low},{close},{volume}") return "\n".join(rows) + "\n" @@ -148,7 +194,8 @@ def test_rest_bars_follow_the_rule_with_quiet_minute_and_carry(self): text = self.output.read_text() self.assertEqual(text, expected_bars(prints, start, start + 240000)) self.assertIn(f"\n{start + 60000},9.90000000,9.90000000,9.90000000,9.90000000,0\n", text) - self.assertIn(f"\n{start},10.10000000,11.20000000,9.90000000,9.90000000,0.6\n", text) + volume = runner_volume(["0.10000000", "0.20000000", "0.30000000"]) + self.assertIn(f"\n{start},10.10000000,11.20000000,9.90000000,9.90000000,{volume}\n", text) manifest = self.manifest() self.assertEqual( {key: manifest[key] for key in ("venue", "market", "symbol", "mode")}, @@ -221,6 +268,45 @@ def test_quiet_first_minute_without_the_warmup_close_stops(self): ) self.assertIsNone(self.manifest()["warmup_close"]) + def test_volume_follows_the_runner_rule_for_its_qty_step(self): + # The engine's #319 vectors: 0.1 + 0.2 + 0.05 on step 0.1 leaves the grid (compensated + # 0.35000000000000003); on step 0.05 it is exact 0.35; without a step it is compensated. + # 0.1 x 10 + 0.3 shows a plain binary64 sum (1.2999999999999998) is not the rule. + start = recent_minute(30) + prints = [(499, start - 1, "10", "1")] + for index, quantity in enumerate(["0.1", "0.2", "0.05"]): + prints.append((500 + index, start + index, "10", quantity)) + for index in range(10): + prints.append((503 + index, start + 60000 + index, "10", "0.1")) + prints.append((513, start + 60010, "10", "0.3")) + prints.append((514, start + 120000, "10", "1")) + server = self.server(prints) + end = start + 120000 + for step, first, second, rule in ( + (None, "0.35000000000000003", "1.3", "compensated-binary64"), + ("0.1", "0.35000000000000003", "1.3", "exact-decimal-grid"), + ("0.05", "0.35000000000000003", "1.3", "compensated-binary64"), + ("0.01", "0.35", "1.3", "exact-decimal-grid"), + ): + with self.subTest(step=step): + extra = ["--qty-step", step] if step else [] + self.export(start, end, server, extra=extra) + rows = self.output.read_text().splitlines() + self.assertEqual(rows[1].split(",")[5], first) + self.assertEqual(rows[2].split(",")[5], second) + self.assertEqual( + self.output.read_text(), expected_bars(prints, start, end, step=step) + ) + manifest = self.manifest() + self.assertEqual((manifest["qty_step"], manifest["volume_rule"]), (step, rule)) + self.output.unlink() + pathlib.Path(str(self.output) + ".manifest.json").unlink() + plain = 0.0 + for quantity in [0.1] * 10 + [0.3]: + plain += quantity + self.assertEqual(plain, 1.2999999999999998) + self.export(start, end, server, extra=["--qty-step", "0"], expected=23) + def test_rest_pages_by_from_id_and_sums_volume_exactly(self): start = recent_minute(40) prints = synthetic(1000, start - 60000, 6, 700, quiet=(2,)) diff --git a/native/feed/tests/public_e2e.py b/native/feed/tests/public_e2e.py index 28ab113..fbaa56b 100644 --- a/native/feed/tests/public_e2e.py +++ b/native/feed/tests/public_e2e.py @@ -7,7 +7,6 @@ import http.server import itertools import json -import math import os import pathlib import signal @@ -62,7 +61,7 @@ class StrategyTick(ctypes.Structure): ] -def runner_bars(strategy, warmup, ledger, symbol): +def runner_bars(strategy, warmup, ledger, symbol, qty_step=None): """The runner's own tick-built bars. Its committed ledger inputs are replayed in order through the observed build of the same strategy; every input must reach the state hash the runner recorded for it, so this is the runner's computation, not a re-implementation. The bar the @@ -79,6 +78,10 @@ def runner_bars(strategy, warmup, ledger, symbol): [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p], ctypes.c_int, ), + "strategy_set_syminfo_metadata": ( + [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_double], + None, + ), "strategy_stream_begin": ( [ ctypes.c_void_p, @@ -128,6 +131,9 @@ def runner_bars(strategy, warmup, ledger, symbol): assert library.strategy_set_syminfo_string(handle, b"tickerid", symbol.encode()) == 0 ticker = symbol.split(":", 1)[-1] assert library.strategy_set_syminfo_string(handle, b"ticker", ticker.encode()) == 0 + if qty_step: + # The runner's --syminfo qty_step=V, read with std::stod. + library.strategy_set_syminfo_metadata(handle, b"qty_step", float(qty_step)) assert library.strategy_stream_begin(handle, array, len(history), b"1", b"1") == 0 library.strategy_stream_order_actions_clear(handle) for index, text, recorded in inputs: @@ -389,6 +395,8 @@ def runner_command(self, mode, name, ledger, source, cursor=0): f"http://127.0.0.1:{self.receiver.server_port}/actions/{name}", "--allow-insecure-http", ] + if getattr(self.options, "qty_step", None): + command += ["--syminfo", f"qty_step={self.options.qty_step}"] if cursor: command += ["--from-input", str(cursor)] return command @@ -634,15 +642,7 @@ def compare_minutes(): ",".join(str(bar[key]) for key in ("ts_open", "o", "h", "l", "c", "v")) + "\n" ) batch_actions = self.directory / (mode + "-batch-actions.jsonl") - subprocess.run( - [ - self.options.batch_probe, - self.options.observed_strategy, - str(combined), - str(batch_actions), - ], - check=True, - ) + subprocess.run(self.batch_command(combined, batch_actions), check=True) expected_actions = [ action_key(record, mode) for record in rows(batch_actions) @@ -703,6 +703,18 @@ def compare_minutes(): "batch_actions_equal": equal, } + def batch_command(self, bars, actions): + """The batch probe over a warmup-format CSV, with the runner's qty_step when it has one.""" + command = [ + self.options.batch_probe, + self.options.observed_strategy, + str(bars), + str(actions), + ] + if getattr(self.options, "qty_step", None): + command.append(self.options.qty_step) + return command + def export_check(self, mode, normalized, actual_actions): """`pineforge-feed export` over the proven window, from venue REST: its prints-built bars must equal the bars the runner built from the same prints (the tape's minutes), value for @@ -729,6 +741,8 @@ def export_check(self, mode, normalized, actual_actions): "--warmup", str(self.directory / "warmup.csv"), ] + if getattr(self.options, "qty_step", None): + command += ["--qty-step", self.options.qty_step] with open(self.directory / (mode + "-export.stderr"), "w") as error: subprocess.run(command, stderr=error, check=True, timeout=600) with open(output) as source: @@ -744,25 +758,24 @@ def export_check(self, mode, normalized, actual_actions): ) for row in (line.split(",") for line in lines[1:]) ] - assert exported == normalized, "exported bars differ from the tape's prints-built bars" - # The runner's own bars, value for value: the batch reads the exported CSV as doubles. The - # export's volume is the exact decimal sum; without a quantity grid (no qty_step) the - # runner sums volume in compensated doubles, which can land one ulp away. - volume_ulp = 0 + prices = ("ts_open", "o", "h", "l", "c") + assert [{key: bar[key] for key in prices} for bar in exported] == [ + {key: bar[key] for key in prices} for bar in normalized + ], "exported prices differ from the tape's prints-built bars" + # The runner's own bars, bit for bit: every field, volume included, must be the double the + # runner built (the batch reads the exported CSV with the same conversion). built = runner_bars( self.options.observed_strategy, self.directory / "warmup.csv", self.directory / (mode + ".sqlite3"), self.venue.label, + getattr(self.options, "qty_step", None), ) for bar in exported: runner = built.get(bar["ts_open"]) assert runner is not None, f"the runner built no bar at {bar['ts_open']}" values = tuple(float(bar[key]) for key in ("o", "h", "l", "c", "v")) - assert values[:4] == runner[:4], f"export {bar} differs from the runner's bar {runner}" - if values[4] != runner[4]: - assert abs(values[4] - runner[4]) <= math.ulp(values[4]), (bar, runner) - volume_ulp += 1 + assert values == runner, f"export {bar} differs from the runner's bar {runner}" manifest = json.loads((self.directory / (mode + "-export.csv.manifest.json")).read_text()) combined = self.directory / (mode + "-export-combined.csv") with open(combined, "w") as target: @@ -770,7 +783,7 @@ def export_check(self, mode, normalized, actual_actions): target.write("\n".join(lines[1:]) + "\n") actions = self.directory / (mode + "-export-batch-actions.jsonl") subprocess.run( - [self.options.batch_probe, self.options.observed_strategy, str(combined), str(actions)], + self.batch_command(combined, actions), check=True, ) expected = [ @@ -782,7 +795,8 @@ def export_check(self, mode, normalized, actual_actions): self.receipt( f"{'PASS' if equal else 'FAIL'} {mode} export prints-built bars equal the runner's " f"tick-built bars (state-hash-proven replay, read via the observer) " - f"minutes={len(exported)} ohlc_exact={len(exported)} volume_one_ulp={volume_ulp} " + f"minutes={len(exported)} bars_bit_exact={len(exported)} " + f"qty_step={manifest['qty_step']} volume_rule={manifest['volume_rule']} " f"prints={manifest['prints']} " f"quiet_minutes={manifest['quiet_minutes']} " f"batch_over_export_actions={len(expected)} runner={len(actual_actions)}" @@ -801,7 +815,7 @@ def kline_batch(self, mode, normalized, actual_actions, kline_differs): output.write(",".join([str(candle[0]), *candle[1]]) + "\n") actions = self.directory / (mode + "-kline-batch-actions.jsonl") subprocess.run( - [self.options.batch_probe, self.options.observed_strategy, str(combined), str(actions)], + self.batch_command(combined, actions), check=True, ) expected = [ @@ -895,6 +909,8 @@ def run(self): "--modes", default="bars,ticks", help="bars, ticks or agg-ticks, comma-separated" ) parser.add_argument("--bar-minutes", type=int, default=46) + # The runner's symbol quantity step (--syminfo qty_step=V); export and the batch use the same. + parser.add_argument("--qty-step") parser.add_argument("--tick-minutes", type=int, default=21) parser.add_argument("--restart-seconds", type=int, default=180) parser.add_argument(