diff --git a/.github/workflows/native-feed.yml b/.github/workflows/native-feed.yml index 1c7aeda..9e41b81 100644 --- a/.github/workflows/native-feed.yml +++ b/.github/workflows/native-feed.yml @@ -55,7 +55,7 @@ jobs: run: | export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_CXX_COMPILER=clang++-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ + -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ -DPINEFORGE_FEED_SANITIZER=address,undefined cmake --build build-feed -j4 ctest --test-dir build-feed --output-on-failure @@ -68,13 +68,13 @@ jobs: - uses: actions/checkout@v7 - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev clang-18 - run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" - - name: Threaded receive, healing and termination + - name: Threaded receive, healing, termination and serve fan-out env: TSAN_OPTIONS: halt_on_error=1 run: | export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_CXX_COMPILER=clang++-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ + -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ -DPINEFORGE_FEED_SANITIZER=thread cmake --build build-feed -j4 ctest --test-dir build-feed --output-on-failure diff --git a/native/feed/CMakeLists.txt b/native/feed/CMakeLists.txt index da33174..48754db 100644 --- a/native/feed/CMakeLists.txt +++ b/native/feed/CMakeLists.txt @@ -46,7 +46,7 @@ if(PINEFORGE_FEED_SANITIZER) endif() add_library(pineforge_feed_core STATIC src/core.cpp src/state.cpp src/transport.cpp src/binance.cpp src/okx.cpp - src/bybit.cpp src/session.cpp) + src/bybit.cpp src/session.cpp src/export.cpp) target_include_directories(pineforge_feed_core PUBLIC src vendor) target_link_libraries(pineforge_feed_core PUBLIC CURL::libcurl OpenSSL::Crypto Threads::Threads) target_compile_options(pineforge_feed_core PRIVATE -Wall -Wextra -Wpedantic -Werror) @@ -55,13 +55,53 @@ target_link_libraries(pineforge-feed PRIVATE pineforge_feed_core) target_compile_options(pineforge-feed PRIVATE -Wall -Wextra -Wpedantic -Werror) install(TARGETS pineforge-feed RUNTIME DESTINATION bin) +# serve: the only code path that links an HTTP/WebSocket server. CivetWeb 1.16 (MIT) is fetched by release +# archive and pinned by SHA-256; only its core is compiled: no TLS, CGI, Lua, Duktape, WebDAV, SSI or files. +option(PINEFORGE_FEED_SERVE "Build the serve subcommand with the pinned CivetWeb release" ON) +if(PINEFORGE_FEED_SERVE) + enable_language(C) + include(FetchContent) + set(PINEFORGE_FEED_CIVETWEB_ARCHIVE "https://github.com/civetweb/civetweb/archive/refs/tags/v1.16.tar.gz" + CACHE STRING "CivetWeb 1.16 release archive, a URL or a local path; its SHA-256 is pinned") + FetchContent_Declare(civetweb URL "${PINEFORGE_FEED_CIVETWEB_ARCHIVE}" + URL_HASH SHA256=f0e471c1bf4e7804a6cfb41ea9d13e7d623b2bcc7bc1e2a4dd54951a24d60285 + DOWNLOAD_EXTRACT_TIMESTAMP TRUE SOURCE_SUBDIR do-not-add-civetweb-cmake) + FetchContent_MakeAvailable(civetweb) + add_library(pineforge_feed_civetweb STATIC "${civetweb_SOURCE_DIR}/src/civetweb.c") + target_include_directories(pineforge_feed_civetweb SYSTEM PUBLIC "${civetweb_SOURCE_DIR}/include") + target_compile_definitions(pineforge_feed_civetweb PRIVATE NO_SSL NO_CGI NO_FILES NO_CACHING STOP_FLAG_NEEDS_LOCK + USE_IPV6 REENTRANT_TIME PUBLIC USE_WEBSOCKET) + set_target_properties(pineforge_feed_civetweb PROPERTIES C_STANDARD 99) + target_compile_options(pineforge_feed_civetweb PRIVATE -w) + target_link_libraries(pineforge_feed_civetweb PUBLIC Threads::Threads ${CMAKE_DL_LIBS}) + add_library(pineforge_feed_serve STATIC src/serve.cpp) + target_link_libraries(pineforge_feed_serve PUBLIC pineforge_feed_core PRIVATE pineforge_feed_civetweb) + target_compile_options(pineforge_feed_serve PRIVATE -Wall -Wextra -Wpedantic -Werror) + target_link_libraries(pineforge-feed PRIVATE pineforge_feed_serve) + target_compile_definitions(pineforge-feed PRIVATE PINEFORGE_FEED_SERVE=1) +endif() + include(CTest) if(BUILD_TESTING) add_executable(feed-unit tests/unit.cpp) target_link_libraries(feed-unit PRIVATE pineforge_feed_core) + if(PINEFORGE_FEED_SERVE) + target_link_libraries(feed-unit PRIVATE pineforge_feed_serve) + target_compile_definitions(feed-unit PRIVATE PINEFORGE_FEED_SERVE=1) + endif() target_compile_options(feed-unit PRIVATE -Wall -Wextra -Wpedantic -Werror -UNDEBUG) add_test(NAME feed-unit COMMAND feed-unit) + add_executable(feed-export-unit tests/export_unit.cpp) + target_link_libraries(feed-export-unit PRIVATE pineforge_feed_core) + target_compile_options(feed-export-unit PRIVATE -Wall -Wextra -Wpedantic -Werror -UNDEBUG) + add_test(NAME feed-export-unit COMMAND feed-export-unit) find_package(Python3 REQUIRED COMPONENTS Interpreter) + add_test(NAME feed-mock-export COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/mock_export.py $) + set_tests_properties(feed-mock-export PROPERTIES TIMEOUT 300) add_test(NAME feed-mock COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/mock_venue.py $) set_tests_properties(feed-mock PROPERTIES TIMEOUT 420) + if(PINEFORGE_FEED_SANITIZER STREQUAL "thread") + set_tests_properties(feed-unit feed-export-unit feed-mock feed-mock-export PROPERTIES ENVIRONMENT + "TSAN_OPTIONS=halt_on_error=1 suppressions=${CMAKE_CURRENT_SOURCE_DIR}/tests/tsan.supp") + endif() endif() diff --git a/native/feed/NOTICE b/native/feed/NOTICE index 362639a..c7513bc 100644 --- a/native/feed/NOTICE +++ b/native/feed/NOTICE @@ -18,6 +18,17 @@ Linked dependencies retain their upstream terms: OpenSSL 3.x: Apache License, Version 2.0, https://openssl.org Preserve upstream licenses/notices when distributing bundled libraries. +The serve subcommand (build option PINEFORGE_FEED_SERVE, on by default) compiles CivetWeb: + CivetWeb 1.16, MIT License. Copyright (c) 2013-2021 The CivetWeb developers; + Copyright (c) 2004-2013 Sergey Lyubka; Copyright (c) 2013 No Face Press, LLC (Thomas Davis); + Copyright (c) 2013 F-Secure Corporation. https://github.com/civetweb/civetweb + Fetched at configure time from the release archive + https://github.com/civetweb/civetweb/archive/refs/tags/v1.16.tar.gz + Archive SHA-256: f0e471c1bf4e7804a6cfb41ea9d13e7d623b2bcc7bc1e2a4dd54951a24d60285 + Only src/civetweb.c is compiled, with NO_SSL, NO_CGI, NO_FILES and NO_CACHING; Lua and Duktape + are not enabled. Its MIT license text (LICENSE.md in the archive) must accompany binary + distributions that include serve. + CI builds libcurl 8.14.1 from curl-8.14.1.tar.xz. Archive SHA-256: f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd Source: https://curl.se/download/curl-8.14.1.tar.xz diff --git a/native/feed/README.md b/native/feed/README.md index 80ea19a..65242dd 100644 --- a/native/feed/README.md +++ b/native/feed/README.md @@ -3,8 +3,9 @@ An independent Apache-2.0 C++17 public-market-data adapter. It produces the native `pineforge-live` runner's JSONL protocol; it does not load strategies, place orders, access accounts, consume fills, or accept API keys. It is not part -of the Python package or wheel. It has `warmup` and `run`; there is no `serve` -subcommand yet. +of the Python package or wheel. It has `warmup` (venue candles), `export` +(bars built from prints), `run` (JSONL on stdout) and `serve` (the same durable +stream to many runners over a local WebSocket). ## Venues and modes @@ -33,7 +34,13 @@ Requirements: CMake >= 3.20, a C++17 compiler, OpenSSL Crypto, threads, and feature check, and every running binary checks the actually loaded libcurl. Cross-compilation without an executable feature check is not qualified. `vendor/json.hpp` is SHA-256 pinned; attribution and dependency pins are in -`NOTICE`. No engine linkage or additional C++ dependency is required. +`NOTICE`. No engine linkage is required. `serve` compiles CivetWeb 1.16 (MIT), +fetched at configure time from its release archive and pinned by SHA-256; only +the serve code path links it, and `-DPINEFORGE_FEED_SERVE=OFF` builds a binary +without it (whose `serve` stops with 23). Only CivetWeb's HTTP/WebSocket core is +compiled: no TLS, CGI, Lua, Duktape, WebDAV, SSI or file serving. For an offline +build, point `-DPINEFORGE_FEED_CIVETWEB_ARCHIVE=/path/to/civetweb-1.16.tar.gz` at +a local copy of the same archive; its hash is still checked. ```sh cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ @@ -68,7 +75,9 @@ build-feed/pineforge-feed run --venue binance --market usdm --symbol BTCUSDT \ ``` Each venue's warmup takes the same flags and writes its own venue's candles: -use a warmup from the venue and market that the live feed reads. +use a warmup from the venue and market that the live feed reads. A tick-mode +deployment (raw ticks or `agg-ticks`) builds its bars from prints, so its +matching backtest history comes from `export` (below), not from candles. Times are minute-aligned Unix milliseconds or strict UTC RFC3339 seconds. Warmup fetches contiguous REST 1m klines only behind an observed `x=true` WS @@ -79,6 +88,81 @@ cut, confirmation watermark, row count, quantity multiplier, and the CSV SHA-256. It refuses to overwrite either output. Keep both files together; an interrupted two-file publication without its manifest is not a qualified warmup. +## Prints-built history: `export` + +```sh +build-feed/pineforge-feed export --venue binance --market usdm --symbol BTCUSDT \ + --mode agg-ticks --start 2026-10-03T00:00:00Z --end 2026-10-03T03:20:00Z --output agg-bars.csv \ + --warmup warmup.csv # add --qty-step V when the runner runs with --syminfo qty_step=V +# or from a local copy of the venue's public daily aggregate-trade archive: +build-feed/pineforge-feed export --venue binance --market usdm --symbol BTCUSDT \ + --mode agg-ticks --start 2026-10-02T00:10:00Z --end 2026-10-02T23:50:00Z --output agg-bars.csv \ + --archive BTCUSDT-aggTrades-2026-10-02.zip --checksum BTCUSDT-aggTrades-2026-10-02.zip.CHECKSUM +``` + +In tick modes the runner builds its 1m bars from the prints, so a backtest that +must equal the forward run needs bars built the same way, not venue candles (see +"Binance USD-M" for how far `agg-ticks` bars and klines can differ). `export` +writes exactly those bars in the warmup CSV format, by the runner's tick-built +bar rule: a print belongs to minute `floor(ts / 60000) * 60000`; open is the first +price, high the maximum, low the minimum, close the last price, and volume the +runner's own tick volume (below); a minute without a print repeats the +previous close with volume `0`, as the runner's carry-forward bar does. Before +the window's first print that close is the runner's last warmup close, not the +last print before `--start`: `--warmup` takes the runner's warmup CSV, whose last +row must be the minute before `--start` (20 otherwise), and the manifest records +it as `warmup_close`. Without `--warmup` a quiet first minute stops with 20 +rather than guess. Price tokens are the venue's own. + +Volume is the double the runner builds, bit for bit, written as the shortest +decimal that reads back as that double. The runner reads each quantity from +the feed's JSON number (`std::stod`) and its rule depends on its symbol's +quantity step (the engine's tick-volume rule): + +- **No quantity step (the default, and the runner's default):** the runner adds + the quantities in print order as a compensated (Neumaier) binary64 sum, so + `0.1 + 0.2 + 0.05` gives `0.35000000000000003`. Use this when the runner runs + without `--syminfo qty_step=...`. +- **`--qty-step V`:** use the same value the runner gets as + `--syminfo qty_step=V`. A decimal step `1/10^k` (`0.001`, `0.1`, `1`) makes the + volume the exact sum of grid units, rounded once to the nearest double; any + other step (for example `0.0004`), a quantity off the grid, or overflow falls + back to the compensated sum, exactly as the runner does. + +The manifest records `qty_step` and `volume_rule` (`exact-decimal-grid` or +`compensated-binary64`). An export made for one setting does not match a runner +deployed with the other. + +Completeness is proven, never assumed: the prints must form a contiguous ID chain +from the last print strictly before `--start` through the fence, the first print +at or after `--end`. A hole, a missing fence (the window has not closed yet), or +a start outside the venue's REST history (USD-M 48 hours) stops with 20 and +writes nothing; time running backwards along the chain stops with 23. From REST +the chain is paged by ID (Binance 1000, OKX 100 per page); a USD-M start is found +by searching hour windows forward from `--start` up to the present (within the +48-hour history), so a quiet first hour is not a stop. + +`--archive` reads the Binance USD-M daily `aggTrades` archive as published (the +`.zip`, inflated in-process and CRC-checked, or the extracted `.csv`, with or +without its header line), streamed rather than loaded. A `transact_time` above +10^14 is microseconds and becomes `floor(us / 1000)` milliseconds. `--checksum` +takes the published ` ` file and must match the archive's +SHA-256 and base name (21 otherwise); without it the export logs +`archive_not_checksum_verified`. The window, its predecessor and its fence must +all lie inside the one archive (else 20, naming the daily file that holds the +missing print). So a window starting at 00:00 UTC, whose predecessor is the +previous day's last print, or one ending at 24:00, whose fence is the next day's +first print, cannot come from the one archive: start or end the window inside +the day, or export it from REST within its retention. Archives are never fetched +automatically. + +An existing output or manifest is refused. The CSV and then +`.manifest.json` are written atomically; the manifest records the source +(REST origin, or archive name, SHA-256 and whether the checksum was verified), +the window, the predecessor and fence prints, the first and last IDs, the print, +bar and quiet-minute counts, the rule, and the CSV's SHA-256. Limits: 100000 +minutes and 64 MiB of CSV (22 beyond). + ## Protocol and proof policy - Bars are `{"type":"bar","bar":{"ts_open":...,"o":...,"h":...,"l":...,"c":...,"v":...}}`. @@ -154,9 +238,12 @@ and `n` count raw trades, not aggregates), so a minute `M` closes with millisecond need not come back in ID order); USD-M takes the first aggregate at or after the start within the next hour and the one before it. Nothing is anchored, and no minute closes, until both sides are visible. A - quiet start waits for the first print (one REST read per closed minute); - once a WebSocket print exists, REST must show it within about 15 seconds, - or the feed stops with 20; + quiet start waits for the first print (each newly closed minute costs one + look-ahead: OKX a time lookup plus an ID page, USD-M one aggregate search + per unsettled hour window, searched hour by hour up to the newest venue time + and at most 48 hours past the start, after which it stops with 20); once a + WebSocket print exists, REST must show it within about 15 seconds, or the + feed stops with 20; 2. the fence: the next print in that chain has a matched time at or after `M + 60000`, so no further print of `M` can exist (matched time never goes backwards along the chain; if it does, the feed stops with 23); @@ -190,8 +277,10 @@ push on it, or any print carrying `count`, stops with 23. The feed sends text `ping` every `--keepalive-seconds` (default 20, at most 25: OKX closes a connection after 30 seconds without traffic) and treats `pong` as control, never data; a `notice` event (service upgrade) reconnects with overlap. Every -connection re-reads the instrument: a changed `ctVal x ctMult` stops with 21, -since quantities already emitted were converted with the old one. +connection re-reads the instrument and re-runs every instrument check: a changed +`ctVal x ctMult` stops with 21, since quantities already emitted were converted +with the old one, and an instrument that is no longer `live` (a suspension) +stops with 23. ### Bybit @@ -201,8 +290,9 @@ forming one; only the confirmed row is a bar. REST klines arrive newest first and a range wider than `limit` keeps only its newest rows, so each request asks for exactly the rows it can hold and the reversed rows must be contiguous. The feed sends `{"op":"ping"}` every `--keepalive-seconds`. Bybit answers HTTP 403 -for its IP rate ban (about ten minutes); the feed stops with 20 there, so a -supervised restart retries after the ban. +for its IP rate ban (about ten minutes) and for a restricted region; the feed +stops with 20 there, so a supervised restart retries; give that restart a backoff +above ten minutes, since a request inside the ban window extends it. ### Binance USD-M @@ -235,7 +325,7 @@ below that minute's kline open: the kline opened at the price of the fill that the aggregate dated in the previous minute, so a backtest on klines fills one tick away. A backtest on the bars built from the same prints equals the runner. The matching backtest for an `agg-ticks` deployment is therefore a backtest on -prints-built 1m bars; a later feed release adds the export of those bars. +prints-built 1m bars, which `export` writes from REST or the daily archive. The subscribed data streams are strict: another event type on them, a malformed trade or kline, or an unknown shape stops with 23 (on every venue). An event outside @@ -245,9 +335,22 @@ reconnects. Curl answers server PING with matching-payload PONG; OKX and Bybit get their text keepalives. Keepalive replies are control messages: they never count as -data, so a silent data stream still reconnects. Reconnect occurs on -`serverShutdown` or an OKX `notice`, transport loss/75 seconds without data, or -at 23h55 (before the 24-hour Binance limit). The reader buffers during REST healing. Every new +data. After `--silence-seconds` (default 75) without data, OKX and Bybit +reconnect; Binance instead sends a `LIST_SUBSCRIPTIONS` probe, and a reply that +lists every stream of the connection proves the connection and its streams +alive, so a quiet symbol no longer pays a reconnect and its REST overlap every +75 seconds (an error reply, a list missing a stream, or no reply within 10 +seconds reconnects; it is never taken for market data). This is safe because +completeness never rests on the connection: every print is chained by ID, every +minute behind a confirmed watermark, and every reconnect re-verifies its overlap. +Reconnect also occurs on `serverShutdown` or an OKX `notice`, transport loss, or +at 23h55 (before the 24-hour Binance limit). The reader buffers during REST +healing; if a long outage keeps the session healing until the reader's bounded +queue (`--max-queue-bytes`) is full, the reader drops its connection instead of +stopping, waits until the queue has drained to a quarter, and reconnects: what +the venue sent meanwhile is healed by ID or behind the watermark from REST after +the reconnect's overlap check, exactly as after a restart. Only a single message +larger than the whole queue still stops with 22. Every new connection re-fetches up to 1000 raw prints and 32 closed-bar proofs, requires exact value/ID/time overlap, then continues the same stream epoch and sequence. A venue's REST row for the minute that just closed can trail its WebSocket @@ -264,12 +367,18 @@ REST routes are allowlisted per venue: Binance spot `exchangeInfo`, `market/kline`, `market/instruments-info` (one request per 100 ms, OKX `50011` and Bybit `10006` rate-limit bodies wait and retry like `429`; OKX `50001`, `50004`, `50013`, `50026` and Bybit `10000`, `10016` mean "try again" and are -retried with backoff like HTTP 5xx, within the same four attempts). Current public weight/raw-request limits are read -from `exchangeInfo`, and requests are spaced to spend at most half of each -published ceiling (about 50% headroom; 20 ms per weight at 6000 per minute, so -500 ms per 1000-print page). The venue's `X-MBX-USED-WEIGHT-1M` header counts -every client on the IP: once it shows half of the 1-minute ceiling, the feed -waits for the next window. `429` honours `Retry-After` as delta-seconds or an +retried with backoff like HTTP 5xx, within the same four attempts). Binance's +public weight and raw-request limits are read from `exchangeInfo`. The one-minute +weight ceiling is spent by the venue's own count: every response's +`X-MBX-USED-WEIGHT-1M` header (which counts every client on the IP) is kept, and +a request whose weight would take the current window past 90% of the ceiling +waits for the next window (`rest_quota_wait`); otherwise requests are not spaced. +The 90% share is per process: each producer sees the count only as of its own +last response, so several producers on one IP (one per symbol) can together cross +the ceiling between their requests. A `429` waits; a `418` ban stops every +producer on that IP with 22. Run one Binance REST client per IP for large +catch-ups, or stagger the producers. +Other published ceilings are paced evenly across their interval. `429` honours `Retry-After` as delta-seconds or an IMF-fixdate HTTP-date; a wait above 24 hours stops with 22 and an unparseable value with 23. A ban/access rejection is not retried as anonymous trading access. TLS verification is always enabled; redirects, URL credentials, and @@ -278,20 +387,53 @@ non-origin overrides are refused. Explicit insecure overrides are ## Durability and restart -One producer owns a state directory via `flock`. `events.jsonl` is append-only. -Messages are group-committed: every event made ready by the source messages -already queued (up to 1024 source messages, or 4096 events) is appended in one -write and fsynced once, then `cursor.json` is atomically replaced once and its -directory fsynced, and only **then** are those lines written to stdout, one -event per line as before. The cursor binds source origins, symbol/mode/units, -random stream epoch, initial/verified cut, venue ID, emitted sequence, message -index, byte count, predecessor, closed-minute proofs, SHA-256 prefix chain, -last-message hash, and fixed message partition. Its canonical document is -itself hashed. Resume verifies the retained committed prefix and fsyncs -truncation of an uncommitted crash tail. - -A pipe write is not a consumer acknowledgment. By default `--resume` replays the -whole verified origin prefix. If the runner has committed N messages: +One producer owns a state directory via `flock`. The normalized log is a +segmented journal under `journal/`: each segment file `.jsonl` holds +consecutive messages and is append-only, and beside it a hashed checkpoint +`.checkpoint.json` records the verified state before its first +message (prefix hash, last-message hash, message index and byte offset, cut, +sequence, last tick time, last bar, predecessor). Messages are group-committed: +every event made ready by the source messages already queued (up to 1024 source +messages, or 4096 events) is appended to the open segment in one write and +fsynced once, then `cursor.json` is atomically replaced once and its directory +fsynced, and only **then** are those lines published (stdout for `run`, clients +for `serve`), one event per line. The cursor binds source origins, +symbol/mode/units, random stream epoch, initial/verified cut, venue ID, emitted +sequence, message index, byte count, the open segment, predecessor, +closed-minute proofs, SHA-256 prefix chain, last-message hash, and fixed message +partition. Its canonical document is itself hashed. + +After a commit leaves the open segment at or above `--segment-bytes` (default +16 MiB), it is sealed: the successor's checkpoint is written atomically, then the +empty successor is created and synced, and only then does the cursor name it. A +crash between those steps leaves files that resume removes, and the old segment +stays open. Sealed segments then expire oldest first while the retained journal +exceeds `--replay-bytes` (default 256 MiB, at least twice `--segment-bytes`) or, +with `--replay-age-seconds`, while a segment's last minute is older than that in +venue time; a segment file goes before its checkpoint, so a crash in between +leaves an orphan checkpoint that resume removes. The open segment and every +segment that can hold a message of the protected window (the open minute and the +32 closed-minute proofs the reconnect overlap re-verifies) never expire, nor, in +tick modes, the segment holding the newest print (a reconnect re-reads recent +prints from the journal, and a quiet next-print-fence stream keeps closing +minutes without any); when they alone exceed the budget the feed keeps them +and logs `replay_budget_held_by_protected_window`. Resume verifies the retained journal +from its oldest checkpoint, checks every segment boundary against its +checkpoint and the end against the cursor, and only then fsyncs truncation of an +uncommitted crash tail. It removes only what a crash can leave (the empty +successor of an interrupted seal, the oldest checkpoint of an interrupted +expiry); any other unexpected segment or checkpoint stops with 21 and is left in +place. Resume reads the whole retained window within `--max-replay-seconds`, so +a very large `--replay-bytes` needs a larger budget there. Every crash point of a commit, a seal and an expiry is +exercised by the test suite (`PINEFORGE_FEED_CRASH_AT`, a test hook that kills +the process at a named step) and by random kills. + +A duplicate print or candle older than the retained journal can no longer be +compared with what was emitted: it is logged as `expired_duplicate_unverified` +and ignored. Inside the retained journal every duplicate is still checked. + +A pipe write is not a consumer acknowledgment. If the runner has committed N +messages: ```sh build-feed/pineforge-feed run --venue binance --market spot --symbol BTCUSDT \ @@ -302,12 +444,17 @@ build-feed/pineforge-feed run --venue binance --market spot --symbol BTCUSDT \ ``` Replace N with the runner's actual committed count; never use the producer's -possibly-ahead count. Bind warmup identity and stream epoch in orchestration. -An unavailable cursor or changed identity is refused. SIGTERM/SIGINT stops -intake, discards staged messages that were never committed or published, and -exits 0. A directory holding only an empty `events.jsonl` (a crash before the -first cursor write) is taken over by a fresh `--start`; other existing state -needs `--resume`. +possibly-ahead count. `--resume` without `--output-from` replays from index 0. +Bind warmup identity and stream epoch in orchestration. An unavailable cursor or +changed identity is refused. A cursor older than the retained journal stops with +22 and says how to recover: a consumer behind the first retained index cannot +resume from this stream; rebuild it from a warmup that ends inside the retained +window, or run with a longer retention. SIGTERM/SIGINT stops intake, discards +staged messages that were never committed or published, and exits 0. A +directory whose journal holds only empty segments and no cursor (a crash before +the first cursor write) is taken over by a fresh `--start`; other existing state +needs `--resume`. A state directory from a release before the segmented journal +(`events.jsonl`) is refused; start a new one. Stdout is nonblocking during `run` for a five-second stop drain, and its original flags are restored on exit (a SIGKILL cannot restore them); a consumer @@ -321,17 +468,18 @@ waits at most 2 seconds for queued records. |---|---| | 0 | Requested clean stop / successful command | | 20 | Unhealable history gap, ambiguous fence, exhausted source retries | -| 21 | Changed overlap, conflicting duplicate, revised bar, changed cursor/prefix | -| 22 | Replay/disk/queue/allocation budget or durable I/O failure | +| 21 | Changed overlap, conflicting duplicate, revised bar, changed cursor/prefix/checkpoint | +| 22 | Expired output cursor, queue/allocation budget or durable I/O failure | | 23 | Unsupported input/source, permanent access, protocol/decimal/dependency failure, matched-time regression | -Default replay disk budget: 256 MiB (`--max-log-bytes`); each prefix traversal: -60 seconds (`--max-replay-seconds`, maximum 3600); source and unconfirmed-print -queues: 16 MiB each (`--max-queue-bytes`); WS text: 1 MiB; REST body: 1 MiB. -Warmup: 100000 rows / 16 MiB. Exhaustion is a cursor-preserving stop, not silent -compaction or dropped events. Raising limits is an explicit operator choice. -`--max-messages` permits bounded qualification. `--reconnect-seconds` can shorten -the 23h55 rotation for synthetic tests, never extend it beyond that limit. +Retained journal: `--segment-bytes` (16 MiB), `--replay-bytes` (256 MiB), +`--replay-age-seconds` (off). Each prefix traversal: 60 seconds +(`--max-replay-seconds`, maximum 3600); source and unconfirmed-print queues: +16 MiB each (`--max-queue-bytes`); WS text: 1 MiB; REST body: 1 MiB. Warmup and +export: 100000 rows / 16 MiB and 64 MiB. Raising limits is an explicit operator +choice. `--max-messages` permits bounded qualification. `--reconnect-seconds` can +shorten the 23h55 rotation for synthetic tests, never extend it beyond that +limit, and `--silence-seconds` shortens the data-silence threshold. ### Throughput @@ -340,36 +488,38 @@ Linux test machine (ext4 root), committing each message separately (three fsyncs per print) sustained about 120 messages/s. With group commit, the loopback bench (synthetic contiguous prints through the real binary and the mock venue, wall time including startup) ran 30,000 prints at 5,811, 11,212 and -10,842 messages/s in three runs on the same machine. The busiest BTCUSDT minute -of a recent 89-day sample averaged 655 prints/s, about one ninth of the slowest -run. +10,842 messages/s in three runs on the same machine under load; with the +segmented journal, on an idle machine, 30,000 prints ran at about 42,000 +messages/s, and through `serve` one client received a 30,000-print burst at about +19,000 messages/s (eight clients at once, about 17,000 each). The busiest BTCUSDT +minute of a recent 89-day sample averaged 655 prints/s, about one ninth of the +slowest run. -### Run-time horizon per mode +### Retention per mode -`--max-log-bytes` is a hard stop (22): this release neither rotates nor -checkpoints the journal. Journal segmentation and a checkpoint (prefix hash, -cut, sequence, current-minute aggregate, the 32 proofs) are planned together -with bounded replay windows. Until then the horizon is the budget divided by -the journal's growth: +The journal no longer has a terminal budget: it keeps the newest +`--replay-bytes` and expires older segments. The retained replay window is the +budget divided by the journal's growth: -| Mode | Bytes per message | Messages | Horizon at the default 256 MiB | +| Mode | Bytes per message | Messages | Window at the default 256 MiB | |---|---|---|---| | bars | about 138 | 1 per minute | about 3.7 years | | ticks, BTCUSDT median day | about 92 | 2.92 M per day | about 24 hours | | ticks, BTCUSDT busiest day | about 92 | 8.93 M per day | about 8 hours | Message sizes come from a public BTCUSDT capture; the daily print counts are -REST kline trade counts over 89 days. - -Raising the budget lengthens every whole-journal scan: at startup, on every -tick-mode reconnect, on a duplicate closed kline and on an old duplicate print, -each bounded by `--max-replay-seconds`. +REST kline trade counts over 89 days. A consumer must resume inside that window. +Startup verifies the whole retained window, and the journal scans for the open +minute, a duplicate closed candle or an old duplicate print start at the +segment that holds the minute or sequence, each bounded by +`--max-replay-seconds`. ### Tick mode needs a liquid symbol -75 seconds without any data message forces a reconnect, and each reconnect -costs an overlap check (on Binance spot a weight-27 `historicalTrades` page and -`klines` call). On Binance spot a minute without a print cannot be fenced, so +On OKX and Bybit 75 seconds without any data message forces a reconnect, and +each reconnect costs an overlap check; Binance proves a quiet connection with a +probe instead (see the reconnect paragraph above). On Binance spot a minute +without a print cannot be fenced, so tick mode stops with 20 there, and again on every resume. OKX closes a quiet minute on the next-print fence and its zero-volume candle, USD-M on the fence alone; the runner then sees a `time` event with no print in that minute. Use bars mode for @@ -385,7 +535,85 @@ verified cursor retained. Run the feed under a supervisor (a systemd unit with backoff, as `--resume --output-from N` with the runner's committed count. Exits 21 and 23 need an operator (changed history, protocol or access failure): an automatic restart repeats the stop. 22 needs a larger budget, more -disk, or a consumer that reads stdout again. +disk, a consumer that reads stdout again, or (an expired cursor) a consumer +rebuilt inside the retained window. + +## Serve: one producer, many runners + +```sh +build-feed/pineforge-feed serve --venue binance --market spot --symbol BTCUSDT \ + --mode bars --state-dir feed-state --start 2026-10-03T03:20:00Z --listen 127.0.0.1:8787 +curl -s http://127.0.0.1:8787/v1/status # epoch, source, retained range, last seq/time, clients +pineforge-live run --strategy strategy.so --warmup warmup.csv --mode bars --script-tf 1 \ + --feed-url 'ws://127.0.0.1:8787/v1/stream?epoch=E&from=0' --allow-insecure-http \ + --ledger runner.sqlite3 --symbol BINANCE:BTCUSDT --webhook-url https://receiver.example/actions +``` + +`serve` is `run` with its stdout replaced by a local HTTP/WebSocket server: one +producer per venue, market, symbol and mode owns the same durable journal (same +proofs, group commit, persist-before-publish, `--resume`, retention) and any +number of runners read it, instead of each opening its own venue connection. +Every WebSocket text message is exactly one journal message, so message +boundaries, and therefore the runner's `--from-input` count, are the journal's. + +- `GET /v1/status`: `epoch`, `source` (venue, market, symbol, mode, origins, + start, quantity multiplier), `retained` (`first_index`, `next_index`), `last` + (sequence, verified cut, last tick time, last bar), `clients` and the limits. +- `GET /v1/snapshot`: the complete committed prefix from index 0 as JSONL, the + only HTTP shape the runner accepts (it starts every snapshot at index 0 and + refuses more than 4 MiB). Once that prefix exceeds 4 MiB the endpoint answers + 413, and once index 0 has expired 410, with the reason in the body: use the + WebSocket stream. +- `ws://HOST:PORT/v1/stream?epoch=E&from=I`: every message from index `I` on, + first from the journal, then live. Cursor negotiation is HTTP, during the + upgrade, never an event in the runner's input: a malformed request answers + 400, a wrong epoch 409, a cursor beyond the committed log 416, an expired + cursor 410, a full server 503, each with a JSON reason. The runner reports any + of them as a failed handshake. + +Resume: bind each runner to the stream's epoch and the index `S` of its first +message (0 when its warmup ends at the producer's start). With N messages in its +ledger, a restarted runner connects with `from=S+N` and `--from-input N` and +receives exactly the remaining messages. A producer restart (`serve --resume`) +keeps the epoch, so runners reconnect the same way; the runner stops when the +producer goes away and is restarted with its own committed count. + +Bounds: each client reads the journal until it reaches the head and then drains +its own queue of released messages, at most `--client-queue-bytes` (default +1 MiB, which must hold one burst: a group commit releases up to 4096 messages at +once); `--max-clients` (default 64). A client whose queue overflows (it reads +slower than the stream, beyond the kernel's socket buffers) is closed with code +1008 and the reason `slow consumer`; a client whose cursor expires while it +catches up is closed with 4410 as soon as it reaches the expired segment. +Neither ever blocks the producer or another client. A client holds a journal +segment open only while it catches up. At start `serve` raises its open-file +limit to the hard limit and refuses to start (23) when that is below +`2 x --max-clients + 64`. A client that stops reading entirely is dropped when its blocked write +times out (10 seconds). A client never sends a message: any frame it sends after +`serve` has closed it ends the connection at once. + +Capacity: the HTTP server runs one worker thread per connection, and each +streaming client holds one for its lifetime. `serve` runs `--max-clients + 16` +workers, so status, snapshot and new runner connects keep spare workers, and it +drops any request not complete within 10 seconds (a half-open connection). +CivetWeb has no separate limit for incomplete requests, and buffers a whole +client frame before `serve` sees it (its own ceiling is 2 GiB), so anything +beyond same-host runners must go through the reverse proxy below, which absorbs +incomplete requests and caps frame sizes. On SIGTERM the producer drains each live client's queue +and closes every client with 1001 (a client still catching up from the journal +is closed at once and resumes from its own count). + +Quiet streams: the runner's WebSocket idle deadline is 15 seconds and libcurl +answers PINGs without telling it, so `serve` sends an unsolicited PONG to a +client after 5 seconds without a message; a bar stream (one message a minute) +stays connected. + +Listening: `--listen` defaults to `127.0.0.1:8787`; `[::1]:PORT` listens on the +IPv6 loopback; port 0 picks a free port, logged as `serve_listening`. **`serve` has no TLS and no authentication.** A +non-loopback address needs `--allow-remote-listen`; expose it only behind a +reverse proxy that terminates TLS and authenticates clients (for example nginx +or Caddy proxying `wss://` to the loopback listener), and never on a public +interface directly. ## Qualification and next adapters @@ -413,7 +641,14 @@ feed dependency. `src/venue.hpp` is the adapter boundary: a venue supplies its WebSocket connection (path, subscription frames, keepalive, frame classifier), decoding, REST history and candles, its tick proof and its print history window. -`src/session.cpp` holds the venue-neutral sessions. Coinbase remains deferred. -`serve` will add approved MIT CivetWeb only to its server target, bounded client -replay/queues, and cursor-bound fan-out; no server library is linked yet. The -engine executable remains `pineforge-live`. +`src/session.cpp` holds the venue-neutral sessions, `src/state.cpp` the +segmented journal, `src/serve.cpp` the fan-out and `src/export.cpp` the +prints-built history. Coinbase remains deferred. The engine executable remains +`pineforge-live`. + +`tests/serve_e2e.py` qualifies `serve` against the public venue and the real +runner: one producer, runners on `--feed-url`, a runner restart with +`--from-input`, a producer SIGKILL and `--resume`, each runner's ledger equal to +the journal, no idle-deadline stop, and the same REST and batch checks. +`public_e2e.py --export` also checks `export` over a tick window: its bars must +equal the bars built from the tape's prints, and the batch over them the runner. diff --git a/native/feed/src/binance.cpp b/native/feed/src/binance.cpp index f9e5f60..6b33e2c 100644 --- a/native/feed/src/binance.cpp +++ b/native/feed/src/binance.cpp @@ -71,9 +71,21 @@ Trade usdm_aggregate(const Json& value) { trade.validate(); return trade; } -Frame binance_frame(const std::string& message) { +Frame binance_frame(const std::string& message, const std::vector& streams) { try { const auto envelope = parse_json(message); + // The reply to the liveness probe: {"result":[subscribed streams],"id":N}. Without them all, reconnect. + if (envelope.find("id") && !envelope.find("stream") && !envelope.find("e")) { + const auto* result = envelope.find("result"); + if (!result || result->kind != Json::Kind::Array || result->items.empty()) return Frame::Stale; + for (const auto& stream : streams) { + bool listed = false; + for (const auto& item : result->items) listed = listed || (item.kind == Json::Kind::String && item.text() == stream); + if (!listed) return Frame::Stale; + } + return Frame::Control; + } + if (envelope.find("result")) throw Error(23, "unexpected public stream reply"); const auto* wrapped = envelope.find("data"); const auto& data = wrapped ? *wrapped : envelope; const auto* event = data.find("e"); @@ -81,9 +93,19 @@ Frame binance_frame(const std::string& message) { } catch (const std::exception&) { throw Error(23, "invalid public WebSocket JSON"); } } +namespace { +// A combined-stream connection whose probe reply must list every one of its streams. +Connection combined(const std::string& prefix, const std::vector& streams) { + std::string path = prefix; + for (const auto& stream : streams) path += (path.size() == prefix.size() ? "" : "/") + stream; + return {path, {}, {}, [streams](const std::string& message) { return binance_frame(message, streams); }, + "{\"method\":\"LIST_SUBSCRIPTIONS\",\"id\":1}"}; +} +} Connection BinanceSpot::connection() const { const auto symbol = lower(config_.symbol); - return {"/stream?streams=" + (config_.mode == "ticks" ? symbol + "@trade/" + symbol + "@kline_1m" : symbol + "@kline_1m"), {}, {}, &binance_frame}; + if (config_.mode == "ticks") return combined("/stream?streams=", {symbol + "@trade", symbol + "@kline_1m"}); + return combined("/stream?streams=", {symbol + "@kline_1m"}); } std::vector BinanceSpot::decode(const std::string& message) const { try { @@ -175,8 +197,8 @@ RestPolicy usdm_rest() { unsigned int usdm_kline_weight(std::size_t limit) { return limit < 100 ? 1 : limit < 500 ? 2 : limit <= 1000 ? 5 : 10; } Connection usdm_connection(const Config& config) { const auto symbol = lower(config.symbol); - return {"/market/stream?streams=" + (config.mode == "agg-ticks" ? symbol + "@aggTrade/" + symbol + "@kline_1m" : symbol + "@kline_1m"), - {}, {}, &binance_frame}; + if (config.mode == "agg-ticks") return combined("/market/stream?streams=", {symbol + "@aggTrade", symbol + "@kline_1m"}); + return combined("/market/stream?streams=", {symbol + "@kline_1m"}); } BinanceUsdm::BinanceUsdm(const Config& config) : config_(config), http_(config, usdm_rest()) { // An unknown symbol is refused before any stream is opened (HTTP 400, code -1121). @@ -199,10 +221,16 @@ std::vector BinanceUsdm::decode(const std::string& message) const { if (wrapped && !(aggregates && type == "aggTrade" && config_.mode == "agg-ticks") && !(klines && type == "kline")) throw Error(23, "unsupported USD-M stream event"); if (data.at("s").text() != config_.symbol) throw Error(23, "public stream symbol changed"); - if (type == "aggTrade") return {{VenueEvent::Kind::Trade, usdm_aggregate(data), {}}}; + if (type == "aggTrade") { + const auto trade = usdm_aggregate(data); + newest_ = std::max(newest_, trade.ts); + return {{VenueEvent::Kind::Trade, trade, {}}}; + } const auto& row = data.at("k"); if (row.at("s").text() != config_.symbol) throw Error(23, "kline symbol changed"); - return {{VenueEvent::Kind::Kline, {}, binance_kline(row, true)}}; + const auto kline = binance_kline(row, true); + newest_ = std::max(newest_, kline.bar.ts + 59999); + return {{VenueEvent::Kind::Kline, {}, kline}}; } if (type == "serverShutdown") return {}; log("warn", "unknown_stream_event", Json::object({{"stream", Json::string(stream)}, {"type", Json::string(type)}})); @@ -224,31 +252,38 @@ std::vector BinanceUsdm::history(std::uint64_t from, std::size_t limit) { } return trades; } -namespace { -// The first aggregate at or after the start, searched over the next hour (the venue's longest time window), -// so a quiet start minute still has a fence; 0 when none is visible yet. -std::uint64_t first_aggregate(HttpClient& http, const std::string& symbol, std::int64_t minute) { - const auto response = http.get("/fapi/v1/aggTrades?symbol=" + symbol + "&startTime=" + std::to_string(minute) + - "&endTime=" + std::to_string(minute + 3599999) + "&limit=1", 20); - const auto& rows = array(response, 1); - if (rows.empty()) return 0; - const auto first = usdm_aggregate(rows.front()); - if (first.ts < minute || first.ts >= minute + 3600000) throw Error(20, "initial aggregate fence is unavailable or ambiguous"); - return first.id; -} +// The first aggregate at or after the start, searched hour window by hour window (the venue's longest time +// window) up to the newest venue time seen on the WebSocket, so a quiet start of any length within the 48-hour +// history finds its fence; 0 when none is visible yet. A window that ended two minutes before that venue time +// is settled (REST trails the WebSocket by seconds) and is not read again. +std::uint64_t BinanceUsdm::first_aggregate(std::int64_t minute) { + for (auto window = std::max(minute, quiet_until_);; window += 3600000) { + if (window - minute >= retention_ms()) throw Error(20, "no aggregate within the venue's 48-hour history after the start minute"); + const auto response = http_.get("/fapi/v1/aggTrades?symbol=" + config_.symbol + "&startTime=" + std::to_string(window) + + "&endTime=" + std::to_string(window + 3599999) + "&limit=1", 20); + const auto& rows = array(response, 1); + if (!rows.empty()) { + const auto first = usdm_aggregate(rows.front()); + if (first.ts < window || first.ts >= window + 3600000) throw Error(20, "initial aggregate fence is unavailable or ambiguous"); + return first.id; + } + if (window + 3600000 + 120000 <= newest_) quiet_until_ = window + 3600000; + if (window + 3600000 > newest_) return 0; + } } std::uint64_t BinanceUsdm::first_trade_id(std::int64_t minute) { for (unsigned int attempt = 0; attempt < 3; ++attempt) { if (attempt) pause_for(std::chrono::milliseconds(500 * attempt)); - if (const auto first = first_aggregate(http_, config_.symbol, minute)) return first; + if (const auto first = first_aggregate(minute)) return first; } throw Error(20, "no aggregate after the start minute is available yet"); } std::optional BinanceUsdm::predecessor_if_ready(std::int64_t minute) { - const auto first = first_aggregate(http_, config_.symbol, minute); + const auto first = first_aggregate(minute); if (!first) return std::nullopt; + if (first <= 1) throw Error(20, "raw predecessor cannot be proven"); const auto previous = history(first - 1, 1); - if (first <= 1 || previous.empty() || previous.front().id != first - 1) throw Error(20, "raw predecessor is unavailable"); + if (previous.empty() || previous.front().id != first - 1) throw Error(20, "raw predecessor is unavailable"); return previous.front(); } std::vector BinanceUsdm::klines(std::int64_t start, std::int64_t end) { diff --git a/native/feed/src/binance.hpp b/native/feed/src/binance.hpp index aed398e..7ecfa32 100644 --- a/native/feed/src/binance.hpp +++ b/native/feed/src/binance.hpp @@ -2,6 +2,7 @@ #pragma once #include "transport.hpp" #include "venue.hpp" +#include namespace pineforge::feed { Trade binance_trade(const Json& trade, bool websocket); @@ -9,7 +10,9 @@ Kline binance_kline(const Json& row, bool websocket); // One USD-M aggregate print: `a` is the sequence, `q` the quantity. Its raw `f..l` range is validated, // never expanded into imaginary raw trades. Trade usdm_aggregate(const Json& row); -Frame binance_frame(const std::string& message); +// A probe reply (a top-level `id`) proves the connection alive only if its `result` lists every stream in +// `streams`; an error or a partial list is a lost subscription (Stale), never market data. +Frame binance_frame(const std::string& message, const std::vector& streams = {}); Connection usdm_connection(const Config& config); unsigned int usdm_kline_weight(std::size_t limit); @@ -22,6 +25,8 @@ class BinanceSpot final : public Venue { std::vector decode(const std::string& message) const override; std::vector history(std::uint64_t from, std::size_t limit = 1000) override; std::uint64_t first_trade_id(std::int64_t minute) override; + // Binance spot ticks close minutes on the kline ID range, never on a next-print fence. + std::optional predecessor_if_ready(std::int64_t) override { throw Error(23, "Binance spot has no next-print fence"); } std::vector klines(std::int64_t start, std::int64_t end) override; std::string kline_source() const override { return "/api/v3/klines"; } }; @@ -31,6 +36,9 @@ class BinanceSpot final : public Venue { class BinanceUsdm final : public Venue { Config config_; HttpClient http_; + mutable std::int64_t newest_ = -1; // newest venue time decoded from the WebSocket (main thread only) + std::int64_t quiet_until_ = -1; // [start, quiet_until_) is settled and holds no aggregate + std::uint64_t first_aggregate(std::int64_t minute); public: explicit BinanceUsdm(const Config& config); Connection connection() const override { return usdm_connection(config_); } @@ -38,6 +46,7 @@ class BinanceUsdm final : public Venue { std::vector history(std::uint64_t from, std::size_t limit = 1000) override; std::uint64_t first_trade_id(std::int64_t minute) override; std::optional predecessor_if_ready(std::int64_t minute) override; + void horizon(std::int64_t ts) override { newest_ = std::max(newest_, ts); } std::vector klines(std::int64_t start, std::int64_t end) override; TickProof tick_proof() const override { return TickProof::NextPrintFence; } std::int64_t retention_ms() const override { return 48LL * 3600000; } diff --git a/native/feed/src/bybit.cpp b/native/feed/src/bybit.cpp index 4cded5f..8a63e09 100644 --- a/native/feed/src/bybit.cpp +++ b/native/feed/src/bybit.cpp @@ -23,7 +23,7 @@ RestPolicy bybit_rest() { policy.throttled = [](const std::string& body) { return body.find("\"retCode\":10006") != std::string::npos; }; // Bybit answers HTTP 403 for its roughly ten-minute IP rate ban: a restart after the ban recovers. policy.rejected = [](long status, const std::string&) { - if (status == 403) throw Error(20, "Bybit IP rate ban (HTTP 403); restart after the venue's ban window"); + if (status == 403) throw Error(20, "Bybit IP rate ban or regional block (HTTP 403); restart with a backoff above ten minutes"); }; // Server timeout and server error: the venue asks to retry. policy.transient = [](long, const std::string& body) { @@ -106,7 +106,7 @@ Json Bybit::result(const std::string& path) { } Connection Bybit::connection() const { return {"/v5/public/" + category(config_), {"{\"op\":\"subscribe\",\"args\":[\"kline.1." + config_.symbol + "\"]}"}, - "{\"op\":\"ping\"}", &bybit_frame}; + "{\"op\":\"ping\"}", &bybit_frame, {}}; } std::vector Bybit::decode(const std::string& message) const { try { diff --git a/native/feed/src/bybit.hpp b/native/feed/src/bybit.hpp index fbb1be4..8ebf78c 100644 --- a/native/feed/src/bybit.hpp +++ b/native/feed/src/bybit.hpp @@ -22,6 +22,7 @@ class Bybit final : public Venue { Connection connection() const override; std::vector decode(const std::string& message) const override; std::vector history(std::uint64_t, std::size_t) override { throw Error(23, "Bybit has no pageable trade history"); } + std::optional predecessor_if_ready(std::int64_t) override { throw Error(23, "Bybit has no pageable trade history"); } std::vector klines(std::int64_t start, std::int64_t end) override; std::string kline_source() const override { return "/v5/market/kline"; } }; diff --git a/native/feed/src/core.cpp b/native/feed/src/core.cpp index 666df49..7355859 100644 --- a/native/feed/src/core.cpp +++ b/native/feed/src/core.cpp @@ -9,6 +9,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -163,6 +166,19 @@ void output_line(std::string_view line) { } } +void crash_point(const char* name) { + static const std::string target = [] { + const char* value = std::getenv("PINEFORGE_FEED_CRASH_AT"); + return std::string(value ? value : ""); + }(); + if (target.empty()) return; + static std::atomic passes{0}; + const auto colon = target.find(':'); + if (target.compare(0, colon, name) != 0 || std::strlen(name) != (colon == std::string::npos ? target.size() : colon)) return; + const auto wanted = colon == std::string::npos ? 1UL : std::strtoul(target.c_str() + colon + 1, nullptr, 10); + if (++passes == wanted) ::raise(SIGKILL); +} + void atomic_file(const std::string& path, std::string_view bytes) { const auto temporary = path + ".tmp"; const int descriptor = ::open(temporary.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600); diff --git a/native/feed/src/core.hpp b/native/feed/src/core.hpp index c35d229..9d9bf7e 100644 --- a/native/feed/src/core.hpp +++ b/native/feed/src/core.hpp @@ -27,6 +27,9 @@ std::int64_t timestamp(const std::string& token); void write_all(int descriptor, std::string_view bytes); void output_line(std::string_view line); void atomic_file(const std::string& path, std::string_view bytes); +// Test hook for crash-safety probes: PINEFORGE_FEED_CRASH_AT=NAME[:N] kills the process with SIGKILL at the +// N-th (default first) pass through the named durable step. Unset, it does nothing. +void crash_point(const char* name); class Decimal { std::string coefficient_; @@ -89,11 +92,21 @@ struct Config { std::int64_t start = -1, end = -1; bool resume = false, allow_insecure = false; std::uint64_t output_from = 0, max_messages = 0; - std::uint64_t max_log_bytes = 256 * 1024 * 1024; + // Journal segments seal at segment_bytes; sealed segments expire beyond replay_bytes retained bytes or, + // when replay_age_ms is set, beyond that much venue time. + std::uint64_t segment_bytes = 16 * 1024 * 1024; + std::uint64_t replay_bytes = 256 * 1024 * 1024; + std::int64_t replay_age_ms = 0; std::uint64_t max_replay_seconds = 60; std::size_t max_queue_bytes = 16 * 1024 * 1024; std::uint64_t reconnect_seconds = 23 * 3600 + 55 * 60; std::uint64_t keepalive_seconds = 20; + std::uint64_t silence_seconds = 75; // data silence before a probe (or, without one, a reconnect) + // serve: the same producer publishes to WebSocket clients instead of stdout. + bool serve = false, allow_remote_listen = false; + std::string listen = "127.0.0.1:8787"; + std::size_t client_queue_bytes = 1024 * 1024; + std::size_t max_clients = 64; }; // Raw ticks and USD-M aggregate prints share the tick/time journal grammar. inline bool tick_mode(const std::string& mode) { return mode == "ticks" || mode == "agg-ticks"; } diff --git a/native/feed/src/export.cpp b/native/feed/src/export.cpp new file mode 100644 index 0000000..1df61fa --- /dev/null +++ b/native/feed/src/export.cpp @@ -0,0 +1,754 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "export.hpp" +#include "venue.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +namespace { +constexpr std::size_t csv_limit = 64 * 1024 * 1024; +constexpr std::string_view archive_header = "agg_trade_id,price,quantity,first_trade_id,last_trade_id,transact_time,is_buyer_maker"; +constexpr const char* bar_rule = "a print belongs to minute floor(ts/60000)*60000; open=first price, high=max, low=min, close=last, " + "volume=the runner's tick volume (exact on a 1/10^k qty_step grid, else the compensated binary64 sum in print " + "order) as the shortest round-trip decimal; a minute without a print has open=high=low=close=the previous close " + "(the runner's last warmup close before the first print) and volume 0"; +Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } +Json signed_number(std::int64_t value) { return Json::number(std::to_string(value)); } +std::string base_name(const std::string& path) { return std::filesystem::path(path).filename().string(); } +std::int64_t now_ms() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); +} +template Integer digits(std::string_view text) { + Integer result{}; + const auto parsed = std::from_chars(text.data(), text.data() + text.size(), result); + if (text.empty() || text.find_first_not_of("0123456789") != std::string_view::npos || parsed.ec != std::errc{} || + parsed.ptr != text.data() + text.size()) + throw Error(23, "archive row has an invalid unsigned integer"); + return result; +} +std::uint32_t little(const std::string& bytes, std::size_t offset, std::size_t width) { + if (offset + width > bytes.size()) throw Error(21, "zip structure is truncated"); + std::uint32_t value = 0; + for (std::size_t index = width; index > 0; --index) value = (value << 8) | static_cast(bytes[offset + index - 1]); + return value; +} +void read_at(std::ifstream& file, std::uint64_t offset, std::string& bytes) { + file.clear(); + file.seekg(static_cast(offset)); + file.read(bytes.data(), static_cast(bytes.size())); + if (static_cast(file.gcount()) != bytes.size()) throw Error(22, "cannot read the archive"); +} + +// Huffman lookup by the next 15 input bits (deflate sends codes least-significant bit first): symbol << 4 | length, +// 0 for a code the table does not assign. +using Table = std::vector; +void build(Table& table, const unsigned char* lengths, std::size_t count) { + std::array counts{}, next{}; + for (std::size_t symbol = 0; symbol < count; ++symbol) ++counts[lengths[symbol]]; + counts[0] = 0; + int left = 1; + for (unsigned int length = 1; length < 16; ++length) { + left = (left << 1) - static_cast(counts[length]); + if (left < 0) throw Error(21, "deflate Huffman code is over-subscribed"); + } + for (unsigned int length = 1, code = 0; length < 16; ++length) { + code = (code + counts[length - 1]) << 1; + next[length] = code; + } + table.assign(1U << 15, 0); + for (std::size_t symbol = 0; symbol < count; ++symbol) { + const unsigned int length = lengths[symbol]; + if (!length) continue; + unsigned int code = next[length]++, reversed = 0; + for (unsigned int bit = 0; bit < length; ++bit, code >>= 1) reversed = (reversed << 1) | (code & 1); + for (auto index = reversed; index < table.size(); index += 1U << length) + table[index] = (static_cast(symbol) << 4) | length; + } +} + +class Inflater { + const std::function& source_; + const std::function& sink_; + std::vector input_ = std::vector(1 << 16); + std::size_t position_ = 0, available_ = 0; + bool exhausted_ = false; + std::uint64_t bits_ = 0; + unsigned int count_ = 0, padding_ = 0; // padding_: zero bytes appended beyond the end of the input + std::string window_; // output not yet handed on, plus the last 32 KiB for back-references + static constexpr std::size_t flush_at = 1 << 20; + + bool byte(unsigned char& value) { + if (position_ == available_) { + if (exhausted_) return false; + available_ = source_(input_.data(), input_.size()); + position_ = 0; + if (!available_) { exhausted_ = true; return false; } + } + value = static_cast(input_[position_++]); + return true; + } + void need(unsigned int wanted) { + while (count_ < wanted) { + unsigned char value = 0; + if (!byte(value)) ++padding_; + bits_ |= static_cast(value) << count_; + count_ += 8; + } + } + void drop(unsigned int used) { + bits_ >>= used; + count_ -= used; + if (count_ < 8 * padding_) throw Error(21, "deflate stream is truncated"); + } + std::uint32_t bits(unsigned int wanted) { + if (!wanted) return 0; + need(wanted); + const auto value = static_cast(bits_ & ((1ULL << wanted) - 1)); + drop(wanted); + return value; + } + unsigned int decode(const Table& table) { + need(15); + const auto entry = table[bits_ & 0x7FFF]; + if (!(entry & 15)) throw Error(21, "deflate stream has an unassigned Huffman code"); + drop(entry & 15); + return entry >> 4; + } + void emit(std::size_t keep) { + keep = std::min(keep, window_.size()); + if (window_.size() > keep) sink_(std::string_view(window_.data(), window_.size() - keep)); + window_.erase(0, window_.size() - keep); + } + void put(char value) { + window_.push_back(value); + if (window_.size() >= flush_at) emit(32768); + } + void stored() { + drop(count_ % 8); + const auto length = bits(16), complement = bits(16); + if (length != (~complement & 0xFFFF)) throw Error(21, "deflate stored block length is corrupt"); + for (std::uint32_t index = 0; index < length; ++index) put(static_cast(bits(8))); + } + void codes(const Table& literal, const Table& distance) { + static constexpr std::uint16_t base[29] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, + 67, 83, 99, 115, 131, 163, 195, 227, 258}; + static constexpr std::uint8_t extra[29] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0}; + static constexpr std::uint16_t offset[30] = {1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, + 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577}; + static constexpr std::uint8_t spread[30] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, + 11, 11, 12, 12, 13, 13}; + for (;;) { + const auto symbol = decode(literal); + if (symbol < 256) { put(static_cast(symbol)); continue; } + if (symbol == 256) return; + if (symbol > 285) throw Error(21, "deflate stream has an invalid length code"); + const auto length = base[symbol - 257] + bits(extra[symbol - 257]); + const auto code = decode(distance); + if (code > 29) throw Error(21, "deflate stream has an invalid distance code"); + const auto back = offset[code] + bits(spread[code]); + if (back > window_.size()) throw Error(21, "deflate distance reaches before the start of the stream"); + for (std::uint32_t copied = 0; copied < length; ++copied) window_.push_back(window_[window_.size() - back]); + if (window_.size() >= flush_at) emit(32768); + } + } + void dynamic(Table& literal, Table& distance, Table& lengths_code) { + static constexpr unsigned char order[19] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + const auto literals = bits(5) + 257, distances = bits(5) + 1, header = bits(4) + 4; + if (literals > 286 || distances > 30) throw Error(21, "deflate dynamic block header is corrupt"); + unsigned char lengths[320] = {}; + for (unsigned int index = 0; index < header; ++index) lengths[order[index]] = static_cast(bits(3)); + build(lengths_code, lengths, 19); + std::fill(std::begin(lengths), std::end(lengths), 0); + for (unsigned int index = 0; index < literals + distances;) { + const auto symbol = decode(lengths_code); + if (symbol < 16) { lengths[index++] = static_cast(symbol); continue; } + unsigned char value = 0; + unsigned int repeat = 0; + if (symbol == 16) { + if (!index) throw Error(21, "deflate code lengths repeat before the first length"); + value = lengths[index - 1]; + repeat = 3 + bits(2); + } else repeat = symbol == 17 ? 3 + bits(3) : 11 + bits(7); + if (index + repeat > literals + distances) throw Error(21, "deflate code lengths overrun the header counts"); + while (repeat--) lengths[index++] = value; + } + if (!lengths[256]) throw Error(21, "deflate dynamic block has no end-of-block code"); + build(literal, lengths, literals); + build(distance, lengths + literals, distances); + } +public: + Inflater(const std::function& source, const std::function& sink) + : source_(source), sink_(sink) { window_.reserve(flush_at + 512); } + void run() { + static const auto fixed = [] { + unsigned char lengths[288 + 30]; + std::fill(lengths, lengths + 144, 8); + std::fill(lengths + 144, lengths + 256, 9); + std::fill(lengths + 256, lengths + 280, 7); + std::fill(lengths + 280, lengths + 288, 8); + std::fill(lengths + 288, lengths + 318, 5); + std::pair tables; + build(tables.first, lengths, 288); + build(tables.second, lengths + 288, 30); + return tables; + }(); + Table literal, distance, lengths_code; + for (bool last = false; !last;) { + last = bits(1) != 0; + const auto type = bits(2); + if (type == 0) stored(); + else if (type == 1) codes(fixed.first, fixed.second); + else if (type == 2) { + dynamic(literal, distance, lengths_code); + codes(literal, distance); + } else throw Error(21, "deflate stream has an invalid block type"); + } + unsigned char spare = 0; + if (count_ - 8 * padding_ >= 8 || byte(spare)) throw Error(21, "bytes follow the final deflate block"); + emit(0); + } +}; +} + +ChainBars::ChainBars(std::int64_t start, std::int64_t end) : start_(start), end_(end), next_(start), + csv_("timestamp,open,high,low,close,volume\n") { + if (start < 0 || end <= start || start % 60000 || end % 60000 || end > std::numeric_limits::max() - 60000) + throw Error(23, "export requires a nonempty minute-aligned exclusive range"); + if ((end - start) / 60000 > 100000) throw Error(22, "export range exceeds 100000 minutes"); +} +void RunnerVolume::reset(const std::optional& step) { + *this = RunnerVolume{}; + if (!step) return; + double scale = 1.0; + for (int places = 0; places <= 22; ++places, scale *= 10.0) { + if (*step != 1.0 / scale) continue; + divisor_ = scale; + places_ = places; + exact_ = true; + return; + } +} +double RunnerVolume::decimal(std::int64_t units) const { + if (units <= (std::int64_t{1} << 53)) return static_cast(units) / divisor_; + // Above 2^53 the quotient units / 10^places is rounded once, to nearest even, in integer arithmetic. + std::uint64_t numerator = static_cast(units), denominator = 1; + for (int places = 0; places < places_; ++places) denominator *= 5; + int exponent = -places_; + while (denominator <= numerator / 2) { + denominator *= 2; + ++exponent; + } + while (numerator < denominator) { + numerator *= 2; + --exponent; + } + std::uint64_t remainder = numerator - denominator, significand = 1; + for (int bit = 0; bit < 52; ++bit) { + remainder *= 2; + significand *= 2; + if (remainder >= denominator) { + remainder -= denominator; + ++significand; + } + } + const std::uint64_t complement = denominator - remainder; + if (remainder > complement || (remainder == complement && (significand & 1))) ++significand; + return std::ldexp(static_cast(significand), exponent - 52); +} +void RunnerVolume::add(double quantity) { + const double next = sum_ + quantity; + if (std::isfinite(next)) compensation_ += sum_ >= quantity ? (sum_ - next) + quantity : (quantity - next) + sum_; + else compensation_ = 0.0; + sum_ = next; + if (!exact_) return; + const double rounded = std::round(quantity * divisor_); + if (!std::isfinite(rounded) || rounded >= std::ldexp(1.0, 63)) { + exact_ = false; + return; + } + const auto units = static_cast(rounded); + if (decimal(units) != quantity || units > std::numeric_limits::max() - units_) { + exact_ = false; + return; + } + units_ += units; +} +double RunnerVolume::value() const { return exact_ ? decimal(units_) : sum_ + compensation_; } + +double runner_double(const std::string& token) { + std::size_t used = 0; + double value = 0; + try { value = std::stod(token, &used); } catch (const std::exception&) { used = 0; } + if (token.empty() || used != token.size() || !std::isfinite(value)) throw Error(23, "not a finite number: " + token); + return value; +} +std::string shortest_decimal(double value) { + char buffer[400]; + const auto written = std::to_chars(buffer, buffer + sizeof(buffer), value, std::chars_format::fixed); + if (written.ec != std::errc{}) throw Error(23, "cannot write a volume"); + return std::string(buffer, written.ptr); +} + +void ChainBars::quantity_grid(double step) { + if (predecessor_) throw Error(23, "the quantity grid is set before the chain's predecessor"); + if (!std::isfinite(step) || step <= 0) throw Error(23, "--qty-step must be a positive number"); + step_ = step; +} +void ChainBars::seed(const std::string& close) { + if (predecessor_) throw Error(23, "the warmup close seeds the chain before its predecessor"); + (void)Decimal(close); + seed_ = close; +} +void ChainBars::anchor(const Trade& predecessor) { + predecessor.validate(); + if (predecessor_) throw Error(23, "the print chain is already anchored"); + if (predecessor.ts >= start_) throw Error(20, "the predecessor print is not strictly before the start"); + predecessor_ = predecessor; + last_id_ = predecessor.id; + last_ts_ = predecessor.ts; + close_ = seed_; +} +bool ChainBars::add(const Trade& trade) { + if (!predecessor_ || fence_) throw Error(23, "a print chain takes its predecessor first and nothing after its fence"); + trade.validate(); + if (last_id_ == std::numeric_limits::max() || trade.id != last_id_ + 1) + throw Error(20, "the print chain has a hole after ID " + std::to_string(last_id_) + " (next print is " + std::to_string(trade.id) + ")"); + if (trade.ts < last_ts_) throw Error(23, "print time goes backwards along the ID chain at ID " + std::to_string(trade.id)); + if (trade.ts < start_) throw Error(20, "a print before the start follows the predecessor: the predecessor is not the last one"); + last_id_ = trade.id; + last_ts_ = trade.ts; + if (trade.ts >= end_) { + flush(); + carry(end_); + fence_ = trade; + return true; + } + const auto minute = trade.ts - trade.ts % 60000; + if (minute != minute_) { + flush(); + carry(minute); + minute_ = minute; + } + const Decimal price(trade.price); + if (!count_) { + open_ = high_ = low_ = trade.price; + high_value_ = low_value_ = price; + volume_.reset(step_); + } else { + if (price.compare(high_value_) > 0) { high_ = trade.price; high_value_ = price; } + if (price.compare(low_value_) < 0) { low_ = trade.price; low_value_ = price; } + } + close_ = trade.price; + volume_.add(runner_double(trade.qty)); + ++count_; + ++prints_; + return false; +} +void ChainBars::flush() { + if (!count_) return; + row(minute_, open_, high_, low_, close_, shortest_decimal(volume_.value())); + count_ = 0; + next_ = minute_ + 60000; +} +void ChainBars::carry(std::int64_t until) { + if (next_ < until && close_.empty()) + throw Error(20, "the window's first minute has no print: the runner carries its last warmup close there; " + "pass that warmup CSV with --warmup"); + for (; next_ < until; next_ += 60000, ++quiet_) row(next_, close_, close_, close_, close_, "0"); +} +void ChainBars::row(std::int64_t minute, const std::string& open, const std::string& high, const std::string& low, + const std::string& close, const std::string& volume) { + csv_ += std::to_string(minute) + ',' + open + ',' + high + ',' + low + ',' + close + ',' + volume + '\n'; + ++bars_; + if (csv_.size() > csv_limit) throw Error(22, "export output exceeds 64 MiB"); +} +const std::string& ChainBars::csv() const { + if (!fence_) throw Error(20, "the window's fence (the first print at or after its end) is not proven"); + return csv_; +} +const Trade& ChainBars::predecessor() const { + if (!predecessor_) throw Error(20, "the window's predecessor print is not proven"); + return *predecessor_; +} +const Trade& ChainBars::fence() const { + if (!fence_) throw Error(20, "the window's fence (the first print at or after its end) is not proven"); + return *fence_; +} + +std::optional archive_row(std::string_view line, bool first_line) { + if (!line.empty() && line.back() == '\r') line.remove_suffix(1); + if (first_line && !line.empty() && (line.front() < '0' || line.front() > '9')) { + if (line != archive_header) throw Error(23, "unexpected archive header; expected " + std::string(archive_header)); + return std::nullopt; + } + std::array fields; + std::size_t count = 0; + for (std::size_t begin = 0;;) { + const auto comma = line.find(',', begin); + if (count == fields.size()) throw Error(23, "archive row must have seven fields"); + fields[count++] = line.substr(begin, comma == std::string_view::npos ? std::string_view::npos : comma - begin); + if (comma == std::string_view::npos) break; + begin = comma + 1; + } + if (count != fields.size()) throw Error(23, "archive row must have seven fields"); + const auto id = digits(fields[0]), first = digits(fields[3]), last = digits(fields[4]); + auto time = digits(fields[5]); + if (!first || last < first) throw Error(23, "invalid aggregate raw-trade range"); + // Binance moved its archives' transact_time to microseconds; no millisecond time reaches 10^14 before 5138. + if (time > 100000000000000LL) time /= 1000; + if (fields[6] != "true" && fields[6] != "false" && fields[6] != "True" && fields[6] != "False") + throw Error(23, "archive is_buyer_maker must be true or false"); + Trade trade{id, time, std::string(fields[1]), std::string(fields[2])}; + trade.validate(); + return trade; +} + +std::uint32_t crc32(std::uint32_t crc, std::string_view bytes) { + static const auto table = [] { + std::array values{}; + for (std::uint32_t index = 0; index < 256; ++index) { + auto value = index; + for (int bit = 0; bit < 8; ++bit) value = (value & 1) ? 0xEDB88320U ^ (value >> 1) : value >> 1; + values[index] = value; + } + return values; + }(); + crc = ~crc; + for (const auto letter : bytes) crc = table[(crc ^ static_cast(letter)) & 0xFF] ^ (crc >> 8); + return ~crc; +} + +void inflate(const std::function& source, const std::function& sink) { + Inflater(source, sink).run(); +} + +std::string unzip(const std::string& path, const std::function& sink) { + std::ifstream file(path, std::ios::binary); + if (!file) throw Error(23, "cannot open the archive " + path); + file.seekg(0, std::ios::end); + const auto end = file.tellg(); + if (end < 0) throw Error(22, "cannot size the archive"); + const auto size = static_cast(end); + // The end-of-central-directory record: 22 bytes, then a comment of at most 65535 bytes. + std::string tail(static_cast(std::min(size, 22 + 65535)), '\0'); + read_at(file, size - tail.size(), tail); + std::size_t record = std::string::npos; + for (auto index = tail.size() >= 22 ? tail.size() - 21 : 0; index-- > 0;) + if (little(tail, index, 4) == 0x06054b50 && index + 22 + little(tail, index + 20, 2) == tail.size()) { record = index; break; } + if (record == std::string::npos) throw Error(23, "the archive is not a zip file (no end-of-central-directory record)"); + if (little(tail, record + 4, 2) || little(tail, record + 6, 2) || little(tail, record + 8, 2) != 1 || little(tail, record + 10, 2) != 1) + throw Error(23, "the zip archive must be a single-disk archive holding exactly one entry"); + const std::uint64_t directory_size = little(tail, record + 12, 4), directory_offset = little(tail, record + 16, 4); + if (directory_size == 0xFFFFFFFF || directory_offset == 0xFFFFFFFF) + throw Error(23, "ZIP64 archives are not supported; pass the extracted .csv"); + if (directory_offset + directory_size > size - tail.size() + record) throw Error(21, "zip central directory is out of bounds"); + std::string directory(static_cast(directory_size), '\0'); + read_at(file, directory_offset, directory); + if (little(directory, 0, 4) != 0x02014b50) throw Error(21, "zip central directory entry is corrupt"); + const auto flags = little(directory, 8, 2), method = little(directory, 10, 2), crc = little(directory, 16, 4); + const std::uint64_t compressed = little(directory, 20, 4), uncompressed = little(directory, 24, 4), local = little(directory, 42, 4); + const auto name_size = little(directory, 28, 2); + if (46 + name_size + little(directory, 30, 2) + little(directory, 32, 2) != directory.size()) + throw Error(21, "zip central directory entry does not fill the directory"); + const auto name = directory.substr(46, name_size); + if (flags & 1) throw Error(23, "encrypted zip entries are not supported"); + if (method != 0 && method != 8) throw Error(23, "the zip entry must be stored or deflated"); + if (compressed == 0xFFFFFFFF || uncompressed == 0xFFFFFFFF || local == 0xFFFFFFFF) + throw Error(23, "ZIP64 archives are not supported; pass the extracted .csv"); + if (local + 30 > directory_offset) throw Error(21, "zip local header is out of bounds"); + std::string header(30, '\0'); + read_at(file, local, header); + if (little(header, 0, 4) != 0x04034b50 || little(header, 8, 2) != method) throw Error(21, "zip local header is corrupt"); + const auto data = local + 30 + little(header, 26, 2) + little(header, 28, 2); + if (data + compressed > directory_offset) throw Error(21, "zip entry data is out of bounds"); + if (method == 0 && compressed != uncompressed) throw Error(21, "stored zip entry sizes disagree"); + file.clear(); + file.seekg(static_cast(data)); + std::uint64_t remaining = compressed, produced = 0; + std::uint32_t check = 0; + const std::function source = [&](char* buffer, std::size_t limit) -> std::size_t { + const auto wanted = static_cast(std::min(remaining, limit)); + if (!wanted) return 0; + file.read(buffer, static_cast(wanted)); + if (static_cast(file.gcount()) != wanted) throw Error(22, "cannot read the zip entry"); + remaining -= wanted; + return wanted; + }; + const std::function output = [&](std::string_view bytes) { + produced += bytes.size(); + if (produced > uncompressed) throw Error(21, "zip entry inflates beyond its recorded size"); + check = crc32(check, bytes); + sink(bytes); + }; + if (method == 8) inflate(source, output); + else { + std::vector buffer(1 << 20); + while (const auto got = source(buffer.data(), buffer.size())) output(std::string_view(buffer.data(), got)); + } + if (produced != uncompressed || check != crc) throw Error(21, "zip entry CRC-32 or size mismatch: the archive is corrupt"); + return name; +} + +std::string file_sha256(const std::string& path) { + std::ifstream file(path, std::ios::binary); + if (!file) throw Error(23, "cannot open " + path); + struct Digest { + EVP_MD_CTX* context = EVP_MD_CTX_new(); + ~Digest() { EVP_MD_CTX_free(context); } + } digest; + if (!digest.context || EVP_DigestInit_ex(digest.context, EVP_sha256(), nullptr) != 1) throw Error(23, "SHA-256 failed"); + std::vector buffer(1 << 20); + while (file) { + file.read(buffer.data(), static_cast(buffer.size())); + const auto got = static_cast(file.gcount()); + if (got && EVP_DigestUpdate(digest.context, buffer.data(), got) != 1) throw Error(23, "SHA-256 failed"); + } + if (file.bad()) throw Error(22, "cannot read " + path); + unsigned char value[EVP_MAX_MD_SIZE]; + unsigned int size = 0; + if (EVP_DigestFinal_ex(digest.context, value, &size) != 1 || size != 32) throw Error(23, "SHA-256 failed"); + constexpr char hex[] = "0123456789abcdef"; + std::string result; + for (unsigned int index = 0; index < size; ++index) { + result += hex[value[index] >> 4]; + result += hex[value[index] & 15]; + } + return result; +} + +void verify_checksum(const std::string& checksum, const std::string& archive, const std::string& digest) { + std::ifstream file(checksum, std::ios::binary); + if (!file) throw Error(23, "cannot open the checksum file " + checksum); + std::string text(4097, '\0'); + file.read(text.data(), static_cast(text.size())); + text.resize(static_cast(file.gcount())); + if (text.size() > 4096) throw Error(23, "the checksum file exceeds 4096 bytes"); + while (!text.empty() && (text.back() == '\n' || text.back() == '\r')) text.pop_back(); + if (text.size() < 66 || text.find('\n') != std::string::npos || text[64] != ' ' || + text.find_first_not_of("0123456789abcdefABCDEF") != 64) + throw Error(23, "the checksum file must hold one line ` `"); + auto hash = text.substr(0, 64); + std::transform(hash.begin(), hash.end(), hash.begin(), [](unsigned char letter) { return static_cast(std::tolower(letter)); }); + const auto name = text.substr(text[65] == ' ' || text[65] == '*' ? 66 : 65); + if (name != base_name(archive)) throw Error(21, "the checksum file names " + name + ", not the archive " + base_name(archive)); + if (hash != digest) throw Error(21, "the archive's SHA-256 does not match its checksum file"); +} + +namespace { +// The venue's daily file that holds `ts`, named like the given one (-aggTrades-YYYY-MM-DD.zip). +std::string daily_name(const std::string& archive, std::int64_t ts) { + const auto seconds = static_cast(ts / 1000); + std::tm day{}; + char date[16]; + if (!::gmtime_r(&seconds, &day) || !std::strftime(date, sizeof(date), "%Y-%m-%d", &day)) return "the neighbouring daily archive"; + const auto name = base_name(archive); + const std::string marker = "-aggTrades-"; + const auto at = name.find(marker); + if (at == std::string::npos || name.size() < at + marker.size() + 10) return std::string("the daily archive of ") + date; + return name.substr(0, at + marker.size()) + date + name.substr(at + marker.size() + 10); +} +} + +std::string warmup_close(const std::string& warmup, std::int64_t start) { + std::ifstream file(warmup, std::ios::binary); + if (!file) throw Error(23, "cannot open the warmup CSV " + warmup); + std::string line, last; + if (!std::getline(file, line) || line != "timestamp,open,high,low,close,volume") + throw Error(23, "the warmup CSV must start with timestamp,open,high,low,close,volume"); + while (std::getline(file, line)) + if (!line.empty()) last = line; + if (file.bad()) throw Error(22, "cannot read the warmup CSV " + warmup); + std::vector fields; + for (std::size_t from = 0;;) { + const auto comma = last.find(',', from); + fields.push_back(last.substr(from, comma == std::string::npos ? std::string::npos : comma - from)); + if (comma == std::string::npos) break; + from = comma + 1; + } + if (last.empty() || fields.size() != 6) throw Error(23, "the warmup CSV's last row must have six fields"); + if (fields[0] != std::to_string(start - 60000)) + throw Error(20, "the warmup CSV must end at the minute before the start (" + std::to_string(start - 60000) + + "); its last row is " + fields[0]); + (void)Decimal(fields[4]); + return fields[4]; +} + +ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end, const std::string& seed, + const std::optional& step) { + ChainBars chain(start, end); + if (!seed.empty()) chain.seed(seed); + if (step) chain.quantity_grid(*step); + std::optional candidate; + std::int64_t last_row = -1; + std::exception_ptr failure; + std::string pending; + bool first = true; + const auto line = [&](std::string_view text) { + const auto row = archive_row(text, first); + first = false; + if (!row) return; + last_row = row->ts; + if (!chain.anchored()) { + if (row->ts < start) { candidate = *row; return; } + // This is the archive's first row: the predecessor is in the previous day's file. + if (!candidate) + throw Error(20, "the archive holds no print before the start: the window's predecessor is in " + + daily_name(archive, row->ts - row->ts % 86400000 - 1) + ", and export reads one daily " + "file; start after the day's first print or export from REST within its retention"); + chain.anchor(*candidate); + } + chain.add(*row); + }; + // Rows stop counting at the fence or the first failure; a zip still inflates to its end so that its CRC-32 is + // checked first: a corrupt archive stops 21, not with whatever its damaged rows would have caused. + const std::function bytes = [&](std::string_view chunk) { + if (failure || chain.fenced()) return; + try { + while (!chunk.empty() && !chain.fenced()) { + const auto newline = chunk.find('\n'); + if (newline == std::string_view::npos) { + if (pending.size() + chunk.size() > 4096) throw Error(23, "archive line exceeds 4096 bytes"); + pending.append(chunk); + return; + } + if (pending.empty()) line(chunk.substr(0, newline)); + else { + pending.append(chunk.substr(0, newline)); + line(pending); + pending.clear(); + } + chunk.remove_prefix(newline + 1); + } + } catch (const Error&) { failure = std::current_exception(); } + }; + const auto name = base_name(archive); + const bool zipped = name.size() > 4 && (name.compare(name.size() - 4, 4, ".zip") == 0 || name.compare(name.size() - 4, 4, ".ZIP") == 0); + if (zipped) { + const auto entry = unzip(archive, bytes); + log("info", "export_archive_entry_verified", Json::object({{"entry", Json::string(entry)}})); + } else { + std::ifstream file(archive, std::ios::binary); + if (!file) throw Error(23, "cannot open the archive " + archive); + std::vector buffer(1 << 20); + while (file && !failure && !chain.fenced()) { + file.read(buffer.data(), static_cast(buffer.size())); + bytes(std::string_view(buffer.data(), static_cast(file.gcount()))); + } + if (file.bad()) throw Error(22, "cannot read the archive " + archive); + } + if (!failure && !chain.fenced() && !pending.empty()) { + try { line(pending); } catch (const Error&) { failure = std::current_exception(); } + } + if (failure) std::rethrow_exception(failure); + if (!chain.anchored()) throw Error(20, "the archive holds no print at or after the start: the window is not inside it"); + if (!chain.fenced()) + throw Error(20, "the archive ends before the window's fence (the first print at or after the end), which is in " + + daily_name(archive, (last_row / 86400000 + 1) * 86400000) + + ", and export reads one daily file; end the window before the day's last print"); + return chain; +} + +namespace { +ChainBars rest_bars(const Config& config, std::string& multiplier, const std::string& seed, + const std::optional& step) { + ChainBars chain(config.start, config.end); + if (!seed.empty()) chain.seed(seed); + if (step) chain.quantity_grid(*step); + const auto now = now_ms(); + if (config.end > now) throw Error(20, "the window has not closed yet: no print at or after its end can exist"); + const auto venue = make_venue(config); + venue->horizon(now); + multiplier = venue->qty_multiplier(); + const auto window = venue->retention_ms(); + if (window && now - config.start > window) + throw Error(20, "the export start is beyond the venue's REST print history window (" + std::to_string(window / 3600000) + + " h); use --archive for older windows"); + chain.anchor(venue->predecessor(config.start)); + log("info", "export_anchored", Json::object({{"predecessor_id", number(chain.predecessor().id)}, + {"predecessor_ts", signed_number(chain.predecessor().ts)}})); + // OKX history-trades pages hold 100 prints; Binance pages hold 1000. + const std::size_t limit = config.venue == "okx" ? 100 : 1000; + for (std::uint64_t pages = 1; !chain.fenced(); ++pages) { + const auto page = venue->history(chain.next_id(), limit); + if (page.empty()) throw Error(20, "no print at or after the window end is available yet: its fence cannot be proven"); + for (const auto& trade : page) + if (chain.add(trade)) break; + if (pages % 20 == 0 || chain.fenced()) + log("info", "export_progress", Json::object({{"pages", number(pages)}, {"prints", number(chain.prints())}, + {"next_id", number(chain.next_id())}, {"ts", signed_number(chain.last_ts())}})); + } + return chain; +} +} + +void export_bars(const Config& config, const ExportOptions& options) { + if (!tick_mode(config.mode)) throw Error(23, "export builds bars from prints: use --mode ticks or agg-ticks"); + const auto manifest_path = options.output + ".manifest.json"; + const auto refuse_existing = [&] { + if (options.output.empty() || std::filesystem::exists(options.output) || std::filesystem::exists(manifest_path)) + throw Error(23, "export output already exists (or is empty)"); + }; + refuse_existing(); + (void)ChainBars(config.start, config.end); // the range and its budget are checked before any network or file access + std::string multiplier = "1"; + Json source; + std::optional chain; + const auto seed = options.warmup.empty() ? std::string() : warmup_close(options.warmup, config.start); + std::optional step; + if (!options.qty_step.empty()) { + step = runner_double(options.qty_step); + if (*step <= 0) throw Error(23, "--qty-step must be a positive number"); + } + RunnerVolume grid; + grid.reset(step); + if (!options.archive.empty()) { + if (config.venue != "binance" || config.market != "usdm" || config.mode != "agg-ticks") + throw Error(23, "--archive reads the Binance USD-M daily aggTrades archive: use --venue binance --market usdm --mode agg-ticks"); + const auto digest = file_sha256(options.archive); + const bool verified = !options.checksum.empty(); + const auto file = Json::object({{"file", Json::string(base_name(options.archive))}, {"sha256", Json::string(digest)}}); + if (verified) verify_checksum(options.checksum, options.archive, digest); + else log("warn", "archive_not_checksum_verified", file); + if (base_name(options.archive).rfind(config.symbol + "-aggTrades-", 0) != 0) log("warn", "archive_name_not_symbol", file); + chain = archive_bars(options.archive, config.start, config.end, seed, step); + source = Json::object({{"kind", Json::string("archive")}, {"file", Json::string(base_name(options.archive))}, + {"sha256", Json::string(digest)}, {"checksum_verified", Json::boolean(verified)}}); + } else { + chain = rest_bars(config, multiplier, seed, step); + source = Json::object({{"kind", Json::string("rest")}, {"origin", Json::string(config.rest_url)}}); + } + const auto& csv = chain->csv(); + const auto& predecessor = chain->predecessor(); + const auto& fence = chain->fence(); + const bool any = chain->prints() != 0; + const auto manifest = Json::object({{"schema", Json::string("pineforge-feed-export/v1")}, + {"venue", Json::string(config.venue)}, {"market", Json::string(config.market)}, {"symbol", Json::string(config.symbol)}, + {"mode", Json::string(config.mode)}, {"source", source}, {"interval", Json::string("1m")}, + {"qty_multiplier", Json::string(multiplier)}, + {"start", signed_number(config.start)}, {"end_exclusive", signed_number(config.end)}, + {"predecessor", Json::object({{"id", number(predecessor.id)}, {"ts", signed_number(predecessor.ts)}, + {"price", Json::string(predecessor.price)}})}, + {"fence", Json::object({{"id", number(fence.id)}, {"ts", signed_number(fence.ts)}})}, + {"warmup_close", seed.empty() ? Json{} : Json::string(seed)}, + {"qty_step", step ? Json::string(options.qty_step) : Json{}}, + {"volume_rule", Json::string(grid.exact() ? "exact-decimal-grid" : "compensated-binary64")}, + {"first_id", any ? number(predecessor.id + 1) : Json{}}, {"last_id", any ? number(fence.id - 1) : Json{}}, + {"prints", number(chain->prints())}, {"bars", number(chain->bars())}, {"quiet_minutes", number(chain->quiet_minutes())}, + {"rule", Json::string(bar_rule)}, {"sha256", Json::string(sha256(csv))}}); + refuse_existing(); + atomic_file(options.output, csv); + atomic_file(manifest_path, manifest.dump() + '\n'); + log("info", "export_written", manifest); +} +} diff --git a/native/feed/src/export.hpp b/native/feed/src/export.hpp new file mode 100644 index 0000000..ca6d13d --- /dev/null +++ b/native/feed/src/export.hpp @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "core.hpp" +#include +#include + +namespace pineforge::feed { +struct ExportOptions { + std::string output; // the warmup-format CSV to write; refused if it exists + std::string archive; // optional: a venue daily aggregate-trade archive (.zip or extracted .csv) + std::string checksum; // optional: the archive's published .CHECKSUM file + std::string warmup; // optional: the runner's warmup CSV, ending at the minute before the start + std::string qty_step; // optional: the runner's syminfo qty_step, as given to it (--syminfo qty_step=...) +}; +// The runner's tick-built bar volume, operation for operation as the engine's NativeExecutionConsumer::TickVolume +// (engine #319): with a quantity grid that is a decimal step 1/10^k, the exact sum of grid units, correctly rounded +// to binary64; otherwise (no step, another step, a quantity off the grid, or overflow) the compensated (Neumaier) +// binary64 sum in print order. Quantities are the doubles the runner reads (runner_double). +class RunnerVolume { + std::int64_t units_ = 0; + double sum_ = 0.0, compensation_ = 0.0, divisor_ = 1.0; + int places_ = 0; + bool exact_ = false; +public: + // Grid units as binary64: units / 10^k, rounded once to nearest even. + double decimal(std::int64_t units) const; + void reset(const std::optional& step); + void add(double quantity); + double value() const; + bool exact() const { return exact_; } +}; +// A JSON number token as the runner converts it (std::stod); 23 unless the whole token is a finite number. +double runner_double(const std::string& token); +// The shortest fixed-notation decimal that reads back as the same binary64. +std::string shortest_decimal(double value); +// Prints-built one-minute bars over [config.start, config.end), by the runner's tick-built bar rule, from venue +// REST within its retention or from a local daily archive. +void export_bars(const Config& config, const ExportOptions& options); + +// The runner's tick-built bar rule over [start, end) on a proven print chain: anchor() takes the predecessor (the +// last print strictly before start), add() every later print in ID order until it returns true at the fence (the +// first print at or after end). A minute is first/max/min/last price and the exact volume sum; a minute without a +// print repeats the previous close with volume 0. Before the window's first print that close is the runner's last +// warmup close (seed()), as the runner carries it; without a seed a quiet first minute stops 20. A hole stops 20, +// a time regression along the chain 23. +class ChainBars { + std::int64_t start_, end_, minute_ = -1, next_, last_ts_ = -1; + std::uint64_t last_id_ = 0, count_ = 0, prints_ = 0, bars_ = 0, quiet_ = 0; + std::optional predecessor_, fence_; + std::string open_, high_, low_, close_, csv_, seed_; + Decimal high_value_{"0"}, low_value_{"0"}; + RunnerVolume volume_; + std::optional step_; + void flush(); + void carry(std::int64_t until); + void row(std::int64_t minute, const std::string& open, const std::string& high, const std::string& low, + const std::string& close, const std::string& volume); +public: + ChainBars(std::int64_t start, std::int64_t end); + // The close a quiet first minute carries: the last warmup bar's close (before anchor()). + void seed(const std::string& close); + // The runner's quantity grid (its qty_step), which decides its volume rule (before anchor()). + void quantity_grid(double step); + void anchor(const Trade& predecessor); + bool add(const Trade& trade); + bool anchored() const { return predecessor_.has_value(); } + bool fenced() const { return fence_.has_value(); } + std::uint64_t next_id() const { return last_id_ + 1; } + std::int64_t last_ts() const { return last_ts_; } + // The warmup CSV (header plus one row per minute); 20 until the fence is in hand. + const std::string& csv() const; + const Trade& predecessor() const; + const Trade& fence() const; + std::uint64_t prints() const { return prints_; } + std::uint64_t bars() const { return bars_; } + std::uint64_t quiet_minutes() const { return quiet_; } + const std::string& seeded_close() const { return seed_; } +}; +// One daily-archive line, columns agg_trade_id,price,quantity,first_trade_id,last_trade_id,transact_time, +// is_buyer_maker: nullopt for the header, which only the first line may be. A transact_time above 10^14 is +// microseconds and becomes floor(us / 1000) milliseconds. +std::optional archive_row(std::string_view line, bool first_line); +// CRC-32 (ISO-HDLC, as zip uses) of `bytes` continued from `crc`; start from 0. +std::uint32_t crc32(std::uint32_t crc, std::string_view bytes); +// RFC 1951 raw deflate: pulls input from `source` (0 at its end), pushes output to `sink`. Corrupt or truncated +// input, or bytes after the final block, stop 21. +void inflate(const std::function& source, const std::function& sink); +// Streams the only entry of a stored or deflated zip to `sink`, then checks its CRC-32 and size (21). Returns +// the entry name. +std::string unzip(const std::string& path, const std::function& sink); +std::string file_sha256(const std::string& path); +// A published ` ` checksum must name the archive's base name and carry its SHA-256 (21). +void verify_checksum(const std::string& checksum, const std::string& archive, const std::string& digest); +// The window's chain from a daily aggregate-trade archive (.zip, else an extracted .csv), streamed: only the +// predecessor candidate, the window's bars and the fence are kept. +ChainBars archive_bars(const std::string& archive, std::int64_t start, std::int64_t end, const std::string& seed = {}, + const std::optional& step = std::nullopt); +// The close of the warmup CSV's last row, which must be the minute before start (20 otherwise; 23 if malformed). +std::string warmup_close(const std::string& warmup, std::int64_t start); +} diff --git a/native/feed/src/main.cpp b/native/feed/src/main.cpp index 0bb6c1b..72c7e0c 100644 --- a/native/feed/src/main.cpp +++ b/native/feed/src/main.cpp @@ -1,6 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 +#include "export.hpp" #include "session.hpp" #include "transport.hpp" +#if PINEFORGE_FEED_SERVE +#include "serve.hpp" +#endif #include #include #include @@ -25,7 +29,13 @@ struct StdoutFlags { }; void help() { std::cout << "pineforge-feed warmup --venue VENUE --market MARKET --symbol SYMBOL --start UTC|MS --end UTC|MS --output FILE\n" + "pineforge-feed export --venue VENUE --market MARKET --symbol SYMBOL --mode ticks|agg-ticks --start UTC|MS --end UTC|MS --output FILE\n" + " [--warmup RUNNER-WARMUP.csv] [--qty-step RUNNER-QTY-STEP] [--archive DAILY.zip|DAILY.csv [--checksum DAILY.zip.CHECKSUM]]\n" + " (prints-built 1m bars; volume by the runner's rule for its qty_step, none by default)\n" "pineforge-feed run --venue VENUE --market MARKET --symbol SYMBOL --mode MODE --state-dir DIR [--start UTC|MS | --resume] [--output-from INDEX]\n" + "pineforge-feed serve --venue VENUE --market MARKET --symbol SYMBOL --mode MODE --state-dir DIR [--start UTC|MS | --resume]\n" + " [--listen HOST:PORT|[IPV6]:PORT (default 127.0.0.1:8787)] [--allow-remote-listen] [--client-queue-bytes N] [--max-clients N]\n" + " GET /v1/status, GET /v1/snapshot (complete prefix from index 0, at most 4 MiB), ws://HOST:PORT/v1/stream?epoch=E&from=I\n" "Venues and modes:\n" " binance spot BTCUSDT bars | ticks\n" " binance usdm BTCUSDT bars | agg-ticks (aggregate prints, not raw trades; bars can\n" @@ -33,7 +43,8 @@ void help() { " okx spot BTC-USDT bars | ticks\n" " okx swap BTC-USDT-SWAP bars | ticks (linear swaps; base quantities)\n" " bybit spot|linear BTCUSDT bars\n" - "Limits: --max-messages N --max-log-bytes N --max-replay-seconds N --max-queue-bytes N --reconnect-seconds N --keepalive-seconds N\n" + "Journal: --segment-bytes N --replay-bytes N --replay-age-seconds N\n" + "Limits: --max-messages N --max-replay-seconds N --max-queue-bytes N --reconnect-seconds N --keepalive-seconds N --silence-seconds N\n" "Testing/public origins: --rest-url ORIGIN --ws-url ORIGIN [--allow-insecure-http (loopback only)]\n"; } // Venue, market and mode gates are checked before any network access. @@ -88,15 +99,18 @@ int run(int argc, char** argv) { if (argc == 2 && std::string(argv[1]) == "--version") { check_runtime_curl(); std::cout << "pineforge-feed 0.1.0\n"; return 0; } if (argc < 2) throw Error(23, "choose warmup or run; see --help"); const std::string command = argv[1]; - if (command != "run" && command != "warmup") throw Error(23, "only warmup and run are implemented"); + if (command != "run" && command != "warmup" && command != "serve" && command != "export") + throw Error(23, "choose warmup, export, run or serve; see --help"); Config config; std::string output; + ExportOptions exported; std::set seen; for (int index = 2; index < argc; ++index) { const std::string key = argv[index]; if (!seen.insert(key).second) throw Error(23, "duplicate CLI option: " + key); if (key == "--resume") { config.resume = true; continue; } if (key == "--allow-insecure-http") { config.allow_insecure = true; continue; } + if (key == "--allow-remote-listen") { config.allow_remote_listen = true; continue; } if (index + 1 >= argc) throw Error(23, "CLI option needs a value: " + key); const std::string value = argv[++index]; if (key == "--venue") config.venue = value; @@ -107,31 +121,71 @@ int run(int argc, char** argv) { else if (key == "--start") config.start = timestamp(value); else if (key == "--end") config.end = timestamp(value); else if (key == "--output") output = value; + else if (key == "--archive") exported.archive = value; + else if (key == "--checksum") exported.checksum = value; + else if (key == "--warmup") exported.warmup = value; + else if (key == "--qty-step") exported.qty_step = value; else if (key == "--output-from") config.output_from = unsigned_value(value); else if (key == "--max-messages") config.max_messages = unsigned_value(value); - else if (key == "--max-log-bytes") config.max_log_bytes = unsigned_value(value); + else if (key == "--segment-bytes") config.segment_bytes = unsigned_value(value); + else if (key == "--replay-bytes") config.replay_bytes = unsigned_value(value); + else if (key == "--replay-age-seconds") { + const auto seconds = unsigned_value(value); + if (seconds > 10ULL * 366 * 86400) throw Error(23, "--replay-age-seconds is at most ten years"); + config.replay_age_ms = static_cast(seconds) * 1000; + } + else if (key == "--listen") config.listen = value; + else if (key == "--client-queue-bytes") config.client_queue_bytes = unsigned_value(value); + else if (key == "--max-clients") config.max_clients = unsigned_value(value); else if (key == "--max-replay-seconds") config.max_replay_seconds = unsigned_value(value); else if (key == "--max-queue-bytes") config.max_queue_bytes = unsigned_value(value); else if (key == "--reconnect-seconds") config.reconnect_seconds = unsigned_value(value); else if (key == "--keepalive-seconds") config.keepalive_seconds = unsigned_value(value); + else if (key == "--silence-seconds") config.silence_seconds = unsigned_value(value); else if (key == "--rest-url") config.rest_url = value; else if (key == "--ws-url") config.ws_url = value; else throw Error(23, "unknown CLI option: " + key); } gate(config, command == "warmup", seen.count("--rest-url") != 0, seen.count("--ws-url") != 0); - if (!config.max_log_bytes || !config.max_queue_bytes || !config.max_replay_seconds || config.max_replay_seconds > 3600 || + if (command != "export" && + (!exported.archive.empty() || !exported.checksum.empty() || !exported.warmup.empty() || !exported.qty_step.empty())) + throw Error(23, "--archive, --checksum, --warmup and --qty-step are export options"); + if (!config.max_queue_bytes || !config.max_replay_seconds || config.max_replay_seconds > 3600 || !config.reconnect_seconds || config.reconnect_seconds > 86100) throw Error(23, "budgets must be positive; reconnect must precede the 24-hour connection limit"); + if (config.segment_bytes < 4096 || config.segment_bytes > (1ULL << 30) || config.replay_bytes / 2 < config.segment_bytes) + throw Error(23, "--segment-bytes must be 4096..1 GiB and --replay-bytes at least twice --segment-bytes"); + // A client queue holds at least one atomic message (at most 4096 bytes). + if (config.client_queue_bytes < 4096 || config.client_queue_bytes > (1ULL << 30) || !config.max_clients || config.max_clients > 1024) + throw Error(23, "--client-queue-bytes must be 4096..1 GiB and --max-clients 1..1024"); // OKX closes a connection after 30 s without traffic; Bybit recommends a ping every 20 s. if (!config.keepalive_seconds || config.keepalive_seconds > 25) throw Error(23, "--keepalive-seconds must be 1..25"); + if (!config.silence_seconds || config.silence_seconds > 75) throw Error(23, "--silence-seconds must be 1..75"); check_runtime_curl(); validate_origin(config.rest_url, false, config.allow_insecure); validate_origin(config.ws_url, true, config.allow_insecure); - if (command == "warmup") { + if (command == "export") { + if (output.empty() || config.resume || !config.state_dir.empty() || seen.count("--output-from") || !tick_mode(config.mode) || + config.start < 0 || config.end < 0) + throw Error(23, "export takes --mode ticks|agg-ticks, --start, --end and --output (plus --warmup, --archive [--checksum])"); + if (!exported.checksum.empty() && exported.archive.empty()) throw Error(23, "--checksum verifies an --archive"); + exported.output = output; + export_bars(config, exported); + } else if (command == "warmup") { if (output.empty() || config.resume || !config.state_dir.empty() || seen.count("--output-from") || seen.count("--mode")) throw Error(23, "warmup requires --output and does not accept run-state options"); warmup(config, output); + } else if (command == "serve") { + if (config.state_dir.empty() || seen.count("--end") || !output.empty() || seen.count("--output-from")) + throw Error(23, "serve requires --state-dir; clients choose their own cursor (from=I), so --output-from is refused"); +#if PINEFORGE_FEED_SERVE + serve_feed(config); +#else + throw Error(23, "this build has no serve subcommand (configured with PINEFORGE_FEED_SERVE=OFF)"); +#endif } else { + if (seen.count("--listen") || config.allow_remote_listen || seen.count("--client-queue-bytes") || seen.count("--max-clients")) + throw Error(23, "--listen, --allow-remote-listen, --client-queue-bytes and --max-clients are serve options"); if (config.state_dir.empty() || seen.count("--end") || !output.empty() || (!config.resume && seen.count("--output-from"))) throw Error(23, "run requires --state-dir; --output-from requires --resume"); const auto flags = ::fcntl(STDOUT_FILENO, F_GETFL); diff --git a/native/feed/src/okx.cpp b/native/feed/src/okx.cpp index 4c3a379..1af908e 100644 --- a/native/feed/src/okx.cpp +++ b/native/feed/src/okx.cpp @@ -119,7 +119,7 @@ Connection Okx::connection() const { const auto subscribe = "{\"op\":\"subscribe\",\"args\":[" + argument("candle1m") + (config_.mode == "ticks" ? "," + argument("trades-all") : std::string()) + "]}"; // Business endpoint: trades-all and candles live there. The text keepalive must precede 30 s idle. - return {"/ws/v5/business", {subscribe}, "ping", &okx_frame}; + return {"/ws/v5/business", {subscribe}, "ping", &okx_frame, {}}; } std::vector Okx::decode(const std::string& message) const { try { diff --git a/native/feed/src/serve.cpp b/native/feed/src/serve.cpp new file mode 100644 index 0000000..ec1ed7e --- /dev/null +++ b/native/feed/src/serve.cpp @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "serve.hpp" +#include "session.hpp" +#include "state.hpp" +#include "transport.hpp" +#include "civetweb.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +namespace { +constexpr std::size_t snapshot_limit = 4 * 1024 * 1024; // the runner's HTTP snapshot bound +constexpr auto keepalive = std::chrono::seconds(5); // the runner's WebSocket idle deadline is 15 s + +Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } + +struct Client { + std::uint64_t next = 0; // the next message index this client receives + std::deque> queue; + std::size_t queue_bytes = 0; + bool live = false, slow = false; +}; + +// Producer-side fan-out state. The producer appends released lines; each client's worker thread reads the +// journal from its cursor until it reaches the released head, then drains its own bounded queue. +struct Hub { + Config config; + std::string epoch, journal, source; + std::mutex mutex; + std::condition_variable changed; + std::uint64_t published = 0, first_retained = 0; + std::string status; // the durable cursor's public fields + std::set clients; + std::size_t connected = 0, serving = 0; + bool stopping = false; + std::atomic halting{false}; // stopping, readable without the mutex by clients catching up + + void publish(const std::string& line) { + const auto shared = std::make_shared(line); + std::lock_guard guard(mutex); + ++published; + for (auto* client : clients) { + if (!client->live || client->slow) continue; + // A client that cannot keep up is cut off; it never slows the producer or another client. + if (shared->size() > config.client_queue_bytes - std::min(client->queue_bytes, config.client_queue_bytes)) { + client->slow = true; + log("warn", "client_slow", Json::object({{"next", number(client->next)}, {"queued_bytes", number(client->queue_bytes)}})); + continue; + } + client->queue_bytes += shared->size(); + client->queue.push_back(shared); + } + changed.notify_all(); + } + void commit(const State& state) { + const auto& durable = state.durable(); + const auto document = Json::object({ + {"schema", Json::string("pineforge-feed-status/v1")}, {"epoch", Json::string(durable.epoch)}, + {"source", parse_json(source)}, + {"retained", Json::object({{"first_index", number(state.first_retained())}, {"next_index", number(durable.message_index)}})}, + {"last", Json::object({{"seq", number(durable.seq)}, {"verified_cut", Json::number(std::to_string(durable.cut))}, + {"last_tick_ts", Json::number(std::to_string(durable.last_tick_ts))}, + {"last_bar", Json::number(std::to_string(durable.last_bar))}})}}); + std::lock_guard guard(mutex); + first_retained = state.first_retained(); + status = document.dump(); + } + // Clients get their close frame (1001) before the server stops its connections. + void stop() { + std::unique_lock guard(mutex); + stopping = true; + halting = true; + changed.notify_all(); + changed.wait_for(guard, std::chrono::seconds(3), [this] { return serving == 0; }); + } +}; + +void respond(mg_connection* connection, int status, const char* type, const std::string& body) { + mg_printf(connection, "HTTP/1.1 %d %s\r\nContent-Type: %s\r\nCache-Control: no-store\r\nContent-Length: %zu\r\n" + "Connection: close\r\n\r\n", status, mg_get_response_code_text(connection, status), type, body.size()); + if (!body.empty()) mg_write(connection, body.data(), body.size()); +} +// Cursor negotiation fails in HTTP, before any WebSocket message, with the reason in the body. +int refuse(mg_connection* connection, int status, const std::string& reason) { + log("warn", "client_refused", Json::object({{"status", Json::number(std::to_string(status))}, {"reason", Json::string(reason)}})); + const auto body = Json::object({{"error", Json::string(reason)}}).dump() + '\n'; + respond(connection, status, "application/json", body); + return status; +} +bool get_only(mg_connection* connection) { + const auto* request = mg_get_request_info(connection); + if (std::string(request->request_method) == "GET") return true; + refuse(connection, 405, "only GET is served"); + return false; +} + +int status_page(mg_connection* connection, void* data) { + if (!get_only(connection)) return 405; + auto& hub = *static_cast(data); + std::string body; + { + std::lock_guard guard(hub.mutex); + auto document = parse_json(hub.status); + document.members["clients"] = number(hub.connected); + document.members["limits"] = Json::object({{"replay_bytes", number(hub.config.replay_bytes)}, + {"replay_age_seconds", number(static_cast(hub.config.replay_age_ms / 1000))}, + {"client_queue_bytes", number(hub.config.client_queue_bytes)}, {"max_clients", number(hub.config.max_clients)}}); + body = document.dump() + '\n'; + } + respond(connection, 200, "application/json", body); + return 200; +} + +// The complete committed prefix from index 0, the only HTTP shape the current runner accepts. +int snapshot_page(mg_connection* connection, void* data) { + if (!get_only(connection)) return 405; + auto& hub = *static_cast(data); + std::uint64_t published = 0, first = 0; + { + std::lock_guard guard(hub.mutex); + published = hub.published; + first = hub.first_retained; + } + const std::string expired = "the snapshot prefix from index 0 has expired from the retained journal; use /v1/stream"; + if (first) return refuse(connection, 410, expired); + try { + const auto body = snapshot_prefix(hub.journal, published, snapshot_limit); + if (!body) return refuse(connection, 413, "the committed prefix exceeds the runner's 4 MiB HTTP snapshot bound; use /v1/stream"); + respond(connection, 200, "application/x-ndjson", *body); + return 200; + } catch (const Error& failure) { return failure.code == 20 ? refuse(connection, 410, expired) : refuse(connection, 500, failure.what()); } +} + +bool query_value(const std::string& query, const char* name, std::string& value) { + char buffer[128]; + const auto length = mg_get_var(query.c_str(), query.size(), name, buffer, sizeof(buffer)); + if (length < 0) return false; + value.assign(buffer, static_cast(length)); + return true; +} + +int admit(const mg_connection* constant, void* data) { + auto* connection = const_cast(constant); + auto& hub = *static_cast(data); + const auto* request = mg_get_request_info(connection); + const std::string query = request->query_string ? request->query_string : ""; + std::string epoch, from; + std::uint64_t index = 0; + if (!query_value(query, "epoch", epoch) || !query_value(query, "from", from) || from.empty() || + from.find_first_not_of("0123456789") != std::string::npos || + std::from_chars(from.data(), from.data() + from.size(), index).ec != std::errc{}) + return refuse(connection, 400, "use /v1/stream?epoch=E&from=I with the stream epoch and a message index") != 0; + int status = 0; + std::string reason; + { + std::lock_guard guard(hub.mutex); + if (hub.stopping) status = 503, reason = "the producer is stopping"; + else if (epoch != hub.epoch) status = 409, reason = "wrong epoch: this stream's epoch is " + hub.epoch; + else if (index > hub.published) + status = 416, reason = "unknown cursor: index " + from + " is beyond the committed log (next index " + + std::to_string(hub.published) + ")"; + else if (index < hub.first_retained) + status = 410, reason = "cursor expired: index " + from + " is older than the retained journal (first retained index " + + std::to_string(hub.first_retained) + "); rebuild the consumer from a warmup inside the retained window"; + else if (hub.connected >= hub.config.max_clients) status = 503, reason = "client limit reached (--max-clients)"; + else { + auto* client = new Client; + client->next = index; + hub.clients.insert(client); + ++hub.connected; + mg_set_user_connection_data(constant, client); + } + } + if (status) return refuse(connection, status, reason) != 0; + log("info", "client_connected", Json::object({{"from", number(index)}})); + return 0; +} + +// No exception may unwind into CivetWeb's C code: a failing request answers 500, a failing upgrade is refused. +int on_status(mg_connection* connection, void* data) { + try { return status_page(connection, data); } + catch (...) { return refuse(connection, 500, "internal error"); } +} +int on_snapshot(mg_connection* connection, void* data) { + try { return snapshot_page(connection, data); } + catch (...) { return refuse(connection, 500, "internal error"); } +} +int on_connect(const mg_connection* connection, void* data) { + try { return admit(connection, data); } + catch (...) { return 1; } +} + +bool send_close(mg_connection* connection, unsigned short code, const std::string& reason) { + std::string payload; + payload += static_cast(code >> 8); + payload += static_cast(code & 0xff); + payload += reason.substr(0, 120); + log(code == 1000 || code == 1001 ? "info" : "warn", "client_closed", + Json::object({{"code", Json::number(std::to_string(code))}, {"reason", Json::string(reason)}})); + return mg_websocket_write(connection, MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE, payload.data(), payload.size()) > 0; +} + +void serve_client(mg_connection* connection, Hub& hub, Client* client) { + auto last_send = std::chrono::steady_clock::now(); + // A write that cannot complete within the request timeout leaves a partial frame: the connection is dead. + const auto send = [&](const std::string& bytes, int opcode) { + const auto written = mg_websocket_write(connection, opcode, bytes.data(), bytes.size()); + if (written <= 0 || (!bytes.empty() && static_cast(written) != bytes.size())) { + log("warn", "client_write_failed", Json::object({{"next", number(client->next)}})); + return false; + } + last_send = std::chrono::steady_clock::now(); + return true; + }; + const auto expired = [&] { + std::lock_guard guard(hub.mutex); + return client->next < hub.first_retained; + }; + // The journal is read only while catching up: a client at the head holds no segment open. + std::optional reader; + for (;;) { + std::uint64_t head = 0; + bool stopping = false; + { + std::lock_guard guard(hub.mutex); + stopping = hub.stopping; + head = hub.published; + if (!stopping && client->next == head) client->live = true; + } + if (stopping) { send_close(connection, 1001, "producer stopping"); return; } + if (client->live) break; + if (!reader) { + reader.emplace(hub.journal); + if (!reader->seek(client->next)) { + send_close(connection, 4410, "cursor expired: older than the retained journal"); + return; + } + } + while (client->next < head) { + if (hub.halting) { send_close(connection, 1001, "producer stopping"); return; } + auto line = reader->next(); + if (!line) { + // A released line is on disk: missing means its segment expired (or the reader cannot open it). + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!line && std::chrono::steady_clock::now() < deadline && !hub.halting) { + if (expired()) { + send_close(connection, 4410, "cursor expired while catching up: the journal moved past it"); + return; + } + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + line = reader->next(); + } + if (!line) { + if (hub.halting) continue; + send_close(connection, 1011, "journal segment unavailable"); + return; + } + } + if (!send(*line, MG_WEBSOCKET_OPCODE_TEXT)) return; + ++client->next; + } + } + reader.reset(); + for (;;) { + std::shared_ptr line; + { + std::unique_lock guard(hub.mutex); + hub.changed.wait_until(guard, last_send + keepalive, [&] { return hub.stopping || client->slow || !client->queue.empty(); }); + // A stopping producer still delivers what it already released, then says goodbye. + if (hub.stopping && client->queue.empty() && !client->slow) { + guard.unlock(); + send_close(connection, 1001, "producer stopping"); + return; + } + if (client->slow) { + guard.unlock(); + send_close(connection, 1008, "slow consumer: queued messages exceeded --client-queue-bytes " + + std::to_string(hub.config.client_queue_bytes)); + return; + } + if (!client->queue.empty()) { + line = std::move(client->queue.front()); + client->queue.pop_front(); + client->queue_bytes -= line->size(); + } + } + if (line) { + if (!send(*line, MG_WEBSOCKET_OPCODE_TEXT)) return; + ++client->next; + } else if (std::chrono::steady_clock::now() - last_send >= keepalive && !send("", MG_WEBSOCKET_OPCODE_PONG)) return; + } +} + +// One client, on its CivetWeb worker thread: the retained journal from its cursor, then live messages from +// its bounded queue, one WebSocket text message per normalized message, and an unsolicited PONG after 5 s +// without a message so a quiet stream never trips the runner's idle deadline. Nothing may unwind into +// CivetWeb's C code. +void on_ready(mg_connection* connection, void* data) { + auto& hub = *static_cast(data); + auto* client = static_cast(mg_get_user_connection_data(connection)); + if (!client) return; + { + std::lock_guard guard(hub.mutex); + ++hub.serving; + } + struct Done { + Hub& hub; + ~Done() { + std::lock_guard guard(hub.mutex); + --hub.serving; + hub.changed.notify_all(); + } + } done{hub}; + try { serve_client(connection, hub, client); } + catch (...) { + try { send_close(connection, 1011, "internal error"); } catch (...) {} + } +} +// The protocol has no client messages: any data frame (CivetWeb reads frames only after the ready handler has +// returned, i.e. after serve's close) ends the connection instead of being read on. +int on_data(mg_connection*, int, char*, std::size_t, void*) { return 0; } +void on_close(const mg_connection*, void*) {} + +// Every connection end, handshake failures included, releases its client. +void on_connection_close(const mg_connection* connection) { + auto* hub = static_cast(mg_get_user_data(mg_get_context(connection))); + auto* client = static_cast(mg_get_user_connection_data(connection)); + if (!hub || !client) return; + { + std::lock_guard guard(hub->mutex); + hub->clients.erase(client); + --hub->connected; + } + const auto next = client->next; + delete client; + mg_set_user_connection_data(connection, nullptr); + try { log("info", "client_disconnected", Json::object({{"next", number(next)}})); } catch (...) {} +} +int on_log(const mg_connection*, const char* message) { + log("warn", "http_server", Json::object({{"message", Json::string(std::string(message).substr(0, 512))}})); + return 1; +} + +class Server { + mg_context* context_ = nullptr; +public: + Server(Hub& hub, const std::string& address) { + mg_init_library(0); + // CivetWeb is thread-per-connection and every streaming client holds its worker. 16 spare workers serve + // status, snapshots and new upgrades; a request that is not complete within 10 s (a half-open + // connection) is dropped, which also bounds a blocked write to a dead client. + const auto threads = std::to_string(hub.config.max_clients + 16); + const char* options[] = {"listening_ports", address.c_str(), "num_threads", threads.c_str(), + "request_timeout_ms", "10000", "websocket_timeout_ms", "10000", + "enable_websocket_ping_pong", "yes", "enable_keep_alive", "no", nullptr}; + mg_callbacks callbacks{}; + callbacks.log_message = &on_log; + callbacks.connection_close = &on_connection_close; + context_ = mg_start(&callbacks, &hub, options); + if (!context_) { + mg_exit_library(); + throw Error(23, "cannot listen on " + address); + } + mg_set_request_handler(context_, "/v1/status$", &on_status, &hub); + mg_set_request_handler(context_, "/v1/snapshot$", &on_snapshot, &hub); + mg_set_websocket_handler(context_, "/v1/stream$", &on_connect, &on_ready, &on_data, &on_close, &hub); + mg_server_port ports[4]{}; + const auto count = mg_get_server_ports(context_, 4, ports); + log("info", "serve_listening", Json::object({{"address", Json::string(address)}, + {"port", Json::number(std::to_string(count > 0 ? ports[0].port : 0))}})); + } + ~Server() { + mg_stop(context_); + mg_exit_library(); + } + Server(const Server&) = delete; + Server& operator=(const Server&) = delete; +}; +} + +JournalReader::~JournalReader() { if (descriptor_ >= 0) ::close(descriptor_); } + +// False only when the segment does not exist (expired, or not yet sealed into existence); any other failure, +// such as running out of descriptors, is an I/O error and never passes for an expired cursor. +bool JournalReader::open(std::uint64_t base) { + const auto path = directory_ + "/" + segment_name(base) + ".jsonl"; + const int descriptor = ::open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (descriptor < 0 && errno == ENOENT) return false; + if (descriptor < 0) throw Error(22, "cannot open a journal segment for a client"); + if (descriptor_ >= 0) ::close(descriptor_); + descriptor_ = descriptor; + buffer_.clear(); + offset_ = 0; + index_ = base; + return true; +} + +bool JournalReader::seek(std::uint64_t index) { + // Retention may expire the chosen segment between the listing and the open: list once more. + for (int attempt = 0; attempt < 2; ++attempt) { + std::uint64_t chosen = 0; + bool found = false; + std::error_code failure; + std::filesystem::directory_iterator entries(directory_, failure), end; + for (; !failure && entries != end; entries.increment(failure)) { + const auto name = entries->path().filename().string(); + if (name.size() != 26 || name.compare(20, 6, ".jsonl") != 0) continue; + std::uint64_t base = 0; + if (std::from_chars(name.data(), name.data() + 20, base).ec != std::errc{}) continue; + if (base <= index && (!found || base > chosen)) { chosen = base; found = true; } + } + if (failure) throw Error(22, "cannot list the journal for a client"); + if (!found) return false; + if (!open(chosen)) continue; + while (index_ < index) { + auto skipped = next(); + if (!skipped) return false; + } + return true; + } + return false; +} + +std::optional JournalReader::next() { + for (;;) { + const auto newline = buffer_.find('\n', offset_); + if (newline != std::string::npos) { + std::string line = buffer_.substr(offset_, newline - offset_); + offset_ = newline + 1; + if (offset_ > 65536) { buffer_.erase(0, offset_); offset_ = 0; } + ++index_; + return line; + } + char chunk[65536]; + const auto count = ::read(descriptor_, chunk, sizeof(chunk)); + if (count < 0 && errno == EINTR) continue; + if (count > 0) { buffer_.append(chunk, static_cast(count)); continue; } + // End of this segment: a sealed segment ends on a line boundary, and its successor starts here. + if (offset_ == buffer_.size() && open(index_)) continue; + return std::nullopt; + } +} + +std::optional snapshot_prefix(const std::string& journal, std::uint64_t published, std::size_t limit) { + JournalReader reader(journal); + if (!reader.seek(0)) throw Error(20, "the snapshot prefix has expired"); + std::string body; + while (reader.index() < published) { + // Every published line is on disk before it is published: a missing one means its segment expired. + auto line = reader.next(); + if (!line) throw Error(20, "a snapshot segment expired while it was read"); + if (line->size() + 1 > limit - body.size()) return std::nullopt; + body += *line; + body += '\n'; + } + return body; +} + +std::string listen_address(const std::string& listen, bool allow_remote) { + const auto colon = listen.rfind(':'); + if (colon == std::string::npos || colon + 1 == listen.size()) throw Error(23, "--listen takes HOST:PORT or [IPV6]:PORT"); + auto host = listen.substr(0, colon); + const auto port = listen.substr(colon + 1); + unsigned int value = 0; + if (port.find_first_not_of("0123456789") != std::string::npos || + std::from_chars(port.data(), port.data() + port.size(), value).ec != std::errc{} || value > 65535) + throw Error(23, "--listen port must be 0..65535"); + bool loopback = false; + if (host.size() > 2 && host.front() == '[' && host.back() == ']') { + in6_addr parsed{}; + if (::inet_pton(AF_INET6, host.substr(1, host.size() - 2).c_str(), &parsed) != 1) + throw Error(23, "--listen host must be an IPv4 address, [IPV6] or localhost"); + loopback = IN6_IS_ADDR_LOOPBACK(&parsed); + } else { + if (host == "localhost") host = "127.0.0.1"; + in_addr parsed{}; + if (::inet_pton(AF_INET, host.c_str(), &parsed) != 1) + throw Error(23, "--listen host must be an IPv4 address, [IPV6] or localhost"); + loopback = (ntohl(parsed.s_addr) >> 24) == 127; + } + if (!loopback && !allow_remote) + throw Error(23, "--listen on a non-loopback address needs --allow-remote-listen; serve has no TLS or authentication"); + return host + ":" + std::to_string(value); +} + +void serve_feed(Config config) { + const auto address = listen_address(config.listen, config.allow_remote_listen); + // Each client holds a socket (and a journal segment while it catches up); the producer needs its own. + rlimit files{}; + if (::getrlimit(RLIMIT_NOFILE, &files) == 0 && files.rlim_cur < files.rlim_max) { + files.rlim_cur = files.rlim_max; + ::setrlimit(RLIMIT_NOFILE, &files); + ::getrlimit(RLIMIT_NOFILE, &files); + } + const auto needed = static_cast(2 * config.max_clients + 64); + if (files.rlim_cur != RLIM_INFINITY && files.rlim_cur < needed) + throw Error(23, "the open-file limit (" + std::to_string(files.rlim_cur) + ") is below " + std::to_string(needed) + + " for --max-clients " + std::to_string(config.max_clients) + "; raise it or lower --max-clients"); + // Instrument metadata fixes the quantity units before the cursor binds them. + const auto venue = make_venue(config); + config.qty_multiplier = venue->qty_multiplier(); + config.serve = true; + State state(config); + Hub hub; + hub.config = config; + hub.epoch = state.durable().epoch; + hub.journal = config.state_dir + "/journal"; + hub.published = state.durable().message_index; + hub.source = Json::object({{"venue", Json::string(config.venue)}, {"market", Json::string(config.market)}, + {"symbol", Json::string(config.symbol)}, {"mode", Json::string(config.mode)}, {"rest_origin", Json::string(config.rest_url)}, + {"ws_origin", Json::string(config.ws_url)}, {"start", Json::number(std::to_string(state.durable().start))}, + {"qty_multiplier", Json::string(config.qty_multiplier)}}).dump(); + hub.commit(state); + Server server(hub, address); + try { + const auto session = make_session(state, *venue, [&](const std::string& line) { hub.publish(line); }); + stream(config, *venue, *session, [&] { hub.commit(state); }); + } catch (const Stopped&) { + hub.stop(); + log("info", "stopped", state.status()); + return; + } catch (...) { + hub.stop(); + log("error", "verified_cursor_retained", state.status()); + throw; + } +} +} diff --git a/native/feed/src/serve.hpp b/native/feed/src/serve.hpp new file mode 100644 index 0000000..e31856a --- /dev/null +++ b/native/feed/src/serve.hpp @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "core.hpp" +#include + +namespace pineforge::feed { +// Reads committed journal lines by message index straight from the segment files, independently of the +// producer's State: a client catching up, and the snapshot, read here while the producer appends. +class JournalReader { + std::string directory_; + int descriptor_ = -1; + std::string buffer_; + std::size_t offset_ = 0; + std::uint64_t index_ = 0; + bool open(std::uint64_t base); +public: + explicit JournalReader(std::string directory) : directory_(std::move(directory)) {} + ~JournalReader(); + JournalReader(const JournalReader&) = delete; + JournalReader& operator=(const JournalReader&) = delete; + // Positions before message `index`; false when no retained segment holds it (expired). + bool seek(std::uint64_t index); + // The next complete line, or nullopt while it is not on disk yet. + std::optional next(); + std::uint64_t index() const { return index_; } +}; + +// The committed prefix [0, published) as JSONL, or nullopt when it exceeds `limit` bytes; throws 20 when index 0 +// has expired or a segment disappears while it is read, 22 on an I/O error. +std::optional snapshot_prefix(const std::string& journal, std::uint64_t published, std::size_t limit); +// The listening address: loopback unless the operator explicitly allows another interface. +std::string listen_address(const std::string& listen, bool allow_remote); +void serve_feed(Config config); +} diff --git a/native/feed/src/session.cpp b/native/feed/src/session.cpp index 415b76e..2402bc0 100644 --- a/native/feed/src/session.cpp +++ b/native/feed/src/session.cpp @@ -89,7 +89,8 @@ void Session::ingest(const VenueEvent& event) { } void Session::replay() { if (!replayed_) { - if (state_.config().resume) state_.visit(state_.config().output_from, output_); + // serve publishes only new messages; its clients read the retained journal by index themselves. + if (state_.config().resume && !state_.config().serve) state_.visit(state_.config().output_from, output_); replayed_ = true; } log("info", "overlap_verified", state_.status()); @@ -97,7 +98,7 @@ void Session::replay() { FeedSession::FeedSession(State& state, Venue& venue, std::function output) : Session(state, venue, std::move(output)) { - if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { + if (state_.config().mode == "ticks") state_.visit(state_.index_for_minute(state_.cursor().cut), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() == "tick") { const auto trade = normalized_trade(event); @@ -115,7 +116,7 @@ void FeedSession::connected() { if (fetched.size() != trades.size()) throw Error(20, "reconnect raw overlap is unavailable"); for (std::size_t index = 0; index < trades.size(); ++index) if (!(trades[index] == fetched[index])) throw Error(21, "reconnect raw overlap changed"); - } else if (state_.cursor().predecessor) { + } else if (!state_.cursor().seq && state_.cursor().predecessor) { const auto fetched = venue_.history(state_.cursor().predecessor->id, 1); if (!(fetched.front() == *state_.cursor().predecessor)) throw Error(21, "initial predecessor overlap changed"); } @@ -125,7 +126,7 @@ void FeedSession::connected() { [&](const auto& rows) { return same_bars(rows, proofs); }); if (fetched.size() != proofs.size()) throw Error(20, "reconnect closed-bar overlap is unavailable"); std::map verified; - if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { + if (state_.config().mode == "ticks") state_.visit(state_.index_for_minute(proofs.front().ts), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() != "tick") return; const auto trade = normalized_trade(event); @@ -158,7 +159,12 @@ void FeedSession::put(const Trade& trade) { trade.validate(); if (trade.ts < state_.cursor().start) return; if (state_.cursor().seq && trade.id <= state_.cursor().seq) { - const auto previous = state_.trade(trade.id); + auto previous = state_.recent_trade(trade.id); + if (!previous && state_.expired_seq(trade.id)) { + log("warn", "expired_duplicate_unverified", Json::object({{"seq", Json::number(std::to_string(trade.id))}})); + return; + } + if (!previous) previous = state_.trade(trade.id); if (!previous || !(*previous == trade)) throw Error(21, "duplicate raw trade conflicts with the immutable prefix"); return; } @@ -247,14 +253,20 @@ void FeedSession::closed(const Kline& kline) { const auto minute = kline.bar.ts; watermark_ = std::max(watermark_, minute + 60000); if (minute < state_.cursor().start) return; + if (minute < state_.cursor().cut && state_.expired_minute(minute)) { + // Retention dropped this minute: an old duplicate can no longer be compared with what was emitted. + log("warn", "expired_duplicate_unverified", Json::object({{"minute", Json::number(std::to_string(minute))}})); + return; + } if (minute < state_.cursor().cut) { - const auto old = state_.bar(minute); + // Tick journals hold no bar lines: only the closed-minute proofs can hold this candle. + const auto old = state_.config().mode == "bars" ? state_.bar(minute) : state_.proof(minute); if (old) { if (!(*old == kline.bar)) throw Error(21, "already-emitted closed bar was revised"); } if (state_.config().mode == "ticks") { Aggregate reconstructed; - state_.visit(0, [&](const std::string& line) { + state_.visit(state_.index_for_minute(minute), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() == "tick") { const auto trade = normalized_trade(event); @@ -313,7 +325,7 @@ void reconcile_fenced(const Aggregate& prints, const Kline& candle) { FenceSession::FenceSession(State& state, Venue& venue, std::function output) : Session(state, venue, std::move(output)) { - state_.visit(0, [&](const std::string& line) { + state_.visit(state_.index_for_minute(state_.cursor().cut), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() == "tick") { const auto trade = normalized_trade(event); @@ -343,7 +355,12 @@ void FenceSession::put(const Trade& trade, bool healed) { throw Error(23, "venue matched time regressed: a print before the proven predecessor is at or after the start minute"); } if (state_.cursor().seq && trade.id <= state_.cursor().seq) { - const auto previous = state_.trade(trade.id); + auto previous = state_.recent_trade(trade.id); + if (!previous && state_.expired_seq(trade.id)) { + log("warn", "expired_duplicate_unverified", Json::object({{"seq", Json::number(std::to_string(trade.id))}})); + return; + } + if (!previous) previous = state_.trade(trade.id); if (!previous || !(*previous == trade)) throw Error(21, "duplicate print conflicts with the immutable prefix"); return; } @@ -453,6 +470,10 @@ void FenceSession::candle(const Kline& kline) { const auto minute = kline.bar.ts; watermark_ = std::max(watermark_, minute + 60000); if (minute < state_.cursor().start) return; + if (minute < state_.cursor().cut && state_.expired_minute(minute)) { + log("warn", "expired_duplicate_unverified", Json::object({{"minute", Json::number(std::to_string(minute))}})); + return; + } if (minute < state_.cursor().cut) { for (const auto& proof : state_.cursor().proofs) if (proof.ts == minute) { @@ -462,7 +483,7 @@ void FenceSession::candle(const Kline& kline) { // Older than the retained proofs: rebuild that minute from the journal, where the candle is a sum. if (!venue_.candle_is_print_sum()) return; Aggregate rebuilt; - state_.visit(0, [&](const std::string& line) { + state_.visit(state_.index_for_minute(minute), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() != "tick") return; const auto trade = normalized_trade(event); @@ -512,7 +533,7 @@ void FenceSession::connected() { if (fetched.size() != trades.size()) throw Error(20, "reconnect print overlap is unavailable"); for (std::size_t index = 0; index < trades.size(); ++index) if (!(trades[index] == fetched[index])) throw Error(21, "reconnect print overlap changed"); - } else if (state_.cursor().predecessor) { + } else if (!state_.cursor().seq && state_.cursor().predecessor) { const auto fetched = venue_.history(state_.cursor().predecessor->id, 1); if (fetched.empty()) throw Error(20, "initial predecessor overlap is unavailable"); if (!(fetched.front() == *state_.cursor().predecessor)) throw Error(21, "initial predecessor overlap changed"); @@ -537,26 +558,35 @@ std::unique_ptr make_session(State& state, Venue& venue, std::function< return std::make_unique(state, venue, std::move(output)); } +void stream(const Config& config, Venue& venue, Session& session, const std::function& committed) { + WebSocketPump pump(config, venue.connection()); + for (;;) { + auto message = pump.take(); + try { + // Group commit: verify every source message already queued (bounded), then persist once. + for (std::size_t taken = 1;; ++taken) { + if (message.connected) session.connected(); + else for (const auto& event : venue.decode(message.text)) session.stage(event); + if (taken >= 1024 || !pump.try_take(message)) break; + } + } catch (const Error&) { + session.salvage(); + if (committed) committed(); + throw; + } + session.publish(); + if (committed) committed(); + } +} + void run_feed(Config config) { // Instrument metadata fixes the quantity units before the cursor binds them. const auto venue = make_venue(config); config.qty_multiplier = venue->qty_multiplier(); State state(config); try { - WebSocketPump pump(config, venue->connection()); const auto session = make_session(state, *venue, &output_line); - for (;;) { - auto message = pump.take(); - try { - // Group commit: verify every source message already queued (bounded), then persist once. - for (std::size_t taken = 1;; ++taken) { - if (message.connected) session->connected(); - else for (const auto& event : venue->decode(message.text)) session->stage(event); - if (taken >= 1024 || !pump.try_take(message)) break; - } - } catch (const Error&) { session->salvage(); throw; } - session->publish(); - } + stream(config, *venue, *session, {}); } catch (const Stopped&) { log("info", "stopped", state.status()); } catch (...) { log("error", "verified_cursor_retained", state.status()); throw; } } diff --git a/native/feed/src/session.hpp b/native/feed/src/session.hpp index 3833ae0..e110de0 100644 --- a/native/feed/src/session.hpp +++ b/native/feed/src/session.hpp @@ -85,6 +85,8 @@ std::vector settled_prints(Venue& venue, const Config& config, const std: void reconcile_fenced(const Aggregate& prints, const Kline& candle); std::unique_ptr make_session(State& state, Venue& venue, std::function output); +// The source loop shared by run and serve: group-commit every queued source message, publish, then report. +void stream(const Config& config, Venue& venue, Session& session, const std::function& committed); void run_feed(Config config); void warmup(const Config& config, const std::string& output); } diff --git a/native/feed/src/state.cpp b/native/feed/src/state.cpp index 571d1b4..2dd0372 100644 --- a/native/feed/src/state.cpp +++ b/native/feed/src/state.cpp @@ -1,11 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 #include "state.hpp" #include +#include #include +#include #include #include #include #include +#include #include #include #include @@ -17,6 +20,11 @@ void Descriptor::reset(int value) { if (value_ >= 0) ::close(value_); value_ = value; } +std::string segment_name(std::uint64_t base) { + char name[32]; + std::snprintf(name, sizeof(name), "%020llu", static_cast(base)); + return name; +} namespace { Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } Json signed_number(std::int64_t value) { return Json::number(std::to_string(value)); } @@ -46,16 +54,69 @@ void advance(Cursor& cursor, const Json& event, const std::string& mode) { cursor.cut = cut; } else throw Error(23, "journal contains an unsupported normalized message"); } +void sync_directory(const std::string& path) { + const int directory = ::open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (directory < 0) throw Error(22, "cannot open journal directory for sync"); + const auto status = ::fsync(directory); + ::close(directory); + if (status != 0) throw Error(22, "cannot sync journal directory"); +} +std::string small_file(const std::string& path) { + if (!std::filesystem::exists(path) || std::filesystem::file_size(path) > 65536) throw Error(22, "missing or oversized durable state file"); + std::ifstream input(path, std::ios::binary); + return std::string((std::istreambuf_iterator(input)), {}); +} +// Every complete line of the first `limit` bytes of `path`; a line cut by `limit` or the end of the file +// is a broken message partition. +void read_lines(const std::string& path, std::uint64_t limit, std::chrono::steady_clock::time_point deadline, + const std::function& visitor) { + std::ifstream input(path, std::ios::binary); + if (!input) throw Error(22, "cannot read retained normalized journal segment"); + std::uint64_t bytes = 0; + std::string line; + while (bytes < limit) { + if (stopping) throw Stopped{}; + if (std::chrono::steady_clock::now() >= deadline) throw Error(22, "retained prefix replay time budget exhausted"); + if (!std::getline(input, line) || input.eof() || line.size() > 4095 || bytes + line.size() + 1 > limit) + throw Error(21, "truncated or oversized verified message"); + bytes += line.size() + 1; + visitor(line); + } +} +Json checkpoint_document(const Cursor& cursor) { + return Json::object({ + {"schema", Json::string("pineforge-feed-checkpoint/v1")}, {"epoch", Json::string(cursor.epoch)}, + {"message_index", number(cursor.message_index)}, {"log_bytes", number(cursor.log_bytes)}, + {"prefix_hash", Json::string(cursor.prefix_hash)}, {"overlap_hash", Json::string(cursor.overlap_hash)}, + {"start", signed_number(cursor.start)}, {"verified_cut", signed_number(cursor.cut)}, + {"emitted_seq", number(cursor.seq)}, {"last_tick_ts", signed_number(cursor.last_tick_ts)}, + {"last_bar", signed_number(cursor.last_bar)}, + {"predecessor", cursor.predecessor ? parse_json(cursor.predecessor->wire()) : Json{}} + }); +} +// The verified fields a segment boundary must reproduce exactly. +bool same_state(const Cursor& left, const Cursor& right) { + return left.message_index == right.message_index && left.log_bytes == right.log_bytes && + left.prefix_hash == right.prefix_hash && left.overlap_hash == right.overlap_hash && left.cut == right.cut && + left.seq == right.seq && left.last_tick_ts == right.last_tick_ts && left.last_bar == right.last_bar; +} +bool parse_base(const std::string& digits, std::uint64_t& base) { + if (digits.size() != 20 || digits.find_first_not_of("0123456789") != std::string::npos) return false; + base = std::stoull(digits); + return true; +} } -std::string State::log_path() const { return config_.state_dir + "/events.jsonl"; } +std::string State::journal_dir() const { return config_.state_dir + "/journal"; } +std::string State::segment_path(std::uint64_t base) const { return journal_dir() + "/" + segment_name(base) + ".jsonl"; } +std::string State::checkpoint_path(std::uint64_t base) const { return journal_dir() + "/" + segment_name(base) + ".checkpoint.json"; } Json State::serialize(const Cursor& cursor) const { Json proofs; proofs.kind = Json::Kind::Array; for (const auto& bar : cursor.proofs) proofs.items.push_back(parse_json(bar.wire())); return Json::object({ - {"schema", Json::string("pineforge-feed-cursor/v1")}, {"epoch", Json::string(cursor.epoch)}, + {"schema", Json::string("pineforge-feed-cursor/v2")}, {"epoch", Json::string(cursor.epoch)}, {"venue", Json::string(config_.venue)}, {"market", Json::string(config_.market)}, {"symbol", Json::string(config_.symbol)}, {"mode", Json::string(config_.mode)}, {"rest_origin", Json::string(config_.rest_url)}, {"ws_origin", Json::string(config_.ws_url)}, @@ -65,8 +126,8 @@ Json State::serialize(const Cursor& cursor) const { {"venue_id", tick_mode(config_.mode) ? number(cursor.seq) : signed_number(cursor.last_bar)}, {"emitted_seq", number(cursor.seq)}, {"last_tick_ts", signed_number(cursor.last_tick_ts)}, {"last_bar", signed_number(cursor.last_bar)}, {"message_index", number(cursor.message_index)}, - {"log_bytes", number(cursor.log_bytes)}, {"prefix_hash", Json::string(cursor.prefix_hash)}, - {"overlap_hash", Json::string(cursor.overlap_hash)}, + {"log_bytes", number(cursor.log_bytes)}, {"segment", number(cursor.segment)}, + {"prefix_hash", Json::string(cursor.prefix_hash)}, {"overlap_hash", Json::string(cursor.overlap_hash)}, {"predecessor", cursor.predecessor ? parse_json(cursor.predecessor->wire()) : Json{}}, {"closed_proofs", std::move(proofs)}, {"partition", Json::string("one-event-per-message")} }); @@ -78,6 +139,38 @@ void State::persist(const Cursor& cursor) const { {"sha256", Json::string(sha256(document.dump()))}}).dump() + '\n'); } +void State::write_checkpoint(const Cursor& cursor) const { + const auto document = checkpoint_document(cursor); + atomic_file(checkpoint_path(cursor.message_index), Json::object({{"checkpoint", document}, + {"sha256", Json::string(sha256(document.dump()))}}).dump() + '\n'); +} + +Cursor State::read_checkpoint(std::uint64_t base) const { + try { + const auto envelope = parse_json(small_file(checkpoint_path(base))); + const auto& document = envelope.at("checkpoint"); + if (envelope.at("sha256").text() != sha256(document.dump())) throw Error(21, "journal checkpoint digest mismatch"); + if (document.at("schema").text() != "pineforge-feed-checkpoint/v1") throw Error(23, "unsupported journal checkpoint schema"); + Cursor checkpoint; + checkpoint.epoch = document.at("epoch").text(); + checkpoint.message_index = document.at("message_index").integer(); + checkpoint.log_bytes = document.at("log_bytes").integer(); + checkpoint.prefix_hash = document.at("prefix_hash").text(); + checkpoint.overlap_hash = document.at("overlap_hash").text(); + checkpoint.start = document.at("start").integer(); + checkpoint.cut = document.at("verified_cut").integer(); + checkpoint.seq = document.at("emitted_seq").integer(); + checkpoint.last_tick_ts = document.at("last_tick_ts").integer(); + checkpoint.last_bar = document.at("last_bar").integer(); + if (document.at("predecessor").kind != Json::Kind::Null) checkpoint.predecessor = normalized_trade(document.at("predecessor")); + if (checkpoint.message_index != base || !digest_token(checkpoint.prefix_hash) || !digest_token(checkpoint.overlap_hash)) + throw Error(21, "journal checkpoint does not describe its segment"); + checkpoint.segment = base; + return checkpoint; + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(21, "invalid journal checkpoint"); } +} + State::State(const Config& config) : config_(config) { std::error_code failure; std::filesystem::create_directories(config_.state_dir, failure); @@ -87,39 +180,55 @@ State::State(const Config& config) : config_(config) { throw Error(22, "state directory has another producer or cannot be locked"); if (config_.resume) { recover(); - journal_.reset(::open(log_path().c_str(), O_WRONLY | O_APPEND | O_CLOEXEC)); + journal_.reset(::open(segment_path(cursor_.segment).c_str(), O_WRONLY | O_APPEND | O_CLOEXEC)); if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot sync rolled-back journal tail"); - } else { - if (config_.start < 0 || config_.start % 60000) throw Error(23, "new streams require a minute-aligned --start"); - // An empty journal without a cursor is an initialization that crashed before its first cursor - // write: nothing was committed or published, so a fresh start may take the directory over. - if (std::filesystem::exists(config_.state_dir + "/cursor.json") || - (std::filesystem::exists(log_path()) && std::filesystem::file_size(log_path()) != 0)) - throw Error(23, "state already exists; use --resume or an explicitly new state directory"); - cursor_.epoch = random_epoch(); - cursor_.start = cursor_.cut = config_.start; - cursor_.prefix_hash = sha256(""); - cursor_.overlap_hash = sha256(""); - journal_.reset(::open(log_path().c_str(), O_WRONLY | O_APPEND | O_CREAT | O_TRUNC | O_CLOEXEC, 0600)); - if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot initialize durable journal"); - persist(cursor_); - durable_ = cursor_; - } + } else initialize(); if (journal_.get() < 0) throw Error(22, "cannot open durable journal"); if (config_.output_from > cursor_.message_index) throw Error(22, "output cursor exceeds the retained verified prefix"); + if (config_.resume && !config_.serve && config_.output_from < first_retained()) + throw Error(22, "output cursor " + std::to_string(config_.output_from) + " has expired: messages before index " + + std::to_string(first_retained()) + " left the retained journal (--replay-bytes, --replay-age-seconds). A consumer " + "behind that index cannot resume from this stream: rebuild it from a warmup that ends inside the retained window, " + "or keep a longer retention"); +} + +void State::initialize() { + if (config_.start < 0 || config_.start % 60000) throw Error(23, "new streams require a minute-aligned --start"); + if (std::filesystem::exists(config_.state_dir + "/events.jsonl")) + throw Error(23, "this state directory holds an unsegmented journal from an earlier release; start a new state directory"); + if (std::filesystem::exists(config_.state_dir + "/cursor.json")) + throw Error(23, "state already exists; use --resume or an explicitly new state directory"); + std::error_code failure; + std::filesystem::create_directories(journal_dir(), failure); + if (failure) throw Error(22, "cannot create journal directory"); + // A journal without a cursor is an initialization that crashed before its first cursor write: nothing + // was committed or published, so a fresh start may take the directory over if no segment holds a byte. + std::vector leftovers; + for (const auto& entry : std::filesystem::directory_iterator(journal_dir())) { + if (entry.path().extension() == ".jsonl" && entry.file_size() != 0) + throw Error(23, "state already exists; use --resume or an explicitly new state directory"); + leftovers.push_back(entry.path()); + } + for (const auto& path : leftovers) std::filesystem::remove(path); + cursor_.epoch = random_epoch(); + cursor_.start = cursor_.cut = config_.start; + cursor_.prefix_hash = sha256(""); + cursor_.overlap_hash = sha256(""); + write_checkpoint(cursor_); + journal_.reset(::open(segment_path(0).c_str(), O_WRONLY | O_APPEND | O_CREAT | O_EXCL | O_CLOEXEC, 0600)); + if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot initialize durable journal"); + sync_directory(journal_dir()); + persist(cursor_); + durable_ = cursor_; + segments_ = {{0, 0, 0, cursor_.start, -1}}; } void State::recover() { try { - const auto path = config_.state_dir + "/cursor.json"; - if (!std::filesystem::exists(path) || std::filesystem::file_size(path) > 65536) - throw Error(22, "missing or oversized durable cursor"); - std::ifstream input(path); - const std::string bytes((std::istreambuf_iterator(input)), {}); - const auto envelope = parse_json(bytes); + const auto envelope = parse_json(small_file(config_.state_dir + "/cursor.json")); const auto& document = envelope.at("cursor"); if (envelope.at("sha256").text() != sha256(document.dump())) throw Error(21, "cursor digest mismatch"); - if (document.at("schema").text() != "pineforge-feed-cursor/v1" || document.at("partition").text() != "one-event-per-message") + if (document.at("schema").text() != "pineforge-feed-cursor/v2" || document.at("partition").text() != "one-event-per-message") throw Error(23, "unsupported cursor schema or message partition"); for (const auto& item : {std::make_pair("venue", config_.venue), {"market", config_.market}, {"symbol", config_.symbol}, {"mode", config_.mode}, {"rest_origin", config_.rest_url}, {"ws_origin", config_.ws_url}}) @@ -128,9 +237,7 @@ void State::recover() { if (units.at("price").text() != "quote/base" || units.at("qty").text() != "base" || units.at("ts").text() != "unix-ms") throw Error(21, "resume units changed"); // Contract quantities were converted with this instrument multiplier; a change would mix units. - const auto* multiplier = document.find("qty_multiplier"); - if ((multiplier ? multiplier->text() : std::string("1")) != config_.qty_multiplier) - throw Error(21, "resume contract quantity multiplier changed"); + if (document.at("qty_multiplier").text() != config_.qty_multiplier) throw Error(21, "resume contract quantity multiplier changed"); cursor_.epoch = document.at("epoch").text(); cursor_.prefix_hash = document.at("prefix_hash").text(); cursor_.overlap_hash = document.at("overlap_hash").text(); @@ -146,38 +253,88 @@ void State::recover() { throw Error(21, "venue ID does not match the emitted cursor"); cursor_.message_index = document.at("message_index").integer(); cursor_.log_bytes = document.at("log_bytes").integer(); + cursor_.segment = document.at("segment").integer(); if (cursor_.start < 0 || cursor_.start % 60000 || cursor_.cut < cursor_.start || cursor_.cut % 60000 || - cursor_.log_bytes > config_.max_log_bytes || (config_.start >= 0 && config_.start != cursor_.start)) + cursor_.segment > cursor_.message_index || (config_.start >= 0 && config_.start != cursor_.start)) throw Error(21, "invalid or changed resume cut"); if (document.at("predecessor").kind != Json::Kind::Null) cursor_.predecessor = normalized_trade(document.at("predecessor")); const auto& proofs = document.at("closed_proofs"); if (proofs.kind != Json::Kind::Array || proofs.items.size() > 32) throw Error(21, "invalid closed-minute proof window"); for (const auto& proof : proofs.items) cursor_.proofs.push_back(normalized_bar(proof)); - if (!std::filesystem::exists(log_path()) || std::filesystem::file_size(log_path()) < cursor_.log_bytes) - throw Error(21, "durable journal is shorter than its verified cursor"); - durable_ = cursor_; - Cursor reconstructed; - reconstructed.start = reconstructed.cut = cursor_.start; - reconstructed.predecessor = cursor_.predecessor; - reconstructed.prefix_hash = reconstructed.overlap_hash = sha256(""); - visit(0, [&](const std::string& line) { - const auto event = parse_json(line); - advance(reconstructed, event, config_.mode); - reconstructed.prefix_hash = sha256(reconstructed.prefix_hash + line + '\n'); - reconstructed.overlap_hash = sha256(line + '\n'); - reconstructed.log_bytes += line.size() + 1; - ++reconstructed.message_index; - remember(event); - }); - if (reconstructed.prefix_hash != cursor_.prefix_hash || reconstructed.overlap_hash != cursor_.overlap_hash || - reconstructed.log_bytes != cursor_.log_bytes || reconstructed.message_index != cursor_.message_index || - reconstructed.seq != cursor_.seq || reconstructed.cut != cursor_.cut || reconstructed.last_bar != cursor_.last_bar || - reconstructed.last_tick_ts != cursor_.last_tick_ts) - throw Error(21, "durable journal prefix does not match its cursor"); - if (::truncate(log_path().c_str(), static_cast(cursor_.log_bytes)) != 0) + // Segments after the cursor's open one are an interrupted rotation: never committed, so removed. A + // checkpoint without its segment is an interrupted expiry. A segment without its checkpoint never occurs. + std::map> files; + for (const auto& entry : std::filesystem::directory_iterator(journal_dir())) { + const auto name = entry.path().filename().string(); + std::uint64_t base = 0; + if (!name.empty() && name.front() == '.') continue; + if (name.size() > 4 && name.compare(name.size() - 4, 4, ".tmp") == 0) { std::filesystem::remove(entry.path()); continue; } + if (name.size() == 26 && name.compare(20, 6, ".jsonl") == 0 && parse_base(name.substr(0, 20), base)) files[base].first = true; + else if (name.size() == 36 && name.compare(20, 16, ".checkpoint.json") == 0 && parse_base(name.substr(0, 20), base)) files[base].second = true; + else throw Error(21, "unexpected file in the journal directory: " + name); + } + std::vector retained; + bool removed = false; + std::uint64_t lowest = cursor_.segment; + for (const auto& [base, kinds] : files) + if (kinds.first) lowest = std::min(lowest, base); + for (const auto& [base, kinds] : files) { + if (base > cursor_.segment) { + // Only the successor an interrupted seal creates: at the cursor's index, and still empty. + if (base != cursor_.message_index || (kinds.first && std::filesystem::file_size(segment_path(base)) != 0)) + throw Error(21, "a journal segment after the cursor's open segment holds data the cursor does not cover"); + std::filesystem::remove(segment_path(base)); + std::filesystem::remove(checkpoint_path(base)); + log("warn", "uncommitted_rotation_removed", Json::object({{"segment", number(base)}})); + removed = true; + } else if (!kinds.first) { + // An interrupted expiry leaves the oldest checkpoint without its segment; anything else is a hole. + if (base >= lowest) throw Error(21, "the retained journal has a hole: a checkpoint without its segment"); + std::filesystem::remove(checkpoint_path(base)); + removed = true; + } else if (!kinds.second) throw Error(21, "a journal segment has no checkpoint"); + else retained.push_back(base); + } + if (removed) sync_directory(journal_dir()); + if (retained.empty() || retained.back() != cursor_.segment) throw Error(21, "the open journal segment is missing"); + Cursor reconstructed = read_checkpoint(retained.front()); + if (reconstructed.epoch != cursor_.epoch || reconstructed.start != cursor_.start) + throw Error(21, "journal checkpoint belongs to another stream"); + // The start predecessor is anchored once, before the first message, and never changes. + if (!reconstructed.predecessor) reconstructed.predecessor = cursor_.predecessor; + else if (!cursor_.predecessor || !(*reconstructed.predecessor == *cursor_.predecessor)) + throw Error(21, "journal checkpoint predecessor does not match the cursor"); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(config_.max_replay_seconds); + for (std::size_t index = 0; index < retained.size(); ++index) { + const auto base = retained[index]; + if (index && !same_state(reconstructed, read_checkpoint(base))) + throw Error(21, "durable journal segment boundary does not match its checkpoint"); + segments_.push_back({base, reconstructed.log_bytes, reconstructed.seq, reconstructed.cut, reconstructed.last_tick_ts}); + const auto path = segment_path(base); + const auto size = std::filesystem::file_size(path); + const bool open = index + 1 == retained.size(); + if (open && cursor_.log_bytes < reconstructed.log_bytes) throw Error(21, "cursor precedes its open journal segment"); + const auto limit = open ? cursor_.log_bytes - reconstructed.log_bytes : size; + if (size < limit) throw Error(21, "durable journal is shorter than its verified cursor"); + read_lines(path, limit, deadline, [&](const std::string& line) { + const auto event = parse_json(line); + advance(reconstructed, event, config_.mode); + reconstructed.prefix_hash = sha256(reconstructed.prefix_hash + line + '\n'); + reconstructed.overlap_hash = sha256(line + '\n'); + reconstructed.log_bytes += line.size() + 1; + ++reconstructed.message_index; + remember(event); + }); + } + if (!same_state(reconstructed, cursor_)) throw Error(21, "durable journal prefix does not match its cursor"); + // Only a verified journal is changed: the open segment's uncommitted tail goes last. + if (::truncate(segment_path(cursor_.segment).c_str(), static_cast(cursor_.log_bytes - segments_.back().bytes)) != 0) throw Error(22, "cannot roll back an uncommitted journal tail"); config_.start = cursor_.start; + durable_ = cursor_; } catch (const Error&) { throw; } + catch (const Stopped&) { throw; } + catch (const std::filesystem::filesystem_error&) { throw Error(22, "journal I/O failed during recovery"); } catch (const std::exception&) { throw Error(21, "invalid durable cursor or journal"); } } @@ -212,9 +369,7 @@ void State::stage(const std::string& line, const std::optional& proof) { next.proofs.push_back(*proof); if (next.proofs.size() > 32) next.proofs.pop_front(); } - if (next.message_index == std::numeric_limits::max() || - line.size() + 1 > config_.max_log_bytes - std::min(cursor_.log_bytes, config_.max_log_bytes)) - throw Error(22, "retained replay budget exhausted"); + if (next.message_index == std::numeric_limits::max()) throw Error(22, "message index exhausted"); next.log_bytes += line.size() + 1; ++next.message_index; next.prefix_hash = sha256(cursor_.prefix_hash + line + '\n'); @@ -232,58 +387,162 @@ std::vector State::flush() { if (unusable_) throw Error(22, "durable journal is unusable after a failed commit"); unusable_ = true; write_all(journal_.get(), staged_bytes_); + crash_point("commit-append"); if (::fsync(journal_.get()) != 0) throw Error(22, "cannot sync normalized messages"); persist(cursor_); + crash_point("commit-cursor"); unusable_ = false; durable_ = cursor_; staged_bytes_.clear(); - return std::exchange(staged_, {}); + auto lines = std::exchange(staged_, {}); + // The lines are committed; a failed seal or expiry leaves files only resume may reconcile. + try { + if (durable_.log_bytes - segments_.back().bytes >= config_.segment_bytes) rotate(); + retain(); + } catch (...) { + unusable_ = true; + throw; + } + return lines; +} + +// Seal the open segment: its successor's checkpoint first, then the empty successor, then the cursor that +// names it. A crash before the cursor leaves files recovery removes; the old segment stays open. +void State::rotate() { + const auto base = durable_.message_index; + if (!staged_.empty() || base == segments_.back().base) return; + write_checkpoint(durable_); + crash_point("rotate-checkpoint"); + Descriptor next(::open(segment_path(base).c_str(), O_WRONLY | O_APPEND | O_CREAT | O_EXCL | O_CLOEXEC, 0600)); + if (next.get() < 0 || ::fsync(next.get()) != 0) throw Error(22, "cannot create the next journal segment"); + sync_directory(journal_dir()); + crash_point("rotate-segment"); + auto rotated = durable_; + rotated.segment = base; + persist(rotated); + crash_point("rotate-cursor"); + durable_.segment = cursor_.segment = base; + journal_.reset(next.release()); + segments_.push_back({base, durable_.log_bytes, durable_.seq, durable_.cut, durable_.last_tick_ts}); + log("info", "journal_segment_sealed", Json::object({{"next_segment", number(base)}, {"segments", number(segments_.size())}})); +} + +// Expire the oldest sealed segments over the byte or venue-time budget. The open segment and every segment +// that can hold a message of the protected window (the open minute and the closed-minute proofs) stay. +void State::retain() { + const auto protected_start = durable_.proofs.empty() ? durable_.cut : std::min(durable_.proofs.front().ts, durable_.cut); + while (segments_.size() > 1) { + const auto oldest = segments_[0]; + const auto& next = segments_[1]; + if (next.cut >= protected_start) break; + // The segment holding the newest print stays: a reconnect re-reads the recent prints from the journal. + if (tick_mode(config_.mode) && next.seq == durable_.seq && oldest.seq < durable_.seq) break; + const bool bytes = durable_.log_bytes - oldest.bytes > config_.replay_bytes; + const bool age = config_.replay_age_ms && durable_.cut - next.cut > config_.replay_age_ms; + if (!bytes && !age) break; + if (::unlink(segment_path(oldest.base).c_str()) != 0) throw Error(22, "cannot expire a journal segment"); + segments_.erase(segments_.begin()); + sync_directory(journal_dir()); + crash_point("retain-segment"); + if (::unlink(checkpoint_path(oldest.base).c_str()) != 0 && errno != ENOENT) throw Error(22, "cannot expire a journal checkpoint"); + sync_directory(journal_dir()); + log("info", "journal_segment_expired", Json::object({{"segment", number(oldest.base)}, + {"first_retained", number(segments_.front().base)}, {"reason", Json::string(bytes ? "bytes" : "age")}})); + } + const bool over = durable_.log_bytes - segments_.front().bytes > config_.replay_bytes; + if (over && !over_budget_) + log("warn", "replay_budget_held_by_protected_window", Json::object({{"retained_bytes", + number(durable_.log_bytes - segments_.front().bytes)}, {"replay_bytes", number(config_.replay_bytes)}})); + over_budget_ = over; +} + +std::size_t State::segment_for(std::uint64_t index) const { + const auto found = std::upper_bound(segments_.begin(), segments_.end(), index, + [](std::uint64_t value, const Segment& segment) { return value < segment.base; }); + return found == segments_.begin() ? 0 : static_cast(found - segments_.begin() - 1); +} + +// Ticks of the first retained segment's open minute can sit in the expired segment before it: that minute +// is expired only when its checkpoint shows a tick of it. +bool State::expired_minute(std::int64_t minute) const { + const auto& first = segments_.front(); + return first_retained() && + (minute < first.cut || (tick_mode(config_.mode) && minute == first.cut && first.last_tick_ts >= first.cut)); +} + +std::uint64_t State::index_for_minute(std::int64_t minute) const { + std::size_t chosen = 0; + for (std::size_t index = 0; index < segments_.size(); ++index) + if (segments_[index].cut < minute) chosen = index; + return segments_[chosen].base; } void State::visit(std::uint64_t from, const std::function& visitor) const { + scan(from, [&](const std::string& line) { visitor(line); return true; }); +} + +void State::scan(std::uint64_t from, const std::function& visitor) const { if (from > cursor_.message_index) throw Error(22, "replay cursor is unavailable"); - std::ifstream input(log_path(), std::ios::binary); - if (!input) throw Error(22, "cannot read retained normalized log"); - std::uint64_t index = 0, bytes = 0; + if (from < first_retained()) throw Error(22, "replay cursor has expired from the retained journal"); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(config_.max_replay_seconds); - std::string line; - while (bytes < durable_.log_bytes) { - if (stopping) throw Stopped{}; - if (std::chrono::steady_clock::now() >= deadline) throw Error(22, "retained prefix replay time budget exhausted"); - if (!std::getline(input, line) || line.size() > 4095 || bytes + line.size() + 1 > durable_.log_bytes) - throw Error(21, "truncated or oversized verified message"); - bytes += line.size() + 1; - if (index >= from) visitor(line); - ++index; - } - if (bytes != durable_.log_bytes || index != durable_.message_index) throw Error(21, "verified message partition changed"); - for (const auto& staged : staged_) { - if (index >= from) visitor(staged); - ++index; - } + struct Enough {}; + std::uint64_t index = durable_.message_index; + try { + for (auto position = segment_for(from); position < segments_.size() && from < durable_.message_index; ++position) { + const auto& segment = segments_[position]; + const bool last = position + 1 == segments_.size(); + const auto end_bytes = last ? durable_.log_bytes : segments_[position + 1].bytes; + const auto end_index = last ? durable_.message_index : segments_[position + 1].base; + index = segment.base; + read_lines(segment_path(segment.base), end_bytes - segment.bytes, deadline, [&](const std::string& line) { + if (index >= from && !visitor(line)) throw Enough{}; + ++index; + }); + if (index != end_index) throw Error(21, "verified message partition changed"); + } + index = durable_.message_index; + for (const auto& staged : staged_) { + if (index >= from && !visitor(staged)) return; + ++index; + } + } catch (const Enough&) {} } -std::optional State::trade(std::uint64_t id) const { +std::optional State::recent_trade(std::uint64_t id) const { for (const auto& trade : recent_trades_) if (trade.id == id) return trade; + return std::nullopt; +} +std::optional State::trade(std::uint64_t id) const { + if (const auto recent = recent_trade(id)) return recent; + std::size_t chosen = 0; + for (std::size_t index = 0; index < segments_.size(); ++index) + if (segments_[index].seq < id) chosen = index; std::optional found; - visit(0, [&](const std::string& line) { + scan(segments_[chosen].base, [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() == "tick" && event.at("seq").integer() == id) found = normalized_trade(event); + return !found; }); return found; } +std::optional State::proof(std::int64_t ts) const { + for (const auto& bar : cursor_.proofs) if (bar.ts == ts) return bar; + return std::nullopt; +} std::optional State::bar(std::int64_t ts) const { for (const auto& bar : recent_bars_) if (bar.ts == ts) return bar; - for (const auto& bar : cursor_.proofs) if (bar.ts == ts) return bar; + if (const auto proven = proof(ts)) return proven; std::optional found; - visit(0, [&](const std::string& line) { + scan(index_for_minute(ts), [&](const std::string& line) { const auto event = parse_json(line); if (event.at("type").text() == "bar" && event.at("bar").at("ts_open").integer() == ts) found = normalized_bar(event); + return !found; }); return found; } Json State::status() const { return Json::object({{"epoch", Json::string(durable_.epoch)}, {"message_index", number(durable_.message_index)}, + {"first_retained", number(first_retained())}, {"segment", number(durable_.segment)}, {"emitted_seq", number(durable_.seq)}, {"verified_cut", signed_number(durable_.cut)}, {"prefix_hash", Json::string(durable_.prefix_hash)}, {"cursor", Json::string(config_.state_dir + "/cursor.json")}}); } diff --git a/native/feed/src/state.hpp b/native/feed/src/state.hpp index a7ff0e3..65fc385 100644 --- a/native/feed/src/state.hpp +++ b/native/feed/src/state.hpp @@ -15,6 +15,7 @@ class Descriptor { Descriptor(const Descriptor&) = delete; Descriptor& operator=(const Descriptor&) = delete; int get() const { return value_; } + int release() { const auto value = value_; value_ = -1; return value; } void reset(int value); }; @@ -24,36 +25,70 @@ struct Cursor { std::int64_t start = 0, cut = 0, last_tick_ts = -1, last_bar = -1; std::optional predecessor; std::deque proofs; + std::uint64_t segment = 0; // base message index of the open journal segment }; +// One retained journal segment: messages [base, next segment's base), its checkpoint holding the verified +// state before its first message. `bytes` and `cut` and `seq` are that state's absolute log bytes, verified +// cut and emitted sequence. +struct Segment { + std::uint64_t base = 0, bytes = 0, seq = 0; + std::int64_t cut = 0, last_tick_ts = -1; +}; +std::string segment_name(std::uint64_t base); + // Messages are staged (verified, cursor advanced in memory), then group-committed: one journal append and -// fsync, one cursor replacement, and only then returned for publication. +// fsync, one cursor replacement, and only then returned for publication. The journal is a sequence of +// segments under `journal/`; a full segment is sealed and a new one opened, and sealed segments expire by +// bytes and by venue time, except those holding the protected window (the open minute and the closed-minute +// proofs the reconnect overlap re-verifies). class State { Config config_; Cursor durable_, cursor_; Descriptor lock_, journal_; + std::vector segments_; std::deque recent_trades_; std::deque recent_bars_; std::vector staged_; std::string staged_bytes_; - bool unusable_ = false; - std::string log_path() const; + bool unusable_ = false, over_budget_ = false; + std::string journal_dir() const; + std::string segment_path(std::uint64_t base) const; + std::string checkpoint_path(std::uint64_t base) const; Json serialize(const Cursor& cursor) const; void persist(const Cursor& cursor) const; + void write_checkpoint(const Cursor& cursor) const; + Cursor read_checkpoint(std::uint64_t base) const; + void initialize(); void recover(); void remember(const Json& event); + void rotate(); + void retain(); + std::size_t segment_for(std::uint64_t index) const; + // visit() that stops as soon as `visitor` returns false. + void scan(std::uint64_t from, const std::function& visitor) const; public: explicit State(const Config& config); const Cursor& cursor() const { return cursor_; } const Cursor& durable() const { return durable_; } const Config& config() const { return config_; } std::size_t staged() const { return staged_.size(); } + const std::vector& segments() const { return segments_; } + std::uint64_t first_retained() const { return segments_.front().base; } + // Whether retention has dropped (part of) this minute or this print, so an old duplicate can no longer be + // compared with what was emitted. + bool expired_minute(std::int64_t minute) const; + bool expired_seq(std::uint64_t id) const { return first_retained() && id <= segments_.front().seq; } + // The first retained message index from which every message of `minute` (and later) is visited. + std::uint64_t index_for_minute(std::int64_t minute) const; void anchor(const Trade& predecessor); void stage(const std::string& line, const std::optional& proof = std::nullopt); std::vector flush(); void visit(std::uint64_t from, const std::function& visitor) const; std::optional trade(std::uint64_t id) const; + std::optional recent_trade(std::uint64_t id) const; std::optional bar(std::int64_t ts) const; + std::optional proof(std::int64_t ts) const; const std::deque& recent_trades() const { return recent_trades_; } const std::deque& recent_bars() const { return recent_bars_; } Json status() const; diff --git a/native/feed/src/transport.cpp b/native/feed/src/transport.cpp index 7a1997d..a0fd3c3 100644 --- a/native/feed/src/transport.cpp +++ b/native/feed/src/transport.cpp @@ -126,6 +126,14 @@ std::uint64_t retry_after_seconds(const std::string& value, std::int64_t now) { return when > now ? static_cast(when - now) : 0; } +std::uint64_t quota_wait_ms(std::uint64_t used, std::uint64_t used_window, unsigned int weight, std::uint64_t limit, std::int64_t now_ms) { + if (!limit || now_ms < 0) return 0; + const auto window = static_cast(now_ms) / 60000; + const auto spent = window == used_window ? used : 0; + if (spent + weight <= limit * 9 / 10) return 0; + return (window + 1) * 60000 + 500 - static_cast(now_ms); +} + void check_runtime_curl() { static GlobalCurl initialization; (void)initialization; @@ -192,17 +200,19 @@ HttpClient::HttpClient(const Config& config, RestPolicy policy) : config_(config interval == "HOUR" ? 3600000ULL : interval == "DAY" ? 86400000ULL : 0ULL; if (!unit || !multiple || multiple > 10000 || !ceiling || ceiling > 1000000000) throw Error(23, "unsupported public rate-limit interval or ceiling"); - // Spend at most half of each published ceiling (about 50% headroom). - const auto pace = (unit * multiple * 2 + ceiling - 1) / ceiling; + // The one-minute weight ceiling is spent by the venue's own count (X-MBX-USED-WEIGHT-1M, shared by + // every client on this IP); any other ceiling is paced evenly across its interval. + const auto pace = (unit * multiple + ceiling - 1) / ceiling; if (type == "REQUEST_WEIGHT") { weighted = true; - milliseconds_per_weight_ = std::max(milliseconds_per_weight_, pace); if (interval == "MINUTE" && multiple == 1) weight_limit_ = ceiling; + else milliseconds_per_weight_ = std::max(milliseconds_per_weight_, pace); } else milliseconds_per_request_ = std::max(milliseconds_per_request_, pace); } if (!weighted) throw Error(23, "public request-weight ceiling is unavailable"); - log("info", "rest_limits_verified", Json::object({{"milliseconds_per_weight", Json::number(std::to_string(milliseconds_per_weight_))}, - {"weight_limit_1m", Json::number(std::to_string(weight_limit_))}})); + log("info", "rest_limits_verified", Json::object({{"weight_limit_1m", Json::number(std::to_string(weight_limit_))}, + {"milliseconds_per_weight", Json::number(std::to_string(milliseconds_per_weight_))}, + {"milliseconds_per_request", Json::number(std::to_string(milliseconds_per_request_))}})); } catch (const Error&) { throw; } catch (const std::exception&) { throw Error(23, "invalid public exchange rate-limit metadata"); } } @@ -218,8 +228,18 @@ std::string HttpClient::body(const std::string& path, unsigned int weight, std:: if (!limit || limit > 4 * 1024 * 1024) throw Error(22, "REST body bound exceeds 4 MiB"); for (unsigned int attempt = 0; attempt < 4; ++attempt) { pause_until(next_request_); + const auto wait = quota_wait_ms(used_weight_, used_window_, weight, weight_limit_, std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + if (wait && !config_.allow_insecure) { + log("warn", "rest_quota_wait", Json::object({{"used_weight_1m", Json::number(std::to_string(used_weight_))}, + {"weight_limit_1m", Json::number(std::to_string(weight_limit_))}, {"wait_ms", Json::number(std::to_string(wait))}})); + pause_for(std::chrono::milliseconds(wait)); + } Progress state; auto handle = handle_for(config_.rest_url + path, false, config_.allow_insecure, &state); + // The venue counts a request in the window it was sent in, not the one its response arrives in. + const auto sent_window = static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()) / 60000; Response response; response.limit = limit; option(handle.get(), CURLOPT_WRITEFUNCTION, &body_callback); @@ -257,12 +277,10 @@ std::string HttpClient::body(const std::string& path, unsigned int weight, std:: continue; } if (status != 200) throw Error(23, "public market-data request rejected (HTTP " + std::to_string(status) + ")"); - if (response.used_weight) log("info", "rest_weight", Json::object({{"used_weight_1m", Json::number(std::to_string(response.used_weight))}})); - // The venue's used-weight header covers every client on this IP: past half the ceiling, wait for the window. - if (weight_limit_ && response.used_weight >= weight_limit_ / 2) { - const auto elapsed = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count() % 60000; - next_request_ = std::max(next_request_, std::chrono::steady_clock::now() + std::chrono::milliseconds(60500 - elapsed)); - log("warn", "rest_shared_quota_pause"); + if (response.used_weight) { + used_weight_ = response.used_weight; + used_window_ = sent_window; + log("info", "rest_weight", Json::object({{"used_weight_1m", Json::number(std::to_string(response.used_weight))}})); } return std::move(response.body); } @@ -283,13 +301,27 @@ WebSocketPump::~WebSocketPump() { ready_.notify_all(); if (worker_.joinable()) worker_.join(); } -void WebSocketPump::push(SourceMessage message) { +// False when the message does not fit the queue now; one message larger than the whole budget stops 22. +bool WebSocketPump::push(SourceMessage message) { std::lock_guard guard(mutex_); const auto bytes = message.text.size() + 32; - if (bytes > config_.max_queue_bytes - std::min(queue_bytes_, config_.max_queue_bytes)) throw Error(22, "WebSocket healing buffer overrun"); + if (bytes > config_.max_queue_bytes) throw Error(22, "one source message exceeds --max-queue-bytes"); + if (bytes > config_.max_queue_bytes - std::min(queue_bytes_, config_.max_queue_bytes)) return false; queue_bytes_ += bytes; queue_.push_back(std::move(message)); ready_.notify_one(); + return true; +} +void WebSocketPump::wait_for_room(std::size_t bytes) { + if (bytes > config_.max_queue_bytes) throw Error(22, "--max-queue-bytes is too small for one source message"); + for (;;) { + { + std::lock_guard guard(mutex_); + if (queue_bytes_ + bytes <= config_.max_queue_bytes) return; + } + if (stopped()) return; + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } } bool WebSocketPump::try_take(SourceMessage& message) { std::lock_guard guard(mutex_); @@ -344,20 +376,34 @@ void WebSocketPump::run() { curl_easy_getinfo(handle.get(), CURLINFO_ACTIVESOCKET, &socket); // The marker precedes every message of this connection: the session verifies its REST // overlap while the subscribed data is buffered behind it. - push({true, {}}); + wait_for_room(32); + if (stopped()) return; + if (!push({true, {}})) throw Error(22, "WebSocket queue cannot take the connection marker"); bool subscribed = true; for (const auto& request : connection_.subscribe) subscribed = subscribed && send_text(handle.get(), socket, request); log(subscribed ? "info" : "warn", subscribed ? "websocket_connected" : "websocket_subscribe_failed"); const auto birth = std::chrono::steady_clock::now(); - auto last_data = birth, message_birth = birth, last_ping = birth; + auto last_data = birth, message_birth = birth, last_ping = birth, probe_sent = birth; std::string message; - bool assembling = false, heard = false; + bool assembling = false, heard = false, probing = false, backpressure = false; std::uint64_t frame_offset = 0; while (subscribed && !stopped()) { const auto now = std::chrono::steady_clock::now(); - // Keepalive replies are control messages: they cannot hide a silent data stream. - if (now - birth >= std::chrono::seconds(config_.reconnect_seconds) || now - last_data > std::chrono::seconds(75) || - (assembling && now - message_birth > std::chrono::seconds(30))) break; + if (now - birth >= std::chrono::seconds(config_.reconnect_seconds) || (assembling && now - message_birth > std::chrono::seconds(30))) break; + // Keepalive replies are control messages: they cannot hide a silent data stream. A venue with a + // probe proves the connection and its subscriptions instead of paying a reconnect; completeness + // never rests on the connection (chain IDs, watermarks and the reconnect overlap prove it). + if (now - last_data > std::chrono::seconds(config_.silence_seconds)) { + if (connection_.probe.empty()) break; + if (!probing) { + if (!send_text(handle.get(), socket, connection_.probe)) break; + probing = true; + probe_sent = now; + } else if (now - probe_sent > std::chrono::seconds(10)) { + log("warn", "source_probe_unanswered"); + break; + } + } if (!connection_.ping.empty() && now - last_ping >= std::chrono::seconds(config_.keepalive_seconds)) { if (!send_text(handle.get(), socket, connection_.ping)) break; last_ping = now; @@ -387,12 +433,28 @@ void WebSocketPump::run() { assembling = false; // A connection counts as established only once the venue answers on it. if (!heard) { heard = true; failures = 0; } - if (kind == Frame::Control) { message.clear(); continue; } + if (kind == Frame::Stale) { log("warn", "source_subscription_lost"); break; } + if (kind == Frame::Control) { + // A probe reply restarts the silence clock: the connection and subscriptions are alive. + if (probing) { probing = false; last_data = now; log("info", "source_probe_answered"); } + message.clear(); + continue; + } last_data = now; - push({false, std::move(message)}); + probing = false; + // A full queue (the session is busy healing a long outage) drops this connection instead of + // stopping: everything from here on is healed from REST after the reconnect's overlap check. + if (!push({false, std::move(message)})) { backpressure = true; break; } message.clear(); if (kind == Frame::Retire) { log("info", "source_retiring_reconnect"); break; } } + if (backpressure && !stopped()) { + log("warn", "websocket_backpressure_reconnect", Json::object({{"max_queue_bytes", Json::number(std::to_string(config_.max_queue_bytes))}})); + handle.reset(); + wait_for_room(config_.max_queue_bytes - config_.max_queue_bytes / 4); + next_attempt = std::chrono::steady_clock::now(); + continue; + } if (!stopped()) { log("warn", "websocket_reconnect"); std::this_thread::sleep_for(std::chrono::milliseconds(250)); diff --git a/native/feed/src/transport.hpp b/native/feed/src/transport.hpp index d4b41c2..bb37eaa 100644 --- a/native/feed/src/transport.hpp +++ b/native/feed/src/transport.hpp @@ -15,6 +15,10 @@ void check_runtime_curl(); void validate_origin(const std::string& origin, bool websocket, bool allow_insecure); void pause_for(std::chrono::milliseconds duration); std::uint64_t retry_after_seconds(const std::string& value, std::int64_t now); +// Milliseconds to wait before a request of `weight`: none while the venue-reported use of the current +// one-minute window (`used`, reported in window `used_window`) leaves room under 90% of `limit`; otherwise +// until the next window opens (500 ms margin). +std::uint64_t quota_wait_ms(std::uint64_t used, std::uint64_t used_window, unsigned int weight, std::uint64_t limit, std::int64_t now_ms); // A venue's public REST contract: allowlisted paths and how requests are paced. struct RestPolicy { @@ -38,6 +42,7 @@ class HttpClient { RestPolicy policy_; std::chrono::steady_clock::time_point next_request_{}; std::uint64_t weight_limit_ = 0, milliseconds_per_weight_ = 0, milliseconds_per_request_ = 0; + std::uint64_t used_weight_ = 0, used_window_ = 0; // the venue's X-MBX-USED-WEIGHT-1M and its minute public: HttpClient(const Config& config, RestPolicy policy); std::string body(const std::string& path, unsigned int weight, std::size_t limit = 1024 * 1024); @@ -59,7 +64,8 @@ class WebSocketPump { std::exception_ptr failure_; std::thread worker_; bool stopped() const; - void push(SourceMessage message); + bool push(SourceMessage message); + void wait_for_room(std::size_t bytes); void run(); public: WebSocketPump(const Config& config, Connection connection); diff --git a/native/feed/src/venue.hpp b/native/feed/src/venue.hpp index f1a25ff..3720377 100644 --- a/native/feed/src/venue.hpp +++ b/native/feed/src/venue.hpp @@ -13,14 +13,17 @@ struct VenueEvent { Kline kline; }; // How the WebSocket reader treats one complete text message. Control messages (subscription -// acknowledgements, keepalive replies) prove liveness but are not queued; Retire queues the message, -// then reconnects. -enum class Frame { Data, Control, Retire }; +// acknowledgements, keepalive and probe replies) prove liveness but are not queued; Retire queues the +// message, then reconnects; Stale (a probe reply without the subscriptions) reconnects without queuing. +enum class Frame { Data, Control, Retire, Stale }; struct Connection { std::string path; // appended to the WebSocket origin std::vector subscribe; // text frames sent after every handshake std::string ping; // venue text keepalive; empty when curl's PONG suffices std::function classify; // runs on the reader thread: no shared state + // A request whose Control reply proves the connection and its subscriptions are alive (Binance + // LIST_SUBSCRIPTIONS): a quiet stream is probed instead of reconnected. Empty: silence reconnects. + std::string probe; }; // How a tick minute is proven complete before its `time` event. // KlineIdRange: the closed kline names the minute's raw-ID range and count (Binance spot). @@ -46,13 +49,14 @@ class Venue { return previous.front(); } // The same predecessor, but only once the venue holds a print at or after `minute`: nullopt while it - // holds none, without waiting or stopping (a quiet start). Fence venues override it. - virtual std::optional predecessor_if_ready(std::int64_t minute) { - try { return predecessor(minute); } - catch (const Error& failure) { if (failure.code == 20) return std::nullopt; throw; } - } + // holds none, without waiting or stopping (a quiet start). Every venue states its own answer: a default + // that mapped stops to "not ready" would wait silently on an unavailable predecessor. + virtual std::optional predecessor_if_ready(std::int64_t minute) = 0; // Re-checks instrument metadata that fixes the stream's units; a change stops 21. Called on connect. virtual void reverify() {} + // The newest venue time the caller knows of without a WebSocket (export passes the clock): bounds a + // start lookup that searches forward through time windows (USD-M). + virtual void horizon(std::int64_t) {} // Contiguous closed one-minute candles in [start, end), ascending. virtual std::vector klines(std::int64_t start, std::int64_t end) = 0; virtual TickProof tick_proof() const { return TickProof::KlineIdRange; } diff --git a/native/feed/tests/batch_probe.cpp b/native/feed/tests/batch_probe.cpp index 6dfee52..03b6b4e 100644 --- a/native/feed/tests/batch_probe.cpp +++ b/native/feed/tests/batch_probe.cpp @@ -16,7 +16,8 @@ template Function load(void* library, const char* name) { int main(int argc, char** argv) { try { - if (argc != 4) throw std::runtime_error("usage: batch-probe observed-strategy.so bars.csv actions.jsonl"); + if (argc != 4 && argc != 5) + throw std::runtime_error("usage: batch-probe observed-strategy.so bars.csv actions.jsonl [qty_step]"); std::ifstream input(argv[2]); if (!input) throw std::runtime_error("cannot open batch input"); std::string line; @@ -45,6 +46,9 @@ int main(int argc, char** argv) { auto error = load(library, "strategy_get_last_error"); auto strategy = create(nullptr); if (!strategy || retain(strategy)) throw std::runtime_error("cannot retain batch execution receipts"); + // The runner's --syminfo qty_step=V, read the same way (std::stod). + if (argc == 5) load(library, "strategy_set_syminfo_metadata")( + strategy, "qty_step", std::stod(argv[4])); pf_report_t report{}; run(strategy, bars.data(), static_cast(bars.size()), "1", "1", 0, 4, PF_MAGNIFIER_ENDPOINTS, &report); const auto* failure = error(strategy); diff --git a/native/feed/tests/export_unit.cpp b/native/feed/tests/export_unit.cpp new file mode 100644 index 0000000..b2357e0 --- /dev/null +++ b/native/feed/tests/export_unit.cpp @@ -0,0 +1,507 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "export.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge::feed; +namespace { +// Made offline with Python's zipfile and zlib from the synthetic rows below: a deflated (dynamic-Huffman) and a +// stored daily archive, and one raw deflate stream of generated() in a fixed, a stored and a dynamic block. +// 362 bytes +const unsigned char deflated_zip[] = { + 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x41, 0x5d, 0xdf, 0xdf, 0x4a, 0xd9, + 0xc6, 0x00, 0x00, 0x00, 0x99, 0x01, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x54, 0x45, 0x53, 0x54, 0x55, 0x53, + 0x44, 0x54, 0x2d, 0x61, 0x67, 0x67, 0x54, 0x72, 0x61, 0x64, 0x65, 0x73, 0x2d, 0x31, 0x39, 0x37, 0x30, 0x2d, + 0x30, 0x31, 0x2d, 0x30, 0x31, 0x2e, 0x63, 0x73, 0x76, 0x4d, 0x8e, 0xd1, 0x0e, 0x82, 0x30, 0x0c, 0x45, 0xdf, + 0xfd, 0x96, 0x86, 0xb4, 0x63, 0x03, 0xfb, 0x35, 0x64, 0xea, 0x20, 0x8b, 0x4a, 0x74, 0x8c, 0x07, 0xff, 0xde, + 0x76, 0x30, 0xe5, 0x61, 0x77, 0x6d, 0xef, 0xc9, 0x6d, 0xfd, 0x34, 0x0d, 0x39, 0xf9, 0x5b, 0x18, 0xe2, 0x0d, + 0x5e, 0x29, 0x5e, 0x03, 0xbc, 0x57, 0x3f, 0xe7, 0x98, 0x3f, 0x30, 0xc6, 0xb4, 0xe4, 0xbf, 0xfb, 0xf0, 0xc7, + 0x4e, 0x8a, 0x79, 0xf1, 0x57, 0x99, 0xc4, 0x67, 0x80, 0xb8, 0x0c, 0x97, 0xf5, 0x13, 0xd2, 0xf0, 0xf4, 0xf7, + 0x90, 0x4e, 0x96, 0xcf, 0x40, 0xd8, 0x20, 0x02, 0x36, 0x0e, 0x98, 0x3b, 0x79, 0x3d, 0x10, 0x31, 0xca, 0x28, + 0xa7, 0x35, 0x08, 0xc1, 0x4a, 0x90, 0x12, 0x24, 0xee, 0x59, 0x1e, 0x2b, 0xa1, 0x5f, 0x21, 0x9c, 0xa0, 0x7f, + 0x82, 0xb0, 0xb4, 0x68, 0x81, 0x8c, 0x94, 0x04, 0xa3, 0x7f, 0x2c, 0x0a, 0x89, 0x45, 0x8d, 0x51, 0xc8, 0xa8, + 0xef, 0x76, 0x11, 0xc8, 0x61, 0x0d, 0x32, 0xc0, 0x0d, 0x2b, 0xd2, 0xaa, 0xdb, 0xa9, 0xc8, 0x7d, 0x7d, 0xd9, + 0x55, 0x73, 0xc4, 0x32, 0xdb, 0xc1, 0x56, 0x7d, 0xde, 0xc4, 0xd8, 0xb2, 0x6c, 0xcf, 0xb1, 0xba, 0x0b, 0xeb, + 0x41, 0x84, 0x9b, 0x28, 0x23, 0xbb, 0x6a, 0x90, 0xfb, 0x05, 0x15, 0x68, 0x97, 0x16, 0x8f, 0x41, 0x9d, 0x32, + 0x54, 0x8f, 0xa6, 0x5d, 0x94, 0xf9, 0x05, 0x7d, 0x01, 0x50, 0x4b, 0x01, 0x02, 0x14, 0x03, 0x14, 0x00, 0x00, + 0x00, 0x08, 0x00, 0x00, 0x00, 0x41, 0x5d, 0xdf, 0xdf, 0x4a, 0xd9, 0xc6, 0x00, 0x00, 0x00, 0x99, 0x01, 0x00, + 0x00, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x54, 0x45, 0x53, 0x54, 0x55, 0x53, 0x44, 0x54, 0x2d, 0x61, 0x67, 0x67, 0x54, 0x72, 0x61, 0x64, 0x65, + 0x73, 0x2d, 0x31, 0x39, 0x37, 0x30, 0x2d, 0x30, 0x31, 0x2d, 0x30, 0x31, 0x2e, 0x63, 0x73, 0x76, 0x50, 0x4b, + 0x05, 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x05, 0x01, 0x00, 0x00, + 0x00, 0x00, +}; +// 487 bytes +const unsigned char stored_zip[] = { + 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x5d, 0x42, 0x8a, 0xb2, 0xcf, + 0x43, 0x01, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x54, 0x45, 0x53, 0x54, 0x55, 0x53, + 0x44, 0x54, 0x2d, 0x61, 0x67, 0x67, 0x54, 0x72, 0x61, 0x64, 0x65, 0x73, 0x2d, 0x31, 0x39, 0x37, 0x30, 0x2d, + 0x30, 0x31, 0x2d, 0x30, 0x31, 0x2e, 0x63, 0x73, 0x76, 0x34, 0x39, 0x38, 0x2c, 0x31, 0x30, 0x2e, 0x30, 0x30, + 0x2c, 0x30, 0x2e, 0x35, 0x2c, 0x39, 0x39, 0x36, 0x2c, 0x39, 0x39, 0x37, 0x2c, 0x31, 0x31, 0x39, 0x30, 0x30, + 0x30, 0x2c, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x34, 0x39, 0x39, 0x2c, 0x31, 0x30, 0x2e, 0x31, 0x30, 0x2c, 0x30, + 0x2e, 0x31, 0x2c, 0x39, 0x39, 0x38, 0x2c, 0x39, 0x39, 0x39, 0x2c, 0x31, 0x31, 0x39, 0x39, 0x39, 0x39, 0x2c, + 0x74, 0x72, 0x75, 0x65, 0x0a, 0x35, 0x30, 0x30, 0x2c, 0x31, 0x30, 0x2e, 0x31, 0x30, 0x2c, 0x30, 0x2e, 0x31, + 0x2c, 0x31, 0x30, 0x30, 0x30, 0x2c, 0x31, 0x30, 0x30, 0x34, 0x2c, 0x31, 0x32, 0x30, 0x30, 0x30, 0x31, 0x2c, + 0x66, 0x61, 0x6c, 0x73, 0x65, 0x0a, 0x35, 0x30, 0x31, 0x2c, 0x31, 0x31, 0x2e, 0x32, 0x30, 0x2c, 0x30, 0x2e, + 0x32, 0x2c, 0x31, 0x30, 0x30, 0x35, 0x2c, 0x31, 0x30, 0x30, 0x35, 0x2c, 0x31, 0x32, 0x30, 0x30, 0x35, 0x30, + 0x2c, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x35, 0x30, 0x32, 0x2c, 0x39, 0x2e, 0x39, 0x30, 0x2c, 0x30, 0x2e, 0x33, + 0x2c, 0x31, 0x30, 0x30, 0x36, 0x2c, 0x31, 0x30, 0x30, 0x38, 0x2c, 0x31, 0x37, 0x39, 0x39, 0x39, 0x39, 0x2c, + 0x66, 0x61, 0x6c, 0x73, 0x65, 0x0a, 0x35, 0x30, 0x33, 0x2c, 0x31, 0x32, 0x2e, 0x30, 0x30, 0x2c, 0x30, 0x2e, + 0x34, 0x2c, 0x31, 0x30, 0x30, 0x39, 0x2c, 0x31, 0x30, 0x30, 0x39, 0x2c, 0x32, 0x34, 0x30, 0x30, 0x30, 0x31, + 0x2c, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x35, 0x30, 0x34, 0x2c, 0x31, 0x31, 0x2e, 0x30, 0x30, 0x2c, 0x30, 0x2e, + 0x31, 0x2c, 0x31, 0x30, 0x31, 0x30, 0x2c, 0x31, 0x30, 0x31, 0x30, 0x2c, 0x32, 0x34, 0x30, 0x30, 0x35, 0x30, + 0x2c, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x0a, 0x35, 0x30, 0x35, 0x2c, 0x31, 0x32, 0x2e, 0x30, 0x30, 0x2c, 0x30, + 0x2e, 0x31, 0x2c, 0x31, 0x30, 0x31, 0x31, 0x2c, 0x31, 0x30, 0x31, 0x31, 0x2c, 0x33, 0x30, 0x30, 0x30, 0x30, + 0x31, 0x2c, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x35, 0x30, 0x36, 0x2c, 0x31, 0x32, 0x2e, 0x31, 0x30, 0x2c, 0x30, + 0x2e, 0x32, 0x2c, 0x31, 0x30, 0x31, 0x32, 0x2c, 0x31, 0x30, 0x31, 0x32, 0x2c, 0x33, 0x30, 0x30, 0x30, 0x35, + 0x30, 0x2c, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x0a, 0x50, 0x4b, 0x01, 0x02, 0x14, 0x03, 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x41, 0x5d, 0x42, 0x8a, 0xb2, 0xcf, 0x43, 0x01, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, + 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x54, 0x45, 0x53, 0x54, 0x55, 0x53, 0x44, 0x54, 0x2d, 0x61, 0x67, 0x67, 0x54, 0x72, 0x61, 0x64, 0x65, 0x73, + 0x2d, 0x31, 0x39, 0x37, 0x30, 0x2d, 0x30, 0x31, 0x2d, 0x30, 0x31, 0x2e, 0x63, 0x73, 0x76, 0x50, 0x4b, 0x05, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x4f, 0x00, 0x00, 0x00, 0x82, 0x01, 0x00, 0x00, 0x00, + 0x00, +}; +// 925 bytes +const unsigned char mixed_deflate[] = { + 0x32, 0x34, 0x30, 0x30, 0xd0, 0x31, 0x34, 0xd0, 0x03, 0x92, 0x20, 0xc2, 0x50, 0xc7, 0x14, 0x24, 0x60, 0x0a, + 0x62, 0x19, 0x1a, 0x19, 0x80, 0xd8, 0x69, 0x89, 0x39, 0xc5, 0xa9, 0x5c, 0x86, 0x60, 0x11, 0x73, 0x3d, 0x63, + 0x73, 0xb0, 0x3a, 0x23, 0x90, 0x12, 0x30, 0x61, 0x0c, 0x54, 0x67, 0x08, 0xa4, 0x75, 0x4a, 0x8a, 0x4a, 0xc1, + 0xca, 0x8c, 0x74, 0x0c, 0x0d, 0xf5, 0xcc, 0x4d, 0xc0, 0xca, 0x8c, 0x41, 0x2a, 0x4c, 0x40, 0x84, 0x29, 0x50, + 0x99, 0x31, 0x8a, 0x71, 0x40, 0x8d, 0x16, 0x7a, 0x86, 0x86, 0x60, 0x75, 0x60, 0x25, 0x66, 0x20, 0xc2, 0x1c, + 0xa8, 0xce, 0x04, 0xd9, 0x38, 0x13, 0xa0, 0x80, 0x9e, 0x89, 0x05, 0x58, 0x99, 0x29, 0x48, 0x85, 0x05, 0x88, + 0xb0, 0x04, 0x8a, 0x9a, 0xa1, 0x18, 0x07, 0xb4, 0xc0, 0x52, 0xcf, 0xc2, 0x14, 0xac, 0x0e, 0x64, 0x92, 0x21, + 0xc8, 0x17, 0x86, 0x20, 0x5f, 0x98, 0x23, 0x1b, 0x67, 0xa6, 0x63, 0x68, 0xac, 0x67, 0x64, 0x04, 0x56, 0x66, + 0x0e, 0x52, 0x01, 0xf2, 0x84, 0x21, 0xc8, 0x13, 0x96, 0x28, 0xc6, 0x99, 0xeb, 0x18, 0x19, 0xe8, 0x99, 0x5a, + 0x82, 0xd5, 0x81, 0x6c, 0x34, 0x04, 0x39, 0xd1, 0x10, 0x68, 0x89, 0xb1, 0x01, 0xb2, 0x71, 0x16, 0x3a, 0x86, + 0x26, 0x7a, 0x96, 0x66, 0x60, 0x65, 0x96, 0x20, 0x15, 0x60, 0xab, 0x81, 0x9e, 0x30, 0x36, 0x42, 0x31, 0xce, + 0x52, 0xc7, 0xc8, 0x50, 0xcf, 0xd8, 0x18, 0x1e, 0xc6, 0x86, 0x60, 0x33, 0x81, 0xbe, 0x30, 0x36, 0x46, 0x32, + 0x0e, 0xe8, 0x66, 0x43, 0x53, 0x3d, 0x73, 0x03, 0x78, 0x10, 0x1b, 0x81, 0x3c, 0x61, 0x04, 0xf4, 0x84, 0xb1, + 0x29, 0xb2, 0x71, 0x40, 0x6f, 0x19, 0x19, 0xe9, 0x19, 0x98, 0xc3, 0xc3, 0xd8, 0x08, 0xac, 0x18, 0xe8, 0x0b, + 0x63, 0x33, 0x64, 0xe3, 0x80, 0x51, 0x61, 0xa6, 0x67, 0x62, 0x02, 0x0f, 0x62, 0x23, 0x30, 0x01, 0xf2, 0x84, + 0x05, 0x8a, 0x71, 0xc6, 0xa0, 0x14, 0x60, 0x61, 0x08, 0x0f, 0x63, 0x23, 0x90, 0x2f, 0x8c, 0x40, 0xbe, 0xb0, + 0x44, 0x36, 0xce, 0x04, 0x94, 0x00, 0x0c, 0x2d, 0xe0, 0x41, 0x6c, 0x04, 0xf2, 0x84, 0x11, 0xd0, 0x13, 0x26, + 0x86, 0x28, 0xc6, 0x99, 0x82, 0x52, 0x80, 0xa9, 0x29, 0x3c, 0x8c, 0x8d, 0x41, 0xbe, 0x30, 0x06, 0xfa, 0xc2, + 0xc4, 0x08, 0xd9, 0x38, 0x33, 0x50, 0x02, 0xb0, 0x34, 0x82, 0x07, 0xb1, 0x31, 0xc8, 0x13, 0xc0, 0x30, 0x32, + 0x34, 0x31, 0x41, 0x31, 0x0e, 0x94, 0x24, 0xf4, 0x8c, 0x2c, 0xe1, 0x61, 0x6c, 0x0c, 0xf2, 0x85, 0x31, 0xd0, + 0x12, 0x13, 0x53, 0x64, 0xe3, 0x2c, 0x40, 0x09, 0xc0, 0xcc, 0x0c, 0x1e, 0xc4, 0xc6, 0x20, 0x27, 0x02, 0x93, + 0xab, 0xa1, 0x89, 0x39, 0x8a, 0x71, 0xa0, 0x40, 0xd7, 0x33, 0x30, 0x86, 0x87, 0xb1, 0x31, 0xd8, 0x6e, 0x90, + 0x2f, 0x2c, 0xe0, 0xc6, 0x01, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x9e, 0x00, 0x61, 0xff, 0x31, 0x30, 0x32, + 0x30, 0x2c, 0x32, 0x30, 0x2e, 0x34, 0x30, 0x2c, 0x30, 0x2e, 0x30, 0x30, 0x33, 0x2c, 0x35, 0x30, 0x34, 0x30, + 0x2c, 0x35, 0x30, 0x34, 0x31, 0x2c, 0x31, 0x35, 0x30, 0x30, 0x30, 0x30, 0x2c, 0x66, 0x61, 0x6c, 0x73, 0x65, + 0x0a, 0x31, 0x30, 0x32, 0x31, 0x2c, 0x31, 0x34, 0x2e, 0x37, 0x37, 0x2c, 0x30, 0x2e, 0x30, 0x30, 0x34, 0x2c, + 0x35, 0x30, 0x34, 0x32, 0x2c, 0x35, 0x30, 0x34, 0x33, 0x2c, 0x31, 0x35, 0x31, 0x35, 0x30, 0x30, 0x2c, 0x74, + 0x72, 0x75, 0x65, 0x0a, 0x31, 0x30, 0x32, 0x32, 0x2c, 0x32, 0x31, 0x2e, 0x31, 0x34, 0x2c, 0x30, 0x2e, 0x30, + 0x30, 0x35, 0x2c, 0x35, 0x30, 0x34, 0x34, 0x2c, 0x35, 0x30, 0x34, 0x35, 0x2c, 0x31, 0x35, 0x33, 0x30, 0x30, + 0x30, 0x2c, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x0a, 0x31, 0x30, 0x32, 0x33, 0x2c, 0x31, 0x35, 0x2e, 0x35, 0x31, + 0x2c, 0x30, 0x2e, 0x30, 0x30, 0x36, 0x2c, 0x35, 0x30, 0x34, 0x36, 0x2c, 0x35, 0x30, 0x34, 0x37, 0x2c, 0x31, + 0x35, 0x34, 0x35, 0x30, 0x30, 0x2c, 0x74, 0x72, 0x75, 0x65, 0x0a, 0x00, 0x00, 0x00, 0xff, 0xff, 0x55, 0x93, + 0x39, 0x92, 0x60, 0x21, 0x0c, 0x43, 0xf3, 0x3e, 0x0b, 0x45, 0xd9, 0xc6, 0x78, 0x39, 0x4e, 0x07, 0x3d, 0xd1, + 0x44, 0xb3, 0xdc, 0xbf, 0x2d, 0x07, 0x34, 0x24, 0x0e, 0x7e, 0xe9, 0x0b, 0x3f, 0x21, 0x98, 0x44, 0x87, 0xc8, + 0x8c, 0x18, 0x34, 0x89, 0x7c, 0x6c, 0xd2, 0xc0, 0xc8, 0xc1, 0xdb, 0x88, 0x68, 0xfc, 0xfa, 0xfc, 0xfd, 0xf7, + 0xeb, 0x83, 0x49, 0xf6, 0x60, 0x9b, 0x35, 0xa1, 0x83, 0x64, 0x13, 0x06, 0x97, 0xce, 0x77, 0xe9, 0xfe, 0xfd, + 0xf9, 0xdf, 0x32, 0x1b, 0x4c, 0xd3, 0xa4, 0x65, 0x09, 0x85, 0x60, 0xac, 0x92, 0xe5, 0x63, 0xe7, 0x83, 0x7d, + 0x66, 0xb6, 0x8e, 0x21, 0x51, 0x0c, 0x1c, 0x42, 0xb7, 0x5d, 0x0c, 0xe6, 0xb9, 0xac, 0x65, 0xed, 0x64, 0x18, + 0xf5, 0xb3, 0xc9, 0x63, 0x57, 0xfb, 0xc6, 0xf4, 0xd5, 0xba, 0x05, 0x49, 0xaf, 0x58, 0x5f, 0x6d, 0x5d, 0x76, + 0x8b, 0x06, 0xcb, 0x64, 0x6a, 0x19, 0x0e, 0x34, 0x40, 0x58, 0x41, 0xd8, 0xbe, 0xed, 0x56, 0x7d, 0xc9, 0xa9, + 0xde, 0xba, 0x0d, 0x09, 0xce, 0xb6, 0xa2, 0x30, 0xbb, 0xed, 0x64, 0xf0, 0x9a, 0xa1, 0x2d, 0xc3, 0x62, 0xd6, + 0x9e, 0x80, 0x88, 0xc7, 0x6e, 0x0d, 0xa1, 0x29, 0x7c, 0x32, 0xb6, 0x16, 0x83, 0x22, 0x6f, 0x3b, 0x1d, 0xac, + 0x73, 0xc7, 0x89, 0xd8, 0x7a, 0x14, 0x84, 0xf3, 0x63, 0xb7, 0x87, 0xf0, 0xcc, 0x7d, 0x32, 0x76, 0x50, 0x78, + 0xed, 0xec, 0x72, 0xdb, 0xd5, 0x55, 0xec, 0xb9, 0xe4, 0x44, 0xec, 0x80, 0xa8, 0x8c, 0xd8, 0xf5, 0xb1, 0x73, + 0x34, 0xc0, 0xf2, 0x64, 0xec, 0xa0, 0xf0, 0xa2, 0xf0, 0x7d, 0xdb, 0x05, 0x0a, 0x40, 0x76, 0x22, 0x76, 0x40, + 0x38, 0xee, 0xd1, 0x1f, 0xbb, 0x44, 0x03, 0x74, 0x9d, 0x8c, 0x1d, 0x14, 0x0e, 0x8a, 0xb8, 0xec, 0x94, 0x50, + 0x80, 0xa0, 0x13, 0x71, 0x00, 0x22, 0x0a, 0x02, 0xd1, 0xfd, 0xd8, 0x29, 0xa3, 0x01, 0xec, 0x27, 0xe3, 0xc0, + 0x8a, 0x51, 0x14, 0xc1, 0xb7, 0x9d, 0xa0, 0x00, 0x5b, 0x4f, 0xc4, 0x81, 0xa3, 0xa3, 0x20, 0x62, 0x3d, 0x76, + 0x0b, 0x0d, 0x48, 0x3e, 0x19, 0x47, 0x7b, 0x16, 0x45, 0xe8, 0x6d, 0xa7, 0x28, 0x80, 0xc4, 0x89, 0x38, 0x5a, + 0x8b, 0x96, 0x3d, 0xaf, 0x42, 0x37, 0x1a, 0x60, 0xfb, 0x64, 0x9c, 0xa0, 0x48, 0x50, 0xdc, 0xaf, 0x42, 0x0d, + 0x05, 0x20, 0x39, 0x11, 0x67, 0x0f, 0x40, 0x3c, 0xaf, 0xa2, 0x1a, 0x57, 0x0d, 0x58, 0x79, 0x32, 0x4e, 0x50, + 0xd4, 0x4d, 0x73, 0xfe, 0xbc, 0x8a, 0x6f, +}; + +struct Temporary { + std::string path; + Temporary() { + char pattern[] = "/tmp/pineforge-feed-export-XXXXXX"; + const auto* result = ::mkdtemp(pattern); + assert(result); + path = result; + } + ~Temporary() { std::filesystem::remove_all(path); } +}; +template void expect(int code, Function operation) { + bool caught = false; + try { operation(); } + catch (const Error& failure) { assert(failure.code == code); caught = true; } + assert(caught); +} +void passed(const char* name) { std::cout << "PASS " << name << '\n'; } +template std::string bytes(const unsigned char (&data)[N]) { return std::string(reinterpret_cast(data), N); } +void put_file(const std::string& path, const std::string& content) { + std::ofstream file(path, std::ios::binary); + file.write(content.data(), static_cast(content.size())); + assert(file); +} +std::string inflated(const std::string& compressed, std::size_t chunk = 4096, std::size_t* calls = nullptr) { + std::size_t offset = 0; + std::string result; + inflate([&](char* buffer, std::size_t limit) { + const auto count = std::min({chunk, limit, compressed.size() - offset}); + std::memcpy(buffer, compressed.data() + offset, count); + offset += count; + return count; + }, [&](std::string_view output) { + result.append(output); + if (calls) ++*calls; + }); + return result; +} +std::string generated() { + std::string text; + for (int index = 0; index < 48; ++index) { + char line[128]; + std::snprintf(line, sizeof(line), "%d,%d.%02d,0.%03d,%d,%d,%d,%s\n", 1000 + index, 10 + index * 7 % 13, index * 37 % 100, + index % 9 + 1, 5000 + 2 * index, 5001 + 2 * index, 120000 + 1500 * index, index % 2 ? "true" : "false"); + text += line; + } + return text; +} +// Deflate bits go least-significant first; Huffman codes are sent from their most significant bit. +struct Bits { + std::string bytes; + std::uint64_t buffer = 0; + unsigned int count = 0; + void put(std::uint32_t value, unsigned int width) { + buffer |= static_cast(value) << count; + for (count += width; count >= 8; count -= 8, buffer >>= 8) bytes += static_cast(buffer & 0xFF); + } + void huffman(std::uint32_t code, unsigned int width) { + std::uint32_t reversed = 0; + for (unsigned int bit = 0; bit < width; ++bit) reversed |= ((code >> bit) & 1U) << (width - 1 - bit); + put(reversed, width); + } + std::string finish() { + if (count) bytes += static_cast(buffer & 0xFF); + count = 0; + return bytes; + } +}; +const std::string bar_header = "timestamp,open,high,low,close,volume\n"; +const std::string archive_header = "agg_trade_id,price,quantity,first_trade_id,last_trade_id,transact_time,is_buyer_maker\n"; +const std::string rows = + "498,10.00,0.5,996,997,119000,true\n" + "499,10.10,0.1,998,999,119999,true\n" + "500,10.10,0.1,1000,1004,120001,false\n" + "501,11.20,0.2,1005,1005,120050,true\n" + "502,9.90,0.3,1006,1008,179999,false\n" + "503,12.00,0.4,1009,1009,240001,true\n" + "504,11.00,0.1,1010,1010,240050,false\n" + "505,12.00,0.1,1011,1011,300001,true\n" + "506,12.10,0.2,1012,1012,300050,false\n"; +// [120000, 300000): predecessor 499, prints 500..504, a quiet minute, fence 505. +const std::string window_bars = bar_header + + "120000,10.10,11.20,9.90,9.90,0.6\n" + "180000,9.90,9.90,9.90,9.90,0\n" + "240000,12.00,12.00,11.00,11.00,0.5\n"; +std::string replaced(std::string text, const std::string& from, const std::string& to) { + const auto at = text.find(from); + assert(at != std::string::npos); + return text.replace(at, from.size(), to); +} +} + +int main() { + { + ChainBars chain(120000, 300000); + chain.anchor({499, 119999, "10.10", "0.1"}); + assert(!chain.add({500, 120001, "10.10", "0.1"}) && !chain.add({501, 120050, "11.20", "0.2"}) && + !chain.add({502, 179999, "9.90", "0.3"}) && !chain.add({503, 240001, "12.00", "0.4"}) && + !chain.add({504, 240050, "11.00", "0.1"})); + assert(chain.add({505, 300001, "12.00", "0.1"})); + assert(chain.csv() == window_bars && chain.prints() == 5 && chain.bars() == 3 && chain.quiet_minutes() == 1); + assert(chain.predecessor().id == 499 && chain.fence().id == 505 && chain.fence().ts == 300001); + expect(23, [&] { chain.add({506, 300050, "12.10", "0.2"}); }); + } + { + // The first minute is quiet: it carries the warmup close, not the predecessor's price, exactly as the + // runner carries its last warmup bar; a later quiet minute carries the last print. A fence exactly at the + // end closes the window. + ChainBars chain(180000, 420000); + chain.seed("9.95"); + chain.anchor({502, 179999, "9.90", "0.3"}); + assert(!chain.add({503, 240001, "12.00", "0.4"})); + assert(chain.add({504, 420000, "11.00", "0.1"})); + assert(chain.csv() == bar_header + "180000,9.95,9.95,9.95,9.95,0\n240000,12.00,12.00,12.00,12.00,0.4\n" + "300000,12.00,12.00,12.00,12.00,0\n360000,12.00,12.00,12.00,12.00,0\n"); + assert(chain.quiet_minutes() == 3 && chain.prints() == 1 && chain.bars() == 4 && chain.seeded_close() == "9.95"); + } + { + // Without the warmup close a quiet first minute is not guessed from the predecessor. + ChainBars chain(180000, 420000); + chain.anchor({502, 179999, "9.90", "0.3"}); + expect(20, [&] { chain.add({503, 240001, "12.00", "0.4"}); }); + ChainBars late(180000, 240000); + expect(23, [&] { late.seed("x"); }); + } + { + // A quiet window: the predecessor's ID + 1 is already the fence. + ChainBars chain(120000, 240000); + chain.seed("100.5"); + chain.anchor({7, 0, "100.5", "1"}); + assert(chain.add({8, 250000, "101", "1"})); + assert(chain.csv() == bar_header + "120000,100.5,100.5,100.5,100.5,0\n180000,100.5,100.5,100.5,100.5,0\n"); + assert(chain.prints() == 0 && chain.quiet_minutes() == 2); + } + { + ChainBars chain(60000, 120000); + chain.anchor({1, 0, "5", "1"}); + chain.add({2, 60000, "5", "0.1"}); + chain.add({3, 119999, "5", "0.2"}); + assert(chain.add({4, 120000, "5", "1"})); + // Without a qty_step the runner's volume is the compensated binary64 sum, written shortest. + assert(chain.csv() == bar_header + "60000,5,5,5,5,0.30000000000000004\n"); + ChainBars grid(60000, 120000); + grid.quantity_grid(0.1); + grid.anchor({1, 0, "5", "1"}); + grid.add({2, 60000, "5", "0.1"}); + grid.add({3, 119999, "5", "0.2"}); + assert(grid.add({4, 120000, "5", "1"})); + assert(grid.csv() == bar_header + "60000,5,5,5,5,0.3\n"); + ChainBars late(60000, 120000); + late.anchor({1, 0, "5", "1"}); + expect(23, [&] { late.quantity_grid(0.1); }); + ChainBars zero(60000, 120000); + expect(23, [&] { zero.quantity_grid(0); }); + } + { + // The engine's tick-volume vectors (#319, tests/test_native_tick_volume.cpp). + const auto volume = [](std::optional step, std::initializer_list quantities) { + RunnerVolume sum; + sum.reset(step); + for (const double quantity : quantities) sum.add(quantity); + return std::make_pair(sum.value(), sum.exact()); + }; + assert(volume(0.1, {0.1, 0.2}) == std::make_pair(0.3, true)); + assert(volume(0.1, {0.1, 0.2, 0.05}) == std::make_pair(std::strtod("0.35000000000000003", nullptr), false)); + assert(volume(0.1, {std::nextafter(0.3, 1.0)}) == std::make_pair(std::nextafter(0.3, 1.0), false)); + assert(volume(0.001, {4e15, 4e15, 4e15, 1, 1}) == std::make_pair(12000000000000002.0, false)); + assert(volume(1.0, {std::ldexp(1.0, 63), 1024, 1024}) == std::make_pair(std::ldexp(1.0, 63) + 2048, false)); + assert(volume(std::nullopt, {0.1, 0.2, 0.05}).first == std::strtod("0.35000000000000003", nullptr)); + assert(volume(0.0004, {0.1, 0.2}) == std::make_pair(0.30000000000000004, false)); + std::uint64_t seed = 84729531; + double divisor = 1.0; + for (int places = 0; places <= 22; ++places, divisor *= 10.0) { + RunnerVolume grid; + grid.reset(1.0 / divisor); + assert(grid.exact()); + for (int sample = 0; sample < 128; ++sample) { + seed = seed * 6364136223846793005ULL + 1442695040888963407ULL; + const auto units = static_cast(seed & 0x7fffffffffffffffULL); + const auto decimal = std::to_string(units) + "e-" + std::to_string(places); + assert(grid.decimal(units) == std::strtod(decimal.c_str(), nullptr)); + } + } + assert(runner_double("0.003") == 0.003 && runner_double("84797.50") == 84797.5); + expect(23, [] { runner_double("1x"); }); + expect(23, [] { runner_double(""); }); + assert(shortest_decimal(0.30000000000000004) == "0.30000000000000004" && shortest_decimal(1e-05) == "0.00001" && + shortest_decimal(3.0) == "3" && shortest_decimal(12000000000000002.0) == "12000000000000002"); + passed("runner_volume_matches_the_engine_tick_volume_vectors"); + } + { + // Price tokens stay verbatim (an equal value keeps the first lexeme); volume is the runner's sum. + ChainBars chain(60000, 180000); + chain.anchor({1, 59999, "1.0", "1"}); + chain.add({2, 60000, "2.50", "0.1"}); + chain.add({3, 60001, "2.5", "0.2"}); + chain.add({4, 60002, "0.70", "9999999999.99999999"}); + chain.add({5, 60003, "0.7", "0.00000001"}); + chain.add({6, 60004, "1.25", "0.000000000000000000000000000001"}); + for (std::uint64_t id = 7; id < 17; ++id) chain.add({id, 120000 + static_cast(id), "3.10", "0.1"}); + assert(chain.add({17, 180000, "3", "1"})); + assert(chain.csv() == bar_header + "60000,2.50,2.50,0.70,1.25,10000000000.3\n" + "120000,3.10,3.10,3.10,3.10,1\n"); + } + { + ChainBars chain(120000, 300000); + expect(23, [&] { chain.add({500, 120001, "1", "1"}); }); + expect(20, [&] { chain.anchor({499, 120000, "1", "1"}); }); + chain.anchor({499, 119999, "10.10", "0.1"}); + expect(23, [&] { chain.anchor({498, 119000, "1", "1"}); }); + chain.add({500, 120001, "10.10", "0.1"}); + expect(20, [&] { chain.add({502, 120002, "10", "1"}); }); + expect(20, [&] { chain.add({500, 120002, "10", "1"}); }); + expect(20, [&] { (void)chain.csv(); }); + expect(20, [&] { (void)chain.fence(); }); + expect(23, [&] { chain.add({501, 120000, "10", "1"}); }); + expect(23, [&] { chain.add({501, 120002, "10", "0"}); }); + } + { + ChainBars chain(120000, 300000); + chain.anchor({499, 119999, "10", "1"}); + expect(20, [&] { chain.add({500, 119999, "10", "1"}); }); + } + expect(23, [] { ChainBars chain(120001, 300000); (void)chain; }); + expect(23, [] { ChainBars chain(120000, 120000); (void)chain; }); + expect(22, [] { ChainBars chain(0, 100001LL * 60000); (void)chain; }); + { ChainBars chain(0, 100000LL * 60000); (void)chain; } + passed("tick_built_bar_rule_and_chain_proof"); + + assert(!archive_row(archive_header.substr(0, archive_header.size() - 1), true)); + expect(23, [] { (void)archive_row(archive_header.substr(0, archive_header.size() - 1), false); }); + expect(23, [] { (void)archive_row("agg_trade_id,price", true); }); + const auto row = archive_row("500,10.10,0.1,1000,1004,120001,false\r", false); + assert(row && row->id == 500 && row->ts == 120001 && row->price == "10.10" && row->qty == "0.1"); + assert(archive_row("7,10.1,0.5,1,1,1759276800123456,True", true)->ts == 1759276800123); + assert(archive_row("7,10.1,0.5,1,1,1759276800123,false", false)->ts == 1759276800123); + for (const auto* invalid : {"7,10.1,0.5,1,1,120001", "7,10.1,0.5,1,1,120001,true,x", "7,10.1,0,1,1,120001,true", "7,-1,0.5,1,1,120001,true", + "0,10.1,0.5,1,1,120001,true", "7,10.1,0.5,2,1,120001,true", "7,10.1,0.5,0,1,120001,true", + "7,10.1,0.5,1,1,12e5,true", "7,10.1,0.5,1,1,-5,true", "7,10.1,0.5,1,1,120001,yes", "7,1e1,0.5,1,1,120001,true", ""}) + expect(23, [&] { (void)archive_row(invalid, false); }); + + Temporary temporary; + const auto with_header = temporary.path + "/TESTUSDT-aggTrades-1970-01-01.csv"; + put_file(with_header, archive_header + rows); + assert(archive_bars(with_header, 120000, 300000).csv() == window_bars); + const auto headerless = temporary.path + "/headerless.csv"; + put_file(headerless, rows); + { + const auto chain = archive_bars(headerless, 120000, 300000); + assert(chain.csv() == window_bars && chain.predecessor().id == 499 && chain.predecessor().price == "10.10"); + assert(chain.fence().id == 505 && chain.prints() == 5 && chain.quiet_minutes() == 1); + } + // A quiet first minute carries the warmup close it is given, and stops 20 without one. + assert(archive_bars(headerless, 180000, 300000, "9.95").csv() == bar_header + "180000,9.95,9.95,9.95,9.95,0\n240000,12.00,12.00,11.00,11.00,0.5\n"); + expect(20, [&] { (void)archive_bars(headerless, 180000, 300000); }); + const auto crlf = temporary.path + "/crlf.csv"; + put_file(crlf, replaced(replaced(archive_header + rows, "\n", "\r\n"), "499,10.10,0.1,998,999,119999,true\n", "499,10.10,0.1,998,999,119999,true\r\n")); + assert(archive_bars(crlf, 120000, 300000).csv() == window_bars); + const auto unterminated = temporary.path + "/unterminated.csv"; + put_file(unterminated, rows.substr(0, rows.find("505,")) + "505,12.00,0.1,1011,1011,300001,true"); + assert(archive_bars(unterminated, 120000, 300000).csv() == window_bars); + expect(20, [&] { (void)archive_bars(headerless, 60000, 180000); }); + expect(20, [&] { (void)archive_bars(headerless, 240000, 360000); }); + const auto empty = temporary.path + "/empty.csv"; + put_file(empty, archive_header); + expect(20, [&] { (void)archive_bars(empty, 120000, 300000); }); + const auto hole = temporary.path + "/hole.csv"; + put_file(hole, replaced(rows, "502,9.90,0.3,1006,1008,179999,false\n", "")); + expect(20, [&] { (void)archive_bars(hole, 120000, 300000); }); + const auto regression = temporary.path + "/regression.csv"; + put_file(regression, replaced(rows, "240001", "170000")); + expect(23, [&] { (void)archive_bars(regression, 120000, 300000); }); + const auto malformed = temporary.path + "/malformed.csv"; + put_file(malformed, replaced(rows, "0.3,", "0.3;")); + expect(23, [&] { (void)archive_bars(malformed, 120000, 300000); }); + expect(23, [&] { (void)archive_bars(temporary.path + "/missing.csv", 120000, 300000); }); + passed("archive_csv_rows_with_and_without_header"); + + assert(crc32(0, "123456789") == 0xCBF43926U && crc32(crc32(0, "1234"), "56789") == 0xCBF43926U); + const auto text = generated(); + assert(text.size() == 1896 && crc32(0, text) == 0x52AFB242U); + const auto mixed = bytes(mixed_deflate); + for (const std::size_t chunk : {1, 7, 4096}) assert(inflated(mixed, chunk) == text); + expect(21, [&] { (void)inflated(mixed.substr(0, mixed.size() - 1)); }); + expect(21, [&] { (void)inflated(mixed.substr(0, 200)); }); + expect(21, [&] { (void)inflated(mixed + std::string(1, '\0')); }); + expect(21, [] { (void)inflated(std::string(1, '\x07')); }); + expect(21, [] { (void)inflated(std::string("\x01\x05\x00\x00\x00", 5)); }); + expect(21, [] { (void)inflated(std::string()); }); + { + Bits far; // a back-reference before any output + far.put(1, 1); + far.put(1, 2); + far.huffman(1, 7); + far.huffman(0, 5); + far.huffman(0, 7); + expect(21, [&] { (void)inflated(far.finish()); }); + } + { + // Fixed-Huffman literals, then 5000 copies of 258 bytes at distance 9: output crosses the 1 MiB hand-off. + Bits big; + big.put(1, 1); + big.put(1, 2); + const std::string seed = "pineforge"; + for (const char letter : seed) big.huffman(0x30 + static_cast(letter), 8); + for (int index = 0; index < 5000; ++index) { + big.huffman(0xC5, 8); + big.huffman(6, 5); + big.put(0, 2); + } + big.huffman(0, 7); + std::string expected; + while (expected.size() < 9 + 5000 * 258) expected += seed; + expected.resize(9 + 5000 * 258); + std::size_t calls = 0; + assert(inflated(big.finish(), 4096, &calls) == expected && calls >= 2); + } + passed("raw_inflate_fixed_stored_dynamic_blocks"); + + const auto zip = temporary.path + "/TESTUSDT-aggTrades-1970-01-01.zip"; + put_file(zip, bytes(deflated_zip)); + std::string entry; + assert(unzip(zip, [&](std::string_view chunk) { entry.append(chunk); }) == "TESTUSDT-aggTrades-1970-01-01.csv"); + assert(entry == archive_header + rows); + assert(archive_bars(zip, 120000, 300000).csv() == window_bars); + const auto stored = temporary.path + "/stored.zip"; + put_file(stored, bytes(stored_zip)); + entry.clear(); + (void)unzip(stored, [&](std::string_view chunk) { entry.append(chunk); }); + assert(entry == rows && archive_bars(stored, 120000, 300000).csv() == window_bars); + const auto damaged = temporary.path + "/damaged.zip"; + auto altered = bytes(deflated_zip); + altered[altered.find("PK\x01\x02") + 16] ^= 1; + put_file(damaged, altered); + expect(21, [&] { (void)unzip(damaged, [](std::string_view) {}); }); + expect(21, [&] { (void)archive_bars(damaged, 120000, 300000); }); + altered = bytes(stored_zip); + altered[30 + 33 + 5] = 'x'; // a row that no longer parses: the CRC-32 mismatch (21) is reported first + put_file(damaged, altered); + expect(21, [&] { (void)archive_bars(damaged, 120000, 300000); }); + altered = bytes(deflated_zip); + altered[30 + 33 + 40] ^= 0x10; + put_file(damaged, altered); + expect(21, [&] { (void)archive_bars(damaged, 120000, 300000); }); + altered = bytes(stored_zip); + altered[altered.find("PK\x01\x02") + 24] ^= 1; // recorded size disagrees + put_file(damaged, altered); + expect(21, [&] { (void)archive_bars(damaged, 120000, 300000); }); + const auto plain = temporary.path + "/plain.zip"; + put_file(plain, rows); + expect(23, [&] { (void)archive_bars(plain, 120000, 300000); }); + passed("zip_deflated_and_stored_entries_crc_checked"); + + const auto digest = file_sha256(zip); + assert(digest == sha256(bytes(deflated_zip)) && file_sha256(headerless) == sha256(rows)); + expect(23, [&] { (void)file_sha256(temporary.path + "/missing.zip"); }); + const auto checksum = temporary.path + "/TESTUSDT-aggTrades-1970-01-01.zip.CHECKSUM"; + put_file(checksum, digest + " TESTUSDT-aggTrades-1970-01-01.zip\n"); + verify_checksum(checksum, zip, digest); + auto upper = digest; + std::transform(upper.begin(), upper.end(), upper.begin(), [](unsigned char letter) { return static_cast(std::toupper(letter)); }); + put_file(checksum, upper + " *TESTUSDT-aggTrades-1970-01-01.zip"); + verify_checksum(checksum, zip, digest); + auto wrong = digest; + wrong[0] = wrong[0] == '0' ? '1' : '0'; + put_file(checksum, wrong + " TESTUSDT-aggTrades-1970-01-01.zip\n"); + expect(21, [&] { verify_checksum(checksum, zip, digest); }); + put_file(checksum, digest + " TESTUSDT-aggTrades-1970-01-02.zip\n"); + expect(21, [&] { verify_checksum(checksum, zip, digest); }); + for (const auto* invalid : {"", "not a checksum\n", "abc TESTUSDT-aggTrades-1970-01-01.zip\n"}) { + put_file(checksum, invalid); + expect(23, [&] { verify_checksum(checksum, zip, digest); }); + } + put_file(checksum, digest + " TESTUSDT-aggTrades-1970-01-01.zip\n" + digest + " other.zip\n"); + expect(23, [&] { verify_checksum(checksum, zip, digest); }); + expect(23, [&] { verify_checksum(temporary.path + "/missing.CHECKSUM", zip, digest); }); + passed("archive_checksum_hash_and_name"); + std::cout << "PASS feed export unit suite\n"; +} diff --git a/native/feed/tests/mock_export.py b/native/feed/tests/mock_export.py new file mode 100644 index 0000000..d518e81 --- /dev/null +++ b/native/feed/tests/mock_export.py @@ -0,0 +1,510 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Synthetic export scenarios: the real binary against the mock USD-M venue and daily archives.""" + +import hashlib +import importlib +import json +import math +import pathlib +import subprocess +import sys +import tempfile +import threading +import time +import unittest +import zipfile +from datetime import UTC, datetime +from decimal import Decimal, getcontext +from fractions import Fraction + +# mock_venue reads the binary from sys.argv[1] at import time. +sys.argv = [sys.argv[0], sys.argv[1]] +mock_venue = importlib.import_module("mock_venue") +BINARY = mock_venue.BINARY +getcontext().prec = 200 +SYMBOL = "TESTUSDT" +HEADER = "agg_trade_id,price,quantity,first_trade_id,last_trade_id,transact_time,is_buyer_maker" + + +def recent_minute(minutes_ago): + return (int(time.time() * 1000) // 60000 - minutes_ago) * 60000 + + +def canonical(value): + text = format(value, "f") + if "." in text: + text = text.rstrip("0").rstrip(".") + return text + + +def runner_volume(quantities, step=None): + """The runner's tick-built volume, independently (engine TickVolume): on a 1/10^k step the + exact grid units, correctly rounded; else, or once a quantity is off it, the compensated + binary64 sum in print order. Quantities are JSON number tokens, read as correctly rounded + doubles (the runner's std::stod).""" + divisor, places, exact = 1.0, 0, False + if step is not None: + scale = 1.0 + for k in range(23): + if float(step) == 1.0 / scale: + divisor, places, exact = scale, k, True + break + scale *= 10.0 + + def decimal(units): + return units / divisor if units <= 2**53 else float(Fraction(units, 10**places)) + + units, total, compensation = 0, 0.0, 0.0 + for token in quantities: + quantity = float(token) + following = total + quantity + if math.isfinite(following): + if total >= quantity: + compensation += (total - following) + quantity + else: + compensation += (quantity - following) + total + else: + compensation = 0.0 + total = following + if not exact: + continue + scaled = quantity * divisor + if not math.isfinite(scaled): + exact = False + continue + rounded = math.floor(scaled) + (1 if scaled - math.floor(scaled) >= 0.5 else 0) + if rounded >= 2**63 or decimal(rounded) != quantity or rounded > 2**63 - 1 - units: + exact = False + continue + units += rounded + value = decimal(units) if exact else total + compensation + return canonical(Decimal(repr(value))) + + +def expected_bars(prints, start, end, warmup_close=None, step=None): + """The tick-built bar rule, independently: prints are (id, ts, price, qty) in ID order. A + quiet first minute carries the runner's last warmup close.""" + close = warmup_close + rows = ["timestamp,open,high,low,close,volume"] + for minute in range(start, end, 60000): + inside = [p for p in prints if minute <= p[1] < minute + 60000] + if not inside: + rows.append(f"{minute},{close},{close},{close},{close},0") + continue + high = max(inside, key=lambda p: Decimal(p[2]))[2] + low = min(inside, key=lambda p: Decimal(p[2]))[2] + volume = runner_volume([p[3] for p in inside], step) + close = inside[-1][2] + rows.append(f"{minute},{inside[0][2]},{high},{low},{close},{volume}") + return "\n".join(rows) + "\n" + + +def synthetic(first_id, start, minutes, per_minute, quiet=()): + """Prints over `minutes` minutes from `start`, none in the `quiet` minute offsets.""" + prints = [] + identifier = first_id + for offset in range(minutes): + if offset in quiet: + continue + for index in range(per_minute): + ts = start + offset * 60000 + index * (59999 // max(per_minute - 1, 1)) + price = f"{100 + (identifier * 37) % 50}.{(identifier * 13) % 100:02d}" + quantity = f"0.{(identifier * 104729) % 99999999 + 1:08d}" + prints.append((identifier, ts, price, quantity)) + identifier += 1 + return prints + + +class ExportMockTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="pineforge-feed-export-") + self.addCleanup(self.temporary.cleanup) + self.directory = pathlib.Path(self.temporary.name) + self.output = self.directory / "bars.csv" + + def server(self, prints): + server = mock_venue.MockServer([[]], "usdm") + server.aggregates = { + identifier: ( + ts, + price, + quantity, + quantity, + 1000 + 2 * identifier, + 1001 + 2 * identifier, + ) + for identifier, ts, price, quantity in prints + } + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return server + + def export(self, start, end, server=None, extra=(), expected=0, venue=("binance", "usdm")): + port = server.server_address[1] if server else 9 + arguments = [ + BINARY, + "export", + "--venue", + venue[0], + "--market", + venue[1], + "--symbol", + SYMBOL, + ] + arguments += ["--mode", "agg-ticks" if venue[1] == "usdm" else "ticks"] + arguments += ["--start", str(start), "--end", str(end), "--output", str(self.output)] + arguments += [ + "--rest-url", + f"http://127.0.0.1:{port}", + "--ws-url", + f"ws://127.0.0.1:{port}", + ] + arguments += ["--allow-insecure-http", *extra] + result = subprocess.run(arguments, capture_output=True, text=True, timeout=40) + self.assertEqual(result.returncode, expected, result.stderr) + if expected: + self.assertFalse(self.output.exists()) + self.assertFalse(pathlib.Path(str(self.output) + ".manifest.json").exists()) + return result + + def manifest(self): + manifest = json.loads(pathlib.Path(str(self.output) + ".manifest.json").read_text()) + self.assertEqual(manifest["sha256"], hashlib.sha256(self.output.read_bytes()).hexdigest()) + self.assertEqual(manifest["schema"], "pineforge-feed-export/v1") + self.assertIn("floor(ts/60000)", manifest["rule"]) + return manifest + + def test_rest_bars_follow_the_rule_with_quiet_minute_and_carry(self): + start = recent_minute(30) + prints = [ + (499, start - 1, "10.10000000", "0.10000000"), + (500, start, "10.10000000", "0.10000000"), + (501, start + 50, "11.20000000", "0.20000000"), + (502, start + 59999, "9.90000000", "0.30000000"), + (503, start + 120001, "12.00000000", "0.40000000"), + (504, start + 120050, "11.00000000", "0.10000000"), + (505, start + 180000, "11.50000000", "0.00100000"), + (506, start + 240001, "12.00000000", "0.10000000"), + (507, start + 240050, "12.10000000", "0.20000000"), + ] + server = self.server(prints) + self.export(start, start + 240000, server) + text = self.output.read_text() + self.assertEqual(text, expected_bars(prints, start, start + 240000)) + self.assertIn(f"\n{start + 60000},9.90000000,9.90000000,9.90000000,9.90000000,0\n", text) + volume = runner_volume(["0.10000000", "0.20000000", "0.30000000"]) + self.assertIn(f"\n{start},10.10000000,11.20000000,9.90000000,9.90000000,{volume}\n", text) + manifest = self.manifest() + self.assertEqual( + {key: manifest[key] for key in ("venue", "market", "symbol", "mode")}, + {"venue": "binance", "market": "usdm", "symbol": SYMBOL, "mode": "agg-ticks"}, + ) + port = server.server_address[1] + self.assertEqual(manifest["source"], {"kind": "rest", "origin": f"http://127.0.0.1:{port}"}) + self.assertEqual((manifest["start"], manifest["end_exclusive"]), (start, start + 240000)) + self.assertEqual( + manifest["predecessor"], {"id": 499, "ts": start - 1, "price": "10.10000000"} + ) + self.assertEqual(manifest["fence"], {"id": 506, "ts": start + 240001}) + self.assertEqual((manifest["first_id"], manifest["last_id"]), (500, 505)) + self.assertEqual( + (manifest["prints"], manifest["bars"], manifest["quiet_minutes"]), (6, 4, 1) + ) + + def warmup(self, start, close): + path = self.directory / "warmup.csv" + path.write_text( + "timestamp,open,high,low,close,volume\n" + f"{start - 120000},9,11,8,10,1\n{start - 60000},10,10.5,9.5,{close},2\n" + ) + return path + + def test_rest_start_survives_a_quiet_first_hour(self): + # The first aggregate after --start lies in the second hour window: the start lookup + # searches forward up to the clock instead of stopping 20 after one window. The quiet + # first minutes carry the warmup close (10.2), not the predecessor's 10.10000000, as the + # runner carries its last warmup bar; the quiet later minute carries the last print. + start = recent_minute(90) + end = start + 73 * 60000 + prints = [(499, start - 1, "10.10000000", "0.10000000")] + prints += synthetic(500, start + 70 * 60000, 4, 4, quiet=(1,)) + server = self.server(prints) + warmup = self.warmup(start, "10.2") + self.export(start, end, server, extra=["--warmup", str(warmup)]) + text = self.output.read_text() + self.assertEqual(text, expected_bars(prints, start, end, warmup_close="10.2")) + self.assertIn(f"\n{start},10.2,10.2,10.2,10.2,0\n", text) + last = [p for p in prints if p[1] < start + 71 * 60000][-1][2] + self.assertIn(f"\n{start + 71 * 60000},{last},{last},{last},{last},0\n", text) + manifest = self.manifest() + self.assertEqual((manifest["quiet_minutes"], manifest["warmup_close"]), (71, "10.2")) + windows = [ + int(query["startTime"][0]) + for path, query in server.queries + if path == "/fapi/v1/aggTrades" and "startTime" in query + ] + self.assertIn(start + 3600000, windows) + + def test_quiet_first_minute_without_the_warmup_close_stops(self): + start = recent_minute(30) + prints = [(499, start - 1, "10.10000000", "0.10000000")] + prints += synthetic(500, start + 60000, 2, 3) + server = self.server(prints) + result = self.export(start, start + 120000, server, expected=20) + self.assertIn("--warmup", result.stderr) + warmup = self.warmup(start + 60000, "10.2") + result = self.export( + start, start + 120000, server, extra=["--warmup", str(warmup)], expected=20 + ) + self.assertIn("minute before the start", result.stderr) + warmup.write_text("timestamp,open,high,low,close,volume\n" + f"{start - 60000},1,1,1,x,1\n") + self.export(start, start + 120000, server, extra=["--warmup", str(warmup)], expected=23) + # A first minute with prints never reads the warmup close. + self.export(start + 60000, start + 120000, server) + self.assertEqual( + self.output.read_text(), expected_bars(prints, start + 60000, start + 120000) + ) + self.assertIsNone(self.manifest()["warmup_close"]) + + def test_volume_follows_the_runner_rule_for_its_qty_step(self): + # The engine's #319 vectors: 0.1 + 0.2 + 0.05 on step 0.1 leaves the grid (compensated + # 0.35000000000000003); on step 0.05 it is exact 0.35; without a step it is compensated. + # 0.1 x 10 + 0.3 shows a plain binary64 sum (1.2999999999999998) is not the rule. + start = recent_minute(30) + prints = [(499, start - 1, "10", "1")] + for index, quantity in enumerate(["0.1", "0.2", "0.05"]): + prints.append((500 + index, start + index, "10", quantity)) + for index in range(10): + prints.append((503 + index, start + 60000 + index, "10", "0.1")) + prints.append((513, start + 60010, "10", "0.3")) + prints.append((514, start + 120000, "10", "1")) + server = self.server(prints) + end = start + 120000 + for step, first, second, rule in ( + (None, "0.35000000000000003", "1.3", "compensated-binary64"), + ("0.1", "0.35000000000000003", "1.3", "exact-decimal-grid"), + ("0.05", "0.35000000000000003", "1.3", "compensated-binary64"), + ("0.01", "0.35", "1.3", "exact-decimal-grid"), + ): + with self.subTest(step=step): + extra = ["--qty-step", step] if step else [] + self.export(start, end, server, extra=extra) + rows = self.output.read_text().splitlines() + self.assertEqual(rows[1].split(",")[5], first) + self.assertEqual(rows[2].split(",")[5], second) + self.assertEqual( + self.output.read_text(), expected_bars(prints, start, end, step=step) + ) + manifest = self.manifest() + self.assertEqual((manifest["qty_step"], manifest["volume_rule"]), (step, rule)) + self.output.unlink() + pathlib.Path(str(self.output) + ".manifest.json").unlink() + plain = 0.0 + for quantity in [0.1] * 10 + [0.3]: + plain += quantity + self.assertEqual(plain, 1.2999999999999998) + self.export(start, end, server, extra=["--qty-step", "0"], expected=23) + + def test_rest_pages_by_from_id_and_sums_volume_exactly(self): + start = recent_minute(40) + prints = synthetic(1000, start - 60000, 6, 700, quiet=(2,)) + server = self.server(prints) + self.export(start, start + 240000, server) + self.assertEqual(self.output.read_text(), expected_bars(prints, start, start + 240000)) + pages = [q for path, q in server.queries if path == "/fapi/v1/aggTrades" and "fromId" in q] + self.assertGreaterEqual(len([q for q in pages if q["limit"] == ["1000"]]), 3) + manifest = self.manifest() + self.assertEqual((manifest["prints"], manifest["quiet_minutes"]), (2100, 1)) + self.assertEqual(manifest["fence"]["id"], manifest["last_id"] + 1) + + def test_missing_fence_stops_20_and_writes_nothing(self): + start = recent_minute(20) + prints = [p for p in synthetic(1, start - 60000, 4, 5) if p[1] < start + 120000] + server = self.server(prints) + result = self.export(start, start + 180000, server, expected=20) + self.assertIn("fence cannot be proven", result.stderr) + + def test_hole_stops_20_and_time_regression_stops_23(self): + start = recent_minute(20) + prints = synthetic(1, start - 60000, 5, 5) + server = self.server([p for p in prints if p[0] != 9]) + result = self.export(start, start + 180000, server, expected=20) + self.assertIn("nonunit aggregate ID", result.stderr) + # ID 12 is dated before ID 11 (the first print of its minute). + regressed = [(i, start + 50000 if i == 12 else ts, p, q) for i, ts, p, q in prints] + server = self.server(regressed) + result = self.export(start, start + 180000, server, expected=23) + self.assertIn("backwards", result.stderr) + + def test_start_beyond_retention_or_open_window_stops_20(self): + start = recent_minute(49 * 60) + server = self.server(synthetic(1, start - 60000, 3, 3)) + result = self.export(start, start + 60000, server, expected=20) + self.assertIn("REST print history window", result.stderr) + self.assertNotIn("/fapi/v1/aggTrades", server.requests) + result = self.export(recent_minute(1), recent_minute(-5), server, expected=20) + self.assertIn("not closed yet", result.stderr) + + def test_existing_output_is_refused(self): + self.output.write_text("keep\n") + start = recent_minute(10) + arguments = [BINARY, "export", "--venue", "binance", "--market", "usdm", "--symbol", SYMBOL] + arguments += ["--mode", "agg-ticks", "--start", str(start), "--end", str(start + 60000)] + arguments += ["--output", str(self.output), "--rest-url", "http://127.0.0.1:9"] + arguments += ["--ws-url", "ws://127.0.0.1:9", "--allow-insecure-http"] + result = subprocess.run(arguments, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 23, result.stderr) + self.assertEqual(self.output.read_text(), "keep\n") + + def archive_prints(self): + start = recent_minute(24 * 60) + prints = synthetic(70000, start - 120000, 9, 400, quiet=(4,)) + return start, prints + + def write_archive(self, name, prints, microseconds=False, header=True): + lines = [HEADER] if header else [] + for identifier, ts, price, quantity in prints: + when = ts * 1000 + 456 if microseconds else ts + raw = 2 * identifier + maker = "true" if identifier % 3 else "false" + lines.append(f"{identifier},{price},{quantity},{raw},{raw + 1},{when},{maker}") + text = "\n".join(lines) + "\n" + path = self.directory / name + if name.endswith(".zip"): + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(name[:-4] + ".csv", text) + else: + path.write_text(text) + return path + + def checksum(self, archive, digest=None, name=None): + digest = digest or hashlib.sha256(archive.read_bytes()).hexdigest() + path = self.directory / (archive.name + ".CHECKSUM") + path.write_text(f"{digest} {name or archive.name}\n") + return path + + def test_archive_csv_and_zip_match_the_rule(self): + start, prints = self.archive_prints() + end = start + 300000 + expected = expected_bars(prints, start, end) + csv_archive = self.write_archive( + f"{SYMBOL}-aggTrades-day.csv", prints, microseconds=True, header=False + ) + checksum = self.checksum(csv_archive) + self.export(start, end, extra=["--archive", str(csv_archive), "--checksum", str(checksum)]) + self.assertEqual(self.output.read_text(), expected) + manifest = self.manifest() + self.assertEqual(manifest["source"]["checksum_verified"], True) + self.output.unlink() + pathlib.Path(str(self.output) + ".manifest.json").unlink() + zipped = self.write_archive(f"{SYMBOL}-aggTrades-day.zip", prints) + checksum = self.checksum(zipped) + self.export(start, end, extra=["--archive", str(zipped), "--checksum", str(checksum)]) + self.assertEqual(self.output.read_text(), expected) + manifest = self.manifest() + digest = hashlib.sha256(zipped.read_bytes()).hexdigest() + self.assertEqual( + manifest["source"], + {"kind": "archive", "file": zipped.name, "sha256": digest, "checksum_verified": True}, + ) + predecessor = [p for p in prints if p[1] < start][-1] + fence = next(p for p in prints if p[1] >= end) + self.assertEqual( + manifest["predecessor"], + {"id": predecessor[0], "ts": predecessor[1], "price": predecessor[2]}, + ) + self.assertEqual(manifest["fence"], {"id": fence[0], "ts": fence[1]}) + self.assertEqual(manifest["quiet_minutes"], 1) + + def test_archive_without_checksum_warns(self): + start, prints = self.archive_prints() + archive = self.write_archive(f"{SYMBOL}-aggTrades-day.zip", prints) + result = self.export(start, start + 120000, extra=["--archive", str(archive)]) + self.assertIn("archive_not_checksum_verified", result.stderr) + self.assertEqual(self.manifest()["source"]["checksum_verified"], False) + + def test_archive_checksum_mismatch_wrong_name_and_corruption_stop_21(self): + start, prints = self.archive_prints() + archive = self.write_archive(f"{SYMBOL}-aggTrades-day.zip", prints) + tampered = self.checksum(archive, digest="0" * 64) + result = self.export( + start, + start + 120000, + extra=["--archive", str(archive), "--checksum", str(tampered)], + expected=21, + ) + self.assertIn("does not match", result.stderr) + renamed = self.checksum(archive, name=f"{SYMBOL}-aggTrades-other.zip") + result = self.export( + start, + start + 120000, + extra=["--archive", str(archive), "--checksum", str(renamed)], + expected=21, + ) + self.assertIn("not the archive", result.stderr) + csv_archive = self.write_archive(f"{SYMBOL}-aggTrades-day.csv", prints) + tampered = self.checksum(csv_archive, digest="f" * 64) + result = self.export( + start, + start + 120000, + extra=["--archive", str(csv_archive), "--checksum", str(tampered)], + expected=21, + ) + self.assertIn("does not match", result.stderr) + data = bytearray(archive.read_bytes()) + data[len(data) // 2] ^= 0x55 + archive.write_bytes(bytes(data)) + result = self.export(start, start + 120000, extra=["--archive", str(archive)], expected=21) + self.assertRegex(result.stderr, "deflate|CRC-32|recorded size") + self.assertNotIn("export_written", result.stderr) + + def test_archive_day_boundaries_name_the_missing_daily_file(self): + # A window from 00:00 UTC has its predecessor in the previous day's file, and one to 24:00 + # has its fence in the next day's: export reads one daily file and says which one is needed. + day = int(datetime(2026, 10, 2, tzinfo=UTC).timestamp() * 1000) + prints = synthetic(1000, day, 3, 4) + synthetic(1012, day + 86400000 - 180000, 3, 4) + archive = self.write_archive(f"{SYMBOL}-aggTrades-2026-10-02.csv", prints) + result = self.export(day, day + 120000, extra=["--archive", str(archive)], expected=20) + self.assertIn(f"{SYMBOL}-aggTrades-2026-10-01.csv", result.stderr) + result = self.export( + day + 86400000 - 120000, + day + 86400000, + extra=["--archive", str(archive)], + expected=20, + ) + self.assertIn(f"{SYMBOL}-aggTrades-2026-10-03.csv", result.stderr) + # A window reaching past the day, or starting before its first print, names the same files. + result = self.export( + day + 86400000 - 120000, + day + 86400000 + 600000, + extra=["--archive", str(archive)], + expected=20, + ) + self.assertIn(f"{SYMBOL}-aggTrades-2026-10-03.csv", result.stderr) + late = self.write_archive(f"{SYMBOL}-aggTrades-2026-10-02.zip", prints[5:]) + result = self.export(day + 60000, day + 120000, extra=["--archive", str(late)], expected=20) + self.assertIn(f"{SYMBOL}-aggTrades-2026-10-01.zip", result.stderr) + + def test_archive_outside_window_and_unsupported_venue(self): + start, prints = self.archive_prints() + archive = self.write_archive(f"{SYMBOL}-aggTrades-day.csv", prints) + self.export(start - 600000, start, extra=["--archive", str(archive)], expected=20) + self.export(start + 360000, start + 900000, extra=["--archive", str(archive)], expected=20) + result = self.export( + start, + start + 60000, + extra=["--archive", str(archive)], + expected=23, + venue=("binance", "spot"), + ) + self.assertIn("--venue binance --market usdm --mode agg-ticks", result.stderr) + + +if __name__ == "__main__": + suite = unittest.defaultTestLoader.loadTestsFromTestCase(ExportMockTests) + result = unittest.TextTestRunner(verbosity=2).run(suite) + if result.wasSuccessful(): + print(f"PASS mock export ({result.testsRun} scenarios)", flush=True) + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py index 4705376..160ab23 100644 --- a/native/feed/tests/mock_venue.py +++ b/native/feed/tests/mock_venue.py @@ -3,6 +3,7 @@ injection.""" import base64 +import contextlib import copy import email.utils import fcntl @@ -12,6 +13,7 @@ import os import pathlib import signal +import socket import struct import subprocess import sys @@ -19,7 +21,9 @@ import threading import time import unittest +import urllib.error import urllib.parse +import urllib.request from types import SimpleNamespace from public_e2e import Soak, action_key, proven_actions @@ -267,6 +271,11 @@ def __init__(self, scripts, venue="binance"): self.exchange_info_bytes = 0 self.pings = [] self.okx_time_lookup_skip = 0 + self.instrument_reads = 0 + self.history_delay = 0.0 + self.probes = [] + self.stale_subscriptions = False + self.probe_reply = "full" self.refuse_once = None self.usdm_unknown_symbol = False self.connections = 0 @@ -300,6 +309,8 @@ def log_message(self, *arguments): def do_GET(self): parsed = urllib.parse.urlsplit(self.path) query = urllib.parse.parse_qs(parsed.query) + if parsed.path == "/api/v3/historicalTrades" and self.server.history_delay: + time.sleep(self.server.history_delay) with self.server.lock: self.server.requests.append(parsed.path) self.server.queries.append((parsed.path, query)) @@ -508,7 +519,11 @@ def rest_okx(self, path, query): server = self.server if path == "/api/v5/public/instruments": match = query["instId"][0] == server.instrument["instId"] - return 200, {"code": "0", "msg": "", "data": [server.instrument] if match else []} + body = {"code": "0", "msg": "", "data": [dict(server.instrument)] if match else []} + # Counted once the answer is built (under the server lock): scripts can wait on the + # feed's reads. + server.instrument_reads += 1 + return 200, body if path == "/api/v5/market/history-trades": limit = int(query["limit"][0]) if query.get("type") == ["2"]: @@ -619,26 +634,63 @@ def websocket(self, path, query): if self.server.venue in ("okx", "bybit"): self.subscribed(script) return - self.connection.settimeout(3) + lock, closed, ponged = threading.Lock(), threading.Event(), threading.Event() + + def send(payload, opcode=1, final=True): + with lock: + frame(self.connection, payload, opcode, final) + + def read(): + # Client frames while the script runs: PONG replies, and liveness probes (answered). + self.connection.settimeout(0.1) + while not closed.is_set(): + try: + opcode, payload = receive(self.connection) + except TimeoutError: + continue + except (EOFError, ConnectionResetError, OSError): + break + if opcode == 8: + break + if opcode == 10: + self.server.pongs.append(payload) + ponged.set() + elif opcode == 1: + request = json.loads(payload) + with self.server.lock: + self.server.probes.append(request) + streams = ( + [] if self.server.stale_subscriptions else query["streams"][0].split("/") + ) + if self.server.probe_reply == "error": + send({"code": 2, "msg": "synthetic refusal", "id": request["id"]}) + elif self.server.probe_reply == "partial": + send({"result": streams[-1:], "id": request["id"]}) + else: + send({"result": streams, "id": request["id"]}) + closed.set() + + reader = threading.Thread(target=read, daemon=True) + reader.start() for action in script: if isinstance(action, dict): - frame(self.connection, action) + send(action) elif action[0] == "pause": time.sleep(action[1]) elif action[0] == "ping": - frame(self.connection, action[1], opcode=9) - opcode, payload = receive(self.connection) - assert opcode == 10 and payload == action[1] - self.server.pongs.append(payload) + ponged.clear() + send(action[1], opcode=9) + assert ponged.wait(3) and self.server.pongs[-1] == action[1] elif action[0] == "fragment": payload = json.dumps(action[1], separators=(",", ":")).encode() middle = len(payload) // 2 - frame(self.connection, payload[:middle], final=False) - frame(self.connection, payload[middle:], opcode=0) + with lock: + frame(self.connection, payload[:middle], final=False) + frame(self.connection, payload[middle:], opcode=0) elif action[0] == "binary": - frame(self.connection, action[1], opcode=2) + send(action[1], opcode=2) elif action[0] == "raw": - frame(self.connection, action[1]) + send(action[1]) elif action[0] == "revise_trade": with self.server.lock: row = list(self.server.trades[action[1]]) @@ -649,6 +701,9 @@ def websocket(self, path, query): row = list(self.server.bars[action[1]]) row[4] = "0.70000000" self.server.bars[action[1]] = tuple(row) + elif action[0] == "add_aggregate": + with self.server.lock: + self.server.aggregates[action[1]] = action[2] elif action[0] == "lag_rest": # The REST row trails the WebSocket close, then catches up. with self.server.lock: @@ -661,17 +716,12 @@ def restore(minute=action[1], row=original): threading.Timer(action[2], restore).start() elif action[0] == "close": - frame(self.connection, b"", opcode=8) + send(b"", opcode=8) + closed.set() + reader.join(timeout=5) self.close_connection = True return - self.connection.settimeout(0.1) - while True: - try: - receive(self.connection) - except TimeoutError: - continue - except (EOFError, ConnectionResetError): - break + reader.join() self.close_connection = True def subscribed(self, script): @@ -740,6 +790,16 @@ def read(): elif action[0] == "set_instrument": with self.server.lock: self.server.instrument = dict(self.server.instrument, **action[1]) + elif action[0] == "wait_instrument_reads": + deadline = time.monotonic() + 15 + while True: + with self.server.lock: + if self.server.instrument_reads >= action[1]: + break + if time.monotonic() > deadline: + self.server.errors.append(f"instrument reads never reached {action[1]}") + break + time.sleep(0.005) elif action[0] == "okx_unconfirm": # REST trails the WebSocket confirm of this minute for a moment. with self.server.lock: @@ -772,6 +832,158 @@ def sequence(output): return [event["seq"] if event["type"] == "tick" else ("time", event["ts"]) for event in output] +def journal_lines(state): + """The retained normalized journal: segment files in message-index order.""" + lines = [] + for path in sorted((pathlib.Path(state) / "journal").glob("*.jsonl")): + lines += path.read_text().splitlines() + return lines + + +def minute_bars(count): + """Synthetic contiguous one-minute bars from 120000, in the mock's BARS tuple shape.""" + return { + 120000 + index * 60000: ( + "10.00000000", + "10.50000000", + "9.50000000", + "10.25000000", + f"{index % 7 + 1}.00000000", + 0, + 0, + 1, + ) + for index in range(count) + } + + +def bar_message(minute, bars): + opening, high, low, close, volume, _first, _last, count = bars[minute] + row = { + "t": minute, + "T": minute + 59999, + "s": "TESTUSDT", + "i": "1m", + "f": 1, + "L": 1, + "o": opening, + "h": high, + "l": low, + "c": close, + "v": volume, + "n": count, + "x": True, + } + return { + "stream": "testusdt@kline_1m", + "data": {"e": "kline", "E": minute + 60000, "s": "TESTUSDT", "k": row}, + } + + +def bar_line(minute, bars): + opening, high, low, close, volume = bars[minute][:5] + return ( + f'{{"type":"bar","bar":{{"ts_open":{minute},"o":{opening},"h":{high},"l":{low},' + f'"c":{close},"v":{volume}}}}}' + ) + + +class StreamClient: + """A minimal RFC 6455 client for the serve endpoint: masked frames out, PING answered.""" + + def __init__(self, port, path, receive_buffer=None, timeout=15): + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if receive_buffer: + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, receive_buffer) + self.socket.settimeout(timeout) + self.socket.connect(("127.0.0.1", port)) + key = base64.b64encode(os.urandom(16)).decode() + self.socket.sendall( + ( + f"GET {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nUpgrade: websocket\r\n" + f"Connection: Upgrade\r\nSec-WebSocket-Key: {key}\r\n" + "Sec-WebSocket-Version: 13\r\n\r\n" + ).encode() + ) + self.buffer = b"" + while b"\r\n\r\n" not in self.buffer: + self.fill() + head, self.buffer = self.buffer.split(b"\r\n\r\n", 1) + self.status = int(head.split(b" ")[1]) + self.body = b"" + if self.status != 101: + length = 0 + for line in head.split(b"\r\n")[1:]: + name, _, value = line.partition(b":") + if name.strip().lower() == b"content-length": + length = int(value) + while len(self.buffer) < length: + self.fill() + self.body = self.buffer[:length] + + def fill(self): + chunk = self.socket.recv(65536) + if not chunk: + raise EOFError + self.buffer += chunk + + def exact(self, size): + while len(self.buffer) < size: + self.fill() + result, self.buffer = self.buffer[:size], self.buffer[size:] + return result + + def send(self, opcode, payload=b""): + mask = os.urandom(4) + header = bytes([128 | opcode, 128 | len(payload)]) + self.socket.sendall(header + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))) + + def message(self): + """("text", str), ("pong", b""), or ("close", code, reason).""" + while True: + first, second = self.exact(2) + size = second & 127 + if size == 126: + size = struct.unpack("!H", self.exact(2))[0] + elif size == 127: + size = struct.unpack("!Q", self.exact(8))[0] + payload = self.exact(size) + opcode = first & 15 + if opcode == 1: + return ("text", payload.decode()) + if opcode == 10: + return ("pong", payload) + if opcode == 9: + self.send(10, payload) + continue + if opcode == 8: + code = struct.unpack("!H", payload[:2])[0] if len(payload) >= 2 else None + return ("close", code, payload[2:].decode()) + raise AssertionError(f"unexpected opcode {opcode}") + + def texts(self, count): + result = [] + while len(result) < count: + kind, *rest = self.message() + if kind == "text": + result.append(rest[0]) + elif kind == "close": + raise AssertionError(f"closed early: {rest}") + return result + + def close(self): + with contextlib.suppress(OSError): + self.socket.close() + + +def http_get(port, path): + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=10) as response: + return response.status, response.read() + except urllib.error.HTTPError as failure: + return failure.code, failure.read() + + class FeedMockTests(unittest.TestCase): def test_public_qualification_fails_on_any_action_difference(self): for ticks_equal, expected in ((False, 1), (True, 0)): @@ -877,10 +1089,8 @@ def run_feed(self, server, mode="bars", extra=(), expected=0, resume=False): cursor = self.directory / "state" / "cursor.json" if cursor.exists(): saved = json.loads(cursor.read_text())["cursor"] - self.assertEqual( - saved["message_index"], - len((cursor.parent / "events.jsonl").read_text().splitlines()), - ) + first = int(sorted((cursor.parent / "journal").glob("*.jsonl"))[0].name[:20]) + self.assertEqual(saved["message_index"], first + len(journal_lines(cursor.parent))) return output, result def test_bars_hole_is_healed_behind_closed_watermark(self): @@ -1247,12 +1457,12 @@ def test_429_retry_after_preserves_prefix(self): self.assertIn("rest_rate_limited", result.stderr) def test_replay_and_queue_budgets_stop_22(self): + # A queue that cannot hold even a connection marker stops at once, never waits forever. server = self.server([[candle(120000)]]) - output, _ = self.run_feed(server, extra=["--max-log-bytes", "4"], expected=22) + output, result = self.run_feed(server, extra=["--max-queue-bytes", "16"], expected=22) self.assertEqual(output, []) - import shutil - - shutil.rmtree(self.directory / "state") + self.assertIn("too small", result.stderr) + self.reset_state() server = self.server([[trade(100)]]) output, _ = self.run_feed(server, "ticks", ["--max-queue-bytes", "64"], expected=22) self.assertEqual(output, []) @@ -1284,7 +1494,7 @@ def test_sigterm_then_resume_tail_has_no_gap_or_duplicate(self): def test_changed_committed_journal_is_refused(self): server = self.server([[candle(120000)], [candle(180000)]]) self.run_feed(server, extra=["--max-messages", "1"]) - journal = self.directory / "state" / "events.jsonl" + journal = sorted((self.directory / "state" / "journal").glob("*.jsonl"))[0] journal.write_text(journal.read_text().replace("10.10000000", "10.20000000")) output, _ = self.run_feed(server, resume=True, expected=21) self.assertEqual(output, []) @@ -1301,6 +1511,82 @@ def history_pages(self, server): if path == "/api/v5/market/history-trades" and query.get("type") == ["1"] ] + def test_long_outage_drops_the_connection_and_heals_instead_of_stopping_22(self): + # The session heals a long hole slowly (REST pages take time) while the venue keeps + # sending: the reader's bounded queue fills, so the connection is dropped and everything + # after it is healed by ID + # from REST once the reconnect's overlap check passes. Exit 22 is gone; the tape is exact. + count = 2400 + trades = {99: TRADES[99]} + for offset in range(count): + trades[100 + offset] = (120001 + offset * 20, "10.00000000", "0.00000001") + + def push(identifier): + matched, price, quantity = trades[identifier] + data = { + "e": "trade", + "E": matched + 1, + "s": "TESTUSDT", + "t": identifier, + "p": price, + "q": quantity, + "T": matched, + "m": False, + } + return {"stream": "testusdt@trade", "data": data} + + # The first connection floods from print 1300 while 100..1299 heal; later ones carry + # the venue's head. + flood = [push(identifier) for identifier in range(1300, 100 + count)] + head = [push(identifier) for identifier in range(100 + count - 50, 100 + count)] + server = self.server([flood, head]) + server.trades = trades + server.history_delay = 0.4 + output, result = self.run_feed( + server, "ticks", ["--max-messages", str(count), "--max-queue-bytes", "16384"] + ) + self.assertEqual([event["seq"] for event in output], list(range(100, 100 + count))) + self.assertIn("websocket_backpressure_reconnect", result.stderr) + self.assertGreaterEqual(server.connections, 2) + + def test_quiet_binance_stream_is_probed_not_reconnected(self): + server = self.server([[candle(120000), ("pause", 3.5), candle(180000)]]) + output, result = self.run_feed( + server, extra=["--max-messages", "2", "--silence-seconds", "1"] + ) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000, 180000]) + self.assertEqual(server.connections, 1) + self.assertGreaterEqual(len(server.probes), 2) + self.assertEqual(server.probes[0], {"method": "LIST_SUBSCRIPTIONS", "id": 1}) + self.assertIn("source_probe_answered", result.stderr) + # A probe reply without the subscriptions is a lost subscription: reconnect with the + # overlap check. + self.reset_state() + server = self.server([[candle(120000), ("pause", 2.0)], [candle(180000)]]) + server.stale_subscriptions = True + output, result = self.run_feed( + server, extra=["--max-messages", "2", "--silence-seconds", "1"] + ) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000, 180000]) + self.assertGreaterEqual(server.connections, 2) + self.assertIn("source_subscription_lost", result.stderr) + # An error reply, or a list missing one of the connection's streams, also reconnects + # (never 23). + for reply, mode, script in ( + ("error", "bars", [[candle(120000), ("pause", 2.0)], [candle(180000)]]), + ("partial", "ticks", [[trade(100), ("pause", 2.0)], [trade(101), trade(102)]]), + ): + with self.subTest(reply=reply): + self.reset_state() + server = self.server(script) + server.probe_reply = reply + output, result = self.run_feed( + server, mode, ["--max-messages", "2", "--silence-seconds", "1"] + ) + self.assertEqual(len(output), 2) + self.assertGreaterEqual(server.connections, 2) + self.assertIn("source_subscription_lost", result.stderr) + def test_okx_swap_bars_confirm_flag_and_contract_units(self): server = self.server([[okx_candle(120000, "0"), okx_candle(180000)]], "okx") output, result = self.run_feed(server, extra=["--max-messages", "2"]) @@ -1524,6 +1810,104 @@ def test_okx_quiet_start_waits_for_the_first_print(self): ) self.assertNotIn('"code":20', result.stderr) + def test_usdm_quiet_start_waits_for_the_first_aggregate(self): + # No aggregate at or after --start for longer than the hour the venue searches at once: + # the feed waits, + # anchors nothing, and then proves the predecessor by searching forward hour by hour. + late = 120000 + 61 * 60000 + quiet = { + minute: ( + "10.10000000", + "10.10000000", + "10.10000000", + "10.10000000", + "0.00000000", + 0, + 0, + 0, + ) + for minute in range(120000, late, 60000) + } + quiet[late] = ( + "12.00000000", + "12.00000000", + "12.00000000", + "12.00000000", + "0.40000000", + 0, + 0, + 1, + ) + row = { + "T": late + 1, + "p": "12.00000000", + "q": "0.40000000", + "nq": "0.40000000", + "f": 1010, + "l": 1012, + } + nxt = { + "T": late + 60001, + "p": "12.00000000", + "q": "0.10000000", + "nq": "0.10000000", + "f": 1013, + "l": 1013, + } + + def closing(minute): + opening, high, low, close, volume = quiet[minute][:5] + return candle( + 120000, + t=minute, + T=minute + 59999, + o=opening, + h=high, + l=low, + c=close, + v=volume, + f=-1, + L=-1, + n=0, + ) + + server = self.server( + [ + [ + closing(120000), + closing(180000), + # Longer than the venue's predecessor retries: a blind anchor would stop 20. + ("pause", 3.0), + ( + "add_aggregate", + 500, + (row["T"], row["p"], row["q"], row["nq"], row["f"], row["l"]), + ), + ( + "add_aggregate", + 501, + (nxt["T"], nxt["p"], nxt["q"], nxt["nq"], nxt["f"], nxt["l"]), + ), + aggregate(500, a=500, **row), + closing(late), + aggregate(501, a=501, **nxt), + ] + ], + "usdm", + ) + server.aggregates = {499: AGGREGATES[499]} + server.bars = quiet + output, result = self.run_feed(server, "agg-ticks", ["--max-messages", "63"]) + minutes = [("time", minute + 60000) for minute in range(120000, late, 60000)] + self.assertEqual(sequence(output), [*minutes, 500, ("time", late + 60000)]) + self.assertNotIn('"code":20', result.stderr) + windows = [ + int(query["startTime"][0]) + for path, query in server.queries + if path == "/fapi/v1/aggTrades" and "startTime" in query + ] + self.assertIn(120000 + 3600000, windows) + def test_bybit_http_403_is_a_retryable_rate_ban(self): server = self.server([[bybit_push(bybit_row(120000, True))]], "bybit") server.fail_status = 403 @@ -1534,17 +1918,37 @@ def test_bybit_http_403_is_a_retryable_rate_ban(self): def test_okx_changed_contract_multiplier_stops_21_on_reconnect(self): notice = {"event": "notice", "code": "64008", "msg": "upgrade", "connId": "mock"} - server = self.server( - [ - # The pause lets the first connection's instrument check and its bar finish first. - [okx_candle(120000), ("pause", 1.0), ("set_instrument", {"ctVal": "0.1"}), notice], - [okx_candle(180000)], - ], - "okx", - ) - output, result = self.run_feed(server, expected=21) - self.assertEqual([event["bar"]["ts_open"] for event in output], [120000]) - self.assertIn("contract multiplier changed", result.stderr) + for mode, first, emitted in ( + ("bars", okx_candle(120000), [120000]), + ("ticks", okx_trades(100, 101), [100, 101]), + ): + with self.subTest(mode=mode): + self.reset_state() + server = self.server( + [ + # Two instrument reads answered (startup, then the first connection's + # re-check) + # before the instrument changes: no timing assumption. + [ + ("wait_instrument_reads", 2), + first, + ("set_instrument", {"ctVal": "0.1"}), + notice, + ], + [okx_candle(180000)], + ], + "okx", + ) + output, result = self.run_feed(server, mode, expected=21) + self.assertEqual( + [ + event["bar"]["ts_open"] if mode == "bars" else event["seq"] + for event in output + ], + emitted, + ) + self.assertIn("contract multiplier changed", result.stderr) + self.assertEqual(server.instrument_reads, 3) def test_okx_websocket_and_rest_lexemes_differ_but_overlap_values_match(self): notice = {"event": "notice", "code": "64008", "msg": "upgrade", "connId": "mock"} @@ -1896,6 +2300,419 @@ def test_coinbase_is_deferred(self): self.assertIn("Coinbase is deferred", result.stderr) self.assertFalse((self.directory / "state").exists()) + def serve( + self, server, mode="bars", extra=(), resume=False, environment=None, listen="127.0.0.1:0" + ): + command = self.command(server, mode, [*extra, "--listen", listen], resume) + command[1] = "serve" + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=dict(os.environ, **(environment or {})), + ) + records, ready = [], threading.Event() + + def read(): + for line in process.stderr: + try: + records.append(json.loads(line)) + except ValueError: + # A sanitizer report, kept so that a failure shows it. + records.append({"event": "raw", "line": line}) + if records[-1]["event"] == "serve_listening": + ready.set() + ready.set() + + threading.Thread(target=read, daemon=True).start() + + def cleanup(): + if process.poll() is None: + process.kill() + process.wait() + + self.addCleanup(cleanup) + self.assertTrue(ready.wait(15)) + listening = [record for record in records if record["event"] == "serve_listening"] + self.assertTrue(listening, records) + return process, listening[0]["port"], records + + def status(self, port): + code, body = http_get(port, "/v1/status") + self.assertEqual(code, 200, body) + return json.loads(body) + + def wait_status(self, port, predicate, timeout=20): + deadline = time.monotonic() + timeout + status = None + while time.monotonic() < deadline: + status = self.status(port) + if predicate(status): + return status + time.sleep(0.05) + self.fail(f"status never satisfied the condition: {status}") + + def stop(self, process, expected=0): + process.send_signal(signal.SIGTERM) + self.assertEqual(process.wait(timeout=30), expected) + + def test_serve_streams_from_cursor_with_status_snapshot_and_keepalive(self): + server = self.server([[candle(240000)]]) + process, port, records = self.serve(server) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] == 3) + journal = journal_lines(self.directory / "state") + self.assertEqual(len(journal), 3) + self.assertEqual(status["retained"]["first_index"], 0) + self.assertEqual(status["source"]["venue"], "binance") + self.assertEqual(status["source"]["mode"], "bars") + self.assertEqual(status["last"]["verified_cut"], 300000) + self.assertEqual(status["clients"], 0) + code, body = http_get(port, "/v1/snapshot") + self.assertEqual((code, body.decode().splitlines()), (200, journal)) + epoch = status["epoch"] + client = StreamClient(port, f"/v1/stream?epoch={epoch}&from=0") + self.assertEqual(client.status, 101) + self.assertEqual(client.texts(3), journal) + # A quiet stream: an unsolicited PONG within 5 s keeps the runner's 15 s idle deadline + # alive. + started = time.monotonic() + self.assertEqual(client.message()[0], "pong") + self.assertLess(time.monotonic() - started, 6.5) + self.assertEqual(client.message()[0], "pong") + self.assertLess(time.monotonic() - started, 12) + tail = StreamClient(port, f"/v1/stream?epoch={epoch}&from=2") + self.assertEqual(tail.texts(1), journal[2:]) + self.assertEqual(self.status(port)["clients"], 2) + for path, expected in ( + (f"/v1/stream?epoch={'0' * 64}&from=0", 409), + (f"/v1/stream?epoch={epoch}&from=4", 416), + (f"/v1/stream?epoch={epoch}", 400), + (f"/v1/stream?epoch={epoch}&from=-1", 400), + ): + with self.subTest(path=path): + refused = StreamClient(port, path) + self.assertEqual(refused.status, expected, refused.body) + self.assertIn("error", json.loads(refused.body)) + refused.close() + self.stop(process) + while True: + kind, *rest = client.message() + if kind == "close": + self.assertEqual(rest[0], 1001) + break + self.assertTrue(any(record["event"] == "client_refused" for record in records)) + + def test_serve_expired_cursor_and_snapshot_fail_explicitly(self): + server = self.server([[]]) + server.bars = minute_bars(300) + script = [] + for index in range(300): + script += [bar_message(120000 + index * 60000, server.bars), ("pause", 0.002)] + server.scripts = [script] + process, port, _ = self.serve( + server, extra=["--segment-bytes", "4096", "--replay-bytes", "8192"] + ) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] == 300) + first = status["retained"]["first_index"] + self.assertGreater(first, 0) + expired = StreamClient(port, f"/v1/stream?epoch={status['epoch']}&from={first - 1}") + self.assertEqual(expired.status, 410) + self.assertIn("cursor expired", json.loads(expired.body)["error"]) + code, body = http_get(port, "/v1/snapshot") + self.assertEqual(code, 410) + self.assertIn("expired", json.loads(body)["error"]) + client = StreamClient(port, f"/v1/stream?epoch={status['epoch']}&from={first}") + expected = [bar_line(120000 + index * 60000, server.bars) for index in range(first, 300)] + self.assertEqual(client.texts(300 - first), expected) + self.assertEqual(journal_lines(self.directory / "state")[-len(expected) :], expected) + self.stop(process) + + def test_serve_resume_after_producer_sigkill_reconnects_at_the_cursor(self): + server = self.server([[]]) + server.bars = minute_bars(20) + server.scripts = [ + [bar_message(120000 + 9 * 60000, server.bars)], + [bar_message(120000 + 19 * 60000, server.bars)], + ] + expected = [bar_line(120000 + index * 60000, server.bars) for index in range(20)] + process, port, _ = self.serve(server) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] == 10) + epoch = status["epoch"] + client = StreamClient(port, f"/v1/stream?epoch={epoch}&from=0") + received = client.texts(6) + client.close() + process.send_signal(signal.SIGKILL) + process.wait(timeout=10) + process, port, _ = self.serve(server, resume=True) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] == 20) + self.assertEqual(status["epoch"], epoch) + resumed = StreamClient(port, f"/v1/stream?epoch={epoch}&from={len(received)}") + received += resumed.texts(20 - len(received)) + self.assertEqual(received, expected) + self.assertEqual(journal_lines(self.directory / "state"), expected) + self.stop(process) + + def test_serve_slow_client_is_closed_with_a_reason_and_never_blocks_others(self): + # A burst well beyond the kernel's socket buffers plus the client queue: a client that + # stops reading overflows its own queue and is closed with a reason; the producer and a + # reading client never wait. + server = self.server([[]]) + count = 70000 + server.bars = minute_bars(count) + last = 120000 + (count - 1) * 60000 + server.scripts = [ + [bar_message(120000, server.bars), ("pause", 2.0), bar_message(last, server.bars)] + ] + process, port, records = self.serve(server, extra=["--client-queue-bytes", "1048576"]) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] >= 1) + path = f"/v1/stream?epoch={status['epoch']}&from=0" + stuck = StreamClient(port, path, receive_buffer=4096, timeout=60) + self.assertEqual(stuck.texts(1), [bar_line(120000, server.bars)]) + received = [] + + def consume(): + stream = StreamClient(port, path, timeout=60) + received.extend(stream.texts(count)) + stream.close() + + reader = threading.Thread(target=consume, daemon=True) + reader.start() + # The stuck client reads again once it is cut off, well inside the 10 s write bound. + deadline = time.monotonic() + 60 + while not any(record["event"] == "client_slow" for record in records): + self.assertLess(time.monotonic(), deadline, "the stuck client was never cut off") + time.sleep(0.01) + lines, closed = 1, None + while closed is None: + kind, *rest = stuck.message() + if kind == "text": + self.assertEqual(rest[0], bar_line(120000 + lines * 60000, server.bars)) + lines += 1 + elif kind == "close": + closed = rest + self.assertEqual(closed[0], 1008) + self.assertIn("slow consumer", closed[1]) + self.assertLess(lines, count) + # After serve's close, a client frame ends the connection instead of being read on. + stuck.socket.settimeout(5) + stuck.send(1, b"after close") + with self.assertRaises((EOFError, ConnectionResetError)): + while True: + stuck.message() + stuck.close() + reader.join(timeout=60) + self.assertEqual(len(received), count) + self.assertEqual(received[-1], bar_line(last, server.bars)) + self.assertIsNone(process.poll()) + self.stop(process) + + def test_serve_fanout_stress_clients_connect_and_disconnect_during_rotation(self): + server = self.server([[]]) + count = 1500 + server.bars = minute_bars(count) + script = [bar_message(120000, server.bars), ("pause", 1.0)] + for index in range(1, count): + script += [bar_message(120000 + index * 60000, server.bars), ("pause", 0.002)] + server.scripts = [script] + expected = [bar_line(120000 + index * 60000, server.bars) for index in range(count)] + process, port, _ = self.serve( + server, + extra=[ + "--segment-bytes", + "4096", + "--replay-bytes", + "16384", + "--client-queue-bytes", + "8192", + ], + ) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] >= 1) + epoch, outcomes, failures = status["epoch"], [], [] + finished = threading.Event() + + def client(seed): + import random + + chooser = random.Random(seed) + while not finished.is_set(): + try: + current = self.status(port)["retained"] + start = chooser.randint(current["first_index"], current["next_index"]) + slow = chooser.random() < 0.2 + stream = StreamClient( + port, + f"/v1/stream?epoch={epoch}&from={start}", + receive_buffer=4096 if slow else None, + ) + if stream.status == 410: + outcomes.append("expired-at-handshake") + stream.close() + continue + if stream.status != 101: + failures.append(("status", stream.status, stream.body)) + return + index, wanted = start, chooser.randint(1, 120) + while index < start + wanted and index < count: + kind, *rest = stream.message() + if kind == "pong": + continue + if kind == "close": + if rest[0] not in (1008, 4410, 1001): + failures.append(("close", rest)) + outcomes.append(f"closed-{rest[0]}") + break + if rest[0] != expected[index]: + failures.append(("line", index, rest[0][:80])) + return + index += 1 + if slow: + time.sleep(0.01) + else: + outcomes.append("read") + stream.close() + except TimeoutError: + # A silent server (a catch-up client stuck on an expired segment) is a failure. + failures.append(("timeout", start)) + return + except (OSError, EOFError) as failure: + outcomes.append(f"transport-{type(failure).__name__}") + + threads = [threading.Thread(target=client, args=(seed,), daemon=True) for seed in range(12)] + for thread in threads: + thread.start() + self.wait_status(port, lambda status: status["retained"]["next_index"] == count, timeout=60) + finished.set() + for thread in threads: + thread.join(timeout=30) + self.assertEqual(failures, []) + self.assertGreater(outcomes.count("read"), 10) + last = self.status(port) + self.assertGreater(last["retained"]["first_index"], 0) + self.assertEqual( + journal_lines(self.directory / "state"), expected[last["retained"]["first_index"] :] + ) + self.stop(process) + + def test_serve_half_open_requests_never_starve_status_or_a_runner(self): + # Eight connections that never finish their request on a listener for four clients: + # status and a runner's upgrade still get a worker at once, and each incomplete request + # is dropped within the 10 s request timeout. + server = self.server([[candle(240000)]]) + process, port, _ = self.serve(server, extra=["--max-clients", "4"]) + status = self.wait_status(port, lambda status: status["retained"]["next_index"] == 3) + stalled = [] + for _ in range(8): + connection = socket.create_connection(("127.0.0.1", port), timeout=20) + self.addCleanup(connection.close) + connection.sendall(b"GET /v1/status HTTP/1.1\r\nHost: x\r\nX-Slow: ") + stalled.append(connection) + time.sleep(0.5) + started = time.monotonic() + self.assertEqual(self.status(port)["epoch"], status["epoch"]) + runner = StreamClient(port, f"/v1/stream?epoch={status['epoch']}&from=0", timeout=5) + self.assertEqual(runner.status, 101) + self.assertEqual(len(runner.texts(3)), 3) + self.assertLess(time.monotonic() - started, 5) + runner.close() + for connection in stalled: + connection.settimeout(max(started + 15 - time.monotonic(), 0.1)) + with contextlib.suppress(ConnectionResetError): + while connection.recv(4096): + pass + self.assertLess(time.monotonic() - started, 15) + self.assertIsNone(process.poll()) + self.stop(process) + + def test_serve_accepts_the_ipv6_loopback(self): + try: + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as probe: + probe.bind(("::1", 0)) + except OSError: + self.skipTest("this host has no IPv6 loopback") + server = self.server([[candle(240000)]]) + process, port, _ = self.serve(server, listen="[::1]:0") + deadline = time.monotonic() + 20 + while True: + with urllib.request.urlopen(f"http://[::1]:{port}/v1/status", timeout=10) as response: + body = json.loads(response.read()) + if body["retained"]["next_index"] == 3: + break + self.assertLess(time.monotonic(), deadline) + time.sleep(0.05) + self.stop(process) + command = self.command(server, extra=["--listen", "[::]:0"]) + command[1] = "serve" + result = subprocess.run(command, capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 23) + self.assertIn("--allow-remote-listen", result.stderr) + + def test_serve_listens_on_loopback_unless_remote_is_explicit(self): + server = self.server([[]]) + command = self.command(server, extra=["--listen", "0.0.0.0:0"]) + command[1] = "serve" + result = subprocess.run(command, capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 23) + self.assertIn("--allow-remote-listen", result.stderr) + self.assertEqual(server.requests, []) + result = subprocess.run( + self.command(server, extra=["--listen", "127.0.0.1:0"]), + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 23) + self.assertIn("serve options", result.stderr) + + def test_rotation_and_expiry_crash_points_resume_with_an_exact_tape(self): + count = 200 + bars = minute_bars(count) + script = [] + for index in range(count): + script += [bar_message(120000 + index * 60000, bars), ("pause", 0.004)] + expected = [bar_line(120000 + index * 60000, bars) for index in range(count)] + journal = ["--segment-bytes", "4096", "--replay-bytes", "8192"] + for point in ( + "commit-append", + "commit-cursor", + "rotate-checkpoint", + "rotate-segment", + "rotate-cursor", + "retain-segment", + ): + with self.subTest(point=point): + self.reset_state() + server = self.server([script]) + server.bars = bars + crashed = subprocess.run( + self.command(server, extra=journal), + capture_output=True, + text=True, + timeout=30, + env=dict(os.environ, PINEFORGE_FEED_CRASH_AT=point + ":3"), + ) + self.assertEqual(crashed.returncode, -signal.SIGKILL, crashed.stderr[-500:]) + received = crashed.stdout.splitlines() + saved = json.loads((self.directory / "state" / "cursor.json").read_text())["cursor"] + self.assertGreaterEqual(saved["message_index"], len(received)) + remaining = count - saved["message_index"] + self.assertGreater(remaining, 0) + output, result = self.run_feed( + server, + extra=[ + *journal, + *("--output-from", str(len(received)), "--max-messages", str(remaining)), + ], + resume=True, + ) + self.assertEqual(len(output), count - len(received)) + received += result.stdout.splitlines() + self.assertEqual(received, expected) + if point == "retain-segment": + self.assertIn("expired_duplicate_unverified", result.stderr) + retained = journal_lines(self.directory / "state") + self.assertEqual(retained, expected[count - len(retained) :]) + def warmup(self, server): port = server.server_address[1] output = self.directory / "warmup.csv" diff --git a/native/feed/tests/public_e2e.py b/native/feed/tests/public_e2e.py index 94ac685..fbaa56b 100644 --- a/native/feed/tests/public_e2e.py +++ b/native/feed/tests/public_e2e.py @@ -2,6 +2,7 @@ """Explicit public-data soak; captures stay in the operator-selected directory.""" import argparse +import ctypes import hashlib import http.server import itertools @@ -9,6 +10,7 @@ import os import pathlib import signal +import sqlite3 import struct import subprocess import threading @@ -28,9 +30,136 @@ def rows(path, exact=False): ] +def journal_rows(state, exact=False): + """The feed's retained durable journal: segment files in message-index order.""" + result = [] + for path in sorted((pathlib.Path(state) / "journal").glob("*.jsonl")): + result += rows(path, exact) + return result + + SCRIPT_TF_MS = 60000 # the runner and batch both run --script-tf 1 +class StrategyBar(ctypes.Structure): + _fields_ = [ + ("open", ctypes.c_double), + ("high", ctypes.c_double), + ("low", ctypes.c_double), + ("close", ctypes.c_double), + ("volume", ctypes.c_double), + ("timestamp", ctypes.c_int64), + ] + + +class StrategyTick(ctypes.Structure): + _fields_ = [ + ("timestamp", ctypes.c_int64), + ("sequence", ctypes.c_uint64), + ("price", ctypes.c_double), + ("quantity", ctypes.c_double), + ] + + +def runner_bars(strategy, warmup, ledger, symbol, qty_step=None): + """The runner's own tick-built bars. Its committed ledger inputs are replayed in order through + the observed build of the same strategy; every input must reach the state hash the runner + recorded for it, so this is the runner's computation, not a re-implementation. The bar the + strategy saw at each time event is read back through the engine observer + (`equivalence_source_bar`). Returns {bar open time: (o, h, l, c, v) as the runner's doubles}.""" + library = ctypes.CDLL(str(strategy)) + signatures = { + "strategy_create": ([ctypes.c_char_p], ctypes.c_void_p), + "strategy_free": ([ctypes.c_void_p], None), + "strategy_set_syminfo_timezone": ([ctypes.c_void_p, ctypes.c_char_p], None), + "strategy_set_chart_timezone": ([ctypes.c_void_p, ctypes.c_char_p], None), + "strategy_set_syminfo_session": ([ctypes.c_void_p, ctypes.c_char_p], None), + "strategy_set_syminfo_string": ( + [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p], + ctypes.c_int, + ), + "strategy_set_syminfo_metadata": ( + [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_double], + None, + ), + "strategy_stream_begin": ( + [ + ctypes.c_void_p, + ctypes.POINTER(StrategyBar), + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_char_p, + ], + ctypes.c_int, + ), + "strategy_stream_push_ticks": ( + [ctypes.c_void_p, ctypes.POINTER(StrategyTick), ctypes.c_int], + ctypes.c_int, + ), + "strategy_stream_advance_time": ([ctypes.c_void_p, ctypes.c_int64], ctypes.c_int), + "strategy_stream_state_hash": ([ctypes.c_void_p], ctypes.c_uint64), + "strategy_stream_order_actions_clear": ([ctypes.c_void_p], None), + "equivalence_source_bar": ( + [ctypes.c_void_p, ctypes.c_int64, ctypes.POINTER(StrategyBar)], + ctypes.c_int, + ), + } + for name, (arguments, result) in signatures.items(): + getattr(library, name).argtypes = arguments + getattr(library, name).restype = result + with open(warmup) as source: + lines = source.read().splitlines() + assert lines[0] == "timestamp,open,high,low,close,volume" + history = [] + for line in lines[1:]: + values = line.split(",") + history.append(StrategyBar(*(float(value) for value in values[1:]), int(values[0]))) + array = (StrategyBar * len(history))(*history) + with sqlite3.connect(f"file:{ledger}?mode=ro", uri=True) as connection: + inputs = connection.execute( + "SELECT input_index, canonical_json, state_hash FROM inputs ORDER BY input_index" + ).fetchall() + assert [row[0] for row in inputs] == list(range(len(inputs))), "ledger inputs have a gap" + handle = library.strategy_create(None) + assert handle, "strategy_create failed" + bars = {} + try: + # The runner's defaults for a run without symbol metadata: UTC, 24x7, its --symbol label. + library.strategy_set_syminfo_timezone(handle, b"UTC") + library.strategy_set_chart_timezone(handle, b"UTC") + library.strategy_set_syminfo_session(handle, b"24x7") + assert library.strategy_set_syminfo_string(handle, b"tickerid", symbol.encode()) == 0 + ticker = symbol.split(":", 1)[-1] + assert library.strategy_set_syminfo_string(handle, b"ticker", ticker.encode()) == 0 + if qty_step: + # The runner's --syminfo qty_step=V, read with std::stod. + library.strategy_set_syminfo_metadata(handle, b"qty_step", float(qty_step)) + assert library.strategy_stream_begin(handle, array, len(history), b"1", b"1") == 0 + library.strategy_stream_order_actions_clear(handle) + for index, text, recorded in inputs: + event = json.loads(text, parse_float=Decimal) + if event["type"] == "tick": + tick = StrategyTick( + event["ts"], event["seq"], float(event["price"]), float(event["qty"]) + ) + assert library.strategy_stream_push_ticks(handle, ctypes.byref(tick), 1) == 0 + else: + assert event["type"] == "time", event + assert library.strategy_stream_advance_time(handle, event["ts"]) == 0 + library.strategy_stream_order_actions_clear(handle) + state = str(library.strategy_stream_state_hash(handle)) + assert state == recorded, f"replay left the runner's state at input {index}" + if event["type"] == "time": + bar = StrategyBar() + assert library.equivalence_source_bar(handle, 60000, ctypes.byref(bar)) == 0 + assert str(library.strategy_stream_state_hash(handle)) == recorded + values = (bar.open, bar.high, bar.low, bar.close, bar.volume) + assert bars.setdefault(bar.timestamp, values) == values + finally: + library.strategy_free(handle) + return bars + + def action_key(record, mode): """Comparable action fields. Ticks use the predeclared R-B2 mapping: an action timestamp becomes floor(ts / script_tf) * script_tf on both sides, because a tick fill carries its @@ -266,6 +395,8 @@ def runner_command(self, mode, name, ledger, source, cursor=0): f"http://127.0.0.1:{self.receiver.server_port}/actions/{name}", "--allow-insecure-http", ] + if getattr(self.options, "qty_step", None): + command += ["--syminfo", f"qty_step={self.options.qty_step}"] if cursor: command += ["--from-input", str(cursor)] return command @@ -387,7 +518,7 @@ def validate(self, mode, duration, cursor, minutes): json.loads(line, parse_float=str, parse_int=str) for line in source if line.strip() ] assert len(tape) == cursor, "capture/runner cursor mismatch" - durable = rows(self.directory / (mode + "-state/events.jsonl"), True) + durable = journal_rows(self.directory / (mode + "-state"), True) assert tape == durable, "restart changed normalized prefix or message boundaries" self.receipt(f"PASS {mode} restart without gap or duplicate messages={cursor}") if mode == "bars": @@ -511,15 +642,7 @@ def compare_minutes(): ",".join(str(bar[key]) for key in ("ts_open", "o", "h", "l", "c", "v")) + "\n" ) batch_actions = self.directory / (mode + "-batch-actions.jsonl") - subprocess.run( - [ - self.options.batch_probe, - self.options.observed_strategy, - str(combined), - str(batch_actions), - ], - check=True, - ) + subprocess.run(self.batch_command(combined, batch_actions), check=True) expected_actions = [ action_key(record, mode) for record in rows(batch_actions) @@ -546,6 +669,9 @@ def compare_minutes(): ) if mode != "bars" and not self.venue.candle_is_print_sum: self.kline_batch(mode, normalized, actual_actions, len(differing)) + exported = None + if mode != "bars" and getattr(self.options, "export", False): + exported = self.export_check(mode, normalized, actual_actions) if mode != "bars": name = mode + "-replay" completed = subprocess.run( @@ -568,6 +694,7 @@ def compare_minutes(): ] self.receipt(f"PASS {mode} same-print runner replay actions equal actions={len(raw)}") return { + "export": exported, "duration_seconds": duration, "messages": cursor, "minutes": len(normalized), @@ -576,6 +703,107 @@ def compare_minutes(): "batch_actions_equal": equal, } + def batch_command(self, bars, actions): + """The batch probe over a warmup-format CSV, with the runner's qty_step when it has one.""" + command = [ + self.options.batch_probe, + self.options.observed_strategy, + str(bars), + str(actions), + ] + if getattr(self.options, "qty_step", None): + command.append(self.options.qty_step) + return command + + def export_check(self, mode, normalized, actual_actions): + """`pineforge-feed export` over the proven window, from venue REST: its prints-built bars + must equal the bars the runner built from the same prints (the tape's minutes), value for + value, and the batch over the exported CSV must equal the runner's actions.""" + end = normalized[-1]["ts_open"] + 60000 + output = self.directory / (mode + "-export.csv") + command = [ + self.options.feed, + "export", + "--venue", + self.venue.venue, + "--market", + self.venue.market, + "--symbol", + self.venue.symbol, + "--mode", + mode, + "--start", + str(self.cut), + "--end", + str(end), + "--output", + str(output), + "--warmup", + str(self.directory / "warmup.csv"), + ] + if getattr(self.options, "qty_step", None): + command += ["--qty-step", self.options.qty_step] + with open(self.directory / (mode + "-export.stderr"), "w") as error: + subprocess.run(command, stderr=error, check=True, timeout=600) + with open(output) as source: + lines = source.read().splitlines() + assert lines[0] == "timestamp,open,high,low,close,volume" + exported = [ + dict( + zip( + ("ts_open", "o", "h", "l", "c", "v"), + [int(row[0]), *(Decimal(v) for v in row[1:])], + strict=True, + ) + ) + for row in (line.split(",") for line in lines[1:]) + ] + prices = ("ts_open", "o", "h", "l", "c") + assert [{key: bar[key] for key in prices} for bar in exported] == [ + {key: bar[key] for key in prices} for bar in normalized + ], "exported prices differ from the tape's prints-built bars" + # The runner's own bars, bit for bit: every field, volume included, must be the double the + # runner built (the batch reads the exported CSV with the same conversion). + built = runner_bars( + self.options.observed_strategy, + self.directory / "warmup.csv", + self.directory / (mode + ".sqlite3"), + self.venue.label, + getattr(self.options, "qty_step", None), + ) + for bar in exported: + runner = built.get(bar["ts_open"]) + assert runner is not None, f"the runner built no bar at {bar['ts_open']}" + values = tuple(float(bar[key]) for key in ("o", "h", "l", "c", "v")) + assert values == runner, f"export {bar} differs from the runner's bar {runner}" + manifest = json.loads((self.directory / (mode + "-export.csv.manifest.json")).read_text()) + combined = self.directory / (mode + "-export-combined.csv") + with open(combined, "w") as target: + target.write((self.directory / "warmup.csv").read_text()) + target.write("\n".join(lines[1:]) + "\n") + actions = self.directory / (mode + "-export-batch-actions.jsonl") + subprocess.run( + self.batch_command(combined, actions), + check=True, + ) + expected = [ + action_key(record, mode) + for record in rows(actions) + if record["origin_input_index"] >= 200 + ] + equal = expected == actual_actions + self.receipt( + f"{'PASS' if equal else 'FAIL'} {mode} export prints-built bars equal the runner's " + f"tick-built bars (state-hash-proven replay, read via the observer) " + f"minutes={len(exported)} bars_bit_exact={len(exported)} " + f"qty_step={manifest['qty_step']} volume_rule={manifest['volume_rule']} " + f"prints={manifest['prints']} " + f"quiet_minutes={manifest['quiet_minutes']} " + f"batch_over_export_actions={len(expected)} runner={len(actual_actions)}" + ) + assert equal, "the batch over the exported bars differs from the runner" + return {"minutes": len(exported), "prints": manifest["prints"], "actions": len(expected)} + def kline_batch(self, mode, normalized, actual_actions, kline_differs): """Information, not a gate: the same strategy over Binance klines instead of the bars built from the prints. Aggregates can straddle a minute edge, so fills there can differ.""" @@ -587,7 +815,7 @@ def kline_batch(self, mode, normalized, actual_actions, kline_differs): output.write(",".join([str(candle[0]), *candle[1]]) + "\n") actions = self.directory / (mode + "-kline-batch-actions.jsonl") subprocess.run( - [self.options.batch_probe, self.options.observed_strategy, str(combined), str(actions)], + self.batch_command(combined, actions), check=True, ) expected = [ @@ -681,6 +909,11 @@ def run(self): "--modes", default="bars,ticks", help="bars, ticks or agg-ticks, comma-separated" ) parser.add_argument("--bar-minutes", type=int, default=46) + # The runner's symbol quantity step (--syminfo qty_step=V); export and the batch use the same. + parser.add_argument("--qty-step") parser.add_argument("--tick-minutes", type=int, default=21) parser.add_argument("--restart-seconds", type=int, default=180) + parser.add_argument( + "--export", action="store_true", help="also check `pineforge-feed export` over tick windows" + ) raise SystemExit(Soak(parser.parse_args()).run()) diff --git a/native/feed/tests/serve_e2e.py b/native/feed/tests/serve_e2e.py new file mode 100644 index 0000000..1132aa6 --- /dev/null +++ b/native/feed/tests/serve_e2e.py @@ -0,0 +1,263 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Explicit public-data soak through `serve`: a producer, runners on --feed-url, cursor resumes.""" + +import argparse +import json +import os +import signal +import sqlite3 +import subprocess +import time +import traceback +import urllib.request +from decimal import Decimal + +from public_e2e import Soak, action_key, journal_rows, proven_actions, rows + + +def ledger_inputs(path): + with sqlite3.connect(path) as ledger: + return [ + row[0] + for row in ledger.execute("SELECT canonical_json FROM inputs ORDER BY input_index") + ] + + +class ServeSoak(Soak): + def __init__(self, options): + options.modes = options.mode + options.bar_minutes = options.tick_minutes = options.minutes + super().__init__(options) + self.mode = options.mode + self.state = self.directory / (self.mode + "-state") + self.producer = None + self.port = None + self.epoch = None + self.generation = 0 + self.runners = {} + + def start_producer(self, resume): + command = [ + self.options.feed, + "serve", + "--venue", + self.venue.venue, + "--market", + self.venue.market, + "--symbol", + self.venue.symbol, + "--mode", + self.mode, + "--state-dir", + str(self.state), + "--listen", + "127.0.0.1:0", + ] + command += ["--resume"] if resume else ["--start", str(self.cut)] + error = self.directory / f"producer-{self.generation}.stderr" + with open(error, "w") as log: + self.producer = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=log) + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + for line in error.read_text().splitlines(): + record = json.loads(line) + if record["event"] == "serve_listening": + self.port = record["port"] + status = self.status() + assert self.epoch in (None, status["epoch"]), ( + "the resumed producer changed the epoch" + ) + self.epoch = status["epoch"] + self.generation += 1 + return status + assert self.producer.poll() is None, f"producer exited {self.producer.returncode}" + time.sleep(0.2) + raise AssertionError("producer never listened") + + def status(self): + with urllib.request.urlopen( + f"http://127.0.0.1:{self.port}/v1/status", timeout=10 + ) as response: + return json.loads(response.read()) + + def start_runner(self, name, cursor): + state = self.runners.setdefault(name, {"generations": 0, "exits": []}) + generation = state["generations"] + url = f"ws://127.0.0.1:{self.port}/v1/stream?epoch={self.epoch}&from={cursor}" + command = self.runner_command( + self.mode, name, self.directory / (name + ".sqlite3"), "-", cursor + ) + index = command.index("--feed") + command[index : index + 2] = ["--feed-url", url] + prefix = self.directory / f"{name}-runner-{generation}" + with open(f"{prefix}.stdout", "w") as output, open(f"{prefix}.stderr", "w") as error: + state["process"] = subprocess.Popen( + command, stdin=subprocess.DEVNULL, stdout=output, stderr=error + ) + state["generations"] += 1 + return url + + def wait_runner(self, name, timeout=60): + state = self.runners[name] + code = state["process"].wait(timeout=timeout) + state["exits"].append(code) + return code, len(ledger_inputs(self.directory / (name + ".sqlite3"))) + + def run(self): + try: + self.warmup() + started = time.monotonic() + self.start_producer(False) + names = [ + f"{self.mode}" if index == 0 else f"{self.mode}-r{index + 1}" + for index in range(self.options.runners) + ] + for name in names: + self.start_runner(name, 0) + self.receipt( + f"START serve {self.mode} public {self.venue.venue} {self.venue.market} " + f"{self.venue.symbol} cut={self.cut} runners={len(names)} " + f"minimum_minutes={self.options.minutes}" + ) + events = [] + if len(names) > 1: + events.append((self.options.runner_restart_seconds, "runner")) + if self.options.producer_kill_seconds: + events.append((self.options.producer_kill_seconds, "producer")) + for moment, kind in sorted(events): + while time.monotonic() - started < moment: + assert self.producer.poll() is None, "producer stopped early" + for name in names: + assert self.runners[name]["process"].poll() is None, ( + f"runner {name} stopped early" + ) + time.sleep(1) + if kind == "runner": + name = names[-1] + self.runners[name]["process"].send_signal(signal.SIGTERM) + code, cursor = self.wait_runner(name) + assert code == 130, f"runner {name} exit={code}" + self.start_runner(name, cursor) + self.receipt(f"RESTART runner {name} --from-input={cursor} from={cursor}") + else: + self.producer.send_signal(signal.SIGKILL) + assert self.producer.wait(timeout=20) == -signal.SIGKILL + cursors = {} + for name in names: + code, cursors[name] = self.wait_runner(name) + assert code == 1, f"runner {name} exit={code} after the producer was killed" + status = self.start_producer(True) + for name in names: + self.start_runner(name, cursors[name]) + self.receipt( + "RESUME producer --resume epoch-unchanged next_index=" + f"{status['retained']['next_index']} " + + " ".join( + f"{name}:--from-input={cursor}" for name, cursor in cursors.items() + ) + ) + while time.monotonic() - started < self.options.minutes * 60: + assert self.producer.poll() is None, "producer stopped early" + for name in names: + assert self.runners[name]["process"].poll() is None, ( + f"runner {name} stopped early" + ) + time.sleep(5) + final = self.status() + self.producer.send_signal(signal.SIGTERM) + assert self.producer.wait(timeout=30) == 0, "producer did not stop cleanly" + cursors = {name: self.wait_runner(name)[1] for name in names} + duration = time.monotonic() - started + journal = journal_rows(self.state, True) + assert final["retained"]["first_index"] == 0 + assert len(journal) >= final["retained"]["next_index"] + for name in names: + ledger = [ + json.loads(text, parse_float=Decimal) + for text in ledger_inputs(self.directory / (name + ".sqlite3")) + ] + assert ledger == journal[: len(ledger)], ( + f"runner {name} ledger differs from the journal prefix" + ) + assert len(ledger) == len(journal), ( + f"runner {name} committed {len(ledger)} of {len(journal)}" + ) + self.receipt( + f"PASS serve {self.mode} runner {name} ledger equals the journal " + f"messages={len(ledger)} generations={self.runners[name]['generations']} " + "(no loss, no duplicate)" + ) + idle = [ + path.name + for path in self.directory.glob("*-runner-*.stderr") + if "idle or message timeout" in path.read_text() + ] + assert not idle, f"runner idle deadline tripped: {idle}" + self.receipt( + f"PASS serve {self.mode} quiet periods never tripped the runner idle deadline " + f"runner_generations={sum(state['generations'] for state in self.runners.values())}" + ) + # The journal is the tape: validate it like the stdout path (REST, batch, replay). + tape = self.directory / (self.mode + "-feed.jsonl") + with open(tape, "w") as output: + for path in sorted((self.state / "journal").glob("*.jsonl")): + output.write(path.read_text()) + result = self.validate(self.mode, duration, len(journal), self.options.minutes) + batch = [ + action_key(record, self.mode) + for record in rows(self.directory / (self.mode + "-batch-actions.jsonl")) + if record["origin_input_index"] >= 200 + ] + window = 200 + result["minutes"] + for name in names[1:]: + actual, trailing = proven_actions( + self.receiver.payloads.get(name, []), self.mode, window + ) + equal = actual == batch + result.setdefault("runners", {})[name] = { + "batch_actions_equal": equal, + "actions": len(actual), + } + result["batch_actions_equal"] = result["batch_actions_equal"] and equal + self.receipt( + f"{'PASS' if equal else 'FAIL'} serve {self.mode} runner {name} actions equal " + f"batch batch={len(batch)} runner={len(actual)} " + f"after_last_proven_minute={trailing}" + ) + self.results[self.mode] = result + except Exception: + failure = traceback.format_exc() + (self.directory / "serve-failure.txt").write_text(failure) + self.receipt(f"FAIL serve {self.mode} qualification; see serve-failure.txt") + self.results[self.mode] = {"error": failure} + finally: + if self.producer and self.producer.poll() is None: + self.producer.kill() + for state in self.runners.values(): + if state.get("process") and state["process"].poll() is None: + state["process"].kill() + self.receiver.shutdown() + (self.directory / "summary.json").write_text( + json.dumps(self.results, indent=2, default=str) + "\n" + ) + failed = any( + "error" in result or not result.get("batch_actions_equal") + for result in self.results.values() + ) + return 1 if failed else 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + for argument in ("feed", "runner", "strategy", "observed-strategy", "batch-probe", "directory"): + parser.add_argument("--" + argument, required=True) + parser.add_argument("--venue", default="binance") + parser.add_argument("--market", default="spot") + parser.add_argument("--symbol", default="BTCUSDT") + parser.add_argument("--mode", default="bars") + parser.add_argument("--minutes", type=int, default=17) + parser.add_argument("--runners", type=int, default=2) + parser.add_argument("--runner-restart-seconds", type=int, default=240) + parser.add_argument("--producer-kill-seconds", type=int, default=480) + os.environ.setdefault("PYTHONUNBUFFERED", "1") + raise SystemExit(ServeSoak(parser.parse_args()).run()) diff --git a/native/feed/tests/tsan.supp b/native/feed/tests/tsan.supp new file mode 100644 index 0000000..518c6cb --- /dev/null +++ b/native/feed/tests/tsan.supp @@ -0,0 +1,5 @@ +# ThreadSanitizer suppressions for the feed's TSan suite; nothing in the feed's own code is suppressed. +# CivetWeb 1.16 (built with STOP_FLAG_NEEDS_LOCK): STOP_FLAG_ASSIGN passes the flag's current value to its +# compare-and-swap loop with a plain read, which races benignly with the atomic reads of mg_stop and the +# master thread during shutdown. +race:STOP_FLAG_ASSIGN diff --git a/native/feed/tests/unit.cpp b/native/feed/tests/unit.cpp index 9ee537f..6a5226b 100644 --- a/native/feed/tests/unit.cpp +++ b/native/feed/tests/unit.cpp @@ -3,6 +3,10 @@ #include "bybit.hpp" #include "okx.hpp" #include "session.hpp" +#if PINEFORGE_FEED_SERVE +#include "serve.hpp" +#endif +#include #include #include #include @@ -55,7 +59,7 @@ struct SyntheticVenue final : Venue { {180000, {{180000, "12.00000000", "12.00000000", "11.00000000", "11.00000000", "0.50000000"}, 103, 104, 2, true}}, {240000, {{240000, "12.00000000", "12.00000000", "12.00000000", "12.00000000", "0.10000000"}, 105, 105, 1, true}} }; - Connection connection() const override { return {"/stream?streams=test@trade/test@kline_1m", {}, {}, &binance_frame}; } + Connection connection() const override { return {"/stream?streams=test@trade/test@kline_1m", {}, {}, [](const std::string& message) { return binance_frame(message); }, {}}; } std::vector decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } std::string kline_source() const override { return "/synthetic"; } std::size_t pages = 0; @@ -71,6 +75,7 @@ struct SyntheticVenue final : Venue { return result; } std::uint64_t first_trade_id(std::int64_t minute) override { return static_cast(bars.at(minute).first); } + std::optional predecessor_if_ready(std::int64_t) override { throw Error(23, "kline-range venue"); } // REST rows carry no x=true and no f..L, exactly like the venue's klines endpoint. std::vector klines(std::int64_t start, std::int64_t end) override { std::vector result; @@ -110,7 +115,7 @@ struct FenceVenue final : Venue { std::size_t pages = 0, candle_requests = 0; // REST lag: the next `empty_reads` history reads return nothing, the next `short_reads` drop their last row. std::size_t empty_reads = 0, short_reads = 0; - Connection connection() const override { return {"/synthetic", {}, {}, &binance_frame}; } + Connection connection() const override { return {"/synthetic", {}, {}, [](const std::string& message) { return binance_frame(message); }, {}}; } std::vector decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } std::vector history(std::uint64_t from, std::size_t limit) override { ++pages; @@ -147,6 +152,30 @@ struct FenceVenue final : Venue { VenueEvent tick(std::uint64_t id) const { return {VenueEvent::Kind::Trade, trades.at(id), {}}; } VenueEvent close(std::int64_t minute) const { return {VenueEvent::Kind::Kline, {}, candles.at(minute)}; } }; +std::string segment_file(const Temporary& temporary, std::uint64_t base) { + return temporary.path + "/journal/" + segment_name(base) + ".jsonl"; +} +std::string checkpoint_file(const Temporary& temporary, std::uint64_t base) { + return temporary.path + "/journal/" + segment_name(base) + ".checkpoint.json"; +} +// Contiguous synthetic minutes from `first`, each staged with its proof and committed in batches of `batch`. +void commit_bars(State& state, std::int64_t first, std::size_t count, std::size_t batch = 7) { + for (std::size_t index = 0; index < count; ++index) { + const Bar bar{first + static_cast(index) * 60000, "10.00000000", "10.50000000", "9.50000000", "10.25000000", "1.00000000"}; + state.stage(bar.wire(), bar); + if ((index + 1) % batch == 0) state.flush(); + } + state.flush(); +} +std::vector journal_bases(const Temporary& temporary, const char* suffix) { + std::vector bases; + for (const auto& entry : std::filesystem::directory_iterator(temporary.path + "/journal")) { + const auto name = entry.path().filename().string(); + if (name.size() > 20 && name.substr(20) == suffix) bases.push_back(std::stoull(name.substr(0, 20))); + } + std::sort(bases.begin(), bases.end()); + return bases; +} std::vector kinds(const std::vector& lines) { std::vector result; for (const auto& line : lines) { @@ -271,14 +300,14 @@ int main() { second.resume = true; expect(22, [&] { State locked(second); }); } - { std::ofstream tail(temporary.path + "/events.jsonl", std::ios::app); tail << "uncommitted-torn-tail"; } + { std::ofstream tail(segment_file(temporary, 0), std::ios::app); tail << "uncommitted-torn-tail"; } options.resume = true; options.start = -1; options.output_from = 2; { State state(options); assert(state.cursor().epoch == epoch); - assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == state.cursor().log_bytes); + assert(std::filesystem::file_size(segment_file(temporary, 0)) == state.cursor().log_bytes); std::vector output; FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); session.connected(); @@ -289,7 +318,7 @@ int main() { assert(parse_json(output.back()).at("type").text() == "time"); } { - std::fstream corrupt(temporary.path + "/events.jsonl", std::ios::in | std::ios::out); + std::fstream corrupt(segment_file(temporary, 0), std::ios::in | std::ios::out); corrupt.seekp(0); corrupt << 'X'; } @@ -309,15 +338,146 @@ int main() { passed("historical_ticks_require_closed_watermark_next_id_fence_and_rest_reconciliation"); } { + // Segments seal at --segment-bytes; sealed segments beyond --replay-bytes expire oldest first, but never + // the open segment or one holding a protected minute (the 32 closed-minute proofs and the open minute). Temporary temporary; auto options = config(temporary, "bars"); - options.max_log_bytes = 4; - State state(options); - SyntheticVenue venue; - FeedSession session(state, venue, [](const auto&) {}); - expect(22, [&] { session.ingest(venue.close(120000)); }); - assert(state.cursor().message_index == 0); - passed("bounded_replay_budget_stops_before_commit"); + options.segment_bytes = 4096; + options.replay_bytes = 8192; + std::string epoch, prefix; + std::uint64_t first = 0; + { + State state(options); + commit_bars(state, 120000, 400); + epoch = state.durable().epoch; + prefix = state.durable().prefix_hash; + first = state.first_retained(); + assert(state.durable().message_index == 400 && first > 0 && state.segments().size() >= 2); + assert(journal_bases(temporary, ".jsonl") == journal_bases(temporary, ".checkpoint.json")); + assert(journal_bases(temporary, ".jsonl").front() == first && journal_bases(temporary, ".jsonl").back() == state.durable().segment); + // The retained bytes stay within the budget plus the open segment. + assert(state.durable().log_bytes - state.segments().front().bytes <= options.replay_bytes + options.segment_bytes); + // The protected window: every proof minute is still readable. + std::vector minutes; + state.visit(state.first_retained(), [&](const std::string& line) { minutes.push_back(normalized_bar(parse_json(line)).ts); }); + assert(minutes.front() <= state.durable().proofs.front().ts && minutes.back() == 120000 + 399 * 60000); + for (std::size_t index = 1; index < minutes.size(); ++index) assert(minutes[index] == minutes[index - 1] + 60000); + expect(22, [&] { state.visit(0, [](const auto&) {}); }); + assert(state.bar(120000 + 399 * 60000) && state.bar(state.durable().proofs.front().ts)); + } + options.resume = true; + options.start = -1; + expect(22, [&] { State replay_from_origin(options); }); + options.output_from = first; + { + State resumed(options); + assert(resumed.durable().epoch == epoch && resumed.durable().prefix_hash == prefix && resumed.first_retained() == first); + assert(resumed.recent_bars().size() == 32 && resumed.durable().message_index == 400); + commit_bars(resumed, 120000 + 400 * 60000, 3); + } + options.output_from = first - 1; + expect(22, [&] { State expired(options); }); + options.output_from = first; + { State exact(options); assert(exact.durable().message_index == 403); } + // Retention by venue time: with an age limit, segments older than it expire once outside the protected window. + Temporary aged; + auto timed = config(aged, "bars"); + timed.segment_bytes = 4096; + timed.replay_bytes = 1ULL << 30; + timed.replay_age_ms = 60 * 60000; + State state(timed); + commit_bars(state, 120000, 300); + assert(state.first_retained() > 0); + assert(state.durable().cut - state.segments().front().cut <= timed.replay_age_ms + 40 * 60000); + passed("journal_segments_rotate_expire_by_bytes_and_age_and_resume"); + } + { + // Crash residue of every rotation and expiry step is recognised: files after the cursor's open segment + // (an interrupted rotation) and a checkpoint without its segment (an interrupted expiry) are removed; a + // segment without its checkpoint, or a changed checkpoint, stops 21. + Temporary temporary; + auto options = config(temporary, "bars"); + options.segment_bytes = 4096; + options.replay_bytes = 8192; + std::uint64_t open = 0, first = 0, expired = 0, next = 0; + std::string prefix; + { + State state(options); + commit_bars(state, 120000, 300); + open = state.durable().segment; + first = state.first_retained(); + next = state.durable().message_index; + prefix = state.durable().prefix_hash; + } + // An interrupted seal: the successor's checkpoint and its empty segment at the cursor's next index. + expired = first - 1; + std::filesystem::copy_file(checkpoint_file(temporary, first), checkpoint_file(temporary, expired)); + std::filesystem::copy_file(checkpoint_file(temporary, open), checkpoint_file(temporary, next)); + { std::ofstream residue(segment_file(temporary, next)); } + options.resume = true; + options.start = -1; + options.output_from = first; + { + State resumed(options); + assert(resumed.durable().prefix_hash == prefix && resumed.durable().segment == open && resumed.first_retained() == first); + assert(!std::filesystem::exists(checkpoint_file(temporary, expired)) && !std::filesystem::exists(segment_file(temporary, next))); + assert(!std::filesystem::exists(checkpoint_file(temporary, next))); + } + // Anything else is not a crash leftover: a segment after the cursor that is not the empty successor, + // or a checkpoint without its segment inside the retained range, stops 21 and is left in place. + std::filesystem::copy_file(checkpoint_file(temporary, open), checkpoint_file(temporary, next + 1000)); + { std::ofstream stray(segment_file(temporary, next + 1000)); stray << "{}\n"; } + expect(21, [&] { State stray(options); }); + assert(std::filesystem::exists(segment_file(temporary, next + 1000))); + std::filesystem::remove(segment_file(temporary, next + 1000)); + std::filesystem::remove(checkpoint_file(temporary, next + 1000)); + std::filesystem::copy_file(checkpoint_file(temporary, open), checkpoint_file(temporary, first + 1)); + expect(21, [&] { State hole(options); }); + std::filesystem::remove(checkpoint_file(temporary, first + 1)); + { State resumed(options); commit_bars(resumed, 120000 + 300 * 60000, 2); } + const auto saved = checkpoint_file(temporary, open) + ".saved"; + std::filesystem::rename(checkpoint_file(temporary, open), saved); + expect(21, [&] { State missing(options); }); + std::filesystem::rename(saved, checkpoint_file(temporary, open)); + { + std::string text; + { std::ifstream input(checkpoint_file(temporary, open)); text.assign(std::istreambuf_iterator(input), {}); } + const auto position = text.find("\"verified_cut\":"); + text[position + 15] = text[position + 15] == '1' ? '2' : '1'; + std::ofstream output(checkpoint_file(temporary, open), std::ios::trunc); + output << text; + } + expect(21, [&] { State changed(options); }); + passed("journal_rotation_and_expiry_crash_residue_recovers_or_stops_21"); + } + { + // A quiet fence-venue tick stream: time events keep closing minutes after the last print, so age and + // byte retention would expire every segment holding a print; the newest print's segment stays, and a + // resume still has the recent prints its reconnect overlap re-reads. + Temporary temporary; + auto options = config(temporary, "ticks"); + options.segment_bytes = 4096; + options.replay_bytes = 8192; + options.replay_age_ms = 5 * 60000; + { + State state(options); + state.anchor({99, 119999, "10.10000000", "0.10000000"}); + state.stage(Trade{100, 120001, "10.10000000", "0.10000000"}.wire()); + state.flush(); + for (std::int64_t minute = 1; minute <= 400; ++minute) { + const Bar proof{120000 + (minute - 1) * 60000, "10.10000000", "10.10000000", "10.10000000", "10.10000000", + minute == 1 ? "0.10000000" : "0"}; + state.stage("{\"type\":\"time\",\"ts\":" + std::to_string(120000 + minute * 60000) + "}", proof); + if (minute % 5 == 0) state.flush(); + } + state.flush(); + assert(state.segments().size() > 2 && state.first_retained() == 0 && state.recent_trades().size() == 1); + } + options.resume = true; + options.start = -1; + State resumed(options); + assert(resumed.recent_trades().size() == 1 && resumed.recent_trades().front().id == 100 && resumed.durable().seq == 100); + passed("retention_keeps_the_segment_of_the_newest_print"); } { Temporary temporary; @@ -353,7 +513,7 @@ int main() { options.resume = true; State recovered(options); assert(recovered.cursor().message_index == 0); - assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == 0); + assert(std::filesystem::file_size(segment_file(temporary, 0)) == 0); passed("failed_cursor_write_rolls_back_uncommitted_message_on_resume"); } { @@ -439,22 +599,28 @@ int main() { const Bar bar{120000, "10.10000000", "11.20000000", "9.90000000", "9.90000000", "0.60000000"}; state.stage(bar.wire(), bar); assert(state.staged() == 1 && state.cursor().message_index == 1 && state.durable().message_index == 0); - assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == 0); + assert(std::filesystem::file_size(segment_file(temporary, 0)) == 0); std::size_t seen = 0; state.visit(0, [&](const auto&) { ++seen; }); assert(seen == 1 && state.bar(120000)); const auto lines = state.flush(); assert(lines.size() == 1 && state.staged() == 0 && state.durable().message_index == 1); - assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == lines.front().size() + 1); + assert(std::filesystem::file_size(segment_file(temporary, 0)) == lines.front().size() + 1); passed("group_commit_stages_then_persists_before_publication"); } { Temporary temporary; - { std::ofstream empty(temporary.path + "/events.jsonl"); } + std::filesystem::create_directories(temporary.path + "/journal"); + { std::ofstream empty(segment_file(temporary, 0)); } + { std::ofstream checkpoint(checkpoint_file(temporary, 0)); checkpoint << "{}"; } { State state(config(temporary, "bars")); assert(state.durable().message_index == 0); } Temporary second; - { std::ofstream used(second.path + "/events.jsonl"); used << "x\n"; } + std::filesystem::create_directories(second.path + "/journal"); + { std::ofstream used(segment_file(second, 0)); used << "x\n"; } expect(23, [&] { State state(config(second, "bars")); }); + Temporary legacy; + { std::ofstream old(legacy.path + "/events.jsonl"); } + expect(23, [&] { State state(config(legacy, "bars")); }); passed("crashed_initialization_without_cursor_is_reinitialized"); } @@ -886,6 +1052,57 @@ int main() { expect(23, [&] { make_session(state, venue, [](const auto&) {}); }); passed("aggregate_mode_needs_a_next_print_fence_venue"); } +#if PINEFORGE_FEED_SERVE + { + // The serve reader follows the journal across segments by message index, as the producer appends. + Temporary temporary; + auto options = config(temporary, "bars"); + options.segment_bytes = 4096; + options.replay_bytes = 1ULL << 30; + State state(options); + commit_bars(state, 120000, 120); + JournalReader reader(temporary.path + "/journal"); + assert(reader.seek(37) && reader.index() == 37); + std::vector minutes; + while (reader.index() < 120) minutes.push_back(normalized_bar(parse_json(*reader.next())).ts); + assert(minutes.size() == 83 && minutes.front() == 120000 + 37 * 60000 && minutes.back() == 120000 + 119 * 60000); + assert(!reader.next()); + commit_bars(state, 120000 + 120 * 60000, 60); + assert(normalized_bar(parse_json(*reader.next())).ts == 120000 + 120 * 60000); + assert(state.segments().size() > 2); + std::filesystem::remove(segment_file(temporary, 0)); + JournalReader expired(temporary.path + "/journal"); + assert(!expired.seek(3) && expired.seek(state.segments()[1].base)); + expect(20, [&] { snapshot_prefix(temporary.path + "/journal", 3, 4096); }); // index 0 expired above + Temporary small; + auto bounded = config(small, "bars"); + State fresh(bounded); + commit_bars(fresh, 120000, 40); + const auto full = snapshot_prefix(small.path + "/journal", 40, 1ULL << 22); + assert(full && std::count(full->begin(), full->end(), '\n') == 40); + assert(!snapshot_prefix(small.path + "/journal", 40, full->size() - 1) && snapshot_prefix(small.path + "/journal", 40, full->size())); + assert(listen_address("localhost:0", false) == "127.0.0.1:0" && listen_address("127.0.0.2:8787", false) == "127.0.0.2:8787"); + expect(23, [] { listen_address("0.0.0.0:8787", false); }); + assert(listen_address("0.0.0.0:8787", true) == "0.0.0.0:8787"); + // The IPv6 loopback is loopback; any other IPv6 address needs --allow-remote-listen. + assert(listen_address("[::1]:8787", false) == "[::1]:8787"); + expect(23, [] { listen_address("[::]:8787", false); }); + assert(listen_address("[::]:8787", true) == "[::]:8787"); + for (const auto* invalid : {"8787", "127.0.0.1:", "127.0.0.1:70000", "[::1", "[]:80", "[127.0.0.1]:80", "::1:8787", + "example.com:80"}) + expect(23, [&] { listen_address(invalid, true); }); + passed("serve_journal_reader_follows_segments_and_listen_is_loopback_by_default"); + } +#endif + // The one-minute weight ceiling is spent by the venue's own count, up to 90%, then the next window. + assert(quota_wait_ms(2000, 100, 20, 2400, 100 * 60000 + 1000) == 0); + assert(quota_wait_ms(2150, 100, 20, 2400, 100 * 60000 + 1000) == 59500); + assert(quota_wait_ms(2150, 99, 20, 2400, 100 * 60000 + 1000) == 0); + assert(quota_wait_ms(0, 0, 20, 0, 1000) == 0); + assert(binance_frame(R"({"result":["testusdt@kline_1m"],"id":1})") == Frame::Control); + assert(binance_frame(R"({"result":[],"id":1})") == Frame::Stale); + expect(23, [] { binance_frame(R"({"result":[]})"); }); + passed("rest_weight_quota_and_liveness_probe_replies"); assert(retry_after_seconds("120", 0) == 120); assert(retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT", 1445412475) == 5); assert(retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT", 1445412490) == 0);