From 1612a3196ac98a7cc67bbc05d1ee6007094f4547 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sat, 3 Oct 2026 23:25:53 +0800 Subject: [PATCH 1/7] Add native Binance spot feed adapter --- .github/workflows/native-feed.yml | 77 +++++ native/feed/CMakeLists.txt | 66 ++++ native/feed/LICENSE | 201 +++++++++++ native/feed/NOTICE | 25 ++ native/feed/README.md | 167 +++++++++ native/feed/src/binance.cpp | 393 +++++++++++++++++++++ native/feed/src/binance.hpp | 50 +++ native/feed/src/core.cpp | 254 ++++++++++++++ native/feed/src/core.hpp | 88 +++++ native/feed/src/main.cpp | 98 ++++++ native/feed/src/state.cpp | 260 ++++++++++++++ native/feed/src/state.hpp | 52 +++ native/feed/src/transport.cpp | 326 ++++++++++++++++++ native/feed/src/transport.hpp | 48 +++ native/feed/src/venue.hpp | 21 ++ native/feed/tests/batch_probe.cpp | 63 ++++ native/feed/tests/build_ci_curl.sh | 19 ++ native/feed/tests/mock_venue.py | 528 +++++++++++++++++++++++++++++ native/feed/tests/public_e2e.py | 311 +++++++++++++++++ native/feed/tests/unit.cpp | 287 ++++++++++++++++ native/feed/vendor/json.hpp | 361 ++++++++++++++++++++ 21 files changed, 3695 insertions(+) create mode 100644 .github/workflows/native-feed.yml create mode 100644 native/feed/CMakeLists.txt create mode 100644 native/feed/LICENSE create mode 100644 native/feed/NOTICE create mode 100644 native/feed/README.md create mode 100644 native/feed/src/binance.cpp create mode 100644 native/feed/src/binance.hpp create mode 100644 native/feed/src/core.cpp create mode 100644 native/feed/src/core.hpp create mode 100644 native/feed/src/main.cpp create mode 100644 native/feed/src/state.cpp create mode 100644 native/feed/src/state.hpp create mode 100644 native/feed/src/transport.cpp create mode 100644 native/feed/src/transport.hpp create mode 100644 native/feed/src/venue.hpp create mode 100644 native/feed/tests/batch_probe.cpp create mode 100644 native/feed/tests/build_ci_curl.sh create mode 100644 native/feed/tests/mock_venue.py create mode 100644 native/feed/tests/public_e2e.py create mode 100644 native/feed/tests/unit.cpp create mode 100644 native/feed/vendor/json.hpp diff --git a/.github/workflows/native-feed.yml b/.github/workflows/native-feed.yml new file mode 100644 index 0000000..bd93497 --- /dev/null +++ b/.github/workflows/native-feed.yml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +name: Native feed +on: + push: + paths: ['native/feed/**', '.github/workflows/native-feed.yml'] + pull_request: + paths: ['native/feed/**', '.github/workflows/native-feed.yml'] + workflow_dispatch: + +permissions: + contents: read + +jobs: + release: + name: Release (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-14] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Linux prerequisites + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev + - name: macOS prerequisites + if: runner.os == 'macOS' + run: | + brew install ninja openssl@3 + echo "OPENSSL_ROOT_DIR=$(brew --prefix openssl@3)" >> "$GITHUB_ENV" + - name: Pinned WebSocket curl + run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" + - name: Configure, build and test + run: | + export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" + export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}" + cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" + cmake --build build-feed -j4 + ctest --test-dir build-feed --output-on-failure + + asan-ubsan: + name: ASan + UBSan + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev clang-18 + - run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" + - name: Instrumented core and mock venue + env: + ASAN_OPTIONS: detect_leaks=1:abort_on_error=1 + UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 + run: | + export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" + cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_CXX_COMPILER=clang++-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ + -DPINEFORGE_FEED_SANITIZER=address,undefined + cmake --build build-feed -j4 + ctest --test-dir build-feed --output-on-failure + + tsan: + name: TSan + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev clang-18 + - run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" + - name: Threaded receive, healing and termination + env: + TSAN_OPTIONS: halt_on_error=1 + run: | + export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" + cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_CXX_COMPILER=clang++-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ + -DPINEFORGE_FEED_SANITIZER=thread + cmake --build build-feed -j4 + ctest --test-dir build-feed --output-on-failure diff --git a/native/feed/CMakeLists.txt b/native/feed/CMakeLists.txt new file mode 100644 index 0000000..ecfebfb --- /dev/null +++ b/native/feed/CMakeLists.txt @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +cmake_minimum_required(VERSION 3.20) +project(pineforge_feed VERSION 0.1.0 LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +find_package(CURL 8.14.1 REQUIRED) +find_package(OpenSSL REQUIRED COMPONENTS Crypto) +find_package(Threads REQUIRED) +include(CheckCXXSourceRuns) +get_target_property(CURL_CHECK_TARGET CURL::libcurl ALIASED_TARGET) +if(NOT CURL_CHECK_TARGET) + set(CURL_CHECK_TARGET CURL::libcurl) +endif() +set(CMAKE_REQUIRED_LIBRARIES ${CURL_CHECK_TARGET}) +check_cxx_source_runs("#include +#include +int main() { + const auto* info = curl_version_info(CURLVERSION_NOW); + if (!info || info->version_num < 0x080e01) return 1; + bool ws = false, wss = false; + for (const char* const* protocol = info->protocols; protocol && *protocol; ++protocol) { + ws |= std::strcmp(*protocol, \"ws\") == 0; + wss |= std::strcmp(*protocol, \"wss\") == 0; + } + return ws && wss ? 0 : 1; +}" PINEFORGE_CURL_WEBSOCKETS) +unset(CMAKE_REQUIRED_LIBRARIES) +if(NOT PINEFORGE_CURL_WEBSOCKETS) + message(FATAL_ERROR "pineforge-feed requires libcurl >= 8.14.1 with both ws and wss") +endif() + +file(SHA256 "${CMAKE_CURRENT_SOURCE_DIR}/vendor/json.hpp" JSON_SHA256) +if(NOT JSON_SHA256 STREQUAL "9431c7c311d74be1cea0e8d1a3942a33528ac11f6e67ee67d000cf4a8af7f830") + message(FATAL_ERROR "The attributed runner/json.hpp pin does not match") +endif() + +set(PINEFORGE_FEED_SANITIZER "" CACHE STRING "Sanitizer: address,undefined or thread") +if(PINEFORGE_FEED_SANITIZER) + if(NOT PINEFORGE_FEED_SANITIZER MATCHES "^(address,undefined|thread)$") + message(FATAL_ERROR "Unsupported sanitizer profile") + endif() + add_compile_options(-fsanitize=${PINEFORGE_FEED_SANITIZER} -fno-omit-frame-pointer -g) + add_link_options(-fsanitize=${PINEFORGE_FEED_SANITIZER}) +endif() + +add_library(pineforge_feed_core STATIC src/core.cpp src/state.cpp src/transport.cpp src/binance.cpp) +target_include_directories(pineforge_feed_core PUBLIC src vendor) +target_link_libraries(pineforge_feed_core PUBLIC CURL::libcurl OpenSSL::Crypto Threads::Threads) +target_compile_options(pineforge_feed_core PRIVATE -Wall -Wextra -Wpedantic -Werror) +add_executable(pineforge-feed src/main.cpp) +target_link_libraries(pineforge-feed PRIVATE pineforge_feed_core) +target_compile_options(pineforge-feed PRIVATE -Wall -Wextra -Wpedantic -Werror) +install(TARGETS pineforge-feed RUNTIME DESTINATION bin) + +include(CTest) +if(BUILD_TESTING) + add_executable(feed-unit tests/unit.cpp) + target_link_libraries(feed-unit PRIVATE pineforge_feed_core) + target_compile_options(feed-unit PRIVATE -Wall -Wextra -Wpedantic -Werror -UNDEBUG) + add_test(NAME feed-unit COMMAND feed-unit) + find_package(Python3 REQUIRED COMPONENTS Interpreter) + add_test(NAME feed-mock COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/mock_venue.py $) + set_tests_properties(feed-mock PROPERTIES TIMEOUT 180) +endif() diff --git a/native/feed/LICENSE b/native/feed/LICENSE new file mode 100644 index 0000000..f07d00b --- /dev/null +++ b/native/feed/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 PineForge contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/native/feed/NOTICE b/native/feed/NOTICE new file mode 100644 index 0000000..362639a --- /dev/null +++ b/native/feed/NOTICE @@ -0,0 +1,25 @@ +PineForge Feed +Apache License, Version 2.0. See LICENSE. + +This product includes runner/json.hpp from PineForge Engine: + PineForge Engine + Copyright 2026 PineForge contributors + This product includes software developed for the PineForge project. + The PineForge runtime is licensed under the Apache License, Version 2.0. + +The header is an unchanged copy from engine revision +227c2236e4769ae926d2c6e4811c37ff5119c4a0, path runner/json.hpp. +SHA-256: 9431c7c311d74be1cea0e8d1a3942a33528ac11f6e67ee67d000cf4a8af7f830 +Source: https://github.com/pineforge-4pass/pineforge-engine +No engine, strategy compiler, Eigen, or Python runtime is linked by the feed. + +Linked dependencies retain their upstream terms: + libcurl: curl license (MIT/X derivative), https://curl.se/docs/copyright.html + OpenSSL 3.x: Apache License, Version 2.0, https://openssl.org +Preserve upstream licenses/notices when distributing bundled libraries. + +CI builds libcurl 8.14.1 from curl-8.14.1.tar.xz. +Archive SHA-256: f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd +Source: https://curl.se/download/curl-8.14.1.tar.xz + +Market-data access and redistribution remain subject to the venue's terms. diff --git a/native/feed/README.md b/native/feed/README.md new file mode 100644 index 0000000..2bd831c --- /dev/null +++ b/native/feed/README.md @@ -0,0 +1,167 @@ +# pineforge-feed + +An independent Apache-2.0 C++17 public-market-data adapter. It produces the +native `pineforge-live` runner's JSONL protocol; it does not load strategies, +place orders, access accounts, consume fills, or accept API keys. It is not part +of the Python package or wheel. This first release supports **Binance spot**, +`warmup` and `run`, with `bars` and raw `ticks`. There is no `serve` subcommand. + +## Build + +Requirements: CMake >= 3.20, a C++17 compiler, OpenSSL Crypto, threads, and +**libcurl >= 8.14.1 with both `ws` and `wss`**. Configure executes a linked +feature check, and every running binary checks the actually loaded libcurl. +Cross-compilation without an executable feature check is not qualified. +`vendor/json.hpp` is SHA-256 pinned; attribution and dependency pins are in +`NOTICE`. No engine linkage or additional C++ dependency is required. + +```sh +cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCURL_DIR=/path/to/qualified-curl/lib/cmake/CURL +cmake --build build-feed -j4 +ctest --test-dir build-feed --output-on-failure +``` + +`tests/build_ci_curl.sh PREFIX` builds the CI-pinned curl with WebSockets and +OpenSSL. The isolated `.github/workflows/native-feed.yml` defines Linux and +macOS Release builds, ASan+UBSan, and a separate Linux TSan build. Python and +the existing Python workflow are unaffected. Python 3.9+ is used only for the +synthetic mock tests; public qualification uses Python 3.10+. + +## A verified warmup cut + +```sh +build-feed/pineforge-feed warmup --venue binance --market spot --symbol BTCUSDT \ + --start 2026-10-03T00:00:00Z --end 2026-10-03T03:20:00Z --output warmup.csv +build-feed/pineforge-feed run --venue binance --market spot --symbol BTCUSDT \ + --mode bars --state-dir feed-state --start 2026-10-03T03:20:00Z | + pineforge-live run --strategy strategy.so --warmup warmup.csv --mode bars \ + --script-tf 1 --feed - --ledger runner.sqlite3 --symbol BINANCE:BTCUSDT \ + --webhook-url https://receiver.example/actions +``` + +Times are minute-aligned Unix milliseconds or strict UTC RFC3339 seconds. +Warmup fetches contiguous REST 1m klines only behind an observed `x=true` WS +watermark. Its end is **exclusive** and must be the live start. It writes the +runner CSV columns `timestamp,open,high,low,close,volume` and an atomic +`warmup.csv.manifest.json` containing source, symbol, units, start, exclusive +cut, confirmation watermark, row count, and the CSV SHA-256. It refuses to +overwrite either output. Keep both files together; an interrupted two-file +publication without its manifest is not a qualified warmup. + +## Protocol and proof policy + +- Bars are `{"type":"bar","bar":{"ts_open":...,"o":...,"h":...,"l":...,"c":...,"v":...}}`. +- Raw ticks are `{"type":"tick","ts":...,"seq":...,"price":...,"qty":...}`. +- Completeness is `{"type":"time","ts":...}`: complete **strictly before** + that timestamp. It is never a heartbeat or a wall-clock assertion. +- Every message contains one event. No batches, control messages, diagnostics, + or status objects are sent on stdout. These immutable boundaries are the + runner's `--from-input` counting unit. + +Venue decimal string tokens become JSON number tokens unchanged, including +trailing zeros. Fixed-point string arithmetic reconciles OHLCV; no price or +quantity enters a binary float. Bounds are 96 token characters, 32 fractional +digits, and 128 arithmetic digits. Invalid/zero tick quantities fail closed. + +The combined stream uses raw `t` as the positive sequence and matched `T` as +time, not dispatch `E`. The real predecessor fetched from REST proves the +initial cut; aggregates locate a raw start ID but are never emitted or expanded. +Each new raw ID must be exactly its predecessor plus one, with nonregressing +matched time. Holes/reordering heal through inclusive `historicalTrades` +`fromId` pages (maximum 1000); identical duplicates are discarded and conflicting +duplicates stop. There is no assumed all-time retention window. + +For the observed minute, `x=true`, the complete `f..L` range, `n`, and exact +OHLCV must all reconcile before `time = open + 60000`. Later-minute prints stay +buffered until that proof. A missed historical minute can be closed only behind +a later observed `x=true` watermark, with a fetched contiguous raw prefix, an +independently fetched next-minute raw-ID fence, and exact REST `n`/OHLCV. This +uses the approved historical-watermark policy, not a fabricated WS confirmation. +Empty/ambiguous tick minutes deliberately stop rather than invent a fence. +Bars heal missing minutes from REST only behind the verified closed watermark. +Any discovered already-emitted bar revision stops; history is never rewritten. + +Curl answers server PING with matching-payload PONG. Reconnect occurs on +`serverShutdown`, transport loss/75-second source silence, or at 23h55 (before +the venue's 24-hour limit). The reader buffers during REST healing. Every new +connection re-fetches up to 1000 raw prints and 32 closed-bar proofs, requires +exact token/ID/time overlap, then continues the same stream epoch and sequence. +An altered duplicate discovered outside that window is checked against the +full retained normalized journal. Older, undiscovered revisions are not claimed +to be continuously polled. + +REST routes are restricted to public `exchangeInfo`, `historicalTrades`, +`aggTrades`, and `klines`. Current public weight/raw-request limits are read +from `exchangeInfo`, combined with conservative pacing, observed used-weight +headers, shared-quota pauses, and `429 Retry-After`. A ban/access rejection is +not retried as anonymous trading access. TLS verification is always enabled; +redirects, URL credentials, and non-origin overrides are refused. Explicit +insecure overrides are **loopback-only**, for tests. + +## Durability and restart + +One producer owns a state directory via `flock`. `events.jsonl` is append-only; +each message is fsynced before `cursor.json` is atomically replaced and its +directory fsynced, **before** stdout publication. The cursor binds source +origins, symbol/mode/units, random stream epoch, initial/verified cut, venue ID, +emitted sequence, message index, byte count, predecessor, closed-minute proofs, +SHA-256 prefix chain, last-message hash, and fixed message partition. Its +canonical document is itself hashed. Resume verifies the retained committed +prefix and fsyncs truncation of an uncommitted crash tail. + +A pipe write is not a consumer acknowledgment. By default `--resume` replays the +whole verified origin prefix. If the runner has committed N messages: + +```sh +build-feed/pineforge-feed run --venue binance --market spot --symbol BTCUSDT \ + --mode bars --state-dir feed-state --resume --output-from N | + pineforge-live run --strategy strategy.so --warmup warmup.csv --mode bars \ + --script-tf 1 --feed - --ledger runner.sqlite3 --from-input N \ + --symbol BINANCE:BTCUSDT --webhook-url https://receiver.example/actions +``` + +Replace N with the runner's actual committed count; never use the producer's +possibly-ahead count. Bind warmup identity and stream epoch in orchestration. +An unavailable cursor or changed identity is refused. SIGTERM/SIGINT stops +intake, finishes a durable current message or leaves a rollback tail, and exits +0. Nonblocking stdout has a five-second stop drain; a consumer that has not +acknowledged a committed message must replay it. Stderr is structured JSON and +nonblocking; a stalled diagnostics sink can lose logs, not stall shutdown. + +| Exit | Meaning | +|---|---| +| 0 | Requested clean stop / successful command | +| 20 | Unhealable history gap, ambiguous fence, exhausted source retries | +| 21 | Changed overlap, conflicting duplicate, revised bar, changed cursor/prefix | +| 22 | Replay/disk/queue/allocation budget or durable I/O failure | +| 23 | Unsupported input/source, permanent access, protocol/decimal/dependency failure | + +Default replay disk budget: 256 MiB (`--max-log-bytes`); each prefix traversal: +60 seconds (`--max-replay-seconds`, maximum 3600); source and unconfirmed-print +queues: 16 MiB each (`--max-queue-bytes`); WS text: 1 MiB; REST body: 4 MiB. +Warmup: 100000 rows / 16 MiB. Exhaustion is a cursor-preserving stop, not silent +compaction or dropped events. Raising limits is an explicit operator choice. +`--max-messages` permits bounded qualification. `--reconnect-seconds` can shorten +the 23h55 rotation for synthetic tests, never extend it beyond that limit. + +## Qualification and next adapters + +`tests/unit.cpp` and `tests/mock_venue.py` contain **synthetic** documented +Binance message shapes only: holes, reordering, duplicates, revisions, access, +reconnect overlap, fragmented text, PING/PONG, retirement, budgets, and recovery. +Never commit public exchange captures. `tests/public_e2e.py` explicitly opts +into public BTCUSDT capture, local receiver delivery, a hard-kill restart, +exact REST verification, bar batch action parity through `run_backtest_full`, +and tick same-print replay determinism. Tick-vs-OHLC-only fills are not assumed +equivalent: the harness exits nonzero for that comparison unless the operator +explicitly passes `--allow-tick-ohlc-difference`; even then it retains the FAIL +receipt and false parity result. Its batch observer is linked separately from the read-only engine's +`tests/native_live_equivalence_observer.cpp`; it is not a feed dependency. + +`src/venue.hpp` is the adapter boundary. Later slices add OKX raw ticks/bars +(contract-unit conversion where needed), Bybit confirmed bars **without ticks**, +and USD-M confirmed bars / separately opted-in aggregate-print ticks. Coinbase +remains deferred. `serve` will add approved MIT CivetWeb only to its server +target, bounded client replay/queues, and cursor-bound fan-out; no server library +is linked in this slice. The engine executable remains `pineforge-live`. diff --git a/native/feed/src/binance.cpp b/native/feed/src/binance.cpp new file mode 100644 index 0000000..eccef22 --- /dev/null +++ b/native/feed/src/binance.cpp @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "binance.hpp" +#include +#include +#include +#include + +namespace pineforge::feed { +namespace { +const std::vector& array(const Json& value, std::size_t maximum) { + if (value.kind != Json::Kind::Array || value.items.size() > maximum) throw Error(23, "invalid public market-data array"); + return value.items; +} +bool boolean(const Json& value) { + if (value.kind != Json::Kind::Bool) throw Error(23, "confirmation must be a JSON boolean"); + return value.value == "true"; +} +std::string lower(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char letter) { return static_cast(std::tolower(letter)); }); + return value; +} +std::size_t trade_bytes(const Trade& trade) { return trade.price.size() + trade.qty.size() + 192; } +} + +Trade binance_trade(const Json& value, bool websocket) { + Trade trade{value.at(websocket ? "t" : "id").integer(), + value.at(websocket ? "T" : "time").integer(), + value.at(websocket ? "p" : "price").text(), value.at(websocket ? "q" : "qty").text()}; + trade.validate(); + return trade; +} +Kline binance_kline(const Json& value, bool websocket) { + Kline kline; + if (websocket) { + if (value.at("i").text() != "1m") throw Error(23, "only UTC one-minute klines are supported"); + kline.bar = {value.at("t").integer(), value.at("o").text(), value.at("h").text(), + value.at("l").text(), value.at("c").text(), value.at("v").text()}; + kline.bar.validate(); + if (value.at("T").integer() != kline.bar.ts + 59999) throw Error(23, "invalid inclusive one-minute kline close"); + kline.first = value.at("f").integer(); + kline.last = value.at("L").integer(); + kline.count = value.at("n").integer(); + kline.confirmed = boolean(value.at("x")); + if (kline.count && (kline.first <= 0 || kline.last < kline.first)) throw Error(23, "invalid kline trade-id fence"); + } else { + const auto& row = array(value, 12); + if (row.size() != 12) throw Error(23, "REST kline must have twelve fields"); + kline.bar = {row[0].integer(), row[1].text(), row[2].text(), row[3].text(), row[4].text(), row[5].text()}; + kline.bar.validate(); + if (row[6].integer() != kline.bar.ts + 59999) throw Error(23, "invalid REST kline close time"); + kline.count = row[8].integer(); + } + return kline; +} + +std::string BinanceSpot::streams() const { + const auto symbol = lower(config_.symbol); + return symbol + "@trade/" + symbol + "@kline_1m"; +} +VenueEvent BinanceSpot::decode(const std::string& message) const { + try { + const auto envelope = parse_json(message); + const auto* wrapped = envelope.find("data"); + const auto& data = wrapped ? *wrapped : envelope; + const auto* event = data.find("e"); + if (!event) throw Error(23, "unexpected public stream control message"); + if (event->text() == "serverShutdown") return {}; + if (data.at("s").text() != config_.symbol) throw Error(23, "public stream symbol changed"); + if (event->text() == "trade") return {VenueEvent::Kind::Trade, binance_trade(data, true), {}}; + if (event->text() == "kline") { + const auto& row = data.at("k"); + if (row.at("s").text() != config_.symbol) throw Error(23, "kline symbol changed"); + return {VenueEvent::Kind::Kline, {}, binance_kline(row, true)}; + } + throw Error(23, "unsupported public stream event; aggregate prints are not raw trades"); + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid Binance spot message shape"); } +} + +std::vector BinanceSpot::history(std::uint64_t from, std::size_t limit) { + if (!from || !limit || limit > 1000) throw Error(23, "invalid raw-history page request"); + for (unsigned int attempt = 0; attempt < 3; ++attempt) { + const auto response = http_.get("/api/v3/historicalTrades?symbol=" + config_.symbol + "&fromId=" + std::to_string(from) + "&limit=" + std::to_string(limit), 25); + const auto& rows = array(response, limit); + if (rows.empty()) continue; + std::vector trades; + std::uint64_t expected = from; + for (const auto& row : rows) { + const auto trade = binance_trade(row, false); + if (trade.id != expected) throw Error(20, "raw-history page has a missing or nonunit venue trade ID"); + trades.push_back(trade); + if (expected == std::numeric_limits::max()) throw Error(23, "venue trade ID exhausted"); + ++expected; + } + return trades; + } + throw Error(20, "raw history exhausted or unavailable within the venue retention window"); +} +std::uint64_t BinanceSpot::first_trade_id(std::int64_t minute) { + const auto response = http_.get("/api/v3/aggTrades?symbol=" + config_.symbol + "&startTime=" + std::to_string(minute) + + "&endTime=" + std::to_string(minute + 59999) + "&limit=1", 4); + const auto& rows = array(response, 1); + if (rows.empty()) throw Error(20, "initial minute has no available raw-trade start fence"); + const auto first = rows.front().at("f").integer(); + const auto matched = rows.front().at("T").integer(); + if (first <= 1 || matched < minute || matched >= minute + 60000) throw Error(20, "initial raw-trade fence is unavailable or ambiguous"); + return first; +} +std::vector BinanceSpot::klines(std::int64_t start, std::int64_t end) { + if (start < 0 || start % 60000 || end <= start || end % 60000) throw Error(23, "invalid exclusive kline range"); + const auto response = http_.get("/api/v3/klines?symbol=" + config_.symbol + "&interval=1m&startTime=" + std::to_string(start) + + "&endTime=" + std::to_string(end - 1) + "&limit=1000", 2); + std::vector result; + auto expected = start; + for (const auto& row : array(response, 1000)) { + const auto kline = binance_kline(row, false); + if (kline.bar.ts != expected || kline.bar.ts >= end) throw Error(20, "historical klines have a missing or unexpected minute"); + expected += 60000; + result.push_back(kline); + } + if (result.empty()) throw Error(20, "historical minute is unavailable"); + return result; +} + +FeedSession::FeedSession(State& state, Venue& venue, std::function output) + : state_(state), venue_(venue), output_(std::move(output)) { + if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() == "tick") { + const auto trade = normalized_trade(event); + if (trade.ts >= state_.cursor().cut) aggregate_.add(trade); + } + }); +} + +void FeedSession::emit(const std::string& line, const std::optional& proof) { + if (stopping) throw Stopped{}; + state_.commit(line, proof); + output_(line); + ++new_messages_; + if (state_.config().max_messages && new_messages_ >= state_.config().max_messages) throw Stopped{}; +} +void FeedSession::connected() { + const auto& trades = state_.recent_trades(); + if (!trades.empty()) { + const auto fetched = venue_.history(trades.front().id, trades.size()); + if (fetched.size() != trades.size()) throw Error(20, "reconnect raw overlap is unavailable"); + for (std::size_t index = 0; index < trades.size(); ++index) + if (!(trades[index] == fetched[index])) throw Error(21, "reconnect raw overlap changed"); + } else if (state_.cursor().predecessor) { + const auto fetched = venue_.history(state_.cursor().predecessor->id, 1); + if (!(fetched.front() == *state_.cursor().predecessor)) throw Error(21, "initial predecessor overlap changed"); + } + const auto& proofs = state_.config().mode == "bars" ? state_.recent_bars() : state_.cursor().proofs; + if (!proofs.empty()) { + const auto fetched = venue_.klines(proofs.front().ts, proofs.back().ts + 60000); + if (fetched.size() != proofs.size()) throw Error(20, "reconnect closed-bar overlap is unavailable"); + std::map verified; + if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() != "tick") return; + const auto trade = normalized_trade(event); + const auto minute = trade.ts - trade.ts % 60000; + if (minute >= proofs.front().ts && minute <= proofs.back().ts) verified[minute].add(trade); + }); + for (std::size_t index = 0; index < proofs.size(); ++index) { + if (!(proofs[index] == fetched[index].bar)) throw Error(21, "reconnect already-emitted bar changed"); + if (state_.config().mode == "ticks") { + auto checked = fetched[index]; + checked.confirmed = true; + try { verified.at(checked.bar.ts).reconcile(checked, false); } + catch (const std::exception&) { throw Error(21, "reconnect closed-minute count or raw proof changed"); } + } + } + } + if (!replayed_) { + if (state_.config().resume) state_.visit(state_.config().output_from, output_); + replayed_ = true; + } + log("info", "overlap_verified", state_.status()); +} + +void FeedSession::anchor(std::uint64_t first) { + if (state_.cursor().predecessor) return; + if (first <= 1) throw Error(20, "raw predecessor cannot be proven"); + const auto predecessor = venue_.history(first - 1, 1).front(); + state_.anchor(predecessor); +} +std::uint64_t FeedSession::next_id() const { + const auto previous = state_.cursor().seq ? state_.cursor().seq : state_.cursor().predecessor ? state_.cursor().predecessor->id : 0; + if (!previous || previous == std::numeric_limits::max()) throw Error(20, "raw-trade cursor is not anchored"); + return previous + 1; +} +void FeedSession::put(const Trade& trade) { + trade.validate(); + if (trade.ts < state_.cursor().start) return; + if (state_.cursor().seq && trade.id <= state_.cursor().seq) { + const auto previous = state_.trade(trade.id); + if (!previous || !(*previous == trade)) throw Error(21, "duplicate raw trade conflicts with the immutable prefix"); + return; + } + const auto existing = pending_.find(trade.id); + if (existing != pending_.end()) { + if (!(existing->second == trade)) throw Error(21, "buffered raw duplicate conflicts"); + return; + } + const auto bytes = trade_bytes(trade); + if (bytes > state_.config().max_queue_bytes - std::min(pending_bytes_, state_.config().max_queue_bytes)) + throw Error(22, "unconfirmed raw-trade buffer overrun"); + pending_bytes_ += bytes; + pending_.emplace(trade.id, trade); +} +void FeedSession::emit_trade(const Trade& trade) { + if (trade.id != next_id() || trade.ts < state_.cursor().cut || trade.ts >= state_.cursor().cut + 60000 || trade.ts < state_.cursor().last_tick_ts) + throw Error(20, "raw trade violates the complete chronological minute prefix"); + const auto buffered = pending_.find(trade.id); + if (buffered != pending_.end() && !(buffered->second == trade)) + throw Error(21, "REST healing changed a buffered raw trade"); + aggregate_.add(trade); + emit(trade.wire()); + if (buffered != pending_.end()) { + pending_bytes_ -= trade_bytes(buffered->second); + pending_.erase(buffered); + } +} +void FeedSession::heal_to(std::uint64_t target) { + while (next_id() <= target) { + log("warn", "raw_gap_healing", Json::object({{"from_id", Json::number(std::to_string(next_id()))}, {"through_id", Json::number(std::to_string(target))}})); + const auto page = venue_.history(next_id()); + bool progressed = false; + for (const auto& trade : page) { + if (trade.id > target) break; + const auto buffered = pending_.find(trade.id); + if (buffered != pending_.end() && !(buffered->second == trade)) throw Error(21, "REST raw healing conflicts with WebSocket data"); + emit_trade(trade); + progressed = true; + } + if (!progressed) throw Error(20, "raw gap cannot advance within available history"); + } +} +void FeedSession::drain() { + while (!pending_.empty()) { + const auto trade = pending_.begin()->second; + if (trade.ts >= state_.cursor().cut + 60000) break; + if (trade.id < next_id()) throw Error(21, "buffered cursor fell behind the immutable prefix"); + if (trade.id > next_id()) heal_to(trade.id - 1); + emit_trade(trade); + } +} +void FeedSession::historical_minute() { + if (state_.cursor().cut + 60000 > watermark_) throw Error(20, "historical tick closure lacks a confirmed WebSocket watermark"); + if (!state_.cursor().predecessor) anchor(venue_.first_trade_id(state_.cursor().start)); + const auto minute = state_.cursor().cut; + bool fenced = false; + while (!fenced) { + const auto page = venue_.history(next_id()); + for (const auto& trade : page) { + const auto buffered = pending_.find(trade.id); + if (buffered != pending_.end() && !(buffered->second == trade)) throw Error(21, "historical raw overlap changed"); + if (trade.ts >= minute + 60000) { + if (trade.id != next_id()) throw Error(20, "historical next-minute raw fence is not contiguous"); + put(trade); + fenced = true; + break; + } + emit_trade(trade); + } + } + auto kline = venue_.klines(minute, minute + 60000).front(); + kline.confirmed = true; + kline.first = static_cast(aggregate_.first); + kline.last = static_cast(aggregate_.last); + aggregate_.reconcile(kline, true); + log("info", "historical_tick_fence_verified", Json::object({{"minute", Json::number(std::to_string(minute))}, + {"first_id", Json::number(std::to_string(aggregate_.first))}, {"last_id", Json::number(std::to_string(aggregate_.last))}})); + emit("{\"type\":\"time\",\"ts\":" + std::to_string(minute + 60000) + "}", kline.bar); + aggregate_ = Aggregate{}; +} + +void FeedSession::closed(const Kline& kline) { + if (!kline.confirmed) return; + const auto minute = kline.bar.ts; + watermark_ = std::max(watermark_, minute + 60000); + if (minute < state_.cursor().start) return; + if (minute < state_.cursor().cut) { + const auto old = state_.bar(minute); + if (old) { + if (!(*old == kline.bar)) throw Error(21, "already-emitted closed bar was revised"); + } + if (state_.config().mode == "ticks") { + Aggregate reconstructed; + state_.visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() == "tick") { + const auto trade = normalized_trade(event); + if (trade.ts >= minute && trade.ts < minute + 60000) reconstructed.add(trade); + } + }); + try { reconstructed.reconcile(kline, true); } + catch (const Error& failure) { + if (failure.code == 20 || failure.code == 21) throw Error(21, "already-emitted closed-minute fence was revised"); + throw; + } + } else if (!old) throw Error(21, "closed duplicate cannot be matched to the retained prefix"); + return; + } + if (state_.config().mode == "bars") { + while (state_.cursor().cut < minute) { + const auto page = venue_.klines(state_.cursor().cut, minute); + for (const auto& historical : page) { + if (historical.bar.ts + 60000 > watermark_) throw Error(20, "historical bar exceeds the confirmed watermark"); + emit(historical.bar.wire(), historical.bar); + } + } + emit(kline.bar.wire(), kline.bar); + return; + } + while (state_.cursor().cut < minute) historical_minute(); + if (!kline.count || kline.first <= 0 || kline.last < kline.first || + static_cast(kline.last - kline.first) + 1 != kline.count) + throw Error(20, "closed kline has ambiguous or noncontiguous raw bounds"); + anchor(static_cast(kline.first)); + if (state_.cursor().seq > static_cast(kline.last)) throw Error(21, "closed kline excludes an already-emitted trade"); + heal_to(static_cast(kline.last)); + aggregate_.reconcile(kline, true); + emit("{\"type\":\"time\",\"ts\":" + std::to_string(minute + 60000) + "}", kline.bar); + aggregate_ = Aggregate{}; + drain(); +} +void FeedSession::ingest(const VenueEvent& event) { + if (event.kind == VenueEvent::Kind::Kline) { closed(event.kline); return; } + if (event.kind != VenueEvent::Kind::Trade || state_.config().mode != "ticks" || event.trade.ts < state_.cursor().start) return; + if (!state_.cursor().predecessor) anchor(venue_.first_trade_id(state_.cursor().start)); + put(event.trade); + drain(); +} + +void run_feed(const Config& config) { + State state(config); + try { + BinanceSpot venue(config); + WebSocketPump pump(config, venue.streams()); + FeedSession session(state, venue, &output_line); + for (;;) { + const auto message = pump.take(); + if (message.connected) session.connected(); + else session.ingest(venue.decode(message.text)); + } + } catch (const Stopped&) { log("info", "stopped", state.status()); } + catch (...) { log("error", "verified_cursor_retained", state.status()); throw; } +} + +void warmup(const Config& config, const std::string& output) { + if (config.start < 0 || config.end <= config.start || config.start % 60000 || config.end % 60000) + throw Error(23, "warmup requires a nonempty minute-aligned exclusive range"); + if ((config.end - config.start) / 60000 > 100000) throw Error(22, "warmup range exceeds 100000 bars"); + if (std::filesystem::exists(output) || std::filesystem::exists(output + ".manifest.json")) throw Error(23, "warmup output already exists"); + BinanceSpot venue(config); + std::int64_t watermark = -1; + { + WebSocketPump pump(config, venue.streams()); + while (watermark < config.end) { + const auto message = pump.take(); + if (message.connected) continue; + const auto event = venue.decode(message.text); + if (event.kind == VenueEvent::Kind::Kline && event.kline.confirmed) watermark = std::max(watermark, event.kline.bar.ts + 60000); + } + } + std::string csv = "timestamp,open,high,low,close,volume\n"; + auto next = config.start; + std::uint64_t count = 0; + while (next < config.end) { + for (const auto& kline : venue.klines(next, config.end)) { + if (kline.bar.ts != next || next + 60000 > watermark) throw Error(20, "warmup does not adjoin a verified closed-minute cut"); + const auto& bar = kline.bar; + csv += std::to_string(bar.ts) + ',' + bar.open + ',' + bar.high + ',' + bar.low + ',' + bar.close + ',' + bar.volume + '\n'; + if (csv.size() > 16 * 1024 * 1024) throw Error(22, "warmup output exceeds 16 MiB"); + next += 60000; + ++count; + } + } + const auto manifest = Json::object({{"schema", Json::string("pineforge-feed-warmup/v1")}, + {"venue", Json::string(config.venue)}, {"market", Json::string(config.market)}, {"symbol", Json::string(config.symbol)}, + {"source", Json::string(config.rest_url + "/api/v3/klines")}, {"interval", Json::string("1m")}, + {"units", Json::object({{"price", Json::string("quote/base")}, {"volume", Json::string("base")}, {"timestamp", Json::string("unix-ms")}})}, + {"start", Json::number(std::to_string(config.start))}, {"end_exclusive", Json::number(std::to_string(config.end))}, + {"cut", Json::number(std::to_string(config.end))}, {"confirmed_ws_watermark", Json::number(std::to_string(watermark))}, + {"bars", Json::number(std::to_string(count))}, {"sha256", Json::string(sha256(csv))}}); + atomic_file(output, csv); + atomic_file(output + ".manifest.json", manifest.dump() + '\n'); + log("info", "warmup_verified", manifest); +} +} diff --git a/native/feed/src/binance.hpp b/native/feed/src/binance.hpp new file mode 100644 index 0000000..deaad62 --- /dev/null +++ b/native/feed/src/binance.hpp @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "state.hpp" +#include "transport.hpp" +#include "venue.hpp" +#include + +namespace pineforge::feed { +Trade binance_trade(const Json& trade, bool websocket); +Kline binance_kline(const Json& row, bool websocket); + +class BinanceSpot final : public Venue { + Config config_; + HttpClient http_; +public: + explicit BinanceSpot(const Config& config) : config_(config), http_(config) {} + std::string streams() const override; + VenueEvent decode(const std::string& message) const override; + std::vector history(std::uint64_t from, std::size_t limit = 1000) override; + std::uint64_t first_trade_id(std::int64_t minute) override; + std::vector klines(std::int64_t start, std::int64_t end) override; +}; + +class FeedSession { + State& state_; + Venue& venue_; + std::function output_; + Aggregate aggregate_; + std::map pending_; + std::size_t pending_bytes_ = 0; + std::int64_t watermark_ = -1; + std::uint64_t new_messages_ = 0; + bool replayed_ = false; + void emit(const std::string& line, const std::optional& proof = std::nullopt); + void anchor(std::uint64_t first); + std::uint64_t next_id() const; + void put(const Trade& trade); + void emit_trade(const Trade& trade); + void heal_to(std::uint64_t target); + void drain(); + void historical_minute(); + void closed(const Kline& kline); +public: + FeedSession(State& state, Venue& venue, std::function output); + void connected(); + void ingest(const VenueEvent& event); +}; +void run_feed(const Config& config); +void warmup(const Config& config, const std::string& output); +} diff --git a/native/feed/src/core.cpp b/native/feed/src/core.cpp new file mode 100644 index 0000000..8dc088a --- /dev/null +++ b/native/feed/src/core.cpp @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "core.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +std::atomic stopping{false}; +static_assert(std::atomic::is_always_lock_free); + +void log(const std::string& level, const std::string& event, const Json& fields) { + static std::mutex mutex; + auto record = fields; + record.members["level"] = Json::string(level); + record.members["event"] = Json::string(event); + record.members["ts"] = Json::number(std::to_string(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count())); + const auto bytes = record.dump() + '\n'; + std::lock_guard guard(mutex); + std::size_t offset = 0; + while (offset < bytes.size()) { + const auto written = ::write(STDERR_FILENO, bytes.data() + offset, bytes.size() - offset); + if (written < 0 && errno == EINTR) continue; + if (written <= 0) break; + offset += static_cast(written); + } +} + +std::string sha256(std::string_view bytes) { + unsigned char digest[EVP_MAX_MD_SIZE]; + unsigned int size = 0; + if (EVP_Digest(bytes.data(), bytes.size(), digest, &size, EVP_sha256(), nullptr) != 1 || size != 32) + throw Error(23, "SHA-256 failed"); + constexpr char digits[] = "0123456789abcdef"; + std::string result; + for (unsigned int index = 0; index < size; ++index) { + result += digits[digest[index] >> 4]; + result += digits[digest[index] & 15]; + } + return result; +} + +std::string random_epoch() { + unsigned char bytes[32]; + if (RAND_bytes(bytes, sizeof(bytes)) != 1) throw Error(23, "stream epoch generation failed"); + return sha256(std::string_view(reinterpret_cast(bytes), sizeof(bytes))); +} + +std::int64_t timestamp(const std::string& token) { + std::int64_t result = -1; + const auto parsed = std::from_chars(token.data(), token.data() + token.size(), result); + if (parsed.ec == std::errc{} && parsed.ptr == token.data() + token.size()) { + if (result < 0 || result > std::numeric_limits::max() - 60000) + throw Error(23, "timestamp outside supported range"); + return result; + } + if (token.size() != 20 || token.back() != 'Z') throw Error(23, "use UTC RFC3339 seconds or Unix milliseconds"); + std::tm broken{}; + std::istringstream input(token); + input >> std::get_time(&broken, "%Y-%m-%dT%H:%M:%SZ"); + if (input.fail()) throw Error(23, "invalid UTC timestamp"); + const auto seconds = timegm(&broken); + char canonical[32]; + std::tm checked{}; + gmtime_r(&seconds, &checked); + std::strftime(canonical, sizeof(canonical), "%Y-%m-%dT%H:%M:%SZ", &checked); + if (seconds < 0 || token != canonical) throw Error(23, "invalid UTC timestamp"); + return static_cast(seconds) * 1000; +} + +void write_all(int descriptor, std::string_view bytes) { + std::size_t offset = 0; + while (offset < bytes.size()) { + const auto written = ::write(descriptor, bytes.data() + offset, bytes.size() - offset); + if (written < 0 && errno == EINTR) continue; + if (written <= 0) throw Error(22, "durable write failed"); + offset += static_cast(written); + } +} + +void output_line(std::string_view line) { + const auto bytes = std::string(line) + '\n'; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + std::size_t offset = 0; + while (offset < bytes.size()) { + const auto written = ::write(STDOUT_FILENO, bytes.data() + offset, bytes.size() - offset); + if (written < 0 && errno == EINTR) continue; + if (written < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) { + if (stopping && std::chrono::steady_clock::now() >= deadline) throw Stopped{}; + pollfd descriptor{STDOUT_FILENO, POLLOUT, 0}; + ::poll(&descriptor, 1, 100); + continue; + } + if (written <= 0) throw Error(22, "stdout unavailable; replay from the consumer's committed message index"); + offset += static_cast(written); + } +} + +void atomic_file(const std::string& path, std::string_view bytes) { + const auto temporary = path + ".tmp"; + const int descriptor = ::open(temporary.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600); + if (descriptor < 0) throw Error(22, "cannot open atomic state file"); + try { + write_all(descriptor, bytes); + if (::fsync(descriptor) != 0) throw Error(22, "cannot sync atomic state file"); + if (::close(descriptor) != 0) throw Error(22, "cannot close atomic state file"); + } catch (...) { + ::close(descriptor); + throw; + } + if (::rename(temporary.c_str(), path.c_str()) != 0) throw Error(22, "cannot replace atomic state file"); + auto parent = std::filesystem::path(path).parent_path(); + if (parent.empty()) parent = "."; + const int directory = ::open(parent.c_str(), O_RDONLY | O_CLOEXEC); + if (directory < 0) throw Error(22, "cannot open state directory for sync"); + const auto status = ::fsync(directory); + ::close(directory); + if (status != 0) throw Error(22, "cannot sync state directory"); +} + +Decimal::Decimal(std::string coefficient, std::size_t scale) + : coefficient_(std::move(coefficient)), scale_(scale) { + const auto first = coefficient_.find_first_not_of('0'); + coefficient_ = first == std::string::npos ? "0" : coefficient_.substr(first); + while (scale_ && coefficient_.size() > 1 && coefficient_.back() == '0') { + coefficient_.pop_back(); + --scale_; + } + if (coefficient_ == "0") scale_ = 0; + if (coefficient_.size() > 128 || scale_ > 32) throw Error(23, "decimal arithmetic exceeds bounded precision"); +} + +Decimal::Decimal(std::string_view token) : Decimal("0", 0) { + if (token.empty() || token.size() > 96) throw Error(23, "invalid decimal token length"); + const auto dot = token.find('.'); + const auto integral = dot == std::string_view::npos ? token.size() : dot; + if (!integral || (integral > 1 && token.front() == '0') || (dot != std::string_view::npos && dot + 1 == token.size())) + throw Error(23, "invalid decimal token grammar"); + std::string coefficient; + for (std::size_t index = 0; index < token.size(); ++index) { + if (index == dot) continue; + if (token[index] < '0' || token[index] > '9') throw Error(23, "decimal must be an unsigned fixed-point token"); + coefficient += token[index]; + } + *this = Decimal(coefficient, dot == std::string_view::npos ? 0 : token.size() - dot - 1); +} + +int Decimal::compare(const Decimal& other) const { + if (zero() || other.zero()) return zero() == other.zero() ? 0 : zero() ? -1 : 1; + const auto scale = std::max(scale_, other.scale_); + const auto left = coefficient_ + std::string(scale - scale_, '0'); + const auto right = other.coefficient_ + std::string(scale - other.scale_, '0'); + if (left.size() != right.size()) return left.size() < right.size() ? -1 : 1; + return left == right ? 0 : left < right ? -1 : 1; +} + +Decimal Decimal::add(const Decimal& other) const { + const auto scale = std::max(scale_, other.scale_); + auto left = coefficient_ + std::string(scale - scale_, '0'); + auto right = other.coefficient_ + std::string(scale - other.scale_, '0'); + const auto width = std::max(left.size(), right.size()); + left.insert(0, width - left.size(), '0'); + right.insert(0, width - right.size(), '0'); + std::string sum(width, '0'); + int carry = 0; + for (std::size_t index = width; index > 0; --index) { + const int digit = left[index - 1] - '0' + right[index - 1] - '0' + carry; + sum[index - 1] = static_cast('0' + digit % 10); + carry = digit / 10; + } + if (carry) sum.insert(sum.begin(), '1'); + return Decimal(sum, scale); +} + +void Trade::validate() const { + if (!id || ts < 0 || ts > std::numeric_limits::max() - 60000 || Decimal(price).zero() || Decimal(qty).zero()) + throw Error(23, "invalid positive venue trade"); +} +std::string Trade::wire() const { + validate(); + return "{\"type\":\"tick\",\"ts\":" + std::to_string(ts) + ",\"seq\":" + std::to_string(id) + + ",\"price\":" + price + ",\"qty\":" + qty + "}"; +} +bool Trade::operator==(const Trade& other) const { + return id == other.id && ts == other.ts && price == other.price && qty == other.qty; +} +void Bar::validate() const { + if (ts < 0 || ts % 60000 || ts > std::numeric_limits::max() - 60000) + throw Error(23, "invalid minute timestamp"); + const Decimal opening(open), highest(high), lowest(low), closing(close), amount(volume); + if (opening.zero() || highest.zero() || lowest.zero() || closing.zero() || highest.compare(lowest) < 0 || + highest.compare(opening) < 0 || highest.compare(closing) < 0 || lowest.compare(opening) > 0 || lowest.compare(closing) > 0) + throw Error(23, "invalid OHLCV envelope"); + (void)amount; +} +std::string Bar::wire() const { + validate(); + return "{\"type\":\"bar\",\"bar\":{\"ts_open\":" + std::to_string(ts) + ",\"o\":" + open + + ",\"h\":" + high + ",\"l\":" + low + ",\"c\":" + close + ",\"v\":" + volume + "}}"; +} +bool Bar::operator==(const Bar& other) const { + return ts == other.ts && open == other.open && high == other.high && low == other.low && close == other.close && volume == other.volume; +} +void Aggregate::add(const Trade& trade) { + trade.validate(); + if (count && (last == std::numeric_limits::max() || trade.id != last + 1 || trade.ts < last_ts)) + throw Error(20, "noncontiguous minute trade prefix"); + if (!count) { + first = trade.id; + open = high = low = trade.price; + } + if (Decimal(trade.price).compare(Decimal(high)) > 0) high = trade.price; + if (Decimal(trade.price).compare(Decimal(low)) < 0) low = trade.price; + close = trade.price; + volume = volume.add(Decimal(trade.qty)); + last = trade.id; + last_ts = trade.ts; + ++count; +} +void Aggregate::reconcile(const Kline& kline, bool require_ids) const { + if (!kline.confirmed || !count || count != kline.count || + (require_ids && (kline.first <= 0 || kline.last < kline.first || first != static_cast(kline.first) || + last != static_cast(kline.last) || last - first + 1 != count))) + throw Error(20, "closed-minute trade coverage is not proven"); + if (!(Decimal(open) == Decimal(kline.bar.open)) || !(Decimal(high) == Decimal(kline.bar.high)) || + !(Decimal(low) == Decimal(kline.bar.low)) || !(Decimal(close) == Decimal(kline.bar.close)) || + !(volume == Decimal(kline.bar.volume))) throw Error(21, "closed-minute exact OHLCV reconciliation failed"); +} +Trade normalized_trade(const Json& event) { + Trade trade{event.at("seq").integer(), event.at("ts").integer(), event.at("price").value, event.at("qty").value}; + trade.validate(); + return trade; +} +Bar normalized_bar(const Json& event) { + const auto& value = event.at("bar"); + Bar bar{value.at("ts_open").integer(), value.at("o").value, value.at("h").value, + value.at("l").value, value.at("c").value, value.at("v").value}; + bar.validate(); + return bar; +} +Json string_field(const std::string& value) { return Json::string(value); } +} diff --git a/native/feed/src/core.hpp b/native/feed/src/core.hpp new file mode 100644 index 0000000..8fd745c --- /dev/null +++ b/native/feed/src/core.hpp @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "json.hpp" +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +using Json = pineforge::live::Json; +using pineforge::live::parse_json; + +struct Error : std::runtime_error { + int code; + Error(int status, const std::string& reason) : std::runtime_error(reason), code(status) {} +}; +struct Stopped {}; +extern std::atomic stopping; +void log(const std::string& level, const std::string& event, const Json& fields = Json::object({})); +std::string sha256(std::string_view bytes); +std::string random_epoch(); +std::int64_t timestamp(const std::string& token); +void write_all(int descriptor, std::string_view bytes); +void output_line(std::string_view line); +void atomic_file(const std::string& path, std::string_view bytes); + +class Decimal { + std::string coefficient_; + std::size_t scale_ = 0; + Decimal(std::string coefficient, std::size_t scale); +public: + explicit Decimal(std::string_view token); + bool zero() const { return coefficient_ == "0"; } + int compare(const Decimal& other) const; + Decimal add(const Decimal& other) const; + bool operator==(const Decimal& other) const { return compare(other) == 0; } +}; + +struct Trade { + std::uint64_t id = 0; + std::int64_t ts = 0; + std::string price; + std::string qty; + void validate() const; + std::string wire() const; + bool operator==(const Trade& other) const; +}; +struct Bar { + std::int64_t ts = 0; + std::string open, high, low, close, volume; + void validate() const; + std::string wire() const; + bool operator==(const Bar& other) const; +}; +struct Kline { + Bar bar; + std::int64_t first = -1; + std::int64_t last = -1; + std::uint64_t count = 0; + bool confirmed = false; +}; +struct Aggregate { + std::uint64_t first = 0, last = 0, count = 0; + std::int64_t last_ts = -1; + std::string open, high, low, close; + Decimal volume{"0"}; + void add(const Trade& trade); + void reconcile(const Kline& kline, bool require_ids) const; +}; +Trade normalized_trade(const Json& event); +Bar normalized_bar(const Json& event); +Json string_field(const std::string& value); + +struct Config { + std::string venue = "binance", market = "spot", symbol, mode = "bars", state_dir; + std::string rest_url = "https://api.binance.com"; + std::string ws_url = "wss://stream.binance.com:443"; + std::int64_t start = -1, end = -1; + bool resume = false, allow_insecure = false; + std::uint64_t output_from = 0, max_messages = 0; + std::uint64_t max_log_bytes = 256 * 1024 * 1024; + std::uint64_t max_replay_seconds = 60; + std::size_t max_queue_bytes = 16 * 1024 * 1024; + std::uint64_t reconnect_seconds = 23 * 3600 + 55 * 60; +}; +} diff --git a/native/feed/src/main.cpp b/native/feed/src/main.cpp new file mode 100644 index 0000000..f821b3f --- /dev/null +++ b/native/feed/src/main.cpp @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "binance.hpp" +#include +#include +#include +#include +#include +#include + +namespace { +void on_signal(int) { pineforge::feed::stopping.store(true); } +std::uint64_t unsigned_value(const std::string& token) { + std::uint64_t value = 0; + const auto parsed = std::from_chars(token.data(), token.data() + token.size(), value); + if (parsed.ec != std::errc{} || parsed.ptr != token.data() + token.size()) throw pineforge::feed::Error(23, "invalid unsigned CLI value"); + return value; +} +void help() { + std::cout << "pineforge-feed warmup --venue binance --market spot --symbol SYMBOL --start UTC|MS --end UTC|MS --output FILE\n" + "pineforge-feed run --venue binance --market spot --symbol SYMBOL --mode bars|ticks --state-dir DIR [--start UTC|MS | --resume] [--output-from INDEX]\n" + "Limits: --max-messages N --max-log-bytes N --max-replay-seconds N --max-queue-bytes N --reconnect-seconds N\n" + "Testing/public origins: --rest-url ORIGIN --ws-url ORIGIN [--allow-insecure-http (loopback only)]\n"; +} +} +int main(int argc, char** argv) { + using namespace pineforge::feed; + try { + const auto error_flags = ::fcntl(STDERR_FILENO, F_GETFL); + if (error_flags < 0 || ::fcntl(STDERR_FILENO, F_SETFL, error_flags | O_NONBLOCK) != 0) + throw Error(22, "cannot configure bounded diagnostic writes"); + if (argc == 2 && (std::string(argv[1]) == "--help" || std::string(argv[1]) == "-h")) { help(); return 0; } + if (argc == 2 && std::string(argv[1]) == "--version") { check_runtime_curl(); std::cout << "pineforge-feed 0.1.0\n"; return 0; } + if (argc < 2) throw Error(23, "choose warmup or run; see --help"); + const std::string command = argv[1]; + if (command != "run" && command != "warmup") throw Error(23, "only warmup and run are implemented"); + Config config; + std::string output; + std::set seen; + for (int index = 2; index < argc; ++index) { + const std::string key = argv[index]; + if (!seen.insert(key).second) throw Error(23, "duplicate CLI option: " + key); + if (key == "--resume") { config.resume = true; continue; } + if (key == "--allow-insecure-http") { config.allow_insecure = true; continue; } + if (index + 1 >= argc) throw Error(23, "CLI option needs a value: " + key); + const std::string value = argv[++index]; + if (key == "--venue") config.venue = value; + else if (key == "--market") config.market = value; + else if (key == "--symbol") config.symbol = value; + else if (key == "--mode") config.mode = value; + else if (key == "--state-dir") config.state_dir = value; + else if (key == "--start") config.start = timestamp(value); + else if (key == "--end") config.end = timestamp(value); + else if (key == "--output") output = value; + else if (key == "--output-from") config.output_from = unsigned_value(value); + else if (key == "--max-messages") config.max_messages = unsigned_value(value); + else if (key == "--max-log-bytes") config.max_log_bytes = unsigned_value(value); + else if (key == "--max-replay-seconds") config.max_replay_seconds = unsigned_value(value); + else if (key == "--max-queue-bytes") config.max_queue_bytes = unsigned_value(value); + else if (key == "--reconnect-seconds") config.reconnect_seconds = unsigned_value(value); + else if (key == "--rest-url") config.rest_url = value; + else if (key == "--ws-url") config.ws_url = value; + else throw Error(23, "unknown CLI option: " + key); + } + if (config.venue != "binance" || config.market != "spot") throw Error(23, "this release supports public Binance spot only"); + if (config.symbol.empty() || config.symbol.size() > 32 || config.symbol.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") != std::string::npos) + throw Error(23, "use an uppercase ASCII Binance symbol"); + if (config.mode != "bars" && config.mode != "ticks") throw Error(23, "mode must be bars or ticks"); + if (!config.max_log_bytes || !config.max_queue_bytes || !config.max_replay_seconds || config.max_replay_seconds > 3600 || + !config.reconnect_seconds || config.reconnect_seconds > 86100) + throw Error(23, "budgets must be positive; reconnect must precede the 24-hour connection limit"); + check_runtime_curl(); + validate_origin(config.rest_url, false, config.allow_insecure); + validate_origin(config.ws_url, true, config.allow_insecure); + std::signal(SIGTERM, &on_signal); + std::signal(SIGINT, &on_signal); + std::signal(SIGPIPE, SIG_IGN); + if (command == "warmup") { + if (output.empty() || config.resume || !config.state_dir.empty() || seen.count("--output-from") || seen.count("--mode")) + throw Error(23, "warmup requires --output and does not accept run-state options"); + warmup(config, output); + } else { + if (config.state_dir.empty() || seen.count("--end") || !output.empty() || (!config.resume && seen.count("--output-from"))) + throw Error(23, "run requires --state-dir; --output-from requires --resume"); + const auto flags = ::fcntl(STDOUT_FILENO, F_GETFL); + if (flags < 0 || ::fcntl(STDOUT_FILENO, F_SETFL, flags | O_NONBLOCK) != 0) throw Error(22, "cannot configure bounded stdout drain"); + run_feed(config); + } + return 0; + } catch (const Stopped&) { log("info", "stopped"); return 0; } + catch (const std::bad_alloc&) { log("error", "allocation_exhausted"); return 22; } + catch (const Error& failure) { + log("error", "stop", Json::object({{"code", Json::number(std::to_string(failure.code))}, {"reason", Json::string(failure.what())}})); + return failure.code; + } catch (const std::exception& failure) { + log("error", "stop", Json::object({{"code", Json::number("23")}, {"reason", Json::string(failure.what())}})); + return 23; + } +} diff --git a/native/feed/src/state.cpp b/native/feed/src/state.cpp new file mode 100644 index 0000000..bba8b9b --- /dev/null +++ b/native/feed/src/state.cpp @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "state.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +Descriptor::~Descriptor() { if (value_ >= 0) ::close(value_); } +void Descriptor::reset(int value) { + if (value_ >= 0) ::close(value_); + value_ = value; +} +namespace { +Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } +Json signed_number(std::int64_t value) { return Json::number(std::to_string(value)); } +bool digest_token(const std::string& token) { + return token.size() == 64 && token.find_first_not_of("0123456789abcdef") == std::string::npos; +} +void advance(Cursor& cursor, const Json& event, const std::string& mode) { + const auto type = event.at("type").text(); + if (type == "tick" && mode == "ticks") { + const auto trade = normalized_trade(event); + const auto previous = cursor.seq ? cursor.seq : cursor.predecessor ? cursor.predecessor->id : 0; + if (!previous || previous == std::numeric_limits::max() || trade.id != previous + 1 || + trade.ts < cursor.cut || trade.ts >= cursor.cut + 60000 || trade.ts < cursor.last_tick_ts) + throw Error(20, "tick does not extend the verified chronological minute prefix"); + cursor.seq = trade.id; + cursor.last_tick_ts = trade.ts; + } else if (type == "bar" && mode == "bars") { + const auto bar = normalized_bar(event); + if (bar.ts != cursor.cut) throw Error(20, "bar does not adjoin the verified cut"); + cursor.last_bar = bar.ts; + cursor.cut += 60000; + } else if (type == "time" && mode == "ticks") { + const auto cut = event.at("ts").integer(); + if (!cursor.seq || cut != cursor.cut + 60000 || cut <= cursor.last_tick_ts) + throw Error(20, "time does not advance exactly one proven closed minute"); + cursor.cut = cut; + } else throw Error(23, "journal contains an unsupported normalized message"); +} +} + +std::string State::log_path() const { return config_.state_dir + "/events.jsonl"; } + +Json State::serialize(const Cursor& cursor) const { + Json proofs; + proofs.kind = Json::Kind::Array; + for (const auto& bar : cursor.proofs) proofs.items.push_back(parse_json(bar.wire())); + return Json::object({ + {"schema", Json::string("pineforge-feed-cursor/v1")}, {"epoch", Json::string(cursor.epoch)}, + {"venue", Json::string(config_.venue)}, {"market", Json::string(config_.market)}, + {"symbol", Json::string(config_.symbol)}, {"mode", Json::string(config_.mode)}, + {"rest_origin", Json::string(config_.rest_url)}, {"ws_origin", Json::string(config_.ws_url)}, + {"units", Json::object({{"price", Json::string("quote/base")}, {"qty", Json::string("base")}, {"ts", Json::string("unix-ms")}})}, + {"start", signed_number(cursor.start)}, {"verified_cut", signed_number(cursor.cut)}, + {"venue_id", config_.mode == "ticks" ? number(cursor.seq) : signed_number(cursor.last_bar)}, + {"emitted_seq", number(cursor.seq)}, {"last_tick_ts", signed_number(cursor.last_tick_ts)}, + {"last_bar", signed_number(cursor.last_bar)}, {"message_index", number(cursor.message_index)}, + {"log_bytes", number(cursor.log_bytes)}, {"prefix_hash", Json::string(cursor.prefix_hash)}, + {"overlap_hash", Json::string(cursor.overlap_hash)}, + {"predecessor", cursor.predecessor ? parse_json(cursor.predecessor->wire()) : Json{}}, + {"closed_proofs", std::move(proofs)}, {"partition", Json::string("one-event-per-message")} + }); +} + +void State::persist(const Cursor& cursor) const { + const auto document = serialize(cursor); + atomic_file(config_.state_dir + "/cursor.json", Json::object({{"cursor", document}, + {"sha256", Json::string(sha256(document.dump()))}}).dump() + '\n'); +} + +State::State(const Config& config) : config_(config) { + std::error_code failure; + std::filesystem::create_directories(config_.state_dir, failure); + if (failure) throw Error(22, "cannot create state directory"); + lock_.reset(::open((config_.state_dir + "/producer.lock").c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600)); + if (lock_.get() < 0 || ::flock(lock_.get(), LOCK_EX | LOCK_NB) != 0) + throw Error(22, "state directory has another producer or cannot be locked"); + if (config_.resume) { + recover(); + journal_.reset(::open(log_path().c_str(), O_WRONLY | O_APPEND | O_CLOEXEC)); + if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot sync rolled-back journal tail"); + } else { + if (config_.start < 0 || config_.start % 60000) throw Error(23, "new streams require a minute-aligned --start"); + if (std::filesystem::exists(config_.state_dir + "/cursor.json") || std::filesystem::exists(log_path())) + throw Error(23, "state already exists; use --resume or an explicitly new state directory"); + cursor_.epoch = random_epoch(); + cursor_.start = cursor_.cut = config_.start; + cursor_.prefix_hash = sha256(""); + cursor_.overlap_hash = sha256(""); + journal_.reset(::open(log_path().c_str(), O_WRONLY | O_APPEND | O_CREAT | O_EXCL | O_CLOEXEC, 0600)); + if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot initialize durable journal"); + persist(cursor_); + } + if (journal_.get() < 0) throw Error(22, "cannot open durable journal"); + if (config_.output_from > cursor_.message_index) throw Error(22, "output cursor exceeds the retained verified prefix"); +} + +void State::recover() { + try { + const auto path = config_.state_dir + "/cursor.json"; + if (!std::filesystem::exists(path) || std::filesystem::file_size(path) > 65536) + throw Error(22, "missing or oversized durable cursor"); + std::ifstream input(path); + const std::string bytes((std::istreambuf_iterator(input)), {}); + const auto envelope = parse_json(bytes); + const auto& document = envelope.at("cursor"); + if (envelope.at("sha256").text() != sha256(document.dump())) throw Error(21, "cursor digest mismatch"); + if (document.at("schema").text() != "pineforge-feed-cursor/v1" || document.at("partition").text() != "one-event-per-message") + throw Error(23, "unsupported cursor schema or message partition"); + for (const auto& item : {std::make_pair("venue", config_.venue), {"market", config_.market}, {"symbol", config_.symbol}, + {"mode", config_.mode}, {"rest_origin", config_.rest_url}, {"ws_origin", config_.ws_url}}) + if (document.at(item.first).text() != item.second) throw Error(21, "resume source identity changed"); + const auto& units = document.at("units"); + if (units.at("price").text() != "quote/base" || units.at("qty").text() != "base" || units.at("ts").text() != "unix-ms") + throw Error(21, "resume units changed"); + cursor_.epoch = document.at("epoch").text(); + cursor_.prefix_hash = document.at("prefix_hash").text(); + cursor_.overlap_hash = document.at("overlap_hash").text(); + if (!digest_token(cursor_.epoch) || !digest_token(cursor_.prefix_hash) || !digest_token(cursor_.overlap_hash)) + throw Error(21, "invalid cursor hash or epoch"); + cursor_.start = document.at("start").integer(); + cursor_.cut = document.at("verified_cut").integer(); + cursor_.seq = document.at("emitted_seq").integer(); + cursor_.last_tick_ts = document.at("last_tick_ts").integer(); + cursor_.last_bar = document.at("last_bar").integer(); + if ((config_.mode == "ticks" && document.at("venue_id").integer() != cursor_.seq) || + (config_.mode == "bars" && document.at("venue_id").integer() != cursor_.last_bar)) + throw Error(21, "venue ID does not match the emitted cursor"); + cursor_.message_index = document.at("message_index").integer(); + cursor_.log_bytes = document.at("log_bytes").integer(); + if (cursor_.start < 0 || cursor_.start % 60000 || cursor_.cut < cursor_.start || cursor_.cut % 60000 || + cursor_.log_bytes > config_.max_log_bytes || (config_.start >= 0 && config_.start != cursor_.start)) + throw Error(21, "invalid or changed resume cut"); + if (document.at("predecessor").kind != Json::Kind::Null) cursor_.predecessor = normalized_trade(document.at("predecessor")); + const auto& proofs = document.at("closed_proofs"); + if (proofs.kind != Json::Kind::Array || proofs.items.size() > 32) throw Error(21, "invalid closed-minute proof window"); + for (const auto& proof : proofs.items) cursor_.proofs.push_back(normalized_bar(proof)); + if (!std::filesystem::exists(log_path()) || std::filesystem::file_size(log_path()) < cursor_.log_bytes) + throw Error(21, "durable journal is shorter than its verified cursor"); + Cursor reconstructed; + reconstructed.start = reconstructed.cut = cursor_.start; + reconstructed.predecessor = cursor_.predecessor; + reconstructed.prefix_hash = reconstructed.overlap_hash = sha256(""); + visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + advance(reconstructed, event, config_.mode); + reconstructed.prefix_hash = sha256(reconstructed.prefix_hash + line + '\n'); + reconstructed.overlap_hash = sha256(line + '\n'); + reconstructed.log_bytes += line.size() + 1; + ++reconstructed.message_index; + remember(event); + }); + if (reconstructed.prefix_hash != cursor_.prefix_hash || reconstructed.overlap_hash != cursor_.overlap_hash || + reconstructed.log_bytes != cursor_.log_bytes || reconstructed.message_index != cursor_.message_index || + reconstructed.seq != cursor_.seq || reconstructed.cut != cursor_.cut || reconstructed.last_bar != cursor_.last_bar || + reconstructed.last_tick_ts != cursor_.last_tick_ts) + throw Error(21, "durable journal prefix does not match its cursor"); + if (::truncate(log_path().c_str(), static_cast(cursor_.log_bytes)) != 0) + throw Error(22, "cannot roll back an uncommitted journal tail"); + config_.start = cursor_.start; + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(21, "invalid durable cursor or journal"); } +} + +void State::anchor(const Trade& predecessor) { + predecessor.validate(); + if (cursor_.predecessor || cursor_.message_index || predecessor.ts >= cursor_.start) + throw Error(20, "initial raw-trade predecessor does not prove the start fence"); + auto next = cursor_; + next.predecessor = predecessor; + persist(next); + cursor_ = std::move(next); +} + +void State::remember(const Json& event) { + if (event.at("type").text() == "tick") { + recent_trades_.push_back(normalized_trade(event)); + if (recent_trades_.size() > 1000) recent_trades_.pop_front(); + } else if (event.at("type").text() == "bar") { + recent_bars_.push_back(normalized_bar(event)); + if (recent_bars_.size() > 32) recent_bars_.pop_front(); + } +} + +void State::commit(const std::string& line, const std::optional& proof) { + if (line.size() > 4095 || line.find('\n') != std::string::npos) throw Error(23, "normalized message exceeds the atomic output bound"); + auto next = cursor_; + const auto event = parse_json(line); + advance(next, event, config_.mode); + if (proof) { + proof->validate(); + if (proof->ts + 60000 != next.cut) throw Error(23, "closed-minute proof does not match the committed cut"); + next.proofs.push_back(*proof); + if (next.proofs.size() > 32) next.proofs.pop_front(); + } + if (next.message_index == std::numeric_limits::max() || + line.size() + 1 > config_.max_log_bytes - std::min(cursor_.log_bytes, config_.max_log_bytes)) + throw Error(22, "retained replay budget exhausted"); + next.log_bytes += line.size() + 1; + ++next.message_index; + next.prefix_hash = sha256(cursor_.prefix_hash + line + '\n'); + next.overlap_hash = sha256(line + '\n'); + write_all(journal_.get(), line + '\n'); + if (::fsync(journal_.get()) != 0) throw Error(22, "cannot sync normalized message"); + persist(next); + cursor_ = std::move(next); + remember(event); +} + +void State::visit(std::uint64_t from, const std::function& visitor) const { + if (from > cursor_.message_index) throw Error(22, "replay cursor is unavailable"); + std::ifstream input(log_path(), std::ios::binary); + if (!input) throw Error(22, "cannot read retained normalized log"); + std::uint64_t index = 0, bytes = 0; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(config_.max_replay_seconds); + std::string line; + while (bytes < cursor_.log_bytes) { + if (stopping) throw Stopped{}; + if (std::chrono::steady_clock::now() >= deadline) throw Error(22, "retained prefix replay time budget exhausted"); + if (!std::getline(input, line) || line.size() > 4095 || bytes + line.size() + 1 > cursor_.log_bytes) + throw Error(21, "truncated or oversized verified message"); + bytes += line.size() + 1; + if (index >= from) visitor(line); + ++index; + } + if (bytes != cursor_.log_bytes || index != cursor_.message_index) throw Error(21, "verified message partition changed"); +} + +std::optional State::trade(std::uint64_t id) const { + for (const auto& trade : recent_trades_) if (trade.id == id) return trade; + std::optional found; + visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() == "tick" && event.at("seq").integer() == id) found = normalized_trade(event); + }); + return found; +} +std::optional State::bar(std::int64_t ts) const { + for (const auto& bar : recent_bars_) if (bar.ts == ts) return bar; + for (const auto& bar : cursor_.proofs) if (bar.ts == ts) return bar; + std::optional found; + visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() == "bar" && event.at("bar").at("ts_open").integer() == ts) found = normalized_bar(event); + }); + return found; +} +Json State::status() const { + return Json::object({{"epoch", Json::string(cursor_.epoch)}, {"message_index", number(cursor_.message_index)}, + {"emitted_seq", number(cursor_.seq)}, {"verified_cut", signed_number(cursor_.cut)}, + {"prefix_hash", Json::string(cursor_.prefix_hash)}, {"cursor", Json::string(config_.state_dir + "/cursor.json")}}); +} +} diff --git a/native/feed/src/state.hpp b/native/feed/src/state.hpp new file mode 100644 index 0000000..1d7e5ee --- /dev/null +++ b/native/feed/src/state.hpp @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "core.hpp" +#include +#include + +namespace pineforge::feed { +class Descriptor { + int value_ = -1; +public: + Descriptor() = default; + explicit Descriptor(int value) : value_(value) {} + ~Descriptor(); + Descriptor(const Descriptor&) = delete; + Descriptor& operator=(const Descriptor&) = delete; + int get() const { return value_; } + void reset(int value); +}; + +struct Cursor { + std::string epoch, prefix_hash, overlap_hash; + std::uint64_t message_index = 0, log_bytes = 0, seq = 0; + std::int64_t start = 0, cut = 0, last_tick_ts = -1, last_bar = -1; + std::optional predecessor; + std::deque proofs; +}; + +class State { + Config config_; + Cursor cursor_; + Descriptor lock_, journal_; + std::deque recent_trades_; + std::deque recent_bars_; + std::string log_path() const; + Json serialize(const Cursor& cursor) const; + void persist(const Cursor& cursor) const; + void recover(); + void remember(const Json& event); +public: + explicit State(const Config& config); + const Cursor& cursor() const { return cursor_; } + const Config& config() const { return config_; } + void anchor(const Trade& predecessor); + void commit(const std::string& line, const std::optional& proof = std::nullopt); + void visit(std::uint64_t from, const std::function& visitor) const; + std::optional trade(std::uint64_t id) const; + std::optional bar(std::int64_t ts) const; + const std::deque& recent_trades() const { return recent_trades_; } + const std::deque& recent_bars() const { return recent_bars_; } + Json status() const; +}; +} diff --git a/native/feed/src/transport.cpp b/native/feed/src/transport.cpp new file mode 100644 index 0000000..f03d19e --- /dev/null +++ b/native/feed/src/transport.cpp @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "transport.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +#if LIBCURL_VERSION_NUM < 0x080e01 +#error pineforge-feed requires libcurl 8.14.1 or newer +#endif + +namespace pineforge::feed { +namespace { +using Handle = std::unique_ptr; +struct GlobalCurl { + GlobalCurl() { + if (curl_global_init(CURL_GLOBAL_DEFAULT) != CURLE_OK) throw Error(23, "curl initialization failed"); + } + ~GlobalCurl() { curl_global_cleanup(); } +}; +template void option(CURL* handle, CURLoption name, Value value) { + if (curl_easy_setopt(handle, name, value) != CURLE_OK) throw Error(23, "curl option rejected"); +} +struct Progress { const std::atomic* stopped = nullptr; }; +int progress(void* context, curl_off_t, curl_off_t, curl_off_t, curl_off_t) { + const auto* state = static_cast(context); + return stopping || (state->stopped && state->stopped->load()) ? 1 : 0; +} +Handle handle_for(const std::string& url, bool websocket, bool insecure, Progress* state) { + check_runtime_curl(); + Handle handle(curl_easy_init(), &curl_easy_cleanup); + if (!handle) throw Error(23, "curl handle allocation failed"); + option(handle.get(), CURLOPT_URL, url.c_str()); + option(handle.get(), CURLOPT_NOSIGNAL, 1L); + option(handle.get(), CURLOPT_NETRC, static_cast(CURL_NETRC_IGNORED)); + option(handle.get(), CURLOPT_FOLLOWLOCATION, 0L); + option(handle.get(), CURLOPT_MAXREDIRS, 0L); + option(handle.get(), CURLOPT_SSL_VERIFYPEER, 1L); + option(handle.get(), CURLOPT_SSL_VERIFYHOST, 2L); + option(handle.get(), CURLOPT_CONNECTTIMEOUT_MS, 10000L); + option(handle.get(), CURLOPT_TIMEOUT_MS, 20000L); + option(handle.get(), CURLOPT_USERAGENT, "pineforge-feed/0.1"); + option(handle.get(), CURLOPT_PROTOCOLS_STR, websocket ? (insecure ? "ws,wss" : "wss") : (insecure ? "http,https" : "https")); + option(handle.get(), CURLOPT_NOPROGRESS, 0L); + option(handle.get(), CURLOPT_XFERINFOFUNCTION, &progress); + option(handle.get(), CURLOPT_XFERINFODATA, state); + return handle; +} +void pause_until(std::chrono::steady_clock::time_point deadline) { + while (std::chrono::steady_clock::now() < deadline) { + if (stopping) throw Stopped{}; + std::this_thread::sleep_for(std::chrono::milliseconds(25)); + } +} +struct Response { + std::string body; + std::uint64_t retry_after = 1; + std::uint64_t used_weight = 0; + bool overflow = false; +}; +std::size_t body_callback(char* data, std::size_t size, std::size_t count, void* context) noexcept { + auto* response = static_cast(context); + const auto bytes = size * count; + if (bytes > 4 * 1024 * 1024 - response->body.size()) { response->overflow = true; return 0; } + try { response->body.append(data, bytes); return bytes; } + catch (...) { response->overflow = true; return 0; } +} +std::size_t header_callback(char* data, std::size_t size, std::size_t count, void* context) noexcept { + const auto bytes = size * count; + try { + auto* response = static_cast(context); + std::string line(data, bytes); + std::transform(line.begin(), line.end(), line.begin(), [](unsigned char value) { return static_cast(std::tolower(value)); }); + const auto colon = line.find(':'); + if (colon == std::string::npos) return bytes; + const auto first = line.find_first_not_of(" \t", colon + 1); + if (first == std::string::npos) return bytes; + std::uint64_t value = 0; + const auto result = std::from_chars(line.data() + first, line.data() + line.size(), value); + if (result.ec != std::errc{}) return bytes; + if (line.substr(0, colon) == "retry-after") response->retry_after = value; + if (line.substr(0, colon) == "x-mbx-used-weight-1m") response->used_weight = value; + } catch (...) {} + return bytes; +} +} + +void check_runtime_curl() { + static GlobalCurl initialization; + (void)initialization; + const auto* info = curl_version_info(CURLVERSION_NOW); + bool ws = false, wss = false; + for (const char* const* protocol = info ? info->protocols : nullptr; protocol && *protocol; ++protocol) { + ws = ws || std::strcmp(*protocol, "ws") == 0; + wss = wss || std::strcmp(*protocol, "wss") == 0; + } + if (!info || info->version_num < 0x080e01 || !ws || !wss) + throw Error(23, "libcurl >= 8.14.1 with both ws and wss is required at runtime"); +} + +void validate_origin(const std::string& origin, bool websocket, bool allow_insecure) { + if (origin.empty() || origin.size() > 4096 || origin.find_first_of("\r\n") != std::string::npos || origin.find('\0') != std::string::npos) + throw Error(23, "invalid public endpoint origin"); + std::unique_ptr parsed(curl_url(), &curl_url_cleanup); + if (!parsed || curl_url_set(parsed.get(), CURLUPART_URL, origin.c_str(), CURLU_NON_SUPPORT_SCHEME) != CURLUE_OK) + throw Error(23, "invalid public endpoint URL"); + auto field = [&](CURLUPart part) { + char* text = nullptr; + const auto status = curl_url_get(parsed.get(), part, &text, 0); + std::string value; + if (status == CURLUE_OK) { value = text; curl_free(text); } + return value; + }; + const auto scheme = field(CURLUPART_SCHEME); + const auto host = field(CURLUPART_HOST); + const auto secure = websocket ? "wss" : "https"; + const auto insecure = websocket ? "ws" : "http"; + if (scheme != secure && !(allow_insecure && scheme == insecure && (host == "127.0.0.1" || host == "localhost" || host == "[::1]"))) + throw Error(23, "public endpoints require TLS; insecure overrides are loopback-only"); + if (!field(CURLUPART_USER).empty() || !field(CURLUPART_PASSWORD).empty() || !field(CURLUPART_FRAGMENT).empty() || + !field(CURLUPART_QUERY).empty() || field(CURLUPART_PATH) != "/") throw Error(23, "endpoint must be a credential-free origin without a path or query"); +} + +HttpClient::HttpClient(const Config& config) : config_(config) { + check_runtime_curl(); + validate_origin(config_.rest_url, false, config_.allow_insecure); + try { + const auto metadata = get("/api/v3/exchangeInfo?symbol=" + config_.symbol, 20); + const auto& limits = metadata.at("rateLimits"); + if (limits.kind != Json::Kind::Array || limits.items.size() > 64) throw Error(23, "invalid public rate-limit metadata"); + bool weighted = false; + for (const auto& limit : limits.items) { + const auto type = limit.at("rateLimitType").text(); + if (type != "REQUEST_WEIGHT" && type != "RAW_REQUESTS") continue; + const auto interval = limit.at("interval").text(); + const auto multiple = limit.at("intervalNum").integer(); + const auto ceiling = limit.at("limit").integer(); + const auto unit = interval == "SECOND" ? 1000ULL : interval == "MINUTE" ? 60000ULL : + interval == "HOUR" ? 3600000ULL : interval == "DAY" ? 86400000ULL : 0ULL; + if (!unit || !multiple || multiple > 10000 || !ceiling || ceiling > 1000000000) + throw Error(23, "unsupported public rate-limit interval or ceiling"); + const auto pace = (unit * multiple * 2 + ceiling - 1) / ceiling; + if (type == "REQUEST_WEIGHT") { + weighted = true; + milliseconds_per_weight_ = std::max(milliseconds_per_weight_, pace); + if (interval == "MINUTE" && multiple == 1) weight_limit_ = ceiling; + } else milliseconds_per_request_ = std::max(milliseconds_per_request_, pace); + } + if (!weighted) throw Error(23, "public request-weight ceiling is unavailable"); + log("info", "rest_limits_verified", Json::object({{"milliseconds_per_weight", Json::number(std::to_string(milliseconds_per_weight_))}, + {"weight_limit_1m", Json::number(std::to_string(weight_limit_))}})); + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid public exchange rate-limit metadata"); } +} +Json HttpClient::get(const std::string& path, unsigned int weight) { + const auto endpoint = path.substr(0, path.find('?')); + if (endpoint != "/api/v3/exchangeInfo" && endpoint != "/api/v3/historicalTrades" && + endpoint != "/api/v3/aggTrades" && endpoint != "/api/v3/klines") + throw Error(23, "REST request is outside the public market-data allowlist"); + for (unsigned int attempt = 0; attempt < 4; ++attempt) { + pause_until(next_request_); + Progress state; + auto handle = handle_for(config_.rest_url + path, false, config_.allow_insecure, &state); + Response response; + option(handle.get(), CURLOPT_WRITEFUNCTION, &body_callback); + option(handle.get(), CURLOPT_WRITEDATA, &response); + option(handle.get(), CURLOPT_HEADERFUNCTION, &header_callback); + option(handle.get(), CURLOPT_HEADERDATA, &response); + const auto code = curl_easy_perform(handle.get()); + long status = 0; + curl_easy_getinfo(handle.get(), CURLINFO_RESPONSE_CODE, &status); + const auto pacing = std::max(milliseconds_per_request_, weight * milliseconds_per_weight_); + if (pacing > 86400000) throw Error(22, "public quota wait exceeds the supported resource bound"); + next_request_ = std::chrono::steady_clock::now() + std::chrono::milliseconds(config_.allow_insecure ? 0 : pacing); + if (stopping || code == CURLE_ABORTED_BY_CALLBACK) throw Stopped{}; + if (response.overflow) throw Error(22, "REST response exceeds the bounded buffer"); + if (status == 401 || status == 403 || status == 418 || status == 451 || (status >= 300 && status < 400)) + throw Error(23, "public market-data access denied or redirected (HTTP " + std::to_string(status) + ")"); + if (status == 429) { + log("warn", "rest_rate_limited", Json::object({{"retry_after", Json::number(std::to_string(response.retry_after))}})); + if (response.retry_after > 86400) throw Error(22, "public Retry-After exceeds the bounded retry window"); + next_request_ = std::chrono::steady_clock::now() + std::chrono::seconds(response.retry_after); + continue; + } + if (status == 404) throw Error(20, "public history is unavailable"); + if (code != CURLE_OK || status >= 500) { + next_request_ = std::chrono::steady_clock::now() + std::chrono::seconds(1U << attempt); + continue; + } + if (status != 200) throw Error(23, "public market-data request rejected (HTTP " + std::to_string(status) + ")"); + if (response.used_weight) log("info", "rest_weight", Json::object({{"used_weight_1m", Json::number(std::to_string(response.used_weight))}})); + if (weight_limit_ && response.used_weight >= weight_limit_ - weight_limit_ / 10) { + const auto elapsed = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count() % 60000; + next_request_ = std::max(next_request_, std::chrono::steady_clock::now() + std::chrono::milliseconds(60500 - elapsed)); + log("warn", "rest_shared_quota_pause"); + } + try { return parse_json(response.body); } + catch (const std::exception&) { throw Error(23, "public market-data response is invalid JSON"); } + } + throw Error(20, "public history retries exhausted; verified prefix retained"); +} + +bool WebSocketPump::stopped() const { return stopped_.load() || stopping.load(); } +WebSocketPump::WebSocketPump(const Config& config, const std::string& streams) : config_(config), url_(config.ws_url + "/stream?streams=" + streams) { + check_runtime_curl(); + validate_origin(config_.ws_url, true, config_.allow_insecure); + worker_ = std::thread([this] { run(); }); +} +WebSocketPump::~WebSocketPump() { + stopped_.store(true); + ready_.notify_all(); + if (worker_.joinable()) worker_.join(); +} +void WebSocketPump::push(SourceMessage message) { + std::lock_guard guard(mutex_); + const auto bytes = message.text.size() + 32; + if (bytes > config_.max_queue_bytes - std::min(queue_bytes_, config_.max_queue_bytes)) throw Error(22, "WebSocket healing buffer overrun"); + queue_bytes_ += bytes; + queue_.push_back(std::move(message)); + ready_.notify_one(); +} +SourceMessage WebSocketPump::take() { + std::unique_lock guard(mutex_); + while (queue_.empty() && !failure_) { + if (stopped()) throw Stopped{}; + ready_.wait_for(guard, std::chrono::milliseconds(100)); + } + if (failure_) std::rethrow_exception(failure_); + auto message = std::move(queue_.front()); + queue_.pop_front(); + queue_bytes_ -= message.text.size() + 32; + return message; +} + +void WebSocketPump::run() { + try { + unsigned int failures = 0; + auto next_attempt = std::chrono::steady_clock::now(); + while (!stopped()) { + while (!stopped() && std::chrono::steady_clock::now() < next_attempt) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + if (stopped()) return; + next_attempt = std::chrono::steady_clock::now() + std::chrono::milliseconds(config_.allow_insecure ? 250 : 5000); + Progress state{&stopped_}; + auto handle = handle_for(url_, true, config_.allow_insecure, &state); + option(handle.get(), CURLOPT_CONNECT_ONLY, 2L); + option(handle.get(), CURLOPT_HTTP_VERSION, static_cast(CURL_HTTP_VERSION_1_1)); + const auto handshake = curl_easy_perform(handle.get()); + long status = 0; + curl_easy_getinfo(handle.get(), CURLINFO_RESPONSE_CODE, &status); + if (stopped()) return; + if (status == 401 || status == 403 || status == 418 || status == 451 || (status >= 300 && status < 400)) + throw Error(23, "public WebSocket access denied or redirected"); + if (handshake != CURLE_OK || status != 101) { + if (++failures >= 8) throw Error(20, "public WebSocket reconnect attempts exhausted"); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(std::min(30U, 1U << failures)); + while (!stopped() && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(50)); + continue; + } + failures = 0; + reconnect_.store(false); + push({true, {}}); + log("info", "websocket_connected"); + const auto birth = std::chrono::steady_clock::now(); + auto last_text = birth, message_birth = birth; + std::string message; + bool assembling = false; + std::uint64_t frame_offset = 0; + curl_socket_t socket = CURL_SOCKET_BAD; + curl_easy_getinfo(handle.get(), CURLINFO_ACTIVESOCKET, &socket); + while (!stopped() && !reconnect_) { + const auto now = std::chrono::steady_clock::now(); + if (now - birth >= std::chrono::seconds(config_.reconnect_seconds) || now - last_text > std::chrono::seconds(75) || + (assembling && now - message_birth > std::chrono::seconds(30))) break; + std::array buffer{}; + std::size_t received = 0; + const curl_ws_frame* metadata = nullptr; + const auto result = curl_ws_recv(handle.get(), buffer.data(), buffer.size(), &received, &metadata); + if (result == CURLE_AGAIN) { + pollfd descriptor{socket, POLLIN, 0}; + ::poll(&descriptor, 1, 100); + continue; + } + if (result != CURLE_OK || !metadata || (metadata->flags & CURLWS_CLOSE)) break; + if (metadata->flags & (CURLWS_PING | CURLWS_PONG)) continue; + if ((metadata->flags & CURLWS_BINARY) || !(metadata->flags & CURLWS_TEXT) || metadata->offset < 0 || metadata->bytesleft < 0 || + static_cast(metadata->offset) != frame_offset) throw Error(23, "public WebSocket requires ordered text frames"); + if (!assembling) { assembling = true; message_birth = now; } + if (received > 1024 * 1024 - message.size() || static_cast(metadata->bytesleft) > 1024 * 1024 - message.size() - received) + throw Error(22, "WebSocket message exceeds 1 MiB"); + message.append(buffer.data(), received); + frame_offset += received; + last_text = now; + if (metadata->bytesleft) continue; + frame_offset = 0; + if (metadata->flags & CURLWS_CONT) continue; + bool retiring = false; + try { + const auto envelope = parse_json(message); + const auto* wrapped = envelope.find("data"); + const auto& data = wrapped ? *wrapped : envelope; + const auto* event = data.find("e"); + retiring = event && event->kind == Json::Kind::String && event->text() == "serverShutdown"; + } catch (const std::exception&) { throw Error(23, "invalid public WebSocket JSON"); } + push({false, std::move(message)}); + message.clear(); + assembling = false; + if (retiring) { log("info", "source_retiring_reconnect"); break; } + } + if (!stopped()) { + log("warn", "websocket_reconnect"); + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + } + } + } catch (const Stopped&) {} + catch (...) { + std::lock_guard guard(mutex_); + failure_ = std::current_exception(); + ready_.notify_all(); + } +} +} diff --git a/native/feed/src/transport.hpp b/native/feed/src/transport.hpp new file mode 100644 index 0000000..95ad62c --- /dev/null +++ b/native/feed/src/transport.hpp @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "core.hpp" +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +void check_runtime_curl(); +void validate_origin(const std::string& origin, bool websocket, bool allow_insecure); + +class HttpClient { + Config config_; + std::chrono::steady_clock::time_point next_request_{}; + std::uint64_t weight_limit_ = 0, milliseconds_per_weight_ = 100, milliseconds_per_request_ = 0; +public: + explicit HttpClient(const Config& config); + Json get(const std::string& path, unsigned int weight); +}; +struct SourceMessage { + bool connected = false; + std::string text; +}; +class WebSocketPump { + Config config_; + std::string url_; + std::atomic stopped_{false}, reconnect_{false}; + std::mutex mutex_; + std::condition_variable ready_; + std::deque queue_; + std::size_t queue_bytes_ = 0; + std::exception_ptr failure_; + std::thread worker_; + bool stopped() const; + void push(SourceMessage message); + void run(); +public: + WebSocketPump(const Config& config, const std::string& streams); + ~WebSocketPump(); + WebSocketPump(const WebSocketPump&) = delete; + WebSocketPump& operator=(const WebSocketPump&) = delete; + SourceMessage take(); + void reconnect() { reconnect_.store(true); } +}; +} diff --git a/native/feed/src/venue.hpp b/native/feed/src/venue.hpp new file mode 100644 index 0000000..8887913 --- /dev/null +++ b/native/feed/src/venue.hpp @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "core.hpp" +#include + +namespace pineforge::feed { +struct VenueEvent { + enum class Kind { Trade, Kline, Control } kind = Kind::Control; + Trade trade; + Kline kline; +}; +class Venue { +public: + virtual ~Venue() = default; + virtual std::string streams() const = 0; + virtual VenueEvent decode(const std::string& message) const = 0; + virtual std::vector history(std::uint64_t from, std::size_t limit = 1000) = 0; + virtual std::uint64_t first_trade_id(std::int64_t minute) = 0; + virtual std::vector klines(std::int64_t start, std::int64_t end) = 0; +}; +} diff --git a/native/feed/tests/batch_probe.cpp b/native/feed/tests/batch_probe.cpp new file mode 100644 index 0000000..6dfee52 --- /dev/null +++ b/native/feed/tests/batch_probe.cpp @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +template Function load(void* library, const char* name) { + auto function = reinterpret_cast(dlsym(library, name)); + if (!function) throw std::runtime_error(std::string("missing qualification export: ") + name); + return function; +} + +int main(int argc, char** argv) { + try { + if (argc != 4) throw std::runtime_error("usage: batch-probe observed-strategy.so bars.csv actions.jsonl"); + std::ifstream input(argv[2]); + if (!input) throw std::runtime_error("cannot open batch input"); + std::string line; + std::getline(input, line); + if (line != "timestamp,open,high,low,close,volume") throw std::runtime_error("unexpected CSV schema"); + std::vector bars; + while (std::getline(input, line)) { + std::istringstream fields(line); + std::vector values; + std::string value; + while (std::getline(fields, value, ',')) values.push_back(value); + if (values.size() != 6) throw std::runtime_error("invalid CSV row"); + bars.push_back({std::stod(values[1]), std::stod(values[2]), std::stod(values[3]), + std::stod(values[4]), std::stod(values[5]), std::stoll(values[0])}); + } + void* library = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (!library) throw std::runtime_error(dlerror()); + if (load(library, "pf_abi_version")() != PF_ABI_VERSION) + throw std::runtime_error("qualification strategy ABI mismatch"); + auto create = load(library, "strategy_create"); + auto release = load(library, "strategy_free"); + auto report_release = load(library, "report_free"); + auto run = load(library, "run_backtest_full"); + auto retain = load(library, "equivalence_retain_events"); + auto actions = load(library, "equivalence_export_actions"); + auto error = load(library, "strategy_get_last_error"); + auto strategy = create(nullptr); + if (!strategy || retain(strategy)) throw std::runtime_error("cannot retain batch execution receipts"); + pf_report_t report{}; + run(strategy, bars.data(), static_cast(bars.size()), "1", "1", 0, 4, PF_MAGNIFIER_ENDPOINTS, &report); + const auto* failure = error(strategy); + if (failure && *failure) throw std::runtime_error(failure); + if (actions(strategy, argv[3])) throw std::runtime_error("cannot export batch execution actions"); + std::cout << "batch_abi=run_backtest_full input_bars=" << bars.size() + << " script_bars=" << report.script_bars_processed << " trades=" << report.trades_len << '\n'; + report_release(&report); + release(strategy); + dlclose(library); + return 0; + } catch (const std::exception& failure) { + std::cerr << failure.what() << '\n'; + return 1; + } +} diff --git a/native/feed/tests/build_ci_curl.sh b/native/feed/tests/build_ci_curl.sh new file mode 100644 index 0000000..c2d50b7 --- /dev/null +++ b/native/feed/tests/build_ci_curl.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +prefix="${1:?provide an installation prefix}" +source_dir="${prefix}-source" +mkdir -p "$source_dir" +archive="$source_dir/curl-8.14.1.tar.xz" +curl --fail --location --retry 3 https://curl.se/download/curl-8.14.1.tar.xz -o "$archive" +printf '%s %s\n' f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd "$archive" | shasum -a 256 --check +tar -xJf "$archive" -C "$source_dir" +cmake -S "$source_dir/curl-8.14.1" -B "$source_dir/build" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$prefix" -DCURL_DISABLE_WEBSOCKETS=OFF \ + -DCURL_USE_OPENSSL=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=OFF \ + -DCURL_USE_LIBPSL=OFF -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=OFF \ + -DCURL_BROTLI=OFF -DCURL_ZSTD=OFF +cmake --build "$source_dir/build" -j4 +cmake --install "$source_dir/build" +LD_LIBRARY_PATH="$prefix/lib:${LD_LIBRARY_PATH:-}" \ + DYLD_LIBRARY_PATH="$prefix/lib:${DYLD_LIBRARY_PATH:-}" "$prefix/bin/curl" --version diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py new file mode 100644 index 0000000..08a4060 --- /dev/null +++ b/native/feed/tests/mock_venue.py @@ -0,0 +1,528 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Synthetic public Binance shapes; standard-library HTTP/WS fault injection.""" +import base64 +import copy +import fcntl +import hashlib +import http.server +import json +import pathlib +import signal +import socket +import struct +import subprocess +import sys +import tempfile +import threading +import time +import urllib.parse +import unittest +from types import SimpleNamespace +from public_e2e import Soak + + +BINARY = str(pathlib.Path(sys.argv[1]).resolve()) +sys.argv = [sys.argv[0]] +TRADES = { + 99: (119999, "10.10000000", "0.10000000"), + 100: (120001, "10.10000000", "0.10000000"), + 101: (120050, "11.20000000", "0.20000000"), + 102: (179999, "9.90000000", "0.30000000"), + 103: (180001, "12.00000000", "0.40000000"), + 104: (180050, "11.00000000", "0.10000000"), + 105: (240001, "12.00000000", "0.10000000"), +} +BARS = { + 120000: ("10.10000000", "11.20000000", "9.90000000", "9.90000000", "0.60000000", 100, 102, 3), + 180000: ("12.00000000", "12.00000000", "11.00000000", "11.00000000", "0.50000000", 103, 104, 2), + 240000: ("12.00000000", "12.00000000", "12.00000000", "12.00000000", "0.10000000", 105, 105, 1), +} + + +def trade(identifier, **changes): + matched, price, quantity = TRADES[identifier] + data = {"e": "trade", "E": matched + 999, "s": "TESTUSDT", "t": identifier, + "p": price, "q": quantity, "T": matched, "m": False, "M": True} + data.update(changes) + return {"stream": "testusdt@trade", "data": data} + + +def candle(minute, confirmed=True, **changes): + opening, high, low, close, volume, first, last, count = BARS[minute] + row = {"t": minute, "T": minute + 59999, "s": "TESTUSDT", "i": "1m", "f": first, + "L": last, "o": opening, "h": high, "l": low, "c": close, "v": volume, + "n": count, "x": confirmed, "q": "0.00000000", "V": "0.00000000", + "Q": "0.00000000", "B": "0"} + row.update(changes) + return {"stream": "testusdt@kline_1m", "data": {"e": "kline", "E": minute + 60000, + "s": "TESTUSDT", "k": row}} + + +def retirement(): + return {"stream": "!serverShutdown", "data": {"e": "serverShutdown", "E": 180000}} + + +def frame(connection, payload, opcode=1, final=True): + if isinstance(payload, dict): + payload = json.dumps(payload, separators=(",", ":")).encode() + if isinstance(payload, str): + payload = payload.encode() + header = bytes([(128 if final else 0) | opcode]) + if len(payload) < 126: + header += bytes([len(payload)]) + elif len(payload) < 65536: + header += bytes([126]) + struct.pack("!H", len(payload)) + else: + header += bytes([127]) + struct.pack("!Q", len(payload)) + connection.sendall(header + payload) + + +def receive(connection): + def exact(size): + result = bytearray() + while len(result) < size: + chunk = connection.recv(size - len(result)) + if not chunk: + raise EOFError + result.extend(chunk) + return bytes(result) + first, second = exact(2) + size = second & 127 + if size == 126: + size = struct.unpack("!H", exact(2))[0] + if size == 127: + size = struct.unpack("!Q", exact(8))[0] + if not second & 128: + raise AssertionError("client frames must be masked") + mask = exact(4) + payload = exact(size) + return first & 15, bytes(value ^ mask[index % 4] for index, value in enumerate(payload)) + + +class MockServer(http.server.ThreadingHTTPServer): + daemon_threads = True + block_on_close = False + + def __init__(self, scripts): + super().__init__(("127.0.0.1", 0), Handler) + self.scripts = scripts + self.connections = 0 + self.trades = copy.deepcopy(TRADES) + self.bars = copy.deepcopy(BARS) + self.requests = [] + self.pongs = [] + self.fail_status = 0 + self.rate_once = False + self.retry_after = 0 + self.rate_limits = None + self.auth_headers = [] + self.lock = threading.Lock() + self.errors = [] + + def handle_error(self, request, client_address): + failure = sys.exc_info()[1] + if not isinstance(failure, (BrokenPipeError, ConnectionResetError, EOFError)): + self.errors.append(repr(failure)) + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *arguments): + pass + + def do_GET(self): + parsed = urllib.parse.urlsplit(self.path) + query = urllib.parse.parse_qs(parsed.query) + with self.server.lock: + self.server.requests.append(parsed.path) + self.server.auth_headers.extend(name for name in ("Authorization", "X-MBX-APIKEY") if self.headers.get(name)) + if parsed.path == "/stream": + self.websocket(query) + return + with self.server.lock: + if self.server.fail_status: + self.respond(self.server.fail_status, {"code": -1, "msg": "synthetic refusal"}) + return + if self.server.rate_once: + self.server.rate_once = False + self.respond(429, {}, retry=True) + return + if parsed.path == "/api/v3/exchangeInfo": + rows = {"rateLimits": [{"rateLimitType": "REQUEST_WEIGHT", "interval": "MINUTE", "intervalNum": 1, "limit": 6000}, + {"rateLimitType": "RAW_REQUESTS", "interval": "MINUTE", "intervalNum": 5, "limit": 61000}], + "symbols": [{"symbol": "TESTUSDT", "baseAsset": "TEST", "quoteAsset": "USDT"}]} + if self.server.rate_limits is not None: + rows["rateLimits"] = self.server.rate_limits + elif parsed.path == "/api/v3/historicalTrades": + start = int(query["fromId"][0]) + limit = int(query["limit"][0]) + rows = [] + for identifier in sorted(self.server.trades): + if identifier >= start and len(rows) < limit: + matched, price, quantity = self.server.trades[identifier] + rows.append({"id": identifier, "price": price, "qty": quantity, + "quoteQty": "0.00000000", "time": matched, + "isBuyerMaker": False, "isBestMatch": True}) + elif parsed.path == "/api/v3/aggTrades": + start, end = int(query["startTime"][0]), int(query["endTime"][0]) + matching = [identifier for identifier in sorted(self.server.trades) + if start <= self.server.trades[identifier][0] <= end] + rows = [] if not matching else [{"a": 42, "f": matching[0], "l": matching[0], + "T": self.server.trades[matching[0]][0], + "p": "10.10000000", "q": "0.10000000", + "m": False, "M": True}] + elif parsed.path == "/api/v3/klines": + start, end = int(query["startTime"][0]), int(query["endTime"][0]) + rows = [] + for minute in sorted(self.server.bars): + if start <= minute <= end and len(rows) < 1000: + opening, high, low, close, volume, first, last, count = self.server.bars[minute] + rows.append([minute, opening, high, low, close, volume, minute + 59999, + "0.00000000", count, "0.00000000", "0.00000000", "0"]) + else: + self.respond(404, {}) + return + self.respond(200, rows) + + def respond(self, status, value, retry=False): + payload = json.dumps(value, separators=(",", ":")).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.send_header("Connection", "close") + if retry: + self.send_header("Retry-After", str(self.server.retry_after)) + self.end_headers() + self.wfile.write(payload) + self.close_connection = True + + def websocket(self, query): + assert query["streams"] == ["testusdt@trade/testusdt@kline_1m"] + key = self.headers["Sec-WebSocket-Key"] + accept = base64.b64encode(hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest()).decode() + self.send_response(101) + self.send_header("Upgrade", "websocket") + self.send_header("Connection", "Upgrade") + self.send_header("Sec-WebSocket-Accept", accept) + self.end_headers() + with self.server.lock: + connection_index = self.server.connections + self.server.connections += 1 + script = self.server.scripts[min(connection_index, len(self.server.scripts) - 1)] + self.connection.settimeout(3) + for action in script: + if isinstance(action, dict): + frame(self.connection, action) + elif action[0] == "pause": + time.sleep(action[1]) + elif action[0] == "ping": + frame(self.connection, action[1], opcode=9) + opcode, payload = receive(self.connection) + assert opcode == 10 and payload == action[1] + self.server.pongs.append(payload) + elif action[0] == "fragment": + payload = json.dumps(action[1], separators=(",", ":")).encode() + middle = len(payload) // 2 + frame(self.connection, payload[:middle], final=False) + frame(self.connection, payload[middle:], opcode=0) + elif action[0] == "binary": + frame(self.connection, action[1], opcode=2) + elif action[0] == "raw": + frame(self.connection, action[1]) + elif action[0] == "revise_rest": + with self.server.lock: + row = list(self.server.bars[action[1]]) + row[4] = "0.70000000" + self.server.bars[action[1]] = tuple(row) + elif action[0] == "close": + frame(self.connection, b"", opcode=8) + self.close_connection = True + return + self.connection.settimeout(0.1) + while True: + try: + receive(self.connection) + except socket.timeout: + continue + except (EOFError, ConnectionResetError): + break + self.close_connection = True + + +class FeedMockTests(unittest.TestCase): + def test_public_qualification_requires_explicit_tick_ohlc_exception(self): + for allow, expected in ((False, 1), (True, 0)): + soak = Soak.__new__(Soak) + soak.options = SimpleNamespace(allow_tick_ohlc_difference=allow, bar_minutes=46, tick_minutes=21) + soak.directory = self.directory / str(allow) + soak.directory.mkdir() + soak.results = {} + soak.receiver = SimpleNamespace(shutdown=lambda: None) + soak.warmup = lambda: None + soak.live = lambda mode, minutes: soak.results.update({mode: {"batch_actions_equal": mode == "bars"}}) + self.assertEqual(soak.run(), expected) + self.assertFalse(json.loads((soak.directory / "summary.json").read_text())["ticks"]["batch_actions_equal"]) + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="pineforge-feed-mock-") + self.addCleanup(self.temporary.cleanup) + self.directory = pathlib.Path(self.temporary.name) + + def server(self, scripts): + server = MockServer(scripts) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return server + + def command(self, server, mode="bars", extra=(), resume=False): + port = server.server_address[1] + arguments = [BINARY, "run", "--venue", "binance", "--market", "spot", "--symbol", "TESTUSDT", + "--mode", mode, "--state-dir", str(self.directory / "state"), + "--rest-url", f"http://127.0.0.1:{port}", "--ws-url", f"ws://127.0.0.1:{port}", + "--allow-insecure-http"] + arguments += ["--resume"] if resume else ["--start", "120000"] + return arguments + list(extra) + + def run_feed(self, server, mode="bars", extra=(), expected=0, resume=False): + result = subprocess.run(self.command(server, mode, extra, resume), capture_output=True, text=True, timeout=12) + self.assertEqual(result.returncode, expected, result.stderr) + self.assertFalse(server.errors, server.errors) + for record in result.stderr.splitlines(): + self.assertIsInstance(json.loads(record), dict) + output = [json.loads(line) for line in result.stdout.splitlines()] + cursor = self.directory / "state" / "cursor.json" + if cursor.exists(): + saved = json.loads(cursor.read_text())["cursor"] + self.assertEqual(saved["message_index"], len((cursor.parent / "events.jsonl").read_text().splitlines())) + return output, result + + def test_bars_hole_is_healed_behind_closed_watermark(self): + server = self.server([[candle(240000)]]) + output, _ = self.run_feed(server, extra=["--max-messages", "3"]) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000, 180000, 240000]) + + def test_unhealable_bar_hole_stops_20(self): + server = self.server([[candle(240000)]]) + del server.bars[180000] + output, _ = self.run_feed(server, expected=20) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000]) + + def test_raw_hole_and_reordering_are_healed_not_resequenced(self): + server = self.server([[trade(102), trade(100), trade(101), trade(103), candle(120000), candle(180000)]]) + output, result = self.run_feed(server, "ticks", ["--max-messages", "7"]) + self.assertEqual([event["seq"] for event in output if event["type"] == "tick"], [100, 101, 102, 103, 104]) + self.assertEqual([event["ts"] for event in output if event["type"] == "time"], [180000, 240000]) + self.assertIn('"price":10.10000000', result.stdout) + self.assertEqual(output[0]["ts"], 120001) + + def test_unhealable_raw_hole_stops_20(self): + server = self.server([[trade(102)]]) + del server.trades[101] + output, _ = self.run_feed(server, "ticks", expected=20) + self.assertTrue(all(event["type"] != "time" for event in output)) + + def test_identical_duplicates_do_not_change_message_boundaries(self): + server = self.server([[candle(120000), candle(120000), candle(180000)]]) + output, _ = self.run_feed(server, extra=["--max-messages", "2"]) + self.assertEqual(len(output), 2) + + def test_conflicting_raw_duplicate_stops_21(self): + server = self.server([[trade(100), trade(100, q="0.20000000")]]) + output, _ = self.run_feed(server, "ticks", expected=21) + self.assertEqual(len(output), 1) + + def test_revised_already_emitted_bar_stops_21(self): + server = self.server([[candle(120000), candle(120000, v="0.70000000")]]) + output, _ = self.run_feed(server, expected=21) + self.assertEqual(len(output), 1) + + def test_ohlcv_mismatch_emits_no_time(self): + server = self.server([[candle(120000, v="0.70000000")]]) + output, _ = self.run_feed(server, "ticks", expected=21) + self.assertEqual([event["type"] for event in output], ["tick", "tick", "tick"]) + + def test_empty_ambiguous_tick_minute_stops_20(self): + server = self.server([[candle(120000, n=0, f=-1, L=-1, v="0.00000000")]]) + output, _ = self.run_feed(server, "ticks", expected=20) + self.assertEqual(output, []) + + def test_forming_candles_never_advance_time(self): + server = self.server([[candle(120000, confirmed=False), trade(100), candle(120000)]]) + output, _ = self.run_feed(server, "ticks", ["--max-messages", "4"]) + self.assertEqual([event["type"] for event in output], ["tick", "tick", "tick", "time"]) + + def test_ping_payload_is_echoed_and_fragmented_text_works(self): + server = self.server([[('ping', b"synthetic-ping"), ("fragment", candle(120000))]]) + output, _ = self.run_feed(server, extra=["--max-messages", "1"]) + self.assertEqual(len(output), 1) + self.assertEqual(server.pongs, [b"synthetic-ping"]) + + def test_server_shutdown_reconnects_with_exact_overlap(self): + server = self.server([[candle(120000), retirement()], [candle(120000), candle(180000)]]) + output, _ = self.run_feed(server, extra=["--max-messages", "2"]) + self.assertEqual(len(output), 2) + self.assertGreaterEqual(server.connections, 2) + + def test_reconnect_changed_rest_overlap_stops_21(self): + server = self.server([[candle(120000), ("revise_rest", 120000), retirement()], [candle(180000)]]) + output, _ = self.run_feed(server, expected=21) + self.assertEqual(len(output), 1) + + def test_planned_24_hour_rotation_uses_overlap(self): + server = self.server([[candle(120000)], [candle(180000)]]) + output, _ = self.run_feed(server, extra=["--max-messages", "2", "--reconnect-seconds", "1"]) + self.assertEqual(len(output), 2) + self.assertGreaterEqual(server.connections, 2) + + def test_protocol_invalid_symbol_and_binary_stop_23(self): + for payload in [trade(100, s="OTHERUSDT"), ("binary", b"not-a-text-event"), ("raw", "{")]: + with self.subTest(payload=payload): + server = self.server([[payload]]) + state = self.directory / "state" + if state.exists(): + import shutil + shutil.rmtree(state) + self.run_feed(server, "ticks", expected=23) + + def test_rest_access_failure_stops_23_no_key_sent(self): + server = self.server([[candle(240000)]]) + server.fail_status = 451 + output, _ = self.run_feed(server, expected=23) + self.assertEqual(output, []) + self.assertTrue(all(path in {"/stream", "/api/v3/klines", "/api/v3/exchangeInfo"} for path in server.requests)) + self.assertEqual(server.auth_headers, []) + + def test_missing_rate_limit_metadata_is_refused(self): + server = self.server([[candle(120000)]]) + server.rate_limits = [] + output, _ = self.run_feed(server, expected=23) + self.assertEqual(output, []) + + def test_excessive_retry_after_stops_instead_of_retrying_early(self): + server = self.server([[candle(120000)]]) + server.rate_once = True + server.retry_after = 86401 + output, _ = self.run_feed(server, expected=22) + self.assertEqual(output, []) + + def test_aggregate_trade_is_never_substituted_for_raw_prints(self): + message = trade(100) + message["data"]["e"] = "aggTrade" + server = self.server([[message]]) + output, _ = self.run_feed(server, "ticks", expected=23) + self.assertEqual(output, []) + + def test_sigterm_drain_is_bounded_with_unread_stdout(self): + messages = [] + for offset in range(2000): + message = candle(120000) + minute = 120000 + offset * 60000 + message["data"]["k"]["t"] = minute + message["data"]["k"]["T"] = minute + 59999 + messages.append(message) + server = self.server([messages]) + process = subprocess.Popen(self.command(server), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + if hasattr(fcntl, "F_SETPIPE_SZ"): + fcntl.fcntl(process.stdout.fileno(), fcntl.F_SETPIPE_SZ, 4096) + cursor = self.directory / "state" / "cursor.json" + deadline = time.monotonic() + 20 + previous_index = 0 + stable_since = time.monotonic() + while time.monotonic() < deadline: + if cursor.exists(): + current_index = json.loads(cursor.read_text())["cursor"]["message_index"] + if current_index != previous_index: + previous_index = current_index + stable_since = time.monotonic() + elif 30 <= current_index < len(messages) and time.monotonic() - stable_since >= 0.4: + break + self.assertIsNone(process.poll()) + time.sleep(0.02) + else: + self.fail(f"unread stdout did not stall before source exhaustion; committed={previous_index}") + self.assertIsNone(process.poll()) + started = time.monotonic() + process.send_signal(signal.SIGTERM) + self.assertEqual(process.wait(timeout=7), 0) + self.assertLess(time.monotonic() - started, 6) + self.assertGreaterEqual(json.loads(cursor.read_text())["cursor"]["message_index"], 30) + finally: + if process.poll() is None: + process.kill() + process.wait() + process.stdout.close() + process.stderr.close() + + def test_429_retry_after_preserves_prefix(self): + server = self.server([[candle(240000)]]) + server.rate_once = True + output, result = self.run_feed(server, extra=["--max-messages", "3"]) + self.assertEqual(len(output), 3) + self.assertIn("rest_rate_limited", result.stderr) + + def test_replay_and_queue_budgets_stop_22(self): + server = self.server([[candle(120000)]]) + output, _ = self.run_feed(server, extra=["--max-log-bytes", "4"], expected=22) + self.assertEqual(output, []) + import shutil + shutil.rmtree(self.directory / "state") + server = self.server([[trade(100)]]) + output, _ = self.run_feed(server, "ticks", ["--max-queue-bytes", "64"], expected=22) + self.assertEqual(output, []) + + def test_sigterm_then_resume_tail_has_no_gap_or_duplicate(self): + server = self.server([[candle(120000)], [candle(120000), candle(240000)]]) + process = subprocess.Popen(self.command(server), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + cursor = self.directory / "state" / "cursor.json" + deadline = time.monotonic() + 8 + while time.monotonic() < deadline: + if cursor.exists() and json.loads(cursor.read_text())["cursor"]["message_index"] == 1: + break + self.assertIsNone(process.poll()) + time.sleep(0.02) + else: + process.kill() + self.fail("first atomic message did not commit") + process.send_signal(signal.SIGTERM) + stdout, stderr = process.communicate(timeout=5) + self.assertEqual(process.returncode, 0, stderr) + self.assertEqual(len(stdout.splitlines()), 1) + output, _ = self.run_feed(server, extra=["--output-from", "1", "--max-messages", "2"], resume=True) + self.assertEqual([event["bar"]["ts_open"] for event in output], [180000, 240000]) + + def test_changed_committed_journal_is_refused(self): + server = self.server([[candle(120000)], [candle(180000)]]) + self.run_feed(server, extra=["--max-messages", "1"]) + journal = self.directory / "state" / "events.jsonl" + journal.write_text(journal.read_text().replace("10.10000000", "10.20000000")) + output, _ = self.run_feed(server, resume=True, expected=21) + self.assertEqual(output, []) + + def test_warmup_exclusive_cut_manifest_and_tokens(self): + server = self.server([[candle(240000)]]) + port = server.server_address[1] + output = self.directory / "warmup.csv" + command = [BINARY, "warmup", "--venue", "binance", "--market", "spot", "--symbol", "TESTUSDT", + "--start", "120000", "--end", "240000", "--output", str(output), + "--rest-url", f"http://127.0.0.1:{port}", "--ws-url", f"ws://127.0.0.1:{port}", + "--allow-insecure-http"] + result = subprocess.run(command, capture_output=True, text=True, timeout=8) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "") + manifest = json.loads(pathlib.Path(str(output) + ".manifest.json").read_text()) + self.assertEqual(manifest["cut"], 240000) + self.assertEqual(manifest["bars"], 2) + self.assertEqual(manifest["sha256"], hashlib.sha256(output.read_bytes()).hexdigest()) + self.assertIn("10.10000000", output.read_text()) + + +if __name__ == "__main__": + suite = unittest.defaultTestLoader.loadTestsFromTestCase(FeedMockTests) + result = unittest.TextTestRunner(verbosity=2).run(suite) + if result.wasSuccessful(): + print(f"PASS local HTTP+WS mock venue ({result.testsRun} scenarios)", flush=True) + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/native/feed/tests/public_e2e.py b/native/feed/tests/public_e2e.py new file mode 100644 index 0000000..866dc96 --- /dev/null +++ b/native/feed/tests/public_e2e.py @@ -0,0 +1,311 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Explicit public-data soak; captures stay in the operator-selected directory.""" +import argparse +import csv +import hashlib +import http.server +import json +import pathlib +import signal +import struct +import subprocess +import threading +import time +import traceback +import urllib.parse +import urllib.request +from decimal import Decimal, getcontext + + +def rows(path, exact=False): + with open(path) as source: + return [json.loads(line, parse_float=Decimal if exact else float) for line in source if line.strip()] + + +def action_key(record): + order = record["order"] + return (record["timestamp"], record["bar_index"], order["id"], order["action"], order["leg"], + struct.pack("!d", float(order["contracts"])).hex(), struct.pack("!d", float(order["price"])).hex(), + order["reduce_only"], order["entry_incarnation"]) + + +class Receiver(http.server.ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, directory): + super().__init__(("127.0.0.1", 0), ReceiverHandler) + self.directory = directory + self.lock = threading.Lock() + self.payloads = {} + threading.Thread(target=self.serve_forever, daemon=True).start() + + +class ReceiverHandler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + name = self.path.removeprefix("/actions/") + if not name.replace("-", "").isalnum(): + self.send_error(404) + return + payload = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + with self.server.lock: + self.server.payloads.setdefault(name, []).append(payload) + with open(self.server.directory / (name + "-actions.jsonl"), "a") as output: + output.write(json.dumps(payload, separators=(",", ":")) + "\n") + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, *_): + pass + + +class Soak: + def __init__(self, options): + self.options = options + self.directory = pathlib.Path(options.directory).resolve() + self.directory.mkdir(parents=True, exist_ok=False) + self.receiver = Receiver(self.directory) + self.cut = int(time.time() * 1000) // 60000 * 60000 + self.results = {} + self.lock = threading.Lock() + + def receipt(self, message): + with self.lock: + print(message, flush=True) + with open(self.directory / "receipts.log", "a") as output: + output.write(message + "\n") + + def runner_command(self, mode, name, ledger, source, cursor=0): + command = [self.options.runner, "run", "--strategy", self.options.strategy, + "--warmup", str(self.directory / "warmup.csv"), "--feed", source, + "--mode", mode, "--script-tf", "1", "--ledger", str(ledger), + "--symbol", "BINANCE:BTCUSDT", "--name", "public-rsi-qualification", + "--webhook-url", f"http://127.0.0.1:{self.receiver.server_port}/actions/{name}", + "--allow-insecure-http"] + if cursor: + command += ["--from-input", str(cursor)] + return command + + def warmup(self): + command = [self.options.feed, "warmup", "--venue", "binance", "--market", "spot", + "--symbol", "BTCUSDT", "--start", str(self.cut - 200 * 60000), + "--end", str(self.cut), "--output", str(self.directory / "warmup.csv")] + with open(self.directory / "warmup.stderr", "w") as error: + subprocess.run(command, stderr=error, check=True, timeout=150) + manifest = json.loads((self.directory / "warmup.csv.manifest.json").read_text()) + assert manifest["end_exclusive"] == self.cut + assert manifest["sha256"] == hashlib.sha256((self.directory / "warmup.csv").read_bytes()).hexdigest() + self.receipt("PASS warmup exclusive cut and SHA-256") + + def generation(self, mode, generation, cursor): + command = [self.options.feed, "run", "--venue", "binance", "--market", "spot", "--symbol", "BTCUSDT", + "--mode", mode, "--state-dir", str(self.directory / (mode + "-state"))] + command += ["--resume", "--output-from", str(cursor)] if generation else ["--start", str(self.cut)] + feed_error = open(self.directory / f"{mode}-feed-{generation}.stderr", "w") + runner_error = open(self.directory / f"{mode}-runner-{generation}.stderr", "w") + runner_output = open(self.directory / f"{mode}-runner-{generation}.stdout", "w") + runner = subprocess.Popen(self.runner_command(mode, mode, self.directory / (mode + ".sqlite3"), "-", cursor), + stdin=subprocess.PIPE, stdout=runner_output, stderr=runner_error) + feed = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=feed_error) + failures = [] + + def pump(): + try: + with open(self.directory / (mode + "-feed.jsonl"), "ab", buffering=0) as capture: + for line in feed.stdout: + if not line.endswith(b"\n"): + continue + runner.stdin.write(line) + runner.stdin.flush() + capture.write(line) + except Exception as failure: + failures.append(str(failure)) + finally: + runner.stdin.close() + + thread = threading.Thread(target=pump, daemon=True) + thread.start() + return feed, runner, thread, failures, [feed_error, runner_error, runner_output] + + def finish(self, mode, generation, processes, crash=False): + feed, runner, thread, failures, files = processes + if feed.poll() is None: + feed.send_signal(signal.SIGKILL if crash else signal.SIGTERM) + code = feed.wait(timeout=20) + thread.join(timeout=30) + assert not thread.is_alive(), "pipe drain exceeded deadline" + runner_code = runner.wait(timeout=40) + for output in files: + output.close() + assert code == (-signal.SIGKILL if crash else 0), f"feed exit={code}" + assert runner_code == 0, f"runner exit={runner_code}" + assert not failures, failures + summary = json.loads((self.directory / f"{mode}-runner-{generation}.stdout").read_text()) + return summary["inputs_committed"] + + def fetch(self, path, query, weight=2): + time.sleep(weight / 10) + url = "https://api.binance.com" + path + "?" + urllib.parse.urlencode(query) + with urllib.request.urlopen(url, timeout=30) as response: + return json.loads(response.read(), parse_float=Decimal) + + def candles(self, start, end): + result = [] + while start < end: + page = self.fetch("/api/v3/klines", {"symbol": "BTCUSDT", "interval": "1m", "startTime": start, + "endTime": end - 1, "limit": 1000}) + assert page and page[0][0] == start, "REST minute missing" + result += page + start = page[-1][0] + 60000 + assert len(result) * 60000 == end - result[0][0] + return result + + def validate(self, mode, duration, cursor): + getcontext().prec = 160 + tape = rows(self.directory / (mode + "-feed.jsonl"), True) + with open(self.directory / (mode + "-feed.jsonl")) as source: + tokens = [json.loads(line, parse_float=str, parse_int=str) for line in source if line.strip()] + assert len(tape) == cursor, "capture/runner cursor mismatch" + durable = rows(self.directory / (mode + "-state/events.jsonl"), True) + assert tape == durable, "restart changed normalized prefix or message boundaries" + self.receipt(f"PASS {mode} restart without gap or duplicate messages={cursor}") + if mode == "bars": + normalized = [event["bar"] for event in tape] + assert len(normalized) >= 45 and duration >= 45 * 60 + expected = self.candles(self.cut, normalized[-1]["ts_open"] + 60000) + for index, (actual, candle) in enumerate(zip(normalized, expected, strict=True)): + assert [actual["ts_open"], *(actual[key] for key in ("o", "h", "l", "c", "v"))] == \ + [candle[0], *(Decimal(candle[index]) for index in range(1, 6))], "bar REST mismatch" + assert tape[index]["type"] == "bar" + assert [tokens[index]["bar"][key] for key in ("o", "h", "l", "c", "v")] == candle[1:6], "bar decimal tokens changed" + self.receipt(f"PASS bars equal REST minutes={len(normalized)} duration_seconds={duration:.3f}") + else: + assert duration >= 20 * 60 + trades = [event for event in tape if event["type"] == "tick"] + trade_tokens = [event for event in tokens if event["type"] == "tick"] + times = [event["ts"] for event in tape if event["type"] == "time"] + assert len(times) >= 20 and trades + for previous, current in zip(trades, trades[1:]): + assert current["seq"] == previous["seq"] + 1 and current["ts"] >= previous["ts"] + next_id = trades[0]["seq"] + trade_offset = 0 + while trade_offset < len(trades): + page = self.fetch("/api/v3/historicalTrades", {"symbol": "BTCUSDT", "fromId": next_id, + "limit": 1000}, 25) + assert page and page[0]["id"] == next_id + for raw in page: + if trade_offset == len(trades): + break + tick = trades[trade_offset] + assert [tick["seq"], tick["ts"], tick["price"], tick["qty"]] == \ + [raw["id"], raw["time"], Decimal(raw["price"]), Decimal(raw["qty"])], "raw REST mismatch" + assert [trade_tokens[trade_offset][key] for key in ("price", "qty")] == [raw["price"], raw["qty"]], "raw decimal tokens changed" + trade_offset += 1 + next_id += 1 + expected = self.candles(self.cut, times[-1]) + normalized = [] + minute_trades = [] + minute = self.cut + for event in tape: + if event["type"] == "tick": + assert minute <= event["ts"] < minute + 60000, "time/print ordering violation" + minute_trades.append(event) + else: + assert event["type"] == "time" and event["ts"] == minute + 60000 + raw = expected[len(normalized)] + prices = [tick["price"] for tick in minute_trades] + assert prices and len(prices) == raw[8] + values = [prices[0], max(prices), min(prices), prices[-1], sum(tick["qty"] for tick in minute_trades)] + assert values == [Decimal(raw[index]) for index in range(1, 6)], "time OHLCV mismatch" + normalized.append(dict(zip(("ts_open", "o", "h", "l", "c", "v"), [minute, *values], strict=True))) + minute += 60000 + minute_trades = [] + self.receipt(f"PASS ticks contiguous and time events equal REST ticks={len(trades)} minutes={len(times)} duration_seconds={duration:.3f}") + combined = self.directory / (mode + "-combined.csv") + with open(combined, "w") as output: + output.write((self.directory / "warmup.csv").read_text()) + for bar in normalized: + output.write(",".join(str(bar[key]) for key in ("ts_open", "o", "h", "l", "c", "v")) + "\n") + batch_actions = self.directory / (mode + "-batch-actions.jsonl") + subprocess.run([self.options.batch_probe, self.options.observed_strategy, str(combined), str(batch_actions)], check=True) + expected_actions = [action_key(record) for record in rows(batch_actions) if record["origin_input_index"] >= 200] + actual_actions = [action_key(record) for record in self.receiver.payloads.get(mode, [])] + event_ids = [record["event_id"] for record in self.receiver.payloads.get(mode, [])] + assert len(event_ids) == len(set(event_ids)), "duplicate delivered action after restart" + equal = expected_actions == actual_actions + (self.directory / (mode + "-action-comparison.json")).write_text(json.dumps( + {"equal": equal, "batch": expected_actions, "runner": actual_actions}, indent=2) + "\n") + self.receipt(f"{'PASS' if equal else 'FAIL'} {mode} actions equal batch via run_backtest_full batch={len(expected_actions)} runner={len(actual_actions)}") + if mode == "ticks": + name = "ticks-replay" + completed = subprocess.run(self.runner_command(mode, name, self.directory / "ticks-replay.sqlite3", + str(self.directory / "ticks-feed.jsonl")), capture_output=True, text=True, timeout=120) + (self.directory / "ticks-replay.stdout").write_text(completed.stdout) + (self.directory / "ticks-replay.stderr").write_text(completed.stderr) + assert completed.returncode == 0, "tick replay runner failed" + assert actual_actions == [action_key(record) for record in self.receiver.payloads.get(name, [])] + self.receipt(f"PASS ticks same-print runner replay actions equal actions={len(actual_actions)}") + assert equal or mode == "ticks", "confirmed-bar batch action parity failed" + return {"duration_seconds": duration, "messages": cursor, "minutes": len(normalized), + "actions": len(actual_actions), "batch_actions_equal": equal} + + def live(self, mode, minutes): + processes = None + try: + started = time.monotonic() + processes = self.generation(mode, 0, 0) + self.receipt(f"START {mode} public BTCUSDT cut={self.cut} minimum_minutes={minutes}") + time.sleep(self.options.restart_seconds) + assert processes[0].poll() is None and processes[1].poll() is None, "process stopped before restart" + cursor = self.finish(mode, 0, processes, crash=True) + processes = self.generation(mode, 1, cursor) + self.receipt(f"RESTART {mode} --output-from={cursor} --from-input={cursor}") + while time.monotonic() - started < minutes * 60: + assert processes[0].poll() is None and processes[1].poll() is None, "live process stopped early" + time.sleep(5) + cursor = self.finish(mode, 1, processes) + processes = None + duration = time.monotonic() - started + self.results[mode] = self.validate(mode, duration, cursor) + except Exception: + if processes: + for process in processes[:2]: + if process.poll() is None: + process.terminate() + for process in processes[:2]: + try: + process.wait(timeout=20) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + failure = traceback.format_exc() + (self.directory / (mode + "-failure.txt")).write_text(failure) + self.receipt(f"FAIL {mode} qualification; see {mode}-failure.txt") + self.results[mode] = {"error": failure} + + def run(self): + self.warmup() + threads = [threading.Thread(target=self.live, args=("bars", self.options.bar_minutes)), + threading.Thread(target=self.live, args=("ticks", self.options.tick_minutes))] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.receiver.shutdown() + (self.directory / "summary.json").write_text(json.dumps(self.results, indent=2) + "\n") + failed = any("error" in result for result in self.results.values()) + if not self.options.allow_tick_ohlc_difference: + failed |= any(result.get("batch_actions_equal") is False for result in self.results.values()) + return 1 if failed else 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + for argument in ("feed", "runner", "strategy", "observed-strategy", "batch-probe", "directory"): + parser.add_argument("--" + argument, required=True) + parser.add_argument("--bar-minutes", type=int, default=46) + parser.add_argument("--tick-minutes", type=int, default=21) + parser.add_argument("--restart-seconds", type=int, default=180) + parser.add_argument("--allow-tick-ohlc-difference", action="store_true", + help="explicitly use same-print tick replay as the gate; still record failed OHLC batch parity") + raise SystemExit(Soak(parser.parse_args()).run()) diff --git a/native/feed/tests/unit.cpp b/native/feed/tests/unit.cpp new file mode 100644 index 0000000..d132315 --- /dev/null +++ b/native/feed/tests/unit.cpp @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "binance.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge::feed; +namespace { +struct Temporary { + std::string path; + Temporary() { + char pattern[] = "/tmp/pineforge-feed-test-XXXXXX"; + const auto* result = ::mkdtemp(pattern); + assert(result); + path = result; + } + ~Temporary() { std::filesystem::remove_all(path); } +}; +template void expect(int code, Function operation) { + bool caught = false; + try { operation(); } + catch (const Error& failure) { assert(failure.code == code); caught = true; } + assert(caught); +} +void passed(const char* name) { std::cout << "PASS " << name << '\n'; } +Config config(const Temporary& temporary, const std::string& mode) { + Config value; + value.symbol = "TESTUSDT"; + value.mode = mode; + value.start = 120000; + value.state_dir = temporary.path; + return value; +} +struct SyntheticVenue final : Venue { + std::map trades{ + {99, {99, 119999, "10.10000000", "0.10000000"}}, + {100, {100, 120001, "10.10000000", "0.10000000"}}, + {101, {101, 120050, "11.20000000", "0.20000000"}}, + {102, {102, 179999, "9.90000000", "0.30000000"}}, + {103, {103, 180001, "12.00000000", "0.40000000"}}, + {104, {104, 180050, "11.00000000", "0.10000000"}}, + {105, {105, 240001, "12.00000000", "0.10000000"}} + }; + std::map bars{ + {120000, {{120000, "10.10000000", "11.20000000", "9.90000000", "9.90000000", "0.60000000"}, 100, 102, 3, true}}, + {180000, {{180000, "12.00000000", "12.00000000", "11.00000000", "11.00000000", "0.50000000"}, 103, 104, 2, true}}, + {240000, {{240000, "12.00000000", "12.00000000", "12.00000000", "12.00000000", "0.10000000"}, 105, 105, 1, true}} + }; + std::string streams() const override { return "test@trade/test@kline_1m"; } + VenueEvent decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } + std::vector history(std::uint64_t from, std::size_t limit) override { + std::vector result; + for (std::size_t offset = 0; offset < limit; ++offset) { + const auto found = trades.find(from + offset); + if (found == trades.end()) break; + result.push_back(found->second); + } + if (result.empty()) throw Error(20, "synthetic exhausted history"); + return result; + } + std::uint64_t first_trade_id(std::int64_t minute) override { return static_cast(bars.at(minute).first); } + std::vector klines(std::int64_t start, std::int64_t end) override { + std::vector result; + for (auto minute = start; minute < end; minute += 60000) { + if (!bars.count(minute)) throw Error(20, "synthetic missing minute"); + result.push_back(bars.at(minute)); + } + return result; + } + VenueEvent tick(std::uint64_t id) const { return {VenueEvent::Kind::Trade, trades.at(id), {}}; } + VenueEvent close(std::int64_t minute) const { return {VenueEvent::Kind::Kline, {}, bars.at(minute)}; } +}; +} + +int main() { + assert(Decimal("0.10000000").add(Decimal("0.20000000")) == Decimal("0.3")); + assert(Decimal("0").compare(Decimal("0.00000001")) < 0); + assert(Decimal("9007199254740993.00000001").compare(Decimal("9007199254740993.00000000")) > 0); + assert(Decimal("9999999999999999.99").add(Decimal("0.01")) == Decimal("10000000000000000.00")); + for (const auto* invalid : {"", "-1", "+1", "01", "1.", ".1", "NaN", "1e3", "1.2.3"}) expect(23, [&] { Decimal value(invalid); (void)value; }); + expect(23, [] { Decimal value("0." + std::string(33, '1')); (void)value; }); + Trade exact{9007199254740993ULL, 120001, "123.45000000", "0.00000001"}; + assert(normalized_trade(parse_json(exact.wire())) == exact); + expect(23, [] { Trade{1, 1, "1", "0"}.validate(); }); + passed("decimal_tokens_and_uint64_identity"); + + const auto raw = parse_json(R"({"e":"trade","E":999999,"s":"TESTUSDT","t":100,"p":"10.10000000","q":"0.10000000","T":120001,"m":false})"); + assert(binance_trade(raw, true).ts == 120001); + const auto source = parse_json(R"({"t":120000,"T":179999,"s":"TESTUSDT","i":"1m","f":100,"L":102,"o":"10.10000000","h":"11.20000000","l":"9.90000000","c":"9.90000000","v":"0.60000000","n":3,"x":false})"); + assert(!binance_kline(source, true).confirmed); + auto bad_source = source; + bad_source.members["T"] = Json::number("180000"); + expect(23, [&] { binance_kline(bad_source, true); }); + bad_source = source; + bad_source.members["x"] = Json::string("true"); + expect(23, [&] { binance_kline(bad_source, true); }); + passed("binance_matched_time_confirmation_and_inclusive_close"); + + { + Temporary temporary; + SyntheticVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.connected(); + session.ingest(venue.tick(102)); + assert(output.size() == 3); + session.ingest(venue.tick(100)); + assert(output.size() == 3); + session.ingest(venue.tick(103)); + assert(output.size() == 3); + auto unconfirmed = venue.close(120000); + unconfirmed.kline.confirmed = false; + session.ingest(unconfirmed); + assert(output.size() == 3); + session.ingest(venue.close(120000)); + assert(output.size() == 5); + assert(parse_json(output[3]).at("type").text() == "time"); + assert(parse_json(output[4]).at("seq").integer() == 103); + session.ingest(venue.close(120000)); + auto conflict = venue.tick(100); + conflict.trade.qty = "0.20000000"; + expect(21, [&] { session.ingest(conflict); }); + passed("raw_gap_healing_reordering_identical_and_conflicting_duplicates"); + passed("time_requires_full_ids_count_exact_ohlcv_and_precedes_next_minute"); + } + { + Temporary temporary; + SyntheticVenue venue; + State state(config(temporary, "ticks")); + FeedSession session(state, venue, [](const auto&) {}); + auto revised = venue.close(120000); + revised.kline.bar.volume = "0.61000000"; + expect(21, [&] { session.ingest(revised); }); + assert(state.cursor().seq == 102 && state.cursor().cut == 120000); + passed("exact_ohlcv_mismatch_stops_without_time"); + } + { + Temporary temporary; + SyntheticVenue venue; + venue.trades.erase(101); + State state(config(temporary, "ticks")); + FeedSession session(state, venue, [](const auto&) {}); + expect(20, [&] { session.ingest(venue.tick(102)); }); + assert(state.cursor().seq == 100); + passed("unhealable_raw_hole_retains_verified_cursor"); + } + { + Temporary temporary; + SyntheticVenue venue; + State state(config(temporary, "bars")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + auto forming = venue.close(240000); + forming.kline.confirmed = false; + session.ingest(forming); + assert(output.empty()); + session.ingest(venue.close(240000)); + assert(output.size() == 3); + assert(normalized_bar(parse_json(output.front())).ts == 120000); + session.ingest(venue.close(180000)); + assert(output.size() == 3); + auto revised = venue.close(180000); + revised.kline.bar.volume = "0.60000000"; + expect(21, [&] { session.ingest(revised); }); + venue.bars.at(240000).bar.volume = "0.20000000"; + expect(21, [&] { session.connected(); }); + passed("confirmed_bars_watermark_healing_duplicate_and_revised_overlap"); + } + { + Temporary temporary; + SyntheticVenue venue; + auto options = config(temporary, "ticks"); + std::string epoch; + { + State state(options); + FeedSession session(state, venue, [](const auto&) {}); + session.ingest(venue.tick(100)); + session.ingest(venue.tick(101)); + epoch = state.cursor().epoch; + auto second = options; + second.resume = true; + expect(22, [&] { State locked(second); }); + } + { std::ofstream tail(temporary.path + "/events.jsonl", std::ios::app); tail << "uncommitted-torn-tail"; } + options.resume = true; + options.start = -1; + options.output_from = 2; + { + State state(options); + assert(state.cursor().epoch == epoch); + assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == state.cursor().log_bytes); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.connected(); + assert(output.empty()); + session.ingest(venue.close(120000)); + assert(output.size() == 2); + assert(parse_json(output.front()).at("seq").integer() == 102); + assert(parse_json(output.back()).at("type").text() == "time"); + } + { + std::fstream corrupt(temporary.path + "/events.jsonl", std::ios::in | std::ios::out); + corrupt.seekp(0); + corrupt << 'X'; + } + expect(21, [&] { State changed(options); }); + passed("atomic_cursor_lock_torn_tail_verified_restart_and_changed_prefix"); + } + { + Temporary temporary; + SyntheticVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.close(180000)); + assert(state.cursor().cut == 240000 && state.cursor().seq == 104); + assert(output.size() == 7); + assert(parse_json(output[3]).at("type").text() == "time"); + passed("historical_ticks_require_closed_watermark_next_id_fence_and_rest_reconciliation"); + } + { + Temporary temporary; + auto options = config(temporary, "bars"); + options.max_log_bytes = 4; + State state(options); + SyntheticVenue venue; + FeedSession session(state, venue, [](const auto&) {}); + expect(22, [&] { session.ingest(venue.close(120000)); }); + assert(state.cursor().message_index == 0); + passed("bounded_replay_budget_stops_before_commit"); + } + { + Temporary temporary; + State state(config(temporary, "ticks")); + SyntheticVenue venue; + FeedSession session(state, venue, [](const auto&) {}); + session.ingest(venue.close(120000)); + auto revised = venue.close(120000); + ++revised.kline.first; + ++revised.kline.last; + expect(21, [&] { session.ingest(revised); }); + ++venue.bars.at(120000).count; + expect(21, [&] { session.connected(); }); + assert(state.cursor().message_index == 4); + passed("closed_tick_fences_and_rest_counts_are_immutable"); + } + { + Temporary temporary; + auto options = config(temporary, "bars"); + { + State state(options); + SyntheticVenue venue; + FeedSession session(state, venue, [](const auto&) {}); + std::filesystem::create_directory(temporary.path + "/cursor.json.tmp"); + expect(22, [&] { session.ingest(venue.close(120000)); }); + assert(state.cursor().message_index == 0); + } + std::filesystem::remove(temporary.path + "/cursor.json.tmp"); + options.resume = true; + State recovered(options); + assert(recovered.cursor().message_index == 0); + assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == 0); + passed("failed_cursor_write_rolls_back_uncommitted_message_on_resume"); + } + { + Temporary temporary; + auto options = config(temporary, "bars"); + options.max_replay_seconds = 1; + State state(options); + SyntheticVenue venue; + FeedSession session(state, venue, [](const auto&) {}); + session.ingest(venue.close(120000)); + session.ingest(venue.close(180000)); + expect(22, [&] { state.visit(0, [](const auto&) { std::this_thread::sleep_for(std::chrono::milliseconds(1100)); }); }); + passed("replay_time_budget_stops_without_mutating_cursor"); + } + assert(timestamp("1970-01-01T00:02:00Z") == 120000); + expect(23, [] { timestamp("2026-02-30T00:00:00Z"); }); + passed("strict_utc_start_and_exclusive_cut"); + std::cout << "PASS feed unit suite\n"; +} diff --git a/native/feed/vendor/json.hpp b/native/feed/vendor/json.hpp new file mode 100644 index 0000000..4de7f0f --- /dev/null +++ b/native/feed/vendor/json.hpp @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::live { + +// Bounded JSON for the native runner's configuration and JSONL wire format. +// Number tokens remain decimal strings so event identities never round uint64. +struct Json { + enum class Kind { Null, Bool, Number, String, Array, Object }; + Kind kind = Kind::Null; + std::string value; + std::vector items; + std::map members; + + static Json string(std::string v) { + Json j; + j.kind = Kind::String; + j.value = std::move(v); + return j; + } + static Json number(std::string v) { + Json j; + j.kind = Kind::Number; + j.value = std::move(v); + return j; + } + static Json boolean(bool v) { + Json j; + j.kind = Kind::Bool; + j.value = v ? "true" : "false"; + return j; + } + static Json object(std::map v) { + Json j; + j.kind = Kind::Object; + j.members = std::move(v); + return j; + } + const Json &at(const std::string &key) const { + if (kind != Kind::Object || members.count(key) != 1) + throw std::runtime_error("missing field: " + key); + return members.at(key); + } + const Json *find(const std::string &key) const { + if (kind != Kind::Object) + throw std::runtime_error("expected object"); + auto i = members.find(key); + return i == members.end() ? nullptr : &i->second; + } + std::string text() const { + if (kind != Kind::String) + throw std::runtime_error("expected string"); + return value; + } + double real() const { + if (kind != Kind::Number) + throw std::runtime_error("expected number"); + std::size_t n = 0; + double d = std::stod(value, &n); + if (n != value.size() || !std::isfinite(d)) + throw std::runtime_error("nonfinite number"); + return d; + } + template T integer() const { + if (kind != Kind::Number) + throw std::runtime_error("expected integer"); + T result{}; + auto r = std::from_chars(value.data(), value.data() + value.size(), result); + if (r.ec != std::errc{} || r.ptr != value.data() + value.size()) + throw std::runtime_error("invalid or overflowing integer"); + return result; + } + static std::string quote(std::string_view s) { + static constexpr char hex[] = "0123456789abcdef"; + std::string out = "\""; + for (unsigned char c : s) { + if (c == '"' || c == '\\') { + out += '\\'; + out += static_cast(c); + } else if (c < 32) { + out += "\\u00"; + out += hex[c >> 4]; + out += hex[c & 15]; + } else + out += static_cast(c); + } + return out + '"'; + } + std::string dump() const { + if (kind == Kind::Null) + return "null"; + if (kind == Kind::String) + return quote(value); + if (kind == Kind::Bool || kind == Kind::Number) + return value; + std::string out = kind == Kind::Array ? "[" : "{"; + bool first = true; + if (kind == Kind::Array) + for (const auto &i : items) { + if (!first) + out += ','; + first = false; + out += i.dump(); + } + else + for (const auto &[key, i] : members) { + if (!first) + out += ','; + first = false; + out += quote(key) + ':' + i.dump(); + } + return out + (kind == Kind::Array ? ']' : '}'); + } +}; + +class JsonParser { + std::string_view s_; + std::size_t p_ = 0; + [[noreturn]] void fail() const { + throw std::runtime_error("invalid JSON at byte " + std::to_string(p_)); + } + void ws() { + while (p_ < s_.size() && + (s_[p_] == ' ' || s_[p_] == '\t' || s_[p_] == '\r' || s_[p_] == '\n')) + ++p_; + } + bool take(char c) { + if (p_ < s_.size() && s_[p_] == c) { + ++p_; + return true; + } + return false; + } + unsigned hex4() { + unsigned value = 0; + for (int i = 0; i < 4; ++i) { + if (p_ == s_.size()) + fail(); + char c = s_[p_++]; + value <<= 4; + if (c >= '0' && c <= '9') + value += c - '0'; + else if (c >= 'a' && c <= 'f') + value += c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + value += c - 'A' + 10; + else + fail(); + } + return value; + } + static void utf8(std::string &out, unsigned c) { + if (c < 0x80) + out += static_cast(c); + else if (c < 0x800) { + out += static_cast(0xc0 | (c >> 6)); + out += static_cast(0x80 | (c & 63)); + } else if (c < 0x10000) { + out += static_cast(0xe0 | (c >> 12)); + out += static_cast(0x80 | ((c >> 6) & 63)); + out += static_cast(0x80 | (c & 63)); + } else { + out += static_cast(0xf0 | (c >> 18)); + out += static_cast(0x80 | ((c >> 12) & 63)); + out += static_cast(0x80 | ((c >> 6) & 63)); + out += static_cast(0x80 | (c & 63)); + } + } + std::string str() { + if (!take('"')) + fail(); + std::string out; + while (p_ < s_.size()) { + unsigned char c = s_[p_++]; + if (c == '"') + return out; + if (c < 32) + fail(); + if (c != '\\') { + if (c < 128) { + out += static_cast(c); + continue; + } + unsigned n = c >= 0xf0 ? 3 : c >= 0xe0 ? 2 : c >= 0xc2 ? 1 : 0; + if (!n || c > 0xf4) + fail(); + unsigned point = c & ((1u << (6 - n)) - 1); + for (unsigned i = 0; i < n; ++i) { + if (p_ == s_.size()) + fail(); + unsigned char tail = s_[p_++]; + if ((tail & 0xc0) != 0x80) + fail(); + point = (point << 6) | (tail & 63); + } + if (point < (n == 1 ? 0x80u + : n == 2 ? 0x800u + : 0x10000u) || + point > 0x10ffff || (point >= 0xd800 && point <= 0xdfff)) + fail(); + utf8(out, point); + continue; + } + if (p_ == s_.size()) + fail(); + char e = s_[p_++]; + switch (e) { + case '"': + case '\\': + case '/': + out += e; + break; + case 'b': + out += '\b'; + break; + case 'f': + out += '\f'; + break; + case 'n': + out += '\n'; + break; + case 'r': + out += '\r'; + break; + case 't': + out += '\t'; + break; + case 'u': { + unsigned point = hex4(); + if (point >= 0xd800 && point <= 0xdbff) { + if (!take('\\') || !take('u')) + fail(); + unsigned low = hex4(); + if (low < 0xdc00 || low > 0xdfff) + fail(); + point = 0x10000 + ((point - 0xd800) << 10) + (low - 0xdc00); + } else if (point >= 0xdc00 && point <= 0xdfff) + fail(); + utf8(out, point); + break; + } + default: + fail(); + } + } + fail(); + } + Json parse(unsigned depth) { + if (depth >= 32) + fail(); + ws(); + if (p_ == s_.size()) + fail(); + if (s_[p_] == '"') + return Json::string(str()); + if (take('{')) { + Json j; + j.kind = Json::Kind::Object; + ws(); + if (take('}')) + return j; + do { + ws(); + auto key = str(); + ws(); + if (!take(':')) + fail(); + auto v = parse(depth + 1); + if (!j.members.emplace(std::move(key), std::move(v)).second) + fail(); + ws(); + if (take('}')) + return j; + } while (take(',')); + fail(); + } + if (take('[')) { + Json j; + j.kind = Json::Kind::Array; + ws(); + if (take(']')) + return j; + do { + j.items.push_back(parse(depth + 1)); + ws(); + if (take(']')) + return j; + } while (take(',')); + fail(); + } + for (auto lit : {"null", "true", "false"}) { + std::string_view v = lit; + if (s_.substr(p_, v.size()) == v) { + p_ += v.size(); + return v == "null" ? Json{} : Json::boolean(v == "true"); + } + } + std::size_t begin = p_; + take('-'); + if (take('0')) { + } else { + if (p_ == s_.size() || s_[p_] < '1' || s_[p_] > '9') + fail(); + while (p_ < s_.size() && s_[p_] >= '0' && s_[p_] <= '9') + ++p_; + } + if (take('.')) { + auto old = p_; + while (p_ < s_.size() && s_[p_] >= '0' && s_[p_] <= '9') + ++p_; + if (p_ == old) + fail(); + } + if (take('e') || take('E')) { + if (!take('+')) + take('-'); + auto old = p_; + while (p_ < s_.size() && s_[p_] >= '0' && s_[p_] <= '9') + ++p_; + if (p_ == old) + fail(); + } + return Json::number(std::string(s_.substr(begin, p_ - begin))); + } + + public: + explicit JsonParser(std::string_view s) : s_(s) { + if (s.size() > 1024 * 1024) + throw std::runtime_error("JSON frame exceeds 1 MiB"); + } + Json run() { + auto j = parse(0); + ws(); + if (p_ != s_.size()) + fail(); + return j; + } +}; +inline Json parse_json(std::string_view s) { return JsonParser(s).run(); } +inline void only_fields(const Json &j, std::initializer_list allowed) { + if (j.kind != Json::Kind::Object) + throw std::runtime_error("expected object"); + for (const auto &[key, value] : j.members) { + bool found = false; + for (const auto &a : allowed) + found = found || a == key; + if (!found) + throw std::runtime_error("unknown field: " + key); + } +} +} // namespace pineforge::live From 5abad8df6d75634866f81e3fdf34f20e3fdc263f Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 02:11:38 +0800 Subject: [PATCH 2/7] Keep the native feed out of the Python sdist Hatchling's sdist took every tracked file, so the next Python release would have shipped native/ inside the Python package. Exclude it from the sdist and fail CI if a built sdist lists anything under native/. The wheel is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 7 +++++++ pyproject.toml | 3 +++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d9d1c1..a252b49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,13 @@ jobs: - run: pytest - if: matrix.python-version == '3.13' run: python -m build + - if: matrix.python-version == '3.13' + name: The sdist does not ship the native feed + run: | + if tar -tzf dist/*.tar.gz | grep -E '^[^/]+/native/'; then + echo "native/ must stay out of the Python sdist" >&2 + exit 1 + fi - if: matrix.python-version == '3.13' run: python -m twine check --strict dist/* diff --git a/pyproject.toml b/pyproject.toml index a72b597..bd27a33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ Source = "https://github.com/pineforge-4pass/pineforge-data" [tool.hatch.build.targets.wheel] packages = ["src/pineforge_data"] +[tool.hatch.build.targets.sdist] +exclude = ["/native"] + [tool.pytest.ini_options] addopts = "-q" testpaths = ["tests"] From ac28bdb37ac5efe418dbff1230e92e93b7455412 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 02:11:38 +0800 Subject: [PATCH 3/7] Build the native feed on Linux arm64; bound every job Align actions/checkout with the other workflows (v7), add ubuntu-24.04-arm to the Release matrix, install cmake on Linux runners, and give each job a 30-minute timeout. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/native-feed.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/native-feed.yml b/.github/workflows/native-feed.yml index bd93497..1c7aeda 100644 --- a/.github/workflows/native-feed.yml +++ b/.github/workflows/native-feed.yml @@ -16,13 +16,14 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-14] + os: [ubuntu-24.04, ubuntu-24.04-arm, macos-14] runs-on: ${{ matrix.os }} + timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Linux prerequisites if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev + run: sudo apt-get update && sudo apt-get install -y cmake ninja-build libssl-dev - name: macOS prerequisites if: runner.os == 'macOS' run: | @@ -42,8 +43,9 @@ jobs: asan-ubsan: name: ASan + UBSan runs-on: ubuntu-24.04 + timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev clang-18 - run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" - name: Instrumented core and mock venue @@ -61,8 +63,9 @@ jobs: tsan: name: TSan runs-on: ubuntu-24.04 + timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev clang-18 - run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" - name: Threaded receive, healing and termination From 4fc3e2801e70c8cd4647c7e415102e94f0abf205 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 02:11:38 +0800 Subject: [PATCH 4/7] Make the native feed test scripts pass ruff The repository's `ruff check .` covers native/feed/tests. Format both scripts and fix the lint findings: open the harness's log files as descriptors, use itertools.pairwise, and drop an unused import and unused names. No behaviour changes. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/tests/mock_venue.py | 262 +++++++++++++++++++++++------ native/feed/tests/public_e2e.py | 290 +++++++++++++++++++++++++------- 2 files changed, 437 insertions(+), 115 deletions(-) diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py index 08a4060..4f9d249 100644 --- a/native/feed/tests/mock_venue.py +++ b/native/feed/tests/mock_venue.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 """Synthetic public Binance shapes; standard-library HTTP/WS fault injection.""" + import base64 import copy import fcntl @@ -8,18 +9,17 @@ import json import pathlib import signal -import socket import struct import subprocess import sys import tempfile import threading import time -import urllib.parse import unittest +import urllib.parse from types import SimpleNamespace -from public_e2e import Soak +from public_e2e import Soak BINARY = str(pathlib.Path(sys.argv[1]).resolve()) sys.argv = [sys.argv[0]] @@ -41,21 +41,47 @@ def trade(identifier, **changes): matched, price, quantity = TRADES[identifier] - data = {"e": "trade", "E": matched + 999, "s": "TESTUSDT", "t": identifier, - "p": price, "q": quantity, "T": matched, "m": False, "M": True} + data = { + "e": "trade", + "E": matched + 999, + "s": "TESTUSDT", + "t": identifier, + "p": price, + "q": quantity, + "T": matched, + "m": False, + "M": True, + } data.update(changes) return {"stream": "testusdt@trade", "data": data} def candle(minute, confirmed=True, **changes): opening, high, low, close, volume, first, last, count = BARS[minute] - row = {"t": minute, "T": minute + 59999, "s": "TESTUSDT", "i": "1m", "f": first, - "L": last, "o": opening, "h": high, "l": low, "c": close, "v": volume, - "n": count, "x": confirmed, "q": "0.00000000", "V": "0.00000000", - "Q": "0.00000000", "B": "0"} + row = { + "t": minute, + "T": minute + 59999, + "s": "TESTUSDT", + "i": "1m", + "f": first, + "L": last, + "o": opening, + "h": high, + "l": low, + "c": close, + "v": volume, + "n": count, + "x": confirmed, + "q": "0.00000000", + "V": "0.00000000", + "Q": "0.00000000", + "B": "0", + } row.update(changes) - return {"stream": "testusdt@kline_1m", "data": {"e": "kline", "E": minute + 60000, - "s": "TESTUSDT", "k": row}} + return { + "stream": "testusdt@kline_1m", + "data": {"e": "kline", "E": minute + 60000, "s": "TESTUSDT", "k": row}, + } def retirement(): @@ -86,6 +112,7 @@ def exact(size): raise EOFError result.extend(chunk) return bytes(result) + first, second = exact(2) size = second & 127 if size == 126: @@ -136,7 +163,9 @@ def do_GET(self): query = urllib.parse.parse_qs(parsed.query) with self.server.lock: self.server.requests.append(parsed.path) - self.server.auth_headers.extend(name for name in ("Authorization", "X-MBX-APIKEY") if self.headers.get(name)) + self.server.auth_headers.extend( + name for name in ("Authorization", "X-MBX-APIKEY") if self.headers.get(name) + ) if parsed.path == "/stream": self.websocket(query) return @@ -149,9 +178,23 @@ def do_GET(self): self.respond(429, {}, retry=True) return if parsed.path == "/api/v3/exchangeInfo": - rows = {"rateLimits": [{"rateLimitType": "REQUEST_WEIGHT", "interval": "MINUTE", "intervalNum": 1, "limit": 6000}, - {"rateLimitType": "RAW_REQUESTS", "interval": "MINUTE", "intervalNum": 5, "limit": 61000}], - "symbols": [{"symbol": "TESTUSDT", "baseAsset": "TEST", "quoteAsset": "USDT"}]} + rows = { + "rateLimits": [ + { + "rateLimitType": "REQUEST_WEIGHT", + "interval": "MINUTE", + "intervalNum": 1, + "limit": 6000, + }, + { + "rateLimitType": "RAW_REQUESTS", + "interval": "MINUTE", + "intervalNum": 5, + "limit": 61000, + }, + ], + "symbols": [{"symbol": "TESTUSDT", "baseAsset": "TEST", "quoteAsset": "USDT"}], + } if self.server.rate_limits is not None: rows["rateLimits"] = self.server.rate_limits elif parsed.path == "/api/v3/historicalTrades": @@ -161,25 +204,64 @@ def do_GET(self): for identifier in sorted(self.server.trades): if identifier >= start and len(rows) < limit: matched, price, quantity = self.server.trades[identifier] - rows.append({"id": identifier, "price": price, "qty": quantity, - "quoteQty": "0.00000000", "time": matched, - "isBuyerMaker": False, "isBestMatch": True}) + rows.append( + { + "id": identifier, + "price": price, + "qty": quantity, + "quoteQty": "0.00000000", + "time": matched, + "isBuyerMaker": False, + "isBestMatch": True, + } + ) elif parsed.path == "/api/v3/aggTrades": start, end = int(query["startTime"][0]), int(query["endTime"][0]) - matching = [identifier for identifier in sorted(self.server.trades) - if start <= self.server.trades[identifier][0] <= end] - rows = [] if not matching else [{"a": 42, "f": matching[0], "l": matching[0], - "T": self.server.trades[matching[0]][0], - "p": "10.10000000", "q": "0.10000000", - "m": False, "M": True}] + matching = [ + identifier + for identifier in sorted(self.server.trades) + if start <= self.server.trades[identifier][0] <= end + ] + rows = ( + [] + if not matching + else [ + { + "a": 42, + "f": matching[0], + "l": matching[0], + "T": self.server.trades[matching[0]][0], + "p": "10.10000000", + "q": "0.10000000", + "m": False, + "M": True, + } + ] + ) elif parsed.path == "/api/v3/klines": start, end = int(query["startTime"][0]), int(query["endTime"][0]) rows = [] for minute in sorted(self.server.bars): if start <= minute <= end and len(rows) < 1000: - opening, high, low, close, volume, first, last, count = self.server.bars[minute] - rows.append([minute, opening, high, low, close, volume, minute + 59999, - "0.00000000", count, "0.00000000", "0.00000000", "0"]) + opening, high, low, close, volume, _first, _last, count = self.server.bars[ + minute + ] + rows.append( + [ + minute, + opening, + high, + low, + close, + volume, + minute + 59999, + "0.00000000", + count, + "0.00000000", + "0.00000000", + "0", + ] + ) else: self.respond(404, {}) return @@ -200,7 +282,9 @@ def respond(self, status, value, retry=False): def websocket(self, query): assert query["streams"] == ["testusdt@trade/testusdt@kline_1m"] key = self.headers["Sec-WebSocket-Key"] - accept = base64.b64encode(hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest()).decode() + accept = base64.b64encode( + hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest() + ).decode() self.send_response(101) self.send_header("Upgrade", "websocket") self.send_header("Connection", "Upgrade") @@ -243,7 +327,7 @@ def websocket(self, query): while True: try: receive(self.connection) - except socket.timeout: + except TimeoutError: continue except (EOFError, ConnectionResetError): break @@ -254,15 +338,23 @@ class FeedMockTests(unittest.TestCase): def test_public_qualification_requires_explicit_tick_ohlc_exception(self): for allow, expected in ((False, 1), (True, 0)): soak = Soak.__new__(Soak) - soak.options = SimpleNamespace(allow_tick_ohlc_difference=allow, bar_minutes=46, tick_minutes=21) + soak.options = SimpleNamespace( + allow_tick_ohlc_difference=allow, bar_minutes=46, tick_minutes=21 + ) soak.directory = self.directory / str(allow) soak.directory.mkdir() soak.results = {} soak.receiver = SimpleNamespace(shutdown=lambda: None) soak.warmup = lambda: None - soak.live = lambda mode, minutes: soak.results.update({mode: {"batch_actions_equal": mode == "bars"}}) + soak.live = lambda mode, minutes, soak=soak: soak.results.update( + {mode: {"batch_actions_equal": mode == "bars"}} + ) self.assertEqual(soak.run(), expected) - self.assertFalse(json.loads((soak.directory / "summary.json").read_text())["ticks"]["batch_actions_equal"]) + self.assertFalse( + json.loads((soak.directory / "summary.json").read_text())["ticks"][ + "batch_actions_equal" + ] + ) def setUp(self): self.temporary = tempfile.TemporaryDirectory(prefix="pineforge-feed-mock-") @@ -279,15 +371,32 @@ def server(self, scripts): def command(self, server, mode="bars", extra=(), resume=False): port = server.server_address[1] - arguments = [BINARY, "run", "--venue", "binance", "--market", "spot", "--symbol", "TESTUSDT", - "--mode", mode, "--state-dir", str(self.directory / "state"), - "--rest-url", f"http://127.0.0.1:{port}", "--ws-url", f"ws://127.0.0.1:{port}", - "--allow-insecure-http"] + arguments = [ + BINARY, + "run", + "--venue", + "binance", + "--market", + "spot", + "--symbol", + "TESTUSDT", + "--mode", + mode, + "--state-dir", + str(self.directory / "state"), + "--rest-url", + f"http://127.0.0.1:{port}", + "--ws-url", + f"ws://127.0.0.1:{port}", + "--allow-insecure-http", + ] arguments += ["--resume"] if resume else ["--start", "120000"] return arguments + list(extra) def run_feed(self, server, mode="bars", extra=(), expected=0, resume=False): - result = subprocess.run(self.command(server, mode, extra, resume), capture_output=True, text=True, timeout=12) + result = subprocess.run( + self.command(server, mode, extra, resume), capture_output=True, text=True, timeout=12 + ) self.assertEqual(result.returncode, expected, result.stderr) self.assertFalse(server.errors, server.errors) for record in result.stderr.splitlines(): @@ -296,7 +405,10 @@ def run_feed(self, server, mode="bars", extra=(), expected=0, resume=False): cursor = self.directory / "state" / "cursor.json" if cursor.exists(): saved = json.loads(cursor.read_text())["cursor"] - self.assertEqual(saved["message_index"], len((cursor.parent / "events.jsonl").read_text().splitlines())) + self.assertEqual( + saved["message_index"], + len((cursor.parent / "events.jsonl").read_text().splitlines()), + ) return output, result def test_bars_hole_is_healed_behind_closed_watermark(self): @@ -311,10 +423,16 @@ def test_unhealable_bar_hole_stops_20(self): self.assertEqual([event["bar"]["ts_open"] for event in output], [120000]) def test_raw_hole_and_reordering_are_healed_not_resequenced(self): - server = self.server([[trade(102), trade(100), trade(101), trade(103), candle(120000), candle(180000)]]) + server = self.server( + [[trade(102), trade(100), trade(101), trade(103), candle(120000), candle(180000)]] + ) output, result = self.run_feed(server, "ticks", ["--max-messages", "7"]) - self.assertEqual([event["seq"] for event in output if event["type"] == "tick"], [100, 101, 102, 103, 104]) - self.assertEqual([event["ts"] for event in output if event["type"] == "time"], [180000, 240000]) + self.assertEqual( + [event["seq"] for event in output if event["type"] == "tick"], [100, 101, 102, 103, 104] + ) + self.assertEqual( + [event["ts"] for event in output if event["type"] == "time"], [180000, 240000] + ) self.assertIn('"price":10.10000000', result.stdout) self.assertEqual(output[0]["ts"], 120001) @@ -355,7 +473,7 @@ def test_forming_candles_never_advance_time(self): self.assertEqual([event["type"] for event in output], ["tick", "tick", "tick", "time"]) def test_ping_payload_is_echoed_and_fragmented_text_works(self): - server = self.server([[('ping', b"synthetic-ping"), ("fragment", candle(120000))]]) + server = self.server([[("ping", b"synthetic-ping"), ("fragment", candle(120000))]]) output, _ = self.run_feed(server, extra=["--max-messages", "1"]) self.assertEqual(len(output), 1) self.assertEqual(server.pongs, [b"synthetic-ping"]) @@ -367,7 +485,9 @@ def test_server_shutdown_reconnects_with_exact_overlap(self): self.assertGreaterEqual(server.connections, 2) def test_reconnect_changed_rest_overlap_stops_21(self): - server = self.server([[candle(120000), ("revise_rest", 120000), retirement()], [candle(180000)]]) + server = self.server( + [[candle(120000), ("revise_rest", 120000), retirement()], [candle(180000)]] + ) output, _ = self.run_feed(server, expected=21) self.assertEqual(len(output), 1) @@ -384,6 +504,7 @@ def test_protocol_invalid_symbol_and_binary_stop_23(self): state = self.directory / "state" if state.exists(): import shutil + shutil.rmtree(state) self.run_feed(server, "ticks", expected=23) @@ -392,7 +513,12 @@ def test_rest_access_failure_stops_23_no_key_sent(self): server.fail_status = 451 output, _ = self.run_feed(server, expected=23) self.assertEqual(output, []) - self.assertTrue(all(path in {"/stream", "/api/v3/klines", "/api/v3/exchangeInfo"} for path in server.requests)) + self.assertTrue( + all( + path in {"/stream", "/api/v3/klines", "/api/v3/exchangeInfo"} + for path in server.requests + ) + ) self.assertEqual(server.auth_headers, []) def test_missing_rate_limit_metadata_is_refused(self): @@ -424,7 +550,9 @@ def test_sigterm_drain_is_bounded_with_unread_stdout(self): message["data"]["k"]["T"] = minute + 59999 messages.append(message) server = self.server([messages]) - process = subprocess.Popen(self.command(server), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process = subprocess.Popen( + self.command(server), stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) try: if hasattr(fcntl, "F_SETPIPE_SZ"): fcntl.fcntl(process.stdout.fileno(), fcntl.F_SETPIPE_SZ, 4096) @@ -438,12 +566,18 @@ def test_sigterm_drain_is_bounded_with_unread_stdout(self): if current_index != previous_index: previous_index = current_index stable_since = time.monotonic() - elif 30 <= current_index < len(messages) and time.monotonic() - stable_since >= 0.4: + elif ( + 30 <= current_index < len(messages) + and time.monotonic() - stable_since >= 0.4 + ): break self.assertIsNone(process.poll()) time.sleep(0.02) else: - self.fail(f"unread stdout did not stall before source exhaustion; committed={previous_index}") + self.fail( + "unread stdout did not stall before source exhaustion; " + f"committed={previous_index}" + ) self.assertIsNone(process.poll()) started = time.monotonic() process.send_signal(signal.SIGTERM) @@ -469,6 +603,7 @@ def test_replay_and_queue_budgets_stop_22(self): output, _ = self.run_feed(server, extra=["--max-log-bytes", "4"], expected=22) self.assertEqual(output, []) import shutil + shutil.rmtree(self.directory / "state") server = self.server([[trade(100)]]) output, _ = self.run_feed(server, "ticks", ["--max-queue-bytes", "64"], expected=22) @@ -476,7 +611,9 @@ def test_replay_and_queue_budgets_stop_22(self): def test_sigterm_then_resume_tail_has_no_gap_or_duplicate(self): server = self.server([[candle(120000)], [candle(120000), candle(240000)]]) - process = subprocess.Popen(self.command(server), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + process = subprocess.Popen( + self.command(server), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) cursor = self.directory / "state" / "cursor.json" deadline = time.monotonic() + 8 while time.monotonic() < deadline: @@ -491,7 +628,9 @@ def test_sigterm_then_resume_tail_has_no_gap_or_duplicate(self): stdout, stderr = process.communicate(timeout=5) self.assertEqual(process.returncode, 0, stderr) self.assertEqual(len(stdout.splitlines()), 1) - output, _ = self.run_feed(server, extra=["--output-from", "1", "--max-messages", "2"], resume=True) + output, _ = self.run_feed( + server, extra=["--output-from", "1", "--max-messages", "2"], resume=True + ) self.assertEqual([event["bar"]["ts_open"] for event in output], [180000, 240000]) def test_changed_committed_journal_is_refused(self): @@ -506,10 +645,27 @@ def test_warmup_exclusive_cut_manifest_and_tokens(self): server = self.server([[candle(240000)]]) port = server.server_address[1] output = self.directory / "warmup.csv" - command = [BINARY, "warmup", "--venue", "binance", "--market", "spot", "--symbol", "TESTUSDT", - "--start", "120000", "--end", "240000", "--output", str(output), - "--rest-url", f"http://127.0.0.1:{port}", "--ws-url", f"ws://127.0.0.1:{port}", - "--allow-insecure-http"] + command = [ + BINARY, + "warmup", + "--venue", + "binance", + "--market", + "spot", + "--symbol", + "TESTUSDT", + "--start", + "120000", + "--end", + "240000", + "--output", + str(output), + "--rest-url", + f"http://127.0.0.1:{port}", + "--ws-url", + f"ws://127.0.0.1:{port}", + "--allow-insecure-http", + ] result = subprocess.run(command, capture_output=True, text=True, timeout=8) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "") diff --git a/native/feed/tests/public_e2e.py b/native/feed/tests/public_e2e.py index 866dc96..cc8ce9e 100644 --- a/native/feed/tests/public_e2e.py +++ b/native/feed/tests/public_e2e.py @@ -1,10 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 """Explicit public-data soak; captures stay in the operator-selected directory.""" + import argparse -import csv import hashlib import http.server +import itertools import json +import os import pathlib import signal import struct @@ -19,14 +21,26 @@ def rows(path, exact=False): with open(path) as source: - return [json.loads(line, parse_float=Decimal if exact else float) for line in source if line.strip()] + return [ + json.loads(line, parse_float=Decimal if exact else float) + for line in source + if line.strip() + ] def action_key(record): order = record["order"] - return (record["timestamp"], record["bar_index"], order["id"], order["action"], order["leg"], - struct.pack("!d", float(order["contracts"])).hex(), struct.pack("!d", float(order["price"])).hex(), - order["reduce_only"], order["entry_incarnation"]) + return ( + record["timestamp"], + record["bar_index"], + order["id"], + order["action"], + order["leg"], + struct.pack("!d", float(order["contracts"])).hex(), + struct.pack("!d", float(order["price"])).hex(), + order["reduce_only"], + order["entry_incarnation"], + ) class Receiver(http.server.ThreadingHTTPServer): @@ -76,36 +90,93 @@ def receipt(self, message): output.write(message + "\n") def runner_command(self, mode, name, ledger, source, cursor=0): - command = [self.options.runner, "run", "--strategy", self.options.strategy, - "--warmup", str(self.directory / "warmup.csv"), "--feed", source, - "--mode", mode, "--script-tf", "1", "--ledger", str(ledger), - "--symbol", "BINANCE:BTCUSDT", "--name", "public-rsi-qualification", - "--webhook-url", f"http://127.0.0.1:{self.receiver.server_port}/actions/{name}", - "--allow-insecure-http"] + command = [ + self.options.runner, + "run", + "--strategy", + self.options.strategy, + "--warmup", + str(self.directory / "warmup.csv"), + "--feed", + source, + "--mode", + mode, + "--script-tf", + "1", + "--ledger", + str(ledger), + "--symbol", + "BINANCE:BTCUSDT", + "--name", + "public-rsi-qualification", + "--webhook-url", + f"http://127.0.0.1:{self.receiver.server_port}/actions/{name}", + "--allow-insecure-http", + ] if cursor: command += ["--from-input", str(cursor)] return command def warmup(self): - command = [self.options.feed, "warmup", "--venue", "binance", "--market", "spot", - "--symbol", "BTCUSDT", "--start", str(self.cut - 200 * 60000), - "--end", str(self.cut), "--output", str(self.directory / "warmup.csv")] + command = [ + self.options.feed, + "warmup", + "--venue", + "binance", + "--market", + "spot", + "--symbol", + "BTCUSDT", + "--start", + str(self.cut - 200 * 60000), + "--end", + str(self.cut), + "--output", + str(self.directory / "warmup.csv"), + ] with open(self.directory / "warmup.stderr", "w") as error: subprocess.run(command, stderr=error, check=True, timeout=150) manifest = json.loads((self.directory / "warmup.csv.manifest.json").read_text()) assert manifest["end_exclusive"] == self.cut - assert manifest["sha256"] == hashlib.sha256((self.directory / "warmup.csv").read_bytes()).hexdigest() + assert ( + manifest["sha256"] + == hashlib.sha256((self.directory / "warmup.csv").read_bytes()).hexdigest() + ) self.receipt("PASS warmup exclusive cut and SHA-256") def generation(self, mode, generation, cursor): - command = [self.options.feed, "run", "--venue", "binance", "--market", "spot", "--symbol", "BTCUSDT", - "--mode", mode, "--state-dir", str(self.directory / (mode + "-state"))] - command += ["--resume", "--output-from", str(cursor)] if generation else ["--start", str(self.cut)] - feed_error = open(self.directory / f"{mode}-feed-{generation}.stderr", "w") - runner_error = open(self.directory / f"{mode}-runner-{generation}.stderr", "w") - runner_output = open(self.directory / f"{mode}-runner-{generation}.stdout", "w") - runner = subprocess.Popen(self.runner_command(mode, mode, self.directory / (mode + ".sqlite3"), "-", cursor), - stdin=subprocess.PIPE, stdout=runner_output, stderr=runner_error) + command = [ + self.options.feed, + "run", + "--venue", + "binance", + "--market", + "spot", + "--symbol", + "BTCUSDT", + "--mode", + mode, + "--state-dir", + str(self.directory / (mode + "-state")), + ] + command += ( + ["--resume", "--output-from", str(cursor)] if generation else ["--start", str(self.cut)] + ) + files = [ + os.open(self.directory / name, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644) + for name in ( + f"{mode}-feed-{generation}.stderr", + f"{mode}-runner-{generation}.stderr", + f"{mode}-runner-{generation}.stdout", + ) + ] + feed_error, runner_error, runner_output = files + runner = subprocess.Popen( + self.runner_command(mode, mode, self.directory / (mode + ".sqlite3"), "-", cursor), + stdin=subprocess.PIPE, + stdout=runner_output, + stderr=runner_error, + ) feed = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=feed_error) failures = [] @@ -125,7 +196,7 @@ def pump(): thread = threading.Thread(target=pump, daemon=True) thread.start() - return feed, runner, thread, failures, [feed_error, runner_error, runner_output] + return feed, runner, thread, failures, files def finish(self, mode, generation, processes, crash=False): feed, runner, thread, failures, files = processes @@ -135,8 +206,8 @@ def finish(self, mode, generation, processes, crash=False): thread.join(timeout=30) assert not thread.is_alive(), "pipe drain exceeded deadline" runner_code = runner.wait(timeout=40) - for output in files: - output.close() + for descriptor in files: + os.close(descriptor) assert code == (-signal.SIGKILL if crash else 0), f"feed exit={code}" assert runner_code == 0, f"runner exit={runner_code}" assert not failures, failures @@ -152,8 +223,16 @@ def fetch(self, path, query, weight=2): def candles(self, start, end): result = [] while start < end: - page = self.fetch("/api/v3/klines", {"symbol": "BTCUSDT", "interval": "1m", "startTime": start, - "endTime": end - 1, "limit": 1000}) + page = self.fetch( + "/api/v3/klines", + { + "symbol": "BTCUSDT", + "interval": "1m", + "startTime": start, + "endTime": end - 1, + "limit": 1000, + }, + ) assert page and page[0][0] == start, "REST minute missing" result += page start = page[-1][0] + 60000 @@ -164,7 +243,9 @@ def validate(self, mode, duration, cursor): getcontext().prec = 160 tape = rows(self.directory / (mode + "-feed.jsonl"), True) with open(self.directory / (mode + "-feed.jsonl")) as source: - tokens = [json.loads(line, parse_float=str, parse_int=str) for line in source if line.strip()] + tokens = [ + json.loads(line, parse_float=str, parse_int=str) for line in source if line.strip() + ] assert len(tape) == cursor, "capture/runner cursor mismatch" durable = rows(self.directory / (mode + "-state/events.jsonl"), True) assert tape == durable, "restart changed normalized prefix or message boundaries" @@ -174,32 +255,48 @@ def validate(self, mode, duration, cursor): assert len(normalized) >= 45 and duration >= 45 * 60 expected = self.candles(self.cut, normalized[-1]["ts_open"] + 60000) for index, (actual, candle) in enumerate(zip(normalized, expected, strict=True)): - assert [actual["ts_open"], *(actual[key] for key in ("o", "h", "l", "c", "v"))] == \ - [candle[0], *(Decimal(candle[index]) for index in range(1, 6))], "bar REST mismatch" + assert [actual["ts_open"], *(actual[key] for key in ("o", "h", "l", "c", "v"))] == [ + candle[0], + *(Decimal(candle[index]) for index in range(1, 6)), + ], "bar REST mismatch" assert tape[index]["type"] == "bar" - assert [tokens[index]["bar"][key] for key in ("o", "h", "l", "c", "v")] == candle[1:6], "bar decimal tokens changed" - self.receipt(f"PASS bars equal REST minutes={len(normalized)} duration_seconds={duration:.3f}") + assert [tokens[index]["bar"][key] for key in ("o", "h", "l", "c", "v")] == candle[ + 1:6 + ], "bar decimal tokens changed" + self.receipt( + f"PASS bars equal REST minutes={len(normalized)} duration_seconds={duration:.3f}" + ) else: assert duration >= 20 * 60 trades = [event for event in tape if event["type"] == "tick"] trade_tokens = [event for event in tokens if event["type"] == "tick"] times = [event["ts"] for event in tape if event["type"] == "time"] assert len(times) >= 20 and trades - for previous, current in zip(trades, trades[1:]): + for previous, current in itertools.pairwise(trades): assert current["seq"] == previous["seq"] + 1 and current["ts"] >= previous["ts"] next_id = trades[0]["seq"] trade_offset = 0 while trade_offset < len(trades): - page = self.fetch("/api/v3/historicalTrades", {"symbol": "BTCUSDT", "fromId": next_id, - "limit": 1000}, 25) + page = self.fetch( + "/api/v3/historicalTrades", + {"symbol": "BTCUSDT", "fromId": next_id, "limit": 1000}, + 25, + ) assert page and page[0]["id"] == next_id for raw in page: if trade_offset == len(trades): break tick = trades[trade_offset] - assert [tick["seq"], tick["ts"], tick["price"], tick["qty"]] == \ - [raw["id"], raw["time"], Decimal(raw["price"]), Decimal(raw["qty"])], "raw REST mismatch" - assert [trade_tokens[trade_offset][key] for key in ("price", "qty")] == [raw["price"], raw["qty"]], "raw decimal tokens changed" + assert [tick["seq"], tick["ts"], tick["price"], tick["qty"]] == [ + raw["id"], + raw["time"], + Decimal(raw["price"]), + Decimal(raw["qty"]), + ], "raw REST mismatch" + assert [trade_tokens[trade_offset][key] for key in ("price", "qty")] == [ + raw["price"], + raw["qty"], + ], "raw decimal tokens changed" trade_offset += 1 next_id += 1 expected = self.candles(self.cut, times[-1]) @@ -215,39 +312,96 @@ def validate(self, mode, duration, cursor): raw = expected[len(normalized)] prices = [tick["price"] for tick in minute_trades] assert prices and len(prices) == raw[8] - values = [prices[0], max(prices), min(prices), prices[-1], sum(tick["qty"] for tick in minute_trades)] - assert values == [Decimal(raw[index]) for index in range(1, 6)], "time OHLCV mismatch" - normalized.append(dict(zip(("ts_open", "o", "h", "l", "c", "v"), [minute, *values], strict=True))) + values = [ + prices[0], + max(prices), + min(prices), + prices[-1], + sum(tick["qty"] for tick in minute_trades), + ] + assert values == [Decimal(raw[index]) for index in range(1, 6)], ( + "time OHLCV mismatch" + ) + normalized.append( + dict( + zip( + ("ts_open", "o", "h", "l", "c", "v"), [minute, *values], strict=True + ) + ) + ) minute += 60000 minute_trades = [] - self.receipt(f"PASS ticks contiguous and time events equal REST ticks={len(trades)} minutes={len(times)} duration_seconds={duration:.3f}") + self.receipt( + f"PASS ticks contiguous and time events equal REST ticks={len(trades)} " + f"minutes={len(times)} duration_seconds={duration:.3f}" + ) combined = self.directory / (mode + "-combined.csv") with open(combined, "w") as output: output.write((self.directory / "warmup.csv").read_text()) for bar in normalized: - output.write(",".join(str(bar[key]) for key in ("ts_open", "o", "h", "l", "c", "v")) + "\n") + output.write( + ",".join(str(bar[key]) for key in ("ts_open", "o", "h", "l", "c", "v")) + "\n" + ) batch_actions = self.directory / (mode + "-batch-actions.jsonl") - subprocess.run([self.options.batch_probe, self.options.observed_strategy, str(combined), str(batch_actions)], check=True) - expected_actions = [action_key(record) for record in rows(batch_actions) if record["origin_input_index"] >= 200] + subprocess.run( + [ + self.options.batch_probe, + self.options.observed_strategy, + str(combined), + str(batch_actions), + ], + check=True, + ) + expected_actions = [ + action_key(record) + for record in rows(batch_actions) + if record["origin_input_index"] >= 200 + ] actual_actions = [action_key(record) for record in self.receiver.payloads.get(mode, [])] event_ids = [record["event_id"] for record in self.receiver.payloads.get(mode, [])] assert len(event_ids) == len(set(event_ids)), "duplicate delivered action after restart" equal = expected_actions == actual_actions - (self.directory / (mode + "-action-comparison.json")).write_text(json.dumps( - {"equal": equal, "batch": expected_actions, "runner": actual_actions}, indent=2) + "\n") - self.receipt(f"{'PASS' if equal else 'FAIL'} {mode} actions equal batch via run_backtest_full batch={len(expected_actions)} runner={len(actual_actions)}") + (self.directory / (mode + "-action-comparison.json")).write_text( + json.dumps( + {"equal": equal, "batch": expected_actions, "runner": actual_actions}, indent=2 + ) + + "\n" + ) + verdict = "PASS" if equal else "FAIL" + self.receipt( + f"{verdict} {mode} actions equal batch via run_backtest_full " + f"batch={len(expected_actions)} runner={len(actual_actions)}" + ) if mode == "ticks": name = "ticks-replay" - completed = subprocess.run(self.runner_command(mode, name, self.directory / "ticks-replay.sqlite3", - str(self.directory / "ticks-feed.jsonl")), capture_output=True, text=True, timeout=120) + completed = subprocess.run( + self.runner_command( + mode, + name, + self.directory / "ticks-replay.sqlite3", + str(self.directory / "ticks-feed.jsonl"), + ), + capture_output=True, + text=True, + timeout=120, + ) (self.directory / "ticks-replay.stdout").write_text(completed.stdout) (self.directory / "ticks-replay.stderr").write_text(completed.stderr) assert completed.returncode == 0, "tick replay runner failed" - assert actual_actions == [action_key(record) for record in self.receiver.payloads.get(name, [])] - self.receipt(f"PASS ticks same-print runner replay actions equal actions={len(actual_actions)}") + assert actual_actions == [ + action_key(record) for record in self.receiver.payloads.get(name, []) + ] + self.receipt( + f"PASS ticks same-print runner replay actions equal actions={len(actual_actions)}" + ) assert equal or mode == "ticks", "confirmed-bar batch action parity failed" - return {"duration_seconds": duration, "messages": cursor, "minutes": len(normalized), - "actions": len(actual_actions), "batch_actions_equal": equal} + return { + "duration_seconds": duration, + "messages": cursor, + "minutes": len(normalized), + "actions": len(actual_actions), + "batch_actions_equal": equal, + } def live(self, mode, minutes): processes = None @@ -256,12 +410,16 @@ def live(self, mode, minutes): processes = self.generation(mode, 0, 0) self.receipt(f"START {mode} public BTCUSDT cut={self.cut} minimum_minutes={minutes}") time.sleep(self.options.restart_seconds) - assert processes[0].poll() is None and processes[1].poll() is None, "process stopped before restart" + assert processes[0].poll() is None and processes[1].poll() is None, ( + "process stopped before restart" + ) cursor = self.finish(mode, 0, processes, crash=True) processes = self.generation(mode, 1, cursor) self.receipt(f"RESTART {mode} --output-from={cursor} --from-input={cursor}") while time.monotonic() - started < minutes * 60: - assert processes[0].poll() is None and processes[1].poll() is None, "live process stopped early" + assert processes[0].poll() is None and processes[1].poll() is None, ( + "live process stopped early" + ) time.sleep(5) cursor = self.finish(mode, 1, processes) processes = None @@ -285,8 +443,10 @@ def live(self, mode, minutes): def run(self): self.warmup() - threads = [threading.Thread(target=self.live, args=("bars", self.options.bar_minutes)), - threading.Thread(target=self.live, args=("ticks", self.options.tick_minutes))] + threads = [ + threading.Thread(target=self.live, args=("bars", self.options.bar_minutes)), + threading.Thread(target=self.live, args=("ticks", self.options.tick_minutes)), + ] for thread in threads: thread.start() for thread in threads: @@ -295,7 +455,9 @@ def run(self): (self.directory / "summary.json").write_text(json.dumps(self.results, indent=2) + "\n") failed = any("error" in result for result in self.results.values()) if not self.options.allow_tick_ohlc_difference: - failed |= any(result.get("batch_actions_equal") is False for result in self.results.values()) + failed |= any( + result.get("batch_actions_equal") is False for result in self.results.values() + ) return 1 if failed else 0 @@ -306,6 +468,10 @@ def run(self): parser.add_argument("--bar-minutes", type=int, default=46) parser.add_argument("--tick-minutes", type=int, default=21) parser.add_argument("--restart-seconds", type=int, default=180) - parser.add_argument("--allow-tick-ohlc-difference", action="store_true", - help="explicitly use same-print tick replay as the gate; still record failed OHLC batch parity") + parser.add_argument( + "--allow-tick-ohlc-difference", + action="store_true", + help="explicitly use same-print tick replay as the gate; " + "still record failed OHLC batch parity", + ) raise SystemExit(Soak(parser.parse_args()).run()) From 24c9fc832bf66f15193e1a086f3c5e9d8e61dcfd Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 02:11:58 +0800 Subject: [PATCH 5/7] Compare tick actions with the batch under the R-B2 mapping A tick fill carries its print's time while the OHLC batch carries the modeled segment's clock, so the strict tick comparison could not pass. Apply the predeclared R-B2 mapping (floor(ts / script_tf) * script_tf on both sides, tick mode only), keep every other field exact, fail the run on any difference, and retire --allow-tick-ohlc-difference. The minimum live duration now follows --bar-minutes and --tick-minutes, and the tick replay gets 600 seconds. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/README.md | 13 +++++--- native/feed/tests/mock_venue.py | 41 +++++++++++++++--------- native/feed/tests/public_e2e.py | 56 ++++++++++++++++++--------------- 3 files changed, 66 insertions(+), 44 deletions(-) diff --git a/native/feed/README.md b/native/feed/README.md index 2bd831c..610fc8f 100644 --- a/native/feed/README.md +++ b/native/feed/README.md @@ -153,11 +153,14 @@ reconnect overlap, fragmented text, PING/PONG, retirement, budgets, and recovery Never commit public exchange captures. `tests/public_e2e.py` explicitly opts into public BTCUSDT capture, local receiver delivery, a hard-kill restart, exact REST verification, bar batch action parity through `run_backtest_full`, -and tick same-print replay determinism. Tick-vs-OHLC-only fills are not assumed -equivalent: the harness exits nonzero for that comparison unless the operator -explicitly passes `--allow-tick-ohlc-difference`; even then it retains the FAIL -receipt and false parity result. Its batch observer is linked separately from the read-only engine's -`tests/native_live_equivalence_observer.cpp`; it is not a feed dependency. +and tick same-print replay determinism. Tick actions are compared with the +batch under the predeclared R-B2 mapping: each action timestamp becomes +`floor(ts / script_tf) * script_tf` on both sides, because a tick fill carries +its print's time while the OHLC batch carries its modeled segment's clock. +Every other field stays exact, bar actions compare raw timestamps, and any +difference fails the run. Its batch observer is linked separately from the +read-only engine's `tests/native_live_equivalence_observer.cpp`; it is not a +feed dependency. `src/venue.hpp` is the adapter boundary. Later slices add OKX raw ticks/bars (contract-unit conversion where needed), Bybit confirmed bars **without ticks**, diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py index 4f9d249..9b8ba55 100644 --- a/native/feed/tests/mock_venue.py +++ b/native/feed/tests/mock_venue.py @@ -19,7 +19,7 @@ import urllib.parse from types import SimpleNamespace -from public_e2e import Soak +from public_e2e import Soak, action_key BINARY = str(pathlib.Path(sys.argv[1]).resolve()) sys.argv = [sys.argv[0]] @@ -335,26 +335,39 @@ def websocket(self, query): class FeedMockTests(unittest.TestCase): - def test_public_qualification_requires_explicit_tick_ohlc_exception(self): - for allow, expected in ((False, 1), (True, 0)): + def test_public_qualification_fails_on_any_action_difference(self): + for ticks_equal, expected in ((False, 1), (True, 0)): soak = Soak.__new__(Soak) - soak.options = SimpleNamespace( - allow_tick_ohlc_difference=allow, bar_minutes=46, tick_minutes=21 - ) - soak.directory = self.directory / str(allow) + soak.options = SimpleNamespace(bar_minutes=21, tick_minutes=21) + soak.directory = self.directory / str(ticks_equal) soak.directory.mkdir() soak.results = {} soak.receiver = SimpleNamespace(shutdown=lambda: None) soak.warmup = lambda: None - soak.live = lambda mode, minutes, soak=soak: soak.results.update( - {mode: {"batch_actions_equal": mode == "bars"}} + soak.live = lambda mode, minutes, equal=ticks_equal, soak=soak: soak.results.update( + {mode: {"batch_actions_equal": mode == "bars" or equal}} ) self.assertEqual(soak.run(), expected) - self.assertFalse( - json.loads((soak.directory / "summary.json").read_text())["ticks"][ - "batch_actions_equal" - ] - ) + + def test_r_b2_maps_only_tick_action_timestamps(self): + record = { + "timestamp": 1791037920021, + "bar_index": 7, + "order": { + "id": "long", + "action": "entry", + "leg": "long", + "contracts": 1.0, + "price": 84816.0, + "reduce_only": False, + "entry_incarnation": 1, + }, + } + batch = dict(record, timestamp=1791037920000) + self.assertEqual(action_key(record, "ticks"), action_key(batch, "ticks")) + self.assertNotEqual(action_key(record, "bars"), action_key(batch, "bars")) + moved = dict(record, order=dict(record["order"], price=84816.5)) + self.assertNotEqual(action_key(moved, "ticks"), action_key(batch, "ticks")) def setUp(self): self.temporary = tempfile.TemporaryDirectory(prefix="pineforge-feed-mock-") diff --git a/native/feed/tests/public_e2e.py b/native/feed/tests/public_e2e.py index cc8ce9e..021b61a 100644 --- a/native/feed/tests/public_e2e.py +++ b/native/feed/tests/public_e2e.py @@ -28,10 +28,20 @@ def rows(path, exact=False): ] -def action_key(record): +SCRIPT_TF_MS = 60000 # the runner and batch both run --script-tf 1 + + +def action_key(record, mode): + """Comparable action fields. Ticks use the predeclared R-B2 mapping: an action timestamp + becomes floor(ts / script_tf) * script_tf on both sides, because a tick fill carries its + print's time while the OHLC batch carries the modeled segment's clock. Every other field + stays exact.""" order = record["order"] + timestamp = record["timestamp"] + if mode == "ticks": + timestamp = timestamp // SCRIPT_TF_MS * SCRIPT_TF_MS return ( - record["timestamp"], + timestamp, record["bar_index"], order["id"], order["action"], @@ -239,7 +249,7 @@ def candles(self, start, end): assert len(result) * 60000 == end - result[0][0] return result - def validate(self, mode, duration, cursor): + def validate(self, mode, duration, cursor, minutes): getcontext().prec = 160 tape = rows(self.directory / (mode + "-feed.jsonl"), True) with open(self.directory / (mode + "-feed.jsonl")) as source: @@ -252,7 +262,7 @@ def validate(self, mode, duration, cursor): self.receipt(f"PASS {mode} restart without gap or duplicate messages={cursor}") if mode == "bars": normalized = [event["bar"] for event in tape] - assert len(normalized) >= 45 and duration >= 45 * 60 + assert len(normalized) >= minutes - 1 and duration >= (minutes - 1) * 60 expected = self.candles(self.cut, normalized[-1]["ts_open"] + 60000) for index, (actual, candle) in enumerate(zip(normalized, expected, strict=True)): assert [actual["ts_open"], *(actual[key] for key in ("o", "h", "l", "c", "v"))] == [ @@ -267,11 +277,11 @@ def validate(self, mode, duration, cursor): f"PASS bars equal REST minutes={len(normalized)} duration_seconds={duration:.3f}" ) else: - assert duration >= 20 * 60 + assert duration >= (minutes - 1) * 60 trades = [event for event in tape if event["type"] == "tick"] trade_tokens = [event for event in tokens if event["type"] == "tick"] times = [event["ts"] for event in tape if event["type"] == "time"] - assert len(times) >= 20 and trades + assert len(times) >= minutes - 1 and trades for previous, current in itertools.pairwise(trades): assert current["seq"] == previous["seq"] + 1 and current["ts"] >= previous["ts"] next_id = trades[0]["seq"] @@ -353,11 +363,13 @@ def validate(self, mode, duration, cursor): check=True, ) expected_actions = [ - action_key(record) + action_key(record, mode) for record in rows(batch_actions) if record["origin_input_index"] >= 200 ] - actual_actions = [action_key(record) for record in self.receiver.payloads.get(mode, [])] + actual_actions = [ + action_key(record, mode) for record in self.receiver.payloads.get(mode, []) + ] event_ids = [record["event_id"] for record in self.receiver.payloads.get(mode, [])] assert len(event_ids) == len(set(event_ids)), "duplicate delivered action after restart" equal = expected_actions == actual_actions @@ -367,9 +379,10 @@ def validate(self, mode, duration, cursor): ) + "\n" ) + mapping = " (R-B2 timestamps)" if mode == "ticks" else "" verdict = "PASS" if equal else "FAIL" self.receipt( - f"{verdict} {mode} actions equal batch via run_backtest_full " + f"{verdict} {mode} actions equal batch via run_backtest_full{mapping} " f"batch={len(expected_actions)} runner={len(actual_actions)}" ) if mode == "ticks": @@ -383,18 +396,18 @@ def validate(self, mode, duration, cursor): ), capture_output=True, text=True, - timeout=120, + timeout=600, ) (self.directory / "ticks-replay.stdout").write_text(completed.stdout) (self.directory / "ticks-replay.stderr").write_text(completed.stderr) assert completed.returncode == 0, "tick replay runner failed" - assert actual_actions == [ - action_key(record) for record in self.receiver.payloads.get(name, []) + raw = [action_key(record, "bars") for record in self.receiver.payloads.get(mode, [])] + assert raw == [ + action_key(record, "bars") for record in self.receiver.payloads.get(name, []) ] self.receipt( f"PASS ticks same-print runner replay actions equal actions={len(actual_actions)}" ) - assert equal or mode == "ticks", "confirmed-bar batch action parity failed" return { "duration_seconds": duration, "messages": cursor, @@ -424,7 +437,7 @@ def live(self, mode, minutes): cursor = self.finish(mode, 1, processes) processes = None duration = time.monotonic() - started - self.results[mode] = self.validate(mode, duration, cursor) + self.results[mode] = self.validate(mode, duration, cursor, minutes) except Exception: if processes: for process in processes[:2]: @@ -453,11 +466,10 @@ def run(self): thread.join() self.receiver.shutdown() (self.directory / "summary.json").write_text(json.dumps(self.results, indent=2) + "\n") - failed = any("error" in result for result in self.results.values()) - if not self.options.allow_tick_ohlc_difference: - failed |= any( - result.get("batch_actions_equal") is False for result in self.results.values() - ) + failed = any( + "error" in result or not result.get("batch_actions_equal") + for result in self.results.values() + ) return 1 if failed else 0 @@ -468,10 +480,4 @@ def run(self): parser.add_argument("--bar-minutes", type=int, default=46) parser.add_argument("--tick-minutes", type=int, default=21) parser.add_argument("--restart-seconds", type=int, default=180) - parser.add_argument( - "--allow-tick-ohlc-difference", - action="store_true", - help="explicitly use same-print tick replay as the gate; " - "still record failed OHLC batch parity", - ) raise SystemExit(Soak(parser.parse_args()).run()) From dfbf3b6592669951e581d4dd9b3235bc9f213d98 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 02:12:30 +0800 Subject: [PATCH 6/7] Group-commit native feed messages and close review gaps - Group commit: verify every event made ready by the source messages already queued (up to 1024 messages or 4096 events), append them in one write, fsync the journal once, replace the cursor once, then publish the lines. One event per line and persist-before-publish are unchanged; a stop publishes what was already verified and a failed commit leaves the journal unusable until resume truncates its tail. - A tick minute whose own x=true kline was missed reconciles the REST kline with reconcile(kline, false) behind an explicit next-minute fence check; no kline field is filled in. - Bars mode subscribes to kline_1m only. An unknown event outside the data streams is a structured warning; the data streams stay strict (23). - REST is spaced at half of each published ceiling and pauses when the used- weight header passes half of the 1-minute limit. Retry-After accepts delta-seconds and an IMF-fixdate HTTP-date. The REST body cap matches the 1 MiB JSON parser limit. - The initial raw-trade fence lookup retries when REST lags the WebSocket. - Warmup compares its REST rows with the x=true bars seen on the WebSocket. - A matched-time regression stops with 23 instead of the gap code. - stdout flags are restored on exit; stderr is left alone and written by one thread in whole records, so a full sink drops records instead of cutting them. Signal handlers are installed first. - A fresh start takes over a directory whose only file is an empty journal. - Remove the unused reconnect hook and string_field. - Tests: tick reconnect overlap change (unit + mock), Retry-After seconds and HTTP-date, multi-page healing, matched-time regression, HTTP 418 and 403, unknown and malformed events, lagging REST fence, warmup cross-check, stdio flags and a full stderr pipe, group commit, crashed initialization; the failed cursor write now asserts that nothing was published. - README: group commit and measured throughput, run-time horizon per mode, the historical closure rule, tick mode's liquid-symbol limit, and the expected restart policy. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/README.md | 149 ++++++++++++++----- native/feed/src/binance.cpp | 130 +++++++++++----- native/feed/src/binance.hpp | 5 + native/feed/src/core.cpp | 82 +++++++++-- native/feed/src/core.hpp | 3 +- native/feed/src/main.cpp | 28 +++- native/feed/src/state.cpp | 52 +++++-- native/feed/src/state.hpp | 13 +- native/feed/src/transport.cpp | 69 ++++++--- native/feed/src/transport.hpp | 8 +- native/feed/tests/mock_venue.py | 252 ++++++++++++++++++++++++++++++-- native/feed/tests/unit.cpp | 111 +++++++++++++- 12 files changed, 754 insertions(+), 148 deletions(-) diff --git a/native/feed/README.md b/native/feed/README.md index 610fc8f..2eff191 100644 --- a/native/feed/README.md +++ b/native/feed/README.md @@ -23,10 +23,11 @@ ctest --test-dir build-feed --output-on-failure ``` `tests/build_ci_curl.sh PREFIX` builds the CI-pinned curl with WebSockets and -OpenSSL. The isolated `.github/workflows/native-feed.yml` defines Linux and -macOS Release builds, ASan+UBSan, and a separate Linux TSan build. Python and -the existing Python workflow are unaffected. Python 3.9+ is used only for the -synthetic mock tests; public qualification uses Python 3.10+. +OpenSSL. The isolated `.github/workflows/native-feed.yml` defines Linux amd64, +Linux arm64 and macOS Release builds, ASan+UBSan, and a separate Linux TSan +build. The Python sdist excludes `native/` and the Python CI asserts it; the +wheel is unchanged. Python 3.9+ is used only for the synthetic mock tests; +public qualification uses Python 3.10+. ## A verified warmup cut @@ -64,24 +65,37 @@ trailing zeros. Fixed-point string arithmetic reconciles OHLCV; no price or quantity enters a binary float. Bounds are 96 token characters, 32 fractional digits, and 128 arithmetic digits. Invalid/zero tick quantities fail closed. -The combined stream uses raw `t` as the positive sequence and matched `T` as -time, not dispatch `E`. The real predecessor fetched from REST proves the -initial cut; aggregates locate a raw start ID but are never emitted or expanded. -Each new raw ID must be exactly its predecessor plus one, with nonregressing -matched time. Holes/reordering heal through inclusive `historicalTrades` -`fromId` pages (maximum 1000); identical duplicates are discarded and conflicting -duplicates stop. There is no assumed all-time retention window. +Bars mode subscribes to `kline_1m` only; ticks mode adds the raw `trade` +stream. Raw `t` is the positive sequence and matched `T` is time, not dispatch +`E`. The real predecessor fetched from REST proves the initial cut; aggregates +locate a raw start ID but are never emitted or expanded. Each new raw ID must be +exactly its predecessor plus one; a matched time that goes backwards along that +chain stops with 23. Holes/reordering heal through inclusive `historicalTrades` +`fromId` pages (maximum 1000); identical duplicates are discarded and +conflicting duplicates stop. There is no assumed all-time retention window. For the observed minute, `x=true`, the complete `f..L` range, `n`, and exact OHLCV must all reconcile before `time = open + 60000`. Later-minute prints stay -buffered until that proof. A missed historical minute can be closed only behind -a later observed `x=true` watermark, with a fetched contiguous raw prefix, an -independently fetched next-minute raw-ID fence, and exact REST `n`/OHLCV. This -uses the approved historical-watermark policy, not a fabricated WS confirmation. +buffered until that proof. A tick minute whose own `x=true` kline was missed +(it fell inside a reconnect or a catch-up) closes only on this proof: + +1. a later observed `x=true` watermark; +2. the contiguous raw-ID chain from the proven predecessor; +3. the first print of a later minute, fetched by ID, with ID exactly the + minute's last ID plus one (the fence); +4. the REST kline's exact `n` and OHLCV. + +A REST kline has no `x=true` and no `f..L`, and the feed does not fill them in. Empty/ambiguous tick minutes deliberately stop rather than invent a fence. Bars heal missing minutes from REST only behind the verified closed watermark. Any discovered already-emitted bar revision stops; history is never rewritten. +The subscribed data streams are strict: another event type on them, a +malformed trade or kline, or an unknown shape stops with 23. An event outside +the data streams with an unknown type (a new venue notice) is logged as a +structured `unknown_stream_event` warning and ignored; `serverShutdown` +reconnects. + Curl answers server PING with matching-payload PONG. Reconnect occurs on `serverShutdown`, transport loss/75-second source silence, or at 23h55 (before the venue's 24-hour limit). The reader buffers during REST healing. Every new @@ -93,22 +107,30 @@ to be continuously polled. REST routes are restricted to public `exchangeInfo`, `historicalTrades`, `aggTrades`, and `klines`. Current public weight/raw-request limits are read -from `exchangeInfo`, combined with conservative pacing, observed used-weight -headers, shared-quota pauses, and `429 Retry-After`. A ban/access rejection is -not retried as anonymous trading access. TLS verification is always enabled; -redirects, URL credentials, and non-origin overrides are refused. Explicit -insecure overrides are **loopback-only**, for tests. +from `exchangeInfo`, and requests are spaced to spend at most half of each +published ceiling (about 50% headroom; 20 ms per weight at 6000 per minute, so +500 ms per 1000-print page). The venue's `X-MBX-USED-WEIGHT-1M` header counts +every client on the IP: once it shows half of the 1-minute ceiling, the feed +waits for the next window. `429` honours `Retry-After` as delta-seconds or an +IMF-fixdate HTTP-date; a wait above 24 hours stops with 22 and an unparseable +value with 23. A ban/access rejection is not retried as anonymous trading +access. TLS verification is always enabled; redirects, URL credentials, and +non-origin overrides are refused. Explicit insecure overrides are +**loopback-only**, for tests. ## Durability and restart -One producer owns a state directory via `flock`. `events.jsonl` is append-only; -each message is fsynced before `cursor.json` is atomically replaced and its -directory fsynced, **before** stdout publication. The cursor binds source -origins, symbol/mode/units, random stream epoch, initial/verified cut, venue ID, -emitted sequence, message index, byte count, predecessor, closed-minute proofs, -SHA-256 prefix chain, last-message hash, and fixed message partition. Its -canonical document is itself hashed. Resume verifies the retained committed -prefix and fsyncs truncation of an uncommitted crash tail. +One producer owns a state directory via `flock`. `events.jsonl` is append-only. +Messages are group-committed: every event made ready by the source messages +already queued (up to 1024 source messages, or 4096 events) is appended in one +write and fsynced once, then `cursor.json` is atomically replaced once and its +directory fsynced, and only **then** are those lines written to stdout, one +event per line as before. The cursor binds source origins, symbol/mode/units, +random stream epoch, initial/verified cut, venue ID, emitted sequence, message +index, byte count, predecessor, closed-minute proofs, SHA-256 prefix chain, +last-message hash, and fixed message partition. Its canonical document is +itself hashed. Resume verifies the retained committed prefix and fsyncs +truncation of an uncommitted crash tail. A pipe write is not a consumer acknowledgment. By default `--resume` replays the whole verified origin prefix. If the runner has committed N messages: @@ -124,10 +146,18 @@ build-feed/pineforge-feed run --venue binance --market spot --symbol BTCUSDT \ Replace N with the runner's actual committed count; never use the producer's possibly-ahead count. Bind warmup identity and stream epoch in orchestration. An unavailable cursor or changed identity is refused. SIGTERM/SIGINT stops -intake, finishes a durable current message or leaves a rollback tail, and exits -0. Nonblocking stdout has a five-second stop drain; a consumer that has not -acknowledged a committed message must replay it. Stderr is structured JSON and -nonblocking; a stalled diagnostics sink can lose logs, not stall shutdown. +intake, discards staged messages that were never committed or published, and +exits 0. A directory holding only an empty `events.jsonl` (a crash before the +first cursor write) is taken over by a fresh `--start`; other existing state +needs `--resume`. + +Stdout is nonblocking during `run` for a five-second stop drain, and its +original flags are restored on exit (a SIGKILL cannot restore them); a consumer +that has not acknowledged a committed message must replay it. Stderr's flags +are never changed: one writer thread writes whole JSON records of at most +`PIPE_BUF` bytes, so a pipe write is atomic and a record is never cut. A full +sink drops whole records, the next record reports `dropped_records`, and exit +waits at most 2 seconds for queued records. | Exit | Meaning | |---|---| @@ -135,16 +165,67 @@ nonblocking; a stalled diagnostics sink can lose logs, not stall shutdown. | 20 | Unhealable history gap, ambiguous fence, exhausted source retries | | 21 | Changed overlap, conflicting duplicate, revised bar, changed cursor/prefix | | 22 | Replay/disk/queue/allocation budget or durable I/O failure | -| 23 | Unsupported input/source, permanent access, protocol/decimal/dependency failure | +| 23 | Unsupported input/source, permanent access, protocol/decimal/dependency failure, matched-time regression | Default replay disk budget: 256 MiB (`--max-log-bytes`); each prefix traversal: 60 seconds (`--max-replay-seconds`, maximum 3600); source and unconfirmed-print -queues: 16 MiB each (`--max-queue-bytes`); WS text: 1 MiB; REST body: 4 MiB. +queues: 16 MiB each (`--max-queue-bytes`); WS text: 1 MiB; REST body: 1 MiB. Warmup: 100000 rows / 16 MiB. Exhaustion is a cursor-preserving stop, not silent compaction or dropped events. Raising limits is an explicit operator choice. `--max-messages` permits bounded qualification. `--reconnect-seconds` can shorten the 23h55 rotation for synthetic tests, never extend it beyond that limit. +### Throughput + +Storage sets the ceiling, so measure on the target disk. On a shared 16-vCPU +Linux test machine (ext4 root), committing each message separately (three +fsyncs per print) sustained about 120 messages/s. With group commit, the +loopback bench (synthetic contiguous prints through the real binary and the +mock venue, wall time including startup) ran 30,000 prints at 5,811, 11,212 and +10,842 messages/s in three runs on the same machine. The busiest BTCUSDT minute +of a recent 89-day sample averaged 655 prints/s, about one ninth of the slowest +run. + +### Run-time horizon per mode + +`--max-log-bytes` is a hard stop (22): this release neither rotates nor +checkpoints the journal. Journal segmentation and a checkpoint (prefix hash, +cut, sequence, current-minute aggregate, the 32 proofs) are planned together +with bounded replay windows. Until then the horizon is the budget divided by +the journal's growth: + +| Mode | Bytes per message | Messages | Horizon at the default 256 MiB | +|---|---|---|---| +| bars | about 138 | 1 per minute | about 3.7 years | +| ticks, BTCUSDT median day | about 92 | 2.92 M per day | about 24 hours | +| ticks, BTCUSDT busiest day | about 92 | 8.93 M per day | about 8 hours | + +Message sizes come from a public BTCUSDT capture; the daily print counts are +REST kline trade counts over 89 days. + +Raising the budget lengthens every whole-journal scan: at startup, on every +tick-mode reconnect, on a duplicate closed kline and on an old duplicate print, +each bounded by `--max-replay-seconds`. + +### Tick mode needs a liquid symbol + +75 seconds without any text frame forces a reconnect, and each reconnect costs +a weight-27 overlap check (one `historicalTrades` page and one `klines` call). +A minute without a print cannot be fenced, so tick mode stops with 20 there, +and again on every resume. Use bars mode for symbols that can go quiet for a +minute. + +### Restart policy + +Retry budgets are short by design: about 2 minutes of failed WebSocket +connects, or about 7 seconds of failed REST attempts, end with 20 and the +verified cursor retained. Run the feed under a supervisor (a systemd unit with +`Restart=on-failure`, or a container restart policy) that restarts on 20 with +backoff, as `--resume --output-from N` with the runner's committed count. +Exits 21 and 23 need an operator (changed history, protocol or access +failure): an automatic restart repeats the stop. 22 needs a larger budget, more +disk, or a consumer that reads stdout again. + ## Qualification and next adapters `tests/unit.cpp` and `tests/mock_venue.py` contain **synthetic** documented diff --git a/native/feed/src/binance.cpp b/native/feed/src/binance.cpp index eccef22..9f8ab38 100644 --- a/native/feed/src/binance.cpp +++ b/native/feed/src/binance.cpp @@ -55,7 +55,7 @@ Kline binance_kline(const Json& value, bool websocket) { std::string BinanceSpot::streams() const { const auto symbol = lower(config_.symbol); - return symbol + "@trade/" + symbol + "@kline_1m"; + return config_.mode == "ticks" ? symbol + "@trade/" + symbol + "@kline_1m" : symbol + "@kline_1m"; } VenueEvent BinanceSpot::decode(const std::string& message) const { try { @@ -63,16 +63,25 @@ VenueEvent BinanceSpot::decode(const std::string& message) const { const auto* wrapped = envelope.find("data"); const auto& data = wrapped ? *wrapped : envelope; const auto* event = data.find("e"); - if (!event) throw Error(23, "unexpected public stream control message"); - if (event->text() == "serverShutdown") return {}; - if (data.at("s").text() != config_.symbol) throw Error(23, "public stream symbol changed"); - if (event->text() == "trade") return {VenueEvent::Kind::Trade, binance_trade(data, true), {}}; - if (event->text() == "kline") { + if (!event || event->kind != Json::Kind::String) throw Error(23, "unexpected public stream control message"); + const auto type = event->text(); + const auto stream = wrapped ? envelope.at("stream").text() : std::string(); + const auto symbol = lower(config_.symbol); + const bool trades = stream == symbol + "@trade", klines = stream == symbol + "@kline_1m"; + // Subscribed data streams and data event types are strict: the documented shape, or stop. + if (type == "trade" || type == "kline" || trades || klines) { + if (wrapped && !(trades && type == "trade") && !(klines && type == "kline")) + throw Error(23, "unsupported public stream event; aggregate prints are not raw trades"); + if (data.at("s").text() != config_.symbol) throw Error(23, "public stream symbol changed"); + if (type == "trade") return {VenueEvent::Kind::Trade, binance_trade(data, true), {}}; const auto& row = data.at("k"); if (row.at("s").text() != config_.symbol) throw Error(23, "kline symbol changed"); return {VenueEvent::Kind::Kline, {}, binance_kline(row, true)}; } - throw Error(23, "unsupported public stream event; aggregate prints are not raw trades"); + if (type == "serverShutdown") return {}; + // A venue notice outside the data streams carries no market data; a new type must not stop the feed. + log("warn", "unknown_stream_event", Json::object({{"stream", Json::string(stream)}, {"type", Json::string(type)}})); + return {}; } catch (const Error&) { throw; } catch (const std::exception&) { throw Error(23, "invalid Binance spot message shape"); } } @@ -97,14 +106,19 @@ std::vector BinanceSpot::history(std::uint64_t from, std::size_t limit) { throw Error(20, "raw history exhausted or unavailable within the venue retention window"); } std::uint64_t BinanceSpot::first_trade_id(std::int64_t minute) { - const auto response = http_.get("/api/v3/aggTrades?symbol=" + config_.symbol + "&startTime=" + std::to_string(minute) + - "&endTime=" + std::to_string(minute + 59999) + "&limit=1", 4); - const auto& rows = array(response, 1); - if (rows.empty()) throw Error(20, "initial minute has no available raw-trade start fence"); - const auto first = rows.front().at("f").integer(); - const auto matched = rows.front().at("T").integer(); - if (first <= 1 || matched < minute || matched >= minute + 60000) throw Error(20, "initial raw-trade fence is unavailable or ambiguous"); - return first; + // REST may lag the WebSocket print that triggered this lookup: retry like history() before stopping. + for (unsigned int attempt = 0; attempt < 3; ++attempt) { + if (attempt) pause_for(std::chrono::milliseconds(500 * attempt)); + const auto response = http_.get("/api/v3/aggTrades?symbol=" + config_.symbol + "&startTime=" + std::to_string(minute) + + "&endTime=" + std::to_string(minute + 59999) + "&limit=1", 4); + const auto& rows = array(response, 1); + if (rows.empty()) continue; + const auto first = rows.front().at("f").integer(); + const auto matched = rows.front().at("T").integer(); + if (first <= 1 || matched < minute || matched >= minute + 60000) throw Error(20, "initial raw-trade fence is unavailable or ambiguous"); + return first; + } + throw Error(20, "initial minute has no available raw-trade start fence"); } std::vector BinanceSpot::klines(std::int64_t start, std::int64_t end) { if (start < 0 || start % 60000 || end <= start || end % 60000) throw Error(23, "invalid exclusive kline range"); @@ -135,12 +149,30 @@ FeedSession::FeedSession(State& state, Venue& venue, std::function& proof) { if (stopping) throw Stopped{}; - state_.commit(line, proof); - output_(line); + state_.stage(line, proof); ++new_messages_; - if (state_.config().max_messages && new_messages_ >= state_.config().max_messages) throw Stopped{}; + if (state_.config().max_messages && new_messages_ >= state_.config().max_messages) { + publish(); + throw Stopped{}; + } + if (state_.staged() >= 4096) publish(); +} +void FeedSession::publish() { + for (const auto& line : state_.flush()) output_(line); +} +void FeedSession::salvage() noexcept { + try { publish(); } + catch (...) { + try { log("error", "verified_messages_not_committed"); } catch (...) {} + } +} +void FeedSession::ingest(const VenueEvent& event) { + try { stage(event); } + catch (const Error&) { salvage(); throw; } + publish(); } void FeedSession::connected() { + publish(); const auto& trades = state_.recent_trades(); if (!trades.empty()) { const auto fetched = venue_.history(trades.front().id, trades.size()); @@ -166,9 +198,7 @@ void FeedSession::connected() { for (std::size_t index = 0; index < proofs.size(); ++index) { if (!(proofs[index] == fetched[index].bar)) throw Error(21, "reconnect already-emitted bar changed"); if (state_.config().mode == "ticks") { - auto checked = fetched[index]; - checked.confirmed = true; - try { verified.at(checked.bar.ts).reconcile(checked, false); } + try { verified.at(fetched[index].bar.ts).reconcile(fetched[index], false); } catch (const std::exception&) { throw Error(21, "reconnect closed-minute count or raw proof changed"); } } } @@ -211,7 +241,8 @@ void FeedSession::put(const Trade& trade) { pending_.emplace(trade.id, trade); } void FeedSession::emit_trade(const Trade& trade) { - if (trade.id != next_id() || trade.ts < state_.cursor().cut || trade.ts >= state_.cursor().cut + 60000 || trade.ts < state_.cursor().last_tick_ts) + if (trade.ts < state_.cursor().last_tick_ts) throw Error(23, "venue matched time regressed along the contiguous raw-trade chain"); + if (trade.id != next_id() || trade.ts < state_.cursor().cut || trade.ts >= state_.cursor().cut + 60000) throw Error(20, "raw trade violates the complete chronological minute prefix"); const auto buffered = pending_.find(trade.id); if (buffered != pending_.end() && !(buffered->second == trade)) @@ -247,30 +278,31 @@ void FeedSession::drain() { emit_trade(trade); } } +// A minute whose own x=true kline was missed closes only on proof: a later x=true watermark, the contiguous +// raw-ID chain from the proven predecessor, the first print of a later minute as the fence, and REST n plus +// exact OHLCV. The REST row has no x=true or f..L, and none is manufactured for it. void FeedSession::historical_minute() { if (state_.cursor().cut + 60000 > watermark_) throw Error(20, "historical tick closure lacks a confirmed WebSocket watermark"); if (!state_.cursor().predecessor) anchor(venue_.first_trade_id(state_.cursor().start)); const auto minute = state_.cursor().cut; - bool fenced = false; - while (!fenced) { + std::optional fence; + while (!fence) { const auto page = venue_.history(next_id()); for (const auto& trade : page) { const auto buffered = pending_.find(trade.id); if (buffered != pending_.end() && !(buffered->second == trade)) throw Error(21, "historical raw overlap changed"); if (trade.ts >= minute + 60000) { - if (trade.id != next_id()) throw Error(20, "historical next-minute raw fence is not contiguous"); - put(trade); - fenced = true; + fence = trade; break; } emit_trade(trade); } } - auto kline = venue_.klines(minute, minute + 60000).front(); - kline.confirmed = true; - kline.first = static_cast(aggregate_.first); - kline.last = static_cast(aggregate_.last); - aggregate_.reconcile(kline, true); + if (!aggregate_.count || fence->id != next_id() || fence->id != aggregate_.last + 1) + throw Error(20, "historical minute lacks a contiguous raw prefix and next-minute fence"); + put(*fence); + const auto kline = venue_.klines(minute, minute + 60000).front(); + aggregate_.reconcile(kline, false); log("info", "historical_tick_fence_verified", Json::object({{"minute", Json::number(std::to_string(minute))}, {"first_id", Json::number(std::to_string(aggregate_.first))}, {"last_id", Json::number(std::to_string(aggregate_.last))}})); emit("{\"type\":\"time\",\"ts\":" + std::to_string(minute + 60000) + "}", kline.bar); @@ -327,7 +359,7 @@ void FeedSession::closed(const Kline& kline) { aggregate_ = Aggregate{}; drain(); } -void FeedSession::ingest(const VenueEvent& event) { +void FeedSession::stage(const VenueEvent& event) { if (event.kind == VenueEvent::Kind::Kline) { closed(event.kline); return; } if (event.kind != VenueEvent::Kind::Trade || state_.config().mode != "ticks" || event.trade.ts < state_.cursor().start) return; if (!state_.cursor().predecessor) anchor(venue_.first_trade_id(state_.cursor().start)); @@ -342,9 +374,16 @@ void run_feed(const Config& config) { WebSocketPump pump(config, venue.streams()); FeedSession session(state, venue, &output_line); for (;;) { - const auto message = pump.take(); - if (message.connected) session.connected(); - else session.ingest(venue.decode(message.text)); + auto message = pump.take(); + try { + // Group commit: verify every source message already queued (bounded), then persist once. + for (std::size_t taken = 1;; ++taken) { + if (message.connected) session.connected(); + else session.stage(venue.decode(message.text)); + if (taken >= 1024 || !pump.try_take(message)) break; + } + } catch (const Error&) { session.salvage(); throw; } + session.publish(); } } catch (const Stopped&) { log("info", "stopped", state.status()); } catch (...) { log("error", "verified_cursor_retained", state.status()); throw; } @@ -357,22 +396,34 @@ void warmup(const Config& config, const std::string& output) { if (std::filesystem::exists(output) || std::filesystem::exists(output + ".manifest.json")) throw Error(23, "warmup output already exists"); BinanceSpot venue(config); std::int64_t watermark = -1; + std::map confirmed; { WebSocketPump pump(config, venue.streams()); while (watermark < config.end) { const auto message = pump.take(); if (message.connected) continue; const auto event = venue.decode(message.text); - if (event.kind == VenueEvent::Kind::Kline && event.kline.confirmed) watermark = std::max(watermark, event.kline.bar.ts + 60000); + if (event.kind != VenueEvent::Kind::Kline || !event.kline.confirmed) continue; + const auto& bar = event.kline.bar; + watermark = std::max(watermark, bar.ts + 60000); + if (bar.ts < config.start || bar.ts >= config.end) continue; + const auto [found, inserted] = confirmed.emplace(bar.ts, bar); + if (!inserted && !(found->second == bar)) throw Error(21, "confirmed WebSocket bar was revised during warmup"); } } std::string csv = "timestamp,open,high,low,close,volume\n"; auto next = config.start; - std::uint64_t count = 0; + std::uint64_t count = 0, cross_checked = 0; while (next < config.end) { for (const auto& kline : venue.klines(next, config.end)) { if (kline.bar.ts != next || next + 60000 > watermark) throw Error(20, "warmup does not adjoin a verified closed-minute cut"); const auto& bar = kline.bar; + // The newest rows come from REST moments after the WS close: the x=true bars in hand must match. + const auto observed = confirmed.find(bar.ts); + if (observed != confirmed.end()) { + if (!(observed->second == bar)) throw Error(21, "warmup REST row differs from the confirmed WebSocket bar"); + ++cross_checked; + } csv += std::to_string(bar.ts) + ',' + bar.open + ',' + bar.high + ',' + bar.low + ',' + bar.close + ',' + bar.volume + '\n'; if (csv.size() > 16 * 1024 * 1024) throw Error(22, "warmup output exceeds 16 MiB"); next += 60000; @@ -385,7 +436,8 @@ void warmup(const Config& config, const std::string& output) { {"units", Json::object({{"price", Json::string("quote/base")}, {"volume", Json::string("base")}, {"timestamp", Json::string("unix-ms")}})}, {"start", Json::number(std::to_string(config.start))}, {"end_exclusive", Json::number(std::to_string(config.end))}, {"cut", Json::number(std::to_string(config.end))}, {"confirmed_ws_watermark", Json::number(std::to_string(watermark))}, - {"bars", Json::number(std::to_string(count))}, {"sha256", Json::string(sha256(csv))}}); + {"bars", Json::number(std::to_string(count))}, {"ws_cross_checked_bars", Json::number(std::to_string(cross_checked))}, + {"sha256", Json::string(sha256(csv))}}); atomic_file(output, csv); atomic_file(output + ".manifest.json", manifest.dump() + '\n'); log("info", "warmup_verified", manifest); diff --git a/native/feed/src/binance.hpp b/native/feed/src/binance.hpp index deaad62..8f5b06f 100644 --- a/native/feed/src/binance.hpp +++ b/native/feed/src/binance.hpp @@ -43,6 +43,11 @@ class FeedSession { public: FeedSession(State& state, Venue& venue, std::function output); void connected(); + // stage() verifies an event and stages its messages; publish() group-commits them, then writes them + // out; salvage() publishes what was already verified before a stop. ingest() is all three for one event. + void stage(const VenueEvent& event); + void publish(); + void salvage() noexcept; void ingest(const VenueEvent& event); }; void run_feed(const Config& config); diff --git a/native/feed/src/core.cpp b/native/feed/src/core.cpp index 8dc088a..a0dd869 100644 --- a/native/feed/src/core.cpp +++ b/native/feed/src/core.cpp @@ -6,7 +6,10 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -14,28 +17,79 @@ #include #include #include +#include #include namespace pineforge::feed { std::atomic stopping{false}; static_assert(std::atomic::is_always_lock_free); +namespace { +// Diagnostics go through one writer thread with ordinary blocking writes, so stderr's flags are never +// changed and no caller waits on a slow sink. A full queue drops whole records and counts them; a +// record is at most PIPE_BUF bytes, so a pipe write is atomic and never leaves half a line. +struct LogSink { + std::mutex mutex; + std::condition_variable ready, idle; + std::deque records; + std::size_t bytes = 0; + std::uint64_t dropped = 0; + bool writing = false; +}; +LogSink& sink() { + static LogSink* const shared = [] { + auto* created = new LogSink(); // never destroyed: the writer may still be blocked at exit + std::thread([created] { + std::unique_lock guard(created->mutex); + for (;;) { + created->ready.wait(guard, [created] { return !created->records.empty(); }); + const auto record = std::move(created->records.front()); + created->records.pop_front(); + created->bytes -= record.size(); + created->writing = true; + guard.unlock(); + std::size_t offset = 0; + while (offset < record.size()) { + const auto written = ::write(STDERR_FILENO, record.data() + offset, record.size() - offset); + if (written < 0 && errno == EINTR) continue; + if (written <= 0) break; + offset += static_cast(written); + } + guard.lock(); + created->writing = false; + if (created->records.empty()) created->idle.notify_all(); + } + }).detach(); + return created; + }(); + return *shared; +} +} + void log(const std::string& level, const std::string& event, const Json& fields) { - static std::mutex mutex; auto record = fields; record.members["level"] = Json::string(level); record.members["event"] = Json::string(event); record.members["ts"] = Json::number(std::to_string(std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count())); - const auto bytes = record.dump() + '\n'; - std::lock_guard guard(mutex); - std::size_t offset = 0; - while (offset < bytes.size()) { - const auto written = ::write(STDERR_FILENO, bytes.data() + offset, bytes.size() - offset); - if (written < 0 && errno == EINTR) continue; - if (written <= 0) break; - offset += static_cast(written); - } + auto& shared = sink(); + std::lock_guard guard(shared.mutex); + if (shared.dropped) record.members["dropped_records"] = Json::number(std::to_string(shared.dropped)); + auto bytes = record.dump() + '\n'; + if (bytes.size() > PIPE_BUF) + bytes = Json::object({{"level", record.members["level"]}, {"event", record.members["event"]}, + {"ts", record.members["ts"]}, {"truncated", Json::boolean(true)}}).dump() + '\n'; + if (bytes.size() > 1024 * 1024 - shared.bytes) { ++shared.dropped; return; } + shared.dropped = 0; + shared.bytes += bytes.size(); + shared.records.push_back(std::move(bytes)); + shared.ready.notify_one(); +} + +void flush_log(std::chrono::milliseconds limit) { + auto& shared = sink(); + std::unique_lock guard(shared.mutex); + shared.idle.wait_for(guard, limit, [&] { return shared.records.empty() && !shared.writing; }); } std::string sha256(std::string_view bytes) { @@ -229,9 +283,12 @@ void Aggregate::add(const Trade& trade) { last_ts = trade.ts; ++count; } +// A WebSocket proof (require_ids) needs the kline's own x=true and venue f..L range. A REST row carries +// neither: its caller proves closure with a later x=true watermark and a next-minute fence print. void Aggregate::reconcile(const Kline& kline, bool require_ids) const { - if (!kline.confirmed || !count || count != kline.count || - (require_ids && (kline.first <= 0 || kline.last < kline.first || first != static_cast(kline.first) || + if (!count || count != kline.count || + (require_ids && (!kline.confirmed || kline.first <= 0 || kline.last < kline.first || + first != static_cast(kline.first) || last != static_cast(kline.last) || last - first + 1 != count))) throw Error(20, "closed-minute trade coverage is not proven"); if (!(Decimal(open) == Decimal(kline.bar.open)) || !(Decimal(high) == Decimal(kline.bar.high)) || @@ -250,5 +307,4 @@ Bar normalized_bar(const Json& event) { bar.validate(); return bar; } -Json string_field(const std::string& value) { return Json::string(value); } } diff --git a/native/feed/src/core.hpp b/native/feed/src/core.hpp index 8fd745c..69b4439 100644 --- a/native/feed/src/core.hpp +++ b/native/feed/src/core.hpp @@ -2,6 +2,7 @@ #pragma once #include "json.hpp" #include +#include #include #include #include @@ -19,6 +20,7 @@ struct Error : std::runtime_error { struct Stopped {}; extern std::atomic stopping; void log(const std::string& level, const std::string& event, const Json& fields = Json::object({})); +void flush_log(std::chrono::milliseconds limit); std::string sha256(std::string_view bytes); std::string random_epoch(); std::int64_t timestamp(const std::string& token); @@ -71,7 +73,6 @@ struct Aggregate { }; Trade normalized_trade(const Json& event); Bar normalized_bar(const Json& event); -Json string_field(const std::string& value); struct Config { std::string venue = "binance", market = "spot", symbol, mode = "bars", state_dir; diff --git a/native/feed/src/main.cpp b/native/feed/src/main.cpp index f821b3f..bfb53e7 100644 --- a/native/feed/src/main.cpp +++ b/native/feed/src/main.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "binance.hpp" #include +#include #include #include #include @@ -15,6 +16,11 @@ std::uint64_t unsigned_value(const std::string& token) { if (parsed.ec != std::errc{} || parsed.ptr != token.data() + token.size()) throw pineforge::feed::Error(23, "invalid unsigned CLI value"); return value; } +// stdout's open file description is shared with whoever handed it to us: its flags are restored on exit. +struct StdoutFlags { + int saved = -1; + ~StdoutFlags() { if (saved >= 0) ::fcntl(STDOUT_FILENO, F_SETFL, saved); } +}; void help() { std::cout << "pineforge-feed warmup --venue binance --market spot --symbol SYMBOL --start UTC|MS --end UTC|MS --output FILE\n" "pineforge-feed run --venue binance --market spot --symbol SYMBOL --mode bars|ticks --state-dir DIR [--start UTC|MS | --resume] [--output-from INDEX]\n" @@ -22,12 +28,13 @@ void help() { "Testing/public origins: --rest-url ORIGIN --ws-url ORIGIN [--allow-insecure-http (loopback only)]\n"; } } -int main(int argc, char** argv) { +int run(int argc, char** argv) { using namespace pineforge::feed; + StdoutFlags restore; + std::signal(SIGTERM, &on_signal); + std::signal(SIGINT, &on_signal); + std::signal(SIGPIPE, SIG_IGN); try { - const auto error_flags = ::fcntl(STDERR_FILENO, F_GETFL); - if (error_flags < 0 || ::fcntl(STDERR_FILENO, F_SETFL, error_flags | O_NONBLOCK) != 0) - throw Error(22, "cannot configure bounded diagnostic writes"); if (argc == 2 && (std::string(argv[1]) == "--help" || std::string(argv[1]) == "-h")) { help(); return 0; } if (argc == 2 && std::string(argv[1]) == "--version") { check_runtime_curl(); std::cout << "pineforge-feed 0.1.0\n"; return 0; } if (argc < 2) throw Error(23, "choose warmup or run; see --help"); @@ -71,9 +78,6 @@ int main(int argc, char** argv) { check_runtime_curl(); validate_origin(config.rest_url, false, config.allow_insecure); validate_origin(config.ws_url, true, config.allow_insecure); - std::signal(SIGTERM, &on_signal); - std::signal(SIGINT, &on_signal); - std::signal(SIGPIPE, SIG_IGN); if (command == "warmup") { if (output.empty() || config.resume || !config.state_dir.empty() || seen.count("--output-from") || seen.count("--mode")) throw Error(23, "warmup requires --output and does not accept run-state options"); @@ -82,7 +86,9 @@ int main(int argc, char** argv) { if (config.state_dir.empty() || seen.count("--end") || !output.empty() || (!config.resume && seen.count("--output-from"))) throw Error(23, "run requires --state-dir; --output-from requires --resume"); const auto flags = ::fcntl(STDOUT_FILENO, F_GETFL); - if (flags < 0 || ::fcntl(STDOUT_FILENO, F_SETFL, flags | O_NONBLOCK) != 0) throw Error(22, "cannot configure bounded stdout drain"); + if (flags < 0) throw Error(22, "cannot configure bounded stdout drain"); + restore.saved = flags; + if (::fcntl(STDOUT_FILENO, F_SETFL, flags | O_NONBLOCK) != 0) throw Error(22, "cannot configure bounded stdout drain"); run_feed(config); } return 0; @@ -96,3 +102,9 @@ int main(int argc, char** argv) { return 23; } } + +int main(int argc, char** argv) { + const int code = run(argc, argv); + pineforge::feed::flush_log(std::chrono::seconds(2)); + return code; +} diff --git a/native/feed/src/state.cpp b/native/feed/src/state.cpp index bba8b9b..4ec6681 100644 --- a/native/feed/src/state.cpp +++ b/native/feed/src/state.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -88,15 +89,19 @@ State::State(const Config& config) : config_(config) { if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot sync rolled-back journal tail"); } else { if (config_.start < 0 || config_.start % 60000) throw Error(23, "new streams require a minute-aligned --start"); - if (std::filesystem::exists(config_.state_dir + "/cursor.json") || std::filesystem::exists(log_path())) + // An empty journal without a cursor is an initialization that crashed before its first cursor + // write: nothing was committed or published, so a fresh start may take the directory over. + if (std::filesystem::exists(config_.state_dir + "/cursor.json") || + (std::filesystem::exists(log_path()) && std::filesystem::file_size(log_path()) != 0)) throw Error(23, "state already exists; use --resume or an explicitly new state directory"); cursor_.epoch = random_epoch(); cursor_.start = cursor_.cut = config_.start; cursor_.prefix_hash = sha256(""); cursor_.overlap_hash = sha256(""); - journal_.reset(::open(log_path().c_str(), O_WRONLY | O_APPEND | O_CREAT | O_EXCL | O_CLOEXEC, 0600)); + journal_.reset(::open(log_path().c_str(), O_WRONLY | O_APPEND | O_CREAT | O_TRUNC | O_CLOEXEC, 0600)); if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot initialize durable journal"); persist(cursor_); + durable_ = cursor_; } if (journal_.get() < 0) throw Error(22, "cannot open durable journal"); if (config_.output_from > cursor_.message_index) throw Error(22, "output cursor exceeds the retained verified prefix"); @@ -144,6 +149,7 @@ void State::recover() { for (const auto& proof : proofs.items) cursor_.proofs.push_back(normalized_bar(proof)); if (!std::filesystem::exists(log_path()) || std::filesystem::file_size(log_path()) < cursor_.log_bytes) throw Error(21, "durable journal is shorter than its verified cursor"); + durable_ = cursor_; Cursor reconstructed; reconstructed.start = reconstructed.cut = cursor_.start; reconstructed.predecessor = cursor_.predecessor; @@ -171,12 +177,12 @@ void State::recover() { void State::anchor(const Trade& predecessor) { predecessor.validate(); - if (cursor_.predecessor || cursor_.message_index || predecessor.ts >= cursor_.start) + if (cursor_.predecessor || cursor_.message_index || !staged_.empty() || predecessor.ts >= cursor_.start) throw Error(20, "initial raw-trade predecessor does not prove the start fence"); auto next = cursor_; next.predecessor = predecessor; persist(next); - cursor_ = std::move(next); + durable_ = cursor_ = std::move(next); } void State::remember(const Json& event) { @@ -189,7 +195,7 @@ void State::remember(const Json& event) { } } -void State::commit(const std::string& line, const std::optional& proof) { +void State::stage(const std::string& line, const std::optional& proof) { if (line.size() > 4095 || line.find('\n') != std::string::npos) throw Error(23, "normalized message exceeds the atomic output bound"); auto next = cursor_; const auto event = parse_json(line); @@ -207,13 +213,27 @@ void State::commit(const std::string& line, const std::optional& proof) { ++next.message_index; next.prefix_hash = sha256(cursor_.prefix_hash + line + '\n'); next.overlap_hash = sha256(line + '\n'); - write_all(journal_.get(), line + '\n'); - if (::fsync(journal_.get()) != 0) throw Error(22, "cannot sync normalized message"); - persist(next); + staged_bytes_ += line; + staged_bytes_ += '\n'; + staged_.push_back(line); cursor_ = std::move(next); remember(event); } +std::vector State::flush() { + if (staged_.empty()) return {}; + // A failed append or cursor write leaves an uncommitted tail that only resume may truncate. + if (unusable_) throw Error(22, "durable journal is unusable after a failed commit"); + unusable_ = true; + write_all(journal_.get(), staged_bytes_); + if (::fsync(journal_.get()) != 0) throw Error(22, "cannot sync normalized messages"); + persist(cursor_); + unusable_ = false; + durable_ = cursor_; + staged_bytes_.clear(); + return std::exchange(staged_, {}); +} + void State::visit(std::uint64_t from, const std::function& visitor) const { if (from > cursor_.message_index) throw Error(22, "replay cursor is unavailable"); std::ifstream input(log_path(), std::ios::binary); @@ -221,16 +241,20 @@ void State::visit(std::uint64_t from, const std::function= deadline) throw Error(22, "retained prefix replay time budget exhausted"); - if (!std::getline(input, line) || line.size() > 4095 || bytes + line.size() + 1 > cursor_.log_bytes) + if (!std::getline(input, line) || line.size() > 4095 || bytes + line.size() + 1 > durable_.log_bytes) throw Error(21, "truncated or oversized verified message"); bytes += line.size() + 1; if (index >= from) visitor(line); ++index; } - if (bytes != cursor_.log_bytes || index != cursor_.message_index) throw Error(21, "verified message partition changed"); + if (bytes != durable_.log_bytes || index != durable_.message_index) throw Error(21, "verified message partition changed"); + for (const auto& staged : staged_) { + if (index >= from) visitor(staged); + ++index; + } } std::optional State::trade(std::uint64_t id) const { @@ -253,8 +277,8 @@ std::optional State::bar(std::int64_t ts) const { return found; } Json State::status() const { - return Json::object({{"epoch", Json::string(cursor_.epoch)}, {"message_index", number(cursor_.message_index)}, - {"emitted_seq", number(cursor_.seq)}, {"verified_cut", signed_number(cursor_.cut)}, - {"prefix_hash", Json::string(cursor_.prefix_hash)}, {"cursor", Json::string(config_.state_dir + "/cursor.json")}}); + return Json::object({{"epoch", Json::string(durable_.epoch)}, {"message_index", number(durable_.message_index)}, + {"emitted_seq", number(durable_.seq)}, {"verified_cut", signed_number(durable_.cut)}, + {"prefix_hash", Json::string(durable_.prefix_hash)}, {"cursor", Json::string(config_.state_dir + "/cursor.json")}}); } } diff --git a/native/feed/src/state.hpp b/native/feed/src/state.hpp index 1d7e5ee..a7ff0e3 100644 --- a/native/feed/src/state.hpp +++ b/native/feed/src/state.hpp @@ -3,6 +3,7 @@ #include "core.hpp" #include #include +#include namespace pineforge::feed { class Descriptor { @@ -25,12 +26,17 @@ struct Cursor { std::deque proofs; }; +// Messages are staged (verified, cursor advanced in memory), then group-committed: one journal append and +// fsync, one cursor replacement, and only then returned for publication. class State { Config config_; - Cursor cursor_; + Cursor durable_, cursor_; Descriptor lock_, journal_; std::deque recent_trades_; std::deque recent_bars_; + std::vector staged_; + std::string staged_bytes_; + bool unusable_ = false; std::string log_path() const; Json serialize(const Cursor& cursor) const; void persist(const Cursor& cursor) const; @@ -39,9 +45,12 @@ class State { public: explicit State(const Config& config); const Cursor& cursor() const { return cursor_; } + const Cursor& durable() const { return durable_; } const Config& config() const { return config_; } + std::size_t staged() const { return staged_.size(); } void anchor(const Trade& predecessor); - void commit(const std::string& line, const std::optional& proof = std::nullopt); + void stage(const std::string& line, const std::optional& proof = std::nullopt); + std::vector flush(); void visit(std::uint64_t from, const std::function& visitor) const; std::optional trade(std::uint64_t id) const; std::optional bar(std::int64_t ts) const; diff --git a/native/feed/src/transport.cpp b/native/feed/src/transport.cpp index f03d19e..6119619 100644 --- a/native/feed/src/transport.cpp +++ b/native/feed/src/transport.cpp @@ -6,8 +6,12 @@ #include #include #include +#include +#include +#include #include #include +#include #if LIBCURL_VERSION_NUM < 0x080e01 #error pineforge-feed requires libcurl 8.14.1 or newer @@ -57,15 +61,15 @@ void pause_until(std::chrono::steady_clock::time_point deadline) { } } struct Response { - std::string body; - std::uint64_t retry_after = 1; + std::string body, retry_after; std::uint64_t used_weight = 0; bool overflow = false; }; std::size_t body_callback(char* data, std::size_t size, std::size_t count, void* context) noexcept { auto* response = static_cast(context); const auto bytes = size * count; - if (bytes > 4 * 1024 * 1024 - response->body.size()) { response->overflow = true; return 0; } + // The JSON parser refuses documents above 1 MiB, so a larger body is refused here. + if (bytes > 1024 * 1024 - response->body.size()) { response->overflow = true; return 0; } try { response->body.append(data, bytes); return bytes; } catch (...) { response->overflow = true; return 0; } } @@ -73,22 +77,41 @@ std::size_t header_callback(char* data, std::size_t size, std::size_t count, voi const auto bytes = size * count; try { auto* response = static_cast(context); - std::string line(data, bytes); - std::transform(line.begin(), line.end(), line.begin(), [](unsigned char value) { return static_cast(std::tolower(value)); }); + const std::string line(data, bytes); const auto colon = line.find(':'); if (colon == std::string::npos) return bytes; + auto name = line.substr(0, colon); + std::transform(name.begin(), name.end(), name.begin(), [](unsigned char value) { return static_cast(std::tolower(value)); }); const auto first = line.find_first_not_of(" \t", colon + 1); - if (first == std::string::npos) return bytes; - std::uint64_t value = 0; - const auto result = std::from_chars(line.data() + first, line.data() + line.size(), value); - if (result.ec != std::errc{}) return bytes; - if (line.substr(0, colon) == "retry-after") response->retry_after = value; - if (line.substr(0, colon) == "x-mbx-used-weight-1m") response->used_weight = value; + const auto last = line.find_last_not_of(" \t\r\n"); + const auto value = first == std::string::npos || last < first ? std::string() : line.substr(first, last - first + 1); + if (name == "retry-after") response->retry_after = value; + if (name == "x-mbx-used-weight-1m") { + std::uint64_t used = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), used); + if (result.ec == std::errc{} && result.ptr == value.data() + value.size()) response->used_weight = used; + } } catch (...) {} return bytes; } } +void pause_for(std::chrono::milliseconds duration) { pause_until(std::chrono::steady_clock::now() + duration); } + +// Retry-After is either delta-seconds or an IMF-fixdate HTTP-date (RFC 9110, section 10.2.3). +std::uint64_t retry_after_seconds(const std::string& value, std::int64_t now) { + std::uint64_t seconds = 0; + const auto parsed = std::from_chars(value.data(), value.data() + value.size(), seconds); + if (!value.empty() && parsed.ec == std::errc{} && parsed.ptr == value.data() + value.size()) return seconds; + std::tm broken{}; + std::istringstream input(value); + input.imbue(std::locale::classic()); + input >> std::get_time(&broken, "%a, %d %b %Y %H:%M:%S GMT"); + if (value.size() != 29 || input.fail()) throw Error(23, "invalid Retry-After on a public rate-limit response"); + const auto when = static_cast(timegm(&broken)); + return when > now ? static_cast(when - now) : 0; +} + void check_runtime_curl() { static GlobalCurl initialization; (void)initialization; @@ -143,6 +166,7 @@ HttpClient::HttpClient(const Config& config) : config_(config) { interval == "HOUR" ? 3600000ULL : interval == "DAY" ? 86400000ULL : 0ULL; if (!unit || !multiple || multiple > 10000 || !ceiling || ceiling > 1000000000) throw Error(23, "unsupported public rate-limit interval or ceiling"); + // Spend at most half of each published ceiling (about 50% headroom). const auto pace = (unit * multiple * 2 + ceiling - 1) / ceiling; if (type == "REQUEST_WEIGHT") { weighted = true; @@ -181,9 +205,11 @@ Json HttpClient::get(const std::string& path, unsigned int weight) { if (status == 401 || status == 403 || status == 418 || status == 451 || (status >= 300 && status < 400)) throw Error(23, "public market-data access denied or redirected (HTTP " + std::to_string(status) + ")"); if (status == 429) { - log("warn", "rest_rate_limited", Json::object({{"retry_after", Json::number(std::to_string(response.retry_after))}})); - if (response.retry_after > 86400) throw Error(22, "public Retry-After exceeds the bounded retry window"); - next_request_ = std::chrono::steady_clock::now() + std::chrono::seconds(response.retry_after); + const auto wait = response.retry_after.empty() ? 1 : retry_after_seconds(response.retry_after, + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count()); + log("warn", "rest_rate_limited", Json::object({{"retry_after", Json::number(std::to_string(wait))}})); + if (wait > 86400) throw Error(22, "public Retry-After exceeds the bounded retry window"); + next_request_ = std::chrono::steady_clock::now() + std::chrono::seconds(wait); continue; } if (status == 404) throw Error(20, "public history is unavailable"); @@ -193,7 +219,8 @@ Json HttpClient::get(const std::string& path, unsigned int weight) { } if (status != 200) throw Error(23, "public market-data request rejected (HTTP " + std::to_string(status) + ")"); if (response.used_weight) log("info", "rest_weight", Json::object({{"used_weight_1m", Json::number(std::to_string(response.used_weight))}})); - if (weight_limit_ && response.used_weight >= weight_limit_ - weight_limit_ / 10) { + // The venue's used-weight header covers every client on this IP: past half the ceiling, wait for the window. + if (weight_limit_ && response.used_weight >= weight_limit_ / 2) { const auto elapsed = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count() % 60000; next_request_ = std::max(next_request_, std::chrono::steady_clock::now() + std::chrono::milliseconds(60500 - elapsed)); log("warn", "rest_shared_quota_pause"); @@ -223,6 +250,15 @@ void WebSocketPump::push(SourceMessage message) { queue_.push_back(std::move(message)); ready_.notify_one(); } +bool WebSocketPump::try_take(SourceMessage& message) { + std::lock_guard guard(mutex_); + if (failure_) std::rethrow_exception(failure_); + if (queue_.empty()) return false; + message = std::move(queue_.front()); + queue_.pop_front(); + queue_bytes_ -= message.text.size() + 32; + return true; +} SourceMessage WebSocketPump::take() { std::unique_lock guard(mutex_); while (queue_.empty() && !failure_) { @@ -262,7 +298,6 @@ void WebSocketPump::run() { continue; } failures = 0; - reconnect_.store(false); push({true, {}}); log("info", "websocket_connected"); const auto birth = std::chrono::steady_clock::now(); @@ -272,7 +307,7 @@ void WebSocketPump::run() { std::uint64_t frame_offset = 0; curl_socket_t socket = CURL_SOCKET_BAD; curl_easy_getinfo(handle.get(), CURLINFO_ACTIVESOCKET, &socket); - while (!stopped() && !reconnect_) { + while (!stopped()) { const auto now = std::chrono::steady_clock::now(); if (now - birth >= std::chrono::seconds(config_.reconnect_seconds) || now - last_text > std::chrono::seconds(75) || (assembling && now - message_birth > std::chrono::seconds(30))) break; diff --git a/native/feed/src/transport.hpp b/native/feed/src/transport.hpp index 95ad62c..2ced947 100644 --- a/native/feed/src/transport.hpp +++ b/native/feed/src/transport.hpp @@ -11,11 +11,13 @@ namespace pineforge::feed { void check_runtime_curl(); void validate_origin(const std::string& origin, bool websocket, bool allow_insecure); +void pause_for(std::chrono::milliseconds duration); +std::uint64_t retry_after_seconds(const std::string& value, std::int64_t now); class HttpClient { Config config_; std::chrono::steady_clock::time_point next_request_{}; - std::uint64_t weight_limit_ = 0, milliseconds_per_weight_ = 100, milliseconds_per_request_ = 0; + std::uint64_t weight_limit_ = 0, milliseconds_per_weight_ = 0, milliseconds_per_request_ = 0; public: explicit HttpClient(const Config& config); Json get(const std::string& path, unsigned int weight); @@ -27,7 +29,7 @@ struct SourceMessage { class WebSocketPump { Config config_; std::string url_; - std::atomic stopped_{false}, reconnect_{false}; + std::atomic stopped_{false}; std::mutex mutex_; std::condition_variable ready_; std::deque queue_; @@ -43,6 +45,6 @@ class WebSocketPump { WebSocketPump(const WebSocketPump&) = delete; WebSocketPump& operator=(const WebSocketPump&) = delete; SourceMessage take(); - void reconnect() { reconnect_.store(true); } + bool try_take(SourceMessage& message); }; } diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py index 9b8ba55..f18dfb8 100644 --- a/native/feed/tests/mock_venue.py +++ b/native/feed/tests/mock_venue.py @@ -3,10 +3,12 @@ import base64 import copy +import email.utils import fcntl import hashlib import http.server import json +import os import pathlib import signal import struct @@ -137,12 +139,15 @@ def __init__(self, scripts): self.trades = copy.deepcopy(TRADES) self.bars = copy.deepcopy(BARS) self.requests = [] + self.request_times = [] + self.agg_empty = 0 self.pongs = [] self.fail_status = 0 self.rate_once = False self.retry_after = 0 self.rate_limits = None self.auth_headers = [] + self.streams = [] self.lock = threading.Lock() self.errors = [] @@ -163,6 +168,7 @@ def do_GET(self): query = urllib.parse.parse_qs(parsed.query) with self.server.lock: self.server.requests.append(parsed.path) + self.server.request_times.append((parsed.path, time.monotonic())) self.server.auth_headers.extend( name for name in ("Authorization", "X-MBX-APIKEY") if self.headers.get(name) ) @@ -222,6 +228,9 @@ def do_GET(self): for identifier in sorted(self.server.trades) if start <= self.server.trades[identifier][0] <= end ] + if self.server.agg_empty: + self.server.agg_empty -= 1 + matching = [] rows = ( [] if not matching @@ -274,13 +283,16 @@ def respond(self, status, value, retry=False): self.send_header("Content-Length", str(len(payload))) self.send_header("Connection", "close") if retry: - self.send_header("Retry-After", str(self.server.retry_after)) + value = self.server.retry_after + self.send_header("Retry-After", value() if callable(value) else str(value)) self.end_headers() self.wfile.write(payload) self.close_connection = True def websocket(self, query): - assert query["streams"] == ["testusdt@trade/testusdt@kline_1m"] + assert query["streams"] in (["testusdt@trade/testusdt@kline_1m"], ["testusdt@kline_1m"]) + with self.server.lock: + self.server.streams.append(query["streams"][0]) key = self.headers["Sec-WebSocket-Key"] accept = base64.b64encode( hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest() @@ -314,6 +326,11 @@ def websocket(self, query): frame(self.connection, action[1], opcode=2) elif action[0] == "raw": frame(self.connection, action[1]) + elif action[0] == "revise_trade": + with self.server.lock: + row = list(self.server.trades[action[1]]) + row[2] = "0.10000001" + self.server.trades[action[1]] = tuple(row) elif action[0] == "revise_rest": with self.server.lock: row = list(self.server.bars[action[1]]) @@ -428,6 +445,7 @@ def test_bars_hole_is_healed_behind_closed_watermark(self): server = self.server([[candle(240000)]]) output, _ = self.run_feed(server, extra=["--max-messages", "3"]) self.assertEqual([event["bar"]["ts_open"] for event in output], [120000, 180000, 240000]) + self.assertEqual(server.streams, ["testusdt@kline_1m"]) def test_unhealable_bar_hole_stops_20(self): server = self.server([[candle(240000)]]) @@ -440,6 +458,7 @@ def test_raw_hole_and_reordering_are_healed_not_resequenced(self): [[trade(102), trade(100), trade(101), trade(103), candle(120000), candle(180000)]] ) output, result = self.run_feed(server, "ticks", ["--max-messages", "7"]) + self.assertEqual(server.streams, ["testusdt@trade/testusdt@kline_1m"]) self.assertEqual( [event["seq"] for event in output if event["type"] == "tick"], [100, 101, 102, 103, 104] ) @@ -511,7 +530,19 @@ def test_planned_24_hour_rotation_uses_overlap(self): self.assertGreaterEqual(server.connections, 2) def test_protocol_invalid_symbol_and_binary_stop_23(self): - for payload in [trade(100, s="OTHERUSDT"), ("binary", b"not-a-text-event"), ("raw", "{")]: + missing_count = candle(120000) + del missing_count["data"]["k"]["n"] + unknown_on_data_stream = trade(100, e="tradeV2") + for payload in [ + trade(100, s="OTHERUSDT"), + ("binary", b"not-a-text-event"), + ("raw", "{"), + trade(100, q="-0.10000000"), + trade(100, p="1e3"), + missing_count, + unknown_on_data_stream, + {"stream": "!notice", "data": {"E": 1}}, + ]: with self.subTest(payload=payload): server = self.server([[payload]]) state = self.directory / "state" @@ -522,17 +553,26 @@ def test_protocol_invalid_symbol_and_binary_stop_23(self): self.run_feed(server, "ticks", expected=23) def test_rest_access_failure_stops_23_no_key_sent(self): - server = self.server([[candle(240000)]]) - server.fail_status = 451 - output, _ = self.run_feed(server, expected=23) - self.assertEqual(output, []) - self.assertTrue( - all( - path in {"/stream", "/api/v3/klines", "/api/v3/exchangeInfo"} - for path in server.requests - ) - ) - self.assertEqual(server.auth_headers, []) + for status in (451, 418, 403): + with self.subTest(status=status): + state = self.directory / "state" + if state.exists(): + import shutil + + shutil.rmtree(state) + server = self.server([[candle(240000)]]) + server.fail_status = status + output, result = self.run_feed(server, expected=23) + self.assertEqual(output, []) + self.assertIn(f"HTTP {status}", result.stderr) + self.assertEqual(server.requests.count("/api/v3/exchangeInfo"), 1) + self.assertTrue( + all( + path in {"/stream", "/api/v3/klines", "/api/v3/exchangeInfo"} + for path in server.requests + ) + ) + self.assertEqual(server.auth_headers, []) def test_missing_rate_limit_metadata_is_refused(self): server = self.server([[candle(120000)]]) @@ -547,6 +587,130 @@ def test_excessive_retry_after_stops_instead_of_retrying_early(self): output, _ = self.run_feed(server, expected=22) self.assertEqual(output, []) + def test_reconnect_changed_raw_overlap_stops_21(self): + server = self.server( + [[trade(100), trade(101), ("revise_trade", 100), retirement()], [trade(102)]] + ) + output, result = self.run_feed(server, "ticks", expected=21) + self.assertEqual([event["seq"] for event in output], [100, 101]) + self.assertIn("reconnect raw overlap changed", result.stderr) + self.assertGreaterEqual(server.connections, 2) + + def test_multi_page_raw_hole_heals_contiguously(self): + server = self.server([[]]) + server.trades = {99: TRADES[99]} + for offset in range(2100): + server.trades[100 + offset] = (120001 + offset * 20, "10.00000000", "0.00000001") + server.bars = { + 120000: ( + "10.00000000", + "10.00000000", + "10.00000000", + "10.00000000", + "0.00002100", + 100, + 2199, + 2100, + ) + } + last = { + "stream": "testusdt@trade", + "data": { + "e": "trade", + "E": 162000, + "s": "TESTUSDT", + "t": 2199, + "p": "10.00000000", + "q": "0.00000001", + "T": 161981, + "m": False, + }, + } + closing = candle( + 120000, + o="10.00000000", + h="10.00000000", + l="10.00000000", + c="10.00000000", + v="0.00002100", + f=100, + L=2199, + n=2100, + ) + server.scripts = [[last, closing]] + output, _ = self.run_feed(server, "ticks", ["--max-messages", "2101"]) + self.assertEqual([event["seq"] for event in output[:-1]], list(range(100, 2200))) + self.assertEqual(output[-1], {"type": "time", "ts": 180000}) + self.assertGreaterEqual(server.requests.count("/api/v3/historicalTrades"), 4) + + def test_matched_time_regression_stops_23(self): + server = self.server([[trade(100), trade(101, T=120000)]]) + server.trades[101] = (120000, "11.20000000", "0.20000000") + output, result = self.run_feed(server, "ticks", expected=23) + self.assertEqual([event["seq"] for event in output], [100]) + self.assertIn("matched time regressed", result.stderr) + + def test_unknown_non_data_event_warns_and_continues(self): + notice = {"stream": "!venueNotice", "data": {"e": "venueNotice", "E": 1}} + server = self.server([[notice, candle(120000)]]) + output, result = self.run_feed(server, extra=["--max-messages", "1"]) + self.assertEqual(len(output), 1) + warnings = [json.loads(record) for record in result.stderr.splitlines()] + self.assertIn( + {"stream": "!venueNotice", "type": "venueNotice"}, + [ + {key: record.get(key) for key in ("stream", "type")} + for record in warnings + if record["event"] == "unknown_stream_event" and record["level"] == "warn" + ], + ) + + def test_initial_fence_retries_lagging_rest(self): + server = self.server([[trade(100), trade(101), trade(102), candle(120000)]]) + server.agg_empty = 1 + output, _ = self.run_feed(server, "ticks", ["--max-messages", "4"]) + self.assertEqual([event["type"] for event in output], ["tick", "tick", "tick", "time"]) + self.assertEqual(server.requests.count("/api/v3/aggTrades"), 2) + + def test_stdio_flags_restored_and_full_stderr_never_cuts_a_record(self): + server = self.server([[candle(120000)]]) + read_error, write_error = os.pipe() + read_output, write_output = os.pipe() + try: + if hasattr(fcntl, "F_SETPIPE_SZ"): + fcntl.fcntl(write_error, fcntl.F_SETPIPE_SZ, 4096) + flags = fcntl.fcntl(write_error, fcntl.F_GETFL) + fcntl.fcntl(write_error, fcntl.F_SETFL, flags | os.O_NONBLOCK) + filler = b'{"filler":true}\n' + try: + while True: + os.write(write_error, filler) + except BlockingIOError: + pass + fcntl.fcntl(write_error, fcntl.F_SETFL, flags) + process = subprocess.Popen( + self.command(server, extra=["--max-messages", "1"]), + stdout=write_output, + stderr=write_error, + ) + started = time.monotonic() + self.assertEqual(process.wait(timeout=10), 0) + self.assertLess(time.monotonic() - started, 8) + self.assertFalse(fcntl.fcntl(write_output, fcntl.F_GETFL) & os.O_NONBLOCK) + self.assertFalse(fcntl.fcntl(write_error, fcntl.F_GETFL) & os.O_NONBLOCK) + os.close(write_error) + os.close(write_output) + write_error = write_output = -1 + with os.fdopen(read_error, "rb") as errors, os.fdopen(read_output, "rb") as output: + read_error = read_output = -1 + for line in errors.read().split(b"\n")[:-1]: + self.assertIsInstance(json.loads(line), dict) + self.assertEqual(len(output.read().splitlines()), 1) + finally: + for descriptor in (read_error, write_error, read_output, write_output): + if descriptor >= 0: + os.close(descriptor) + def test_aggregate_trade_is_never_substituted_for_raw_prints(self): message = trade(100) message["data"]["e"] = "aggTrade" @@ -604,6 +768,27 @@ def test_sigterm_drain_is_bounded_with_unread_stdout(self): process.stdout.close() process.stderr.close() + def test_retry_after_seconds_and_http_date_are_honoured(self): + for retry_after in (2, lambda: email.utils.formatdate(time.time() + 4, usegmt=True)): + with self.subTest(retry_after=retry_after): + server = self.server([[candle(120000)]]) + server.rate_once = True + server.retry_after = retry_after + state = self.directory / "state" + if state.exists(): + import shutil + + shutil.rmtree(state) + output, result = self.run_feed(server, extra=["--max-messages", "1"]) + self.assertEqual(len(output), 1) + self.assertIn("rest_rate_limited", result.stderr) + limited, retried = [ + moment + for path, moment in server.request_times + if path == "/api/v3/exchangeInfo" + ] + self.assertGreaterEqual(retried - limited, 1.9) + def test_429_retry_after_preserves_prefix(self): server = self.server([[candle(240000)]]) server.rate_once = True @@ -654,6 +839,45 @@ def test_changed_committed_journal_is_refused(self): output, _ = self.run_feed(server, resume=True, expected=21) self.assertEqual(output, []) + def warmup(self, server): + port = server.server_address[1] + output = self.directory / "warmup.csv" + command = [ + BINARY, + "warmup", + "--venue", + "binance", + "--market", + "spot", + "--symbol", + "TESTUSDT", + "--start", + "120000", + "--end", + "240000", + "--output", + str(output), + "--rest-url", + f"http://127.0.0.1:{port}", + "--ws-url", + f"ws://127.0.0.1:{port}", + "--allow-insecure-http", + ] + return output, subprocess.run(command, capture_output=True, text=True, timeout=8) + + def test_warmup_rest_rows_must_equal_confirmed_ws_bars(self): + output, result = self.warmup(self.server([[candle(180000, v="0.70000000")]])) + self.assertEqual(result.returncode, 21, result.stderr) + self.assertFalse(output.exists()) + output, result = self.warmup(self.server([[candle(180000)]])) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + json.loads(pathlib.Path(str(output) + ".manifest.json").read_text())[ + "ws_cross_checked_bars" + ], + 1, + ) + def test_warmup_exclusive_cut_manifest_and_tokens(self): server = self.server([[candle(240000)]]) port = server.server_address[1] diff --git a/native/feed/tests/unit.cpp b/native/feed/tests/unit.cpp index d132315..5fe6967 100644 --- a/native/feed/tests/unit.cpp +++ b/native/feed/tests/unit.cpp @@ -53,7 +53,9 @@ struct SyntheticVenue final : Venue { }; std::string streams() const override { return "test@trade/test@kline_1m"; } VenueEvent decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } + std::size_t pages = 0; std::vector history(std::uint64_t from, std::size_t limit) override { + ++pages; std::vector result; for (std::size_t offset = 0; offset < limit; ++offset) { const auto found = trades.find(from + offset); @@ -64,11 +66,15 @@ struct SyntheticVenue final : Venue { return result; } std::uint64_t first_trade_id(std::int64_t minute) override { return static_cast(bars.at(minute).first); } + // REST rows carry no x=true and no f..L, exactly like the venue's klines endpoint. std::vector klines(std::int64_t start, std::int64_t end) override { std::vector result; for (auto minute = start; minute < end; minute += 60000) { if (!bars.count(minute)) throw Error(20, "synthetic missing minute"); - result.push_back(bars.at(minute)); + auto row = bars.at(minute); + row.first = row.last = -1; + row.confirmed = false; + result.push_back(row); } return result; } @@ -256,10 +262,14 @@ int main() { { State state(options); SyntheticVenue venue; - FeedSession session(state, venue, [](const auto&) {}); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); std::filesystem::create_directory(temporary.path + "/cursor.json.tmp"); expect(22, [&] { session.ingest(venue.close(120000)); }); - assert(state.cursor().message_index == 0); + assert(output.empty()); + assert(state.durable().message_index == 0); + expect(22, [&] { session.publish(); }); + assert(output.empty()); } std::filesystem::remove(temporary.path + "/cursor.json.tmp"); options.resume = true; @@ -280,6 +290,101 @@ int main() { expect(22, [&] { state.visit(0, [](const auto&) { std::this_thread::sleep_for(std::chrono::milliseconds(1100)); }); }); passed("replay_time_budget_stops_without_mutating_cursor"); } + { + Temporary temporary; + SyntheticVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.connected(); + session.ingest(venue.tick(100)); + session.ingest(venue.tick(101)); + assert(output.size() == 2); + venue.trades.at(100).qty = "0.10000001"; + expect(21, [&] { session.connected(); }); + assert(output.size() == 2 && state.durable().message_index == 2); + passed("reconnect_raw_overlap_change_stops_without_output"); + } + { + Temporary temporary; + SyntheticVenue venue; + venue.trades.erase(venue.trades.find(101), venue.trades.end()); + for (std::uint64_t id = 100; id < 2600; ++id) + venue.trades[id] = {id, 120001 + static_cast(id - 100) * 20, "10.00000000", "0.00000001"}; + venue.trades[2600] = {2600, 180001, "10.00000000", "0.00000001"}; + venue.bars.at(120000) = {{120000, "10.00000000", "10.00000000", "10.00000000", "10.00000000", "0.00002500"}, 100, 2599, 2500, true}; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.tick(2599)); + session.ingest(venue.close(120000)); + assert(output.size() == 2501 && venue.pages >= 4); + for (std::size_t index = 0; index < 2500; ++index) + assert(parse_json(output[index]).at("seq").integer() == 100 + index); + assert(parse_json(output.back()).at("type").text() == "time"); + passed("multi_page_raw_healing_stays_contiguous"); + } + { + Temporary temporary; + SyntheticVenue venue; + venue.trades.at(101).ts = 120000; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.tick(100)); + expect(23, [&] { session.ingest(venue.tick(101)); }); + assert(output.size() == 1 && state.durable().seq == 100); + passed("matched_time_regression_stops_23"); + } + { + Temporary temporary; + SyntheticVenue venue; + ++venue.bars.at(120000).count; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + expect(20, [&] { session.ingest(venue.close(180000)); }); + assert(state.durable().cut == 120000); + for (const auto& line : output) assert(parse_json(line).at("type").text() == "tick"); + venue.bars.at(120000).count = 3; + venue.trades.erase(103); + Temporary second; + State fresh(config(second, "ticks")); + FeedSession fenced(fresh, venue, [](const auto&) {}); + expect(20, [&] { fenced.ingest(venue.close(180000)); }); + assert(fresh.durable().cut == 120000); + passed("historical_closure_needs_rest_count_and_next_minute_fence"); + } + { + Temporary temporary; + State state(config(temporary, "bars")); + const Bar bar{120000, "10.10000000", "11.20000000", "9.90000000", "9.90000000", "0.60000000"}; + state.stage(bar.wire(), bar); + assert(state.staged() == 1 && state.cursor().message_index == 1 && state.durable().message_index == 0); + assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == 0); + std::size_t seen = 0; + state.visit(0, [&](const auto&) { ++seen; }); + assert(seen == 1 && state.bar(120000)); + const auto lines = state.flush(); + assert(lines.size() == 1 && state.staged() == 0 && state.durable().message_index == 1); + assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == lines.front().size() + 1); + passed("group_commit_stages_then_persists_before_publication"); + } + { + Temporary temporary; + { std::ofstream empty(temporary.path + "/events.jsonl"); } + { State state(config(temporary, "bars")); assert(state.durable().message_index == 0); } + Temporary second; + { std::ofstream used(second.path + "/events.jsonl"); used << "x\n"; } + expect(23, [&] { State state(config(second, "bars")); }); + passed("crashed_initialization_without_cursor_is_reinitialized"); + } + assert(retry_after_seconds("120", 0) == 120); + assert(retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT", 1445412475) == 5); + assert(retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT", 1445412490) == 0); + for (const auto* invalid : {"", "soon", "-1", "1.5", "Wed, 21 Oct 2015 07:28:00 UTC", "21 Oct 2015 07:28:00 GMT"}) + expect(23, [&] { retry_after_seconds(invalid, 0); }); + passed("retry_after_delta_seconds_and_http_date"); assert(timestamp("1970-01-01T00:02:00Z") == 120000); expect(23, [] { timestamp("2026-02-30T00:00:00Z"); }); passed("strict_utc_start_and_exclusive_cut"); From ec11077bd1325094d3be03a91d50fad4ca3addf7 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 4 Oct 2026 02:22:05 +0800 Subject: [PATCH 7/7] Compare live actions with the batch only on proven bars The batch replays warmup plus the minutes the tape proved complete. In tick mode the runner also sees the prints of the minute after the last `time` event and may act on them; those actions have no batch counterpart. Compare the actions on proven bars and report how many fell after them; the same-print replay still checks every runner action. Co-Authored-By: Claude Opus 5.5 (1M context) --- native/feed/tests/mock_venue.py | 21 ++++++++++++++++++++- native/feed/tests/public_e2e.py | 22 +++++++++++++++------- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py index f18dfb8..a4b9c8f 100644 --- a/native/feed/tests/mock_venue.py +++ b/native/feed/tests/mock_venue.py @@ -21,7 +21,7 @@ import urllib.parse from types import SimpleNamespace -from public_e2e import Soak, action_key +from public_e2e import Soak, action_key, proven_actions BINARY = str(pathlib.Path(sys.argv[1]).resolve()) sys.argv = [sys.argv[0]] @@ -386,6 +386,25 @@ def test_r_b2_maps_only_tick_action_timestamps(self): moved = dict(record, order=dict(record["order"], price=84816.5)) self.assertNotEqual(action_key(moved, "ticks"), action_key(batch, "ticks")) + def test_only_actions_on_proven_bars_are_compared(self): + record = { + "timestamp": 1791051480451, + "bar_index": 221, + "order": { + "id": "S", + "action": "sell", + "leg": "entry", + "contracts": 1.0, + "price": 84980.0, + "reduce_only": False, + "entry_incarnation": 15, + }, + } + open_minute = dict(record, bar_index=222) + inside, trailing = proven_actions([record, open_minute], "ticks", 222) + self.assertEqual(inside, [action_key(record, "ticks")]) + self.assertEqual(trailing, 1) + def setUp(self): self.temporary = tempfile.TemporaryDirectory(prefix="pineforge-feed-mock-") self.addCleanup(self.temporary.cleanup) diff --git a/native/feed/tests/public_e2e.py b/native/feed/tests/public_e2e.py index 021b61a..4142d8f 100644 --- a/native/feed/tests/public_e2e.py +++ b/native/feed/tests/public_e2e.py @@ -53,6 +53,14 @@ def action_key(record, mode): ) +def proven_actions(records, mode, window): + """Runner actions on the bars the batch replays, and how many fell after them. The batch + sees only proven minutes; ticks of a minute without its `time` event can still drive + actions in the runner, and those have no batch counterpart.""" + inside = [action_key(record, mode) for record in records if record["bar_index"] < window] + return inside, len(records) - len(inside) + + class Receiver(http.server.ThreadingHTTPServer): daemon_threads = True @@ -367,9 +375,9 @@ def validate(self, mode, duration, cursor, minutes): for record in rows(batch_actions) if record["origin_input_index"] >= 200 ] - actual_actions = [ - action_key(record, mode) for record in self.receiver.payloads.get(mode, []) - ] + actual_actions, trailing = proven_actions( + self.receiver.payloads.get(mode, []), mode, 200 + len(normalized) + ) event_ids = [record["event_id"] for record in self.receiver.payloads.get(mode, [])] assert len(event_ids) == len(set(event_ids)), "duplicate delivered action after restart" equal = expected_actions == actual_actions @@ -383,7 +391,8 @@ def validate(self, mode, duration, cursor, minutes): verdict = "PASS" if equal else "FAIL" self.receipt( f"{verdict} {mode} actions equal batch via run_backtest_full{mapping} " - f"batch={len(expected_actions)} runner={len(actual_actions)}" + f"batch={len(expected_actions)} runner={len(actual_actions)} " + f"after_last_proven_minute={trailing}" ) if mode == "ticks": name = "ticks-replay" @@ -405,14 +414,13 @@ def validate(self, mode, duration, cursor, minutes): assert raw == [ action_key(record, "bars") for record in self.receiver.payloads.get(name, []) ] - self.receipt( - f"PASS ticks same-print runner replay actions equal actions={len(actual_actions)}" - ) + self.receipt(f"PASS ticks same-print runner replay actions equal actions={len(raw)}") return { "duration_seconds": duration, "messages": cursor, "minutes": len(normalized), "actions": len(actual_actions), + "actions_after_last_proven_minute": trailing, "batch_actions_equal": equal, }