diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d9d1c1..a252b49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,13 @@ jobs: - run: pytest - if: matrix.python-version == '3.13' run: python -m build + - if: matrix.python-version == '3.13' + name: The sdist does not ship the native feed + run: | + if tar -tzf dist/*.tar.gz | grep -E '^[^/]+/native/'; then + echo "native/ must stay out of the Python sdist" >&2 + exit 1 + fi - if: matrix.python-version == '3.13' run: python -m twine check --strict dist/* diff --git a/.github/workflows/native-feed.yml b/.github/workflows/native-feed.yml new file mode 100644 index 0000000..1c7aeda --- /dev/null +++ b/.github/workflows/native-feed.yml @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +name: Native feed +on: + push: + paths: ['native/feed/**', '.github/workflows/native-feed.yml'] + pull_request: + paths: ['native/feed/**', '.github/workflows/native-feed.yml'] + workflow_dispatch: + +permissions: + contents: read + +jobs: + release: + name: Release (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm, macos-14] + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - name: Linux prerequisites + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y cmake ninja-build libssl-dev + - name: macOS prerequisites + if: runner.os == 'macOS' + run: | + brew install ninja openssl@3 + echo "OPENSSL_ROOT_DIR=$(brew --prefix openssl@3)" >> "$GITHUB_ENV" + - name: Pinned WebSocket curl + run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" + - name: Configure, build and test + run: | + export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" + export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}" + cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" + cmake --build build-feed -j4 + ctest --test-dir build-feed --output-on-failure + + asan-ubsan: + name: ASan + UBSan + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev clang-18 + - run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" + - name: Instrumented core and mock venue + env: + ASAN_OPTIONS: detect_leaks=1:abort_on_error=1 + UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 + run: | + export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" + cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_CXX_COMPILER=clang++-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ + -DPINEFORGE_FEED_SANITIZER=address,undefined + cmake --build build-feed -j4 + ctest --test-dir build-feed --output-on-failure + + tsan: + name: TSan + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev clang-18 + - run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" + - name: Threaded receive, healing and termination + env: + TSAN_OPTIONS: halt_on_error=1 + run: | + export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" + cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_CXX_COMPILER=clang++-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ + -DPINEFORGE_FEED_SANITIZER=thread + cmake --build build-feed -j4 + ctest --test-dir build-feed --output-on-failure diff --git a/native/feed/CMakeLists.txt b/native/feed/CMakeLists.txt new file mode 100644 index 0000000..ecfebfb --- /dev/null +++ b/native/feed/CMakeLists.txt @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +cmake_minimum_required(VERSION 3.20) +project(pineforge_feed VERSION 0.1.0 LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +find_package(CURL 8.14.1 REQUIRED) +find_package(OpenSSL REQUIRED COMPONENTS Crypto) +find_package(Threads REQUIRED) +include(CheckCXXSourceRuns) +get_target_property(CURL_CHECK_TARGET CURL::libcurl ALIASED_TARGET) +if(NOT CURL_CHECK_TARGET) + set(CURL_CHECK_TARGET CURL::libcurl) +endif() +set(CMAKE_REQUIRED_LIBRARIES ${CURL_CHECK_TARGET}) +check_cxx_source_runs("#include +#include +int main() { + const auto* info = curl_version_info(CURLVERSION_NOW); + if (!info || info->version_num < 0x080e01) return 1; + bool ws = false, wss = false; + for (const char* const* protocol = info->protocols; protocol && *protocol; ++protocol) { + ws |= std::strcmp(*protocol, \"ws\") == 0; + wss |= std::strcmp(*protocol, \"wss\") == 0; + } + return ws && wss ? 0 : 1; +}" PINEFORGE_CURL_WEBSOCKETS) +unset(CMAKE_REQUIRED_LIBRARIES) +if(NOT PINEFORGE_CURL_WEBSOCKETS) + message(FATAL_ERROR "pineforge-feed requires libcurl >= 8.14.1 with both ws and wss") +endif() + +file(SHA256 "${CMAKE_CURRENT_SOURCE_DIR}/vendor/json.hpp" JSON_SHA256) +if(NOT JSON_SHA256 STREQUAL "9431c7c311d74be1cea0e8d1a3942a33528ac11f6e67ee67d000cf4a8af7f830") + message(FATAL_ERROR "The attributed runner/json.hpp pin does not match") +endif() + +set(PINEFORGE_FEED_SANITIZER "" CACHE STRING "Sanitizer: address,undefined or thread") +if(PINEFORGE_FEED_SANITIZER) + if(NOT PINEFORGE_FEED_SANITIZER MATCHES "^(address,undefined|thread)$") + message(FATAL_ERROR "Unsupported sanitizer profile") + endif() + add_compile_options(-fsanitize=${PINEFORGE_FEED_SANITIZER} -fno-omit-frame-pointer -g) + add_link_options(-fsanitize=${PINEFORGE_FEED_SANITIZER}) +endif() + +add_library(pineforge_feed_core STATIC src/core.cpp src/state.cpp src/transport.cpp src/binance.cpp) +target_include_directories(pineforge_feed_core PUBLIC src vendor) +target_link_libraries(pineforge_feed_core PUBLIC CURL::libcurl OpenSSL::Crypto Threads::Threads) +target_compile_options(pineforge_feed_core PRIVATE -Wall -Wextra -Wpedantic -Werror) +add_executable(pineforge-feed src/main.cpp) +target_link_libraries(pineforge-feed PRIVATE pineforge_feed_core) +target_compile_options(pineforge-feed PRIVATE -Wall -Wextra -Wpedantic -Werror) +install(TARGETS pineforge-feed RUNTIME DESTINATION bin) + +include(CTest) +if(BUILD_TESTING) + add_executable(feed-unit tests/unit.cpp) + target_link_libraries(feed-unit PRIVATE pineforge_feed_core) + target_compile_options(feed-unit PRIVATE -Wall -Wextra -Wpedantic -Werror -UNDEBUG) + add_test(NAME feed-unit COMMAND feed-unit) + find_package(Python3 REQUIRED COMPONENTS Interpreter) + add_test(NAME feed-mock COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/mock_venue.py $) + set_tests_properties(feed-mock PROPERTIES TIMEOUT 180) +endif() diff --git a/native/feed/LICENSE b/native/feed/LICENSE new file mode 100644 index 0000000..f07d00b --- /dev/null +++ b/native/feed/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 PineForge contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/native/feed/NOTICE b/native/feed/NOTICE new file mode 100644 index 0000000..362639a --- /dev/null +++ b/native/feed/NOTICE @@ -0,0 +1,25 @@ +PineForge Feed +Apache License, Version 2.0. See LICENSE. + +This product includes runner/json.hpp from PineForge Engine: + PineForge Engine + Copyright 2026 PineForge contributors + This product includes software developed for the PineForge project. + The PineForge runtime is licensed under the Apache License, Version 2.0. + +The header is an unchanged copy from engine revision +227c2236e4769ae926d2c6e4811c37ff5119c4a0, path runner/json.hpp. +SHA-256: 9431c7c311d74be1cea0e8d1a3942a33528ac11f6e67ee67d000cf4a8af7f830 +Source: https://github.com/pineforge-4pass/pineforge-engine +No engine, strategy compiler, Eigen, or Python runtime is linked by the feed. + +Linked dependencies retain their upstream terms: + libcurl: curl license (MIT/X derivative), https://curl.se/docs/copyright.html + OpenSSL 3.x: Apache License, Version 2.0, https://openssl.org +Preserve upstream licenses/notices when distributing bundled libraries. + +CI builds libcurl 8.14.1 from curl-8.14.1.tar.xz. +Archive SHA-256: f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd +Source: https://curl.se/download/curl-8.14.1.tar.xz + +Market-data access and redistribution remain subject to the venue's terms. diff --git a/native/feed/README.md b/native/feed/README.md new file mode 100644 index 0000000..2eff191 --- /dev/null +++ b/native/feed/README.md @@ -0,0 +1,251 @@ +# pineforge-feed + +An independent Apache-2.0 C++17 public-market-data adapter. It produces the +native `pineforge-live` runner's JSONL protocol; it does not load strategies, +place orders, access accounts, consume fills, or accept API keys. It is not part +of the Python package or wheel. This first release supports **Binance spot**, +`warmup` and `run`, with `bars` and raw `ticks`. There is no `serve` subcommand. + +## Build + +Requirements: CMake >= 3.20, a C++17 compiler, OpenSSL Crypto, threads, and +**libcurl >= 8.14.1 with both `ws` and `wss`**. Configure executes a linked +feature check, and every running binary checks the actually loaded libcurl. +Cross-compilation without an executable feature check is not qualified. +`vendor/json.hpp` is SHA-256 pinned; attribution and dependency pins are in +`NOTICE`. No engine linkage or additional C++ dependency is required. + +```sh +cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCURL_DIR=/path/to/qualified-curl/lib/cmake/CURL +cmake --build build-feed -j4 +ctest --test-dir build-feed --output-on-failure +``` + +`tests/build_ci_curl.sh PREFIX` builds the CI-pinned curl with WebSockets and +OpenSSL. The isolated `.github/workflows/native-feed.yml` defines Linux amd64, +Linux arm64 and macOS Release builds, ASan+UBSan, and a separate Linux TSan +build. The Python sdist excludes `native/` and the Python CI asserts it; the +wheel is unchanged. Python 3.9+ is used only for the synthetic mock tests; +public qualification uses Python 3.10+. + +## A verified warmup cut + +```sh +build-feed/pineforge-feed warmup --venue binance --market spot --symbol BTCUSDT \ + --start 2026-10-03T00:00:00Z --end 2026-10-03T03:20:00Z --output warmup.csv +build-feed/pineforge-feed run --venue binance --market spot --symbol BTCUSDT \ + --mode bars --state-dir feed-state --start 2026-10-03T03:20:00Z | + pineforge-live run --strategy strategy.so --warmup warmup.csv --mode bars \ + --script-tf 1 --feed - --ledger runner.sqlite3 --symbol BINANCE:BTCUSDT \ + --webhook-url https://receiver.example/actions +``` + +Times are minute-aligned Unix milliseconds or strict UTC RFC3339 seconds. +Warmup fetches contiguous REST 1m klines only behind an observed `x=true` WS +watermark. Its end is **exclusive** and must be the live start. It writes the +runner CSV columns `timestamp,open,high,low,close,volume` and an atomic +`warmup.csv.manifest.json` containing source, symbol, units, start, exclusive +cut, confirmation watermark, row count, and the CSV SHA-256. It refuses to +overwrite either output. Keep both files together; an interrupted two-file +publication without its manifest is not a qualified warmup. + +## Protocol and proof policy + +- Bars are `{"type":"bar","bar":{"ts_open":...,"o":...,"h":...,"l":...,"c":...,"v":...}}`. +- Raw ticks are `{"type":"tick","ts":...,"seq":...,"price":...,"qty":...}`. +- Completeness is `{"type":"time","ts":...}`: complete **strictly before** + that timestamp. It is never a heartbeat or a wall-clock assertion. +- Every message contains one event. No batches, control messages, diagnostics, + or status objects are sent on stdout. These immutable boundaries are the + runner's `--from-input` counting unit. + +Venue decimal string tokens become JSON number tokens unchanged, including +trailing zeros. Fixed-point string arithmetic reconciles OHLCV; no price or +quantity enters a binary float. Bounds are 96 token characters, 32 fractional +digits, and 128 arithmetic digits. Invalid/zero tick quantities fail closed. + +Bars mode subscribes to `kline_1m` only; ticks mode adds the raw `trade` +stream. Raw `t` is the positive sequence and matched `T` is time, not dispatch +`E`. The real predecessor fetched from REST proves the initial cut; aggregates +locate a raw start ID but are never emitted or expanded. Each new raw ID must be +exactly its predecessor plus one; a matched time that goes backwards along that +chain stops with 23. Holes/reordering heal through inclusive `historicalTrades` +`fromId` pages (maximum 1000); identical duplicates are discarded and +conflicting duplicates stop. There is no assumed all-time retention window. + +For the observed minute, `x=true`, the complete `f..L` range, `n`, and exact +OHLCV must all reconcile before `time = open + 60000`. Later-minute prints stay +buffered until that proof. A tick minute whose own `x=true` kline was missed +(it fell inside a reconnect or a catch-up) closes only on this proof: + +1. a later observed `x=true` watermark; +2. the contiguous raw-ID chain from the proven predecessor; +3. the first print of a later minute, fetched by ID, with ID exactly the + minute's last ID plus one (the fence); +4. the REST kline's exact `n` and OHLCV. + +A REST kline has no `x=true` and no `f..L`, and the feed does not fill them in. +Empty/ambiguous tick minutes deliberately stop rather than invent a fence. +Bars heal missing minutes from REST only behind the verified closed watermark. +Any discovered already-emitted bar revision stops; history is never rewritten. + +The subscribed data streams are strict: another event type on them, a +malformed trade or kline, or an unknown shape stops with 23. An event outside +the data streams with an unknown type (a new venue notice) is logged as a +structured `unknown_stream_event` warning and ignored; `serverShutdown` +reconnects. + +Curl answers server PING with matching-payload PONG. Reconnect occurs on +`serverShutdown`, transport loss/75-second source silence, or at 23h55 (before +the venue's 24-hour limit). The reader buffers during REST healing. Every new +connection re-fetches up to 1000 raw prints and 32 closed-bar proofs, requires +exact token/ID/time overlap, then continues the same stream epoch and sequence. +An altered duplicate discovered outside that window is checked against the +full retained normalized journal. Older, undiscovered revisions are not claimed +to be continuously polled. + +REST routes are restricted to public `exchangeInfo`, `historicalTrades`, +`aggTrades`, and `klines`. Current public weight/raw-request limits are read +from `exchangeInfo`, and requests are spaced to spend at most half of each +published ceiling (about 50% headroom; 20 ms per weight at 6000 per minute, so +500 ms per 1000-print page). The venue's `X-MBX-USED-WEIGHT-1M` header counts +every client on the IP: once it shows half of the 1-minute ceiling, the feed +waits for the next window. `429` honours `Retry-After` as delta-seconds or an +IMF-fixdate HTTP-date; a wait above 24 hours stops with 22 and an unparseable +value with 23. A ban/access rejection is not retried as anonymous trading +access. TLS verification is always enabled; redirects, URL credentials, and +non-origin overrides are refused. Explicit insecure overrides are +**loopback-only**, for tests. + +## Durability and restart + +One producer owns a state directory via `flock`. `events.jsonl` is append-only. +Messages are group-committed: every event made ready by the source messages +already queued (up to 1024 source messages, or 4096 events) is appended in one +write and fsynced once, then `cursor.json` is atomically replaced once and its +directory fsynced, and only **then** are those lines written to stdout, one +event per line as before. The cursor binds source origins, symbol/mode/units, +random stream epoch, initial/verified cut, venue ID, emitted sequence, message +index, byte count, predecessor, closed-minute proofs, SHA-256 prefix chain, +last-message hash, and fixed message partition. Its canonical document is +itself hashed. Resume verifies the retained committed prefix and fsyncs +truncation of an uncommitted crash tail. + +A pipe write is not a consumer acknowledgment. By default `--resume` replays the +whole verified origin prefix. If the runner has committed N messages: + +```sh +build-feed/pineforge-feed run --venue binance --market spot --symbol BTCUSDT \ + --mode bars --state-dir feed-state --resume --output-from N | + pineforge-live run --strategy strategy.so --warmup warmup.csv --mode bars \ + --script-tf 1 --feed - --ledger runner.sqlite3 --from-input N \ + --symbol BINANCE:BTCUSDT --webhook-url https://receiver.example/actions +``` + +Replace N with the runner's actual committed count; never use the producer's +possibly-ahead count. Bind warmup identity and stream epoch in orchestration. +An unavailable cursor or changed identity is refused. SIGTERM/SIGINT stops +intake, discards staged messages that were never committed or published, and +exits 0. A directory holding only an empty `events.jsonl` (a crash before the +first cursor write) is taken over by a fresh `--start`; other existing state +needs `--resume`. + +Stdout is nonblocking during `run` for a five-second stop drain, and its +original flags are restored on exit (a SIGKILL cannot restore them); a consumer +that has not acknowledged a committed message must replay it. Stderr's flags +are never changed: one writer thread writes whole JSON records of at most +`PIPE_BUF` bytes, so a pipe write is atomic and a record is never cut. A full +sink drops whole records, the next record reports `dropped_records`, and exit +waits at most 2 seconds for queued records. + +| Exit | Meaning | +|---|---| +| 0 | Requested clean stop / successful command | +| 20 | Unhealable history gap, ambiguous fence, exhausted source retries | +| 21 | Changed overlap, conflicting duplicate, revised bar, changed cursor/prefix | +| 22 | Replay/disk/queue/allocation budget or durable I/O failure | +| 23 | Unsupported input/source, permanent access, protocol/decimal/dependency failure, matched-time regression | + +Default replay disk budget: 256 MiB (`--max-log-bytes`); each prefix traversal: +60 seconds (`--max-replay-seconds`, maximum 3600); source and unconfirmed-print +queues: 16 MiB each (`--max-queue-bytes`); WS text: 1 MiB; REST body: 1 MiB. +Warmup: 100000 rows / 16 MiB. Exhaustion is a cursor-preserving stop, not silent +compaction or dropped events. Raising limits is an explicit operator choice. +`--max-messages` permits bounded qualification. `--reconnect-seconds` can shorten +the 23h55 rotation for synthetic tests, never extend it beyond that limit. + +### Throughput + +Storage sets the ceiling, so measure on the target disk. On a shared 16-vCPU +Linux test machine (ext4 root), committing each message separately (three +fsyncs per print) sustained about 120 messages/s. With group commit, the +loopback bench (synthetic contiguous prints through the real binary and the +mock venue, wall time including startup) ran 30,000 prints at 5,811, 11,212 and +10,842 messages/s in three runs on the same machine. The busiest BTCUSDT minute +of a recent 89-day sample averaged 655 prints/s, about one ninth of the slowest +run. + +### Run-time horizon per mode + +`--max-log-bytes` is a hard stop (22): this release neither rotates nor +checkpoints the journal. Journal segmentation and a checkpoint (prefix hash, +cut, sequence, current-minute aggregate, the 32 proofs) are planned together +with bounded replay windows. Until then the horizon is the budget divided by +the journal's growth: + +| Mode | Bytes per message | Messages | Horizon at the default 256 MiB | +|---|---|---|---| +| bars | about 138 | 1 per minute | about 3.7 years | +| ticks, BTCUSDT median day | about 92 | 2.92 M per day | about 24 hours | +| ticks, BTCUSDT busiest day | about 92 | 8.93 M per day | about 8 hours | + +Message sizes come from a public BTCUSDT capture; the daily print counts are +REST kline trade counts over 89 days. + +Raising the budget lengthens every whole-journal scan: at startup, on every +tick-mode reconnect, on a duplicate closed kline and on an old duplicate print, +each bounded by `--max-replay-seconds`. + +### Tick mode needs a liquid symbol + +75 seconds without any text frame forces a reconnect, and each reconnect costs +a weight-27 overlap check (one `historicalTrades` page and one `klines` call). +A minute without a print cannot be fenced, so tick mode stops with 20 there, +and again on every resume. Use bars mode for symbols that can go quiet for a +minute. + +### Restart policy + +Retry budgets are short by design: about 2 minutes of failed WebSocket +connects, or about 7 seconds of failed REST attempts, end with 20 and the +verified cursor retained. Run the feed under a supervisor (a systemd unit with +`Restart=on-failure`, or a container restart policy) that restarts on 20 with +backoff, as `--resume --output-from N` with the runner's committed count. +Exits 21 and 23 need an operator (changed history, protocol or access +failure): an automatic restart repeats the stop. 22 needs a larger budget, more +disk, or a consumer that reads stdout again. + +## Qualification and next adapters + +`tests/unit.cpp` and `tests/mock_venue.py` contain **synthetic** documented +Binance message shapes only: holes, reordering, duplicates, revisions, access, +reconnect overlap, fragmented text, PING/PONG, retirement, budgets, and recovery. +Never commit public exchange captures. `tests/public_e2e.py` explicitly opts +into public BTCUSDT capture, local receiver delivery, a hard-kill restart, +exact REST verification, bar batch action parity through `run_backtest_full`, +and tick same-print replay determinism. Tick actions are compared with the +batch under the predeclared R-B2 mapping: each action timestamp becomes +`floor(ts / script_tf) * script_tf` on both sides, because a tick fill carries +its print's time while the OHLC batch carries its modeled segment's clock. +Every other field stays exact, bar actions compare raw timestamps, and any +difference fails the run. Its batch observer is linked separately from the +read-only engine's `tests/native_live_equivalence_observer.cpp`; it is not a +feed dependency. + +`src/venue.hpp` is the adapter boundary. Later slices add OKX raw ticks/bars +(contract-unit conversion where needed), Bybit confirmed bars **without ticks**, +and USD-M confirmed bars / separately opted-in aggregate-print ticks. Coinbase +remains deferred. `serve` will add approved MIT CivetWeb only to its server +target, bounded client replay/queues, and cursor-bound fan-out; no server library +is linked in this slice. The engine executable remains `pineforge-live`. diff --git a/native/feed/src/binance.cpp b/native/feed/src/binance.cpp new file mode 100644 index 0000000..9f8ab38 --- /dev/null +++ b/native/feed/src/binance.cpp @@ -0,0 +1,445 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "binance.hpp" +#include +#include +#include +#include + +namespace pineforge::feed { +namespace { +const std::vector& array(const Json& value, std::size_t maximum) { + if (value.kind != Json::Kind::Array || value.items.size() > maximum) throw Error(23, "invalid public market-data array"); + return value.items; +} +bool boolean(const Json& value) { + if (value.kind != Json::Kind::Bool) throw Error(23, "confirmation must be a JSON boolean"); + return value.value == "true"; +} +std::string lower(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char letter) { return static_cast(std::tolower(letter)); }); + return value; +} +std::size_t trade_bytes(const Trade& trade) { return trade.price.size() + trade.qty.size() + 192; } +} + +Trade binance_trade(const Json& value, bool websocket) { + Trade trade{value.at(websocket ? "t" : "id").integer(), + value.at(websocket ? "T" : "time").integer(), + value.at(websocket ? "p" : "price").text(), value.at(websocket ? "q" : "qty").text()}; + trade.validate(); + return trade; +} +Kline binance_kline(const Json& value, bool websocket) { + Kline kline; + if (websocket) { + if (value.at("i").text() != "1m") throw Error(23, "only UTC one-minute klines are supported"); + kline.bar = {value.at("t").integer(), value.at("o").text(), value.at("h").text(), + value.at("l").text(), value.at("c").text(), value.at("v").text()}; + kline.bar.validate(); + if (value.at("T").integer() != kline.bar.ts + 59999) throw Error(23, "invalid inclusive one-minute kline close"); + kline.first = value.at("f").integer(); + kline.last = value.at("L").integer(); + kline.count = value.at("n").integer(); + kline.confirmed = boolean(value.at("x")); + if (kline.count && (kline.first <= 0 || kline.last < kline.first)) throw Error(23, "invalid kline trade-id fence"); + } else { + const auto& row = array(value, 12); + if (row.size() != 12) throw Error(23, "REST kline must have twelve fields"); + kline.bar = {row[0].integer(), row[1].text(), row[2].text(), row[3].text(), row[4].text(), row[5].text()}; + kline.bar.validate(); + if (row[6].integer() != kline.bar.ts + 59999) throw Error(23, "invalid REST kline close time"); + kline.count = row[8].integer(); + } + return kline; +} + +std::string BinanceSpot::streams() const { + const auto symbol = lower(config_.symbol); + return config_.mode == "ticks" ? symbol + "@trade/" + symbol + "@kline_1m" : symbol + "@kline_1m"; +} +VenueEvent BinanceSpot::decode(const std::string& message) const { + try { + const auto envelope = parse_json(message); + const auto* wrapped = envelope.find("data"); + const auto& data = wrapped ? *wrapped : envelope; + const auto* event = data.find("e"); + if (!event || event->kind != Json::Kind::String) throw Error(23, "unexpected public stream control message"); + const auto type = event->text(); + const auto stream = wrapped ? envelope.at("stream").text() : std::string(); + const auto symbol = lower(config_.symbol); + const bool trades = stream == symbol + "@trade", klines = stream == symbol + "@kline_1m"; + // Subscribed data streams and data event types are strict: the documented shape, or stop. + if (type == "trade" || type == "kline" || trades || klines) { + if (wrapped && !(trades && type == "trade") && !(klines && type == "kline")) + throw Error(23, "unsupported public stream event; aggregate prints are not raw trades"); + if (data.at("s").text() != config_.symbol) throw Error(23, "public stream symbol changed"); + if (type == "trade") return {VenueEvent::Kind::Trade, binance_trade(data, true), {}}; + const auto& row = data.at("k"); + if (row.at("s").text() != config_.symbol) throw Error(23, "kline symbol changed"); + return {VenueEvent::Kind::Kline, {}, binance_kline(row, true)}; + } + if (type == "serverShutdown") return {}; + // A venue notice outside the data streams carries no market data; a new type must not stop the feed. + log("warn", "unknown_stream_event", Json::object({{"stream", Json::string(stream)}, {"type", Json::string(type)}})); + return {}; + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid Binance spot message shape"); } +} + +std::vector BinanceSpot::history(std::uint64_t from, std::size_t limit) { + if (!from || !limit || limit > 1000) throw Error(23, "invalid raw-history page request"); + for (unsigned int attempt = 0; attempt < 3; ++attempt) { + const auto response = http_.get("/api/v3/historicalTrades?symbol=" + config_.symbol + "&fromId=" + std::to_string(from) + "&limit=" + std::to_string(limit), 25); + const auto& rows = array(response, limit); + if (rows.empty()) continue; + std::vector trades; + std::uint64_t expected = from; + for (const auto& row : rows) { + const auto trade = binance_trade(row, false); + if (trade.id != expected) throw Error(20, "raw-history page has a missing or nonunit venue trade ID"); + trades.push_back(trade); + if (expected == std::numeric_limits::max()) throw Error(23, "venue trade ID exhausted"); + ++expected; + } + return trades; + } + throw Error(20, "raw history exhausted or unavailable within the venue retention window"); +} +std::uint64_t BinanceSpot::first_trade_id(std::int64_t minute) { + // REST may lag the WebSocket print that triggered this lookup: retry like history() before stopping. + for (unsigned int attempt = 0; attempt < 3; ++attempt) { + if (attempt) pause_for(std::chrono::milliseconds(500 * attempt)); + const auto response = http_.get("/api/v3/aggTrades?symbol=" + config_.symbol + "&startTime=" + std::to_string(minute) + + "&endTime=" + std::to_string(minute + 59999) + "&limit=1", 4); + const auto& rows = array(response, 1); + if (rows.empty()) continue; + const auto first = rows.front().at("f").integer(); + const auto matched = rows.front().at("T").integer(); + if (first <= 1 || matched < minute || matched >= minute + 60000) throw Error(20, "initial raw-trade fence is unavailable or ambiguous"); + return first; + } + throw Error(20, "initial minute has no available raw-trade start fence"); +} +std::vector BinanceSpot::klines(std::int64_t start, std::int64_t end) { + if (start < 0 || start % 60000 || end <= start || end % 60000) throw Error(23, "invalid exclusive kline range"); + const auto response = http_.get("/api/v3/klines?symbol=" + config_.symbol + "&interval=1m&startTime=" + std::to_string(start) + + "&endTime=" + std::to_string(end - 1) + "&limit=1000", 2); + std::vector result; + auto expected = start; + for (const auto& row : array(response, 1000)) { + const auto kline = binance_kline(row, false); + if (kline.bar.ts != expected || kline.bar.ts >= end) throw Error(20, "historical klines have a missing or unexpected minute"); + expected += 60000; + result.push_back(kline); + } + if (result.empty()) throw Error(20, "historical minute is unavailable"); + return result; +} + +FeedSession::FeedSession(State& state, Venue& venue, std::function output) + : state_(state), venue_(venue), output_(std::move(output)) { + if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() == "tick") { + const auto trade = normalized_trade(event); + if (trade.ts >= state_.cursor().cut) aggregate_.add(trade); + } + }); +} + +void FeedSession::emit(const std::string& line, const std::optional& proof) { + if (stopping) throw Stopped{}; + state_.stage(line, proof); + ++new_messages_; + if (state_.config().max_messages && new_messages_ >= state_.config().max_messages) { + publish(); + throw Stopped{}; + } + if (state_.staged() >= 4096) publish(); +} +void FeedSession::publish() { + for (const auto& line : state_.flush()) output_(line); +} +void FeedSession::salvage() noexcept { + try { publish(); } + catch (...) { + try { log("error", "verified_messages_not_committed"); } catch (...) {} + } +} +void FeedSession::ingest(const VenueEvent& event) { + try { stage(event); } + catch (const Error&) { salvage(); throw; } + publish(); +} +void FeedSession::connected() { + publish(); + const auto& trades = state_.recent_trades(); + if (!trades.empty()) { + const auto fetched = venue_.history(trades.front().id, trades.size()); + if (fetched.size() != trades.size()) throw Error(20, "reconnect raw overlap is unavailable"); + for (std::size_t index = 0; index < trades.size(); ++index) + if (!(trades[index] == fetched[index])) throw Error(21, "reconnect raw overlap changed"); + } else if (state_.cursor().predecessor) { + const auto fetched = venue_.history(state_.cursor().predecessor->id, 1); + if (!(fetched.front() == *state_.cursor().predecessor)) throw Error(21, "initial predecessor overlap changed"); + } + const auto& proofs = state_.config().mode == "bars" ? state_.recent_bars() : state_.cursor().proofs; + if (!proofs.empty()) { + const auto fetched = venue_.klines(proofs.front().ts, proofs.back().ts + 60000); + if (fetched.size() != proofs.size()) throw Error(20, "reconnect closed-bar overlap is unavailable"); + std::map verified; + if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() != "tick") return; + const auto trade = normalized_trade(event); + const auto minute = trade.ts - trade.ts % 60000; + if (minute >= proofs.front().ts && minute <= proofs.back().ts) verified[minute].add(trade); + }); + for (std::size_t index = 0; index < proofs.size(); ++index) { + if (!(proofs[index] == fetched[index].bar)) throw Error(21, "reconnect already-emitted bar changed"); + if (state_.config().mode == "ticks") { + try { verified.at(fetched[index].bar.ts).reconcile(fetched[index], false); } + catch (const std::exception&) { throw Error(21, "reconnect closed-minute count or raw proof changed"); } + } + } + } + if (!replayed_) { + if (state_.config().resume) state_.visit(state_.config().output_from, output_); + replayed_ = true; + } + log("info", "overlap_verified", state_.status()); +} + +void FeedSession::anchor(std::uint64_t first) { + if (state_.cursor().predecessor) return; + if (first <= 1) throw Error(20, "raw predecessor cannot be proven"); + const auto predecessor = venue_.history(first - 1, 1).front(); + state_.anchor(predecessor); +} +std::uint64_t FeedSession::next_id() const { + const auto previous = state_.cursor().seq ? state_.cursor().seq : state_.cursor().predecessor ? state_.cursor().predecessor->id : 0; + if (!previous || previous == std::numeric_limits::max()) throw Error(20, "raw-trade cursor is not anchored"); + return previous + 1; +} +void FeedSession::put(const Trade& trade) { + trade.validate(); + if (trade.ts < state_.cursor().start) return; + if (state_.cursor().seq && trade.id <= state_.cursor().seq) { + const auto previous = state_.trade(trade.id); + if (!previous || !(*previous == trade)) throw Error(21, "duplicate raw trade conflicts with the immutable prefix"); + return; + } + const auto existing = pending_.find(trade.id); + if (existing != pending_.end()) { + if (!(existing->second == trade)) throw Error(21, "buffered raw duplicate conflicts"); + return; + } + const auto bytes = trade_bytes(trade); + if (bytes > state_.config().max_queue_bytes - std::min(pending_bytes_, state_.config().max_queue_bytes)) + throw Error(22, "unconfirmed raw-trade buffer overrun"); + pending_bytes_ += bytes; + pending_.emplace(trade.id, trade); +} +void FeedSession::emit_trade(const Trade& trade) { + if (trade.ts < state_.cursor().last_tick_ts) throw Error(23, "venue matched time regressed along the contiguous raw-trade chain"); + if (trade.id != next_id() || trade.ts < state_.cursor().cut || trade.ts >= state_.cursor().cut + 60000) + throw Error(20, "raw trade violates the complete chronological minute prefix"); + const auto buffered = pending_.find(trade.id); + if (buffered != pending_.end() && !(buffered->second == trade)) + throw Error(21, "REST healing changed a buffered raw trade"); + aggregate_.add(trade); + emit(trade.wire()); + if (buffered != pending_.end()) { + pending_bytes_ -= trade_bytes(buffered->second); + pending_.erase(buffered); + } +} +void FeedSession::heal_to(std::uint64_t target) { + while (next_id() <= target) { + log("warn", "raw_gap_healing", Json::object({{"from_id", Json::number(std::to_string(next_id()))}, {"through_id", Json::number(std::to_string(target))}})); + const auto page = venue_.history(next_id()); + bool progressed = false; + for (const auto& trade : page) { + if (trade.id > target) break; + const auto buffered = pending_.find(trade.id); + if (buffered != pending_.end() && !(buffered->second == trade)) throw Error(21, "REST raw healing conflicts with WebSocket data"); + emit_trade(trade); + progressed = true; + } + if (!progressed) throw Error(20, "raw gap cannot advance within available history"); + } +} +void FeedSession::drain() { + while (!pending_.empty()) { + const auto trade = pending_.begin()->second; + if (trade.ts >= state_.cursor().cut + 60000) break; + if (trade.id < next_id()) throw Error(21, "buffered cursor fell behind the immutable prefix"); + if (trade.id > next_id()) heal_to(trade.id - 1); + emit_trade(trade); + } +} +// A minute whose own x=true kline was missed closes only on proof: a later x=true watermark, the contiguous +// raw-ID chain from the proven predecessor, the first print of a later minute as the fence, and REST n plus +// exact OHLCV. The REST row has no x=true or f..L, and none is manufactured for it. +void FeedSession::historical_minute() { + if (state_.cursor().cut + 60000 > watermark_) throw Error(20, "historical tick closure lacks a confirmed WebSocket watermark"); + if (!state_.cursor().predecessor) anchor(venue_.first_trade_id(state_.cursor().start)); + const auto minute = state_.cursor().cut; + std::optional fence; + while (!fence) { + const auto page = venue_.history(next_id()); + for (const auto& trade : page) { + const auto buffered = pending_.find(trade.id); + if (buffered != pending_.end() && !(buffered->second == trade)) throw Error(21, "historical raw overlap changed"); + if (trade.ts >= minute + 60000) { + fence = trade; + break; + } + emit_trade(trade); + } + } + if (!aggregate_.count || fence->id != next_id() || fence->id != aggregate_.last + 1) + throw Error(20, "historical minute lacks a contiguous raw prefix and next-minute fence"); + put(*fence); + const auto kline = venue_.klines(minute, minute + 60000).front(); + aggregate_.reconcile(kline, false); + log("info", "historical_tick_fence_verified", Json::object({{"minute", Json::number(std::to_string(minute))}, + {"first_id", Json::number(std::to_string(aggregate_.first))}, {"last_id", Json::number(std::to_string(aggregate_.last))}})); + emit("{\"type\":\"time\",\"ts\":" + std::to_string(minute + 60000) + "}", kline.bar); + aggregate_ = Aggregate{}; +} + +void FeedSession::closed(const Kline& kline) { + if (!kline.confirmed) return; + const auto minute = kline.bar.ts; + watermark_ = std::max(watermark_, minute + 60000); + if (minute < state_.cursor().start) return; + if (minute < state_.cursor().cut) { + const auto old = state_.bar(minute); + if (old) { + if (!(*old == kline.bar)) throw Error(21, "already-emitted closed bar was revised"); + } + if (state_.config().mode == "ticks") { + Aggregate reconstructed; + state_.visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() == "tick") { + const auto trade = normalized_trade(event); + if (trade.ts >= minute && trade.ts < minute + 60000) reconstructed.add(trade); + } + }); + try { reconstructed.reconcile(kline, true); } + catch (const Error& failure) { + if (failure.code == 20 || failure.code == 21) throw Error(21, "already-emitted closed-minute fence was revised"); + throw; + } + } else if (!old) throw Error(21, "closed duplicate cannot be matched to the retained prefix"); + return; + } + if (state_.config().mode == "bars") { + while (state_.cursor().cut < minute) { + const auto page = venue_.klines(state_.cursor().cut, minute); + for (const auto& historical : page) { + if (historical.bar.ts + 60000 > watermark_) throw Error(20, "historical bar exceeds the confirmed watermark"); + emit(historical.bar.wire(), historical.bar); + } + } + emit(kline.bar.wire(), kline.bar); + return; + } + while (state_.cursor().cut < minute) historical_minute(); + if (!kline.count || kline.first <= 0 || kline.last < kline.first || + static_cast(kline.last - kline.first) + 1 != kline.count) + throw Error(20, "closed kline has ambiguous or noncontiguous raw bounds"); + anchor(static_cast(kline.first)); + if (state_.cursor().seq > static_cast(kline.last)) throw Error(21, "closed kline excludes an already-emitted trade"); + heal_to(static_cast(kline.last)); + aggregate_.reconcile(kline, true); + emit("{\"type\":\"time\",\"ts\":" + std::to_string(minute + 60000) + "}", kline.bar); + aggregate_ = Aggregate{}; + drain(); +} +void FeedSession::stage(const VenueEvent& event) { + if (event.kind == VenueEvent::Kind::Kline) { closed(event.kline); return; } + if (event.kind != VenueEvent::Kind::Trade || state_.config().mode != "ticks" || event.trade.ts < state_.cursor().start) return; + if (!state_.cursor().predecessor) anchor(venue_.first_trade_id(state_.cursor().start)); + put(event.trade); + drain(); +} + +void run_feed(const Config& config) { + State state(config); + try { + BinanceSpot venue(config); + WebSocketPump pump(config, venue.streams()); + FeedSession session(state, venue, &output_line); + for (;;) { + auto message = pump.take(); + try { + // Group commit: verify every source message already queued (bounded), then persist once. + for (std::size_t taken = 1;; ++taken) { + if (message.connected) session.connected(); + else session.stage(venue.decode(message.text)); + if (taken >= 1024 || !pump.try_take(message)) break; + } + } catch (const Error&) { session.salvage(); throw; } + session.publish(); + } + } catch (const Stopped&) { log("info", "stopped", state.status()); } + catch (...) { log("error", "verified_cursor_retained", state.status()); throw; } +} + +void warmup(const Config& config, const std::string& output) { + if (config.start < 0 || config.end <= config.start || config.start % 60000 || config.end % 60000) + throw Error(23, "warmup requires a nonempty minute-aligned exclusive range"); + if ((config.end - config.start) / 60000 > 100000) throw Error(22, "warmup range exceeds 100000 bars"); + if (std::filesystem::exists(output) || std::filesystem::exists(output + ".manifest.json")) throw Error(23, "warmup output already exists"); + BinanceSpot venue(config); + std::int64_t watermark = -1; + std::map confirmed; + { + WebSocketPump pump(config, venue.streams()); + while (watermark < config.end) { + const auto message = pump.take(); + if (message.connected) continue; + const auto event = venue.decode(message.text); + if (event.kind != VenueEvent::Kind::Kline || !event.kline.confirmed) continue; + const auto& bar = event.kline.bar; + watermark = std::max(watermark, bar.ts + 60000); + if (bar.ts < config.start || bar.ts >= config.end) continue; + const auto [found, inserted] = confirmed.emplace(bar.ts, bar); + if (!inserted && !(found->second == bar)) throw Error(21, "confirmed WebSocket bar was revised during warmup"); + } + } + std::string csv = "timestamp,open,high,low,close,volume\n"; + auto next = config.start; + std::uint64_t count = 0, cross_checked = 0; + while (next < config.end) { + for (const auto& kline : venue.klines(next, config.end)) { + if (kline.bar.ts != next || next + 60000 > watermark) throw Error(20, "warmup does not adjoin a verified closed-minute cut"); + const auto& bar = kline.bar; + // The newest rows come from REST moments after the WS close: the x=true bars in hand must match. + const auto observed = confirmed.find(bar.ts); + if (observed != confirmed.end()) { + if (!(observed->second == bar)) throw Error(21, "warmup REST row differs from the confirmed WebSocket bar"); + ++cross_checked; + } + csv += std::to_string(bar.ts) + ',' + bar.open + ',' + bar.high + ',' + bar.low + ',' + bar.close + ',' + bar.volume + '\n'; + if (csv.size() > 16 * 1024 * 1024) throw Error(22, "warmup output exceeds 16 MiB"); + next += 60000; + ++count; + } + } + const auto manifest = Json::object({{"schema", Json::string("pineforge-feed-warmup/v1")}, + {"venue", Json::string(config.venue)}, {"market", Json::string(config.market)}, {"symbol", Json::string(config.symbol)}, + {"source", Json::string(config.rest_url + "/api/v3/klines")}, {"interval", Json::string("1m")}, + {"units", Json::object({{"price", Json::string("quote/base")}, {"volume", Json::string("base")}, {"timestamp", Json::string("unix-ms")}})}, + {"start", Json::number(std::to_string(config.start))}, {"end_exclusive", Json::number(std::to_string(config.end))}, + {"cut", Json::number(std::to_string(config.end))}, {"confirmed_ws_watermark", Json::number(std::to_string(watermark))}, + {"bars", Json::number(std::to_string(count))}, {"ws_cross_checked_bars", Json::number(std::to_string(cross_checked))}, + {"sha256", Json::string(sha256(csv))}}); + atomic_file(output, csv); + atomic_file(output + ".manifest.json", manifest.dump() + '\n'); + log("info", "warmup_verified", manifest); +} +} diff --git a/native/feed/src/binance.hpp b/native/feed/src/binance.hpp new file mode 100644 index 0000000..8f5b06f --- /dev/null +++ b/native/feed/src/binance.hpp @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "state.hpp" +#include "transport.hpp" +#include "venue.hpp" +#include + +namespace pineforge::feed { +Trade binance_trade(const Json& trade, bool websocket); +Kline binance_kline(const Json& row, bool websocket); + +class BinanceSpot final : public Venue { + Config config_; + HttpClient http_; +public: + explicit BinanceSpot(const Config& config) : config_(config), http_(config) {} + std::string streams() const override; + VenueEvent decode(const std::string& message) const override; + std::vector history(std::uint64_t from, std::size_t limit = 1000) override; + std::uint64_t first_trade_id(std::int64_t minute) override; + std::vector klines(std::int64_t start, std::int64_t end) override; +}; + +class FeedSession { + State& state_; + Venue& venue_; + std::function output_; + Aggregate aggregate_; + std::map pending_; + std::size_t pending_bytes_ = 0; + std::int64_t watermark_ = -1; + std::uint64_t new_messages_ = 0; + bool replayed_ = false; + void emit(const std::string& line, const std::optional& proof = std::nullopt); + void anchor(std::uint64_t first); + std::uint64_t next_id() const; + void put(const Trade& trade); + void emit_trade(const Trade& trade); + void heal_to(std::uint64_t target); + void drain(); + void historical_minute(); + void closed(const Kline& kline); +public: + FeedSession(State& state, Venue& venue, std::function output); + void connected(); + // stage() verifies an event and stages its messages; publish() group-commits them, then writes them + // out; salvage() publishes what was already verified before a stop. ingest() is all three for one event. + void stage(const VenueEvent& event); + void publish(); + void salvage() noexcept; + void ingest(const VenueEvent& event); +}; +void run_feed(const Config& config); +void warmup(const Config& config, const std::string& output); +} diff --git a/native/feed/src/core.cpp b/native/feed/src/core.cpp new file mode 100644 index 0000000..a0dd869 --- /dev/null +++ b/native/feed/src/core.cpp @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "core.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +std::atomic stopping{false}; +static_assert(std::atomic::is_always_lock_free); + +namespace { +// Diagnostics go through one writer thread with ordinary blocking writes, so stderr's flags are never +// changed and no caller waits on a slow sink. A full queue drops whole records and counts them; a +// record is at most PIPE_BUF bytes, so a pipe write is atomic and never leaves half a line. +struct LogSink { + std::mutex mutex; + std::condition_variable ready, idle; + std::deque records; + std::size_t bytes = 0; + std::uint64_t dropped = 0; + bool writing = false; +}; +LogSink& sink() { + static LogSink* const shared = [] { + auto* created = new LogSink(); // never destroyed: the writer may still be blocked at exit + std::thread([created] { + std::unique_lock guard(created->mutex); + for (;;) { + created->ready.wait(guard, [created] { return !created->records.empty(); }); + const auto record = std::move(created->records.front()); + created->records.pop_front(); + created->bytes -= record.size(); + created->writing = true; + guard.unlock(); + std::size_t offset = 0; + while (offset < record.size()) { + const auto written = ::write(STDERR_FILENO, record.data() + offset, record.size() - offset); + if (written < 0 && errno == EINTR) continue; + if (written <= 0) break; + offset += static_cast(written); + } + guard.lock(); + created->writing = false; + if (created->records.empty()) created->idle.notify_all(); + } + }).detach(); + return created; + }(); + return *shared; +} +} + +void log(const std::string& level, const std::string& event, const Json& fields) { + auto record = fields; + record.members["level"] = Json::string(level); + record.members["event"] = Json::string(event); + record.members["ts"] = Json::number(std::to_string(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count())); + auto& shared = sink(); + std::lock_guard guard(shared.mutex); + if (shared.dropped) record.members["dropped_records"] = Json::number(std::to_string(shared.dropped)); + auto bytes = record.dump() + '\n'; + if (bytes.size() > PIPE_BUF) + bytes = Json::object({{"level", record.members["level"]}, {"event", record.members["event"]}, + {"ts", record.members["ts"]}, {"truncated", Json::boolean(true)}}).dump() + '\n'; + if (bytes.size() > 1024 * 1024 - shared.bytes) { ++shared.dropped; return; } + shared.dropped = 0; + shared.bytes += bytes.size(); + shared.records.push_back(std::move(bytes)); + shared.ready.notify_one(); +} + +void flush_log(std::chrono::milliseconds limit) { + auto& shared = sink(); + std::unique_lock guard(shared.mutex); + shared.idle.wait_for(guard, limit, [&] { return shared.records.empty() && !shared.writing; }); +} + +std::string sha256(std::string_view bytes) { + unsigned char digest[EVP_MAX_MD_SIZE]; + unsigned int size = 0; + if (EVP_Digest(bytes.data(), bytes.size(), digest, &size, EVP_sha256(), nullptr) != 1 || size != 32) + throw Error(23, "SHA-256 failed"); + constexpr char digits[] = "0123456789abcdef"; + std::string result; + for (unsigned int index = 0; index < size; ++index) { + result += digits[digest[index] >> 4]; + result += digits[digest[index] & 15]; + } + return result; +} + +std::string random_epoch() { + unsigned char bytes[32]; + if (RAND_bytes(bytes, sizeof(bytes)) != 1) throw Error(23, "stream epoch generation failed"); + return sha256(std::string_view(reinterpret_cast(bytes), sizeof(bytes))); +} + +std::int64_t timestamp(const std::string& token) { + std::int64_t result = -1; + const auto parsed = std::from_chars(token.data(), token.data() + token.size(), result); + if (parsed.ec == std::errc{} && parsed.ptr == token.data() + token.size()) { + if (result < 0 || result > std::numeric_limits::max() - 60000) + throw Error(23, "timestamp outside supported range"); + return result; + } + if (token.size() != 20 || token.back() != 'Z') throw Error(23, "use UTC RFC3339 seconds or Unix milliseconds"); + std::tm broken{}; + std::istringstream input(token); + input >> std::get_time(&broken, "%Y-%m-%dT%H:%M:%SZ"); + if (input.fail()) throw Error(23, "invalid UTC timestamp"); + const auto seconds = timegm(&broken); + char canonical[32]; + std::tm checked{}; + gmtime_r(&seconds, &checked); + std::strftime(canonical, sizeof(canonical), "%Y-%m-%dT%H:%M:%SZ", &checked); + if (seconds < 0 || token != canonical) throw Error(23, "invalid UTC timestamp"); + return static_cast(seconds) * 1000; +} + +void write_all(int descriptor, std::string_view bytes) { + std::size_t offset = 0; + while (offset < bytes.size()) { + const auto written = ::write(descriptor, bytes.data() + offset, bytes.size() - offset); + if (written < 0 && errno == EINTR) continue; + if (written <= 0) throw Error(22, "durable write failed"); + offset += static_cast(written); + } +} + +void output_line(std::string_view line) { + const auto bytes = std::string(line) + '\n'; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + std::size_t offset = 0; + while (offset < bytes.size()) { + const auto written = ::write(STDOUT_FILENO, bytes.data() + offset, bytes.size() - offset); + if (written < 0 && errno == EINTR) continue; + if (written < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) { + if (stopping && std::chrono::steady_clock::now() >= deadline) throw Stopped{}; + pollfd descriptor{STDOUT_FILENO, POLLOUT, 0}; + ::poll(&descriptor, 1, 100); + continue; + } + if (written <= 0) throw Error(22, "stdout unavailable; replay from the consumer's committed message index"); + offset += static_cast(written); + } +} + +void atomic_file(const std::string& path, std::string_view bytes) { + const auto temporary = path + ".tmp"; + const int descriptor = ::open(temporary.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0600); + if (descriptor < 0) throw Error(22, "cannot open atomic state file"); + try { + write_all(descriptor, bytes); + if (::fsync(descriptor) != 0) throw Error(22, "cannot sync atomic state file"); + if (::close(descriptor) != 0) throw Error(22, "cannot close atomic state file"); + } catch (...) { + ::close(descriptor); + throw; + } + if (::rename(temporary.c_str(), path.c_str()) != 0) throw Error(22, "cannot replace atomic state file"); + auto parent = std::filesystem::path(path).parent_path(); + if (parent.empty()) parent = "."; + const int directory = ::open(parent.c_str(), O_RDONLY | O_CLOEXEC); + if (directory < 0) throw Error(22, "cannot open state directory for sync"); + const auto status = ::fsync(directory); + ::close(directory); + if (status != 0) throw Error(22, "cannot sync state directory"); +} + +Decimal::Decimal(std::string coefficient, std::size_t scale) + : coefficient_(std::move(coefficient)), scale_(scale) { + const auto first = coefficient_.find_first_not_of('0'); + coefficient_ = first == std::string::npos ? "0" : coefficient_.substr(first); + while (scale_ && coefficient_.size() > 1 && coefficient_.back() == '0') { + coefficient_.pop_back(); + --scale_; + } + if (coefficient_ == "0") scale_ = 0; + if (coefficient_.size() > 128 || scale_ > 32) throw Error(23, "decimal arithmetic exceeds bounded precision"); +} + +Decimal::Decimal(std::string_view token) : Decimal("0", 0) { + if (token.empty() || token.size() > 96) throw Error(23, "invalid decimal token length"); + const auto dot = token.find('.'); + const auto integral = dot == std::string_view::npos ? token.size() : dot; + if (!integral || (integral > 1 && token.front() == '0') || (dot != std::string_view::npos && dot + 1 == token.size())) + throw Error(23, "invalid decimal token grammar"); + std::string coefficient; + for (std::size_t index = 0; index < token.size(); ++index) { + if (index == dot) continue; + if (token[index] < '0' || token[index] > '9') throw Error(23, "decimal must be an unsigned fixed-point token"); + coefficient += token[index]; + } + *this = Decimal(coefficient, dot == std::string_view::npos ? 0 : token.size() - dot - 1); +} + +int Decimal::compare(const Decimal& other) const { + if (zero() || other.zero()) return zero() == other.zero() ? 0 : zero() ? -1 : 1; + const auto scale = std::max(scale_, other.scale_); + const auto left = coefficient_ + std::string(scale - scale_, '0'); + const auto right = other.coefficient_ + std::string(scale - other.scale_, '0'); + if (left.size() != right.size()) return left.size() < right.size() ? -1 : 1; + return left == right ? 0 : left < right ? -1 : 1; +} + +Decimal Decimal::add(const Decimal& other) const { + const auto scale = std::max(scale_, other.scale_); + auto left = coefficient_ + std::string(scale - scale_, '0'); + auto right = other.coefficient_ + std::string(scale - other.scale_, '0'); + const auto width = std::max(left.size(), right.size()); + left.insert(0, width - left.size(), '0'); + right.insert(0, width - right.size(), '0'); + std::string sum(width, '0'); + int carry = 0; + for (std::size_t index = width; index > 0; --index) { + const int digit = left[index - 1] - '0' + right[index - 1] - '0' + carry; + sum[index - 1] = static_cast('0' + digit % 10); + carry = digit / 10; + } + if (carry) sum.insert(sum.begin(), '1'); + return Decimal(sum, scale); +} + +void Trade::validate() const { + if (!id || ts < 0 || ts > std::numeric_limits::max() - 60000 || Decimal(price).zero() || Decimal(qty).zero()) + throw Error(23, "invalid positive venue trade"); +} +std::string Trade::wire() const { + validate(); + return "{\"type\":\"tick\",\"ts\":" + std::to_string(ts) + ",\"seq\":" + std::to_string(id) + + ",\"price\":" + price + ",\"qty\":" + qty + "}"; +} +bool Trade::operator==(const Trade& other) const { + return id == other.id && ts == other.ts && price == other.price && qty == other.qty; +} +void Bar::validate() const { + if (ts < 0 || ts % 60000 || ts > std::numeric_limits::max() - 60000) + throw Error(23, "invalid minute timestamp"); + const Decimal opening(open), highest(high), lowest(low), closing(close), amount(volume); + if (opening.zero() || highest.zero() || lowest.zero() || closing.zero() || highest.compare(lowest) < 0 || + highest.compare(opening) < 0 || highest.compare(closing) < 0 || lowest.compare(opening) > 0 || lowest.compare(closing) > 0) + throw Error(23, "invalid OHLCV envelope"); + (void)amount; +} +std::string Bar::wire() const { + validate(); + return "{\"type\":\"bar\",\"bar\":{\"ts_open\":" + std::to_string(ts) + ",\"o\":" + open + + ",\"h\":" + high + ",\"l\":" + low + ",\"c\":" + close + ",\"v\":" + volume + "}}"; +} +bool Bar::operator==(const Bar& other) const { + return ts == other.ts && open == other.open && high == other.high && low == other.low && close == other.close && volume == other.volume; +} +void Aggregate::add(const Trade& trade) { + trade.validate(); + if (count && (last == std::numeric_limits::max() || trade.id != last + 1 || trade.ts < last_ts)) + throw Error(20, "noncontiguous minute trade prefix"); + if (!count) { + first = trade.id; + open = high = low = trade.price; + } + if (Decimal(trade.price).compare(Decimal(high)) > 0) high = trade.price; + if (Decimal(trade.price).compare(Decimal(low)) < 0) low = trade.price; + close = trade.price; + volume = volume.add(Decimal(trade.qty)); + last = trade.id; + last_ts = trade.ts; + ++count; +} +// A WebSocket proof (require_ids) needs the kline's own x=true and venue f..L range. A REST row carries +// neither: its caller proves closure with a later x=true watermark and a next-minute fence print. +void Aggregate::reconcile(const Kline& kline, bool require_ids) const { + if (!count || count != kline.count || + (require_ids && (!kline.confirmed || kline.first <= 0 || kline.last < kline.first || + first != static_cast(kline.first) || + last != static_cast(kline.last) || last - first + 1 != count))) + throw Error(20, "closed-minute trade coverage is not proven"); + if (!(Decimal(open) == Decimal(kline.bar.open)) || !(Decimal(high) == Decimal(kline.bar.high)) || + !(Decimal(low) == Decimal(kline.bar.low)) || !(Decimal(close) == Decimal(kline.bar.close)) || + !(volume == Decimal(kline.bar.volume))) throw Error(21, "closed-minute exact OHLCV reconciliation failed"); +} +Trade normalized_trade(const Json& event) { + Trade trade{event.at("seq").integer(), event.at("ts").integer(), event.at("price").value, event.at("qty").value}; + trade.validate(); + return trade; +} +Bar normalized_bar(const Json& event) { + const auto& value = event.at("bar"); + Bar bar{value.at("ts_open").integer(), value.at("o").value, value.at("h").value, + value.at("l").value, value.at("c").value, value.at("v").value}; + bar.validate(); + return bar; +} +} diff --git a/native/feed/src/core.hpp b/native/feed/src/core.hpp new file mode 100644 index 0000000..69b4439 --- /dev/null +++ b/native/feed/src/core.hpp @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "json.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +using Json = pineforge::live::Json; +using pineforge::live::parse_json; + +struct Error : std::runtime_error { + int code; + Error(int status, const std::string& reason) : std::runtime_error(reason), code(status) {} +}; +struct Stopped {}; +extern std::atomic stopping; +void log(const std::string& level, const std::string& event, const Json& fields = Json::object({})); +void flush_log(std::chrono::milliseconds limit); +std::string sha256(std::string_view bytes); +std::string random_epoch(); +std::int64_t timestamp(const std::string& token); +void write_all(int descriptor, std::string_view bytes); +void output_line(std::string_view line); +void atomic_file(const std::string& path, std::string_view bytes); + +class Decimal { + std::string coefficient_; + std::size_t scale_ = 0; + Decimal(std::string coefficient, std::size_t scale); +public: + explicit Decimal(std::string_view token); + bool zero() const { return coefficient_ == "0"; } + int compare(const Decimal& other) const; + Decimal add(const Decimal& other) const; + bool operator==(const Decimal& other) const { return compare(other) == 0; } +}; + +struct Trade { + std::uint64_t id = 0; + std::int64_t ts = 0; + std::string price; + std::string qty; + void validate() const; + std::string wire() const; + bool operator==(const Trade& other) const; +}; +struct Bar { + std::int64_t ts = 0; + std::string open, high, low, close, volume; + void validate() const; + std::string wire() const; + bool operator==(const Bar& other) const; +}; +struct Kline { + Bar bar; + std::int64_t first = -1; + std::int64_t last = -1; + std::uint64_t count = 0; + bool confirmed = false; +}; +struct Aggregate { + std::uint64_t first = 0, last = 0, count = 0; + std::int64_t last_ts = -1; + std::string open, high, low, close; + Decimal volume{"0"}; + void add(const Trade& trade); + void reconcile(const Kline& kline, bool require_ids) const; +}; +Trade normalized_trade(const Json& event); +Bar normalized_bar(const Json& event); + +struct Config { + std::string venue = "binance", market = "spot", symbol, mode = "bars", state_dir; + std::string rest_url = "https://api.binance.com"; + std::string ws_url = "wss://stream.binance.com:443"; + std::int64_t start = -1, end = -1; + bool resume = false, allow_insecure = false; + std::uint64_t output_from = 0, max_messages = 0; + std::uint64_t max_log_bytes = 256 * 1024 * 1024; + std::uint64_t max_replay_seconds = 60; + std::size_t max_queue_bytes = 16 * 1024 * 1024; + std::uint64_t reconnect_seconds = 23 * 3600 + 55 * 60; +}; +} diff --git a/native/feed/src/main.cpp b/native/feed/src/main.cpp new file mode 100644 index 0000000..bfb53e7 --- /dev/null +++ b/native/feed/src/main.cpp @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "binance.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { +void on_signal(int) { pineforge::feed::stopping.store(true); } +std::uint64_t unsigned_value(const std::string& token) { + std::uint64_t value = 0; + const auto parsed = std::from_chars(token.data(), token.data() + token.size(), value); + if (parsed.ec != std::errc{} || parsed.ptr != token.data() + token.size()) throw pineforge::feed::Error(23, "invalid unsigned CLI value"); + return value; +} +// stdout's open file description is shared with whoever handed it to us: its flags are restored on exit. +struct StdoutFlags { + int saved = -1; + ~StdoutFlags() { if (saved >= 0) ::fcntl(STDOUT_FILENO, F_SETFL, saved); } +}; +void help() { + std::cout << "pineforge-feed warmup --venue binance --market spot --symbol SYMBOL --start UTC|MS --end UTC|MS --output FILE\n" + "pineforge-feed run --venue binance --market spot --symbol SYMBOL --mode bars|ticks --state-dir DIR [--start UTC|MS | --resume] [--output-from INDEX]\n" + "Limits: --max-messages N --max-log-bytes N --max-replay-seconds N --max-queue-bytes N --reconnect-seconds N\n" + "Testing/public origins: --rest-url ORIGIN --ws-url ORIGIN [--allow-insecure-http (loopback only)]\n"; +} +} +int run(int argc, char** argv) { + using namespace pineforge::feed; + StdoutFlags restore; + std::signal(SIGTERM, &on_signal); + std::signal(SIGINT, &on_signal); + std::signal(SIGPIPE, SIG_IGN); + try { + if (argc == 2 && (std::string(argv[1]) == "--help" || std::string(argv[1]) == "-h")) { help(); return 0; } + if (argc == 2 && std::string(argv[1]) == "--version") { check_runtime_curl(); std::cout << "pineforge-feed 0.1.0\n"; return 0; } + if (argc < 2) throw Error(23, "choose warmup or run; see --help"); + const std::string command = argv[1]; + if (command != "run" && command != "warmup") throw Error(23, "only warmup and run are implemented"); + Config config; + std::string output; + std::set seen; + for (int index = 2; index < argc; ++index) { + const std::string key = argv[index]; + if (!seen.insert(key).second) throw Error(23, "duplicate CLI option: " + key); + if (key == "--resume") { config.resume = true; continue; } + if (key == "--allow-insecure-http") { config.allow_insecure = true; continue; } + if (index + 1 >= argc) throw Error(23, "CLI option needs a value: " + key); + const std::string value = argv[++index]; + if (key == "--venue") config.venue = value; + else if (key == "--market") config.market = value; + else if (key == "--symbol") config.symbol = value; + else if (key == "--mode") config.mode = value; + else if (key == "--state-dir") config.state_dir = value; + else if (key == "--start") config.start = timestamp(value); + else if (key == "--end") config.end = timestamp(value); + else if (key == "--output") output = value; + else if (key == "--output-from") config.output_from = unsigned_value(value); + else if (key == "--max-messages") config.max_messages = unsigned_value(value); + else if (key == "--max-log-bytes") config.max_log_bytes = unsigned_value(value); + else if (key == "--max-replay-seconds") config.max_replay_seconds = unsigned_value(value); + else if (key == "--max-queue-bytes") config.max_queue_bytes = unsigned_value(value); + else if (key == "--reconnect-seconds") config.reconnect_seconds = unsigned_value(value); + else if (key == "--rest-url") config.rest_url = value; + else if (key == "--ws-url") config.ws_url = value; + else throw Error(23, "unknown CLI option: " + key); + } + if (config.venue != "binance" || config.market != "spot") throw Error(23, "this release supports public Binance spot only"); + if (config.symbol.empty() || config.symbol.size() > 32 || config.symbol.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") != std::string::npos) + throw Error(23, "use an uppercase ASCII Binance symbol"); + if (config.mode != "bars" && config.mode != "ticks") throw Error(23, "mode must be bars or ticks"); + if (!config.max_log_bytes || !config.max_queue_bytes || !config.max_replay_seconds || config.max_replay_seconds > 3600 || + !config.reconnect_seconds || config.reconnect_seconds > 86100) + throw Error(23, "budgets must be positive; reconnect must precede the 24-hour connection limit"); + check_runtime_curl(); + validate_origin(config.rest_url, false, config.allow_insecure); + validate_origin(config.ws_url, true, config.allow_insecure); + if (command == "warmup") { + if (output.empty() || config.resume || !config.state_dir.empty() || seen.count("--output-from") || seen.count("--mode")) + throw Error(23, "warmup requires --output and does not accept run-state options"); + warmup(config, output); + } else { + if (config.state_dir.empty() || seen.count("--end") || !output.empty() || (!config.resume && seen.count("--output-from"))) + throw Error(23, "run requires --state-dir; --output-from requires --resume"); + const auto flags = ::fcntl(STDOUT_FILENO, F_GETFL); + if (flags < 0) throw Error(22, "cannot configure bounded stdout drain"); + restore.saved = flags; + if (::fcntl(STDOUT_FILENO, F_SETFL, flags | O_NONBLOCK) != 0) throw Error(22, "cannot configure bounded stdout drain"); + run_feed(config); + } + return 0; + } catch (const Stopped&) { log("info", "stopped"); return 0; } + catch (const std::bad_alloc&) { log("error", "allocation_exhausted"); return 22; } + catch (const Error& failure) { + log("error", "stop", Json::object({{"code", Json::number(std::to_string(failure.code))}, {"reason", Json::string(failure.what())}})); + return failure.code; + } catch (const std::exception& failure) { + log("error", "stop", Json::object({{"code", Json::number("23")}, {"reason", Json::string(failure.what())}})); + return 23; + } +} + +int main(int argc, char** argv) { + const int code = run(argc, argv); + pineforge::feed::flush_log(std::chrono::seconds(2)); + return code; +} diff --git a/native/feed/src/state.cpp b/native/feed/src/state.cpp new file mode 100644 index 0000000..4ec6681 --- /dev/null +++ b/native/feed/src/state.cpp @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "state.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +Descriptor::~Descriptor() { if (value_ >= 0) ::close(value_); } +void Descriptor::reset(int value) { + if (value_ >= 0) ::close(value_); + value_ = value; +} +namespace { +Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } +Json signed_number(std::int64_t value) { return Json::number(std::to_string(value)); } +bool digest_token(const std::string& token) { + return token.size() == 64 && token.find_first_not_of("0123456789abcdef") == std::string::npos; +} +void advance(Cursor& cursor, const Json& event, const std::string& mode) { + const auto type = event.at("type").text(); + if (type == "tick" && mode == "ticks") { + const auto trade = normalized_trade(event); + const auto previous = cursor.seq ? cursor.seq : cursor.predecessor ? cursor.predecessor->id : 0; + if (!previous || previous == std::numeric_limits::max() || trade.id != previous + 1 || + trade.ts < cursor.cut || trade.ts >= cursor.cut + 60000 || trade.ts < cursor.last_tick_ts) + throw Error(20, "tick does not extend the verified chronological minute prefix"); + cursor.seq = trade.id; + cursor.last_tick_ts = trade.ts; + } else if (type == "bar" && mode == "bars") { + const auto bar = normalized_bar(event); + if (bar.ts != cursor.cut) throw Error(20, "bar does not adjoin the verified cut"); + cursor.last_bar = bar.ts; + cursor.cut += 60000; + } else if (type == "time" && mode == "ticks") { + const auto cut = event.at("ts").integer(); + if (!cursor.seq || cut != cursor.cut + 60000 || cut <= cursor.last_tick_ts) + throw Error(20, "time does not advance exactly one proven closed minute"); + cursor.cut = cut; + } else throw Error(23, "journal contains an unsupported normalized message"); +} +} + +std::string State::log_path() const { return config_.state_dir + "/events.jsonl"; } + +Json State::serialize(const Cursor& cursor) const { + Json proofs; + proofs.kind = Json::Kind::Array; + for (const auto& bar : cursor.proofs) proofs.items.push_back(parse_json(bar.wire())); + return Json::object({ + {"schema", Json::string("pineforge-feed-cursor/v1")}, {"epoch", Json::string(cursor.epoch)}, + {"venue", Json::string(config_.venue)}, {"market", Json::string(config_.market)}, + {"symbol", Json::string(config_.symbol)}, {"mode", Json::string(config_.mode)}, + {"rest_origin", Json::string(config_.rest_url)}, {"ws_origin", Json::string(config_.ws_url)}, + {"units", Json::object({{"price", Json::string("quote/base")}, {"qty", Json::string("base")}, {"ts", Json::string("unix-ms")}})}, + {"start", signed_number(cursor.start)}, {"verified_cut", signed_number(cursor.cut)}, + {"venue_id", config_.mode == "ticks" ? number(cursor.seq) : signed_number(cursor.last_bar)}, + {"emitted_seq", number(cursor.seq)}, {"last_tick_ts", signed_number(cursor.last_tick_ts)}, + {"last_bar", signed_number(cursor.last_bar)}, {"message_index", number(cursor.message_index)}, + {"log_bytes", number(cursor.log_bytes)}, {"prefix_hash", Json::string(cursor.prefix_hash)}, + {"overlap_hash", Json::string(cursor.overlap_hash)}, + {"predecessor", cursor.predecessor ? parse_json(cursor.predecessor->wire()) : Json{}}, + {"closed_proofs", std::move(proofs)}, {"partition", Json::string("one-event-per-message")} + }); +} + +void State::persist(const Cursor& cursor) const { + const auto document = serialize(cursor); + atomic_file(config_.state_dir + "/cursor.json", Json::object({{"cursor", document}, + {"sha256", Json::string(sha256(document.dump()))}}).dump() + '\n'); +} + +State::State(const Config& config) : config_(config) { + std::error_code failure; + std::filesystem::create_directories(config_.state_dir, failure); + if (failure) throw Error(22, "cannot create state directory"); + lock_.reset(::open((config_.state_dir + "/producer.lock").c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600)); + if (lock_.get() < 0 || ::flock(lock_.get(), LOCK_EX | LOCK_NB) != 0) + throw Error(22, "state directory has another producer or cannot be locked"); + if (config_.resume) { + recover(); + journal_.reset(::open(log_path().c_str(), O_WRONLY | O_APPEND | O_CLOEXEC)); + if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot sync rolled-back journal tail"); + } else { + if (config_.start < 0 || config_.start % 60000) throw Error(23, "new streams require a minute-aligned --start"); + // An empty journal without a cursor is an initialization that crashed before its first cursor + // write: nothing was committed or published, so a fresh start may take the directory over. + if (std::filesystem::exists(config_.state_dir + "/cursor.json") || + (std::filesystem::exists(log_path()) && std::filesystem::file_size(log_path()) != 0)) + throw Error(23, "state already exists; use --resume or an explicitly new state directory"); + cursor_.epoch = random_epoch(); + cursor_.start = cursor_.cut = config_.start; + cursor_.prefix_hash = sha256(""); + cursor_.overlap_hash = sha256(""); + journal_.reset(::open(log_path().c_str(), O_WRONLY | O_APPEND | O_CREAT | O_TRUNC | O_CLOEXEC, 0600)); + if (journal_.get() < 0 || ::fsync(journal_.get()) != 0) throw Error(22, "cannot initialize durable journal"); + persist(cursor_); + durable_ = cursor_; + } + if (journal_.get() < 0) throw Error(22, "cannot open durable journal"); + if (config_.output_from > cursor_.message_index) throw Error(22, "output cursor exceeds the retained verified prefix"); +} + +void State::recover() { + try { + const auto path = config_.state_dir + "/cursor.json"; + if (!std::filesystem::exists(path) || std::filesystem::file_size(path) > 65536) + throw Error(22, "missing or oversized durable cursor"); + std::ifstream input(path); + const std::string bytes((std::istreambuf_iterator(input)), {}); + const auto envelope = parse_json(bytes); + const auto& document = envelope.at("cursor"); + if (envelope.at("sha256").text() != sha256(document.dump())) throw Error(21, "cursor digest mismatch"); + if (document.at("schema").text() != "pineforge-feed-cursor/v1" || document.at("partition").text() != "one-event-per-message") + throw Error(23, "unsupported cursor schema or message partition"); + for (const auto& item : {std::make_pair("venue", config_.venue), {"market", config_.market}, {"symbol", config_.symbol}, + {"mode", config_.mode}, {"rest_origin", config_.rest_url}, {"ws_origin", config_.ws_url}}) + if (document.at(item.first).text() != item.second) throw Error(21, "resume source identity changed"); + const auto& units = document.at("units"); + if (units.at("price").text() != "quote/base" || units.at("qty").text() != "base" || units.at("ts").text() != "unix-ms") + throw Error(21, "resume units changed"); + cursor_.epoch = document.at("epoch").text(); + cursor_.prefix_hash = document.at("prefix_hash").text(); + cursor_.overlap_hash = document.at("overlap_hash").text(); + if (!digest_token(cursor_.epoch) || !digest_token(cursor_.prefix_hash) || !digest_token(cursor_.overlap_hash)) + throw Error(21, "invalid cursor hash or epoch"); + cursor_.start = document.at("start").integer(); + cursor_.cut = document.at("verified_cut").integer(); + cursor_.seq = document.at("emitted_seq").integer(); + cursor_.last_tick_ts = document.at("last_tick_ts").integer(); + cursor_.last_bar = document.at("last_bar").integer(); + if ((config_.mode == "ticks" && document.at("venue_id").integer() != cursor_.seq) || + (config_.mode == "bars" && document.at("venue_id").integer() != cursor_.last_bar)) + throw Error(21, "venue ID does not match the emitted cursor"); + cursor_.message_index = document.at("message_index").integer(); + cursor_.log_bytes = document.at("log_bytes").integer(); + if (cursor_.start < 0 || cursor_.start % 60000 || cursor_.cut < cursor_.start || cursor_.cut % 60000 || + cursor_.log_bytes > config_.max_log_bytes || (config_.start >= 0 && config_.start != cursor_.start)) + throw Error(21, "invalid or changed resume cut"); + if (document.at("predecessor").kind != Json::Kind::Null) cursor_.predecessor = normalized_trade(document.at("predecessor")); + const auto& proofs = document.at("closed_proofs"); + if (proofs.kind != Json::Kind::Array || proofs.items.size() > 32) throw Error(21, "invalid closed-minute proof window"); + for (const auto& proof : proofs.items) cursor_.proofs.push_back(normalized_bar(proof)); + if (!std::filesystem::exists(log_path()) || std::filesystem::file_size(log_path()) < cursor_.log_bytes) + throw Error(21, "durable journal is shorter than its verified cursor"); + durable_ = cursor_; + Cursor reconstructed; + reconstructed.start = reconstructed.cut = cursor_.start; + reconstructed.predecessor = cursor_.predecessor; + reconstructed.prefix_hash = reconstructed.overlap_hash = sha256(""); + visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + advance(reconstructed, event, config_.mode); + reconstructed.prefix_hash = sha256(reconstructed.prefix_hash + line + '\n'); + reconstructed.overlap_hash = sha256(line + '\n'); + reconstructed.log_bytes += line.size() + 1; + ++reconstructed.message_index; + remember(event); + }); + if (reconstructed.prefix_hash != cursor_.prefix_hash || reconstructed.overlap_hash != cursor_.overlap_hash || + reconstructed.log_bytes != cursor_.log_bytes || reconstructed.message_index != cursor_.message_index || + reconstructed.seq != cursor_.seq || reconstructed.cut != cursor_.cut || reconstructed.last_bar != cursor_.last_bar || + reconstructed.last_tick_ts != cursor_.last_tick_ts) + throw Error(21, "durable journal prefix does not match its cursor"); + if (::truncate(log_path().c_str(), static_cast(cursor_.log_bytes)) != 0) + throw Error(22, "cannot roll back an uncommitted journal tail"); + config_.start = cursor_.start; + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(21, "invalid durable cursor or journal"); } +} + +void State::anchor(const Trade& predecessor) { + predecessor.validate(); + if (cursor_.predecessor || cursor_.message_index || !staged_.empty() || predecessor.ts >= cursor_.start) + throw Error(20, "initial raw-trade predecessor does not prove the start fence"); + auto next = cursor_; + next.predecessor = predecessor; + persist(next); + durable_ = cursor_ = std::move(next); +} + +void State::remember(const Json& event) { + if (event.at("type").text() == "tick") { + recent_trades_.push_back(normalized_trade(event)); + if (recent_trades_.size() > 1000) recent_trades_.pop_front(); + } else if (event.at("type").text() == "bar") { + recent_bars_.push_back(normalized_bar(event)); + if (recent_bars_.size() > 32) recent_bars_.pop_front(); + } +} + +void State::stage(const std::string& line, const std::optional& proof) { + if (line.size() > 4095 || line.find('\n') != std::string::npos) throw Error(23, "normalized message exceeds the atomic output bound"); + auto next = cursor_; + const auto event = parse_json(line); + advance(next, event, config_.mode); + if (proof) { + proof->validate(); + if (proof->ts + 60000 != next.cut) throw Error(23, "closed-minute proof does not match the committed cut"); + next.proofs.push_back(*proof); + if (next.proofs.size() > 32) next.proofs.pop_front(); + } + if (next.message_index == std::numeric_limits::max() || + line.size() + 1 > config_.max_log_bytes - std::min(cursor_.log_bytes, config_.max_log_bytes)) + throw Error(22, "retained replay budget exhausted"); + next.log_bytes += line.size() + 1; + ++next.message_index; + next.prefix_hash = sha256(cursor_.prefix_hash + line + '\n'); + next.overlap_hash = sha256(line + '\n'); + staged_bytes_ += line; + staged_bytes_ += '\n'; + staged_.push_back(line); + cursor_ = std::move(next); + remember(event); +} + +std::vector State::flush() { + if (staged_.empty()) return {}; + // A failed append or cursor write leaves an uncommitted tail that only resume may truncate. + if (unusable_) throw Error(22, "durable journal is unusable after a failed commit"); + unusable_ = true; + write_all(journal_.get(), staged_bytes_); + if (::fsync(journal_.get()) != 0) throw Error(22, "cannot sync normalized messages"); + persist(cursor_); + unusable_ = false; + durable_ = cursor_; + staged_bytes_.clear(); + return std::exchange(staged_, {}); +} + +void State::visit(std::uint64_t from, const std::function& visitor) const { + if (from > cursor_.message_index) throw Error(22, "replay cursor is unavailable"); + std::ifstream input(log_path(), std::ios::binary); + if (!input) throw Error(22, "cannot read retained normalized log"); + std::uint64_t index = 0, bytes = 0; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(config_.max_replay_seconds); + std::string line; + while (bytes < durable_.log_bytes) { + if (stopping) throw Stopped{}; + if (std::chrono::steady_clock::now() >= deadline) throw Error(22, "retained prefix replay time budget exhausted"); + if (!std::getline(input, line) || line.size() > 4095 || bytes + line.size() + 1 > durable_.log_bytes) + throw Error(21, "truncated or oversized verified message"); + bytes += line.size() + 1; + if (index >= from) visitor(line); + ++index; + } + if (bytes != durable_.log_bytes || index != durable_.message_index) throw Error(21, "verified message partition changed"); + for (const auto& staged : staged_) { + if (index >= from) visitor(staged); + ++index; + } +} + +std::optional State::trade(std::uint64_t id) const { + for (const auto& trade : recent_trades_) if (trade.id == id) return trade; + std::optional found; + visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() == "tick" && event.at("seq").integer() == id) found = normalized_trade(event); + }); + return found; +} +std::optional State::bar(std::int64_t ts) const { + for (const auto& bar : recent_bars_) if (bar.ts == ts) return bar; + for (const auto& bar : cursor_.proofs) if (bar.ts == ts) return bar; + std::optional found; + visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() == "bar" && event.at("bar").at("ts_open").integer() == ts) found = normalized_bar(event); + }); + return found; +} +Json State::status() const { + return Json::object({{"epoch", Json::string(durable_.epoch)}, {"message_index", number(durable_.message_index)}, + {"emitted_seq", number(durable_.seq)}, {"verified_cut", signed_number(durable_.cut)}, + {"prefix_hash", Json::string(durable_.prefix_hash)}, {"cursor", Json::string(config_.state_dir + "/cursor.json")}}); +} +} diff --git a/native/feed/src/state.hpp b/native/feed/src/state.hpp new file mode 100644 index 0000000..a7ff0e3 --- /dev/null +++ b/native/feed/src/state.hpp @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "core.hpp" +#include +#include +#include + +namespace pineforge::feed { +class Descriptor { + int value_ = -1; +public: + Descriptor() = default; + explicit Descriptor(int value) : value_(value) {} + ~Descriptor(); + Descriptor(const Descriptor&) = delete; + Descriptor& operator=(const Descriptor&) = delete; + int get() const { return value_; } + void reset(int value); +}; + +struct Cursor { + std::string epoch, prefix_hash, overlap_hash; + std::uint64_t message_index = 0, log_bytes = 0, seq = 0; + std::int64_t start = 0, cut = 0, last_tick_ts = -1, last_bar = -1; + std::optional predecessor; + std::deque proofs; +}; + +// Messages are staged (verified, cursor advanced in memory), then group-committed: one journal append and +// fsync, one cursor replacement, and only then returned for publication. +class State { + Config config_; + Cursor durable_, cursor_; + Descriptor lock_, journal_; + std::deque recent_trades_; + std::deque recent_bars_; + std::vector staged_; + std::string staged_bytes_; + bool unusable_ = false; + std::string log_path() const; + Json serialize(const Cursor& cursor) const; + void persist(const Cursor& cursor) const; + void recover(); + void remember(const Json& event); +public: + explicit State(const Config& config); + const Cursor& cursor() const { return cursor_; } + const Cursor& durable() const { return durable_; } + const Config& config() const { return config_; } + std::size_t staged() const { return staged_.size(); } + void anchor(const Trade& predecessor); + void stage(const std::string& line, const std::optional& proof = std::nullopt); + std::vector flush(); + void visit(std::uint64_t from, const std::function& visitor) const; + std::optional trade(std::uint64_t id) const; + std::optional bar(std::int64_t ts) const; + const std::deque& recent_trades() const { return recent_trades_; } + const std::deque& recent_bars() const { return recent_bars_; } + Json status() const; +}; +} diff --git a/native/feed/src/transport.cpp b/native/feed/src/transport.cpp new file mode 100644 index 0000000..6119619 --- /dev/null +++ b/native/feed/src/transport.cpp @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "transport.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if LIBCURL_VERSION_NUM < 0x080e01 +#error pineforge-feed requires libcurl 8.14.1 or newer +#endif + +namespace pineforge::feed { +namespace { +using Handle = std::unique_ptr; +struct GlobalCurl { + GlobalCurl() { + if (curl_global_init(CURL_GLOBAL_DEFAULT) != CURLE_OK) throw Error(23, "curl initialization failed"); + } + ~GlobalCurl() { curl_global_cleanup(); } +}; +template void option(CURL* handle, CURLoption name, Value value) { + if (curl_easy_setopt(handle, name, value) != CURLE_OK) throw Error(23, "curl option rejected"); +} +struct Progress { const std::atomic* stopped = nullptr; }; +int progress(void* context, curl_off_t, curl_off_t, curl_off_t, curl_off_t) { + const auto* state = static_cast(context); + return stopping || (state->stopped && state->stopped->load()) ? 1 : 0; +} +Handle handle_for(const std::string& url, bool websocket, bool insecure, Progress* state) { + check_runtime_curl(); + Handle handle(curl_easy_init(), &curl_easy_cleanup); + if (!handle) throw Error(23, "curl handle allocation failed"); + option(handle.get(), CURLOPT_URL, url.c_str()); + option(handle.get(), CURLOPT_NOSIGNAL, 1L); + option(handle.get(), CURLOPT_NETRC, static_cast(CURL_NETRC_IGNORED)); + option(handle.get(), CURLOPT_FOLLOWLOCATION, 0L); + option(handle.get(), CURLOPT_MAXREDIRS, 0L); + option(handle.get(), CURLOPT_SSL_VERIFYPEER, 1L); + option(handle.get(), CURLOPT_SSL_VERIFYHOST, 2L); + option(handle.get(), CURLOPT_CONNECTTIMEOUT_MS, 10000L); + option(handle.get(), CURLOPT_TIMEOUT_MS, 20000L); + option(handle.get(), CURLOPT_USERAGENT, "pineforge-feed/0.1"); + option(handle.get(), CURLOPT_PROTOCOLS_STR, websocket ? (insecure ? "ws,wss" : "wss") : (insecure ? "http,https" : "https")); + option(handle.get(), CURLOPT_NOPROGRESS, 0L); + option(handle.get(), CURLOPT_XFERINFOFUNCTION, &progress); + option(handle.get(), CURLOPT_XFERINFODATA, state); + return handle; +} +void pause_until(std::chrono::steady_clock::time_point deadline) { + while (std::chrono::steady_clock::now() < deadline) { + if (stopping) throw Stopped{}; + std::this_thread::sleep_for(std::chrono::milliseconds(25)); + } +} +struct Response { + std::string body, retry_after; + std::uint64_t used_weight = 0; + bool overflow = false; +}; +std::size_t body_callback(char* data, std::size_t size, std::size_t count, void* context) noexcept { + auto* response = static_cast(context); + const auto bytes = size * count; + // The JSON parser refuses documents above 1 MiB, so a larger body is refused here. + if (bytes > 1024 * 1024 - response->body.size()) { response->overflow = true; return 0; } + try { response->body.append(data, bytes); return bytes; } + catch (...) { response->overflow = true; return 0; } +} +std::size_t header_callback(char* data, std::size_t size, std::size_t count, void* context) noexcept { + const auto bytes = size * count; + try { + auto* response = static_cast(context); + const std::string line(data, bytes); + const auto colon = line.find(':'); + if (colon == std::string::npos) return bytes; + auto name = line.substr(0, colon); + std::transform(name.begin(), name.end(), name.begin(), [](unsigned char value) { return static_cast(std::tolower(value)); }); + const auto first = line.find_first_not_of(" \t", colon + 1); + const auto last = line.find_last_not_of(" \t\r\n"); + const auto value = first == std::string::npos || last < first ? std::string() : line.substr(first, last - first + 1); + if (name == "retry-after") response->retry_after = value; + if (name == "x-mbx-used-weight-1m") { + std::uint64_t used = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), used); + if (result.ec == std::errc{} && result.ptr == value.data() + value.size()) response->used_weight = used; + } + } catch (...) {} + return bytes; +} +} + +void pause_for(std::chrono::milliseconds duration) { pause_until(std::chrono::steady_clock::now() + duration); } + +// Retry-After is either delta-seconds or an IMF-fixdate HTTP-date (RFC 9110, section 10.2.3). +std::uint64_t retry_after_seconds(const std::string& value, std::int64_t now) { + std::uint64_t seconds = 0; + const auto parsed = std::from_chars(value.data(), value.data() + value.size(), seconds); + if (!value.empty() && parsed.ec == std::errc{} && parsed.ptr == value.data() + value.size()) return seconds; + std::tm broken{}; + std::istringstream input(value); + input.imbue(std::locale::classic()); + input >> std::get_time(&broken, "%a, %d %b %Y %H:%M:%S GMT"); + if (value.size() != 29 || input.fail()) throw Error(23, "invalid Retry-After on a public rate-limit response"); + const auto when = static_cast(timegm(&broken)); + return when > now ? static_cast(when - now) : 0; +} + +void check_runtime_curl() { + static GlobalCurl initialization; + (void)initialization; + const auto* info = curl_version_info(CURLVERSION_NOW); + bool ws = false, wss = false; + for (const char* const* protocol = info ? info->protocols : nullptr; protocol && *protocol; ++protocol) { + ws = ws || std::strcmp(*protocol, "ws") == 0; + wss = wss || std::strcmp(*protocol, "wss") == 0; + } + if (!info || info->version_num < 0x080e01 || !ws || !wss) + throw Error(23, "libcurl >= 8.14.1 with both ws and wss is required at runtime"); +} + +void validate_origin(const std::string& origin, bool websocket, bool allow_insecure) { + if (origin.empty() || origin.size() > 4096 || origin.find_first_of("\r\n") != std::string::npos || origin.find('\0') != std::string::npos) + throw Error(23, "invalid public endpoint origin"); + std::unique_ptr parsed(curl_url(), &curl_url_cleanup); + if (!parsed || curl_url_set(parsed.get(), CURLUPART_URL, origin.c_str(), CURLU_NON_SUPPORT_SCHEME) != CURLUE_OK) + throw Error(23, "invalid public endpoint URL"); + auto field = [&](CURLUPart part) { + char* text = nullptr; + const auto status = curl_url_get(parsed.get(), part, &text, 0); + std::string value; + if (status == CURLUE_OK) { value = text; curl_free(text); } + return value; + }; + const auto scheme = field(CURLUPART_SCHEME); + const auto host = field(CURLUPART_HOST); + const auto secure = websocket ? "wss" : "https"; + const auto insecure = websocket ? "ws" : "http"; + if (scheme != secure && !(allow_insecure && scheme == insecure && (host == "127.0.0.1" || host == "localhost" || host == "[::1]"))) + throw Error(23, "public endpoints require TLS; insecure overrides are loopback-only"); + if (!field(CURLUPART_USER).empty() || !field(CURLUPART_PASSWORD).empty() || !field(CURLUPART_FRAGMENT).empty() || + !field(CURLUPART_QUERY).empty() || field(CURLUPART_PATH) != "/") throw Error(23, "endpoint must be a credential-free origin without a path or query"); +} + +HttpClient::HttpClient(const Config& config) : config_(config) { + check_runtime_curl(); + validate_origin(config_.rest_url, false, config_.allow_insecure); + try { + const auto metadata = get("/api/v3/exchangeInfo?symbol=" + config_.symbol, 20); + const auto& limits = metadata.at("rateLimits"); + if (limits.kind != Json::Kind::Array || limits.items.size() > 64) throw Error(23, "invalid public rate-limit metadata"); + bool weighted = false; + for (const auto& limit : limits.items) { + const auto type = limit.at("rateLimitType").text(); + if (type != "REQUEST_WEIGHT" && type != "RAW_REQUESTS") continue; + const auto interval = limit.at("interval").text(); + const auto multiple = limit.at("intervalNum").integer(); + const auto ceiling = limit.at("limit").integer(); + const auto unit = interval == "SECOND" ? 1000ULL : interval == "MINUTE" ? 60000ULL : + interval == "HOUR" ? 3600000ULL : interval == "DAY" ? 86400000ULL : 0ULL; + if (!unit || !multiple || multiple > 10000 || !ceiling || ceiling > 1000000000) + throw Error(23, "unsupported public rate-limit interval or ceiling"); + // Spend at most half of each published ceiling (about 50% headroom). + const auto pace = (unit * multiple * 2 + ceiling - 1) / ceiling; + if (type == "REQUEST_WEIGHT") { + weighted = true; + milliseconds_per_weight_ = std::max(milliseconds_per_weight_, pace); + if (interval == "MINUTE" && multiple == 1) weight_limit_ = ceiling; + } else milliseconds_per_request_ = std::max(milliseconds_per_request_, pace); + } + if (!weighted) throw Error(23, "public request-weight ceiling is unavailable"); + log("info", "rest_limits_verified", Json::object({{"milliseconds_per_weight", Json::number(std::to_string(milliseconds_per_weight_))}, + {"weight_limit_1m", Json::number(std::to_string(weight_limit_))}})); + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid public exchange rate-limit metadata"); } +} +Json HttpClient::get(const std::string& path, unsigned int weight) { + const auto endpoint = path.substr(0, path.find('?')); + if (endpoint != "/api/v3/exchangeInfo" && endpoint != "/api/v3/historicalTrades" && + endpoint != "/api/v3/aggTrades" && endpoint != "/api/v3/klines") + throw Error(23, "REST request is outside the public market-data allowlist"); + for (unsigned int attempt = 0; attempt < 4; ++attempt) { + pause_until(next_request_); + Progress state; + auto handle = handle_for(config_.rest_url + path, false, config_.allow_insecure, &state); + Response response; + option(handle.get(), CURLOPT_WRITEFUNCTION, &body_callback); + option(handle.get(), CURLOPT_WRITEDATA, &response); + option(handle.get(), CURLOPT_HEADERFUNCTION, &header_callback); + option(handle.get(), CURLOPT_HEADERDATA, &response); + const auto code = curl_easy_perform(handle.get()); + long status = 0; + curl_easy_getinfo(handle.get(), CURLINFO_RESPONSE_CODE, &status); + const auto pacing = std::max(milliseconds_per_request_, weight * milliseconds_per_weight_); + if (pacing > 86400000) throw Error(22, "public quota wait exceeds the supported resource bound"); + next_request_ = std::chrono::steady_clock::now() + std::chrono::milliseconds(config_.allow_insecure ? 0 : pacing); + if (stopping || code == CURLE_ABORTED_BY_CALLBACK) throw Stopped{}; + if (response.overflow) throw Error(22, "REST response exceeds the bounded buffer"); + if (status == 401 || status == 403 || status == 418 || status == 451 || (status >= 300 && status < 400)) + throw Error(23, "public market-data access denied or redirected (HTTP " + std::to_string(status) + ")"); + if (status == 429) { + const auto wait = response.retry_after.empty() ? 1 : retry_after_seconds(response.retry_after, + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count()); + log("warn", "rest_rate_limited", Json::object({{"retry_after", Json::number(std::to_string(wait))}})); + if (wait > 86400) throw Error(22, "public Retry-After exceeds the bounded retry window"); + next_request_ = std::chrono::steady_clock::now() + std::chrono::seconds(wait); + continue; + } + if (status == 404) throw Error(20, "public history is unavailable"); + if (code != CURLE_OK || status >= 500) { + next_request_ = std::chrono::steady_clock::now() + std::chrono::seconds(1U << attempt); + continue; + } + if (status != 200) throw Error(23, "public market-data request rejected (HTTP " + std::to_string(status) + ")"); + if (response.used_weight) log("info", "rest_weight", Json::object({{"used_weight_1m", Json::number(std::to_string(response.used_weight))}})); + // The venue's used-weight header covers every client on this IP: past half the ceiling, wait for the window. + if (weight_limit_ && response.used_weight >= weight_limit_ / 2) { + const auto elapsed = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count() % 60000; + next_request_ = std::max(next_request_, std::chrono::steady_clock::now() + std::chrono::milliseconds(60500 - elapsed)); + log("warn", "rest_shared_quota_pause"); + } + try { return parse_json(response.body); } + catch (const std::exception&) { throw Error(23, "public market-data response is invalid JSON"); } + } + throw Error(20, "public history retries exhausted; verified prefix retained"); +} + +bool WebSocketPump::stopped() const { return stopped_.load() || stopping.load(); } +WebSocketPump::WebSocketPump(const Config& config, const std::string& streams) : config_(config), url_(config.ws_url + "/stream?streams=" + streams) { + check_runtime_curl(); + validate_origin(config_.ws_url, true, config_.allow_insecure); + worker_ = std::thread([this] { run(); }); +} +WebSocketPump::~WebSocketPump() { + stopped_.store(true); + ready_.notify_all(); + if (worker_.joinable()) worker_.join(); +} +void WebSocketPump::push(SourceMessage message) { + std::lock_guard guard(mutex_); + const auto bytes = message.text.size() + 32; + if (bytes > config_.max_queue_bytes - std::min(queue_bytes_, config_.max_queue_bytes)) throw Error(22, "WebSocket healing buffer overrun"); + queue_bytes_ += bytes; + queue_.push_back(std::move(message)); + ready_.notify_one(); +} +bool WebSocketPump::try_take(SourceMessage& message) { + std::lock_guard guard(mutex_); + if (failure_) std::rethrow_exception(failure_); + if (queue_.empty()) return false; + message = std::move(queue_.front()); + queue_.pop_front(); + queue_bytes_ -= message.text.size() + 32; + return true; +} +SourceMessage WebSocketPump::take() { + std::unique_lock guard(mutex_); + while (queue_.empty() && !failure_) { + if (stopped()) throw Stopped{}; + ready_.wait_for(guard, std::chrono::milliseconds(100)); + } + if (failure_) std::rethrow_exception(failure_); + auto message = std::move(queue_.front()); + queue_.pop_front(); + queue_bytes_ -= message.text.size() + 32; + return message; +} + +void WebSocketPump::run() { + try { + unsigned int failures = 0; + auto next_attempt = std::chrono::steady_clock::now(); + while (!stopped()) { + while (!stopped() && std::chrono::steady_clock::now() < next_attempt) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + if (stopped()) return; + next_attempt = std::chrono::steady_clock::now() + std::chrono::milliseconds(config_.allow_insecure ? 250 : 5000); + Progress state{&stopped_}; + auto handle = handle_for(url_, true, config_.allow_insecure, &state); + option(handle.get(), CURLOPT_CONNECT_ONLY, 2L); + option(handle.get(), CURLOPT_HTTP_VERSION, static_cast(CURL_HTTP_VERSION_1_1)); + const auto handshake = curl_easy_perform(handle.get()); + long status = 0; + curl_easy_getinfo(handle.get(), CURLINFO_RESPONSE_CODE, &status); + if (stopped()) return; + if (status == 401 || status == 403 || status == 418 || status == 451 || (status >= 300 && status < 400)) + throw Error(23, "public WebSocket access denied or redirected"); + if (handshake != CURLE_OK || status != 101) { + if (++failures >= 8) throw Error(20, "public WebSocket reconnect attempts exhausted"); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(std::min(30U, 1U << failures)); + while (!stopped() && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(50)); + continue; + } + failures = 0; + push({true, {}}); + log("info", "websocket_connected"); + const auto birth = std::chrono::steady_clock::now(); + auto last_text = birth, message_birth = birth; + std::string message; + bool assembling = false; + std::uint64_t frame_offset = 0; + curl_socket_t socket = CURL_SOCKET_BAD; + curl_easy_getinfo(handle.get(), CURLINFO_ACTIVESOCKET, &socket); + while (!stopped()) { + const auto now = std::chrono::steady_clock::now(); + if (now - birth >= std::chrono::seconds(config_.reconnect_seconds) || now - last_text > std::chrono::seconds(75) || + (assembling && now - message_birth > std::chrono::seconds(30))) break; + std::array buffer{}; + std::size_t received = 0; + const curl_ws_frame* metadata = nullptr; + const auto result = curl_ws_recv(handle.get(), buffer.data(), buffer.size(), &received, &metadata); + if (result == CURLE_AGAIN) { + pollfd descriptor{socket, POLLIN, 0}; + ::poll(&descriptor, 1, 100); + continue; + } + if (result != CURLE_OK || !metadata || (metadata->flags & CURLWS_CLOSE)) break; + if (metadata->flags & (CURLWS_PING | CURLWS_PONG)) continue; + if ((metadata->flags & CURLWS_BINARY) || !(metadata->flags & CURLWS_TEXT) || metadata->offset < 0 || metadata->bytesleft < 0 || + static_cast(metadata->offset) != frame_offset) throw Error(23, "public WebSocket requires ordered text frames"); + if (!assembling) { assembling = true; message_birth = now; } + if (received > 1024 * 1024 - message.size() || static_cast(metadata->bytesleft) > 1024 * 1024 - message.size() - received) + throw Error(22, "WebSocket message exceeds 1 MiB"); + message.append(buffer.data(), received); + frame_offset += received; + last_text = now; + if (metadata->bytesleft) continue; + frame_offset = 0; + if (metadata->flags & CURLWS_CONT) continue; + bool retiring = false; + try { + const auto envelope = parse_json(message); + const auto* wrapped = envelope.find("data"); + const auto& data = wrapped ? *wrapped : envelope; + const auto* event = data.find("e"); + retiring = event && event->kind == Json::Kind::String && event->text() == "serverShutdown"; + } catch (const std::exception&) { throw Error(23, "invalid public WebSocket JSON"); } + push({false, std::move(message)}); + message.clear(); + assembling = false; + if (retiring) { log("info", "source_retiring_reconnect"); break; } + } + if (!stopped()) { + log("warn", "websocket_reconnect"); + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + } + } + } catch (const Stopped&) {} + catch (...) { + std::lock_guard guard(mutex_); + failure_ = std::current_exception(); + ready_.notify_all(); + } +} +} diff --git a/native/feed/src/transport.hpp b/native/feed/src/transport.hpp new file mode 100644 index 0000000..2ced947 --- /dev/null +++ b/native/feed/src/transport.hpp @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "core.hpp" +#include +#include +#include +#include +#include +#include + +namespace pineforge::feed { +void check_runtime_curl(); +void validate_origin(const std::string& origin, bool websocket, bool allow_insecure); +void pause_for(std::chrono::milliseconds duration); +std::uint64_t retry_after_seconds(const std::string& value, std::int64_t now); + +class HttpClient { + Config config_; + std::chrono::steady_clock::time_point next_request_{}; + std::uint64_t weight_limit_ = 0, milliseconds_per_weight_ = 0, milliseconds_per_request_ = 0; +public: + explicit HttpClient(const Config& config); + Json get(const std::string& path, unsigned int weight); +}; +struct SourceMessage { + bool connected = false; + std::string text; +}; +class WebSocketPump { + Config config_; + std::string url_; + std::atomic stopped_{false}; + std::mutex mutex_; + std::condition_variable ready_; + std::deque queue_; + std::size_t queue_bytes_ = 0; + std::exception_ptr failure_; + std::thread worker_; + bool stopped() const; + void push(SourceMessage message); + void run(); +public: + WebSocketPump(const Config& config, const std::string& streams); + ~WebSocketPump(); + WebSocketPump(const WebSocketPump&) = delete; + WebSocketPump& operator=(const WebSocketPump&) = delete; + SourceMessage take(); + bool try_take(SourceMessage& message); +}; +} diff --git a/native/feed/src/venue.hpp b/native/feed/src/venue.hpp new file mode 100644 index 0000000..8887913 --- /dev/null +++ b/native/feed/src/venue.hpp @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "core.hpp" +#include + +namespace pineforge::feed { +struct VenueEvent { + enum class Kind { Trade, Kline, Control } kind = Kind::Control; + Trade trade; + Kline kline; +}; +class Venue { +public: + virtual ~Venue() = default; + virtual std::string streams() const = 0; + virtual VenueEvent decode(const std::string& message) const = 0; + virtual std::vector history(std::uint64_t from, std::size_t limit = 1000) = 0; + virtual std::uint64_t first_trade_id(std::int64_t minute) = 0; + virtual std::vector klines(std::int64_t start, std::int64_t end) = 0; +}; +} diff --git a/native/feed/tests/batch_probe.cpp b/native/feed/tests/batch_probe.cpp new file mode 100644 index 0000000..6dfee52 --- /dev/null +++ b/native/feed/tests/batch_probe.cpp @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +template Function load(void* library, const char* name) { + auto function = reinterpret_cast(dlsym(library, name)); + if (!function) throw std::runtime_error(std::string("missing qualification export: ") + name); + return function; +} + +int main(int argc, char** argv) { + try { + if (argc != 4) throw std::runtime_error("usage: batch-probe observed-strategy.so bars.csv actions.jsonl"); + std::ifstream input(argv[2]); + if (!input) throw std::runtime_error("cannot open batch input"); + std::string line; + std::getline(input, line); + if (line != "timestamp,open,high,low,close,volume") throw std::runtime_error("unexpected CSV schema"); + std::vector bars; + while (std::getline(input, line)) { + std::istringstream fields(line); + std::vector values; + std::string value; + while (std::getline(fields, value, ',')) values.push_back(value); + if (values.size() != 6) throw std::runtime_error("invalid CSV row"); + bars.push_back({std::stod(values[1]), std::stod(values[2]), std::stod(values[3]), + std::stod(values[4]), std::stod(values[5]), std::stoll(values[0])}); + } + void* library = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (!library) throw std::runtime_error(dlerror()); + if (load(library, "pf_abi_version")() != PF_ABI_VERSION) + throw std::runtime_error("qualification strategy ABI mismatch"); + auto create = load(library, "strategy_create"); + auto release = load(library, "strategy_free"); + auto report_release = load(library, "report_free"); + auto run = load(library, "run_backtest_full"); + auto retain = load(library, "equivalence_retain_events"); + auto actions = load(library, "equivalence_export_actions"); + auto error = load(library, "strategy_get_last_error"); + auto strategy = create(nullptr); + if (!strategy || retain(strategy)) throw std::runtime_error("cannot retain batch execution receipts"); + pf_report_t report{}; + run(strategy, bars.data(), static_cast(bars.size()), "1", "1", 0, 4, PF_MAGNIFIER_ENDPOINTS, &report); + const auto* failure = error(strategy); + if (failure && *failure) throw std::runtime_error(failure); + if (actions(strategy, argv[3])) throw std::runtime_error("cannot export batch execution actions"); + std::cout << "batch_abi=run_backtest_full input_bars=" << bars.size() + << " script_bars=" << report.script_bars_processed << " trades=" << report.trades_len << '\n'; + report_release(&report); + release(strategy); + dlclose(library); + return 0; + } catch (const std::exception& failure) { + std::cerr << failure.what() << '\n'; + return 1; + } +} diff --git a/native/feed/tests/build_ci_curl.sh b/native/feed/tests/build_ci_curl.sh new file mode 100644 index 0000000..c2d50b7 --- /dev/null +++ b/native/feed/tests/build_ci_curl.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +prefix="${1:?provide an installation prefix}" +source_dir="${prefix}-source" +mkdir -p "$source_dir" +archive="$source_dir/curl-8.14.1.tar.xz" +curl --fail --location --retry 3 https://curl.se/download/curl-8.14.1.tar.xz -o "$archive" +printf '%s %s\n' f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd "$archive" | shasum -a 256 --check +tar -xJf "$archive" -C "$source_dir" +cmake -S "$source_dir/curl-8.14.1" -B "$source_dir/build" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$prefix" -DCURL_DISABLE_WEBSOCKETS=OFF \ + -DCURL_USE_OPENSSL=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=OFF \ + -DCURL_USE_LIBPSL=OFF -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=OFF \ + -DCURL_BROTLI=OFF -DCURL_ZSTD=OFF +cmake --build "$source_dir/build" -j4 +cmake --install "$source_dir/build" +LD_LIBRARY_PATH="$prefix/lib:${LD_LIBRARY_PATH:-}" \ + DYLD_LIBRARY_PATH="$prefix/lib:${DYLD_LIBRARY_PATH:-}" "$prefix/bin/curl" --version diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py new file mode 100644 index 0000000..a4b9c8f --- /dev/null +++ b/native/feed/tests/mock_venue.py @@ -0,0 +1,940 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Synthetic public Binance shapes; standard-library HTTP/WS fault injection.""" + +import base64 +import copy +import email.utils +import fcntl +import hashlib +import http.server +import json +import os +import pathlib +import signal +import struct +import subprocess +import sys +import tempfile +import threading +import time +import unittest +import urllib.parse +from types import SimpleNamespace + +from public_e2e import Soak, action_key, proven_actions + +BINARY = str(pathlib.Path(sys.argv[1]).resolve()) +sys.argv = [sys.argv[0]] +TRADES = { + 99: (119999, "10.10000000", "0.10000000"), + 100: (120001, "10.10000000", "0.10000000"), + 101: (120050, "11.20000000", "0.20000000"), + 102: (179999, "9.90000000", "0.30000000"), + 103: (180001, "12.00000000", "0.40000000"), + 104: (180050, "11.00000000", "0.10000000"), + 105: (240001, "12.00000000", "0.10000000"), +} +BARS = { + 120000: ("10.10000000", "11.20000000", "9.90000000", "9.90000000", "0.60000000", 100, 102, 3), + 180000: ("12.00000000", "12.00000000", "11.00000000", "11.00000000", "0.50000000", 103, 104, 2), + 240000: ("12.00000000", "12.00000000", "12.00000000", "12.00000000", "0.10000000", 105, 105, 1), +} + + +def trade(identifier, **changes): + matched, price, quantity = TRADES[identifier] + data = { + "e": "trade", + "E": matched + 999, + "s": "TESTUSDT", + "t": identifier, + "p": price, + "q": quantity, + "T": matched, + "m": False, + "M": True, + } + data.update(changes) + return {"stream": "testusdt@trade", "data": data} + + +def candle(minute, confirmed=True, **changes): + opening, high, low, close, volume, first, last, count = BARS[minute] + row = { + "t": minute, + "T": minute + 59999, + "s": "TESTUSDT", + "i": "1m", + "f": first, + "L": last, + "o": opening, + "h": high, + "l": low, + "c": close, + "v": volume, + "n": count, + "x": confirmed, + "q": "0.00000000", + "V": "0.00000000", + "Q": "0.00000000", + "B": "0", + } + row.update(changes) + return { + "stream": "testusdt@kline_1m", + "data": {"e": "kline", "E": minute + 60000, "s": "TESTUSDT", "k": row}, + } + + +def retirement(): + return {"stream": "!serverShutdown", "data": {"e": "serverShutdown", "E": 180000}} + + +def frame(connection, payload, opcode=1, final=True): + if isinstance(payload, dict): + payload = json.dumps(payload, separators=(",", ":")).encode() + if isinstance(payload, str): + payload = payload.encode() + header = bytes([(128 if final else 0) | opcode]) + if len(payload) < 126: + header += bytes([len(payload)]) + elif len(payload) < 65536: + header += bytes([126]) + struct.pack("!H", len(payload)) + else: + header += bytes([127]) + struct.pack("!Q", len(payload)) + connection.sendall(header + payload) + + +def receive(connection): + def exact(size): + result = bytearray() + while len(result) < size: + chunk = connection.recv(size - len(result)) + if not chunk: + raise EOFError + result.extend(chunk) + return bytes(result) + + first, second = exact(2) + size = second & 127 + if size == 126: + size = struct.unpack("!H", exact(2))[0] + if size == 127: + size = struct.unpack("!Q", exact(8))[0] + if not second & 128: + raise AssertionError("client frames must be masked") + mask = exact(4) + payload = exact(size) + return first & 15, bytes(value ^ mask[index % 4] for index, value in enumerate(payload)) + + +class MockServer(http.server.ThreadingHTTPServer): + daemon_threads = True + block_on_close = False + + def __init__(self, scripts): + super().__init__(("127.0.0.1", 0), Handler) + self.scripts = scripts + self.connections = 0 + self.trades = copy.deepcopy(TRADES) + self.bars = copy.deepcopy(BARS) + self.requests = [] + self.request_times = [] + self.agg_empty = 0 + self.pongs = [] + self.fail_status = 0 + self.rate_once = False + self.retry_after = 0 + self.rate_limits = None + self.auth_headers = [] + self.streams = [] + self.lock = threading.Lock() + self.errors = [] + + def handle_error(self, request, client_address): + failure = sys.exc_info()[1] + if not isinstance(failure, (BrokenPipeError, ConnectionResetError, EOFError)): + self.errors.append(repr(failure)) + + +class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *arguments): + pass + + def do_GET(self): + parsed = urllib.parse.urlsplit(self.path) + query = urllib.parse.parse_qs(parsed.query) + with self.server.lock: + self.server.requests.append(parsed.path) + self.server.request_times.append((parsed.path, time.monotonic())) + self.server.auth_headers.extend( + name for name in ("Authorization", "X-MBX-APIKEY") if self.headers.get(name) + ) + if parsed.path == "/stream": + self.websocket(query) + return + with self.server.lock: + if self.server.fail_status: + self.respond(self.server.fail_status, {"code": -1, "msg": "synthetic refusal"}) + return + if self.server.rate_once: + self.server.rate_once = False + self.respond(429, {}, retry=True) + return + if parsed.path == "/api/v3/exchangeInfo": + rows = { + "rateLimits": [ + { + "rateLimitType": "REQUEST_WEIGHT", + "interval": "MINUTE", + "intervalNum": 1, + "limit": 6000, + }, + { + "rateLimitType": "RAW_REQUESTS", + "interval": "MINUTE", + "intervalNum": 5, + "limit": 61000, + }, + ], + "symbols": [{"symbol": "TESTUSDT", "baseAsset": "TEST", "quoteAsset": "USDT"}], + } + if self.server.rate_limits is not None: + rows["rateLimits"] = self.server.rate_limits + elif parsed.path == "/api/v3/historicalTrades": + start = int(query["fromId"][0]) + limit = int(query["limit"][0]) + rows = [] + for identifier in sorted(self.server.trades): + if identifier >= start and len(rows) < limit: + matched, price, quantity = self.server.trades[identifier] + rows.append( + { + "id": identifier, + "price": price, + "qty": quantity, + "quoteQty": "0.00000000", + "time": matched, + "isBuyerMaker": False, + "isBestMatch": True, + } + ) + elif parsed.path == "/api/v3/aggTrades": + start, end = int(query["startTime"][0]), int(query["endTime"][0]) + matching = [ + identifier + for identifier in sorted(self.server.trades) + if start <= self.server.trades[identifier][0] <= end + ] + if self.server.agg_empty: + self.server.agg_empty -= 1 + matching = [] + rows = ( + [] + if not matching + else [ + { + "a": 42, + "f": matching[0], + "l": matching[0], + "T": self.server.trades[matching[0]][0], + "p": "10.10000000", + "q": "0.10000000", + "m": False, + "M": True, + } + ] + ) + elif parsed.path == "/api/v3/klines": + start, end = int(query["startTime"][0]), int(query["endTime"][0]) + rows = [] + for minute in sorted(self.server.bars): + if start <= minute <= end and len(rows) < 1000: + opening, high, low, close, volume, _first, _last, count = self.server.bars[ + minute + ] + rows.append( + [ + minute, + opening, + high, + low, + close, + volume, + minute + 59999, + "0.00000000", + count, + "0.00000000", + "0.00000000", + "0", + ] + ) + else: + self.respond(404, {}) + return + self.respond(200, rows) + + def respond(self, status, value, retry=False): + payload = json.dumps(value, separators=(",", ":")).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.send_header("Connection", "close") + if retry: + value = self.server.retry_after + self.send_header("Retry-After", value() if callable(value) else str(value)) + self.end_headers() + self.wfile.write(payload) + self.close_connection = True + + def websocket(self, query): + assert query["streams"] in (["testusdt@trade/testusdt@kline_1m"], ["testusdt@kline_1m"]) + with self.server.lock: + self.server.streams.append(query["streams"][0]) + key = self.headers["Sec-WebSocket-Key"] + accept = base64.b64encode( + hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest() + ).decode() + self.send_response(101) + self.send_header("Upgrade", "websocket") + self.send_header("Connection", "Upgrade") + self.send_header("Sec-WebSocket-Accept", accept) + self.end_headers() + with self.server.lock: + connection_index = self.server.connections + self.server.connections += 1 + script = self.server.scripts[min(connection_index, len(self.server.scripts) - 1)] + self.connection.settimeout(3) + for action in script: + if isinstance(action, dict): + frame(self.connection, action) + elif action[0] == "pause": + time.sleep(action[1]) + elif action[0] == "ping": + frame(self.connection, action[1], opcode=9) + opcode, payload = receive(self.connection) + assert opcode == 10 and payload == action[1] + self.server.pongs.append(payload) + elif action[0] == "fragment": + payload = json.dumps(action[1], separators=(",", ":")).encode() + middle = len(payload) // 2 + frame(self.connection, payload[:middle], final=False) + frame(self.connection, payload[middle:], opcode=0) + elif action[0] == "binary": + frame(self.connection, action[1], opcode=2) + elif action[0] == "raw": + frame(self.connection, action[1]) + elif action[0] == "revise_trade": + with self.server.lock: + row = list(self.server.trades[action[1]]) + row[2] = "0.10000001" + self.server.trades[action[1]] = tuple(row) + elif action[0] == "revise_rest": + with self.server.lock: + row = list(self.server.bars[action[1]]) + row[4] = "0.70000000" + self.server.bars[action[1]] = tuple(row) + elif action[0] == "close": + frame(self.connection, b"", opcode=8) + self.close_connection = True + return + self.connection.settimeout(0.1) + while True: + try: + receive(self.connection) + except TimeoutError: + continue + except (EOFError, ConnectionResetError): + break + self.close_connection = True + + +class FeedMockTests(unittest.TestCase): + def test_public_qualification_fails_on_any_action_difference(self): + for ticks_equal, expected in ((False, 1), (True, 0)): + soak = Soak.__new__(Soak) + soak.options = SimpleNamespace(bar_minutes=21, tick_minutes=21) + soak.directory = self.directory / str(ticks_equal) + soak.directory.mkdir() + soak.results = {} + soak.receiver = SimpleNamespace(shutdown=lambda: None) + soak.warmup = lambda: None + soak.live = lambda mode, minutes, equal=ticks_equal, soak=soak: soak.results.update( + {mode: {"batch_actions_equal": mode == "bars" or equal}} + ) + self.assertEqual(soak.run(), expected) + + def test_r_b2_maps_only_tick_action_timestamps(self): + record = { + "timestamp": 1791037920021, + "bar_index": 7, + "order": { + "id": "long", + "action": "entry", + "leg": "long", + "contracts": 1.0, + "price": 84816.0, + "reduce_only": False, + "entry_incarnation": 1, + }, + } + batch = dict(record, timestamp=1791037920000) + self.assertEqual(action_key(record, "ticks"), action_key(batch, "ticks")) + self.assertNotEqual(action_key(record, "bars"), action_key(batch, "bars")) + moved = dict(record, order=dict(record["order"], price=84816.5)) + self.assertNotEqual(action_key(moved, "ticks"), action_key(batch, "ticks")) + + def test_only_actions_on_proven_bars_are_compared(self): + record = { + "timestamp": 1791051480451, + "bar_index": 221, + "order": { + "id": "S", + "action": "sell", + "leg": "entry", + "contracts": 1.0, + "price": 84980.0, + "reduce_only": False, + "entry_incarnation": 15, + }, + } + open_minute = dict(record, bar_index=222) + inside, trailing = proven_actions([record, open_minute], "ticks", 222) + self.assertEqual(inside, [action_key(record, "ticks")]) + self.assertEqual(trailing, 1) + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="pineforge-feed-mock-") + self.addCleanup(self.temporary.cleanup) + self.directory = pathlib.Path(self.temporary.name) + + def server(self, scripts): + server = MockServer(scripts) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + return server + + def command(self, server, mode="bars", extra=(), resume=False): + port = server.server_address[1] + arguments = [ + BINARY, + "run", + "--venue", + "binance", + "--market", + "spot", + "--symbol", + "TESTUSDT", + "--mode", + mode, + "--state-dir", + str(self.directory / "state"), + "--rest-url", + f"http://127.0.0.1:{port}", + "--ws-url", + f"ws://127.0.0.1:{port}", + "--allow-insecure-http", + ] + arguments += ["--resume"] if resume else ["--start", "120000"] + return arguments + list(extra) + + def run_feed(self, server, mode="bars", extra=(), expected=0, resume=False): + result = subprocess.run( + self.command(server, mode, extra, resume), capture_output=True, text=True, timeout=12 + ) + self.assertEqual(result.returncode, expected, result.stderr) + self.assertFalse(server.errors, server.errors) + for record in result.stderr.splitlines(): + self.assertIsInstance(json.loads(record), dict) + output = [json.loads(line) for line in result.stdout.splitlines()] + cursor = self.directory / "state" / "cursor.json" + if cursor.exists(): + saved = json.loads(cursor.read_text())["cursor"] + self.assertEqual( + saved["message_index"], + len((cursor.parent / "events.jsonl").read_text().splitlines()), + ) + return output, result + + def test_bars_hole_is_healed_behind_closed_watermark(self): + server = self.server([[candle(240000)]]) + output, _ = self.run_feed(server, extra=["--max-messages", "3"]) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000, 180000, 240000]) + self.assertEqual(server.streams, ["testusdt@kline_1m"]) + + def test_unhealable_bar_hole_stops_20(self): + server = self.server([[candle(240000)]]) + del server.bars[180000] + output, _ = self.run_feed(server, expected=20) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000]) + + def test_raw_hole_and_reordering_are_healed_not_resequenced(self): + server = self.server( + [[trade(102), trade(100), trade(101), trade(103), candle(120000), candle(180000)]] + ) + output, result = self.run_feed(server, "ticks", ["--max-messages", "7"]) + self.assertEqual(server.streams, ["testusdt@trade/testusdt@kline_1m"]) + self.assertEqual( + [event["seq"] for event in output if event["type"] == "tick"], [100, 101, 102, 103, 104] + ) + self.assertEqual( + [event["ts"] for event in output if event["type"] == "time"], [180000, 240000] + ) + self.assertIn('"price":10.10000000', result.stdout) + self.assertEqual(output[0]["ts"], 120001) + + def test_unhealable_raw_hole_stops_20(self): + server = self.server([[trade(102)]]) + del server.trades[101] + output, _ = self.run_feed(server, "ticks", expected=20) + self.assertTrue(all(event["type"] != "time" for event in output)) + + def test_identical_duplicates_do_not_change_message_boundaries(self): + server = self.server([[candle(120000), candle(120000), candle(180000)]]) + output, _ = self.run_feed(server, extra=["--max-messages", "2"]) + self.assertEqual(len(output), 2) + + def test_conflicting_raw_duplicate_stops_21(self): + server = self.server([[trade(100), trade(100, q="0.20000000")]]) + output, _ = self.run_feed(server, "ticks", expected=21) + self.assertEqual(len(output), 1) + + def test_revised_already_emitted_bar_stops_21(self): + server = self.server([[candle(120000), candle(120000, v="0.70000000")]]) + output, _ = self.run_feed(server, expected=21) + self.assertEqual(len(output), 1) + + def test_ohlcv_mismatch_emits_no_time(self): + server = self.server([[candle(120000, v="0.70000000")]]) + output, _ = self.run_feed(server, "ticks", expected=21) + self.assertEqual([event["type"] for event in output], ["tick", "tick", "tick"]) + + def test_empty_ambiguous_tick_minute_stops_20(self): + server = self.server([[candle(120000, n=0, f=-1, L=-1, v="0.00000000")]]) + output, _ = self.run_feed(server, "ticks", expected=20) + self.assertEqual(output, []) + + def test_forming_candles_never_advance_time(self): + server = self.server([[candle(120000, confirmed=False), trade(100), candle(120000)]]) + output, _ = self.run_feed(server, "ticks", ["--max-messages", "4"]) + self.assertEqual([event["type"] for event in output], ["tick", "tick", "tick", "time"]) + + def test_ping_payload_is_echoed_and_fragmented_text_works(self): + server = self.server([[("ping", b"synthetic-ping"), ("fragment", candle(120000))]]) + output, _ = self.run_feed(server, extra=["--max-messages", "1"]) + self.assertEqual(len(output), 1) + self.assertEqual(server.pongs, [b"synthetic-ping"]) + + def test_server_shutdown_reconnects_with_exact_overlap(self): + server = self.server([[candle(120000), retirement()], [candle(120000), candle(180000)]]) + output, _ = self.run_feed(server, extra=["--max-messages", "2"]) + self.assertEqual(len(output), 2) + self.assertGreaterEqual(server.connections, 2) + + def test_reconnect_changed_rest_overlap_stops_21(self): + server = self.server( + [[candle(120000), ("revise_rest", 120000), retirement()], [candle(180000)]] + ) + output, _ = self.run_feed(server, expected=21) + self.assertEqual(len(output), 1) + + def test_planned_24_hour_rotation_uses_overlap(self): + server = self.server([[candle(120000)], [candle(180000)]]) + output, _ = self.run_feed(server, extra=["--max-messages", "2", "--reconnect-seconds", "1"]) + self.assertEqual(len(output), 2) + self.assertGreaterEqual(server.connections, 2) + + def test_protocol_invalid_symbol_and_binary_stop_23(self): + missing_count = candle(120000) + del missing_count["data"]["k"]["n"] + unknown_on_data_stream = trade(100, e="tradeV2") + for payload in [ + trade(100, s="OTHERUSDT"), + ("binary", b"not-a-text-event"), + ("raw", "{"), + trade(100, q="-0.10000000"), + trade(100, p="1e3"), + missing_count, + unknown_on_data_stream, + {"stream": "!notice", "data": {"E": 1}}, + ]: + with self.subTest(payload=payload): + server = self.server([[payload]]) + state = self.directory / "state" + if state.exists(): + import shutil + + shutil.rmtree(state) + self.run_feed(server, "ticks", expected=23) + + def test_rest_access_failure_stops_23_no_key_sent(self): + for status in (451, 418, 403): + with self.subTest(status=status): + state = self.directory / "state" + if state.exists(): + import shutil + + shutil.rmtree(state) + server = self.server([[candle(240000)]]) + server.fail_status = status + output, result = self.run_feed(server, expected=23) + self.assertEqual(output, []) + self.assertIn(f"HTTP {status}", result.stderr) + self.assertEqual(server.requests.count("/api/v3/exchangeInfo"), 1) + self.assertTrue( + all( + path in {"/stream", "/api/v3/klines", "/api/v3/exchangeInfo"} + for path in server.requests + ) + ) + self.assertEqual(server.auth_headers, []) + + def test_missing_rate_limit_metadata_is_refused(self): + server = self.server([[candle(120000)]]) + server.rate_limits = [] + output, _ = self.run_feed(server, expected=23) + self.assertEqual(output, []) + + def test_excessive_retry_after_stops_instead_of_retrying_early(self): + server = self.server([[candle(120000)]]) + server.rate_once = True + server.retry_after = 86401 + output, _ = self.run_feed(server, expected=22) + self.assertEqual(output, []) + + def test_reconnect_changed_raw_overlap_stops_21(self): + server = self.server( + [[trade(100), trade(101), ("revise_trade", 100), retirement()], [trade(102)]] + ) + output, result = self.run_feed(server, "ticks", expected=21) + self.assertEqual([event["seq"] for event in output], [100, 101]) + self.assertIn("reconnect raw overlap changed", result.stderr) + self.assertGreaterEqual(server.connections, 2) + + def test_multi_page_raw_hole_heals_contiguously(self): + server = self.server([[]]) + server.trades = {99: TRADES[99]} + for offset in range(2100): + server.trades[100 + offset] = (120001 + offset * 20, "10.00000000", "0.00000001") + server.bars = { + 120000: ( + "10.00000000", + "10.00000000", + "10.00000000", + "10.00000000", + "0.00002100", + 100, + 2199, + 2100, + ) + } + last = { + "stream": "testusdt@trade", + "data": { + "e": "trade", + "E": 162000, + "s": "TESTUSDT", + "t": 2199, + "p": "10.00000000", + "q": "0.00000001", + "T": 161981, + "m": False, + }, + } + closing = candle( + 120000, + o="10.00000000", + h="10.00000000", + l="10.00000000", + c="10.00000000", + v="0.00002100", + f=100, + L=2199, + n=2100, + ) + server.scripts = [[last, closing]] + output, _ = self.run_feed(server, "ticks", ["--max-messages", "2101"]) + self.assertEqual([event["seq"] for event in output[:-1]], list(range(100, 2200))) + self.assertEqual(output[-1], {"type": "time", "ts": 180000}) + self.assertGreaterEqual(server.requests.count("/api/v3/historicalTrades"), 4) + + def test_matched_time_regression_stops_23(self): + server = self.server([[trade(100), trade(101, T=120000)]]) + server.trades[101] = (120000, "11.20000000", "0.20000000") + output, result = self.run_feed(server, "ticks", expected=23) + self.assertEqual([event["seq"] for event in output], [100]) + self.assertIn("matched time regressed", result.stderr) + + def test_unknown_non_data_event_warns_and_continues(self): + notice = {"stream": "!venueNotice", "data": {"e": "venueNotice", "E": 1}} + server = self.server([[notice, candle(120000)]]) + output, result = self.run_feed(server, extra=["--max-messages", "1"]) + self.assertEqual(len(output), 1) + warnings = [json.loads(record) for record in result.stderr.splitlines()] + self.assertIn( + {"stream": "!venueNotice", "type": "venueNotice"}, + [ + {key: record.get(key) for key in ("stream", "type")} + for record in warnings + if record["event"] == "unknown_stream_event" and record["level"] == "warn" + ], + ) + + def test_initial_fence_retries_lagging_rest(self): + server = self.server([[trade(100), trade(101), trade(102), candle(120000)]]) + server.agg_empty = 1 + output, _ = self.run_feed(server, "ticks", ["--max-messages", "4"]) + self.assertEqual([event["type"] for event in output], ["tick", "tick", "tick", "time"]) + self.assertEqual(server.requests.count("/api/v3/aggTrades"), 2) + + def test_stdio_flags_restored_and_full_stderr_never_cuts_a_record(self): + server = self.server([[candle(120000)]]) + read_error, write_error = os.pipe() + read_output, write_output = os.pipe() + try: + if hasattr(fcntl, "F_SETPIPE_SZ"): + fcntl.fcntl(write_error, fcntl.F_SETPIPE_SZ, 4096) + flags = fcntl.fcntl(write_error, fcntl.F_GETFL) + fcntl.fcntl(write_error, fcntl.F_SETFL, flags | os.O_NONBLOCK) + filler = b'{"filler":true}\n' + try: + while True: + os.write(write_error, filler) + except BlockingIOError: + pass + fcntl.fcntl(write_error, fcntl.F_SETFL, flags) + process = subprocess.Popen( + self.command(server, extra=["--max-messages", "1"]), + stdout=write_output, + stderr=write_error, + ) + started = time.monotonic() + self.assertEqual(process.wait(timeout=10), 0) + self.assertLess(time.monotonic() - started, 8) + self.assertFalse(fcntl.fcntl(write_output, fcntl.F_GETFL) & os.O_NONBLOCK) + self.assertFalse(fcntl.fcntl(write_error, fcntl.F_GETFL) & os.O_NONBLOCK) + os.close(write_error) + os.close(write_output) + write_error = write_output = -1 + with os.fdopen(read_error, "rb") as errors, os.fdopen(read_output, "rb") as output: + read_error = read_output = -1 + for line in errors.read().split(b"\n")[:-1]: + self.assertIsInstance(json.loads(line), dict) + self.assertEqual(len(output.read().splitlines()), 1) + finally: + for descriptor in (read_error, write_error, read_output, write_output): + if descriptor >= 0: + os.close(descriptor) + + def test_aggregate_trade_is_never_substituted_for_raw_prints(self): + message = trade(100) + message["data"]["e"] = "aggTrade" + server = self.server([[message]]) + output, _ = self.run_feed(server, "ticks", expected=23) + self.assertEqual(output, []) + + def test_sigterm_drain_is_bounded_with_unread_stdout(self): + messages = [] + for offset in range(2000): + message = candle(120000) + minute = 120000 + offset * 60000 + message["data"]["k"]["t"] = minute + message["data"]["k"]["T"] = minute + 59999 + messages.append(message) + server = self.server([messages]) + process = subprocess.Popen( + self.command(server), stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + try: + if hasattr(fcntl, "F_SETPIPE_SZ"): + fcntl.fcntl(process.stdout.fileno(), fcntl.F_SETPIPE_SZ, 4096) + cursor = self.directory / "state" / "cursor.json" + deadline = time.monotonic() + 20 + previous_index = 0 + stable_since = time.monotonic() + while time.monotonic() < deadline: + if cursor.exists(): + current_index = json.loads(cursor.read_text())["cursor"]["message_index"] + if current_index != previous_index: + previous_index = current_index + stable_since = time.monotonic() + elif ( + 30 <= current_index < len(messages) + and time.monotonic() - stable_since >= 0.4 + ): + break + self.assertIsNone(process.poll()) + time.sleep(0.02) + else: + self.fail( + "unread stdout did not stall before source exhaustion; " + f"committed={previous_index}" + ) + self.assertIsNone(process.poll()) + started = time.monotonic() + process.send_signal(signal.SIGTERM) + self.assertEqual(process.wait(timeout=7), 0) + self.assertLess(time.monotonic() - started, 6) + self.assertGreaterEqual(json.loads(cursor.read_text())["cursor"]["message_index"], 30) + finally: + if process.poll() is None: + process.kill() + process.wait() + process.stdout.close() + process.stderr.close() + + def test_retry_after_seconds_and_http_date_are_honoured(self): + for retry_after in (2, lambda: email.utils.formatdate(time.time() + 4, usegmt=True)): + with self.subTest(retry_after=retry_after): + server = self.server([[candle(120000)]]) + server.rate_once = True + server.retry_after = retry_after + state = self.directory / "state" + if state.exists(): + import shutil + + shutil.rmtree(state) + output, result = self.run_feed(server, extra=["--max-messages", "1"]) + self.assertEqual(len(output), 1) + self.assertIn("rest_rate_limited", result.stderr) + limited, retried = [ + moment + for path, moment in server.request_times + if path == "/api/v3/exchangeInfo" + ] + self.assertGreaterEqual(retried - limited, 1.9) + + def test_429_retry_after_preserves_prefix(self): + server = self.server([[candle(240000)]]) + server.rate_once = True + output, result = self.run_feed(server, extra=["--max-messages", "3"]) + self.assertEqual(len(output), 3) + self.assertIn("rest_rate_limited", result.stderr) + + def test_replay_and_queue_budgets_stop_22(self): + server = self.server([[candle(120000)]]) + output, _ = self.run_feed(server, extra=["--max-log-bytes", "4"], expected=22) + self.assertEqual(output, []) + import shutil + + shutil.rmtree(self.directory / "state") + server = self.server([[trade(100)]]) + output, _ = self.run_feed(server, "ticks", ["--max-queue-bytes", "64"], expected=22) + self.assertEqual(output, []) + + def test_sigterm_then_resume_tail_has_no_gap_or_duplicate(self): + server = self.server([[candle(120000)], [candle(120000), candle(240000)]]) + process = subprocess.Popen( + self.command(server), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + cursor = self.directory / "state" / "cursor.json" + deadline = time.monotonic() + 8 + while time.monotonic() < deadline: + if cursor.exists() and json.loads(cursor.read_text())["cursor"]["message_index"] == 1: + break + self.assertIsNone(process.poll()) + time.sleep(0.02) + else: + process.kill() + self.fail("first atomic message did not commit") + process.send_signal(signal.SIGTERM) + stdout, stderr = process.communicate(timeout=5) + self.assertEqual(process.returncode, 0, stderr) + self.assertEqual(len(stdout.splitlines()), 1) + output, _ = self.run_feed( + server, extra=["--output-from", "1", "--max-messages", "2"], resume=True + ) + self.assertEqual([event["bar"]["ts_open"] for event in output], [180000, 240000]) + + def test_changed_committed_journal_is_refused(self): + server = self.server([[candle(120000)], [candle(180000)]]) + self.run_feed(server, extra=["--max-messages", "1"]) + journal = self.directory / "state" / "events.jsonl" + journal.write_text(journal.read_text().replace("10.10000000", "10.20000000")) + output, _ = self.run_feed(server, resume=True, expected=21) + self.assertEqual(output, []) + + def warmup(self, server): + port = server.server_address[1] + output = self.directory / "warmup.csv" + command = [ + BINARY, + "warmup", + "--venue", + "binance", + "--market", + "spot", + "--symbol", + "TESTUSDT", + "--start", + "120000", + "--end", + "240000", + "--output", + str(output), + "--rest-url", + f"http://127.0.0.1:{port}", + "--ws-url", + f"ws://127.0.0.1:{port}", + "--allow-insecure-http", + ] + return output, subprocess.run(command, capture_output=True, text=True, timeout=8) + + def test_warmup_rest_rows_must_equal_confirmed_ws_bars(self): + output, result = self.warmup(self.server([[candle(180000, v="0.70000000")]])) + self.assertEqual(result.returncode, 21, result.stderr) + self.assertFalse(output.exists()) + output, result = self.warmup(self.server([[candle(180000)]])) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + json.loads(pathlib.Path(str(output) + ".manifest.json").read_text())[ + "ws_cross_checked_bars" + ], + 1, + ) + + def test_warmup_exclusive_cut_manifest_and_tokens(self): + server = self.server([[candle(240000)]]) + port = server.server_address[1] + output = self.directory / "warmup.csv" + command = [ + BINARY, + "warmup", + "--venue", + "binance", + "--market", + "spot", + "--symbol", + "TESTUSDT", + "--start", + "120000", + "--end", + "240000", + "--output", + str(output), + "--rest-url", + f"http://127.0.0.1:{port}", + "--ws-url", + f"ws://127.0.0.1:{port}", + "--allow-insecure-http", + ] + result = subprocess.run(command, capture_output=True, text=True, timeout=8) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "") + manifest = json.loads(pathlib.Path(str(output) + ".manifest.json").read_text()) + self.assertEqual(manifest["cut"], 240000) + self.assertEqual(manifest["bars"], 2) + self.assertEqual(manifest["sha256"], hashlib.sha256(output.read_bytes()).hexdigest()) + self.assertIn("10.10000000", output.read_text()) + + +if __name__ == "__main__": + suite = unittest.defaultTestLoader.loadTestsFromTestCase(FeedMockTests) + result = unittest.TextTestRunner(verbosity=2).run(suite) + if result.wasSuccessful(): + print(f"PASS local HTTP+WS mock venue ({result.testsRun} scenarios)", flush=True) + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/native/feed/tests/public_e2e.py b/native/feed/tests/public_e2e.py new file mode 100644 index 0000000..4142d8f --- /dev/null +++ b/native/feed/tests/public_e2e.py @@ -0,0 +1,491 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Explicit public-data soak; captures stay in the operator-selected directory.""" + +import argparse +import hashlib +import http.server +import itertools +import json +import os +import pathlib +import signal +import struct +import subprocess +import threading +import time +import traceback +import urllib.parse +import urllib.request +from decimal import Decimal, getcontext + + +def rows(path, exact=False): + with open(path) as source: + return [ + json.loads(line, parse_float=Decimal if exact else float) + for line in source + if line.strip() + ] + + +SCRIPT_TF_MS = 60000 # the runner and batch both run --script-tf 1 + + +def action_key(record, mode): + """Comparable action fields. Ticks use the predeclared R-B2 mapping: an action timestamp + becomes floor(ts / script_tf) * script_tf on both sides, because a tick fill carries its + print's time while the OHLC batch carries the modeled segment's clock. Every other field + stays exact.""" + order = record["order"] + timestamp = record["timestamp"] + if mode == "ticks": + timestamp = timestamp // SCRIPT_TF_MS * SCRIPT_TF_MS + return ( + timestamp, + record["bar_index"], + order["id"], + order["action"], + order["leg"], + struct.pack("!d", float(order["contracts"])).hex(), + struct.pack("!d", float(order["price"])).hex(), + order["reduce_only"], + order["entry_incarnation"], + ) + + +def proven_actions(records, mode, window): + """Runner actions on the bars the batch replays, and how many fell after them. The batch + sees only proven minutes; ticks of a minute without its `time` event can still drive + actions in the runner, and those have no batch counterpart.""" + inside = [action_key(record, mode) for record in records if record["bar_index"] < window] + return inside, len(records) - len(inside) + + +class Receiver(http.server.ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, directory): + super().__init__(("127.0.0.1", 0), ReceiverHandler) + self.directory = directory + self.lock = threading.Lock() + self.payloads = {} + threading.Thread(target=self.serve_forever, daemon=True).start() + + +class ReceiverHandler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + name = self.path.removeprefix("/actions/") + if not name.replace("-", "").isalnum(): + self.send_error(404) + return + payload = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + with self.server.lock: + self.server.payloads.setdefault(name, []).append(payload) + with open(self.server.directory / (name + "-actions.jsonl"), "a") as output: + output.write(json.dumps(payload, separators=(",", ":")) + "\n") + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, *_): + pass + + +class Soak: + def __init__(self, options): + self.options = options + self.directory = pathlib.Path(options.directory).resolve() + self.directory.mkdir(parents=True, exist_ok=False) + self.receiver = Receiver(self.directory) + self.cut = int(time.time() * 1000) // 60000 * 60000 + self.results = {} + self.lock = threading.Lock() + + def receipt(self, message): + with self.lock: + print(message, flush=True) + with open(self.directory / "receipts.log", "a") as output: + output.write(message + "\n") + + def runner_command(self, mode, name, ledger, source, cursor=0): + command = [ + self.options.runner, + "run", + "--strategy", + self.options.strategy, + "--warmup", + str(self.directory / "warmup.csv"), + "--feed", + source, + "--mode", + mode, + "--script-tf", + "1", + "--ledger", + str(ledger), + "--symbol", + "BINANCE:BTCUSDT", + "--name", + "public-rsi-qualification", + "--webhook-url", + f"http://127.0.0.1:{self.receiver.server_port}/actions/{name}", + "--allow-insecure-http", + ] + if cursor: + command += ["--from-input", str(cursor)] + return command + + def warmup(self): + command = [ + self.options.feed, + "warmup", + "--venue", + "binance", + "--market", + "spot", + "--symbol", + "BTCUSDT", + "--start", + str(self.cut - 200 * 60000), + "--end", + str(self.cut), + "--output", + str(self.directory / "warmup.csv"), + ] + with open(self.directory / "warmup.stderr", "w") as error: + subprocess.run(command, stderr=error, check=True, timeout=150) + manifest = json.loads((self.directory / "warmup.csv.manifest.json").read_text()) + assert manifest["end_exclusive"] == self.cut + assert ( + manifest["sha256"] + == hashlib.sha256((self.directory / "warmup.csv").read_bytes()).hexdigest() + ) + self.receipt("PASS warmup exclusive cut and SHA-256") + + def generation(self, mode, generation, cursor): + command = [ + self.options.feed, + "run", + "--venue", + "binance", + "--market", + "spot", + "--symbol", + "BTCUSDT", + "--mode", + mode, + "--state-dir", + str(self.directory / (mode + "-state")), + ] + command += ( + ["--resume", "--output-from", str(cursor)] if generation else ["--start", str(self.cut)] + ) + files = [ + os.open(self.directory / name, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644) + for name in ( + f"{mode}-feed-{generation}.stderr", + f"{mode}-runner-{generation}.stderr", + f"{mode}-runner-{generation}.stdout", + ) + ] + feed_error, runner_error, runner_output = files + runner = subprocess.Popen( + self.runner_command(mode, mode, self.directory / (mode + ".sqlite3"), "-", cursor), + stdin=subprocess.PIPE, + stdout=runner_output, + stderr=runner_error, + ) + feed = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=feed_error) + failures = [] + + def pump(): + try: + with open(self.directory / (mode + "-feed.jsonl"), "ab", buffering=0) as capture: + for line in feed.stdout: + if not line.endswith(b"\n"): + continue + runner.stdin.write(line) + runner.stdin.flush() + capture.write(line) + except Exception as failure: + failures.append(str(failure)) + finally: + runner.stdin.close() + + thread = threading.Thread(target=pump, daemon=True) + thread.start() + return feed, runner, thread, failures, files + + def finish(self, mode, generation, processes, crash=False): + feed, runner, thread, failures, files = processes + if feed.poll() is None: + feed.send_signal(signal.SIGKILL if crash else signal.SIGTERM) + code = feed.wait(timeout=20) + thread.join(timeout=30) + assert not thread.is_alive(), "pipe drain exceeded deadline" + runner_code = runner.wait(timeout=40) + for descriptor in files: + os.close(descriptor) + assert code == (-signal.SIGKILL if crash else 0), f"feed exit={code}" + assert runner_code == 0, f"runner exit={runner_code}" + assert not failures, failures + summary = json.loads((self.directory / f"{mode}-runner-{generation}.stdout").read_text()) + return summary["inputs_committed"] + + def fetch(self, path, query, weight=2): + time.sleep(weight / 10) + url = "https://api.binance.com" + path + "?" + urllib.parse.urlencode(query) + with urllib.request.urlopen(url, timeout=30) as response: + return json.loads(response.read(), parse_float=Decimal) + + def candles(self, start, end): + result = [] + while start < end: + page = self.fetch( + "/api/v3/klines", + { + "symbol": "BTCUSDT", + "interval": "1m", + "startTime": start, + "endTime": end - 1, + "limit": 1000, + }, + ) + assert page and page[0][0] == start, "REST minute missing" + result += page + start = page[-1][0] + 60000 + assert len(result) * 60000 == end - result[0][0] + return result + + def validate(self, mode, duration, cursor, minutes): + getcontext().prec = 160 + tape = rows(self.directory / (mode + "-feed.jsonl"), True) + with open(self.directory / (mode + "-feed.jsonl")) as source: + tokens = [ + json.loads(line, parse_float=str, parse_int=str) for line in source if line.strip() + ] + assert len(tape) == cursor, "capture/runner cursor mismatch" + durable = rows(self.directory / (mode + "-state/events.jsonl"), True) + assert tape == durable, "restart changed normalized prefix or message boundaries" + self.receipt(f"PASS {mode} restart without gap or duplicate messages={cursor}") + if mode == "bars": + normalized = [event["bar"] for event in tape] + assert len(normalized) >= minutes - 1 and duration >= (minutes - 1) * 60 + expected = self.candles(self.cut, normalized[-1]["ts_open"] + 60000) + for index, (actual, candle) in enumerate(zip(normalized, expected, strict=True)): + assert [actual["ts_open"], *(actual[key] for key in ("o", "h", "l", "c", "v"))] == [ + candle[0], + *(Decimal(candle[index]) for index in range(1, 6)), + ], "bar REST mismatch" + assert tape[index]["type"] == "bar" + assert [tokens[index]["bar"][key] for key in ("o", "h", "l", "c", "v")] == candle[ + 1:6 + ], "bar decimal tokens changed" + self.receipt( + f"PASS bars equal REST minutes={len(normalized)} duration_seconds={duration:.3f}" + ) + else: + assert duration >= (minutes - 1) * 60 + trades = [event for event in tape if event["type"] == "tick"] + trade_tokens = [event for event in tokens if event["type"] == "tick"] + times = [event["ts"] for event in tape if event["type"] == "time"] + assert len(times) >= minutes - 1 and trades + for previous, current in itertools.pairwise(trades): + assert current["seq"] == previous["seq"] + 1 and current["ts"] >= previous["ts"] + next_id = trades[0]["seq"] + trade_offset = 0 + while trade_offset < len(trades): + page = self.fetch( + "/api/v3/historicalTrades", + {"symbol": "BTCUSDT", "fromId": next_id, "limit": 1000}, + 25, + ) + assert page and page[0]["id"] == next_id + for raw in page: + if trade_offset == len(trades): + break + tick = trades[trade_offset] + assert [tick["seq"], tick["ts"], tick["price"], tick["qty"]] == [ + raw["id"], + raw["time"], + Decimal(raw["price"]), + Decimal(raw["qty"]), + ], "raw REST mismatch" + assert [trade_tokens[trade_offset][key] for key in ("price", "qty")] == [ + raw["price"], + raw["qty"], + ], "raw decimal tokens changed" + trade_offset += 1 + next_id += 1 + expected = self.candles(self.cut, times[-1]) + normalized = [] + minute_trades = [] + minute = self.cut + for event in tape: + if event["type"] == "tick": + assert minute <= event["ts"] < minute + 60000, "time/print ordering violation" + minute_trades.append(event) + else: + assert event["type"] == "time" and event["ts"] == minute + 60000 + raw = expected[len(normalized)] + prices = [tick["price"] for tick in minute_trades] + assert prices and len(prices) == raw[8] + values = [ + prices[0], + max(prices), + min(prices), + prices[-1], + sum(tick["qty"] for tick in minute_trades), + ] + assert values == [Decimal(raw[index]) for index in range(1, 6)], ( + "time OHLCV mismatch" + ) + normalized.append( + dict( + zip( + ("ts_open", "o", "h", "l", "c", "v"), [minute, *values], strict=True + ) + ) + ) + minute += 60000 + minute_trades = [] + self.receipt( + f"PASS ticks contiguous and time events equal REST ticks={len(trades)} " + f"minutes={len(times)} duration_seconds={duration:.3f}" + ) + combined = self.directory / (mode + "-combined.csv") + with open(combined, "w") as output: + output.write((self.directory / "warmup.csv").read_text()) + for bar in normalized: + output.write( + ",".join(str(bar[key]) for key in ("ts_open", "o", "h", "l", "c", "v")) + "\n" + ) + batch_actions = self.directory / (mode + "-batch-actions.jsonl") + subprocess.run( + [ + self.options.batch_probe, + self.options.observed_strategy, + str(combined), + str(batch_actions), + ], + check=True, + ) + expected_actions = [ + action_key(record, mode) + for record in rows(batch_actions) + if record["origin_input_index"] >= 200 + ] + actual_actions, trailing = proven_actions( + self.receiver.payloads.get(mode, []), mode, 200 + len(normalized) + ) + event_ids = [record["event_id"] for record in self.receiver.payloads.get(mode, [])] + assert len(event_ids) == len(set(event_ids)), "duplicate delivered action after restart" + equal = expected_actions == actual_actions + (self.directory / (mode + "-action-comparison.json")).write_text( + json.dumps( + {"equal": equal, "batch": expected_actions, "runner": actual_actions}, indent=2 + ) + + "\n" + ) + mapping = " (R-B2 timestamps)" if mode == "ticks" else "" + verdict = "PASS" if equal else "FAIL" + self.receipt( + f"{verdict} {mode} actions equal batch via run_backtest_full{mapping} " + f"batch={len(expected_actions)} runner={len(actual_actions)} " + f"after_last_proven_minute={trailing}" + ) + if mode == "ticks": + name = "ticks-replay" + completed = subprocess.run( + self.runner_command( + mode, + name, + self.directory / "ticks-replay.sqlite3", + str(self.directory / "ticks-feed.jsonl"), + ), + capture_output=True, + text=True, + timeout=600, + ) + (self.directory / "ticks-replay.stdout").write_text(completed.stdout) + (self.directory / "ticks-replay.stderr").write_text(completed.stderr) + assert completed.returncode == 0, "tick replay runner failed" + raw = [action_key(record, "bars") for record in self.receiver.payloads.get(mode, [])] + assert raw == [ + action_key(record, "bars") for record in self.receiver.payloads.get(name, []) + ] + self.receipt(f"PASS ticks same-print runner replay actions equal actions={len(raw)}") + return { + "duration_seconds": duration, + "messages": cursor, + "minutes": len(normalized), + "actions": len(actual_actions), + "actions_after_last_proven_minute": trailing, + "batch_actions_equal": equal, + } + + def live(self, mode, minutes): + processes = None + try: + started = time.monotonic() + processes = self.generation(mode, 0, 0) + self.receipt(f"START {mode} public BTCUSDT cut={self.cut} minimum_minutes={minutes}") + time.sleep(self.options.restart_seconds) + assert processes[0].poll() is None and processes[1].poll() is None, ( + "process stopped before restart" + ) + cursor = self.finish(mode, 0, processes, crash=True) + processes = self.generation(mode, 1, cursor) + self.receipt(f"RESTART {mode} --output-from={cursor} --from-input={cursor}") + while time.monotonic() - started < minutes * 60: + assert processes[0].poll() is None and processes[1].poll() is None, ( + "live process stopped early" + ) + time.sleep(5) + cursor = self.finish(mode, 1, processes) + processes = None + duration = time.monotonic() - started + self.results[mode] = self.validate(mode, duration, cursor, minutes) + except Exception: + if processes: + for process in processes[:2]: + if process.poll() is None: + process.terminate() + for process in processes[:2]: + try: + process.wait(timeout=20) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + failure = traceback.format_exc() + (self.directory / (mode + "-failure.txt")).write_text(failure) + self.receipt(f"FAIL {mode} qualification; see {mode}-failure.txt") + self.results[mode] = {"error": failure} + + def run(self): + self.warmup() + threads = [ + threading.Thread(target=self.live, args=("bars", self.options.bar_minutes)), + threading.Thread(target=self.live, args=("ticks", self.options.tick_minutes)), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.receiver.shutdown() + (self.directory / "summary.json").write_text(json.dumps(self.results, indent=2) + "\n") + failed = any( + "error" in result or not result.get("batch_actions_equal") + for result in self.results.values() + ) + return 1 if failed else 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + for argument in ("feed", "runner", "strategy", "observed-strategy", "batch-probe", "directory"): + parser.add_argument("--" + argument, required=True) + parser.add_argument("--bar-minutes", type=int, default=46) + parser.add_argument("--tick-minutes", type=int, default=21) + parser.add_argument("--restart-seconds", type=int, default=180) + raise SystemExit(Soak(parser.parse_args()).run()) diff --git a/native/feed/tests/unit.cpp b/native/feed/tests/unit.cpp new file mode 100644 index 0000000..5fe6967 --- /dev/null +++ b/native/feed/tests/unit.cpp @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "binance.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge::feed; +namespace { +struct Temporary { + std::string path; + Temporary() { + char pattern[] = "/tmp/pineforge-feed-test-XXXXXX"; + const auto* result = ::mkdtemp(pattern); + assert(result); + path = result; + } + ~Temporary() { std::filesystem::remove_all(path); } +}; +template void expect(int code, Function operation) { + bool caught = false; + try { operation(); } + catch (const Error& failure) { assert(failure.code == code); caught = true; } + assert(caught); +} +void passed(const char* name) { std::cout << "PASS " << name << '\n'; } +Config config(const Temporary& temporary, const std::string& mode) { + Config value; + value.symbol = "TESTUSDT"; + value.mode = mode; + value.start = 120000; + value.state_dir = temporary.path; + return value; +} +struct SyntheticVenue final : Venue { + std::map trades{ + {99, {99, 119999, "10.10000000", "0.10000000"}}, + {100, {100, 120001, "10.10000000", "0.10000000"}}, + {101, {101, 120050, "11.20000000", "0.20000000"}}, + {102, {102, 179999, "9.90000000", "0.30000000"}}, + {103, {103, 180001, "12.00000000", "0.40000000"}}, + {104, {104, 180050, "11.00000000", "0.10000000"}}, + {105, {105, 240001, "12.00000000", "0.10000000"}} + }; + std::map bars{ + {120000, {{120000, "10.10000000", "11.20000000", "9.90000000", "9.90000000", "0.60000000"}, 100, 102, 3, true}}, + {180000, {{180000, "12.00000000", "12.00000000", "11.00000000", "11.00000000", "0.50000000"}, 103, 104, 2, true}}, + {240000, {{240000, "12.00000000", "12.00000000", "12.00000000", "12.00000000", "0.10000000"}, 105, 105, 1, true}} + }; + std::string streams() const override { return "test@trade/test@kline_1m"; } + VenueEvent decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } + std::size_t pages = 0; + std::vector history(std::uint64_t from, std::size_t limit) override { + ++pages; + std::vector result; + for (std::size_t offset = 0; offset < limit; ++offset) { + const auto found = trades.find(from + offset); + if (found == trades.end()) break; + result.push_back(found->second); + } + if (result.empty()) throw Error(20, "synthetic exhausted history"); + return result; + } + std::uint64_t first_trade_id(std::int64_t minute) override { return static_cast(bars.at(minute).first); } + // REST rows carry no x=true and no f..L, exactly like the venue's klines endpoint. + std::vector klines(std::int64_t start, std::int64_t end) override { + std::vector result; + for (auto minute = start; minute < end; minute += 60000) { + if (!bars.count(minute)) throw Error(20, "synthetic missing minute"); + auto row = bars.at(minute); + row.first = row.last = -1; + row.confirmed = false; + result.push_back(row); + } + return result; + } + VenueEvent tick(std::uint64_t id) const { return {VenueEvent::Kind::Trade, trades.at(id), {}}; } + VenueEvent close(std::int64_t minute) const { return {VenueEvent::Kind::Kline, {}, bars.at(minute)}; } +}; +} + +int main() { + assert(Decimal("0.10000000").add(Decimal("0.20000000")) == Decimal("0.3")); + assert(Decimal("0").compare(Decimal("0.00000001")) < 0); + assert(Decimal("9007199254740993.00000001").compare(Decimal("9007199254740993.00000000")) > 0); + assert(Decimal("9999999999999999.99").add(Decimal("0.01")) == Decimal("10000000000000000.00")); + for (const auto* invalid : {"", "-1", "+1", "01", "1.", ".1", "NaN", "1e3", "1.2.3"}) expect(23, [&] { Decimal value(invalid); (void)value; }); + expect(23, [] { Decimal value("0." + std::string(33, '1')); (void)value; }); + Trade exact{9007199254740993ULL, 120001, "123.45000000", "0.00000001"}; + assert(normalized_trade(parse_json(exact.wire())) == exact); + expect(23, [] { Trade{1, 1, "1", "0"}.validate(); }); + passed("decimal_tokens_and_uint64_identity"); + + const auto raw = parse_json(R"({"e":"trade","E":999999,"s":"TESTUSDT","t":100,"p":"10.10000000","q":"0.10000000","T":120001,"m":false})"); + assert(binance_trade(raw, true).ts == 120001); + const auto source = parse_json(R"({"t":120000,"T":179999,"s":"TESTUSDT","i":"1m","f":100,"L":102,"o":"10.10000000","h":"11.20000000","l":"9.90000000","c":"9.90000000","v":"0.60000000","n":3,"x":false})"); + assert(!binance_kline(source, true).confirmed); + auto bad_source = source; + bad_source.members["T"] = Json::number("180000"); + expect(23, [&] { binance_kline(bad_source, true); }); + bad_source = source; + bad_source.members["x"] = Json::string("true"); + expect(23, [&] { binance_kline(bad_source, true); }); + passed("binance_matched_time_confirmation_and_inclusive_close"); + + { + Temporary temporary; + SyntheticVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.connected(); + session.ingest(venue.tick(102)); + assert(output.size() == 3); + session.ingest(venue.tick(100)); + assert(output.size() == 3); + session.ingest(venue.tick(103)); + assert(output.size() == 3); + auto unconfirmed = venue.close(120000); + unconfirmed.kline.confirmed = false; + session.ingest(unconfirmed); + assert(output.size() == 3); + session.ingest(venue.close(120000)); + assert(output.size() == 5); + assert(parse_json(output[3]).at("type").text() == "time"); + assert(parse_json(output[4]).at("seq").integer() == 103); + session.ingest(venue.close(120000)); + auto conflict = venue.tick(100); + conflict.trade.qty = "0.20000000"; + expect(21, [&] { session.ingest(conflict); }); + passed("raw_gap_healing_reordering_identical_and_conflicting_duplicates"); + passed("time_requires_full_ids_count_exact_ohlcv_and_precedes_next_minute"); + } + { + Temporary temporary; + SyntheticVenue venue; + State state(config(temporary, "ticks")); + FeedSession session(state, venue, [](const auto&) {}); + auto revised = venue.close(120000); + revised.kline.bar.volume = "0.61000000"; + expect(21, [&] { session.ingest(revised); }); + assert(state.cursor().seq == 102 && state.cursor().cut == 120000); + passed("exact_ohlcv_mismatch_stops_without_time"); + } + { + Temporary temporary; + SyntheticVenue venue; + venue.trades.erase(101); + State state(config(temporary, "ticks")); + FeedSession session(state, venue, [](const auto&) {}); + expect(20, [&] { session.ingest(venue.tick(102)); }); + assert(state.cursor().seq == 100); + passed("unhealable_raw_hole_retains_verified_cursor"); + } + { + Temporary temporary; + SyntheticVenue venue; + State state(config(temporary, "bars")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + auto forming = venue.close(240000); + forming.kline.confirmed = false; + session.ingest(forming); + assert(output.empty()); + session.ingest(venue.close(240000)); + assert(output.size() == 3); + assert(normalized_bar(parse_json(output.front())).ts == 120000); + session.ingest(venue.close(180000)); + assert(output.size() == 3); + auto revised = venue.close(180000); + revised.kline.bar.volume = "0.60000000"; + expect(21, [&] { session.ingest(revised); }); + venue.bars.at(240000).bar.volume = "0.20000000"; + expect(21, [&] { session.connected(); }); + passed("confirmed_bars_watermark_healing_duplicate_and_revised_overlap"); + } + { + Temporary temporary; + SyntheticVenue venue; + auto options = config(temporary, "ticks"); + std::string epoch; + { + State state(options); + FeedSession session(state, venue, [](const auto&) {}); + session.ingest(venue.tick(100)); + session.ingest(venue.tick(101)); + epoch = state.cursor().epoch; + auto second = options; + second.resume = true; + expect(22, [&] { State locked(second); }); + } + { std::ofstream tail(temporary.path + "/events.jsonl", std::ios::app); tail << "uncommitted-torn-tail"; } + options.resume = true; + options.start = -1; + options.output_from = 2; + { + State state(options); + assert(state.cursor().epoch == epoch); + assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == state.cursor().log_bytes); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.connected(); + assert(output.empty()); + session.ingest(venue.close(120000)); + assert(output.size() == 2); + assert(parse_json(output.front()).at("seq").integer() == 102); + assert(parse_json(output.back()).at("type").text() == "time"); + } + { + std::fstream corrupt(temporary.path + "/events.jsonl", std::ios::in | std::ios::out); + corrupt.seekp(0); + corrupt << 'X'; + } + expect(21, [&] { State changed(options); }); + passed("atomic_cursor_lock_torn_tail_verified_restart_and_changed_prefix"); + } + { + Temporary temporary; + SyntheticVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.close(180000)); + assert(state.cursor().cut == 240000 && state.cursor().seq == 104); + assert(output.size() == 7); + assert(parse_json(output[3]).at("type").text() == "time"); + passed("historical_ticks_require_closed_watermark_next_id_fence_and_rest_reconciliation"); + } + { + Temporary temporary; + auto options = config(temporary, "bars"); + options.max_log_bytes = 4; + State state(options); + SyntheticVenue venue; + FeedSession session(state, venue, [](const auto&) {}); + expect(22, [&] { session.ingest(venue.close(120000)); }); + assert(state.cursor().message_index == 0); + passed("bounded_replay_budget_stops_before_commit"); + } + { + Temporary temporary; + State state(config(temporary, "ticks")); + SyntheticVenue venue; + FeedSession session(state, venue, [](const auto&) {}); + session.ingest(venue.close(120000)); + auto revised = venue.close(120000); + ++revised.kline.first; + ++revised.kline.last; + expect(21, [&] { session.ingest(revised); }); + ++venue.bars.at(120000).count; + expect(21, [&] { session.connected(); }); + assert(state.cursor().message_index == 4); + passed("closed_tick_fences_and_rest_counts_are_immutable"); + } + { + Temporary temporary; + auto options = config(temporary, "bars"); + { + State state(options); + SyntheticVenue venue; + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + std::filesystem::create_directory(temporary.path + "/cursor.json.tmp"); + expect(22, [&] { session.ingest(venue.close(120000)); }); + assert(output.empty()); + assert(state.durable().message_index == 0); + expect(22, [&] { session.publish(); }); + assert(output.empty()); + } + std::filesystem::remove(temporary.path + "/cursor.json.tmp"); + options.resume = true; + State recovered(options); + assert(recovered.cursor().message_index == 0); + assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == 0); + passed("failed_cursor_write_rolls_back_uncommitted_message_on_resume"); + } + { + Temporary temporary; + auto options = config(temporary, "bars"); + options.max_replay_seconds = 1; + State state(options); + SyntheticVenue venue; + FeedSession session(state, venue, [](const auto&) {}); + session.ingest(venue.close(120000)); + session.ingest(venue.close(180000)); + expect(22, [&] { state.visit(0, [](const auto&) { std::this_thread::sleep_for(std::chrono::milliseconds(1100)); }); }); + passed("replay_time_budget_stops_without_mutating_cursor"); + } + { + Temporary temporary; + SyntheticVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.connected(); + session.ingest(venue.tick(100)); + session.ingest(venue.tick(101)); + assert(output.size() == 2); + venue.trades.at(100).qty = "0.10000001"; + expect(21, [&] { session.connected(); }); + assert(output.size() == 2 && state.durable().message_index == 2); + passed("reconnect_raw_overlap_change_stops_without_output"); + } + { + Temporary temporary; + SyntheticVenue venue; + venue.trades.erase(venue.trades.find(101), venue.trades.end()); + for (std::uint64_t id = 100; id < 2600; ++id) + venue.trades[id] = {id, 120001 + static_cast(id - 100) * 20, "10.00000000", "0.00000001"}; + venue.trades[2600] = {2600, 180001, "10.00000000", "0.00000001"}; + venue.bars.at(120000) = {{120000, "10.00000000", "10.00000000", "10.00000000", "10.00000000", "0.00002500"}, 100, 2599, 2500, true}; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.tick(2599)); + session.ingest(venue.close(120000)); + assert(output.size() == 2501 && venue.pages >= 4); + for (std::size_t index = 0; index < 2500; ++index) + assert(parse_json(output[index]).at("seq").integer() == 100 + index); + assert(parse_json(output.back()).at("type").text() == "time"); + passed("multi_page_raw_healing_stays_contiguous"); + } + { + Temporary temporary; + SyntheticVenue venue; + venue.trades.at(101).ts = 120000; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.tick(100)); + expect(23, [&] { session.ingest(venue.tick(101)); }); + assert(output.size() == 1 && state.durable().seq == 100); + passed("matched_time_regression_stops_23"); + } + { + Temporary temporary; + SyntheticVenue venue; + ++venue.bars.at(120000).count; + State state(config(temporary, "ticks")); + std::vector output; + FeedSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + expect(20, [&] { session.ingest(venue.close(180000)); }); + assert(state.durable().cut == 120000); + for (const auto& line : output) assert(parse_json(line).at("type").text() == "tick"); + venue.bars.at(120000).count = 3; + venue.trades.erase(103); + Temporary second; + State fresh(config(second, "ticks")); + FeedSession fenced(fresh, venue, [](const auto&) {}); + expect(20, [&] { fenced.ingest(venue.close(180000)); }); + assert(fresh.durable().cut == 120000); + passed("historical_closure_needs_rest_count_and_next_minute_fence"); + } + { + Temporary temporary; + State state(config(temporary, "bars")); + const Bar bar{120000, "10.10000000", "11.20000000", "9.90000000", "9.90000000", "0.60000000"}; + state.stage(bar.wire(), bar); + assert(state.staged() == 1 && state.cursor().message_index == 1 && state.durable().message_index == 0); + assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == 0); + std::size_t seen = 0; + state.visit(0, [&](const auto&) { ++seen; }); + assert(seen == 1 && state.bar(120000)); + const auto lines = state.flush(); + assert(lines.size() == 1 && state.staged() == 0 && state.durable().message_index == 1); + assert(std::filesystem::file_size(temporary.path + "/events.jsonl") == lines.front().size() + 1); + passed("group_commit_stages_then_persists_before_publication"); + } + { + Temporary temporary; + { std::ofstream empty(temporary.path + "/events.jsonl"); } + { State state(config(temporary, "bars")); assert(state.durable().message_index == 0); } + Temporary second; + { std::ofstream used(second.path + "/events.jsonl"); used << "x\n"; } + expect(23, [&] { State state(config(second, "bars")); }); + passed("crashed_initialization_without_cursor_is_reinitialized"); + } + assert(retry_after_seconds("120", 0) == 120); + assert(retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT", 1445412475) == 5); + assert(retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT", 1445412490) == 0); + for (const auto* invalid : {"", "soon", "-1", "1.5", "Wed, 21 Oct 2015 07:28:00 UTC", "21 Oct 2015 07:28:00 GMT"}) + expect(23, [&] { retry_after_seconds(invalid, 0); }); + passed("retry_after_delta_seconds_and_http_date"); + assert(timestamp("1970-01-01T00:02:00Z") == 120000); + expect(23, [] { timestamp("2026-02-30T00:00:00Z"); }); + passed("strict_utc_start_and_exclusive_cut"); + std::cout << "PASS feed unit suite\n"; +} diff --git a/native/feed/vendor/json.hpp b/native/feed/vendor/json.hpp new file mode 100644 index 0000000..4de7f0f --- /dev/null +++ b/native/feed/vendor/json.hpp @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::live { + +// Bounded JSON for the native runner's configuration and JSONL wire format. +// Number tokens remain decimal strings so event identities never round uint64. +struct Json { + enum class Kind { Null, Bool, Number, String, Array, Object }; + Kind kind = Kind::Null; + std::string value; + std::vector items; + std::map members; + + static Json string(std::string v) { + Json j; + j.kind = Kind::String; + j.value = std::move(v); + return j; + } + static Json number(std::string v) { + Json j; + j.kind = Kind::Number; + j.value = std::move(v); + return j; + } + static Json boolean(bool v) { + Json j; + j.kind = Kind::Bool; + j.value = v ? "true" : "false"; + return j; + } + static Json object(std::map v) { + Json j; + j.kind = Kind::Object; + j.members = std::move(v); + return j; + } + const Json &at(const std::string &key) const { + if (kind != Kind::Object || members.count(key) != 1) + throw std::runtime_error("missing field: " + key); + return members.at(key); + } + const Json *find(const std::string &key) const { + if (kind != Kind::Object) + throw std::runtime_error("expected object"); + auto i = members.find(key); + return i == members.end() ? nullptr : &i->second; + } + std::string text() const { + if (kind != Kind::String) + throw std::runtime_error("expected string"); + return value; + } + double real() const { + if (kind != Kind::Number) + throw std::runtime_error("expected number"); + std::size_t n = 0; + double d = std::stod(value, &n); + if (n != value.size() || !std::isfinite(d)) + throw std::runtime_error("nonfinite number"); + return d; + } + template T integer() const { + if (kind != Kind::Number) + throw std::runtime_error("expected integer"); + T result{}; + auto r = std::from_chars(value.data(), value.data() + value.size(), result); + if (r.ec != std::errc{} || r.ptr != value.data() + value.size()) + throw std::runtime_error("invalid or overflowing integer"); + return result; + } + static std::string quote(std::string_view s) { + static constexpr char hex[] = "0123456789abcdef"; + std::string out = "\""; + for (unsigned char c : s) { + if (c == '"' || c == '\\') { + out += '\\'; + out += static_cast(c); + } else if (c < 32) { + out += "\\u00"; + out += hex[c >> 4]; + out += hex[c & 15]; + } else + out += static_cast(c); + } + return out + '"'; + } + std::string dump() const { + if (kind == Kind::Null) + return "null"; + if (kind == Kind::String) + return quote(value); + if (kind == Kind::Bool || kind == Kind::Number) + return value; + std::string out = kind == Kind::Array ? "[" : "{"; + bool first = true; + if (kind == Kind::Array) + for (const auto &i : items) { + if (!first) + out += ','; + first = false; + out += i.dump(); + } + else + for (const auto &[key, i] : members) { + if (!first) + out += ','; + first = false; + out += quote(key) + ':' + i.dump(); + } + return out + (kind == Kind::Array ? ']' : '}'); + } +}; + +class JsonParser { + std::string_view s_; + std::size_t p_ = 0; + [[noreturn]] void fail() const { + throw std::runtime_error("invalid JSON at byte " + std::to_string(p_)); + } + void ws() { + while (p_ < s_.size() && + (s_[p_] == ' ' || s_[p_] == '\t' || s_[p_] == '\r' || s_[p_] == '\n')) + ++p_; + } + bool take(char c) { + if (p_ < s_.size() && s_[p_] == c) { + ++p_; + return true; + } + return false; + } + unsigned hex4() { + unsigned value = 0; + for (int i = 0; i < 4; ++i) { + if (p_ == s_.size()) + fail(); + char c = s_[p_++]; + value <<= 4; + if (c >= '0' && c <= '9') + value += c - '0'; + else if (c >= 'a' && c <= 'f') + value += c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + value += c - 'A' + 10; + else + fail(); + } + return value; + } + static void utf8(std::string &out, unsigned c) { + if (c < 0x80) + out += static_cast(c); + else if (c < 0x800) { + out += static_cast(0xc0 | (c >> 6)); + out += static_cast(0x80 | (c & 63)); + } else if (c < 0x10000) { + out += static_cast(0xe0 | (c >> 12)); + out += static_cast(0x80 | ((c >> 6) & 63)); + out += static_cast(0x80 | (c & 63)); + } else { + out += static_cast(0xf0 | (c >> 18)); + out += static_cast(0x80 | ((c >> 12) & 63)); + out += static_cast(0x80 | ((c >> 6) & 63)); + out += static_cast(0x80 | (c & 63)); + } + } + std::string str() { + if (!take('"')) + fail(); + std::string out; + while (p_ < s_.size()) { + unsigned char c = s_[p_++]; + if (c == '"') + return out; + if (c < 32) + fail(); + if (c != '\\') { + if (c < 128) { + out += static_cast(c); + continue; + } + unsigned n = c >= 0xf0 ? 3 : c >= 0xe0 ? 2 : c >= 0xc2 ? 1 : 0; + if (!n || c > 0xf4) + fail(); + unsigned point = c & ((1u << (6 - n)) - 1); + for (unsigned i = 0; i < n; ++i) { + if (p_ == s_.size()) + fail(); + unsigned char tail = s_[p_++]; + if ((tail & 0xc0) != 0x80) + fail(); + point = (point << 6) | (tail & 63); + } + if (point < (n == 1 ? 0x80u + : n == 2 ? 0x800u + : 0x10000u) || + point > 0x10ffff || (point >= 0xd800 && point <= 0xdfff)) + fail(); + utf8(out, point); + continue; + } + if (p_ == s_.size()) + fail(); + char e = s_[p_++]; + switch (e) { + case '"': + case '\\': + case '/': + out += e; + break; + case 'b': + out += '\b'; + break; + case 'f': + out += '\f'; + break; + case 'n': + out += '\n'; + break; + case 'r': + out += '\r'; + break; + case 't': + out += '\t'; + break; + case 'u': { + unsigned point = hex4(); + if (point >= 0xd800 && point <= 0xdbff) { + if (!take('\\') || !take('u')) + fail(); + unsigned low = hex4(); + if (low < 0xdc00 || low > 0xdfff) + fail(); + point = 0x10000 + ((point - 0xd800) << 10) + (low - 0xdc00); + } else if (point >= 0xdc00 && point <= 0xdfff) + fail(); + utf8(out, point); + break; + } + default: + fail(); + } + } + fail(); + } + Json parse(unsigned depth) { + if (depth >= 32) + fail(); + ws(); + if (p_ == s_.size()) + fail(); + if (s_[p_] == '"') + return Json::string(str()); + if (take('{')) { + Json j; + j.kind = Json::Kind::Object; + ws(); + if (take('}')) + return j; + do { + ws(); + auto key = str(); + ws(); + if (!take(':')) + fail(); + auto v = parse(depth + 1); + if (!j.members.emplace(std::move(key), std::move(v)).second) + fail(); + ws(); + if (take('}')) + return j; + } while (take(',')); + fail(); + } + if (take('[')) { + Json j; + j.kind = Json::Kind::Array; + ws(); + if (take(']')) + return j; + do { + j.items.push_back(parse(depth + 1)); + ws(); + if (take(']')) + return j; + } while (take(',')); + fail(); + } + for (auto lit : {"null", "true", "false"}) { + std::string_view v = lit; + if (s_.substr(p_, v.size()) == v) { + p_ += v.size(); + return v == "null" ? Json{} : Json::boolean(v == "true"); + } + } + std::size_t begin = p_; + take('-'); + if (take('0')) { + } else { + if (p_ == s_.size() || s_[p_] < '1' || s_[p_] > '9') + fail(); + while (p_ < s_.size() && s_[p_] >= '0' && s_[p_] <= '9') + ++p_; + } + if (take('.')) { + auto old = p_; + while (p_ < s_.size() && s_[p_] >= '0' && s_[p_] <= '9') + ++p_; + if (p_ == old) + fail(); + } + if (take('e') || take('E')) { + if (!take('+')) + take('-'); + auto old = p_; + while (p_ < s_.size() && s_[p_] >= '0' && s_[p_] <= '9') + ++p_; + if (p_ == old) + fail(); + } + return Json::number(std::string(s_.substr(begin, p_ - begin))); + } + + public: + explicit JsonParser(std::string_view s) : s_(s) { + if (s.size() > 1024 * 1024) + throw std::runtime_error("JSON frame exceeds 1 MiB"); + } + Json run() { + auto j = parse(0); + ws(); + if (p_ != s_.size()) + fail(); + return j; + } +}; +inline Json parse_json(std::string_view s) { return JsonParser(s).run(); } +inline void only_fields(const Json &j, std::initializer_list allowed) { + if (j.kind != Json::Kind::Object) + throw std::runtime_error("expected object"); + for (const auto &[key, value] : j.members) { + bool found = false; + for (const auto &a : allowed) + found = found || a == key; + if (!found) + throw std::runtime_error("unknown field: " + key); + } +} +} // namespace pineforge::live diff --git a/pyproject.toml b/pyproject.toml index a72b597..bd27a33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ Source = "https://github.com/pineforge-4pass/pineforge-data" [tool.hatch.build.targets.wheel] packages = ["src/pineforge_data"] +[tool.hatch.build.targets.sdist] +exclude = ["/native"] + [tool.pytest.ini_options] addopts = "-q" testpaths = ["tests"]