diff --git a/.github/actions/setup-live-curl/action.yml b/.github/actions/setup-live-curl/action.yml new file mode 100644 index 000000000..948ec610b --- /dev/null +++ b/.github/actions/setup-live-curl/action.yml @@ -0,0 +1,37 @@ +name: Set up native runner libcurl +description: Cache and build the checksum-pinned WebSocket-enabled libcurl +outputs: + curl-dir: + description: CMake package directory for ci_verify.py --curl-dir + value: ${{ steps.dependencies.outputs.curl-dir }} + cache-hit: + description: Whether the pinned installation was restored from cache + value: ${{ steps.curl-cache.outputs.cache-hit }} +runs: + using: composite + steps: + - name: Identify pinned dependency environment + id: dependencies + shell: bash + run: | + bash scripts/build_live_curl.sh --metadata >> "$GITHUB_OUTPUT" + mkdir -p build-native-deps + { + printf '%s\n' "${ImageOS:-unknown}" "${ImageVersion:-unknown}" "${GITHUB_WORKSPACE}" + cc --version + cmake --version + dpkg-query -W libssl-dev zlib1g-dev + } > build-native-deps/environment.txt + echo "identity=$(sha256sum build-native-deps/environment.txt | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" + echo "curl-dir=${GITHUB_WORKSPACE}/build-native-deps/curl-install/lib/cmake/CURL" >> "$GITHUB_OUTPUT" + - name: Restore pinned WebSocket-enabled libcurl + id: curl-cache + uses: actions/cache@v4 + with: + path: build-native-deps/curl-install + key: curl-${{ steps.dependencies.outputs.version }}-${{ steps.dependencies.outputs.sha256 }}-${{ steps.dependencies.outputs.identity }}-${{ hashFiles('.github/actions/setup-live-curl/action.yml', 'scripts/build_live_curl.sh') }}-${{ runner.os }}-${{ runner.arch }} + - name: Build pinned WebSocket-enabled libcurl + shell: bash + env: + CURL_CACHE_HIT: ${{ steps.curl-cache.outputs.cache-hit }} + run: bash scripts/build_live_curl.sh build-native-deps 4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 731c280f3..dda7ea6e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,7 +284,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libeigen3-dev tzdata-legacy libsqlite3-dev libssl-dev libcurl4-openssl-dev ccache + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cmake ninja-build libeigen3-dev tzdata-legacy libsqlite3-dev libssl-dev zlib1g-dev ccache - name: Restore ccache uses: actions/cache@v4 @@ -294,12 +294,16 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ runner.arch }}-kernel- + - name: Set up pinned WebSocket-enabled libcurl + id: curl-deps + uses: ./.github/actions/setup-live-curl + - name: Verify (kernel) - run: python3 scripts/ci_verify.py kernel --build-dir build-kernel --jobs "$(getconf _NPROCESSORS_ONLN)" --ccache + run: python3 scripts/ci_verify.py kernel --build-dir build-kernel --jobs "$(getconf _NPROCESSORS_ONLN)" --ccache --curl-dir "${{ steps.curl-deps.outputs.curl-dir }}" - name: Stage and summarize diagnostics if: always() - run: python3 scripts/collect_ci_diagnostics.py --build-dir build-kernel --profile kernel + run: python3 scripts/collect_ci_diagnostics.py --build-dir build-kernel --profile kernel --dependency-dir build-native-deps - name: Retain CI diagnostics if: always() diff --git a/.github/workflows/native-live.yml b/.github/workflows/native-live.yml index 1b3d32c09..37fbc5401 100644 --- a/.github/workflows/native-live.yml +++ b/.github/workflows/native-live.yml @@ -24,8 +24,6 @@ env: CCACHE_COMPILERCHECK: content CCACHE_COMPRESS: "1" CCACHE_MAXSIZE: 500M - CURL_VERSION: "8.14.1" - CURL_SHA256: "f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd" # The organization's 16-core larger runner for a push, the schedule, a # dispatch and a pull request from a branch of this repository; every other @@ -65,61 +63,35 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ runner.arch }}-native- - - name: Identify native dependency environment - id: native-deps - shell: bash - run: | - mkdir -p build-native-deps - { - printf '%s\n' "${ImageOS:-unknown}" "${ImageVersion:-unknown}" "${GITHUB_WORKSPACE}" - cc --version - cmake --version - dpkg-query -W libssl-dev zlib1g-dev - } > build-native-deps/environment.txt - echo "identity=$(sha256sum build-native-deps/environment.txt | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" - - - name: Restore pinned WebSocket-enabled libcurl - id: curl-cache - uses: actions/cache@v4 - with: - path: build-native-deps/curl-install - key: curl-${{ env.CURL_VERSION }}-${{ env.CURL_SHA256 }}-${{ steps.native-deps.outputs.identity }}-${{ hashFiles('.github/workflows/native-live.yml') }}-${{ runner.os }}-${{ runner.arch }} - - - name: Build pinned WebSocket-enabled libcurl - env: - CURL_CACHE_HIT: ${{ steps.curl-cache.outputs.cache-hit }} - run: | - set -euo pipefail - cmake_file="build-native-deps/curl-install/lib/cmake/CURL/CURLConfig.cmake" - if [[ "${CURL_CACHE_HIT}" == "true" && -f "$cmake_file" ]]; then - echo "Using cached libcurl at ${cmake_file}" - exit 0 - fi - mkdir -p build-native-deps - cd build-native-deps - curl --fail --location --proto '=https' --tlsv1.2 \ - "https://curl.se/download/curl-${CURL_VERSION}.tar.xz" -o curl.tar.xz - echo "${CURL_SHA256} curl.tar.xz" | sha256sum -c - - tar -xf curl.tar.xz - cmake -S "curl-${CURL_VERSION}" -B curl-build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PWD/curl-install" \ - -DBUILD_CURL_EXE=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON \ - -DBUILD_TESTING=OFF -DCURL_USE_OPENSSL=ON -DENABLE_WEBSOCKETS=ON \ - -DHTTP_ONLY=ON -DCURL_USE_LIBPSL=OFF -DUSE_LIBIDN2=OFF \ - -DCURL_USE_LIBSSH2=OFF -DCURL_BROTLI=OFF -DCURL_ZSTD=OFF \ - 2>&1 | tee curl-configure.log - cmake --build curl-build -j 4 2>&1 | tee curl-build.log - cmake --install curl-build 2>&1 | tee curl-install.log + - name: Set up pinned WebSocket-enabled libcurl + id: curl-deps + uses: ./.github/actions/setup-live-curl - name: Verify native live - run: python3 scripts/ci_verify.py native --build-dir build-live --jobs "$(getconf _NPROCESSORS_ONLN)" --generator Ninja --curl-dir "${{ github.workspace }}/build-native-deps/curl-install/lib/cmake/CURL" --ccache --require-websocket ${{ inputs.exclude_slow && '--exclude-label slow' || '' }} + run: python3 scripts/ci_verify.py native --build-dir build-live --jobs "$(getconf _NPROCESSORS_ONLN)" --generator Ninja --curl-dir "${{ steps.curl-deps.outputs.curl-dir }}" --ccache --require-websocket ${{ inputs.exclude_slow && '--exclude-label slow' || '' }} + + - name: Verify every live runner target with ASan and UBSan + if: ${{ !cancelled() }} + run: python3 scripts/ci_verify.py live-sanitizers --build-dir build-live-sanitizers --jobs "$(getconf _NPROCESSORS_ONLN)" --generator Ninja --curl-dir "${{ steps.curl-deps.outputs.curl-dir }}" --ccache - name: Stage and summarize diagnostics if: always() env: - CURL_CACHE_HIT: ${{ steps.curl-cache.outputs.cache-hit }} + CURL_CACHE_HIT: ${{ steps.curl-deps.outputs.cache-hit }} run: python3 scripts/collect_ci_diagnostics.py --build-dir build-live --profile native --dependency-dir build-native-deps + - name: Verify every live runner target with ThreadSanitizer + if: ${{ !cancelled() }} + run: python3 scripts/ci_verify.py live-tsan --build-dir build-live-tsan --jobs "$(getconf _NPROCESSORS_ONLN)" --generator Ninja --curl-dir "${{ steps.curl-deps.outputs.curl-dir }}" --ccache + + - name: Stage thread sanitizer diagnostics + if: always() + run: python3 scripts/collect_ci_diagnostics.py --build-dir build-live-tsan --profile live-tsan --output ci-diagnostics/live-tsan --dependency-dir build-native-deps + + - name: Stage live sanitizer diagnostics + if: always() + run: python3 scripts/collect_ci_diagnostics.py --build-dir build-live-sanitizers --profile live-sanitizers --output ci-diagnostics/live-sanitizers --dependency-dir build-native-deps + - name: Retain CI diagnostics if: always() uses: actions/upload-artifact@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f4a75ee1..bd2933cf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,41 @@ README's *Releases* section and on the GitHub releases page. From 1.0.0 the version number follows semantic versioning over the surfaces the [public contract](docs/pages/public-contract.md) lists. +## Unreleased + +- **Native runner routing and delivery:** webhooks are optional; `--webhook-routes` + adds first-match per-action targets and payload `pineforge-native-order-action/v2`. + New `actions`, `status` and offline `redeliver` commands expose the journal and + delivery audit. `redeliver` requires the ledger's `--deployment` identity. + Existing `--webhook-url` deployments keep one default target, exact v1 payload + bytes and event-id idempotency keys, but delivery behavior changes: HTTP errors + are final, normal runs exit 0 even with failed deliveries, and `--max-attempts N` + caps transport retries at `min(2, N-1)` rather than stopping computation. + Default delivery timeouts are now 2 s connect / 5 s total (formerly 5 s / 15 s). + Fatal exits drain for at most one total timeout and report unsent actions; + SIGINT/SIGTERM exit 130. Redelivery exits 2 for failed or pending selections. + Schema-1 native ledgers migrate additively to an append-only delivery log without + rewriting actions. This migration is one-way: older runner binaries cannot open + the migrated ledger. Back up the ledger before upgrading. The engine is unchanged. + +- **Runner tooling removal:** the native live runner accepts only normalized + PineForge feed events from stdin, files, or a user's own HTTP/WebSocket feed + service. The installed `pineforge/live_parser.h` header, its + `PF_LIVE_PARSER_*` types/constants and `pf_live_parser_abi_version` / + `pf_live_parse_message` plugin exports, the `--parser` / `--parser-config` + flags, and the example plugin are removed. This breaks callers that included + that header, authored native parser plugins, or passed raw provider messages + through the runner. Migrate translation to an **external feed adapter** that + emits [PineForge feed events](runner/README.md#feed-format); feed URLs address + that adapter, not an exchange. The outbound order-action webhook remains. + Strict native runner configurations now also require 1m input; higher script + timeframes still aggregate those minutes. + No versioned engine `PF_API` export, native C++ surface, script ABI epoch or + engine behavior changes. Frozen historical ABI header manifests and archives + remain intact. Plugin-free ledger identity bytes remain unchanged; ledgers + bound to removed plugins cannot be resumed by this runner and must not be + silently rewritten. + ## 1.0.1 — 2026-10-02 A documentation-only release that pairs with pineforge-codegen 1.0.1, the pair diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a0b103059..d5629f940 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -158,8 +158,8 @@ What each gate refuses: ### The floors `ci_verify.py` counts the CTest rows that actually **ran** and fails below a -floor — `KERNEL_MIN_TESTS` ci_verify.py:320 and `RELEASE_MIN_TESTS` -ci_verify.py:594. +floor — `KERNEL_MIN_TESTS` ci_verify.py:321 and `RELEASE_MIN_TESTS` +ci_verify.py:595. A deleted or silently skipped row is a failure, not a quieter run. If your change adds rows, raise the floor in the same commit and say by how much; if it legitimately removes one, lower it deliberately and say why. `--min-tests` diff --git a/README.md b/README.md index 1e669197c..d4b738fe9 100644 --- a/README.md +++ b/README.md @@ -211,8 +211,9 @@ reference underneath it. ## Native live runner The optional C++17 `pineforge-live` executable uses this engine's native -warmup-to-stream lifecycle. It accepts normalized ticks or confirmed OHLCV bars, -supports user-defined C++ parsers for broker/provider messages, and commits +warmup-to-stream lifecycle. It accepts only normalized ticks or confirmed OHLCV +bars as PineForge feed events from stdin, a file or your own +feed service (exchange translation belongs in an external feed adapter), and commits inputs plus order-action webhooks to a durable SQLite ledger. Hand-written C++ strategies use the native contract; generated Pine strategies retain their compatibility path. Both expose the versioned C ABI used by the runner. @@ -220,7 +221,7 @@ compatibility path. Both expose the versioned C ABI used by the runner. Build with `-DPINEFORGE_BUILD_LIVE_RUNNER=ON`; the option is off by default, so core-only users do not acquire SQLite/libcurl/OpenSSL dependencies. See the [native runner guide](runner/README.md) for feed modes, symbol metadata, -parser ABI, recovery and execution limitations. The existing validation +feed format, recovery and execution limitations. The existing validation scoreboard below describes batch backtests; it does not certify new native live behavior or real broker fills. diff --git a/docs/ci.md b/docs/ci.md index fecaa0fe9..58aab2610 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -55,7 +55,9 @@ even when the current benchmark files are correct. | `release` | Release, tutorial enabled | Standard CI checks behind a row floor (`RELEASE_MIN_TESTS`; `--min-tests N` overrides it), installed package, and installed native include-independence proof | | `debug` | Debug, tutorial enabled | The standard checks and installed package without Release optimization; no default row floor, examples, include-independence proof or twin-parity guard | | `sanitizers` | Debug, ASan and UBSan | Instrumented library, tests and installed consumer; Linux CI also requires leak detection | -| `native` | Release, live runner enabled | Parser, journal, transport tests, installed runner help, and installed native include-independence proof | +| `native` | Release, live runner enabled | JSON, journal, transport tests, installed runner help, and installed native include-independence proof | +| `live-sanitizers` | Debug, live runner enabled, ASan and UBSan | Every runner target instrumented and audited from compile commands; all runner CTest rows, Python E2Es and installed runner help; WebSocket-enabled curl mandatory while `runner/transport.cpp` exists, no skips accepted | +| `live-tsan` | Debug, live runner enabled, ThreadSanitizer | Separate build instruments every runner target, audits compile commands and runs all runner rows and Python E2Es; WebSocket-enabled curl is mandatory and skips are refused | | `kernel` | Release, live runner enabled, Pine source layer OFF | The source-free CTest set behind a row floor (`KERNEL_MIN_TESTS`; `--min-tests N` overrides it), installed package, and the `nm` half of the include-independence proof over `libpineforge_kernel.a` | By default each profile uses `build-ci-`. Keep separate build directories @@ -73,6 +75,10 @@ python3 scripts/ci_verify.py debug --jobs 4 python3 scripts/ci_verify.py sanitizers --jobs 4 python3 scripts/ci_verify.py native --curl-dir /path/to/curl/lib/cmake/CURL \ --require-websocket --jobs 4 +python3 scripts/ci_verify.py live-sanitizers --curl-dir /path/to/curl/lib/cmake/CURL \ + --jobs 4 +python3 scripts/ci_verify.py live-tsan --curl-dir /path/to/curl/lib/cmake/CURL \ + --jobs 4 ``` The sanitizer profile uses the Linux CI ASan/UBSan environment, including @@ -82,6 +88,26 @@ That flag turns a transport skip into failure. A local native run using system curl may report the existing unsupported-WebSocket skip; that is not the required Linux transport proof. +`live-sanitizers` performs a full all-target build, then runs the complete +`build-ci-live-sanitizers/runner` CTest inventory (at least twelve rows while +`runner/transport.cpp` exists, eleven after its removal), including +`native_live_e2e` and `native_live_startup_e2e`, and requires any additional +runner `tests/native_live*_e2e.py` on the tree to be registered too. The standalone +corpus equivalence driver is not a runner CTest row; its harness unit tests remain +in the engine-wide inventory. The profile does not rerun +the engine-wide CTest set or prepare historical ABI providers; those remain +covered by `sanitizers`. It refuses label exclusions, missing runner compile +commands, missing ASan/UBSan or frame-pointer flags, skipped/disabled tests, +and, while `runner/transport.cpp` exists, a missing WebSocket row or transport +skip even without `--require-websocket`. After transport removal those two +automatic gates no longer apply; an explicit `--require-websocket` still +requires the WebSocket test. +The runner's curl version floor still applies at configure time. +`live-tsan` runs the same inventory in a separate ThreadSanitizer build, +checks every runner compile unit for instrumentation, and stops on a race. +`native-live.yml` runs both profiles alongside `native`, reusing the same +checksum-pinned WebSocket-enabled curl, and retains all profiles' diagnostics. + The kernel profile also gates the CTest row count: `tests/CMakeLists.txt` drops every test TU whose include closure reaches `pineforge/source/` or `compat/pine/`, so a lane whose native witnesses shared a TU with an adapter @@ -111,7 +137,7 @@ plus wave G's six rows, wave H's twenty-four, INT26's own tape row, INT27's thirteen, XSYM-D's four, K-SESSION-WINDOWS' four, INT28's twenty-four, INT28-FIX's four, INT29's seven, W15-KERNEL-CAL's two, INT30's twenty-four, TAIL-I's one, XAU-CAL's four and FIX-E1E2's two). The full-run release and -kernel floors are 792 and 300 rows that ran; full runs do not exclude a label. +kernel floors are 792 and 299 rows that ran; full runs do not exclude a label. Preflight also runs the detached-comment census of the kernel compile closure (`detached-comments`: `scripts/measure_detached_comments.py --check-ceiling`) diff --git a/docs/design/native-feature-parity.md b/docs/design/native-feature-parity.md index 07d233fa3..b509dfe8b 100644 --- a/docs/design/native-feature-parity.md +++ b/docs/design/native-feature-parity.md @@ -121,7 +121,7 @@ Columns: **Feature** (adapter mechanism, cited) · **Native today** (`yes` / `pa | CT4 | Partial-bar view / open-bar lookahead guard | **no** — the open hook receives the *complete* script bar: `invoke_bar_open_callback(engine, bar, point)` native_execution_consumer.cpp:7190 with `engine.current_bar_ = open_view` native_execution_consumer.cpp:7241 (the complete bar unless `OpenOnly`) **Closed:** `current_partial_bar()` native_host.hpp:1073 is the lookahead-free bar so far, through the last path point consumed (a fill inside a segment reads it at the segment's origin), and `NativeOpenBarView::OpenOnly` native_run_spec.hpp:109 masks `on_native_bar_open`'s bar to `H = L = C = open` as presentation only — the complete bar is restored before the open match (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E3 | (single-source) The adapter needs the full bar, so the guard is opt-in. | | CT5 | Raw input observation before aggregation | **yes** — `on_native_input` native_host.hpp:875, `NativeInputContext` native_host.hpp:773-778 | K | — | O:F3 | (single-source row) | | CT6 | Calculate on order fills (scheduler `PineScheduler::recalculate` pine_scheduler_native.cpp:652, `coof_recalculation_due` pine_scheduler_native.cpp:618, the first-open chain `kFirstOpenLoopGuard` pine_scheduler_native.cpp:766; `begin_coof_recalc` pine_adapter.cpp:5845, `flush_coof_tail` pine_adapter.cpp:18889, `end_coof_recalc` pine_adapter.cpp:5887) | **partial** — `on_native_applied` native_host.hpp:938 fires mid-path after each fill native_execution_consumer.cpp:7096, native_execution_consumer.cpp:7069-7099; commands are legal there `commands_allowed` native_execution_consumer.cpp:1419-1427; a request born there is eligible on the unconsumed rest of the bar `born_on_remaining_path` native_execution_consumer.cpp:5763-5797, `remaining_path_delivery` native_order.cpp:2061-2069. Missing: calculation re-entry, a documented chronology, a bar-so-far view **Closed:** `NativeCalculationTrigger::BarCloseAndFills` native_run_spec.hpp:96 drives one `OrderFill` recalculation per applied execution at that event's own cursor, delivered as `on_native_recalculate(..., OrderFill, cause)` and bounded by `max_recalculations_per_point` (`tests/test_native_calc_timing.cpp`). | K+A | L5 | F:E4 O:F5 S:C2 | **Disagree:** F partial / O, S no; F, S v1 / O deferrable. R5-5: v1, sub-bar hook + chronology contract, adapter path unchanged. TV parts → CT11. | -| CT7 | Calculate on every tick | **partial** — `on_native_tick` native_host.hpp:878 fires per accepted realtime print native_execution_consumer.cpp:7329-7333; no per-tick calculation in batch; the guide pinned "close-only" (`docs/pages/native-engine.md`, "Batch OHLCV vs ticks vs quiet") **Closed:** `NativeCalculationTrigger::EveryModeledPoint` native_run_spec.hpp:97 adds one `Tick` recalculation at every modeled point of the delivered path and at every observed print, in batch and in a stream alike (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E5 O:F4 S:C3 | **Disagree:** F yes / O partial / S no. R5-5: v1. Not an adapter feature either: absent from `PineStrategyConfig` pine_adapter.hpp:62-76, the runner rejects it (`calc_on_every_tick` main.cpp:213), the Pine stream route refuses `calc_on_order_fills` pine_strategy_host.cpp:213-215. | +| CT7 | Calculate on every tick | **partial** — `on_native_tick` native_host.hpp:878 fires per accepted realtime print native_execution_consumer.cpp:7329-7333; no per-tick calculation in batch; the guide pinned "close-only" (`docs/pages/native-engine.md`, "Batch OHLCV vs ticks vs quiet") **Closed:** `NativeCalculationTrigger::EveryModeledPoint` native_run_spec.hpp:97 adds one `Tick` recalculation at every modeled point of the delivered path and at every observed print, in batch and in a stream alike (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E5 O:F4 S:C3 | **Disagree:** F yes / O partial / S no. R5-5: v1. Not an adapter feature either: absent from `PineStrategyConfig` pine_adapter.hpp:62-76, the runner rejects it (`calc_on_every_tick` main.cpp:216), the Pine stream route refuses `calc_on_order_fills` pine_strategy_host.cpp:213-215. | | CT8 | Historical intrabar calculation (per lower-TF sub-bar / magnifier sample) | **no** — with an `IntrabarPath` the callback still fires once per script bar native_execution_consumer.cpp:7764-7906 **Closed:** `EveryModeledPoint` recalculates at each intrabar sample, and `on_native_sub_bar` native_host.hpp:928 fires once after each retained lower-timeframe sub-bar's whole matching path (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E6 O:F8 S:C4 | — | | CT9 | Intrabar (magnifier) matching path + path-order policy (`volume_weighted_cap` pine_adapter.cpp:2407-2463) | **yes** — `IntrabarPath` native_run_spec.hpp:362-403, delivery `deliver_confirmed_script` native_execution_consumer.cpp:7549-7815; `NativePathOrder` native_run_spec.hpp:335-339 | K+A | — | F:E7 F:G5 O:F7 S:C4 | **Disagree:** S partial (it wants callback cadence → CT8). TV sampler choice is A: TradingView's own intrabars at its table's timeframe, each owned by the chart bar of its last minute, built from the finer feed (`tradingview_magnifier_bars` magnifier_intrabars.hpp:44, R5 lane MAG-INTRABAR). | | CT10 | Batch → warmup → realtime forward execution | **yes** — `stream_begin` engine.hpp:1771, `stream_push_bar` engine.hpp:1777, `stream_end` engine.hpp:1781, `NativeRunPhase` native_host.hpp:41, C ABI `strategy_stream_begin` c_abi.cpp:613 | K+A | — | F:E8 O:J3 S:C5 | **Disagree:** S partial. R5-1: the lifecycle is native; Pine trade-start suppression and realtime-tail / probe rules (`trading_window_active` pine_strategy_commands.cpp:15-21, `prepare_native_begin` pine_strategy_host.cpp:153-239) are A. S's "neutral tail + trade-window policy" is single-source and unassigned (§4 Q8). | @@ -299,9 +299,9 @@ Depends on L7 (working view), L3 (so `Sized` is in v1 of the C request) and the **Status (L10): done.** `examples/native/` holds eighteen hosts, sixteen C++ and two C, built by `PINEFORGE_BUILD_EXAMPLES`; `include/pineforge/native_module.hpp` defines `PINEFORGE_EXPORT_NATIVE_STRATEGY(Class)`. The bullets below are the plan L10 worked from. -- Move `runner/examples/native_market_strategy.cpp` and `native_selected_strategy.cpp` to a top-level `examples/native/`, built by `PINEFORGE_BUILD_EXAMPLES` (then "none yet" and guarding nothing CMakeLists.txt:37; the examples build only under `PINEFORGE_BUILD_LIVE_RUNNER` CMakeLists.txt:465-466, with `native_market_example` runner/CMakeLists.txt:45). Link the kernel target (S), or `PineForge::pineforge` during migration (O) — no SQLite / curl / OpenSSL (runner/CMakeLists.txt:1-4). +- Move `runner/examples/native_market_strategy.cpp` and `native_selected_strategy.cpp` to a top-level `examples/native/`, built by `PINEFORGE_BUILD_EXAMPLES` (then "none yet" and guarding nothing CMakeLists.txt:37; the examples build only under `PINEFORGE_BUILD_LIVE_RUNNER` CMakeLists.txt:465-466, with `native_market_example` runner/CMakeLists.txt:54). Link the kernel target (S), or `PineForge::pineforge` during migration (O) — no SQLite / curl / OpenSSL (runner/CMakeLists.txt:1-4). - Build (1) standalone executables with a `main()` that runs batch + stream on embedded bars (the guide already contains that `main` native-engine.md:3237-3305) and (2) the same MODULE targets the runner tests load; keep the C-ABI shims so the live runner can still `dlopen` them (F, O, S). -- Update the three places that pin the paths: `native_market_example` runner/CMakeLists.txt:45, `NATIVE_EXAMPLES` check_native_include_independence.py:49, the tests at `test_native_example_batch` runner/CMakeLists.txt:128. +- Update the three places that pin the paths: `native_market_example` runner/CMakeLists.txt:54, `NATIVE_EXAMPLES` check_native_include_independence.py:49, the tests at `test_native_example_batch` runner/CMakeLists.txt:122. - Add a minimal "hello, kernel" host (~60 lines, no C ABI — O) and one example per v1 feature lane as it lands; each doubles as that lane's twin host (F). - Add `include/pineforge/native_module.hpp` with `PINEFORGE_EXPORT_NATIVE_STRATEGY(Class)` to replace the ~70 hand-written `extern "C"` lines per example (F, single-source). - Ship the native API reference and the standard dual-run harness of §3.1 b (S lane J). diff --git a/docs/pages/contributing-llm.md b/docs/pages/contributing-llm.md index 1648028f1..322e6eec0 100644 --- a/docs/pages/contributing-llm.md +++ b/docs/pages/contributing-llm.md @@ -132,8 +132,8 @@ finding to report, not a step to take. `scripts/check_doc_reverts.py`. 10. **A test row never silently disappears.** Each profile counts the rows that - *ran* against a floor: `KERNEL_MIN_TESTS` ci_verify.py:320 and - `RELEASE_MIN_TESTS` ci_verify.py:594. Adding rows means raising the floor + *ran* against a floor: `KERNEL_MIN_TESTS` ci_verify.py:321 and + `RELEASE_MIN_TESTS` ci_verify.py:595. Adding rows means raising the floor in the same commit. ## The recipe for a lane @@ -281,7 +281,7 @@ measurement that produced it, so a later change to it is visible as a change to the record, not as an edit to a literal. **floor** — the minimum number of CTest rows a profile must actually run -(`KERNEL_MIN_TESTS` ci_verify.py:320, `RELEASE_MIN_TESTS` ci_verify.py:594). It +(`KERNEL_MIN_TESTS` ci_verify.py:321, `RELEASE_MIN_TESTS` ci_verify.py:595). It counts rows that ran, so a skipped row does not pad it. **receipt** — the recorded evidence an ABI-comparison row needs (a prepared diff --git a/docs/pages/getting-started.md b/docs/pages/getting-started.md index 6853ccc61..f70027de0 100644 --- a/docs/pages/getting-started.md +++ b/docs/pages/getting-started.md @@ -40,7 +40,7 @@ This installs: `native_toolkit.hpp`) and internal ones; `source/` and `compat/pine/` hold the Pine adapter's - `include/pineforge/pineforge.h` — **the public C ABI**, with - `native_c_api.h` (the C native-host API it includes) and `live_parser.h` + `native_c_api.h` (the C native-host API it includes) - `include/pineforge/version.h` — generated version macros - `lib/cmake/PineForge/PineForge{Config,Targets,ConfigVersion}.cmake` diff --git a/docs/pages/install.md b/docs/pages/install.md index 7061cfaf5..5fa9988b7 100644 --- a/docs/pages/install.md +++ b/docs/pages/install.md @@ -42,7 +42,6 @@ ${prefix}/ └── include/pineforge/ ├── pineforge.h # public C ABI ├── native_c_api.h # C native-host API (included by pineforge.h) - ├── live_parser.h # parser-plugin ABI of the native live runner ├── version.h # generated version macros ├── native_*.hpp # kernel / native C++ API (see Public contract) ├── *.hpp # the other C++ headers (no stability guarantee) diff --git a/docs/pages/native-engine.md b/docs/pages/native-engine.md index a7dd7b02b..d802bfc54 100644 --- a/docs/pages/native-engine.md +++ b/docs/pages/native-engine.md @@ -3938,10 +3938,9 @@ same sources, as the MODULE targets the live runner `dlopen`s `-UNDEBUG` too, and `examples-assert-live` checks them wherever the runner is built: in the `kernel` profile beside the `example_*` executables, and in the `native` profile, which builds no `example_*` target, for them alone. The -runner's own two example modules — `native_live_example` and -`native_live_parser_example`, built from `runner/examples/` — also compile -with `-UNDEBUG`. Neither holds an `assert()`, and the stage does not read -them: neither is an `examples/native` source. +runner's own example module, `native_live_example`, built from +`runner/examples/`, also compiles with `-UNDEBUG`. It holds no `assert()`, +and the stage does not read it: it is not an `examples/native` source. A host becomes such a module through one macro from ``: diff --git a/docs/pages/pine-to-native.md b/docs/pages/pine-to-native.md index 56fbc784d..ad8682cab 100644 --- a/docs/pages/pine-to-native.md +++ b/docs/pages/pine-to-native.md @@ -1059,7 +1059,7 @@ is what makes the engine's protected presentation-error string (returned by Two example modules are built twice — as executables by `PINEFORGE_BUILD_EXAMPLES` CMakeLists.txt:37, and as the runner's MODULE -targets `native_market_example` runner/CMakeLists.txt:45 and `native_selected_example` runner/CMakeLists.txt:51 — from the same sources. +targets `native_market_example` runner/CMakeLists.txt:54 and `native_selected_example` runner/CMakeLists.txt:60 — from the same sources. ## Verifying parity {#pine_to_native_parity} diff --git a/docs/pages/streaming.md b/docs/pages/streaming.md index 1935f6f28..dfacbff42 100644 --- a/docs/pages/streaming.md +++ b/docs/pages/streaming.md @@ -17,9 +17,12 @@ already-active stream keeps its begin-time settings until it ends. After a failed stream begin, recreate the strategy handle before starting again. The optional **native C++ `pineforge-live` executable** in this repository -uses this lifecycle directly. It includes native transport, custom C++ parser -plugins, SQLite recovery and order-action webhooks; see the +uses this lifecycle directly. It includes native normalized-feed transport, +SQLite recovery and order-action webhooks; see the [native runner guide](https://github.com/pineforge-4pass/pineforge-engine/blob/main/runner/README.md). +Webhooks are optional: use `--webhook-routes` for per-action routing or read +committed actions with `pineforge-live actions --ledger L --after 0 --follow`. +Delivery timing and receiver errors never influence computation. A host can instead recompute `run_backtest_full` over its accumulated bars with the [ABI v4 live surface](@ref live_surface); the earlier Python runtime diff --git a/include/pineforge/live_parser.h b/include/pineforge/live_parser.h deleted file mode 100644 index 0d00afe05..000000000 --- a/include/pineforge/live_parser.h +++ /dev/null @@ -1,96 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 */ -#ifndef PINEFORGE_LIVE_PARSER_H -#define PINEFORGE_LIVE_PARSER_H - -#include -#include - -#define PF_LIVE_PARSER_ABI_VERSION 1u -#define PF_LIVE_PARSER_MAX_MESSAGE_BYTES (1024u * 1024u) -#define PF_LIVE_PARSER_MAX_EVENTS 1024u -#define PF_LIVE_PARSER_MAX_OUTPUT_BYTES (1024u * 1024u) - -#if defined(_WIN32) || defined(__CYGWIN__) -#define PF_LIVE_PARSER_API __declspec(dllexport) -#elif defined(__GNUC__) || defined(__clang__) -#define PF_LIVE_PARSER_API __attribute__((visibility("default"))) -#else -#define PF_LIVE_PARSER_API -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -enum pf_live_parser_event_kind { - PF_LIVE_PARSER_TICK = 1, - PF_LIVE_PARSER_BAR = 2, - PF_LIVE_PARSER_TIME = 3 -}; - -/* ABI v1 is fixed-layout POD; a changed layout requires a new ABI version. - * Initialize the whole struct to zero before assigning its active fields. - * All timestamps are Unix milliseconds and must be nonnegative. - * - * TICK: timestamp, strictly positive source sequence, price and quantity. - * BAR: timestamp is a confirmed one-minute bar's open, aligned to 60000ms; - * open/high/low/close are positive finite prices; volume is finite >= 0. - * TIME: timestamp is the provider's explicit completeness boundary, declaring - * that no earlier ticks remain. A wall clock/heartbeat is not sufficient. - * - * reserved must be zero. Inactive fields are ignored and normalized to zero by - * the host. Sequence continuity, mode and stream ordering are checked by the - * runner across messages; parsers must preserve source sequence, never invent - * a mutable counter. BAR/TIME do not use sequence. - */ -typedef struct pf_live_parser_event_v1 { - uint32_t kind; - uint32_t reserved; - int64_t timestamp; - uint64_t sequence; - double open; - double high; - double low; - double close; - double volume; - double price; - double quantity; -} pf_live_parser_event_v1_t; - -/* The event pointer is borrowed only until this callback returns. The callback - * copies it synchronously and returns 0 on acceptance or -1 on failure. A plugin - * MUST stop and propagate a callback failure; it must not retain callback/user - * pointers, call concurrently, or emit after parse_message returns. - */ -typedef int (*pf_live_parser_emit_v1_fn)( - const pf_live_parser_event_v1_t* event, void* user); - -PF_LIVE_PARSER_API uint32_t pf_live_parser_abi_version(void); - -/* Parse ONE complete provider message into zero or more ordered events. - * message/config_json are length-delimited borrowed bytes, not NUL-terminated - * strings. config_json is an immutable JSON object; it remains in host memory - * and must not be logged or copied into normalized events. Heartbeats/control - * messages may succeed with zero events. Malformed input MUST fail. - * - * Return 0 on success, -1 on any failure. Never throw across this C ABI. The - * host stages events until success, so a failed message emits no partial output - * to the strategy or journal. Limits: 1MiB message, 1024 events, 1MiB POD output. - * With the v1 event size the event-count bound is the tighter output bound. - * - * REQUIRED: stateless and deterministic for identical (message, config) bytes. - * No I/O, clock, random values, mutable globals or retained state may influence - * output. The journal stores canonical normalized events, not parser state. - * Providers requiring cross-message assembly/state need an upstream adapter. - * Plugins are trusted native executable code, not sandboxed. The host cannot - * enforce this contract against malicious code or recover from native crashes. - */ -PF_LIVE_PARSER_API int pf_live_parse_message( - const char* message, size_t message_size, - const char* config_json, size_t config_size, - pf_live_parser_emit_v1_fn emit, void* user); - -#ifdef __cplusplus -} /* extern "C" */ -#endif -#endif /* PINEFORGE_LIVE_PARSER_H */ diff --git a/runner/CMakeLists.txt b/runner/CMakeLists.txt index a00227889..da8de6fb9 100644 --- a/runner/CMakeLists.txt +++ b/runner/CMakeLists.txt @@ -1,10 +1,23 @@ # Dependencies belong only to the optional executable, not PineForge::pineforge. find_package(SQLite3 REQUIRED) -find_package(CURL 7.86 REQUIRED) +find_package(CURL 8.14.1 REQUIRED) find_package(OpenSSL REQUIRED COMPONENTS Crypto) find_package(Threads REQUIRED) -add_library(pineforge_live_support STATIC store.cpp transport.cpp) +option(PINEFORGE_LIVE_TSAN "Instrument all native runner targets with ThreadSanitizer" OFF) +if(PINEFORGE_LIVE_TSAN) + if(PINEFORGE_ENABLE_SANITIZERS) + message(FATAL_ERROR "Runner ThreadSanitizer must use a separate build from ASan/UBSan") + endif() + add_compile_options(-fsanitize=thread -fno-omit-frame-pointer) + add_link_options(-fsanitize=thread) +endif() + +if(PINEFORGE_ENABLE_SANITIZERS) + add_compile_options(-fsanitize=address,undefined -fno-omit-frame-pointer) + add_link_options(-fsanitize=address,undefined) +endif() +add_library(pineforge_live_support STATIC store.cpp transport.cpp routing.cpp delivery.cpp) target_compile_features(pineforge_live_support PUBLIC cxx_std_17) target_include_directories(pineforge_live_support PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) if(TARGET SQLite3::SQLite3) @@ -14,21 +27,18 @@ else() endif() target_link_libraries(pineforge_live_support PUBLIC ${_pf_sqlite_target} CURL::libcurl OpenSSL::Crypto Threads::Threads) target_compile_options(pineforge_live_support PRIVATE -Wall -Wextra -Wpedantic) - add_library(pineforge_live_startup STATIC native_startup.cpp) target_compile_features(pineforge_live_startup PUBLIC cxx_std_17) target_include_directories(pineforge_live_startup PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/include ${PROJECT_BINARY_DIR}/include) target_link_libraries(pineforge_live_startup PUBLIC pineforge_live_support PineForge::pineforge) target_compile_options(pineforge_live_startup PRIVATE -Wall -Wextra -Wpedantic -ffp-contract=off) - -add_executable(pineforge-live main.cpp parser.cpp) +add_executable(pineforge-live main.cpp) target_include_directories(pineforge-live PRIVATE ${PROJECT_SOURCE_DIR}/include ${PROJECT_BINARY_DIR}/include) target_link_libraries(pineforge-live PRIVATE pineforge_live_startup ${CMAKE_DL_LIBS}) target_compile_options(pineforge-live PRIVATE -Wall -Wextra -Wpedantic -ffp-contract=off) install(TARGETS pineforge-live RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - # examples/strategy.cpp derives from pineforge::source::PineStrategyHost, so # it is the one example that a kernel-only build cannot compile. if(PINEFORGE_BUILD_SOURCE_LAYER) @@ -41,7 +51,6 @@ else() "Live runner example strategy.cpp skipped: it is Pine-bound and " "PINEFORGE_BUILD_SOURCE_LAYER is OFF") endif() - add_library(native_market_example MODULE ${PROJECT_SOURCE_DIR}/examples/native/native_market_strategy.cpp) target_link_libraries(native_market_example PRIVATE PineForge::pineforge) @@ -65,15 +74,6 @@ foreach(_pf_example_module native_market_example native_selected_example) target_compile_options(${_pf_example_module} PRIVATE -UNDEBUG) endforeach() -add_library(native_live_parser_example MODULE examples/demo_parser.cpp) -target_include_directories(native_live_parser_example PRIVATE ${PROJECT_SOURCE_DIR}/include) -# The runner's own two example modules (examples/strategy.cpp above and this -# parser) take -UNDEBUG as well, so an assert() added to either checks -# instead of compiling to a no-op. Neither holds one today, and neither is an -# examples/native source, so the examples-assert-live stage does not read them. -target_compile_options(native_live_parser_example PRIVATE -UNDEBUG) -set_target_properties(native_live_parser_example PROPERTIES PREFIX "" OUTPUT_NAME "native-live-parser-example") - if(PINEFORGE_BUILD_TESTS) add_executable(test_live_json ${PROJECT_SOURCE_DIR}/tests/test_live_json.cpp) target_include_directories(test_live_json PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) @@ -84,24 +84,11 @@ if(PINEFORGE_BUILD_TESTS) target_compile_options(test_native_live_startup PRIVATE -UNDEBUG) add_test(NAME test_native_live_startup COMMAND test_native_live_startup) add_test(NAME native_live_help COMMAND pineforge-live --help) - add_executable(test_live_parser ${PROJECT_SOURCE_DIR}/tests/test_live_parser.cpp parser.cpp) - target_include_directories(test_live_parser PRIVATE ${PROJECT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}) - target_compile_options(test_live_parser PRIVATE -UNDEBUG) - target_link_libraries(test_live_parser PRIVATE ${CMAKE_DL_LIBS}) - foreach(parser_kind fixture wrong_abi) - add_library(live_parser_${parser_kind} MODULE ${PROJECT_SOURCE_DIR}/tests/test_live_parser.cpp) - target_include_directories(live_parser_${parser_kind} PRIVATE ${PROJECT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}) - target_compile_definitions(live_parser_${parser_kind} PRIVATE PF_LIVE_PARSER_TEST_PLUGIN=1) - endforeach() - target_compile_definitions(live_parser_wrong_abi PRIVATE PF_LIVE_PARSER_TEST_ABI=2) - add_test(NAME test_live_parser COMMAND test_live_parser $ - $ $) find_package(Python3 REQUIRED COMPONENTS Interpreter) if(PINEFORGE_BUILD_SOURCE_LAYER) add_test(NAME native_live_e2e COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tests/native_live_e2e.py $ - $ $ - $) + $) set_tests_properties(native_live_e2e PROPERTIES TIMEOUT 120) endif() foreach(contract_kind absent unknown native_no_export native_stub) @@ -131,11 +118,6 @@ if(PINEFORGE_BUILD_TESTS) ${PROJECT_SOURCE_DIR}/include ${PROJECT_BINARY_DIR}/include) target_compile_options(test_native_example_batch PRIVATE -UNDEBUG -ffp-contract=off) target_link_libraries(test_native_example_batch PRIVATE ${CMAKE_DL_LIBS}) - if(PINEFORGE_ENABLE_SANITIZERS) - target_compile_options(test_native_example_batch PRIVATE - -fsanitize=address,undefined -fno-omit-frame-pointer) - target_link_options(test_native_example_batch PRIVATE -fsanitize=address,undefined) - endif() add_dependencies(test_native_example_batch native_market_example) add_test(NAME test_native_example_batch COMMAND test_native_example_batch $) @@ -147,23 +129,38 @@ if(PINEFORGE_BUILD_TESTS) ${PROJECT_SOURCE_DIR}/include ${PROJECT_BINARY_DIR}/include) target_compile_options(test_native_example_selected PRIVATE -UNDEBUG -ffp-contract=off) target_link_libraries(test_native_example_selected PRIVATE ${CMAKE_DL_LIBS}) - if(PINEFORGE_ENABLE_SANITIZERS) - target_compile_options(test_native_example_selected PRIVATE - -fsanitize=address,undefined -fno-omit-frame-pointer) - target_link_options(test_native_example_selected PRIVATE -fsanitize=address,undefined) - endif() add_dependencies(test_native_example_selected native_selected_example) add_test(NAME test_native_example_selected COMMAND test_native_example_selected $) set_tests_properties(test_native_example_selected PROPERTIES TIMEOUT 60) - foreach(native_test test_native_live_store test_native_live_websocket) + add_library(pineforge_live_legacy_test_support STATIC store.cpp transport.cpp) + target_compile_features(pineforge_live_legacy_test_support PUBLIC cxx_std_17) + target_include_directories(pineforge_live_legacy_test_support PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + target_compile_definitions(pineforge_live_legacy_test_support PUBLIC PINEFORGE_LIVE_LEGACY_TEST_API) + target_link_libraries(pineforge_live_legacy_test_support PUBLIC + ${_pf_sqlite_target} CURL::libcurl OpenSSL::Crypto Threads::Threads) + foreach(native_test test_native_live_store test_native_live_websocket test_native_live_routing) add_executable(${native_test} ${PROJECT_SOURCE_DIR}/tests/${native_test}.cpp) - target_link_libraries(${native_test} PRIVATE pineforge_live_support) + if(native_test STREQUAL "test_native_live_store") + target_link_libraries(${native_test} PRIVATE pineforge_live_legacy_test_support) + else() + target_link_libraries(${native_test} PRIVATE pineforge_live_support) + endif() target_compile_options(${native_test} PRIVATE -UNDEBUG) add_test(NAME ${native_test} COMMAND ${native_test}) set_tests_properties(${native_test} PROPERTIES TIMEOUT 120) endforeach() set_tests_properties(test_native_live_websocket PROPERTIES SKIP_RETURN_CODE 77) + if(PINEFORGE_BUILD_SOURCE_LAYER) + add_test(NAME native_live_routing_e2e COMMAND ${Python3_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/tests/native_live_routing_e2e.py $ + $) + set_tests_properties(native_live_routing_e2e PROPERTIES TIMEOUT 120) + add_test(NAME native_live_websocket_e2e COMMAND ${Python3_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/tests/native_live_websocket_e2e.py $ + $) + set_tests_properties(native_live_websocket_e2e PROPERTIES TIMEOUT 120 SKIP_RETURN_CODE 77) + endif() endif() if(PINEFORGE_BUILD_TESTS AND PINEFORGE_BUILD_SOURCE_LAYER) diff --git a/runner/README.md b/runner/README.md index 1cb71c9ec..eb10852b0 100644 --- a/runner/README.md +++ b/runner/README.md @@ -10,7 +10,7 @@ refused rather than treated as a legacy library. `pineforge-live` is an optional C++17 executable built with this engine. It loads a compiled strategy, warms it on confirmed historical bars, keeps that same native strategy instance alive, and sends simulated fill actions to a -broker-neutral webhook. Feed transport, parsing, aggregation, strategy +broker-neutral webhook. Normalized-feed transport, aggregation, strategy execution, SQLite journaling, recovery and HTTP delivery run in C++. Both hand-written C++ strategies and `pineforge-codegen-oss` output can expose @@ -20,6 +20,13 @@ old ABI-v4 library without the native extension is refused before execution. The engine and runner are Apache-2.0. The separately distributed compiler retains its own source-available/commercial terms. +## Boundary + +The engine only computes. Hosts push bars or ticks; it never connects to an +exchange or ingests venue fills. The consumer owns venue execution, +reconciliation and risk. The outbound webhook is alert-style delivery of +computed order actions, not an exchange connection. + ## Build ```sh @@ -32,19 +39,66 @@ ctest --test-dir build-live --output-on-failure The option defaults to **OFF**. Core-only engine users gain no SQLite, networking or cryptography dependencies. The optional runner requires -SQLite3, libcurl 7.86+ and OpenSSL Crypto. WebSockets additionally require a +SQLite3, libcurl 8.14.1+ and OpenSSL Crypto. Older curl versions are refused +at configure time, including for a runner intended only for file/HTTP feeds. +Core-only engine builds remain unaffected. WebSockets additionally require a libcurl build with the `ws`/`wss` protocols enabled; some system curl builds omit them even when the version is recent. Such a build refuses WebSocket -input explicitly. Set `CURL_DIR` to a curl CMake package when using a custom +input explicitly, before opening or binding the ledger. A WebSocket startup +check also refuses a runtime libcurl older than 8.14.1, even if the runner was +built with newer headers. File/HTTP feeds do not use this WebSocket runtime +check. + +The floor is necessary for reliable fragmented-message finality. Curl 8.13.0 +first corrected FIN/CONT reporting: [curl's release notes](https://curl.se/ch/8.13.0.html) list +“ws: corrected curlws_cont to reflect its documented purpose” +([`fa3d1e7`](https://github.com/curl/curl/commit/fa3d1e7d43bf0e4f589aeae73715348645318a83)) +and “ws: fix and extend CURLWS_CONT handling” +([`3588df9`](https://github.com/curl/curl/commit/3588df9478d7c27046b34cdb510728a26bedabc7)). +Older versions can report an unfinished FIN=0 text frame as complete. However, +8.13.0 and 8.14.0 still lose fragmentation state around control frames and +accept a new text message before the unfinished one terminates. Curl 8.14.1 +(June 4, 2025) qualifies both cases: its [release notes](https://curl.se/ch/8.14.1.html) +include “ws: tests and fixes” +([`d3594be`](https://github.com/curl/curl/commit/d3594be6531df3d5eafcdd09f84ad9dee1777028)), +which preserves state across interleaved ping/pong and rejects invalid +fragment sequences. In 8.14.1+, `CURLWS_CONT` reflects a non-final data frame, consistently across +all chunks of that frame; `bytesleft == 0` alone proves only the end of a +frame, not the end of a message. The runner checks ordered chunks and stable +frame flags, and delivers text only after the entire final frame arrives +(`bytesleft == 0` and no `CURLWS_CONT`). libcurl's public metadata exposes no +separate raw FIN bit, so assembly cannot repair unreliable older metadata. + +Set `CURL_DIR` to a curl CMake package when using a custom build; its imported target must include any transitive static dependencies. The executable targets POSIX macOS/Linux. Python is used only by optional -integration tests (`tests/native_live_e2e.py`, `tests/native_live_startup_e2e.py`), +integration tests (`tests/native_live_e2e.py`, `tests/native_live_startup_e2e.py`, +`tests/native_live_websocket_e2e.py`), never by the running executable. +For sanitizer verification, use a WebSocket-enabled curl meeting the version +floor above while `runner/transport.cpp` exists, and run: + +```sh +python3 scripts/ci_verify.py live-sanitizers --jobs 4 \ + --curl-dir /path/to/curl/lib/cmake/CURL +``` + +This enables ASan and UBSan for every target declared by the runner build, +including main, startup, store, examples, strategy test modules and transport +while it exists. No global compiler flags are necessary. Compile-command +coverage is checked before building; all runner CTest rows and Python E2Es +run with strict ASan/UBSan settings (including Linux leak detection). +While `runner/transport.cpp` exists, WebSocket support and its CTest row are +mandatory, with a default floor of nine runner rows. If transport is removed, +the automatic WebSocket requirement and row pin go away and the floor becomes +eight; an explicit `--require-websocket` still requires the WebSocket test. +No runner row may skip in either state. This focused profile supplements, +rather than replaces, engine-wide sanitizer verification. + The build includes `build-live/lib/native-live-example.so`, a hand-written -C++ example, `native-market-example.so`, `native-selected-example.so`, and -`native-live-parser-example.so`, an illustrative parser. The platform's CMake -module suffix may differ. A Pine/generated strategy's factory returns a +C++ example, `native-market-example.so`, and `native-selected-example.so`. +The platform's CMake module suffix may differ. A Pine/generated strategy's factory returns a `pineforge::source::PineStrategyHost`-derived instance, exports `strategy_create/free` and the input/override setters, and links the engine's C ABI object. See [examples/strategy.cpp](examples/strategy.cpp). Codegen-generated libraries @@ -86,7 +140,8 @@ Native strategies use `--native-config FILE` instead of `--input` / `--override` / `--syminfo`. The file is a strict JSON object with `run`, `clock`, `instrument` and `execution` keys. Explicit CLI clock/symbol flags must equal the file; omitted CLI clock values take the file. Monthly stream -input is refused before the ledger is bound. Legacy 1m identity bytes are +input and any `input_tf` other than `"1"` are refused before the ledger is +bound. Legacy 1m identity bytes are unchanged when `--native-config` is absent. The native examples are `native-market-example` and `native-selected-example`; the latter demonstrates host-sized terms, exact reversal, and a selected current-point close. @@ -103,20 +158,38 @@ settings. The bundled example has no inputs or configurable overrides. It also e `run_backtest`/`run_backtest_full`/`report_free`, so the same C++ strategy can be loaded by either a batch or native live harness. -`--feed -` reads stdin. `--feed-url https://...` polls a **complete JSONL +## Feed format + +The feed is a small, broker-neutral JSON contract. Your external feed adapter +normalizes a provider's bars or ticks into PineForge events; the runner accepts +**only** these events. Raw exchange messages, venue fills, incomplete +`forming` bars and provider heartbeats are refused with an error pointing +here. Translation, subscriptions to exchanges and gap healing belong outside +the engine and runner, in that adapter. + +`--feed -` (the default) reads stdin; `--feed events.jsonl` reads a file, +one JSON event per nonempty line. `--feed-url https://your-feed.example/events` +polls a **complete JSONL snapshot** (maximum 4 MiB) every `--poll-ms` (default 1000); `--check` fetches once. It is intended for bounded snapshots, not an indefinitely growing archive. Redirects and URL user information are refused; TLS is verified. Plain HTTP/WS requires an explicit `--allow-insecure-http` for local testing. -`--feed-url wss://...` receives complete UTF-8 WebSocket messages up to 1 MiB. +`--feed-url wss://your-feed.example/events` receives complete UTF-8 WebSocket +messages up to 1 MiB. An optional `--subscribe subscription.json` sends that file as the initial text subscription/authentication message. Feed requests never receive webhook HMAC or idempotency headers. WebSocket close, malformed/binary frames or an idle/message timeout stop the runner. Reconnection and gap healing are not -guessed: reconnect using the provider's resume mechanism and a verified +guessed: reconnect using your feed service's resume mechanism and a verified input prefix/tail. The default native transport timeout is 15 seconds. +HTTP and WebSocket URLs identify **your own normalized feed service**, never +an exchange. All sources use the same event shapes below. A JSON array of +1..1024 events, or `{"type":"batch","events":[...]}`, is an atomic input +message; nesting and empty batches are refused. For HTTP, each snapshot line +contains one event or one such batch. No source receives webhook credentials. + ### Tick mode ```json @@ -125,7 +198,11 @@ input prefix/tail. The default native transport timeout is 15 seconds. {"type":"time","ts":180000} ``` -Trade sequence numbers must be positive and contiguous. Timestamps cannot +Use `--mode ticks`. Trade sequence numbers must be positive and contiguous +(`last + 1`); a sequence hole stops processing rather than fabricating trades. +Exact duplicates may be replayed as the recorded input prefix on reconnect; +changed records or duplicates introduced at new input indexes are refused. +Timestamps cannot regress. Every tick updates native OHLCV and runs resting-order evaluation at its observed price/time. Strategies calculate on script-bar close in this version. An explicit `time` event declares input completeness before that @@ -134,6 +211,26 @@ does not prove that completeness. Crossing a boundary also closes prior input bars. Existing native semantics create zero-volume carry-forward bars for quiet in-session intervals and skip configured out-of-session intervals. +### Feed field validation + +Fields are strict: unknown keys are fatal and JSON numeric strings are not +numbers. All fields shown below are required, except `trade_count`. + +| Field | JSON type | Range | +| --- | --- | --- | +| `type` | string | `tick`, `time`, `bar`, or `batch` | +| `ts`, `bar.ts_open` | integer | Non-negative milliseconds, within int64 | +| `seq` | integer | Positive uint64; subsequent ticks are contiguous | +| `price`, `qty` | number | Finite and positive | +| `bar.o`, `bar.h`, `bar.l`, `bar.c` | number | Finite, positive, valid OHLC range | +| `bar.v` | number | Finite and non-negative | +| `bar.trade_count` | integer, optional | Non-negative uint64; accepted but ignored | +| `events` | array | 1..1024 non-batch events | + +Unlike the retired provider parser, `forming` is refused, tick mode uses +explicit `time` boundaries, and duplicate input is accepted only as an +index-aligned identical prefix, including identical message framing. + ### Confirmed OHLCV mode ```json @@ -159,47 +256,10 @@ configuration. Ordinary security evaluations derived from the input stream retain the existing native engine behavior. -## User-defined broker parsers - -Use `--parser your-parser.so --parser-config mapping.json` to translate a -provider's complete message into normalized ticks, bars or time boundaries. -The parser can emit zero events for a recognized control message, or several -ordered events for a provider batch. The runner stages and validates the whole -parser result before applying any events. Parser errors emit no partial parse. -All normalized events from one provider message then commit as one ledger -input and outbox transaction; a failure during application discards the -instance without committing any part of that message. - -The public [C ABI header](../include/pineforge/live_parser.h) defines the -versioned POD and callbacks. A C++ plugin exports: - -```cpp -uint32_t pf_live_parser_abi_version(void); -int pf_live_parse_message(const char* message, size_t message_size, - const char* config_json, size_t config_size, - pf_live_parser_emit_v1_fn emit, void* user); -``` - -Initialize `pf_live_parser_event_v1_t` to zero, set its kind and active fields, -then call `emit(&event, user)`. Return failure if parsing or the callback fails; -never throw across the ABI. Limits are 1 MiB per message and 1024 events. -For stdin/files each line is one provider message; for WebSocket feeds one -complete text message is passed intact. HTTP uses one provider message per -snapshot line. See [examples/demo_parser.cpp](examples/demo_parser.cpp), which -maps `trade,sequence,timestamp,price,quantity` and ignores `heartbeat`. - -Parsers are trusted executable native code. Version 1 requires stateless, -deterministic output for identical message/config bytes; cross-message state -or protocol assembly belongs in an upstream feed adapter. Do not infer -sequence numbers from mutable parser globals. The runner validates ordering -and configured feed mode after parsing. Parser library/config hashes are -part of deployment identity; configuration bytes are not written to the -order ledger or exposed in transport errors. - ## Durable orders and recovery One process holds an exclusive lock beside the SQLite ledger. Each successful -provider message commits its normalized event or batch, observable engine/stream hash and +feed message commits its normalized event or batch, observable engine/stream hash and ordered immutable webhook events in one database transaction. Network delivery starts only after commit. A failed engine/database operation ends that process; it never continues with an advanced but uncommitted strategy. @@ -213,16 +273,15 @@ and stream state, not arbitrary C++ private variables. Files and HTTP snapshots normally repeat the full normalized event prefix; matching records are skipped, changed records are refused. `--from-input N` -declares the zero-based first **nonempty provider message** in a resumed -file/stdin/WS tail; it cannot skip beyond the recorded count. A parser batch -of multiple events uses one index, and heartbeats emitting no events use -none. HTTP snapshots always begin at index zero. The final -stdout JSON reports `inputs_committed`; use that cursor with a provider that -can resume exactly. `--max-events N` counts newly committed nonempty provider +declares the zero-based first **nonempty feed message** in a resumed +file/stdin/WS tail; it cannot skip beyond the recorded count. An atomic batch +of multiple events uses one index. HTTP snapshots always begin at index zero. The final +stdout JSON reports `inputs_committed`; use that cursor with a feed adapter that +can resume exactly. `--max-events N` counts newly committed nonempty feed messages, not individual batch elements or order actions. It stops only between atomic messages. `last_tick_sequence` reports the last committed -source tick sequence for provider-specific resumption. The built-in JSONL -parser also accepts `{"type":"batch","events":[...]}` with 1..1024 +source tick sequence for adapter-specific resumption. JSONL +also accepts `{"type":"batch","events":[...]}` with 1..1024 normalized tick/bar/time events. The webhook payload schema is `pineforge-native-order-action/v1`: @@ -238,16 +297,142 @@ price come from actual engine fill observations, including same-input roundtrips and partial exits; no net-position-only inference is used. Pending-order create/replace/cancel operations are not fill actions. -Delivery is **at least once**: a receiver may accept a request before the -runner can persist its acknowledgment. Deduplicate `event_id` before trading. -Attempts are saved before requests. The oldest unacknowledged event blocks -later delivery; bounded exponential delays retry transient errors. Permanent -HTTP refusal, including redirects, stops the process. After investigating, restart to retry, raising -`--max-attempts` beyond the durable count if its default of 8 was exhausted. -This limit does not discard events or reset their IDs. There is no implicit -skip/rewrite of a failed order. +Delivery runs independently of computation, in commit order with bounded +per-target concurrency. A receiver can accept a request before its result is +durable, so it must deduplicate the idempotency key. HTTP errors are final; +only transport errors receive bounded retries. Failures stay visible in the +append-only delivery log and `status`; use `redeliver` to resend them. No HTTP +failure stops or influences the strategy. With no webhook, actions are still +committed and can be read with `actions --follow`. See routing below for v2, +per-target HMAC, delivery configuration and migration details. This runner has its own ledger/schema and native tick semantics. It does not open journals of the retired Python `pineforge-live` runtime, and that runtime's evidence does not validate this execution path: record native parity and recovery evidence before replacing a Python deployment. + +## Routing order actions to webhook targets + +Implemented B1 routing is runner-only. The engine only computes; receivers are +your own applications, never exchanges or fill-ingestion endpoints. + +### 1. Configuration +`pineforge-live run ... --webhook-routes routes.json` reads a strict JSON file. Unknown keys, an undefined target, or an unsupported selector fail at startup, before any input is read. +```json +{ + "schema_version": 1, + "default_target": "default", + "targets": { + "default": {"url": "https://consumer.example/actions", "secret_env": "DEFAULT_HMAC"}, + "entries": {"url": "https://entry-consumer.example/actions", "secret_env": "ENTRY_HMAC"} + }, + "rules": [ + {"match": {"order_id": "Long", "kind": "entry", "side": "long"}, "target": "entries"}, + {"match": {"order_id": "Hedge"}, "target": null} + ], + "delivery": {"max_in_flight": 8, "connect_timeout_ms": 2000, "total_timeout_ms": 5000, + "transport_retries": 2, "retry_backoff_ms": [1000, 2000]} +} +``` +- Without `--webhook-routes`, `--webhook-url` / `--webhook-secret-env` retain one default target, exact v1 payload bytes and `Idempotency-Key = event_id`. Delivery behavior is not unchanged: HTTP errors are final, normal runs exit 0 despite delivery failures, `--max-attempts N` caps transport retries at `min(2, N-1)`, and timeouts default to 2 s connect / 5 s total. The webhook URL is optional; see §5 and the changelog for migration and recovery. +- With it, those two flags set the default target, and they must agree with the file. +- URLs are HTTPS with no user info and no redirects. Plain HTTP stays a test-only opt-in. +- Secret values never appear in the file, the ledger, a payload or a log. +- Target URLs are stored in clear in the ledger's routing configuration for offline redelivery. A URL must not carry a token; use `secret_env` for signing credentials. + +### 2. Matching +- Rules are checked in file order and the **first match wins**; with no match, the default target is used. Every predicate given must match; an omitted predicate matches anything. Matching is exact and case-sensitive. +- **One target per action** (no fan-out), so the same action is never sent to two venues by accident. +- Selectors: + - **B1:** `order_id` (the `strategy.entry` / `strategy.exit` / `strategy.order` id), `kind` (`entry` / `exit`) and `side` (the position side: long / short). These are available today. + - **B2 (planned):** `alert_message` and `kind: "close"`. These need a small additive engine + codegen extension that carries Pine's `alert_message` and a distinct close provenance with each action. Until a strategy library provides it, a rule using these selectors is refused at startup, never guessed. + +### 3. Webhooks are optional +- Every action is always committed to the runner's ledger first. +- `"target": null`, in a rule or as `default_target`, means journal-only: the action is recorded but sent nowhere. +- A runner with no webhook configured runs fully journal-only. +- Programs that do not want HTTP read the actions from the ledger with `pineforge-live actions --ledger L --after [--follow]`, which prints one JSON action per line. This serves the hosted app and self-hosted scripts. +- `actions` and `status` accept optional `--deployment ` and compare it with the ledger's deployment identity. `redeliver` requires `--deployment ` and is an offline operation: stop the runner first; it resumes from its ledger. Live redelivery is planned for the runner-service follow-up. + +### 4. Payload `pineforge-native-order-action/v2` +- It keeps all of v1's fields: event, event_id, deployment, strategy, symbol, timeframe, sequence, timestamp, bar_index, order id/comment, buy/sell, leg, contracts, price, reduce_only, entry_incarnation. +- It adds `target_id`, `delivery_id`, `order.kind` and `order.side`. `order.alert_message` is added only where B2 provides it. +- The routing decision and the payload bytes are fixed when the action is committed. A restart never re-routes queued actions under changed rules. +- Rotating a secret changes only the signature. + +### 5. Delivery: alert-like, never blocking +- **Send once, in commit order.** Each action is committed to the ledger first, then POSTed in commit order when its target has capacity. +- **Configurable, validated at startup.** The `delivery` block sets `max_in_flight`, `connect_timeout_ms`, `total_timeout_ms`, `transport_retries` and `retry_backoff_ms`. The defaults are the numbers below, and any omitted key takes its default. +- **Computation never blocks on HTTP.** Up to 8 requests per target can be in flight. Each request has a 2 s connect timeout and a 5 s total timeout. Requests can complete out of order: receivers order by `sequence`. +- **Saturation.** An action whose target is at its in-flight limit remains durably unsent until that target has a slot. It never blocks computation or another target. Waiting actions start in commit order, ahead of due transport retries. +- **Errors are shown, then sending continues.** When the receiver answers non-2xx, or the request times out or cannot connect, the runner: + - records the result as an append-only delivery-log row; + - updates that target's status (last error, redacted; failure count; time of last success); + - writes one structured log line; + - and goes on with the next actions. A failed action never parks newer actions behind its retries. +- **Bounded transport retries.** A connection failure, a timeout before any response, or a reset is retried at most 2 times, after 1 s and then 2 s. The retries run beside newer actions and never delay them. An HTTP error response (any non-2xx, redirects included) is final, so it is shown and not retried. +- **Nothing is lost.** Every action and every delivery result stays in the ledger. `pineforge-live redeliver --ledger L --deployment D --target T [--from ] [--failed-only]` re-sends selected actions in commit order, with the same `delivery_id`, and records each new attempt. Deployment D must match the ledger's `metadata.identity`. Redelivery is offline: while the runner owns the ledger it says exactly "the runner is running: stop it first; it resumes from its ledger". Live redelivery is a runner-service follow-up. Redelivery counts selected, delivered, failed and pending actions, exits 2 if any selected action failed or is pending, and reads only the selected target's secret. `pineforge-live actions --follow` streams every committed action, whatever happened to its delivery. +- **Restart.** After the usual replay verification, an action that was committed but has no delivery result yet (the process died before sending, or mid-request) is sent once. An action whose delivery failed is not re-sent automatically; `redeliver` does that. +- **Fatal exit:** delivery drains without retries for at most one `total_timeout_ms` in total, regardless of targets or action count. The final error reports actions with no delivery result and gives `pineforge-live redeliver --ledger L --deployment D --target T` guidance. SIGINT/SIGTERM promptly cancel delivery, including EOF drain and redelivery, and exit 130. +- **Status:** `pineforge-live status --ledger L [--deployment D]` prints a consistent read snapshot as JSON, per target: sent, failed, unsent (committed actions with no delivery result), last success, last error (redacted), last attempt. Use offline `redeliver` for failed or unsent actions; omit `--failed-only` to include unsent actions. +- **Audit (closes audit finding F11):** every attempt is a new delivery-log row: target, delivery_id, attempt, start/end time, HTTP status or error class. Nothing is updated in place. + +### 6. Idempotency and security +- `delivery_id` = SHA-256 of `{"event_id","target_id"}`, sent as `Idempotency-Key`; the original `event_id` goes in `X-PineForge-Event-Id`. Both are stable across retries and restarts. +- Receivers must deduplicate on `delivery_id` and answer 2xx only after accepting the action. +- Each target signs with its own `X-PineForge-Signature: sha256=`. + +#### Validation and ledger compatibility + +Target names contain 1..128 identifier characters (letters, digits, `_`, `.`, +`:` or `-`). A file supports at most 128 targets and 4096 rules. Environment +variable names must be valid identifiers, not literal secrets. Every named +target requires `url` and `secret_env`; the legacy CLI may omit signing. + +`max_in_flight` is an integer in 1..1024. Timeouts are integer milliseconds in +1..300000, with connect timeout no greater than total timeout. Retries are +0..2. Backoffs are an array of at most two integer delays in 1..300000 ms, +with a delay for each enabled retry. Omitting a key uses its documented default. +An action at a target whose in-flight limit is reached stays durable and +unsent until that target has a slot; it never blocks computation or another +target. New actions take priority over due transport retries. + +The routing file bytes join deployment identity. Restore the original file +or choose a new ledger after any routing configuration change. Secrets are +read from the environment at startup, so rotating their values does not +change deployment identity or payload bytes. + +The phase-A schema-1 ledger receives additive `event_routes`, +`routing_configuration` and `delivery_log` tables. Existing event IDs, payloads, +input records and acknowledged flags are not rewritten. Old acknowledged +events remain delivered; old unacknowledged events with no new delivery result +are sent once after replay. Old v1 events retain `Idempotency-Key = event_id`. + +Each attempt appends a `started` row before HTTP and a `completed` row with +its result afterwards, sharing the attempt number and start time. Interrupted +attempts therefore remain visible without mutating a row. SQLite triggers +refuse updates or deletions of delivery-log rows. The main thread and the +single delivery worker serialize database operations; no database lock is +held during HTTP. Only the main thread accesses the strategy. No delivery +worker starts in fully journal-only mode. Delivery scans advance an in-memory +low-water ledger ordinal using constant-size statements, never rescanning old +journal-only or failed actions during normal delivery. Proxy environment values +are captured once on the main thread and supplied explicitly to libcurl handles. +The system libcurl, SQLite, libc and resolver are not instrumented by TSan; +resolver/library environment access beyond proxy settings remains outside its coverage. + +`status` reports `targets`, including targets with no attempts. `sent` counts +successful attempts (including previously acknowledged v1 events), `failed` +counts failed attempts, and timestamps are epoch milliseconds. `last_error` +contains only a fixed category, HTTP status and time, never receiver response +text or a URL. `actions --after N` is exclusive; `redeliver --from N` is +inclusive. Both cursors are ledger action ordinals, equal to the action sequence +in this runner. `--failed-only` selects actions +whose latest completed attempt failed, not successful or never-attempted +actions. Redelivery reads target endpoints and environment variable names +from the ledger; it cannot change an action’s target. + +The legacy `--max-attempts` option remains accepted; when explicitly supplied +without a routes file, it caps transport retries to `min(2, N-1)`. HTTP errors +are always final. Delivery failures do not stop feed computation or change +the run exit status; they are reported in stderr, the audit log and status. diff --git a/runner/delivery.cpp b/runner/delivery.cpp new file mode 100644 index 000000000..0ea5bcdf7 --- /dev/null +++ b/runner/delivery.cpp @@ -0,0 +1,163 @@ +#include "delivery.hpp" + +#include +#include +#include +#include +#include + +namespace pineforge::live { +namespace { +using DeliveryClock = std::chrono::steady_clock; + +std::uint64_t wall_time() { + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); +} + +std::int64_t clock_time() { + return std::chrono::duration_cast( + DeliveryClock::now().time_since_epoch()).count(); +} + +void error_line(const std::string& line) { + const auto written = ::write(STDERR_FILENO, line.data(), line.size()); + (void)written; +} + +struct ActiveDelivery { + DeliveryAttempt attempt; + unsigned retries = 0; +}; + +struct RetryDelivery { + StoredEvent event; + unsigned retries = 0; + DeliveryClock::time_point due; +}; +} + +DeliveryWorker::DeliveryWorker(Ledger& ledger, DeliveryOptions settings, + std::map targets, + std::optional> redeliver, + std::function stopped) + : ledger_(ledger), settings_(std::move(settings)), targets_(std::move(targets)), + redeliver_(std::move(redeliver)), stopped_(std::move(stopped)) { + if (!targets_.empty()) worker_ = std::thread([this] { + try { run(); } catch (...) { + error_ = std::current_exception(); + worker_failed_.store(true, std::memory_order_release); + error_line("{\"event\":\"webhook_worker_failed\",\"error_class\":\"delivery_worker_failure\"}\n"); + } + }); +} + +DeliveryWorker::~DeliveryWorker() { + cancelling_ = true; + if (worker_.joinable()) worker_.join(); +} + +void DeliveryWorker::check() const { + if (worker_failed_.load(std::memory_order_acquire)) + throw std::runtime_error("webhook delivery worker failed; committed actions remain in the ledger"); +} + +void DeliveryWorker::limit_drain() { + drain_until_ = clock_time() + settings_.total_timeout_ms; +} + +void DeliveryWorker::finish(bool cancel) { + cancelling_ = cancel; + finishing_ = true; + if (worker_.joinable()) worker_.join(); + if (error_) std::rethrow_exception(error_); +} + +void DeliveryWorker::run() { + WebhookMulti transport; + std::map active; + std::map in_flight; + std::map> pending; + std::vector retries; + std::uint64_t next_key = 0, low = 0; + std::size_t redelivery_index = 0; + const auto start = [&](const StoredEvent& event, unsigned retry) { + const auto& target = *event.target_id; + const auto options = targets_.find(target); + if (options == targets_.end()) throw std::runtime_error("stored action refers to an unavailable webhook target"); + const auto key = ++next_key; + auto attempt = ledger_.start_attempt(event, wall_time()); + active.emplace(key, ActiveDelivery{std::move(attempt), retry}); + transport.add(key, options->second, event); + ++in_flight[target]; + }; + while (!cancelling_ && !(stopped_ && stopped_())) { + const auto deadline = drain_until_.load(); + if (deadline && clock_time() >= deadline) return; + if (deadline) retries.clear(); + bool exhausted = false; + for (unsigned scanned = 0; scanned < 256; ++scanned) { + if (cancelling_ || (stopped_ && stopped_()) || + (drain_until_ && clock_time() >= drain_until_)) return; + if (redeliver_) { + if (redelivery_index == redeliver_->size()) { exhausted = true; break; } + const auto& event = redeliver_->at(redelivery_index++); + pending[*event.target_id].push_back(event); + } else { + const auto next = ledger_.next_delivery_event(low); + if (!next) { exhausted = true; break; } + low = next->event.ordinal; + if (next->unsent) pending[*next->event.target_id].push_back(next->event); + } + } + for (;;) { + auto first = pending.end(); + for (auto position = pending.begin(); position != pending.end(); ++position) + if (!position->second.empty() && in_flight[position->first] < settings_.max_in_flight && + (first == pending.end() || position->second.front().ordinal < first->second.front().ordinal)) + first = position; + if (first == pending.end()) break; + if (cancelling_ || (stopped_ && stopped_()) || + (drain_until_ && clock_time() >= drain_until_)) return; + start(first->second.front(), 0); + first->second.pop_front(); + } + const bool waiting = std::any_of(pending.begin(), pending.end(), + [](const auto& target) { return !target.second.empty(); }); + if (!drain_until_) for (auto position = retries.begin(); position != retries.end();) { + if (drain_until_ || cancelling_ || (stopped_ && stopped_())) break; + if (position->due <= DeliveryClock::now() && in_flight[*position->event.target_id] < settings_.max_in_flight) { + start(position->event, position->retries); + position = retries.erase(position); + } else ++position; + } + if (finishing_ && exhausted && !waiting && active.empty() && retries.empty()) return; + const auto remaining = drain_until_ ? std::max(0, drain_until_ - clock_time()) : 20; + for (const auto& completed : transport.poll(static_cast(std::min(20, remaining)))) { + const auto found = active.find(completed.key); + if (found == active.end()) throw std::runtime_error("unknown delivery result"); + const auto& attempt = found->second.attempt; + const auto& result = completed.result; + const auto ended = wall_time(); + ledger_.finish_attempt(attempt, ended, result.status, result.success, result.error); + --in_flight[*attempt.event.target_id]; + if (result.success) ++delivered_; + else { + ++failed_; + error_line(Json::object({{"event", Json::string("webhook_delivery_error")}, + {"event_id", Json::string(attempt.event.id)}, {"target_id", Json::string(*attempt.event.target_id)}, + {"delivery_id", Json::string(attempt.event.delivery_id)}, {"attempt", Json::number(std::to_string(attempt.attempt))}, + {"started_at", Json::number(std::to_string(attempt.started_at))}, {"ended_at", Json::number(std::to_string(ended))}, + {"http_status", Json::number(std::to_string(result.status))}, {"error_class", Json::string(result.error)}}).dump() + '\n'); + if (!drain_until_ && result.retryable && found->second.retries < settings_.transport_retries) { + const auto retry = found->second.retries; + retries.push_back({attempt.event, retry + 1, + DeliveryClock::now() + std::chrono::milliseconds(settings_.retry_backoff_ms.at(retry))}); + } + } + active.erase(found); + } + } +} + +} diff --git a/runner/delivery.hpp b/runner/delivery.hpp new file mode 100644 index 000000000..987ec3c54 --- /dev/null +++ b/runner/delivery.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include "routing.hpp" +#include "store.hpp" + +#include +#include +#include +#include +#include + +namespace pineforge::live { + +class DeliveryWorker { +public: + DeliveryWorker(Ledger& ledger, DeliveryOptions settings, + std::map targets, + std::optional> redeliver = std::nullopt, + std::function stopped = {}); + ~DeliveryWorker(); + DeliveryWorker(const DeliveryWorker&) = delete; + DeliveryWorker& operator=(const DeliveryWorker&) = delete; + void finish(bool cancel = false); + void limit_drain(); + void check() const; + std::uint64_t delivered() const { return delivered_.load(); } + std::uint64_t failed() const { return failed_.load(); } +private: + void run(); + Ledger& ledger_; + DeliveryOptions settings_; + std::map targets_; + std::optional> redeliver_; + std::function stopped_; + std::atomic finishing_{false}, cancelling_{false}; + std::atomic worker_failed_{false}; + std::atomic drain_until_{0}; + std::atomic delivered_{0}, failed_{0}; + std::exception_ptr error_; + std::thread worker_; +}; + +} diff --git a/runner/examples/demo_parser.cpp b/runner/examples/demo_parser.cpp deleted file mode 100644 index d6b699cbe..000000000 --- a/runner/examples/demo_parser.cpp +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Illustrative provider-neutral mapping, not an exchange protocol: -// trade,,,, -// heartbeat -// One complete message per call. No state, I/O, or configuration is needed. -// Example Linux build: -// c++ -std=c++17 -shared -fPIC -Iinclude runner/examples/demo_parser.cpp -o demo_parser.so -// macOS: replace -shared with -dynamiclib and use demo_parser.dylib. -#include - -#include -#include -#include -#include - -namespace { -template bool number(std::string_view text, T& value) { - if (text.empty()) return false; - const auto parsed = std::from_chars(text.data(), text.data() + text.size(), value); - return parsed.ec == std::errc{} && parsed.ptr == text.data() + text.size(); -} -} - -extern "C" PF_LIVE_PARSER_API uint32_t pf_live_parser_abi_version(void) { - return PF_LIVE_PARSER_ABI_VERSION; -} - -extern "C" PF_LIVE_PARSER_API int pf_live_parse_message( - const char* message, size_t message_size, - const char* config_json, size_t config_size, - pf_live_parser_emit_v1_fn emit, void* user) { - if ((!message && message_size) || (!config_json && config_size) || !emit - || message_size > PF_LIVE_PARSER_MAX_MESSAGE_BYTES) return -1; - const std::string_view text(message ? message : "", message_size); - if (text == "heartbeat") return 0; - std::array cells{}; - size_t offset = 0; - for (size_t i = 0; i < cells.size(); ++i) { - const auto end = text.find(',', offset); - if ((i + 1 == cells.size()) != (end == std::string_view::npos)) return -1; - cells[i] = text.substr(offset, end == std::string_view::npos ? end : end - offset); - if (end != std::string_view::npos) offset = end + 1; - } - pf_live_parser_event_v1_t event{}; - event.kind = PF_LIVE_PARSER_TICK; - if (cells[0] != "trade" || !number(cells[1], event.sequence) - || !number(cells[2], event.timestamp) || !number(cells[3], event.price) - || !number(cells[4], event.quantity) || !event.sequence || event.timestamp < 0 - || !std::isfinite(event.price) || event.price <= 0 - || !std::isfinite(event.quantity) || event.quantity <= 0) return -1; - return emit(&event, user) == 0 ? 0 : -1; -} diff --git a/runner/main.cpp b/runner/main.cpp index 641b9cdc9..5bc9e88b7 100644 --- a/runner/main.cpp +++ b/runner/main.cpp @@ -1,14 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 #include "json.hpp" #include "native_startup.hpp" -#include "parser.hpp" #include "store.hpp" #include "transport.hpp" +#include "delivery.hpp" #include #include #include -#include #include #include #include @@ -30,7 +29,8 @@ namespace { using namespace pineforge::live; namespace fs = std::filesystem; -volatile std::sig_atomic_t stopped = 0; +std::atomic stopped{0}; +static_assert(std::atomic::is_always_lock_free); void signal_stop(int) { stopped = 1; } constexpr std::size_t MAX_FRAME = 1024 * 1024; @@ -38,7 +38,8 @@ struct Config { std::string strategy, warmup, feed = "-", ledger, webhook, mode = "", input_tf = "1", script_tf, symbol, name = "strategy"; std::string session = "24x7", timezone = "UTC", chart_timezone = "UTC", secret_env, feed_url, - parser_path, parser_config_path, subscribe_path, native_config; + subscribe_path, native_config, routes_path; + RoutingConfig routing; std::vector> inputs, overrides, syminfo; std::set explicit_flags; NativeConfigValues native; @@ -51,14 +52,14 @@ void help() { std::cout << "PineForge native live runner (C++17)\n" "Usage: pineforge-live run --strategy strategy.so --warmup history.csv\n" " --script-tf 15 --mode ticks|bars --ledger orders.sqlite3\n" - " --webhook-url https://receiver.example/events --symbol EXCHANGE:SYMBOL\n" + " --symbol EXCHANGE:SYMBOL [--webhook-url https://receiver.example/events]\n" " [--feed events.jsonl|- | --feed-url https://...|wss://...]\n" "Options: --input-tf 1 --name NAME --session 24x7 --timezone UTC\n" " --input TITLE=VALUE --override KEY=VALUE (repeatable)\n" " --syminfo KEY=VALUE --chart-timezone UTC\n" - " --parser parser.so --parser-config mapping.json\n" " --subscribe subscription.json (WebSocket only)\n" " --webhook-secret-env NAME --allow-insecure-http\n" + " --webhook-routes FILE (strict per-action routing; optional webhooks)\n" " --from-input N --max-events N --max-attempts 8\n" " --native-config FILE (strict native run specification)\n" " --check (one HTTP snapshot) --poll-ms 1000\n" @@ -69,7 +70,11 @@ void help() { " {\"type\":\"time\",\"ts\":120000} (tick mode only)\n" "Recovery replays immutable warmup + ledger inputs before any delivery.\n" "File/HTTP input defaults to the full recorded prefix; --from-input declares\n" - "the zero-based start of a resumed tail. HTTP polls use full snapshots.\n"; + "the zero-based start of a resumed tail. HTTP polls use full snapshots.\n" + "Usage: pineforge-live actions --ledger L --after N [--follow] [--deployment D]\n" + " pineforge-live status --ledger L [--deployment D]\n" + " pineforge-live redeliver --ledger L --deployment D --target T [--from N] [--failed-only]\n" + "Redeliver is offline: stop the runner first; it resumes from its ledger.\n"; } std::uint64_t unsigned_arg(const std::string &s) { return Json::number(s).integer(); @@ -100,6 +105,8 @@ Config args(int argc, char **argv) { std::set seen; for (int i = 2; i < argc; ++i) { std::string a = argv[i]; + if (a == "--parser" || a == "--parser-config") + throw std::runtime_error("in-runner parsers were removed; use an external feed adapter (runner/README.md#feed-format)"); if (a != "--input" && a != "--override" && a != "--syminfo" && !seen.insert(a).second) throw std::runtime_error("duplicate option: " + a); c.explicit_flags.insert(a); @@ -126,6 +133,8 @@ Config args(int argc, char **argv) { c.ledger = v; else if (a == "--webhook-url") c.webhook = v; + else if (a == "--webhook-routes") + c.routes_path = v; else if (a == "--mode") c.mode = v; else if (a == "--input-tf") @@ -146,10 +155,6 @@ Config args(int argc, char **argv) { c.chart_timezone = v; else if (a == "--syminfo") c.syminfo.push_back(pair_arg(v)); - else if (a == "--parser") - c.parser_path = v; - else if (a == "--parser-config") - c.parser_config_path = v; else if (a == "--subscribe") c.subscribe_path = v; else if (a == "--input") @@ -172,8 +177,8 @@ Config args(int argc, char **argv) { } else throw std::runtime_error("unknown option: " + a); } - if (c.strategy.empty() || c.warmup.empty() || c.ledger.empty() || c.webhook.empty()) - throw std::runtime_error("strategy, warmup, ledger and webhook-url are required"); + if (c.strategy.empty() || c.warmup.empty() || c.ledger.empty()) + throw std::runtime_error("strategy, warmup and ledger are required"); if (c.mode != "bars" && c.mode != "ticks") throw std::runtime_error("mode must be bars or ticks"); if (!c.native_config.empty()) { @@ -195,8 +200,6 @@ Config args(int argc, char **argv) { throw std::runtime_error("HTTP snapshots require from-input 0"); if (!c.subscribe_path.empty() && !websocket) throw std::runtime_error("subscribe requires WebSocket feed-url"); - if (!c.parser_config_path.empty() && c.parser_path.empty()) - throw std::runtime_error("parser-config requires parser"); if (!c.max_attempts || c.max_attempts > 1000) throw std::runtime_error("max-attempts must be 1..1000"); if (!c.allow_http && (c.webhook.rfind("http://", 0) == 0 || @@ -214,7 +217,7 @@ Config args(int argc, char **argv) { throw std::runtime_error("native runner supports close-only strategy calculation"); auto ledger = fs::weakly_canonical(fs::absolute(c.ledger)); for (const auto &src : {c.strategy, c.warmup, c.feed == "-" ? std::string{} : c.feed, - c.parser_path, c.parser_config_path, c.subscribe_path}) + c.subscribe_path, c.routes_path, c.native_config}) if (!src.empty()) { auto path = fs::weakly_canonical(fs::absolute(src)); for (const auto &suffix : {"", "-wal", "-shm", ".lock"}) @@ -524,7 +527,9 @@ void apply(Strategy &s, const Config &c, Cursor &cursor, const Json &frame) { if (c.mode != "bars") throw std::runtime_error("ticks mode uses trade ticks and explicit time boundaries"); const auto &j = frame.at("bar"); - only_fields(j, {"ts_open", "o", "h", "l", "c", "v"}); + only_fields(j, {"ts_open", "o", "h", "l", "c", "v", "trade_count"}); + if (j.find("trade_count")) + j.at("trade_count").integer(); pf_bar_t b{}; b.timestamp = j.at("ts_open").integer(); b.open = j.at("o").real(); @@ -585,7 +590,18 @@ std::vector actions(Strategy &s, const Config &c, const std::string &depl {"price", real(a.price)}, {"reduce_only", Json::boolean(!a.is_entry)}, {"entry_incarnation", num(a.entry_incarnation)}})}}); - out.push_back({std::move(id), payload.dump()}); + const std::string kind = a.is_entry ? "entry" : "exit"; + const std::string side = a.is_long ? "long" : "short"; + const auto target = c.routing.select(a.order_id ? a.order_id : "", kind, side); + const auto delivery_id = c.routing.routed ? delivery_identity(id, target) : id; + if (c.routing.routed) { + payload.members["schema_version"] = Json::string("pineforge-native-order-action/v2"); + payload.members["target_id"] = target ? Json::string(*target) : Json{}; + payload.members["delivery_id"] = Json::string(delivery_id); + payload.members["order"].members["kind"] = Json::string(kind); + payload.members["order"].members["side"] = Json::string(side); + } + out.push_back({std::move(id), payload.dump(), target, delivery_id}); } return out; } @@ -601,55 +617,49 @@ void apply_record(Strategy &strategy, const Config &c, Cursor &cursor, const Jso for (const auto &event : events.items) apply(strategy, c, cursor, event); } -std::vector normalize(Parser *parser, const std::string &message) { - if (!parser) - return {parse_json(message)}; - std::vector result; - for (const auto &event : parser->parse(message)) { - auto ts = Json::number(std::to_string(event.timestamp)); - if (event.kind == PF_LIVE_PARSER_TICK) - result.push_back(Json::object({{"type", Json::string("tick")}, - {"ts", ts}, - {"seq", num(event.sequence)}, - {"price", real(event.price)}, - {"qty", real(event.quantity)}})); - else if (event.kind == PF_LIVE_PARSER_TIME) - result.push_back(Json::object({{"type", Json::string("time")}, {"ts", ts}})); - else - result.push_back(Json::object({{"type", Json::string("bar")}, - {"bar", Json::object({{"ts_open", ts}, - {"o", real(event.open)}, - {"h", real(event.high)}, - {"l", real(event.low)}, - {"c", real(event.close)}, - {"v", real(event.volume)}})}})); - } - return result; +void require_feed_event(const Json &event) { + const auto type = event.at("type").text(); + if (type == "tick") { + only_fields(event, {"type", "ts", "seq", "price", "qty"}); + event.at("ts").integer(); + event.at("seq").integer(); + event.at("price").real(); + event.at("qty").real(); + } else if (type == "time") { + only_fields(event, {"type", "ts"}); + event.at("ts").integer(); + } else if (type == "bar") { + only_fields(event, {"type", "bar"}); + const auto &bar = event.at("bar"); + only_fields(bar, {"ts_open", "o", "h", "l", "c", "v", "trade_count"}); + if (bar.find("trade_count")) + bar.at("trade_count").integer(); + bar.at("ts_open").integer(); + for (const auto *field : {"o", "h", "l", "c", "v"}) + bar.at(field).real(); + } else + throw std::runtime_error("expected tick, time or confirmed bar event"); } -bool drain(Ledger &ledger, const HttpOptions &options, const Config &c, std::uint64_t &delivered) { - while (!stopped) { - auto e = ledger.pending_event(); - if (!e) - return true; - if (e->attempts >= c.max_attempts) - throw std::runtime_error("webhook retry limit reached; queued event remains in ledger"); - ledger.begin_delivery(e->id); - auto result = post_webhook(options, *e); - if (result.success) { - ledger.acknowledge(e->id); - ++delivered; - continue; - } - ledger.record_delivery_failure(e->id, result.error); - if (result.status >= 300 && result.status < 500 && result.status != 408 && - result.status != 429) - throw std::runtime_error("webhook receiver refused event; event remains queued"); - const auto delay = - std::min(5000, 100ULL << std::min(e->attempts, 5)); - for (std::uint64_t n = 0; n < delay && !stopped; n += 100) - std::this_thread::sleep_for(std::chrono::milliseconds(100)); +Json feed_record(const std::string &message) { + try { + auto record = parse_json(message); + if (record.kind == Json::Kind::Array) + record = Json::object({{"type", Json::string("batch")}, {"events", std::move(record)}}); + if (record.at("type").text() == "batch") { + only_fields(record, {"type", "events"}); + const auto &events = record.at("events"); + if (events.kind != Json::Kind::Array || events.items.empty() || events.items.size() > 1024) + throw std::runtime_error("normalized batch requires 1..1024 events"); + for (const auto &event : events.items) + require_feed_event(event); + } else + require_feed_event(record); + return record; + } catch (const std::exception &error) { + throw std::runtime_error( + std::string("invalid feed message: ") + error.what() + + "; PineForge feed events required; use an external feed adapter (runner/README.md#feed-format)"); } - return false; } LegacyIdentityFields legacy_fields(const Config &c) { return {c.mode, c.input_tf, c.script_tf, c.session, c.timezone, c.chart_timezone, @@ -657,6 +667,20 @@ LegacyIdentityFields legacy_fields(const Config &c) { } int run(Config c) { + c.routing = c.routes_path.empty() + ? single_target(c.webhook, c.secret_env, c.allow_http) + : parse_routes(read_file(c.routes_path, MAX_FRAME), c.allow_http, c.webhook, c.secret_env); + auto targets = c.routing.load_secrets(); + if (c.routing.routed) + c.webhook = c.routing.default_target ? c.routing.targets.at(*c.routing.default_target).url : ""; + else if (c.explicit_flags.count("--max-attempts")) + c.routing.delivery.transport_retries = static_cast(std::min(2, c.max_attempts - 1)); + if (c.feed_url.rfind("ws://", 0) == 0 || c.feed_url.rfind("wss://", 0) == 0) { + HttpOptions feed; + feed.url = c.feed_url; + feed.allow_insecure_http = c.allow_http; + validate_websocket(feed); + } if (!c.native_config.empty()) { c.native = parse_native_config(read_file(c.native_config, MAX_FRAME)); NativeClockBindings clock{c.input_tf, c.script_tf, c.timezone, c.session, @@ -669,19 +693,11 @@ int run(Config c) { c.chart_timezone = clock.chart_timezone; c.symbol = clock.symbol; validate_native_config(c.native); + if (c.input_tf != "1") + throw std::runtime_error("native runner input-tf currently must be 1 minute"); } auto original = read_file(c.warmup, 512ULL * 1024 * 1024); auto library = read_file(c.strategy, 512ULL * 1024 * 1024); - std::string parser_bytes = - c.parser_path.empty() ? "" : read_file(c.parser_path, 64ULL * 1024 * 1024); - std::string parser_config = - c.parser_config_path.empty() ? "{}" : read_file(c.parser_config_path, MAX_FRAME); - std::unique_ptr parser; - if (!c.parser_path.empty()) - parser = std::make_unique(c.parser_path, parser_config); - if (!c.parser_path.empty() && - sha256_hex(read_file(c.parser_path, 64ULL * 1024 * 1024)) != sha256_hex(parser_bytes)) - throw std::runtime_error("parser library changed during initialization"); Strategy strategy; strategy.load(c.strategy); if (sha256_hex(read_file(c.strategy, 512ULL * 1024 * 1024)) != sha256_hex(library)) @@ -692,11 +708,13 @@ int run(Config c) { const auto settings_receipt = strategy.effective_settings(); std::string deployment = c.native.present - ? native_identity(c.native, c.mode, c.name, c.webhook, original, library, parser_bytes, - parser_config) - : identity(legacy_fields(c), original, library, parser_bytes, parser_config); + ? native_identity(c.native, c.mode, c.name, c.webhook, original, library) + : identity(legacy_fields(c), original, library); if (!settings_receipt.empty()) deployment = sha256_hex(deployment + ":settings-v1:" + settings_receipt); + if (c.routing.routed) + deployment = sha256_hex(Json::object({{"deployment", Json::string(deployment)}, + {"webhook_routes", Json::string(c.routing.file_identity)}}).dump()); try { // A switched PineStrategyHost is native-bound but owns its run spec // through prepare_native_begin. Let that provider admit the stream @@ -721,6 +739,7 @@ int run(Config c) { if (c.native.present) require_native_warmup(c.native, warmup); Ledger ledger(c.ledger, deployment); + ledger.bind_routing(c.routing.stored_document()); Cursor cursor; auto recorded = ledger.input_count(); for (std::uint64_t i = 0; i < recorded; ++i) { @@ -735,36 +754,19 @@ int run(Config c) { events.size() != row->events.size()) throw std::runtime_error("native replay state/action count mismatch"); for (std::size_t k = 0; k < events.size(); ++k) - if (events[k].id != row->events[k].id || events[k].payload != row->events[k].payload) + if (events[k].id != row->events[k].id || events[k].payload != row->events[k].payload || + events[k].target_id != row->events[k].target_id || events[k].delivery_id != row->events[k].delivery_id) throw std::runtime_error("native replay order-action mismatch"); strategy.clear(strategy.state); } - HttpOptions webhook; - webhook.url = c.webhook; - webhook.allow_insecure_http = c.allow_http; - if (!c.secret_env.empty()) { - const char *value = std::getenv(c.secret_env.c_str()); - if (!value || !*value) - throw std::runtime_error("webhook secret environment variable is missing or empty"); - webhook.hmac_secret = value; - } if (c.from_input > recorded) throw std::runtime_error("from-input skips unrecorded inputs"); - std::uint64_t delivered = 0, processed = 0, replayed_prefix = 0; - drain(ledger, webhook, c, delivered); + std::uint64_t processed = 0, replayed_prefix = 0; + DeliveryWorker delivery(ledger, c.routing.delivery, std::move(targets), std::nullopt, + [] { return stopped != 0; }); auto consume_message = [&](const std::string &message, std::uint64_t &index) { - auto frames = normalize(parser.get(), message); - if (frames.empty()) - return; - Json frame; - if (frames.size() == 1) - frame = std::move(frames.front()); - else { - Json events; - events.kind = Json::Kind::Array; - events.items = std::move(frames); - frame = Json::object({{"type", Json::string("batch")}, {"events", std::move(events)}}); - } + delivery.check(); + auto frame = feed_record(message); auto canonical = frame.dump(); if (auto previous = ledger.input(index)) { if (previous->canonical_json != canonical) @@ -773,7 +775,7 @@ int run(Config c) { } else { if (index != ledger.input_count()) throw std::runtime_error("input sequence is not contiguous"); - // The entire provider message advances in memory before one + // The entire feed message advances in memory before one // input/state/outbox transaction. Failure discards this instance; // interruption and delivery begin only after every event commits. apply_record(strategy, c, cursor, frame); @@ -781,8 +783,6 @@ int run(Config c) { ledger.commit_input(index, canonical, strategy.hash(strategy.state), events); strategy.clear(strategy.state); ++processed; - if (!drain(ledger, webhook, c, delivered)) - return; } ++index; }; @@ -800,101 +800,196 @@ int run(Config c) { index < recorded) throw std::runtime_error("input snapshot omits committed prefix"); }; - if (c.feed_url.empty()) { - if (c.feed == "-") { - std::uint64_t index = c.from_input; - std::string pending; - char buffer[65536]; - while (!stopped && !(c.max_events && processed >= c.max_events)) { - pollfd fd{STDIN_FILENO, POLLIN, 0}; - int rc = poll(&fd, 1, 100); - if (rc < 0) { - if (errno == EINTR) - continue; - throw std::runtime_error("stdin poll failed"); - } - if (!rc) - continue; - auto n = read(STDIN_FILENO, buffer, sizeof buffer); - if (n < 0) { - if (errno == EINTR) + try { + if (c.feed_url.empty()) { + if (c.feed == "-") { + std::uint64_t index = c.from_input; + std::string pending; + char buffer[65536]; + while (!stopped && !(c.max_events && processed >= c.max_events)) { + delivery.check(); + pollfd fd{STDIN_FILENO, POLLIN, 0}; + int rc = poll(&fd, 1, 100); + if (rc < 0) { + if (errno == EINTR) + continue; + throw std::runtime_error("stdin poll failed"); + } + if (!rc) continue; - throw std::runtime_error("stdin read failed"); - } - if (!n) { - if (!blank(pending)) - consume_message(pending, index); - break; - } - pending.append(buffer, static_cast(n)); - for (;;) { - auto end = pending.find('\n'); - if (end == std::string::npos) + auto n = read(STDIN_FILENO, buffer, sizeof buffer); + if (n < 0) { + if (errno == EINTR) + continue; + throw std::runtime_error("stdin read failed"); + } + if (!n) { + if (!blank(pending)) + consume_message(pending, index); break; - if (end > MAX_FRAME) + } + pending.append(buffer, static_cast(n)); + for (;;) { + auto end = pending.find('\n'); + if (end == std::string::npos) + break; + if (end > MAX_FRAME) + throw std::runtime_error("input line exceeds 1 MiB"); + auto message = pending.substr(0, end); + pending.erase(0, end + 1); + if (!blank(message)) + consume_message(message, index); + if (stopped || (c.max_events && processed >= c.max_events)) + break; + } + if (pending.size() > MAX_FRAME) throw std::runtime_error("input line exceeds 1 MiB"); - auto message = pending.substr(0, end); - pending.erase(0, end + 1); - if (!blank(message)) - consume_message(message, index); - if (stopped || (c.max_events && processed >= c.max_events)) - break; } - if (pending.size() > MAX_FRAME) - throw std::runtime_error("input line exceeds 1 MiB"); + } else { + std::ifstream input(c.feed); + if (!input) + throw std::runtime_error("cannot open feed"); + consume(input, c.from_input, true); } + } else if (c.feed_url.rfind("ws://", 0) == 0 || c.feed_url.rfind("wss://", 0) == 0) { + HttpOptions feed; + feed.url = c.feed_url; + feed.allow_insecure_http = c.allow_http; + std::string subscription = + c.subscribe_path.empty() ? "" : read_file(c.subscribe_path, MAX_FRAME); + std::uint64_t index = c.from_input; + receive_websocket( + feed, subscription, + [&](std::string_view bytes) { + consume_message(std::string(bytes), index); + return !stopped && !(c.max_events && processed >= c.max_events); + }, + [&] { delivery.check(); return stopped != 0; }); } else { - std::ifstream input(c.feed); - if (!input) - throw std::runtime_error("cannot open feed"); - consume(input, c.from_input, true); + HttpOptions feed; + feed.url = c.feed_url; + feed.allow_insecure_http = c.allow_http; + do { + delivery.check(); + auto snapshot = get_feed_snapshot(feed); + std::istringstream input(snapshot); + consume(input, 0, true); + recorded = ledger.input_count(); + if (c.check || stopped || (c.max_events && processed >= c.max_events)) + break; + for (long n = 0; n < c.poll_ms && !stopped; n += 100) { + delivery.check(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } while (!stopped); } - } else if (c.feed_url.rfind("ws://", 0) == 0 || c.feed_url.rfind("wss://", 0) == 0) { - HttpOptions feed; - feed.url = c.feed_url; - feed.allow_insecure_http = c.allow_http; - std::string subscription = - c.subscribe_path.empty() ? "" : read_file(c.subscribe_path, MAX_FRAME); - std::uint64_t index = c.from_input; - receive_websocket( - feed, subscription, - [&](std::string_view bytes) { - consume_message(std::string(bytes), index); - return !stopped && !(c.max_events && processed >= c.max_events); - }, - [] { return stopped != 0; }); - } else { - HttpOptions feed; - feed.url = c.feed_url; - feed.allow_insecure_http = c.allow_http; - do { - auto snapshot = get_feed_snapshot(feed); - std::istringstream input(snapshot); - consume(input, 0, true); - recorded = ledger.input_count(); - if (c.check || stopped || (c.max_events && processed >= c.max_events)) - break; - for (long n = 0; n < c.poll_ms && !stopped; n += 100) - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } while (!stopped); + delivery.finish(stopped != 0); + } catch (const std::exception& error) { + delivery.limit_drain(); + try { delivery.finish(false); } catch (...) {} + const auto pending = ledger.unsent_count(); + const auto status = parse_json(LedgerView(c.ledger).status_json()); + std::string guidance; + for (const auto& [target, value] : status.at("targets").members) + if (value.at("unsent").integer()) + guidance += "; run `pineforge-live redeliver --ledger " + c.ledger + + " --deployment " + deployment + " --target " + target + "`"; + throw std::runtime_error(std::string(error.what()) + "; " + std::to_string(pending) + + " actions not sent" + guidance); } - auto pending = ledger.pending_count(); + auto pending = ledger.unsent_count(); std::cout << Json::object( {{"deployment", Json::string(deployment)}, {"effective_settings", settings_receipt.empty() ? Json{} : parse_json(settings_receipt)}, {"inputs_committed", num(ledger.input_count())}, {"inputs_processed", num(processed)}, {"prefix_skipped", num(replayed_prefix)}, - {"webhooks_delivered", num(delivered)}, + {"webhooks_delivered", num(delivery.delivered())}, + {"webhook_failures", num(delivery.failed())}, {"webhooks_pending", num(pending)}, {"last_tick_sequence", cursor.seen_tick ? num(cursor.tick_seq) : Json{}}}) .dump() << '\n'; - return stopped ? 130 : pending ? 2 : 0; + return stopped ? 130 : 0; +} + +int ledger_command(int argc, char** argv) { + const std::string command = argv[1]; + std::string path, target, deployment; + std::uint64_t after = 0, from = 1; + bool follow = false, failed_only = false; + std::set seen; + for (int index = 2; index < argc; ++index) { + const std::string option = argv[index]; + if (!seen.insert(option).second) throw std::runtime_error("duplicate option: " + option); + if (command == "actions" && option == "--follow") { follow = true; continue; } + if (command == "redeliver" && option == "--failed-only") { failed_only = true; continue; } + if (index + 1 == argc) throw std::runtime_error("missing option value: " + option); + const std::string value = argv[++index]; + if (option == "--ledger") path = value; + else if (option == "--deployment") deployment = value; + else if (command == "actions" && option == "--after") after = unsigned_arg(value); + else if (command == "redeliver" && option == "--from") from = unsigned_arg(value); + else if (command == "redeliver" && option == "--target") target = value; + else throw std::runtime_error("unknown option: " + option); + } + if (path.empty()) throw std::runtime_error("ledger is required"); + if (command == "redeliver" && deployment.empty()) + throw std::runtime_error("redeliver requires --deployment "); + LedgerView view(path); + if (!deployment.empty() && deployment != view.identity()) + throw std::runtime_error("ledger deployment identity mismatch"); + if (command == "status") { + std::cout << view.status_json() << '\n'; + return 0; + } + if (command == "actions") { + do { + const auto events = view.actions_after(after); + for (const auto& event : events) { + std::cout << event.payload << '\n' << std::flush; + after = event.ordinal; + } + if (events.size() == 256) continue; + if (!follow) break; + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } while (!stopped); + return stopped ? 130 : 0; + } + if (target.empty()) throw std::runtime_error("redeliver requires target"); + const auto document = view.routing_document(); + if (document.empty()) throw std::runtime_error("resume this phase-A ledger with run before redelivering"); + auto routing = restore_routes(document); + if (!routing.targets.count(target)) throw std::runtime_error("undefined webhook target: " + target); + for (auto position = routing.targets.begin(); position != routing.targets.end();) { + if (position->first != target) position = routing.targets.erase(position); + else ++position; + } + auto targets = routing.load_secrets(); + Ledger ledger(path, view.identity()); + auto events = view.redelivery_events(target, from, failed_only); + const auto selected = events.size(); + std::set selected_ids; + for (const auto& event : events) selected_ids.insert(event.id); + DeliveryWorker delivery(ledger, routing.delivery, std::move(targets), std::move(events), + [] { return stopped != 0; }); + delivery.finish(); + std::uint64_t failed = 0, pending = 0; + for (const auto& event : view.redelivery_events(target, from, true)) + if (selected_ids.count(event.id)) ++failed; + auto low = from ? from - 1 : 0; + while (const auto next = ledger.next_delivery_event(low)) { + low = next->event.ordinal; + if (next->unsent && selected_ids.count(next->event.id)) ++pending; + } + std::cout << Json::object({{"selected", num(selected)}, {"delivered", num(delivery.delivered())}, + {"failed", num(failed)}, {"pending", num(pending)}}).dump() << '\n'; + return stopped ? 130 : (failed || pending ? 2 : 0); } } // namespace int main(int argc, char **argv) { std::locale::global(std::locale::classic()); + capture_proxy_environment(); std::signal(SIGINT, signal_stop); std::signal(SIGTERM, signal_stop); try { @@ -903,6 +998,9 @@ int main(int argc, char **argv) { help(); return 0; } + const std::string command = argv[1]; + if (command == "actions" || command == "status" || command == "redeliver") + return ledger_command(argc, argv); return run(args(argc, argv)); } catch (const std::exception &e) { std::cerr << "pineforge-live: " << e.what() << '\n'; diff --git a/runner/native_startup.cpp b/runner/native_startup.cpp index 6ca68f117..da4425df7 100644 --- a/runner/native_startup.cpp +++ b/runner/native_startup.cpp @@ -513,8 +513,7 @@ Json timezone_rule_identity(std::string_view timezone, bool required) { } std::string identity_document(const LegacyIdentityFields& fields, const std::string& warmup, - const std::string& library, const std::string& parser_bytes, - const std::string& parser_config) { + const std::string& library) { Json j = Json::object({{"schema", Json::string("pineforge-native-ledger/v1")}, {"library", Json::string(sha256_hex(library))}, {"warmup", Json::string(sha256_hex(warmup))}, @@ -527,8 +526,8 @@ std::string identity_document(const LegacyIdentityFields& fields, const std::str {"symbol", Json::string(fields.symbol)}, {"name", Json::string(fields.name)}, {"webhook", Json::string(fields.webhook)}, - {"parser", Json::string(sha256_hex(parser_bytes))}, - {"parser_config", Json::string(sha256_hex(parser_config))}}); + {"parser", Json::string(sha256_hex(""))}, + {"parser_config", Json::string(sha256_hex("{}"))}}); Json input, overrides; input.kind = overrides.kind = Json::Kind::Array; for (const auto& [k, v] : fields.inputs) @@ -546,16 +545,13 @@ std::string identity_document(const LegacyIdentityFields& fields, const std::str } std::string identity(const LegacyIdentityFields& fields, const std::string& warmup, - const std::string& library, const std::string& parser_bytes, - const std::string& parser_config) { - return sha256_hex(identity_document(fields, warmup, library, parser_bytes, parser_config)); + const std::string& library) { + return sha256_hex(identity_document(fields, warmup, library)); } std::string native_identity_document(const NativeConfigValues& native, const std::string& mode, const std::string& name, const std::string& webhook, - const std::string& warmup, const std::string& library, - const std::string& parser_bytes, - const std::string& parser_config) { + const std::string& warmup, const std::string& library) { Json execution = Json::object({ {"initial_capital", json_real(native.initial_capital)}, {"point_value", json_real(native.point_value)}, @@ -580,8 +576,8 @@ std::string native_identity_document(const NativeConfigValues& native, const std {"library", Json::string(sha256_hex(library))}, {"warmup", Json::string(sha256_hex(warmup))}, {"mode", Json::string(mode)}, - {"parser", Json::string(sha256_hex(parser_bytes))}, - {"parser_config", Json::string(sha256_hex(parser_config))}, + {"parser", Json::string(sha256_hex(""))}, + {"parser_config", Json::string(sha256_hex("{}"))}, {"timezone_dependency", timezone_rule_identity(native.timezone, true)}, {"chart_timezone_dependency", timezone_rule_identity(native.chart_timezone, false)}, {"run", Json::object({{"session_key", Json::string(native.session_key)}, @@ -608,10 +604,8 @@ std::string native_identity_document(const NativeConfigValues& native, const std std::string native_identity(const NativeConfigValues& native, const std::string& mode, const std::string& name, const std::string& webhook, - const std::string& warmup, const std::string& library, - const std::string& parser_bytes, const std::string& parser_config) { - return sha256_hex(native_identity_document(native, mode, name, webhook, warmup, library, - parser_bytes, parser_config)); + const std::string& warmup, const std::string& library) { + return sha256_hex(native_identity_document(native, mode, name, webhook, warmup, library)); } } // namespace pineforge::live diff --git a/runner/native_startup.hpp b/runner/native_startup.hpp index 18fe77e6a..8c319b0ab 100644 --- a/runner/native_startup.hpp +++ b/runner/native_startup.hpp @@ -51,20 +51,15 @@ void require_native_warmup(const NativeConfigValues& spec, const std::vector -#include -#include -#include -#include -#include -#include -#include - -namespace pineforge::live { -namespace { - -static_assert(std::is_standard_layout::value, "parser ABI must be POD"); -static_assert(std::is_trivial::value, "parser ABI must be POD"); - -enum class EmitFailure { none, invalid_event, too_many_events, allocation }; - -struct Emission { - std::vector events; - EmitFailure failure = EmitFailure::none; -}; - -bool positive(double value) { return std::isfinite(value) && value > 0; } - -bool normalize(const ParsedEvent& event, ParsedEvent& result) { - if (event.reserved != 0 || event.timestamp < 0) return false; - result = {}; - result.kind = event.kind; - result.timestamp = event.timestamp; - switch (event.kind) { - case PF_LIVE_PARSER_TICK: - if (!event.sequence || !positive(event.price) || !positive(event.quantity)) return false; - result.sequence = event.sequence; - result.price = event.price; - result.quantity = event.quantity; - return true; - case PF_LIVE_PARSER_BAR: - if (event.timestamp % 60000 != 0 - || event.timestamp > std::numeric_limits::max() - 60000 - || !positive(event.open) || !positive(event.high) - || !positive(event.low) || !positive(event.close) - || !std::isfinite(event.volume) || event.volume < 0 - || event.high < std::max(event.open, event.close) - || event.low > std::min(event.open, event.close) - || event.high < event.low) return false; - result.open = event.open; - result.high = event.high; - result.low = event.low; - result.close = event.close; - result.volume = event.volume == 0 ? 0 : event.volume; - return true; - case PF_LIVE_PARSER_TIME: - return true; - default: - return false; - } -} - -int collect(const ParsedEvent* event, void* user) noexcept { - auto& emission = *static_cast(user); - if (emission.failure != EmitFailure::none) return -1; - if (emission.events.size() >= PF_LIVE_PARSER_MAX_EVENTS - || emission.events.size() >= PF_LIVE_PARSER_MAX_OUTPUT_BYTES / sizeof(ParsedEvent)) { - emission.failure = EmitFailure::too_many_events; - return -1; - } - ParsedEvent normalized{}; - if (!event || !normalize(*event, normalized)) { - emission.failure = EmitFailure::invalid_event; - return -1; - } - try { - emission.events.push_back(normalized); - } catch (...) { - emission.failure = EmitFailure::allocation; - return -1; - } - return 0; -} - -template -Function symbol(void* library, const char* name) { - dlerror(); - void* address = dlsym(library, name); - const char* error = dlerror(); - if (error || !address) throw std::runtime_error(std::string("parser plugin lacks ABI symbol: ") + name); - return reinterpret_cast(address); -} - -} // namespace - -struct Parser::Impl { - void* library = nullptr; - decltype(&pf_live_parse_message) parse = nullptr; - std::string config; - - ~Impl() { if (library) dlclose(library); } -}; - -Parser::Parser(std::string path, std::string config_json) : impl_(std::make_unique()) { - if (path.empty() || path.find('\0') != std::string::npos) - throw std::runtime_error("parser plugin path is invalid"); - if (config_json.size() > PF_LIVE_PARSER_MAX_MESSAGE_BYTES) - throw std::runtime_error("parser configuration exceeds 1 MiB"); - try { - if (parse_json(config_json).kind != Json::Kind::Object) - throw std::runtime_error("configuration must be an object"); - } catch (...) { - // Never include provider configuration (which may carry credentials). - throw std::runtime_error("parser configuration must be a valid JSON object"); - } - impl_->config = std::move(config_json); - // Resolve even a bare filename against cwd, not the loader's library search - // path, so the caller fingerprints the same artifact that is actually loaded. - const auto absolute = std::filesystem::absolute(path).string(); - impl_->library = dlopen(absolute.c_str(), RTLD_NOW | RTLD_LOCAL); - if (!impl_->library) throw std::runtime_error("cannot load parser plugin"); - const auto version = symbol(impl_->library, "pf_live_parser_abi_version"); - std::uint32_t abi = 0; - try { abi = version(); } - catch (...) { throw std::runtime_error("parser ABI version function threw an exception"); } - if (abi != PF_LIVE_PARSER_ABI_VERSION) throw std::runtime_error("parser plugin ABI version mismatch"); - impl_->parse = symbolparse)>(impl_->library, "pf_live_parse_message"); -} - -Parser::~Parser() = default; -Parser::Parser(Parser&&) noexcept = default; -Parser& Parser::operator=(Parser&&) noexcept = default; - -std::vector Parser::parse(std::string_view message) const { - if (!impl_) throw std::runtime_error("parser was moved from"); - if (message.size() > PF_LIVE_PARSER_MAX_MESSAGE_BYTES) - throw std::runtime_error("parser message exceeds 1 MiB"); - Emission emission; - int status = -1; - try { - status = impl_->parse(message.empty() ? "" : message.data(), message.size(), - impl_->config.data(), impl_->config.size(), collect, &emission); - } catch (...) { - // A conforming C plugin never throws; avoid leaking partially parsed - // data or provider error text if a C++ plugin violates that contract. - throw std::runtime_error("parser plugin threw an exception"); - } - switch (emission.failure) { - case EmitFailure::invalid_event: throw std::runtime_error("parser emitted an invalid normalized event"); - case EmitFailure::too_many_events: throw std::runtime_error("parser output exceeds the event/byte limit"); - case EmitFailure::allocation: throw std::runtime_error("cannot allocate parser output"); - case EmitFailure::none: break; - } - if (status != 0) throw std::runtime_error("parser rejected provider message"); - return std::move(emission.events); -} - -} // namespace pineforge::live diff --git a/runner/parser.hpp b/runner/parser.hpp deleted file mode 100644 index 63ac51bc9..000000000 --- a/runner/parser.hpp +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -#pragma once - -#include -#include -#include -#include -#include - -namespace pineforge::live { - -using ParsedEvent = pf_live_parser_event_v1_t; - -// Loads a trusted C ABI parser plugin (.so on Linux, .dylib on macOS). -// Each parse stages a complete message before returning any output. The caller -// stores canonical normalized events and pins plugin/config identity separately. -// A parser instance is not intended for concurrent calls. -class Parser { -public: - explicit Parser(std::string path, std::string config_json = "{}"); - ~Parser(); - Parser(const Parser&) = delete; - Parser& operator=(const Parser&) = delete; - Parser(Parser&&) noexcept; - Parser& operator=(Parser&&) noexcept; - - std::vector parse(std::string_view message) const; - -private: - struct Impl; - std::unique_ptr impl_; -}; - -} // namespace pineforge::live diff --git a/runner/routing.cpp b/runner/routing.cpp new file mode 100644 index 000000000..64f4ba9c7 --- /dev/null +++ b/runner/routing.cpp @@ -0,0 +1,202 @@ +#include "routing.hpp" + +#include +#include + +namespace pineforge::live { +namespace { + +std::optional target_name(const Json& value) { + if (value.kind == Json::Kind::Null) return std::nullopt; + auto name = value.text(); + if (name.empty() || name.size() > 128 || + name.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.:-") != std::string::npos) + throw std::runtime_error("webhook target id must be 1..128 identifier characters"); + return name; +} + +void secret_name(const std::string& name) { + if (name.empty() || name.size() > 128 || + name.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_") != std::string::npos || + (name.front() >= '0' && name.front() <= '9')) + throw std::runtime_error("webhook secret_env must name an environment variable, not contain a secret"); +} + +long bounded(const Json& value, long minimum, long maximum, const char* name) { + const auto number = value.integer(); + if (number < static_cast(minimum) || number > static_cast(maximum)) + throw std::runtime_error(std::string("webhook delivery ") + name + " out of range"); + return static_cast(number); +} + +void validate_targets(const RoutingConfig& config) { + for (const auto& [name, target] : config.targets) { + (void)name; + HttpOptions options; + options.url = target.url; + options.allow_insecure_http = config.allow_insecure_http; + options.connect_timeout_ms = config.delivery.connect_timeout_ms; + options.total_timeout_ms = config.delivery.total_timeout_ms; + validate_http(options); + } +} + +} + +RoutingConfig parse_routes(const std::string& text, bool allow_http, + const std::string& default_url, const std::string& default_secret_env) { + RoutingConfig config; + config.routed = true; + config.allow_insecure_http = allow_http; + config.document = parse_json(text); + config.file_identity = sha256_hex(text); + const auto& root = config.document; + only_fields(root, {"schema_version", "default_target", "targets", "rules", "delivery"}); + if (root.at("schema_version").integer() != 1) + throw std::runtime_error("unsupported webhook routes schema_version; expected 1"); + config.default_target = target_name(root.at("default_target")); + const auto& targets = root.at("targets"); + if (targets.kind != Json::Kind::Object || targets.members.size() > 128) + throw std::runtime_error("webhook targets must be an object with at most 128 targets"); + for (const auto& [name, value] : targets.members) { + target_name(Json::string(name)); + only_fields(value, {"url", "secret_env"}); + WebhookTarget target{value.at("url").text(), value.at("secret_env").text()}; + secret_name(target.secret_env); + config.targets.emplace(name, std::move(target)); + } + const auto require_target = [&](const std::optional& target) { + if (target && !config.targets.count(*target)) + throw std::runtime_error("undefined webhook target: " + *target); + }; + require_target(config.default_target); + const auto& rules = root.at("rules"); + if (rules.kind != Json::Kind::Array || rules.items.size() > 4096) + throw std::runtime_error("webhook rules must be an array with at most 4096 rules"); + for (const auto& value : rules.items) { + only_fields(value, {"match", "target"}); + const auto& match = value.at("match"); + if (match.find("alert_message") || + (match.find("kind") && match.at("kind").kind == Json::Kind::String && + match.at("kind").text() == "close")) + throw std::runtime_error("B2 webhook selectors require the missing strategy-metadata extension: Pine alert_message and distinct close provenance"); + only_fields(match, {"order_id", "kind", "side"}); + RoutingRule rule; + if (const auto* order = match.find("order_id")) rule.order_id = order->text(); + if (const auto* kind = match.find("kind")) { + rule.kind = kind->text(); + if (*rule.kind != "entry" && *rule.kind != "exit") + throw std::runtime_error("webhook kind must be entry or exit"); + } + if (const auto* side = match.find("side")) { + rule.side = side->text(); + if (*rule.side != "long" && *rule.side != "short") + throw std::runtime_error("webhook side must be long or short"); + } + rule.target = target_name(value.at("target")); + require_target(rule.target); + config.rules.push_back(std::move(rule)); + } + if (const auto* delivery = root.find("delivery")) { + only_fields(*delivery, {"max_in_flight", "connect_timeout_ms", "total_timeout_ms", "transport_retries", "retry_backoff_ms"}); + if (const auto* value = delivery->find("max_in_flight")) + config.delivery.max_in_flight = static_cast(bounded(*value, 1, 1024, "max_in_flight")); + if (const auto* value = delivery->find("connect_timeout_ms")) + config.delivery.connect_timeout_ms = bounded(*value, 1, 300000, "connect_timeout_ms"); + if (const auto* value = delivery->find("total_timeout_ms")) + config.delivery.total_timeout_ms = bounded(*value, 1, 300000, "total_timeout_ms"); + if (const auto* value = delivery->find("transport_retries")) + config.delivery.transport_retries = static_cast(bounded(*value, 0, 2, "transport_retries")); + if (const auto* value = delivery->find("retry_backoff_ms")) { + if (value->kind != Json::Kind::Array || value->items.size() > 2 || + value->items.size() < config.delivery.transport_retries) + throw std::runtime_error("retry_backoff_ms must provide a delay for each transport retry (at most 2)"); + config.delivery.retry_backoff_ms.clear(); + for (const auto& delay : value->items) + config.delivery.retry_backoff_ms.push_back(bounded(delay, 1, 300000, "retry_backoff_ms")); + } + } + if (config.delivery.connect_timeout_ms > config.delivery.total_timeout_ms) + throw std::runtime_error("connect_timeout_ms must not exceed total_timeout_ms"); + if (!default_url.empty() || !default_secret_env.empty()) { + if (!config.default_target) + throw std::runtime_error("webhook CLI flags conflict with journal-only default_target"); + const auto& target = config.targets.at(*config.default_target); + if ((!default_url.empty() && target.url != default_url) || + (!default_secret_env.empty() && target.secret_env != default_secret_env)) + throw std::runtime_error("webhook CLI flags must agree with the routes file's default target"); + } + validate_targets(config); + return config; +} + +RoutingConfig single_target(const std::string& url, const std::string& secret_env, bool allow_http) { + RoutingConfig config; + config.allow_insecure_http = allow_http; + if (!secret_env.empty()) secret_name(secret_env); + if (url.empty() && !secret_env.empty()) + throw std::runtime_error("webhook-secret-env requires a webhook target"); + if (!url.empty()) { + config.default_target = "default"; + config.targets.emplace("default", WebhookTarget{url, secret_env}); + } + validate_targets(config); + config.document = Json::object({{"url", Json::string(url)}, {"secret_env", Json::string(secret_env)}}); + return config; +} + +std::optional RoutingConfig::select(const std::string& order_id, + const std::string& kind, const std::string& side) const { + for (const auto& rule : rules) + if ((!rule.order_id || *rule.order_id == order_id) && + (!rule.kind || *rule.kind == kind) && (!rule.side || *rule.side == side)) + return rule.target; + return default_target; +} + +std::map RoutingConfig::load_secrets() const { + std::map result; + for (const auto& [name, target] : targets) { + HttpOptions options; + options.url = target.url; + options.allow_insecure_http = allow_insecure_http; + options.connect_timeout_ms = delivery.connect_timeout_ms; + options.total_timeout_ms = delivery.total_timeout_ms; + if (!target.secret_env.empty()) { + const char* secret = std::getenv(target.secret_env.c_str()); + if (!secret || !*secret) + throw std::runtime_error("webhook secret environment variable is missing or empty for target " + name); + options.hmac_secret = secret; + } + result.emplace(name, std::move(options)); + } + return result; +} + +std::string RoutingConfig::stored_document() const { + return Json::object({{"routed", Json::boolean(routed)}, + {"allow_insecure_http", Json::boolean(allow_insecure_http)}, + {"file_identity", Json::string(file_identity)}, {"configuration", document}}).dump(); +} + +RoutingConfig restore_routes(const std::string& document) { + const auto value = parse_json(document); + only_fields(value, {"routed", "allow_insecure_http", "file_identity", "configuration"}); + for (const auto* name : {"routed", "allow_insecure_http"}) + if (value.at(name).kind != Json::Kind::Bool) + throw std::runtime_error("invalid stored routing configuration"); + const bool insecure = value.at("allow_insecure_http").value == "true"; + const auto& configuration = value.at("configuration"); + auto config = value.at("routed").value == "true" + ? parse_routes(configuration.dump(), insecure) + : single_target(configuration.at("url").text(), configuration.at("secret_env").text(), insecure); + config.file_identity = value.at("file_identity").text(); + return config; +} + +std::string delivery_identity(const std::string& event_id, const std::optional& target) { + return sha256_hex(Json::object({{"event_id", Json::string(event_id)}, + {"target_id", target ? Json::string(*target) : Json{}}}).dump()); +} + +} diff --git a/runner/routing.hpp b/runner/routing.hpp new file mode 100644 index 000000000..f1ddf3f79 --- /dev/null +++ b/runner/routing.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include "json.hpp" +#include "transport.hpp" + +#include +#include +#include +#include + +namespace pineforge::live { + +struct DeliveryOptions { + std::size_t max_in_flight = 8; + long connect_timeout_ms = 2000; + long total_timeout_ms = 5000; + unsigned transport_retries = 2; + std::vector retry_backoff_ms{1000, 2000}; +}; + +struct WebhookTarget { + std::string url; + std::string secret_env; +}; + +struct RoutingRule { + std::optional order_id, kind, side, target; +}; + +struct RoutingConfig { + bool routed = false; + bool allow_insecure_http = false; + std::optional default_target; + std::map targets; + std::vector rules; + DeliveryOptions delivery; + Json document; + std::string file_identity; + + std::optional select(const std::string& order_id, + const std::string& kind, const std::string& side) const; + std::map load_secrets() const; + std::string stored_document() const; +}; + +RoutingConfig parse_routes(const std::string& text, bool allow_http, + const std::string& default_url = "", + const std::string& default_secret_env = ""); +RoutingConfig single_target(const std::string& url, const std::string& secret_env, bool allow_http); +RoutingConfig restore_routes(const std::string& document); +std::string delivery_identity(const std::string& event_id, + const std::optional& target); + +} diff --git a/runner/store.cpp b/runner/store.cpp index 67afb81ce..d4fafbc65 100644 --- a/runner/store.cpp +++ b/runner/store.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 #include "store.hpp" +#include "json.hpp" #include @@ -7,6 +8,7 @@ #include #include #include +#include #include #include @@ -48,6 +50,13 @@ class Statement { if (sqlite3_bind_int64(stmt_, col, static_cast(value)) != SQLITE_OK) database_error(); } + void bind_null(int col) { + if (sqlite3_bind_null(stmt_, col) != SQLITE_OK) database_error(); + } + bool is_null(int col) const { return sqlite3_column_type(stmt_, col) == SQLITE_NULL; } + std::uint64_t steps() const { + return static_cast(sqlite3_stmt_status(stmt_, SQLITE_STMTSTATUS_VM_STEP, 0)); + } bool row() { const int rc = sqlite3_step(stmt_); if (rc == SQLITE_ROW) return true; @@ -111,12 +120,24 @@ StoredEvent read_event(const Statement& q, int start = 0) { return event; } +StoredEvent read_routed_event(const Statement& query) { + auto event = read_event(query); + event.target_id = query.is_null(4) ? std::nullopt : std::optional(query.text(4)); + event.delivery_id = query.text(5); + return event; +} + +constexpr const char* unsent_predicate = + "e.acknowledged=0 AND r.target_id IS NOT NULL AND NOT EXISTS " + "(SELECT 1 FROM delivery_log d WHERE d.event_id=e.event_id AND d.phase='completed')"; + } // namespace struct Ledger::Impl { sqlite3* db = nullptr; int lock_fd = -1; int database_fd = -1; + mutable std::recursive_mutex mutex; ~Impl() { if (db) sqlite3_close_v2(db); if (database_fd >= 0) close(database_fd); @@ -140,8 +161,10 @@ Ledger::Ledger(const std::string& path, const std::string& deployment_identity) throw std::runtime_error("native ledger requires an on-disk path"); const std::string canonical = std::filesystem::weakly_canonical(path).string(); impl_->lock_fd = open((canonical + ".lock").c_str(), O_RDWR | O_CREAT | O_CLOEXEC | O_NOFOLLOW, 0600); - if (impl_->lock_fd < 0 || flock(impl_->lock_fd, LOCK_EX | LOCK_NB) != 0) - throw std::runtime_error("native ledger is locked or its lock file cannot be opened"); + if (impl_->lock_fd < 0) + throw std::runtime_error("native ledger lock file cannot be opened"); + if (flock(impl_->lock_fd, LOCK_EX | LOCK_NB) != 0) + throw std::runtime_error("the runner is running: stop it first; it resumes from its ledger"); impl_->database_fd = open(canonical.c_str(), O_RDWR | O_CREAT | O_CLOEXEC | O_NOFOLLOW, 0600); struct stat st {}; if (impl_->database_fd < 0 || fstat(impl_->database_fd, &st) != 0 || @@ -177,15 +200,29 @@ Ledger::Ledger(const std::string& path, const std::string& deployment_identity) Statement q(impl_->db, "INSERT INTO metadata VALUES(1,1,?)"); q.bind(1, deployment_identity); q.done(); - } else if (table_count != 3) { + } else if (table_count != 3 && table_count != 6) { throw std::runtime_error("native ledger is not an empty or supported ledger database"); } { Statement q(impl_->db, "SELECT schema_version,identity FROM metadata WHERE singleton=1"); if (!q.row() || q.integer(0) != 1 || q.text(1) != deployment_identity) - throw std::runtime_error("native ledger deployment identity or schema mismatch"); + throw std::runtime_error("native ledger deployment identity or schema mismatch (including webhook routing); restore the original configuration or use a new ledger"); if (q.row() || scalar(impl_->db, "SELECT COUNT(*) FROM metadata") != 1) database_error(); } + exec(impl_->db, + "CREATE TABLE IF NOT EXISTS routing_configuration (singleton INTEGER PRIMARY KEY CHECK(singleton=1),document TEXT NOT NULL);" + "CREATE TABLE IF NOT EXISTS event_routes (ordinal INTEGER PRIMARY KEY REFERENCES events(ordinal)," + "target_id TEXT,delivery_id TEXT NOT NULL);" + "CREATE TABLE IF NOT EXISTS delivery_log (log_id INTEGER PRIMARY KEY,event_id TEXT NOT NULL REFERENCES events(event_id)," + "target_id TEXT NOT NULL,delivery_id TEXT NOT NULL,attempt INTEGER NOT NULL CHECK(attempt>0)," + "phase TEXT NOT NULL CHECK(phase IN ('started','completed')),started_at INTEGER NOT NULL,ended_at INTEGER," + "http_status INTEGER NOT NULL DEFAULT 0,error_category TEXT NOT NULL DEFAULT '',success INTEGER NOT NULL DEFAULT 0 CHECK(success IN (0,1))," + "UNIQUE(event_id,attempt,phase));" + "CREATE INDEX IF NOT EXISTS delivery_results ON delivery_log(event_id,phase,log_id);" + "CREATE TRIGGER IF NOT EXISTS delivery_log_no_update BEFORE UPDATE ON delivery_log BEGIN SELECT RAISE(ABORT,'delivery log is append-only'); END;" + "CREATE TRIGGER IF NOT EXISTS delivery_log_no_delete BEFORE DELETE ON delivery_log BEGIN SELECT RAISE(ABORT,'delivery log is append-only'); END;" + "INSERT INTO event_routes(ordinal,target_id,delivery_id) SELECT ordinal,'default',event_id FROM events " + "WHERE ordinal NOT IN (SELECT ordinal FROM event_routes);"); // Refuse holes/corruption before any delivery can occur. All source rows // and immutable event bytes are also compared by the runner during replay. const auto count = scalar(impl_->db, "SELECT COUNT(*) FROM inputs"); @@ -206,10 +243,6 @@ Ledger::Ledger(const std::string& path, const std::string& deployment_identity) "OR (current.input_index=previous.input_index AND current.input_position<>previous.input_position+1) " "OR (current.input_index>previous.input_index AND current.input_position<>0)") != 0) throw std::runtime_error("native ledger event order differs from input order"); - if (scalar(impl_->db, - "SELECT COUNT(*) FROM events WHERE acknowledged=1 AND ordinal>" - "(SELECT MIN(ordinal) FROM events WHERE acknowledged=0)") != 0) - throw std::runtime_error("native ledger acknowledged events violate delivery order"); { Statement q(impl_->db, "PRAGMA foreign_key_check"); if (q.row()) database_error(); @@ -224,10 +257,12 @@ Ledger::Ledger(const std::string& path, const std::string& deployment_identity) Ledger::~Ledger() = default; std::uint64_t Ledger::input_count() const { + std::lock_guard lock(impl_->mutex); return scalar(impl_->db, "SELECT COALESCE(MAX(input_index)+1,0) FROM inputs"); } std::optional Ledger::input(std::uint64_t index) const { + std::lock_guard lock(impl_->mutex); Statement q(impl_->db, "SELECT canonical_json,state_hash FROM inputs WHERE input_index=?"); q.bind(1, index); if (!q.row()) return std::nullopt; @@ -235,15 +270,16 @@ std::optional Ledger::input(std::uint64_t index) const { result.index = index; result.canonical_json = q.text(0); result.state_hash = q.text(1); - Statement e(impl_->db, "SELECT ordinal,event_id,payload,attempts FROM events " - "WHERE input_index=? ORDER BY input_position"); + Statement e(impl_->db, "SELECT e.ordinal,e.event_id,e.payload,e.attempts,r.target_id,r.delivery_id FROM events e " + "JOIN event_routes r ON r.ordinal=e.ordinal WHERE input_index=? ORDER BY input_position"); e.bind(1, index); - while (e.row()) result.events.push_back(read_event(e)); + while (e.row()) result.events.push_back(read_routed_event(e)); return result; } void Ledger::commit_input(std::uint64_t index, const std::string& canonical_json, std::uint64_t state_hash, const std::vector& events) { + std::lock_guard lock(impl_->mutex); validate_bytes(canonical_json, "input"); for (const auto& e : events) { validate_bytes(e.id, "event id"); @@ -255,7 +291,9 @@ void Ledger::commit_input(std::uint64_t index, const std::string& canonical_json bool same = old->canonical_json == canonical_json && old->state_hash == hash && old->events.size() == events.size(); for (std::size_t i = 0; same && i < events.size(); ++i) - same = old->events[i].id == events[i].id && old->events[i].payload == events[i].payload; + same = old->events[i].id == events[i].id && old->events[i].payload == events[i].payload && + old->events[i].target_id == events[i].target_id && old->events[i].delivery_id == + (events[i].delivery_id.empty() ? events[i].id : events[i].delivery_id); if (!same) throw std::runtime_error("native ledger replay differs from committed input, state or events"); tx.commit(); return; @@ -271,11 +309,19 @@ void Ledger::commit_input(std::uint64_t index, const std::string& canonical_json "VALUES(?,?,?,?,?)"); q.bind(1, ++ordinal); q.bind(2, index); q.bind(3, static_cast(i)); q.bind(4, events[i].id); q.bind(5, events[i].payload); q.done(); + Statement route(impl_->db, "INSERT INTO event_routes(ordinal,target_id,delivery_id) VALUES(?,?,?)"); + route.bind(1, ordinal); + if (events[i].target_id) route.bind(2, *events[i].target_id); + else route.bind_null(2); + route.bind(3, events[i].delivery_id.empty() ? events[i].id : events[i].delivery_id); + route.done(); } tx.commit(); } +#ifdef PINEFORGE_LIVE_LEGACY_TEST_API std::optional Ledger::pending_event() const { + std::lock_guard lock(impl_->mutex); Statement q(impl_->db, "SELECT ordinal,event_id,payload,attempts FROM events " "WHERE acknowledged=0 ORDER BY ordinal LIMIT 1"); if (!q.row()) return std::nullopt; @@ -283,10 +329,12 @@ std::optional Ledger::pending_event() const { } std::uint64_t Ledger::pending_count() const { + std::lock_guard lock(impl_->mutex); return scalar(impl_->db, "SELECT COUNT(*) FROM events WHERE acknowledged=0"); } void Ledger::begin_delivery(const std::string& event_id) { + std::lock_guard lock(impl_->mutex); Transaction tx(impl_->db); const auto head = impl_->require_head(event_id); if (head.attempts == std::numeric_limits::max()) @@ -296,6 +344,7 @@ void Ledger::begin_delivery(const std::string& event_id) { } void Ledger::record_delivery_failure(const std::string& event_id, const std::string& error_category) { + std::lock_guard lock(impl_->mutex); Transaction tx(impl_->db); if (impl_->require_head(event_id).attempts == 0) throw std::runtime_error("native ledger delivery has not begun"); @@ -304,6 +353,7 @@ void Ledger::record_delivery_failure(const std::string& event_id, const std::str } void Ledger::acknowledge(const std::string& event_id) { + std::lock_guard lock(impl_->mutex); Transaction tx(impl_->db); if (impl_->require_head(event_id).attempts == 0) throw std::runtime_error("native ledger delivery has not begun"); @@ -311,4 +361,204 @@ void Ledger::acknowledge(const std::string& event_id) { q.bind(1, event_id); q.done(); tx.commit(); } +#endif + +void Ledger::bind_routing(const std::string& document) { + std::lock_guard lock(impl_->mutex); + validate_bytes(document, "routing configuration"); + Statement query(impl_->db, "INSERT INTO routing_configuration(singleton,document) VALUES(1,?) " + "ON CONFLICT(singleton) DO UPDATE SET document=excluded.document"); + query.bind(1, document); + query.done(); +} + +std::vector Ledger::unsent_events(std::uint64_t after, std::size_t limit) const { + std::lock_guard lock(impl_->mutex); + const auto sql = std::string("SELECT e.ordinal,e.event_id,e.payload,e.attempts,r.target_id,r.delivery_id " + "FROM events e JOIN event_routes r ON r.ordinal=e.ordinal WHERE e.ordinal>? AND ") + + unsent_predicate + " ORDER BY e.ordinal LIMIT ?"; + Statement query(impl_->db, sql.c_str()); + query.bind(1, after); query.bind(2, static_cast(limit)); + std::vector result; + while (query.row()) result.push_back(read_routed_event(query)); + return result; +} + +std::uint64_t Ledger::unsent_count() const { + std::lock_guard lock(impl_->mutex); + const auto sql = std::string("SELECT COUNT(*) FROM events e JOIN event_routes r ON r.ordinal=e.ordinal WHERE ") + unsent_predicate; + return scalar(impl_->db, sql.c_str()); +} + +std::optional Ledger::next_delivery_event(std::uint64_t after, + std::uint64_t* steps) const { + std::lock_guard lock(impl_->mutex); + Statement query(impl_->db, "SELECT e.ordinal,e.event_id,e.payload,e.attempts,r.target_id,r.delivery_id," + "e.acknowledged=0 AND r.target_id IS NOT NULL AND NOT EXISTS " + "(SELECT 1 FROM delivery_log d WHERE d.event_id=e.event_id AND d.phase='completed') " + "FROM events e JOIN event_routes r ON r.ordinal=e.ordinal WHERE e.ordinal>? ORDER BY e.ordinal LIMIT 1"); + query.bind(1, after); + const bool found = query.row(); + if (steps) *steps = query.steps(); + if (!found) return std::nullopt; + return DeliveryScan{read_routed_event(query), query.integer(6) != 0}; +} + +DeliveryAttempt Ledger::start_attempt(const StoredEvent& event, std::uint64_t started_at) { + std::lock_guard lock(impl_->mutex); + if (!event.target_id) throw std::runtime_error("journal-only action cannot be delivered"); + Transaction transaction(impl_->db); + Statement query(impl_->db, "SELECT e.ordinal,e.event_id,e.payload,e.attempts,r.target_id,r.delivery_id " + "FROM events e JOIN event_routes r ON r.ordinal=e.ordinal WHERE e.event_id=?"); + query.bind(1, event.id); + if (!query.row()) database_error(); + DeliveryAttempt result{read_routed_event(query), 0, started_at}; + if (result.event.target_id != event.target_id || result.event.delivery_id != event.delivery_id || + result.event.payload != event.payload || result.event.attempts == UINT32_MAX) + throw std::runtime_error("native ledger delivery identity mismatch or attempt counter exhausted"); + result.attempt = result.event.attempts + 1; + Statement update(impl_->db, "UPDATE events SET attempts=attempts+1 WHERE event_id=?"); + update.bind(1, event.id); update.done(); + Statement log(impl_->db, "INSERT INTO delivery_log(event_id,target_id,delivery_id,attempt,phase,started_at) " + "VALUES(?,?,?,?,'started',?)"); + log.bind(1, event.id); log.bind(2, *event.target_id); log.bind(3, event.delivery_id); + log.bind(4, result.attempt); log.bind(5, started_at); log.done(); + transaction.commit(); + return result; +} + +void Ledger::finish_attempt(const DeliveryAttempt& attempt, std::uint64_t ended_at, + long http_status, bool success, const std::string& error) { + std::lock_guard lock(impl_->mutex); + Transaction transaction(impl_->db); + const auto& event = attempt.event; + Statement log(impl_->db, "INSERT INTO delivery_log(event_id,target_id,delivery_id,attempt,phase,started_at,ended_at,http_status,error_category,success) " + "VALUES(?,?,?,?,'completed',?,?,?,?,?)"); + log.bind(1, event.id); log.bind(2, *event.target_id); log.bind(3, event.delivery_id); + log.bind(4, attempt.attempt); log.bind(5, attempt.started_at); log.bind(6, ended_at); + log.bind(7, static_cast(http_status)); + log.bind(8, success ? "" : safe_category(error)); log.bind(9, static_cast(success)); log.done(); + Statement update(impl_->db, "UPDATE events SET acknowledged=?,last_error=? WHERE event_id=?"); + update.bind(1, static_cast(success)); + update.bind(2, success ? "" : safe_category(error)); update.bind(3, event.id); update.done(); + transaction.commit(); +} + +struct LedgerView::Impl { + sqlite3* db = nullptr; + bool routed = false; + ~Impl() { if (db) sqlite3_close_v2(db); } +}; + +LedgerView::LedgerView(const std::string& path) : impl_(std::make_unique()) { + if (path.empty() || path == ":memory:" || path.find('\0') != std::string::npos || + sqlite3_open_v2(path.c_str(), &impl_->db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nullptr) != SQLITE_OK) + throw std::runtime_error("cannot open native ledger for reading"); + sqlite3_busy_timeout(impl_->db, 5000); + Statement metadata(impl_->db, "SELECT schema_version FROM metadata WHERE singleton=1"); + if (!metadata.row() || metadata.integer(0) != 1) database_error(); + impl_->routed = scalar(impl_->db, "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='event_routes'") == 1; +} + +LedgerView::~LedgerView() = default; + +std::string LedgerView::identity() const { + Statement query(impl_->db, "SELECT identity FROM metadata WHERE singleton=1"); + if (!query.row()) database_error(); + return query.text(0); +} + +std::string LedgerView::routing_document() const { + if (!impl_->routed) return ""; + Statement query(impl_->db, "SELECT document FROM routing_configuration WHERE singleton=1"); + return query.row() ? query.text(0) : ""; +} + +std::vector LedgerView::actions_after(std::uint64_t after, std::size_t limit) const { + Statement query(impl_->db, impl_->routed + ? "SELECT e.ordinal,e.event_id,e.payload,e.attempts,r.target_id,r.delivery_id FROM events e " + "JOIN event_routes r ON r.ordinal=e.ordinal WHERE e.ordinal>? ORDER BY e.ordinal LIMIT ?" + : "SELECT ordinal,event_id,payload,attempts,'default',event_id FROM events WHERE ordinal>? ORDER BY ordinal LIMIT ?"); + query.bind(1, after); query.bind(2, static_cast(limit)); + std::vector result; + while (query.row()) result.push_back(read_routed_event(query)); + return result; +} + +std::vector LedgerView::redelivery_events(const std::string& target, + std::uint64_t from, bool failed_only) const { + if (!impl_->routed) throw std::runtime_error("resume this phase-A ledger with run before redelivering"); + std::string sql = "SELECT e.ordinal,e.event_id,e.payload,e.attempts,r.target_id,r.delivery_id FROM events e " + "JOIN event_routes r ON r.ordinal=e.ordinal WHERE r.target_id=? AND e.ordinal>=?"; + if (failed_only) + sql += " AND (SELECT success FROM delivery_log d WHERE d.event_id=e.event_id AND d.phase='completed' " + "ORDER BY d.log_id DESC LIMIT 1)=0"; + sql += " ORDER BY e.ordinal"; + Statement query(impl_->db, sql.c_str()); + query.bind(1, target); query.bind(2, from); + std::vector result; + while (query.row()) result.push_back(read_routed_event(query)); + return result; +} + +std::string LedgerView::status_json() const { + exec(impl_->db, "BEGIN"); + struct ReadEnd { + sqlite3* database; + ~ReadEnd() { sqlite3_exec(database, "ROLLBACK", nullptr, nullptr, nullptr); } + } read_end{impl_->db}; + Json targets = Json::object({}); + const auto initialize = [&](const std::string& name) -> Json& { + auto [position, inserted] = targets.members.emplace(name, Json{}); + if (inserted) position->second = Json::object({{"sent", Json::number("0")}, {"failed", Json::number("0")}, + {"unsent", Json::number("0")}, {"last_success", Json{}}, {"last_error", Json{}}, {"last_attempt", Json{}}}); + return position->second; + }; + if (const auto document = routing_document(); !document.empty()) { + const auto stored = parse_json(document); + const auto& config = stored.at("configuration"); + if (stored.at("routed").value == "true") + for (const auto& [name, value] : config.at("targets").members) { (void)value; initialize(name); } + else if (!config.at("url").text().empty()) initialize("default"); + } + if (impl_->routed) { + Statement log(impl_->db, "SELECT target_id,SUM(phase='completed' AND success=1)," + "SUM(phase='completed' AND success=0),MAX(started_at)," + "MAX(CASE WHEN phase='completed' AND success=1 THEN ended_at END)," + "MAX(CASE WHEN phase='completed' AND success=0 THEN log_id END) FROM delivery_log GROUP BY target_id"); + while (log.row()) { + auto& status = initialize(log.text(0)); + status.members["sent"] = Json::number(std::to_string(log.integer(1))); + status.members["failed"] = Json::number(std::to_string(log.integer(2))); + status.members["last_attempt"] = Json::number(std::to_string(log.integer(3))); + if (!log.is_null(4)) status.members["last_success"] = Json::number(std::to_string(log.integer(4))); + if (!log.is_null(5)) { + Statement error(impl_->db, "SELECT error_category,http_status,ended_at FROM delivery_log WHERE log_id=?"); + error.bind(1, log.integer(5)); + if (!error.row()) database_error(); + status.members["last_error"] = Json::object({{"category", Json::string(error.text(0))}, + {"http_status", Json::number(std::to_string(error.integer(1)))}, + {"at", Json::number(std::to_string(error.integer(2)))}}); + } + } + } + Statement legacy(impl_->db, impl_->routed + ? "SELECT r.target_id,COUNT(*) FROM events e JOIN event_routes r ON r.ordinal=e.ordinal WHERE e.acknowledged=1 " + "AND NOT EXISTS(SELECT 1 FROM delivery_log d WHERE d.event_id=e.event_id) GROUP BY r.target_id" + : "SELECT 'default',COUNT(*) FROM events WHERE acknowledged=1 HAVING COUNT(*)>0"); + while (legacy.row()) { + if (legacy.is_null(0)) continue; + auto& count = initialize(legacy.text(0)).members["sent"]; + count = Json::number(std::to_string(count.integer() + legacy.integer(1))); + } + Statement unsent(impl_->db, impl_->routed + ? "SELECT r.target_id,COUNT(*) FROM events e JOIN event_routes r ON r.ordinal=e.ordinal WHERE " + "e.acknowledged=0 AND r.target_id IS NOT NULL AND NOT EXISTS " + "(SELECT 1 FROM delivery_log d WHERE d.event_id=e.event_id AND d.phase='completed') GROUP BY r.target_id" + : "SELECT 'default',COUNT(*) FROM events WHERE acknowledged=0 HAVING COUNT(*)>0"); + while (unsent.row()) initialize(unsent.text(0)).members["unsent"] = Json::number(std::to_string(unsent.integer(1))); + return Json::object({{"schema_version", Json::number("1")}, {"targets", std::move(targets)}, + {"actions", Json::number(std::to_string(scalar(impl_->db, "SELECT COUNT(*) FROM events")))}}).dump(); +} + } // namespace pineforge::live diff --git a/runner/store.hpp b/runner/store.hpp index f5d925193..139b8a964 100644 --- a/runner/store.hpp +++ b/runner/store.hpp @@ -12,6 +12,8 @@ namespace pineforge::live { struct Event { std::string id; std::string payload; + std::optional target_id = "default"; + std::string delivery_id; }; struct StoredEvent : Event { @@ -26,6 +28,17 @@ struct RecordedInput { std::vector events; }; +struct DeliveryAttempt { + StoredEvent event; + std::uint32_t attempt = 0; + std::uint64_t started_at = 0; +}; + +struct DeliveryScan { + StoredEvent event; + bool unsent = false; +}; + // One process owns a ledger for its entire lifetime. All exceptions are fatal // to an advanced in-memory strategy: recreate it and replay durable inputs. // Payloads are opaque bytes and are never reserialized or changed on delivery. @@ -45,6 +58,7 @@ class Ledger { void commit_input(std::uint64_t index, const std::string& canonical_json, std::uint64_t state_hash, const std::vector& events); +#ifdef PINEFORGE_LIVE_LEGACY_TEST_API std::optional pending_event() const; std::uint64_t pending_count() const; // Persist the attempt before sending. A crash leaves an unacknowledged @@ -53,7 +67,33 @@ class Ledger { void record_delivery_failure(const std::string& event_id, const std::string& error_category); void acknowledge(const std::string& event_id); +#endif + void bind_routing(const std::string& document); + std::vector unsent_events(std::uint64_t after, std::size_t limit = 256) const; + std::optional next_delivery_event(std::uint64_t after, + std::uint64_t* steps = nullptr) const; + DeliveryAttempt start_attempt(const StoredEvent& event, std::uint64_t started_at); + void finish_attempt(const DeliveryAttempt& attempt, std::uint64_t ended_at, + long http_status, bool success, const std::string& error); + std::uint64_t unsent_count() const; + +private: + struct Impl; + std::unique_ptr impl_; +}; +class LedgerView { +public: + explicit LedgerView(const std::string& path); + ~LedgerView(); + LedgerView(const LedgerView&) = delete; + LedgerView& operator=(const LedgerView&) = delete; + std::string identity() const; + std::string routing_document() const; + std::vector actions_after(std::uint64_t after, std::size_t limit = 256) const; + std::vector redelivery_events(const std::string& target, + std::uint64_t from, bool failed_only) const; + std::string status_json() const; private: struct Impl; std::unique_ptr impl_; diff --git a/runner/transport.cpp b/runner/transport.cpp index 61e2b54ec..c4ea3998c 100644 --- a/runner/transport.cpp +++ b/runner/transport.cpp @@ -9,7 +9,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -21,6 +23,7 @@ namespace { constexpr std::size_t max_feed_bytes = 4 * 1024 * 1024; constexpr std::size_t max_event_bytes = 1024 * 1024; +std::string http_proxy, https_proxy, all_proxy, no_proxy; std::string hex(const unsigned char* data, std::size_t length) { constexpr char digits[] = "0123456789abcdef"; @@ -40,6 +43,11 @@ struct CurlGlobal { ~CurlGlobal() { curl_global_cleanup(); } }; +void initialize_curl() { + static CurlGlobal global; + (void)global; +} + using CurlHandle = std::unique_ptr; using UrlHandle = std::unique_ptr; @@ -90,19 +98,16 @@ void check_url(const HttpOptions& options, bool websocket) { } CurlHandle make_handle(const HttpOptions& options, bool websocket = false) { - static CurlGlobal global; - (void)global; - check_url(options, websocket); - if (websocket) { - const auto* info = curl_version_info(CURLVERSION_NOW); - bool found = false; - for (const char* const* p = info ? info->protocols : nullptr; p && *p; ++p) - if (std::string_view(*p) == (options.url.rfind("wss:", 0) == 0 ? "wss" : "ws")) found = true; - if (!found) throw std::runtime_error("native WebSocket requires a libcurl build with WS/WSS support enabled"); - } + initialize_curl(); + if (websocket) validate_websocket(options); + else check_url(options, false); CurlHandle curl(curl_easy_init(), &curl_easy_cleanup); if (!curl) throw std::runtime_error("native HTTP handle allocation failed"); option(curl.get(), CURLOPT_URL, options.url.c_str()); + const auto& proxy = (options.url.rfind("https:", 0) == 0 || options.url.rfind("wss:", 0) == 0) + ? https_proxy : http_proxy; + option(curl.get(), CURLOPT_PROXY, (proxy.empty() ? all_proxy : proxy).c_str()); + option(curl.get(), CURLOPT_NOPROXY, no_proxy.c_str()); option(curl.get(), CURLOPT_CONNECTTIMEOUT_MS, options.connect_timeout_ms); option(curl.get(), CURLOPT_TIMEOUT_MS, options.total_timeout_ms); option(curl.get(), CURLOPT_NOSIGNAL, 1L); @@ -148,13 +153,14 @@ std::size_t receive(char* data, std::size_t size, std::size_t nmemb, void* userd return count; } -DeliveryResult perform(CURL* curl, Response& response) { - option(curl, CURLOPT_WRITEFUNCTION, &receive); - option(curl, CURLOPT_WRITEDATA, &response); - const auto code = curl_easy_perform(curl); +DeliveryResult response_result(CURL* curl, const Response& response, CURLcode code) { DeliveryResult result; if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &result.status) != CURLE_OK) throw std::runtime_error("native HTTP response status unavailable"); + if (result.status >= 300) { + result.error = "http_status"; + return result; + } if (code != CURLE_OK) { if (response.too_large) result.error = "response_too_large"; else if (response.allocation_failed) result.error = "response_allocation_failed"; @@ -163,6 +169,10 @@ DeliveryResult perform(CURL* curl, Response& response) { code == CURLE_SSL_CERTPROBLEM || code == CURLE_SSL_CACERT_BADFILE) result.error = "tls_failure"; else result.error = "network_error"; + result.retryable = result.status == 0 && !response.too_large && !response.allocation_failed && + (code == CURLE_COULDNT_CONNECT || code == CURLE_COULDNT_RESOLVE_HOST || + code == CURLE_COULDNT_RESOLVE_PROXY || code == CURLE_OPERATION_TIMEDOUT || + code == CURLE_SEND_ERROR || code == CURLE_RECV_ERROR || code == CURLE_GOT_NOTHING); return result; } result.success = result.status >= 200 && result.status <= 299; @@ -170,6 +180,12 @@ DeliveryResult perform(CURL* curl, Response& response) { return result; } +DeliveryResult perform(CURL* curl, Response& response) { + option(curl, CURLOPT_WRITEFUNCTION, &receive); + option(curl, CURLOPT_WRITEDATA, &response); + return response_result(curl, response, curl_easy_perform(curl)); +} + bool valid_utf8(std::string_view bytes) { std::uint32_t value = 0, minimum = 0; unsigned remaining = 0; @@ -257,6 +273,7 @@ std::string hmac_sha256_hex(std::string_view secret, std::string_view bytes) { return hex(digest.data(), length); } +#ifdef PINEFORGE_LIVE_LEGACY_TEST_API DeliveryResult post_webhook(const HttpOptions& options, const StoredEvent& event) { if (event.id.empty() || event.id.size() > 256 || event.id.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.:") @@ -266,7 +283,7 @@ DeliveryResult post_webhook(const HttpOptions& options, const StoredEvent& event auto curl = make_handle(options); Headers headers; headers.add("Content-Type: application/json"); - headers.add("Idempotency-Key: " + event.id); + headers.add("Idempotency-Key: " + (event.delivery_id.empty() ? event.id : event.delivery_id)); headers.add("X-PineForge-Event-Id: " + event.id); headers.add("Expect:"); if (!options.hmac_secret.empty()) @@ -278,6 +295,7 @@ DeliveryResult post_webhook(const HttpOptions& options, const StoredEvent& event Response response; return perform(curl.get(), response); } +#endif std::string get_feed_snapshot(const HttpOptions& options) { auto curl = make_handle(options); @@ -292,15 +310,129 @@ std::string get_feed_snapshot(const HttpOptions& options) { return std::move(response.body); } +void validate_http(const HttpOptions& options) { + initialize_curl(); + check_url(options, false); +} + +struct WebhookMulti::Impl { + struct Request { + StoredEvent event; + std::uint64_t key; + Headers headers; + Response response; + CurlHandle curl; + + Request(std::uint64_t request_key, const HttpOptions& options, const StoredEvent& stored) + : event(stored), key(request_key), curl(make_handle(options)) { + if (event.id.empty() || event.delivery_id.empty() || event.id.size() > 256 || + event.delivery_id.size() > 256 || + event.id.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.:") != std::string::npos || + event.delivery_id.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.:") != std::string::npos || + event.payload.empty() || event.payload.size() > max_event_bytes) + throw std::runtime_error("native HTTP event id or payload is invalid"); + headers.add("Content-Type: application/json"); + headers.add("Idempotency-Key: " + event.delivery_id); + headers.add("X-PineForge-Event-Id: " + event.id); + headers.add("Expect:"); + if (!options.hmac_secret.empty()) + headers.add("X-PineForge-Signature: sha256=" + hmac_sha256_hex(options.hmac_secret, event.payload)); + option(curl.get(), CURLOPT_HTTPHEADER, headers.value); + option(curl.get(), CURLOPT_POST, 1L); + option(curl.get(), CURLOPT_POSTFIELDS, event.payload.data()); + option(curl.get(), CURLOPT_POSTFIELDSIZE_LARGE, static_cast(event.payload.size())); + option(curl.get(), CURLOPT_WRITEFUNCTION, &receive); + option(curl.get(), CURLOPT_WRITEDATA, &response); + } + }; + CURLM* multi = nullptr; + std::map> requests; + ~Impl() { + for (const auto& [handle, request] : requests) { + (void)request; + curl_multi_remove_handle(multi, handle); + } + requests.clear(); + if (multi) curl_multi_cleanup(multi); + } +}; + +WebhookMulti::WebhookMulti() : impl_(std::make_unique()) { + initialize_curl(); + impl_->multi = curl_multi_init(); + if (!impl_->multi) throw std::runtime_error("native HTTP multi allocation failed"); +} + +WebhookMulti::~WebhookMulti() = default; + +void WebhookMulti::add(std::uint64_t key, const HttpOptions& options, const StoredEvent& event) { + auto request = std::make_unique(key, options, event); + auto* handle = request->curl.get(); + impl_->requests.emplace(handle, std::move(request)); + if (curl_multi_add_handle(impl_->multi, handle) != CURLM_OK) { + impl_->requests.erase(handle); + throw std::runtime_error("native HTTP multi add failed"); + } + int running = 0; + if (curl_multi_perform(impl_->multi, &running) != CURLM_OK) + throw std::runtime_error("native HTTP multi perform failed"); +} + +std::vector WebhookMulti::poll(int timeout_ms) { + int running = 0; + if (curl_multi_perform(impl_->multi, &running) != CURLM_OK || + curl_multi_poll(impl_->multi, nullptr, 0, timeout_ms, nullptr) != CURLM_OK || + curl_multi_perform(impl_->multi, &running) != CURLM_OK) + throw std::runtime_error("native HTTP multi polling failed"); + std::vector completed; + int remaining = 0; + while (auto* message = curl_multi_info_read(impl_->multi, &remaining)) { + if (message->msg != CURLMSG_DONE) continue; + const auto found = impl_->requests.find(message->easy_handle); + if (found == impl_->requests.end()) throw std::runtime_error("native HTTP unknown completed request"); + const auto& request = *found->second; + completed.push_back({request.key, response_result(message->easy_handle, request.response, message->data.result)}); + if (curl_multi_remove_handle(impl_->multi, message->easy_handle) != CURLM_OK) + throw std::runtime_error("native HTTP multi remove failed"); + impl_->requests.erase(found); + } + return completed; +} + +void capture_proxy_environment() { + const auto read = [](const char* lower, const char* upper) { + const char* value = std::getenv(lower); + if (!value && upper) value = std::getenv(upper); + return std::string(value ? value : ""); + }; + http_proxy = read("http_proxy", nullptr); + https_proxy = read("https_proxy", "HTTPS_PROXY"); + all_proxy = read("all_proxy", "ALL_PROXY"); + no_proxy = read("no_proxy", "NO_PROXY"); +} + +void validate_websocket(const HttpOptions& options) { + initialize_curl(); + check_url(options, true); + const auto* info = curl_version_info(CURLVERSION_NOW); + if (!info || info->version_num < 0x080e01) + throw std::runtime_error("native WebSocket requires libcurl 8.14.1 or newer for complete-message finality (loaded " + + std::string(info && info->version ? info->version : "unknown") + ")"); + const std::string_view protocol = options.url.rfind("wss:", 0) == 0 ? "wss" : "ws"; + for (const char* const* entry = info->protocols; entry && *entry; ++entry) + if (std::string_view(*entry) == protocol) return; + throw std::runtime_error("native WebSocket requires a libcurl build with WS/WSS support enabled"); +} + void receive_websocket(const HttpOptions& options, std::string_view subscription, const std::function& on_message, const std::function& stopped) { if (!on_message || !stopped || subscription.size() > max_event_bytes || !valid_utf8(subscription)) throw std::runtime_error("native WebSocket invalid callbacks or subscription"); if (stopped()) return; -#if LIBCURL_VERSION_NUM < 0x075600 +#if LIBCURL_VERSION_NUM < 0x080e01 (void)options; - throw std::runtime_error("native WebSocket requires libcurl 7.86 or newer with WS/WSS support enabled"); + throw std::runtime_error("native WebSocket requires libcurl 8.14.1 or newer with WS/WSS support enabled"); #else auto curl = make_handle(options, true); option(curl.get(), CURLOPT_CONNECT_ONLY, 2L); @@ -316,6 +448,7 @@ void receive_websocket(const HttpOptions& options, std::string_view subscription bool assembling = false; std::string message; std::uint64_t frame_offset = 0; + int frame_flags = 0; for (;;) { if (stopped()) return; const auto deadline = assembling ? std::min(idle_deadline, message_deadline) : idle_deadline; @@ -342,8 +475,10 @@ void receive_websocket(const HttpOptions& options, std::string_view subscription } if ((meta->flags & CURLWS_BINARY) || !(meta->flags & CURLWS_TEXT) || meta->offset < 0 || meta->bytesleft < 0 || - static_cast(meta->offset) != frame_offset || received > buffer.size()) + static_cast(meta->offset) != frame_offset || received > buffer.size() || + meta->len != received || (frame_offset && meta->flags != frame_flags)) throw std::runtime_error("native WebSocket requires ordered text frames"); + frame_flags = meta->flags; if (!assembling) { message_deadline = Clock::now() + timeout; assembling = true; diff --git a/runner/transport.hpp b/runner/transport.hpp index b7590eff2..7fbb738e7 100644 --- a/runner/transport.hpp +++ b/runner/transport.hpp @@ -3,8 +3,10 @@ #include #include +#include #include #include +#include #include "store.hpp" @@ -21,18 +23,42 @@ struct HttpOptions { struct DeliveryResult { long status = 0; bool success = false; + bool retryable = false; // A fixed category, never a URL, response body, header or libcurl error buffer. std::string error; }; std::string sha256_hex(std::string_view bytes); std::string hmac_sha256_hex(std::string_view secret, std::string_view bytes); +void validate_http(const HttpOptions& options); +void capture_proxy_environment(); + +struct CompletedWebhook { + std::uint64_t key = 0; + DeliveryResult result; +}; + +class WebhookMulti { +public: + WebhookMulti(); + ~WebhookMulti(); + WebhookMulti(const WebhookMulti&) = delete; + WebhookMulti& operator=(const WebhookMulti&) = delete; + void add(std::uint64_t key, const HttpOptions& options, const StoredEvent& event); + std::vector poll(int timeout_ms); +private: + struct Impl; + std::unique_ptr impl_; +}; // Blocking, bounded native libcurl delivery. Only HTTP(S), no redirects, // verified TLS, JSON body, stable idempotency and optional HMAC headers. +#ifdef PINEFORGE_LIVE_LEGACY_TEST_API DeliveryResult post_webhook(const HttpOptions& options, const StoredEvent& event); +#endif // Fetch a finite provider-neutral JSONL snapshot. Maximum response 4 MiB; // requires HTTP 2xx and never attaches webhook HMAC/idempotency headers. std::string get_feed_snapshot(const HttpOptions& options); +void validate_websocket(const HttpOptions& options); // Native WS/WSS intake. Complete UTF-8 text messages only, at most 1 MiB. // total_timeout_ms bounds idle time and assembly of each message. A closure // or transport failure throws; reconnection/continuity is never inferred. diff --git a/scripts/build_live_curl.sh b/scripts/build_live_curl.sh new file mode 100644 index 000000000..22c869433 --- /dev/null +++ b/scripts/build_live_curl.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly CURL_VERSION="8.14.1" +readonly CURL_SHA256="f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd" + +if [[ "${1:-}" == "--metadata" ]]; then + printf 'version=%s\nsha256=%s\n' "$CURL_VERSION" "$CURL_SHA256" + exit 0 +fi + +dependency_dir="${1:-build-native-deps}" +build_jobs="${2:-4}" +mkdir -p "$dependency_dir" +cd "$dependency_dir" +cmake_file="curl-install/lib/cmake/CURL/CURLConfig.cmake" +if [[ "${CURL_CACHE_HIT:-false}" == "true" && -f "$cmake_file" ]]; then + echo "Using cached libcurl at ${cmake_file}" + exit 0 +fi + +curl --fail --location --proto '=https' --tlsv1.2 \ + "https://curl.se/download/curl-${CURL_VERSION}.tar.xz" -o curl.tar.xz +echo "${CURL_SHA256} curl.tar.xz" | sha256sum -c - +tar -xf curl.tar.xz +cmake -S "curl-${CURL_VERSION}" -B curl-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PWD/curl-install" \ + -DBUILD_CURL_EXE=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON \ + -DBUILD_TESTING=OFF -DCURL_USE_OPENSSL=ON -DENABLE_WEBSOCKETS=ON \ + -DHTTP_ONLY=ON -DCURL_USE_LIBPSL=OFF -DUSE_LIBIDN2=OFF \ + -DCURL_USE_LIBSSH2=OFF -DCURL_BROTLI=OFF -DCURL_ZSTD=OFF \ + 2>&1 | tee curl-configure.log +cmake --build curl-build -j "$build_jobs" 2>&1 | tee curl-build.log +cmake --install curl-build 2>&1 | tee curl-install.log diff --git a/scripts/ci_verify.py b/scripts/ci_verify.py index 3c2fb64ee..06be6b15f 100644 --- a/scripts/ci_verify.py +++ b/scripts/ci_verify.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Shared local/CI verification driver. Stdlib only. Not a command generator. -Profiles: release, debug, sanitizers, native, kernel. Default build dir build-ci-PROFILE. +Profiles: release, debug, sanitizers, native, live-sanitizers, live-tsan, kernel. Default dir build-ci-PROFILE. Source guards, explicit configure, full rebuild, pinned e60/0e/v13/v14/v15/v16 ABI prepare/reuse, CTest, install+find_package+VERSION smoke, native help / required WebSocket. Fail fast on configure/build. After a successful build collect independent @@ -33,7 +33,8 @@ ) ROOT = Path(__file__).resolve().parents[1] -PROFILES = ('release', 'debug', 'sanitizers', 'native', 'kernel') +PROFILES = ('release', 'debug', 'sanitizers', 'native', 'live-sanitizers', 'live-tsan', 'kernel') +LIVE_ONLY_PROFILES = frozenset(('live-sanitizers', 'live-tsan')) DEFAULT_JOBS = 4 JOBS_MIN, JOBS_MAX = 1, 64 SCHEMA = 'pineforge-ci-verify/v1' @@ -315,9 +316,9 @@ # labelled by its stamp: OANDA's 17:00 ET break stamps) # +1 test_symbol_calendar (TradingView's symbol calendar as a run input: the # tape converter, the loader, the harness's syminfo-metadata transport) -# Both register in release too. 301 registered, 300 run: the WebSocket row +# Both register in release too. 300 registered, 299 run after parser removal: the WebSocket row # still skips on a system libcurl. -KERNEL_MIN_TESTS = 300 +KERNEL_MIN_TESTS = 299 # Release-row floor, the same gate for the default profile. Before lane P7 # only the kernel profile had one, so a row that left release alone (a # source-bound TU dropped from TEST_SOURCES, a deleted twin or ABI row) left a @@ -643,6 +644,7 @@ # minutes. CTEST_TIMEOUT = 1800 SANITIZERS_FULL_CTEST_TIMEOUT = 3600 +LIVE_SANITIZERS_MIN_TESTS = 12 def ctest_timeout(cfg: 'VerifyConfig') -> int: @@ -720,6 +722,7 @@ class Profile: source_layer: bool # Minimum CTest rows the profile must run; None leaves the count ungated. min_tests: int | None = None + thread_sanitizers: bool = False PROFILE = { @@ -727,10 +730,25 @@ class Profile: 'debug': Profile('debug', 'Debug', False, False, True, True), 'sanitizers': Profile('sanitizers', 'Debug', True, False, True, True), 'native': Profile('native', 'Release', False, True, False, True), + 'live-sanitizers': Profile('live-sanitizers', 'Debug', True, True, False, True, + LIVE_SANITIZERS_MIN_TESTS), + 'live-tsan': Profile('live-tsan', 'Debug', False, True, False, True, + LIVE_SANITIZERS_MIN_TESTS, True), 'kernel': Profile('kernel', 'Release', False, True, False, False, KERNEL_MIN_TESTS), } +def profile_min_tests(profile: Profile, source: Path) -> int | None: + if profile.name in LIVE_ONLY_PROFILES and not (source / 'runner' / 'transport.cpp').is_file(): + return LIVE_SANITIZERS_MIN_TESTS - 1 + return profile.min_tests + + +def runner_e2e_test_names(source: Path) -> set[str]: + return {path.stem for path in (source / 'tests').glob('native_live*_e2e.py') + if path.stem != 'native_live_equivalence_e2e'} + + @dataclass class Completed: returncode: int @@ -913,6 +931,8 @@ def cmake_cache_definitions(cfg: VerifyConfig) -> dict[str, str]: 'PINEFORGE_REQUIRE_ABI_RECEIPTS': 'ON', 'PINEFORGE_VERSION_SOURCE': 'FILE', } + if profile.live_runner: + values['PINEFORGE_LIVE_TSAN'] = 'ON' if profile.thread_sanitizers else 'OFF' if cfg.curl_dir is not None: values['CURL_DIR'] = str(cfg.curl_dir) if cfg.ccache_path: @@ -938,6 +958,7 @@ def parse_args(argv: list[str] | None, *, source: Path = ROOT) -> argparse.Names 'profile', choices=PROFILES, help='release/debug keep tutorial ON and native OFF; ' 'sanitizers enable PUBLIC ASan/UBSan; native enables the live runner; ' + 'live-sanitizers/live-tsan instrument every runner target and run all runner tests; ' 'kernel is native with the Pine source layer built OFF') parser.add_argument('--build-dir', type=Path, default=None, help='default: /build-ci-PROFILE') @@ -948,7 +969,8 @@ def parse_args(argv: list[str] | None, *, source: Path = ROOT) -> argparse.Names parser.add_argument('--ccache', action='store_true', help='require installed ccache and bind CMAKE_*_COMPILER_LAUNCHER') parser.add_argument('--require-websocket', action='store_true', - help='native only: execute test_native_live_websocket and refuse skip (77)') + help='native/live-sanitizers/live-tsan: execute test_native_live_websocket and ' + 'refuse skip (77); automatic in runner sanitizer profiles while transport.cpp exists') parser.add_argument('--exclude-label', default=None, help='exclude one CTest label; verify the run count against ' 'CTest discovery with and without -LE') @@ -956,7 +978,10 @@ def parse_args(argv: list[str] | None, *, source: Path = ROOT) -> argparse.Names help='fail the ctest-floor stage unless at least N CTest rows ran ' '(a skipped or not-run row is listed, never counted); ' f'the kernel profile defaults to {KERNEL_MIN_TESTS}, the release ' - f'profile to {RELEASE_MIN_TESTS}, the others to no floor') + f'profile to {RELEASE_MIN_TESTS}, runner sanitizer profiles to ' + f'{LIVE_SANITIZERS_MIN_TESTS} with transport.cpp or ' + f'{LIVE_SANITIZERS_MIN_TESTS - 1} without it, ' + 'the others to no floor') args = parser.parse_args(argv) if args.build_dir is None: args.build_dir = default_build_dir(source, args.profile) @@ -967,8 +992,11 @@ def validate_config(args: argparse.Namespace, *, source: Path = ROOT, which: Callable[[str], str | None] = shutil.which) -> VerifyConfig: if not JOBS_MIN <= args.jobs <= JOBS_MAX: raise ConfigError(f'--jobs must be {JOBS_MIN}..{JOBS_MAX}') - if args.require_websocket and args.profile != 'native': - raise ConfigError('--require-websocket is only valid with the native profile') + if args.require_websocket and args.profile not in {'native', 'live-sanitizers', 'live-tsan'}: + raise ConfigError('--require-websocket is only valid with the native or ' + 'live-sanitizers or live-tsan profile') + if args.profile in LIVE_ONLY_PROFILES and args.exclude_label is not None: + raise ConfigError(f'{args.profile} must run every runner row; --exclude-label is invalid') if args.exclude_label is not None: label = args.exclude_label.strip() if not label or any(not (char.isalnum() or char in '_.-') for char in label): @@ -1000,10 +1028,13 @@ def validate_config(args: argparse.Namespace, *, source: Path = ROOT, generator=args.generator, curl_dir=args.curl_dir.resolve() if args.curl_dir is not None else None, ccache_path=ccache_path, - require_websocket=bool(args.require_websocket), + require_websocket=bool(args.require_websocket or + (args.profile in LIVE_ONLY_PROFILES and + (source / 'runner' / 'transport.cpp').is_file())), runner=default_runner, exclude_label=args.exclude_label, - min_tests=args.min_tests if args.min_tests is not None else PROFILE[args.profile].min_tests, + min_tests=(args.min_tests if args.min_tests is not None else + profile_min_tests(PROFILE[args.profile], source)), ) @@ -1091,6 +1122,49 @@ def compile_target(entry: dict, argv: list[str]) -> str | None: return match.group(1) if match else None +def runner_translation_units(source: Path) -> list[Path]: + return sorted(unit for unit in (source / 'runner').rglob('*') + if unit.suffix.lower() in {'.c', '.cc', '.cpp', '.cxx'} and unit.is_file()) + + +def runner_sanitizer_coverage(build_dir: Path, source: Path, + sanitizer_flag: str = SANITIZER_FLAG) -> list[str]: + path = build_dir / 'compile_commands.json' + if not path.is_file(): + raise RuntimeError('compile_commands.json missing; runner instrumentation cannot be verified') + runner_build = (build_dir / 'runner').resolve() + compiled = set() + coverage = [] + for entry in json.loads(path.read_text()): + argv = compile_argv(entry) + output = entry.get('output') or next( + (value for flag, value in zip(argv, argv[1:]) if flag == '-o'), '') + object_path = Path(output) + if not object_path.is_absolute(): + object_path = Path(entry.get('directory') or build_dir) / object_path + if runner_build not in object_path.resolve().parents: + continue + unit = compile_unit(entry) + target = compile_target(entry, argv) + location = (str(unit.relative_to(source.resolve())) if unit.is_relative_to(source.resolve()) + else f'{unit} (outside source tree)') + label = f'{target}: {location}' + disabled = any(flag.startswith('-fno-sanitize=') or + flag == '-fomit-frame-pointer' for flag in argv) + if sanitizer_flag not in argv or '-fno-omit-frame-pointer' not in argv or disabled: + instrumentation = 'ASan/UBSan' if sanitizer_flag == SANITIZER_FLAG else 'ThreadSanitizer' + raise RuntimeError(f'runner compile lacks {instrumentation}/frame pointers: {label}') + compiled.add(unit) + coverage.append(label) + required = {unit.resolve() for unit in runner_translation_units(source)} + missing = required - compiled + if not coverage or missing: + raise RuntimeError('runner compile commands missing: ' + + ', '.join(os.path.relpath(unit, source.resolve()) + for unit in sorted(missing))) + return sorted(coverage) + + def example_targets_with_ndebug(build_dir: Path, source: Path) -> tuple[list[str], list[str]]: """The targets compile_commands.json builds from an examples/native source, and those built with NDEBUG. @@ -1323,6 +1397,8 @@ def write_summary(self) -> None: def invoke(self, name: str, argv: list[str], *, extra_env: dict[str, str] | None = None, timeout: int = 600, combine_stderr: bool = True, stream_output: bool | None = None) -> Completed: + if self.cfg.profile.thread_sanitizers and name in {'ctest', 'native-help', 'require-websocket'}: + argv = ['setarch', os.uname().machine, '-R', *argv] if self.cfg.ccache_path: extra_env = {**(extra_env or {}), 'CCACHE_COMPILERCHECK': 'content'} log = self.logs / f'{name}.log' @@ -1396,6 +1472,8 @@ def pass_stage(self, name: str, message: str, *, argv: list[str] | None = None) self.write_summary() def sanitizer_env(self) -> dict[str, str] | None: + if self.cfg.profile.thread_sanitizers: + return {'TSAN_OPTIONS': 'halt_on_error=1:second_deadlock_stack=1'} return dict(SANITIZER_RUN_ENV) if self.cfg.profile.sanitizers else None def collect_tool_versions(self) -> bool: @@ -1465,6 +1543,7 @@ def verify_configured_profile(self, cache: dict[str, str]) -> str | None: ('PINEFORGE_BUILD_LIVE_RUNNER', profile.live_runner), ('PINEFORGE_BUILD_SOURCE_LAYER', profile.source_layer), ('PINEFORGE_ENABLE_SANITIZERS', profile.sanitizers), + ('PINEFORGE_LIVE_TSAN', profile.thread_sanitizers), ('PINEFORGE_BUILD_EXAMPLES', profile.name in EXAMPLES_PROFILES), ('PINEFORGE_REQUIRE_ABI_RECEIPTS', True), ): @@ -1672,6 +1751,23 @@ def run(self) -> int: self.fail_stage('sanitizer-public-flag', str(error)) return self.finish('failed', 1) self.pass_stage('sanitizer-public-flag', f'library compile uses {SANITIZER_FLAG}') + if self.cfg.profile.live_runner: + try: + coverage = runner_sanitizer_coverage(self.cfg.build_dir, self.cfg.source) + except Exception as error: + self.fail_stage('live-sanitizer-coverage', str(error)) + return self.finish('failed', 1) + self.pass_stage('live-sanitizer-coverage', + f'all {len(coverage)} runner compiles use ASan/UBSan and ' + 'frame pointers:\n' + '\n'.join(coverage)) + if self.cfg.profile.thread_sanitizers: + try: + coverage = runner_sanitizer_coverage(self.cfg.build_dir, self.cfg.source, '-fsanitize=thread') + except Exception as error: + self.fail_stage('runner-thread-sanitizer-coverage', str(error)) + return self.finish('failed', 1) + self.pass_stage('runner-thread-sanitizer-coverage', + f'{len(coverage)} runner compile units use ThreadSanitizer and frame pointers:\n' + '\n'.join(coverage)) # The examples (release, kernel) and the live runner's two modules # built from example sources (kernel, native). if self.cfg.profile.name in EXAMPLES_PROFILES or self.cfg.profile.live_runner: @@ -1732,7 +1828,10 @@ def run(self) -> int: else: self.pass_stage('native-binary', 'live runner absent as required for this profile') - if self.cfg.profile.source_layer: + if self.cfg.profile.name in LIVE_ONLY_PROFILES: + self.pass_stage('abi-providers-skipped', + 'runner-only CTest inventory has no receipt-backed ABI rows') + elif self.cfg.profile.source_layer: self.ensure_abi_base() self.ensure_abi_prior() self.ensure_abi_v13() @@ -1755,6 +1854,29 @@ def run(self) -> int: apple_asan = (self.cfg.profile.sanitizers and sys.platform == 'darwin' and not cxx_name.startswith('g++')) ctest_jobs = 1 if apple_asan else self.cfg.jobs + test_dir = self.cfg.build_dir + if self.cfg.profile.name in LIVE_ONLY_PROFILES: + test_dir /= 'runner' + inventory = self.invoke('live-test-inventory', + ['ctest', '--test-dir', str(test_dir), '--show-only=json-v1'], + timeout=120, stream_output=False) + try: + names = {test['name'] for test in json.loads(inventory.stdout)['tests']} + required = {'native_live_help', 'test_live_json', + 'test_native_live_startup', 'test_native_live_store', + 'test_native_live_routing', + 'test_native_example_batch', 'test_native_example_selected'} + if (self.cfg.source / 'runner' / 'transport.cpp').is_file(): + required.add('test_native_live_websocket') + required.update(runner_e2e_test_names(self.cfg.source)) + if inventory.returncode != 0 or required - names: + raise RuntimeError('missing runner CTest rows: ' + + ', '.join(sorted(required - names))) + except Exception as error: + self.fail_stage('live-test-inventory-required', str(error)) + return self.finish('failed', 1) + self.pass_stage('live-test-inventory-required', + 'all required runner rows registered: ' + ', '.join(sorted(names))) registered = selected = None if self.cfg.exclude_label: listing = ['ctest', '--test-dir', str(self.cfg.build_dir), '-N'] @@ -1770,7 +1892,7 @@ def run(self) -> int: self.summary['ctestRegistered'] = registered self.summary['ctestSelected'] = selected self.write_summary() - ctest = ['ctest', '--test-dir', str(self.cfg.build_dir), + ctest = ['ctest', '--test-dir', str(test_dir), '--output-on-failure', '--no-tests=error', '--parallel', str(ctest_jobs)] if self.cfg.exclude_label: ctest += ['-LE', self.cfg.exclude_label] @@ -1779,6 +1901,13 @@ def run(self) -> int: ran = self.invoke('ctest', ctest, extra_env=self.sanitizer_env(), timeout=ctest_timeout(self.cfg)) self.enforce_test_floor(ran, registered=registered, selected=selected) + if self.cfg.profile.name in LIVE_ONLY_PROFILES: + try: + rows = ctest_rows(ran.stdout + ran.stderr) + if rows is None or rows.skipped or rows.not_run or rows.disabled: + raise RuntimeError(f'{self.cfg.profile.name} cannot accept skipped, disabled or unreadable rows') + except Exception as error: + self.fail_stage('live-test-skips', str(error)) installed = self.invoke( 'install', @@ -1791,7 +1920,8 @@ def run(self) -> int: if not help_bin.is_file(): self.fail_stage('native-help', f'installed native executable missing: {help_bin}') else: - self.invoke('native-help', [str(help_bin), '--help'], timeout=30) + self.invoke('native-help', [str(help_bin), '--help'], + extra_env=self.sanitizer_env(), timeout=30) if self.cfg.require_websocket: ws = self.cfg.build_dir / 'bin' / 'test_native_live_websocket' if not ws.is_file(): diff --git a/scripts/collect_ci_diagnostics.py b/scripts/collect_ci_diagnostics.py index 40ead1651..0ba84b1d9 100644 --- a/scripts/collect_ci_diagnostics.py +++ b/scripts/collect_ci_diagnostics.py @@ -13,6 +13,7 @@ "ci-logs": "ci-logs", "ctest-junit.xml": "ctest-junit.xml", "Testing/Temporary/LastTest.log": "LastTest.log", + "runner/Testing/Temporary/LastTest.log": "runner-LastTest.log", "settlement-abi-base/receipt.json": "settlement-abi-base-receipt.json", "settlement-abi-base/configure.log": "settlement-abi-base-configure.log", "settlement-abi-base/build.log": "settlement-abi-base-build.log", @@ -59,7 +60,7 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--build-dir", type=Path, required=True) parser.add_argument("--profile", - choices=("release", "debug", "sanitizers", "native", "kernel"), + choices=("release", "debug", "sanitizers", "native", "live-sanitizers", "live-tsan", "kernel"), required=True) parser.add_argument("--output", type=Path, default=Path("ci-diagnostics")) parser.add_argument("--dependency-dir", type=Path, diff --git a/scripts/test_ci_preflight.py b/scripts/test_ci_preflight.py index 633d9ba61..a51707d4e 100644 --- a/scripts/test_ci_preflight.py +++ b/scripts/test_ci_preflight.py @@ -17,7 +17,8 @@ ROOT = Path(__file__).resolve().parents[1] KERNEL_VERIFY = ('run: python3 scripts/ci_verify.py kernel --build-dir build-kernel ' - '--jobs "$(getconf _NPROCESSORS_ONLN)" --ccache') + '--jobs "$(getconf _NPROCESSORS_ONLN)" --ccache ' + '--curl-dir "${{ steps.curl-deps.outputs.curl-dir }}"') # ci_workflow_findings' arguments, in order. CI_SOURCES = ('.github/workflows/ci.yml', '.github/workflows/native-live.yml', '.github/workflows/promote-baseline.yml', 'tests/CMakeLists.txt', @@ -268,8 +269,8 @@ def test_ci_contract_pins_runners_time_limits_and_the_fork_guard(self): 'ci.yml job kernel-only must size'), (0, 'kernel-only', f'run: python3 scripts/ci_verify.py kernel --build-dir build-kernel --jobs {CORES} --ccache', 'run: $VERIFY', 'ci.yml job kernel-only must size'), - (0, 'kernel-only', f'run: python3 scripts/ci_verify.py kernel --build-dir build-kernel --jobs {CORES} --ccache\n', - f'run: python3 scripts/ci_verify.py kernel --build-dir build-kernel --jobs {CORES} --ccache\n\n' + (0, 'kernel-only', KERNEL_VERIFY + '\n', + KERNEL_VERIFY + '\n\n' ' - run: PYTHONPATH=scripts python3 -m ci_verify kernel --build-dir build-kernel --ccache\n', 'ci.yml job kernel-only must size'), # A second, unsized call however it is chained, blocked or carried. @@ -686,5 +687,33 @@ def test_success_is_explicitly_only_preflight(self): self.assertIn('full verification still required', summary['scope']) +class PinnedLiveCurl(unittest.TestCase): + def test_all_live_profiles_use_the_shared_pinned_dependency(self): + kernel = _jobs((ROOT / CI_SOURCES[0]).read_text())['kernel-only'] + native = _jobs((ROOT / CI_SOURCES[1]).read_text())['native-live'] + for job in (kernel, native): + self.assertIn('uses: ./.github/actions/setup-live-curl', job) + self.assertNotIn('libcurl4-openssl-dev', job) + for line in job.splitlines(): + if 'run: python3 scripts/ci_verify.py ' in line: + self.assertIn('--curl-dir "${{ steps.curl-deps.outputs.curl-dir }}"', line) + action = (ROOT / '.github/actions/setup-live-curl/action.yml').read_text() + self.assertIn('bash scripts/build_live_curl.sh --metadata', action) + self.assertIn('uses: actions/cache@v4', action) + self.assertIn('bash scripts/build_live_curl.sh build-native-deps 4', action) + + def test_shared_curl_pin_and_websocket_build_flags(self): + script = ROOT / 'scripts/build_live_curl.sh' + result = subprocess.run(['bash', str(script), '--metadata'], + check=True, capture_output=True, text=True) + self.assertEqual(result.stdout, 'version=8.14.1\nsha256=' + 'f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd\n') + build = script.read_text() + self.assertIn('sha256sum -c -', build) + self.assertIn('-DENABLE_WEBSOCKETS=ON', build) + self.assertIn('-DBUILD_STATIC_LIBS=ON', build) + self.assertIn('-DCURL_USE_OPENSSL=ON', build) + + if __name__ == '__main__': unittest.main() diff --git a/scripts/test_ci_verify.py b/scripts/test_ci_verify.py index 4eb24673a..f0e009d47 100644 --- a/scripts/test_ci_verify.py +++ b/scripts/test_ci_verify.py @@ -26,6 +26,7 @@ Completed, ConfigError, KERNEL_MIN_TESTS, + LIVE_SANITIZERS_MIN_TESTS, PROFILES, RELEASE_MIN_TESTS, ROOT, @@ -134,10 +135,14 @@ def __init__(self, build_dir: Path, source: Path, profile: str = 'release', **ex def __call__(self, argv, *, extra_env=None, timeout=600, combine_stderr=True, stream_output=False) -> Completed: argv = list(map(str, argv)) + if argv[0] == 'setarch': + if argv[2] != '-R': + raise AssertionError('ThreadSanitizer must disable ASLR only for its child') + argv = argv[3:] self.timeouts.append((argv, timeout)) # Discovery invocations are asserted through Driver stages. Keep the # existing execution-call fixture stable for pre-existing tests. - if argv[0] != 'ctest' or '-N' not in argv: + if argv[0] != 'ctest' or ('-N' not in argv and '--show-only=json-v1' not in argv): self.calls.append(argv) extra_env = extra_env or {} joined = ' '.join(argv) @@ -219,13 +224,25 @@ def __call__(self, argv, *, extra_env=None, timeout=600, combine_stderr=True, if argv[0] == 'cmake' and '--install' in argv: return self._install() if argv[0] == 'ctest': + if '--show-only=json-v1' in argv: + names = {'native_live_help', 'test_live_json', + 'test_native_live_startup', 'test_native_live_store', + 'test_native_live_routing', + 'test_native_example_batch', 'test_native_example_selected'} + if (self.source / 'runner' / 'transport.cpp').is_file(): + names.add('test_native_live_websocket') + names.update(ci_verify.runner_e2e_test_names(self.source)) + names.discard(self.exits.get('live_missing_test')) + return Completed(0, json.dumps({'tests': [{'name': name} + for name in sorted(names)]}).encode(), b'') if self.exits.get('actual_empty_ctest'): return default_runner(argv, extra_env=extra_env, timeout=timeout, combine_stderr=combine_stderr, stream_output=False) registered = self.exits.get( 'ctest_registered', ci_verify.EXCLUDED_REGISTERED_MIN.get( - self.profile, ci_verify.PROFILE[self.profile].min_tests or KERNEL_MIN_TESTS)) + self.profile, ci_verify.profile_min_tests( + ci_verify.PROFILE[self.profile], self.source) or KERNEL_MIN_TESTS)) labelled = self.exits.get('ctest_labelled', 5) if '-N' in argv: stage = 'ctest-list-selected' if '-LE' in argv else 'ctest-list-all' @@ -235,7 +252,7 @@ def __call__(self, argv, *, extra_env=None, timeout=600, combine_stderr=True, return Completed(int(self.exits.get(stage, 0)), f'Total Tests: {count}\n'.encode(), b'') env_ok = True - if self.profile == 'sanitizers': + if ci_verify.PROFILE[self.profile].sanitizers: env_ok = extra_env == SANITIZER_RUN_ENV if not env_ok: return Completed(1, b'', b'sanitizer env missing\n') @@ -247,7 +264,8 @@ def __call__(self, argv, *, extra_env=None, timeout=600, combine_stderr=True, if 'ctest_raw' in self.exits: return Completed(int(self.exits.get('ctest', 0)), self.exits['ctest_raw'], b'') default_rows = (registered - labelled if '-LE' in argv else - ci_verify.PROFILE[self.profile].min_tests or KERNEL_MIN_TESTS) + ci_verify.profile_min_tests( + ci_verify.PROFILE[self.profile], self.source) or KERNEL_MIN_TESTS) rows = self.exits.get('ctest_rows', default_rows) if rows == 'absent': return Completed(int(self.exits.get('ctest', 0)), b'tests\n', b'') @@ -267,12 +285,13 @@ def __call__(self, argv, *, extra_env=None, timeout=600, combine_stderr=True, return Completed(1, b'', f'unhandled command: {argv}\n'.encode()) def _cache_values(self) -> dict[str, str]: - tutorial = 'OFF' if self.profile in {'native', 'kernel'} else 'ON' - live = 'ON' if self.profile in {'native', 'kernel'} else 'OFF' - sanitizers = 'ON' if self.profile == 'sanitizers' else 'OFF' + profile = ci_verify.PROFILE[self.profile] + tutorial = 'ON' if profile.tutorial else 'OFF' + live = 'ON' if profile.live_runner else 'OFF' + sanitizers = 'ON' if profile.sanitizers else 'OFF' source_layer = 'OFF' if self.profile == 'kernel' else 'ON' examples = 'ON' if self.profile in {'release', 'kernel'} else 'OFF' - build_type = 'Debug' if self.profile in {'debug', 'sanitizers'} else 'Release' + build_type = profile.build_type values = { 'CMAKE_HOME_DIRECTORY': str(self.source), 'CMAKE_CXX_COMPILER': self.cxx, @@ -282,6 +301,7 @@ def _cache_values(self) -> dict[str, str]: 'PINEFORGE_BUILD_TESTS': 'ON', 'PINEFORGE_BUILD_TUTORIAL': tutorial, 'PINEFORGE_BUILD_LIVE_RUNNER': live, + 'PINEFORGE_LIVE_TSAN': 'ON' if ci_verify.PROFILE[self.profile].thread_sanitizers else 'OFF', 'PINEFORGE_BUILD_SOURCE_LAYER': source_layer, 'PINEFORGE_ENABLE_SANITIZERS': sanitizers, 'PINEFORGE_BUILD_EXAMPLES': examples, @@ -308,7 +328,8 @@ def _configure(self) -> Completed: # 'example_commands' == 'absent' scripts a configure that wrote no # compile database at all. if self.exits.get('example_commands') != 'absent': - commands = self._library_compile_commands() + self._example_compile_commands() + commands = (self._library_compile_commands() + self._example_compile_commands() + + self._runner_compile_commands()) (self.build_dir / 'compile_commands.json').write_text(json.dumps(commands)) for role in ('e60', '0e', 'v13', 'v14', 'v15-frozen', 'v16-frozen', 'v18-frozen'): self._maybe_seed_abi_base(role) @@ -323,7 +344,8 @@ def _library_compile_commands(self) -> list[dict]: if self.exits.get('library_commands') == 'none': return [] flag = '' - if self.profile == 'sanitizers' and self.exits.get('sanitizer_flag') != 'absent': + if (ci_verify.PROFILE[self.profile].sanitizers and + self.exits.get('sanitizer_flag') != 'absent'): flag = ' ' + SANITIZER_FLAG unit = self.source / 'src/matrix.cpp' commands = [] @@ -360,6 +382,10 @@ def _example_compile_commands(self) -> list[dict]: ('example_hello_kernel_c', 'hello_kernel_c.c'))] commands = [] for directory, target, object_path, source, extra in rows: + if ci_verify.PROFILE[self.profile].thread_sanitizers: + extra += ' -fsanitize=thread -fno-omit-frame-pointer' + elif ci_verify.PROFILE[self.profile].sanitizers: + extra += ' ' + SANITIZER_FLAG + ' -fno-omit-frame-pointer' undebug = '' if self.exits.get('example_ndebug') == target else ' -UNDEBUG' obj = f'CMakeFiles/{target}.dir/{object_path}.o' commands.append({ @@ -371,6 +397,23 @@ def _example_compile_commands(self) -> list[dict]: }) return commands + def _runner_compile_commands(self) -> list[dict]: + if self.profile not in ci_verify.LIVE_ONLY_PROFILES: + return [] + commands = [] + for unit in ci_verify.runner_translation_units(self.source): + if self.exits.get('runner_command_missing') == unit.name: + continue + sanitizer = '-fsanitize=thread' if self.profile == 'live-tsan' else SANITIZER_FLAG + flags = sanitizer + ' -fno-omit-frame-pointer' + if self.exits.get('runner_flag_missing') == unit.name: + flags = '-fno-omit-frame-pointer' + obj = f'runner/CMakeFiles/runner_{unit.stem}.dir/{unit.name}.o' + commands.append({'directory': str(self.build_dir), 'file': str(unit), + 'output': obj, + 'arguments': [self.cxx, *flags.split(), '-o', obj, '-c', str(unit)]}) + return commands + def _build(self) -> Completed: code = int(self.exits.get('build', 0)) if code != 0: @@ -383,12 +426,13 @@ def _build(self) -> Completed: archive.write_bytes(b'!\nci-verify-test\n') if self.exits.get('stale_archive') == name: os.utime(archive, (1, 1)) - if self.profile in {'native', 'kernel'} or self.exits.get('create_native_binaries'): + if ci_verify.PROFILE[self.profile].live_runner or self.exits.get('create_native_binaries'): binary = self.build_dir / 'bin' / 'pineforge-live' binary.parent.mkdir(parents=True, exist_ok=True) binary.write_bytes(b'live') - ws = self.build_dir / 'bin' / 'test_native_live_websocket' - ws.write_bytes(b'ws') + if (self.source / 'runner' / 'transport.cpp').is_file(): + ws = self.build_dir / 'bin' / 'test_native_live_websocket' + ws.write_bytes(b'ws') if self.exits.get('omit_ws_binary'): ws = self.build_dir / 'bin' / 'test_native_live_websocket' if ws.exists(): @@ -401,7 +445,7 @@ def _install(self) -> Completed: return Completed(code, b'', b'install failed\n') prefix = self.build_dir / 'ci-install' (prefix / 'lib' / 'cmake' / 'PineForge').mkdir(parents=True, exist_ok=True) - if self.profile in {'native', 'kernel'}: + if ci_verify.PROFILE[self.profile].live_runner: help_bin = prefix / 'bin' / 'pineforge-live' help_bin.parent.mkdir(parents=True, exist_ok=True) help_bin.write_bytes(b'live') @@ -695,6 +739,55 @@ def test_sanitizers_public_flag_expected_and_native_off(self): self.assertEqual(values['PINEFORGE_BUILD_TUTORIAL'], 'ON') self.assertIn('-DPINEFORGE_ENABLE_SANITIZERS=ON', argv) + def test_live_sanitizers_enable_runner_and_gate_websockets_without_opt_in(self): + values, _ = self.definitions('live-sanitizers') + self.assertEqual(values['CMAKE_BUILD_TYPE'], 'Debug') + self.assertEqual(values['PINEFORGE_BUILD_LIVE_RUNNER'], 'ON') + self.assertEqual(values['PINEFORGE_ENABLE_SANITIZERS'], 'ON') + self.assertEqual(values['PINEFORGE_BUILD_SOURCE_LAYER'], 'ON') + self.assertEqual(values['PINEFORGE_BUILD_TUTORIAL'], 'OFF') + cfg = validate_config(parse_args(['live-sanitizers'], source=ROOT)) + present = (ROOT / 'runner' / 'transport.cpp').is_file() + self.assertEqual(cfg.require_websocket, present) + self.assertEqual(cfg.min_tests, LIVE_SANITIZERS_MIN_TESTS - int(not present)) + with self.assertRaisesRegex(ConfigError, 'must run every runner row'): + self.definitions('live-sanitizers', ['--exclude-label', 'slow']) + + def test_live_sanitizers_config_follows_transport_presence_and_preserves_overrides(self): + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / 'source' + transport = source / 'runner' / 'transport.cpp' + transport.parent.mkdir(parents=True) + for present in (True, False): + if present: + transport.write_text('int transport_fixture;\n') + else: + transport.unlink() + with self.subTest(transport_present=present): + args = parse_args(['live-sanitizers'], source=source) + cfg = validate_config(args, source=source) + self.assertEqual(cfg.require_websocket, present) + self.assertEqual(cfg.min_tests, LIVE_SANITIZERS_MIN_TESTS - int(not present)) + override = validate_config(parse_args( + ['live-sanitizers', '--min-tests', '4', '--require-websocket'], + source=source), source=source) + self.assertTrue(override.require_websocket) + self.assertEqual(override.min_tests, 4) + + def test_live_tsan_is_separate_and_requires_complete_runner_coverage(self): + values, _ = self.definitions('live-tsan') + self.assertEqual(values['CMAKE_BUILD_TYPE'], 'Debug') + self.assertEqual(values['PINEFORGE_BUILD_LIVE_RUNNER'], 'ON') + self.assertEqual(values['PINEFORGE_LIVE_TSAN'], 'ON') + self.assertEqual(values['PINEFORGE_ENABLE_SANITIZERS'], 'OFF') + config = validate_config(parse_args(['live-tsan'], source=ROOT)) + self.assertEqual(config.require_websocket, (ROOT / 'runner' / 'transport.cpp').is_file()) + self.assertEqual(config.min_tests, LIVE_SANITIZERS_MIN_TESTS) + with self.assertRaisesRegex(ConfigError, 'must run every runner row'): + self.definitions('live-tsan', ['--exclude-label', 'slow']) + self.assertIn('native_live_routing_e2e', ci_verify.runner_e2e_test_names(ROOT)) + self.assertNotIn('native_live_equivalence_e2e', ci_verify.runner_e2e_test_names(ROOT)) + def test_kernel_drops_the_source_layer_and_keeps_the_live_runner(self): values, argv = self.definitions('kernel') self.assertEqual(values['CMAKE_BUILD_TYPE'], 'Release') @@ -736,6 +829,59 @@ def test_native_live_on_tutorial_off(self): self.assertIn('Ninja', argv) +class RunnerSanitizerCoverage(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + self.source = root / 'source' + self.build_dir = root / 'build' + (self.source / 'runner').mkdir(parents=True) + self.build_dir.mkdir() + self.commands = [] + for suffix in ('.c', '.cc', '.cpp', '.cxx'): + unit = self.source / 'runner' / ('fixture' + suffix) + unit.write_text('int coverage_fixture;\n') + self.commands.append(self.command(unit)) + + def command(self, unit: Path) -> dict: + output = f'runner/CMakeFiles/coverage.dir/{unit.name}.o' + return {'directory': str(self.build_dir), 'file': str(unit), 'output': output, + 'arguments': ['c++', SANITIZER_FLAG, '-fno-omit-frame-pointer', + '-o', output, '-c', str(unit)]} + + def coverage(self, commands: list[dict]) -> list[str]: + (self.build_dir / 'compile_commands.json').write_text(json.dumps(commands)) + return ci_verify.runner_sanitizer_coverage(self.build_dir, self.source) + + def test_all_c_and_cxx_suffixes_require_compile_commands(self): + self.assertEqual(len(self.coverage(self.commands)), len(self.commands)) + for command in self.commands: + with self.subTest(unit=command['file']): + without_unit = [entry for entry in self.commands if entry is not command] + with self.assertRaisesRegex(RuntimeError, re.escape(Path(command['file']).name)): + self.coverage(without_unit) + + def test_out_of_tree_units_are_instrumented_and_reported(self): + generated = self.build_dir / 'generated.cc' + generated.write_text('int generated_fixture;\n') + command = self.command(generated) + coverage = self.coverage([*self.commands, command]) + self.assertIn(f'coverage: {generated.resolve()} (outside source tree)', coverage) + command['arguments'].remove(SANITIZER_FLAG) + with self.assertRaisesRegex(RuntimeError, 'outside source tree'): + self.coverage([*self.commands, command]) + + def test_sanitizer_and_frame_pointer_opt_out_flags_are_rejected(self): + for flag in ('-fno-sanitize=address', '-fno-sanitize=undefined', + '-fno-sanitize=all', '-fomit-frame-pointer'): + with self.subTest(flag=flag): + command = dict(self.commands[0]) + command['arguments'] = [*command['arguments'], flag] + with self.assertRaisesRegex(RuntimeError, 'lacks ASan/UBSan/frame pointers'): + self.coverage([command, *self.commands[1:]]) + + class RuntimeBudgetLanes(unittest.TestCase): """Every Release lane of the CI build matrix runs the relative runtime gate. @@ -1687,6 +1833,91 @@ def test_sanitizers_profile_passes_public_flag_and_asan_env(self): self.assertEqual(ctest['extraEnvKeys'], ['ASAN_OPTIONS', 'UBSAN_OPTIONS']) self.assertNotIn('ctest', failure_stages(summary)) + def test_live_sanitizers_follow_transport_presence_and_run_e2es_under_strict_env(self): + original_is_file = Path.is_file + for present in (True, False): + def is_file(path): + if path == ROOT / 'runner' / 'transport.cpp': + return present + return original_is_file(path) + with self.subTest(transport_present=present), mock.patch.object( + Path, 'is_file', autospec=True, side_effect=is_file): + code, summary, scripted, build_dir = self.run_profile('live-sanitizers') + self.assertEqual(code, 0, summary['failures']) + self.assertIn('live-sanitizer-coverage', stage_names(summary)) + self.assertIn('live-test-inventory-required', stage_names(summary)) + self.assertIn('abi-providers-skipped', stage_names(summary)) + self.assertEqual(summary['requireWebsocket'], present) + self.assertEqual(summary['minTests'], LIVE_SANITIZERS_MIN_TESTS - int(not present)) + self.assertEqual(summary['ctestRows'], summary['minTests']) + inventory = next(stage for stage in summary['stages'] + if stage['name'] == 'live-test-inventory-required') + inventory_log = (build_dir / inventory['log']).read_text() + self.assertEqual('test_native_live_websocket' in inventory_log, present) + self.assertEqual('require-websocket' in stage_names(summary), present) + self.assertEqual((build_dir / 'bin' / 'test_native_live_websocket').is_file(), present) + ctest = next(stage for stage in summary['stages'] if stage['name'] == 'ctest') + self.assertIn(str((build_dir / 'runner').resolve()), ctest['argv']) + for name in ('ctest', 'native-help', *(['require-websocket'] if present else [])): + stage = next(stage for stage in summary['stages'] if stage['name'] == name) + self.assertEqual(stage['extraEnvKeys'], ['ASAN_OPTIONS', 'UBSAN_OPTIONS']) + self.assertNotIn('abi-base-prepare', scripted.names()) + code, below_floor, _, _ = self.run_profile( + 'live-sanitizers', ctest_rows=summary['minTests'] - 1) + self.assertEqual(code, 1) + self.assertIn('ctest-floor', failure_stages(below_floor)) + + def test_live_sanitizers_refuse_an_uninstrumented_or_missing_runner_compile(self): + for unit in ci_verify.runner_translation_units(ROOT): + for key in ('runner_flag_missing', 'runner_command_missing'): + with self.subTest(unit=unit, defect=key): + code, summary, scripted, _ = self.run_profile('live-sanitizers', **{key: unit.name}) + self.assertEqual(code, 1) + self.assertIn('live-sanitizer-coverage', failure_stages(summary)) + self.assertNotIn('build', scripted.names()) + + def test_live_tsan_runs_the_same_inventory_and_catches_missing_instrumentation(self): + code, summary, scripted, _ = self.run_profile('live-tsan') + self.assertEqual(code, 0, summary['failures']) + self.assertIn('runner-thread-sanitizer-coverage', stage_names(summary)) + self.assertIn('live-test-inventory-required', stage_names(summary)) + self.assertNotIn('sanitizer-public-flag', stage_names(summary)) + self.assertNotIn('live-sanitizer-coverage', stage_names(summary)) + for name in ('ctest', 'native-help', 'require-websocket'): + stage = next(stage for stage in summary['stages'] if stage['name'] == name) + self.assertEqual(stage['extraEnvKeys'], ['TSAN_OPTIONS']) + self.assertEqual(stage['argv'][:3], ['setarch', os.uname().machine, '-R']) + for unit in ci_verify.runner_translation_units(ROOT): + for key in ('runner_flag_missing', 'runner_command_missing'): + with self.subTest(unit=unit.name, defect=key): + code, summary, scripted, _ = self.run_profile('live-tsan', **{key: unit.name}) + self.assertEqual(code, 1) + self.assertIn('runner-thread-sanitizer-coverage', failure_stages(summary)) + self.assertNotIn('build', scripted.names()) + + def test_live_sanitizers_refuse_a_missing_required_row(self): + names = ci_verify.runner_e2e_test_names(ROOT) + if (ROOT / 'runner' / 'transport.cpp').is_file(): + names.add('test_native_live_websocket') + for name in sorted(names): + with self.subTest(row=name): + code, summary, scripted, _ = self.run_profile( + 'live-sanitizers', live_missing_test=name) + self.assertEqual(code, 1) + self.assertIn('live-test-inventory-required', failure_stages(summary)) + self.assertNotIn('ctest', scripted.names()) + + def test_live_sanitizers_refuse_skips_and_unsupported_websockets(self): + cases = [({'ctest_rows': LIVE_SANITIZERS_MIN_TESTS + 1, + 'ctest_skipped': ['native_live_e2e']}, 'live-test-skips')] + if (ROOT / 'runner' / 'transport.cpp').is_file(): + cases.append(({'require-websocket': 77}, 'require-websocket-skip')) + for defects, stage in cases: + with self.subTest(defects=defects): + code, summary, _, _ = self.run_profile('live-sanitizers', **defects) + self.assertEqual(code, 1) + self.assertIn(stage, failure_stages(summary)) + def test_smoke_ignores_stderr_noise_and_checks_stdout(self): code, summary, _, _ = self.run_profile(**{'smoke-stderr': 'AddressSanitizer noise'}) self.assertEqual(code, 0, summary['failures']) @@ -2143,15 +2374,34 @@ def test_the_recipe_prepares_every_provider_with_ci_verify_argv(self): for role in check_abi_receipt_skips.PROVIDER_ROLES]) class DiagnosticsCollection(unittest.TestCase): - def collect(self, build: Path, output: Path) -> None: + def collect(self, build: Path, output: Path, profile='release') -> None: env = os.environ.copy() env.pop('GITHUB_STEP_SUMMARY', None) result = subprocess.run( [sys.executable, str(ROOT / 'scripts/collect_ci_diagnostics.py'), - '--build-dir', str(build), '--profile', 'release', '--output', str(output)], + '--build-dir', str(build), '--profile', profile, '--output', str(output)], capture_output=True, text=True, env=env, timeout=30) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + def test_live_sanitizers_retain_runner_last_test_log_separately(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + build = root / 'build' + logs = {'Testing/Temporary/LastTest.log': 'engine CTest output\n', + 'runner/Testing/Temporary/LastTest.log': 'runner ASan/UBSan output\n'} + for relative, content in logs.items(): + path = build / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + output = root / 'diagnostics' + self.collect(build, output, profile='live-sanitizers') + self.assertEqual((output / 'LastTest.log').read_text(), + logs['Testing/Temporary/LastTest.log']) + self.assertEqual((output / 'runner-LastTest.log').read_text(), + logs['runner/Testing/Temporary/LastTest.log']) + missing = json.loads((output / 'missing.json').read_text()) + self.assertFalse(set(logs) & set(missing)) + def test_v13_provider_diagnostics_survive_without_binaries(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) diff --git a/tests/live_phase_a_migration.py b/tests/live_phase_a_migration.py new file mode 100644 index 000000000..1775012b0 --- /dev/null +++ b/tests/live_phase_a_migration.py @@ -0,0 +1,79 @@ +"""Run with an actual phase-A executable, the new executable and the same strategy library.""" +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import sqlite3 +import subprocess +import sys +import tempfile +import threading + + +previous, current, library = sys.argv[1:] +received = [] + + +class Receiver(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def do_POST(self): + body = self.rfile.read(int(self.headers['Content-Length'])) + event = json.loads(body) + assert self.headers['Idempotency-Key'] == self.headers['X-PineForge-Event-Id'] == event['event_id'] + received.append((event['sequence'], event['event_id'], body)) + self.send_response(204 if event['sequence'] == 1 or allow_success else 500) + self.send_header('Content-Length', '0') + self.end_headers() + + def log_message(self, *args): + pass + + +allow_success = False +server = ThreadingHTTPServer(('127.0.0.1', 0), Receiver) +server.daemon_threads = True +thread = threading.Thread(target=server.serve_forever, daemon=True) +thread.start() +try: + with tempfile.TemporaryDirectory(prefix='pineforge-phase-a-migration-') as directory: + root = Path(directory) + warmup = root / 'warmup.csv' + warmup.write_text('timestamp,open,high,low,close,volume\n' + + ''.join(f'{index*60000},100,101,99,100,4\n' for index in range(3))) + feed = root / 'feed.jsonl' + feed.write_text(''.join(json.dumps({'type': 'bar', 'bar': {'ts_open': index*60000, + 'o': 100+index, 'h': 102+index, 'l': 99+index, 'c': 101+index, 'v': 4}}) + '\n' + for index in range(3, 27))) + ledger = root / 'phase-a.sqlite' + arguments = ['run', '--strategy', library, '--warmup', str(warmup), '--script-tf', '3', + '--symbol', 'TEST:MOCK', '--mode', 'bars', '--feed', str(feed), '--ledger', str(ledger), + '--webhook-url', f'http://127.0.0.1:{server.server_port}/actions', '--allow-insecure-http', + '--max-attempts', '1'] + old = subprocess.run([previous] + arguments, capture_output=True, text=True, env=os.environ, timeout=25) + assert old.returncode == 1, (old.stdout, old.stderr) + with sqlite3.connect(ledger) as database: + assert database.execute('SELECT schema_version FROM metadata').fetchone() == (1,) + assert database.execute("SELECT count(*) FROM sqlite_master WHERE type='table'").fetchone() == (3,) + immutable = database.execute('SELECT ordinal,event_id,payload,acknowledged FROM events ORDER BY ordinal').fetchall() + assert [row[3] for row in immutable] == [1, 0] + received.clear() + allow_success = True + new = subprocess.run([current] + arguments, capture_output=True, text=True, env=os.environ, timeout=25) + assert new.returncode == 0, (new.stdout, new.stderr) + assert sorted(row[0] for row in received) == [2, 3, 4] + recovered = next(row for row in received if row[0] == 2) + assert recovered[1] == immutable[1][1] and recovered[2].decode() == immutable[1][2] + with sqlite3.connect(ledger) as database: + migrated = database.execute('SELECT ordinal,event_id,payload FROM events ORDER BY ordinal').fetchall() + assert migrated[:2] == [row[:3] for row in immutable] + assert database.execute("SELECT count(*) FROM sqlite_master WHERE type='table'").fetchone() == (6,) + assert database.execute("SELECT count(*) FROM delivery_log WHERE phase='completed'").fetchone() == (3,) + assert database.execute('SELECT delivery_id=event_id FROM event_routes JOIN events USING(ordinal)').fetchall() == [(1,)] * 4 + resumed = subprocess.run([current] + arguments, capture_output=True, text=True, env=os.environ, timeout=25) + assert resumed.returncode == 0 and len(received) == 3, resumed.stderr + print('PASS actual phase-A schema-1 ledger migration: additive audit tables, unchanged payload/event IDs, acknowledged events never resent, unacknowledged events sent once') +finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) diff --git a/tests/native_live_e2e.py b/tests/native_live_e2e.py index ed39794bf..eb11edd23 100644 --- a/tests/native_live_e2e.py +++ b/tests/native_live_e2e.py @@ -7,6 +7,7 @@ import hmac from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json +import os from pathlib import Path import sqlite3 import subprocess @@ -16,7 +17,7 @@ import base64 import struct -runner, library, parser, batch_parser = sys.argv[1:] +runner, library = sys.argv[1:] received = [] responses = [] secret = "synthetic-loopback-test-key" @@ -24,6 +25,10 @@ stream_messages = [] +def ordered_receipts(): + return sorted(received, key=lambda event: event['sequence']) + + class Receiver(BaseHTTPRequestHandler): protocol_version = 'HTTP/1.1' def do_GET(self): @@ -76,10 +81,11 @@ def log_message(self, *args): '--webhook-url', f'http://127.0.0.1:{server.server_port}/webhook', '--webhook-secret-env', 'PINEFORGE_TEST_HMAC'] - def invoke(extra, success=True): + def invoke(extra, success=True, stdin=None): import os env = dict(os.environ, PINEFORGE_TEST_HMAC=secret) - p = subprocess.run(base + extra, capture_output=True, text=True, env=env, timeout=25) + p = subprocess.run(base + extra, input=stdin, capture_output=True, text=True, + env=env, timeout=25) assert (p.returncode == 0) is success, (p.returncode, p.stdout, p.stderr) assert secret not in p.stdout + p.stderr return json.loads(p.stdout) if success else p @@ -108,21 +114,21 @@ def invoke(extra, success=True): assert result['inputs_committed'] == len(events) assert result['webhooks_pending'] == 0 assert len(received) == 4, received - assert [e['order']['leg'] for e in received] == ['entry', 'exit', 'entry', 'exit'] - assert [e['sequence'] for e in received] == [1, 2, 3, 4] + assert [e['order']['leg'] for e in ordered_receipts()] == ['entry', 'exit', 'entry', 'exit'] + assert [e['sequence'] for e in ordered_receipts()] == [1, 2, 3, 4] assert len({e['event_id'] for e in received}) == 4 - prior = list(received) + prior = ordered_receipts() final = invoke(options) assert final['inputs_processed'] == final['webhooks_delivered'] == 0 - assert received == prior + assert ordered_receipts() == prior assert final['prefix_skipped'] == len(events) same_timezone = invoke(options+['--chart-timezone', 'UTC']) - assert same_timezone['webhooks_delivered'] == 0 and received == prior + assert same_timezone['webhooks_delivered'] == 0 and ordered_receipts() == prior with sqlite3.connect(ledger) as db: assert db.execute('SELECT count(*) FROM inputs').fetchone()[0] == len(events) # Changing a declared strategy input changes deployment identity. invoke(options+['--input', 'changed=1'], success=False) - assert received == prior + assert ordered_receipts() == prior # A changed historical frame must refuse without another webhook. bad = root/(mode+'-changed.jsonl') changed = [dict(e) for e in events] @@ -132,43 +138,67 @@ def invoke(extra, success=True): changed[0]['bar'] = dict(changed[0]['bar'], v=5) bad.write_text(''.join(json.dumps(e)+'\n' for e in changed)) invoke(['--mode', mode, '--feed', str(bad), '--ledger', str(ledger)], success=False) - assert received == prior + assert ordered_receipts() == prior + + received.clear() + stdin_feed = ''.join(json.dumps(event)+'\n' for event in events) + stdin_options = ['--mode', mode, '--feed', '-', + '--ledger', str(root/(mode+'-stdin.sqlite3'))] + result = invoke(stdin_options, stdin=stdin_feed) + assert result['inputs_committed'] == len(events) + assert ordered_receipts() == prior + assert invoke(stdin_options, stdin=stdin_feed)['inputs_processed'] == 0 + assert ordered_receipts() == prior - # Failed HTTP response leaves the same immutable event queued. Restart - # replays input before retrying that event, with its identical id/body. + # HTTP errors are final; restart does not retry them. received.clear(); responses[:] = [503] feed = root/'retry.jsonl'; feed.write_text(''.join(json.dumps(e)+'\n' for e in bar_events)) ledger = root/'retry.sqlite3' options = ['--mode', 'bars', '--feed', str(feed), '--ledger', str(ledger)] - invoke(options+['--max-attempts', '1'], success=False) - assert len(received) == 1 + invoke(options+['--max-attempts', '1']) + assert len(received) == 4 first = received[0] invoke(options) - assert received[1] == first + assert len(received) == 4 + with sqlite3.connect(ledger) as db: + deployment = db.execute('SELECT identity FROM metadata WHERE singleton=1').fetchone()[0] + retry = subprocess.run([runner, 'redeliver', '--ledger', str(ledger), '--deployment', deployment, + '--target', 'default', '--failed-only'], + capture_output=True, text=True, env=dict(os.environ, PINEFORGE_TEST_HMAC=secret), timeout=25) + assert retry.returncode == 0, retry.stderr + assert received[4] == first assert len({e['event_id'] for e in received}) == 4 - # Redirect refusal is permanent, with one durable attempt per invocation. + # Redirect refusal is permanent and does not stop later actions. received.clear(); responses[:] = [302] redirect_ledger = root/'redirect.sqlite3' - invoke(['--mode','bars','--feed',str(feed),'--ledger',str(redirect_ledger)],success=False) - assert len(received) == 1 + invoke(['--mode','bars','--feed',str(feed),'--ledger',str(redirect_ledger)]) + assert len(received) == 4 with sqlite3.connect(redirect_ledger) as db: assert db.execute('SELECT attempts FROM events WHERE acknowledged=0 ORDER BY ordinal LIMIT 1').fetchone()[0] == 1 - # A complete provider batch is one durable input, even when delivery + # A complete feed batch is one durable input, even when delivery # fails or max-events requests a stop. No remaining event is lost. received.clear(); responses[:] = [503] feed = root/'batch.jsonl' feed.write_text(json.dumps({'type':'batch','events':bar_events})+'\n') ledger = root/'batch.sqlite3' options = ['--mode','bars','--feed',str(feed),'--ledger',str(ledger),'--max-events','1'] - invoke(options+['--max-attempts','1'],success=False) + result = invoke(options+['--max-attempts','1']) + assert result['webhook_failures'] == 1 with sqlite3.connect(ledger) as db: assert db.execute('SELECT COUNT(*) FROM inputs').fetchone()[0] == 1 assert db.execute('SELECT COUNT(*) FROM events').fetchone()[0] == 4 first = received[0] invoke(options) - assert received[1] == first and len({e['event_id'] for e in received}) == 4 + assert len(received) == 4 and len({e['event_id'] for e in received}) == 4 + with sqlite3.connect(ledger) as db: + deployment = db.execute('SELECT identity FROM metadata WHERE singleton=1').fetchone()[0] + retry = subprocess.run([runner, 'redeliver', '--ledger', str(ledger), '--deployment', deployment, + '--target', 'default', '--failed-only'], + capture_output=True, text=True, env=dict(os.environ, PINEFORGE_TEST_HMAC=secret), timeout=25) + assert retry.returncode == 0, retry.stderr + assert received[4] == first # A later invalid event in a batch must not commit any of the message. received.clear() @@ -181,65 +211,110 @@ def invoke(extra, success=True): assert db.execute('SELECT COUNT(*) FROM events').fetchone()[0] == 0 assert not received - # A plugin may emit multiple trades; its message is still one input. - feed = root/'parser-batch.txt'; feed.write_text('two-trades\n') - ledger = root/'parser-batch.sqlite3' - options=['--mode','ticks','--feed',str(feed),'--ledger',str(ledger),'--parser',batch_parser,'--max-events','1'] - result=invoke(options) - assert result['inputs_committed']==1 and result['last_tick_sequence']==2 - assert invoke(options)['inputs_processed']==0 + websocket_available = True + for mode, events in [('bars', bar_events), ('ticks', tick_events)]: + received.clear() + snapshot = ''.join(json.dumps(event)+'\n' for event in events).encode() + options = ['--mode', mode, '--feed-url', + f'http://127.0.0.1:{server.server_port}/snapshot', + '--check', '--ledger', str(root/(mode+'-http.sqlite3'))] + result = invoke(options) + assert result['inputs_committed'] == len(events) + assert len(received) == 4 + prior = ordered_receipts() + assert invoke(options)['inputs_processed'] == 0 + assert ordered_receipts() == prior - # A custom C++ parser accepts provider messages without a Python adapter. - # The next-minute tick finalizes the previous minute in the native engine. - received.clear() - feed = root/'provider.txt' - messages = ['heartbeat'] + [f"trade,{e['seq']},{e['ts']},{e['price']},{e['qty']}" - for e in tick_events if e['type'] == 'tick'] - messages.append(f'trade,{seq},1620000,127,1') - feed.write_text('\n'.join(messages)+'\n') - options = ['--mode', 'ticks', '--feed', str(feed), '--ledger', str(root/'parser.sqlite3'), - '--parser', parser] - invoke(options) - assert len(received) == 4 - prior = list(received) - invoke(options) - assert received == prior - # Polling snapshots are consumed by the native libcurl transport. - received.clear() - snapshot = ''.join(json.dumps(e)+'\n' for e in bar_events).encode() - options = ['--mode', 'bars', '--feed-url', f'http://127.0.0.1:{server.server_port}/snapshot', - '--check', '--ledger', str(root/'http.sqlite3')] - invoke(options) - assert len(received) == 4 - prior = list(received) - invoke(options) - assert received == prior + received.clear() + stream_messages = [json.dumps(event) for event in events] + websocket_ledger = root/(mode+'-websocket.sqlite3') + options = ['--mode', mode, '--feed-url', + f'ws://127.0.0.1:{server.server_port}/stream', + '--max-events', str(len(events)), '--ledger', str(websocket_ledger)] + import os + result = subprocess.run(base+options, capture_output=True, text=True, + env=dict(os.environ, PINEFORGE_TEST_HMAC=secret), timeout=25) + if result.returncode: + assert ('libcurl build with WS/WSS support enabled' in result.stderr or + 'libcurl 8.14.1 or newer' in result.stderr), (result.stdout,result.stderr) + assert not received and not websocket_ledger.exists() + websocket_available = False + print('WebSocket CLI integration unavailable in this libcurl build') + else: + assert ordered_receipts() == prior + assert json.loads(result.stdout)['inputs_committed'] == len(events) - # WebSocket wire intake, custom native parser, engine and HTTP outbox - # execute in the same C++ process. A system curl lacking WebSockets - # refuses explicitly; separate native transport tests report a skip. - received.clear() - stream_messages = messages - options = ['--mode', 'ticks', '--feed-url', f'ws://127.0.0.1:{server.server_port}/stream', - '--max-events', str(len(messages)-1), '--parser', parser, - '--ledger', str(root/'websocket.sqlite3')] - import os - result = subprocess.run(base+options, capture_output=True, text=True, - env=dict(os.environ, PINEFORGE_TEST_HMAC=secret), timeout=25) - if result.returncode: - assert 'libcurl build with WS/WSS support enabled' in result.stderr, (result.stdout,result.stderr) - assert not received - print('WebSocket CLI integration unavailable in this libcurl build') - else: - assert len(received) == 4 - assert json.loads(result.stdout)['inputs_committed'] == len(messages)-1 + for source in ('stdin', 'file', 'http', 'websocket'): + if source == 'websocket' and not websocket_available: + continue received.clear() - stream_messages = ['two-trades'] - result = invoke(['--mode','ticks','--feed-url',f'ws://127.0.0.1:{server.server_port}/stream', - '--max-events','1','--parser',batch_parser, - '--ledger',str(root/'websocket-batch.sqlite3')]) - assert result['inputs_committed']==1 and result['last_tick_sequence']==2 - print('native C++ runner: bars/ticks/parser, HMAC HTTP, partial restart, exact recovery and immutable retry passed') + array_message = json.dumps(tick_events[:2]) + ledger = root/(source+'-array.sqlite3') + options = ['--mode', 'ticks', '--ledger', str(ledger), '--max-events', '1'] + stdin = None + if source == 'stdin': + options += ['--feed', '-'] + stdin = array_message+'\n' + elif source == 'file': + feed = root/'array.jsonl' + feed.write_text(array_message+'\n') + options += ['--feed', str(feed)] + elif source == 'http': + snapshot = (array_message+'\n').encode() + options += ['--feed-url', f'http://127.0.0.1:{server.server_port}/snapshot', '--check'] + else: + stream_messages = [array_message] + options += ['--feed-url', f'ws://127.0.0.1:{server.server_port}/stream'] + result = invoke(options, stdin=stdin) + assert result['inputs_committed'] == 1 and result['last_tick_sequence'] == 2 + assert not received + + raw_messages = [ + 'trade,1,180000,103,1', + json.dumps({'event': 'trade', 'timestamp': 180000, 'price': '103', 'quantity': '1'}), + json.dumps({'type': 'trade', 'ts': 180000, 'seq': 1, 'price': 103, 'qty': 1}), + json.dumps({'type': 'forming', 'bar': bar_events[0]['bar']}), + json.dumps(dict(tick_events[0], venue='raw-provider')), + json.dumps({'type': 'tick'}), + json.dumps(dict(tick_events[0], price='103')), + json.dumps(dict(tick_events[0], seq='1')), + json.dumps({'type': 'time', 'ts': '180000'}), + json.dumps({'type': 'bar', 'bar': {'ts_open': 180000}}), + json.dumps([tick_events[0], {'event': 'trade'}]), + '[]', + json.dumps({'type': 'batch', 'events': [{'type': 'batch', 'events': tick_events[:2]}]}), + ] + for source in ('stdin', 'file', 'http', 'websocket'): + if source == 'websocket' and not websocket_available: + continue + for message_index, message in enumerate(raw_messages): + received.clear() + ledger = root/(source+'-raw-'+str(message_index)+'.sqlite3') + options = ['--mode', 'ticks', '--ledger', str(ledger)] + stdin = None + if source == 'stdin': + options += ['--feed', '-'] + stdin = message+'\n' + elif source == 'file': + feed = root/'raw.jsonl' + feed.write_text(message+'\n') + options += ['--feed', str(feed)] + elif source == 'http': + snapshot = (message+'\n').encode() + options += ['--feed-url', f'http://127.0.0.1:{server.server_port}/snapshot', '--check'] + else: + stream_messages = [message] + options += ['--feed-url', f'ws://127.0.0.1:{server.server_port}/stream'] + result = invoke(options, success=False, stdin=stdin) + assert 'PineForge feed events required' in result.stderr, result.stderr + assert 'external feed adapter' in result.stderr, result.stderr + assert 'runner/README.md#feed-format' in result.stderr, result.stderr + with sqlite3.connect(ledger) as db: + assert db.execute('SELECT COUNT(*) FROM inputs').fetchone()[0] == 0 + assert db.execute('SELECT COUNT(*) FROM events').fetchone()[0] == 0 + assert not received + print('native C++ runner: normalized bars/ticks over stdin/file/HTTP/WebSocket, ' + 'raw-message refusal, HMAC, exact recovery and immutable retry passed') finally: server.shutdown() server.server_close() diff --git a/tests/native_live_routing_e2e.py b/tests/native_live_routing_e2e.py new file mode 100644 index 000000000..0c6eff803 --- /dev/null +++ b/tests/native_live_routing_e2e.py @@ -0,0 +1,468 @@ +"""Local receivers exercise immutable routing, delivery isolation and ledger readers.""" +import copy +import hashlib +import hmac +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import queue +import signal +import socket +import sqlite3 +import subprocess +import sys +import tempfile +import threading +import time + + +runner, library = sys.argv[1:3] +secrets = {'MAIN_HMAC': 'main-loopback-secret', 'ENTRY_HMAC': 'entry-loopback-secret', + 'EXIT_HMAC': 'exit-loopback-secret'} +environment = dict(os.environ, **secrets) + + +class Receiver: + def __init__(self, secret, port=0): + self.secret = secret + self.mode = 'ok' + self.rows = [] + self.errors = [] + self.lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def do_POST(self): + try: + body = self.rfile.read(int(self.headers['Content-Length'])) + action = json.loads(body) + signature = 'sha256=' + hmac.new(owner.secret.encode(), body, hashlib.sha256).hexdigest() + assert self.headers['X-PineForge-Signature'] == signature + assert self.headers['X-PineForge-Event-Id'] == action['event_id'] + assert self.headers['Idempotency-Key'] == action.get('delivery_id', action['event_id']) + with owner.lock: + owner.rows.append({'body': body, 'action': action, 'key': self.headers['Idempotency-Key'], + 'at': time.monotonic()}) + mode = owner.mode + if mode == 'reset': + self.connection.shutdown(socket.SHUT_RDWR) + return + if mode == 'hang' or (mode == 'hang-first' and action['sequence'] == 1): + time.sleep(4) + status = 500 if mode == 'fail' or (mode == 'fail-first' and action['sequence'] == 2) else 204 + self.send_response(status) + self.send_header('Content-Length', '0') + self.end_headers() + except (BrokenPipeError, ConnectionResetError): + pass + except Exception as error: + with owner.lock: + owner.errors.append(str(error)) + + def log_message(self, *args): + pass + + self.server = ThreadingHTTPServer(('127.0.0.1', port), Handler) + self.server.daemon_threads = True + self.server.request_queue_size = 128 + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + + @property + def url(self): + return f'http://127.0.0.1:{self.server.server_port}/actions' + + def clear(self): + with self.lock: + self.rows.clear() + self.errors.clear() + + def close(self): + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=2) + + +def wait_for(predicate, timeout=5): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + raise AssertionError('timed out waiting for local test condition') + + +def query(ledger, sql): + with sqlite3.connect(ledger, timeout=2) as database: + return database.execute(sql).fetchall() + + +def committed_actions(ledger): + if not ledger.exists(): + return False + try: + return query(ledger, 'SELECT count(*) FROM events')[0][0] == 4 + except sqlite3.OperationalError: + return False + + +def invoke(arguments, expected=0, env=None, auto_deployment=True): + if arguments[0] == 'redeliver' and '--deployment' not in arguments and auto_deployment: + ledger = arguments[arguments.index('--ledger') + 1] + arguments = arguments + ['--deployment', query(ledger, 'SELECT identity FROM metadata')[0][0]] + process = subprocess.run([runner] + arguments, capture_output=True, text=True, + env=environment if env is None else env, timeout=25) + assert process.returncode == expected, (arguments, process.returncode, process.stdout, process.stderr) + for secret in secrets.values(): + assert secret not in process.stdout + process.stderr + return process + + +with tempfile.TemporaryDirectory(prefix='pineforge-routing-e2e-') as directory: + root = Path(directory) + warmup = root / 'warmup.csv' + warmup.write_text('timestamp,open,high,low,close,volume\n' + + ''.join(f'{index * 60000},100,101,99,100,4\n' for index in range(3))) + events = [{'type': 'bar', 'bar': {'ts_open': index * 60000, 'o': 100 + index, 'h': 102 + index, + 'l': 99 + index, 'c': 101 + index, 'v': 4, 'trade_count': 4}} + for index in range(3, 27)] + feed = root / 'feed.jsonl' + feed.write_text(''.join(json.dumps(event) + '\n' for event in events)) + base = ['run', '--strategy', library, '--warmup', str(warmup), '--script-tf', '3', + '--symbol', 'TEST:MOCK', '--mode', 'bars', '--feed', str(feed), '--allow-insecure-http'] + receivers = [Receiver(secrets[name]) for name in ('MAIN_HMAC', 'ENTRY_HMAC', 'EXIT_HMAC')] + main_receiver, entries, exits = receivers + + def run(ledger, routes=None, extra=None, expected=0): + arguments = base + ['--ledger', str(ledger)] + if routes is not None: + arguments += ['--webhook-routes', str(routes)] + return invoke(arguments + (extra or []), expected) + + def save_routes(name, document): + path = root / (name + '.json') + path.write_text(json.dumps(document, sort_keys=True, separators=(',', ':'))) + return path + + def stored(ledger): + return (query(ledger, 'SELECT input_index,canonical_json,state_hash FROM inputs ORDER BY input_index'), + query(ledger, 'SELECT ordinal,event_id,payload FROM events ORDER BY ordinal')) + + try: + journal = root / 'journal.sqlite' + partial = json.loads(run(journal, extra=['--max-events', '2']).stdout) + assert partial['inputs_committed'] == 2 + follower = subprocess.Popen([runner, 'actions', '--ledger', str(journal), '--after', '0', '--follow'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=environment) + streamed = queue.Queue() + reader = threading.Thread(target=lambda: [streamed.put(line.rstrip('\n')) for line in follower.stdout], daemon=True) + reader.start() + run(journal) + observed = [streamed.get(timeout=5) for _ in range(4)] + expected_payloads = [row[0] for row in query(journal, 'SELECT payload FROM events ORDER BY ordinal')] + assert observed == expected_payloads + follower.send_signal(signal.SIGTERM) + follower.wait(timeout=5) + reader.join(timeout=2) + assert follower.returncode == 130 and not follower.stderr.read() + exported = invoke(['actions', '--ledger', str(journal), '--after', '2']) + assert exported.stdout.splitlines() == expected_payloads[2:] + assert json.loads(invoke(['status', '--ledger', str(journal)]).stdout)['targets'] == {} + assert not query(journal, 'SELECT * FROM delivery_log') + assert all(json.loads(payload)['schema_version'].endswith('/v1') for payload in expected_payloads) + print('PASS journal-only runner, optional webhook, actions export and concurrent actions --follow', flush=True) + + document = {'schema_version': 1, 'default_target': 'main', + 'targets': {'main': {'url': main_receiver.url, 'secret_env': 'MAIN_HMAC'}, + 'entries': {'url': entries.url, 'secret_env': 'ENTRY_HMAC'}, + 'exits': {'url': exits.url, 'secret_env': 'EXIT_HMAC'}}, + 'rules': [{'match': {'order_id': 'Long', 'kind': 'entry', 'side': 'long'}, 'target': 'entries'}, + {'match': {'kind': 'entry'}, 'target': None}, + {'match': {'kind': 'exit', 'side': 'long'}, 'target': 'exits'}]} + routes = save_routes('routes', document) + ledger = root / 'routed.sqlite' + result = json.loads(run(ledger, routes).stdout) + assert result['webhooks_delivered'] == 4 + assert not main_receiver.rows and len(entries.rows) == len(exits.rows) == 2 + payloads = [json.loads(row[0]) for row in query(ledger, 'SELECT payload FROM events ORDER BY ordinal')] + assert [action['target_id'] for action in payloads] == ['entries', 'exits', 'entries', 'exits'] + assert all(action['schema_version'] == 'pineforge-native-order-action/v2' for action in payloads) + assert all(action['order']['side'] == 'long' for action in payloads) + for action in payloads: + canonical = json.dumps({'event_id': action['event_id'], 'target_id': action['target_id']}, + sort_keys=True, separators=(',', ':')).encode() + assert action['delivery_id'] == hashlib.sha256(canonical).hexdigest() + assert action['order']['kind'] == action['order']['leg'] + immutable = stored(ledger) + run(ledger, routes, ['--webhook-url', main_receiver.url, '--webhook-secret-env', 'MAIN_HMAC']) + assert stored(ledger) == immutable and len(entries.rows) == len(exits.rows) == 2 + changed = copy.deepcopy(document) + changed['rules'].reverse() + refusal = run(ledger, save_routes('changed', changed), expected=1) + assert 'routing' in refusal.stderr and 'new ledger' in refusal.stderr + assert stored(ledger) == immutable + print('PASS multiple targets, first-match B1 selectors, v2 identity, HMAC and routing identity refusal', flush=True) + + null_document = copy.deepcopy(document) + null_document['default_target'] = None + null_document['rules'] = [{'match': {'kind': 'entry'}, 'target': None}, + {'match': {'kind': 'exit'}, 'target': 'exits'}] + exits.clear() + null_ledger = root / 'null.sqlite' + run(null_ledger, save_routes('null', null_document)) + assert len(exits.rows) == 2 + assert query(null_ledger, 'SELECT target_id FROM event_routes ORDER BY ordinal') == [(None,), ('exits',), (None,), ('exits',)] + assert len(query(null_ledger, "SELECT * FROM delivery_log WHERE phase='completed'")) == 2 + print('PASS null rule/default target journals without sending or losing actions', flush=True) + + invalid_documents = [] + for selector in ({'alert_message': 'x'}, {'kind': 'close'}, {'side': 'buy'}, {'order_id': 1}, {'venue': 'x'}): + invalid = copy.deepcopy(document) + invalid['rules'][0]['match'] = selector + invalid_documents.append((invalid, 'strategy-metadata extension' if 'alert_message' in selector or selector.get('kind') == 'close' else None)) + for delivery in ({'max_in_flight': 0}, {'transport_retries': 3}, {'connect_timeout_ms': -1}, + {'total_timeout_ms': 0}, {'retry_backoff_ms': [10]}, {'unknown': 1}): + invalid = copy.deepcopy(document) + invalid['delivery'] = delivery + invalid_documents.append((invalid, None)) + invalid = copy.deepcopy(document); invalid['extra'] = True + invalid_documents.append((invalid, 'unknown')) + invalid = copy.deepcopy(document); invalid['rules'][0]['target'] = 'undefined' + invalid_documents.append((invalid, 'undefined')) + invalid = copy.deepcopy(document); invalid['targets']['entries']['url'] = 'https://user:secret@receiver.example/' + invalid_documents.append((invalid, 'user information')) + for index, (invalid, message) in enumerate(invalid_documents): + path = root / f'invalid-{index}.sqlite' + process = run(path, save_routes(f'invalid-{index}', invalid), expected=1) + assert not path.exists() + if message: + assert message in process.stderr, process.stderr + missing_secret = dict(environment); missing_secret.pop('ENTRY_HMAC') + path = root / 'missing-secret.sqlite' + process = invoke(base + ['--ledger', str(path), '--webhook-routes', str(routes)], expected=1, env=missing_secret) + assert 'missing or empty' in process.stderr and not path.exists() + path = root / 'cli-conflict.sqlite' + assert 'agree' in run(path, routes, ['--webhook-url', entries.url], expected=1).stderr + assert not path.exists() + print('PASS strict startup validation, B2 refusal, missing secrets and CLI conflicts before input', flush=True) + + isolation_document = copy.deepcopy(document) + isolation_document['delivery'] = {'max_in_flight': 8, 'connect_timeout_ms': 100, + 'total_timeout_ms': 500, 'transport_retries': 2, + 'retry_backoff_ms': [100, 200]} + isolation = save_routes('isolation', isolation_document) + entries.clear(); exits.clear() + entries.mode = 'hang-first'; exits.mode = 'fail-first' + isolation_ledger = root / 'isolation.sqlite' + process = run(isolation_ledger, isolation) + newer = [row for row in entries.rows if row['action']['sequence'] == 3] + assert len(newer) == 1 + first_completed = query(isolation_ledger, "SELECT min(ended_at) FROM delivery_log WHERE target_id='entries' AND phase='completed' AND success=0")[0][0] + initial_starts = query(isolation_ledger, "SELECT started_at FROM delivery_log WHERE phase='started' AND attempt=1") + assert len(initial_starts) == 4 and all(row[0] < first_completed for row in initial_starts) + assert len([row for row in exits.rows if row['action']['sequence'] == 2]) == 1 + assert len([row for row in entries.rows if row['action']['sequence'] == 1]) == 3 + assert query(isolation_ledger, 'SELECT count(*) FROM inputs')[0][0] == len(events) + warning = ('pineforge-live: warning: compiled strategy lacks checked settings; ' + 'legacy settings may be ignored or defaulted') + log_lines = process.stderr.splitlines() + assert log_lines.count(warning) == 1, process.stderr + errors = [json.loads(line) for line in log_lines if line != warning] + assert len(errors) == 4 and all(row['event'] == 'webhook_delivery_error' for row in errors) + status = json.loads(invoke(['status', '--ledger', str(isolation_ledger)]).stdout)['targets'] + assert status['entries']['failed'] == 3 and status['exits']['failed'] == 1 + assert status['entries']['last_error']['category'] == 'timeout' + assert status['exits']['last_error']['http_status'] == 500 + assert status['entries']['last_success'] is not None and status['main']['sent'] == 0 + attempts = query(isolation_ledger, "SELECT event_id,attempt,started_at,ended_at,http_status,error_category FROM delivery_log WHERE phase='completed'") + assert len(attempts) == 6 and all(row[3] >= row[2] for row in attempts) + run(isolation_ledger, isolation) + assert len(entries.rows) == 4 and len(exits.rows) == 2 + entries.mode = exits.mode = 'ok' + result = json.loads(invoke(['redeliver', '--ledger', str(isolation_ledger), '--target', 'entries', '--failed-only']).stdout) + assert result == {'selected': 1, 'delivered': 1, 'failed': 0, 'pending': 0} + repeated = [row for row in entries.rows if row['action']['sequence'] == 1] + assert len(repeated) == 4 and len({row['body'] for row in repeated}) == len({row['key'] for row in repeated}) == 1 + rotated_environment = dict(environment, ENTRY_HMAC='rotated-loopback-signing-key') + entries.secret = rotated_environment['ENTRY_HMAC'] + invoke(['redeliver', '--ledger', str(isolation_ledger), '--target', 'entries', '--from', '1'], + env=rotated_environment) + assert all(row['body'] == next(original['body'] for original in repeated + if original['action']['sequence'] == row['action']['sequence']) + for row in entries.rows if row['action']['sequence'] == 1) + entries.secret = secrets['ENTRY_HMAC'] + result = json.loads(invoke(['redeliver', '--ledger', str(isolation_ledger), '--target', 'exits', '--from', '3', '--failed-only']).stdout) + assert result['selected'] == 0 + invoke(['redeliver', '--ledger', str(isolation_ledger), '--target', 'exits', '--failed-only']) + assert len(exits.rows) == 3 + with sqlite3.connect(isolation_ledger) as database: + for statement in ('UPDATE delivery_log SET error_category=\'hidden\'', 'DELETE FROM delivery_log'): + try: + database.execute(statement) + except sqlite3.DatabaseError as error: + assert 'append-only' in str(error) + else: + raise AssertionError('delivery log was mutable') + print('PASS hanging/500 targets never delay later actions, bounded retries, append-only rows, status and stable redelivery', flush=True) + + disconnected = copy.deepcopy(isolation_document) + with socket.socket() as reserved: + reserved.bind(('127.0.0.1', 0)) + refused_port = reserved.getsockname()[1] + disconnected['targets']['entries']['url'] = f'http://127.0.0.1:{refused_port}/actions' + disconnected['delivery'].pop('retry_backoff_ms') + disconnected_routes = save_routes('disconnected', disconnected) + exits.clear() + disconnected_ledger = root / 'disconnected.sqlite' + run(disconnected_ledger, disconnected_routes) + initial_starts = query(disconnected_ledger, "SELECT started_at FROM delivery_log WHERE phase='started' AND attempt=1") + assert max(row[0] for row in initial_starts) - min(row[0] for row in initial_starts) < 900 + failed = query(disconnected_ledger, "SELECT event_id,attempt,started_at FROM delivery_log WHERE phase='completed' AND target_id='entries' ORDER BY log_id") + for event_id in {row[0] for row in failed}: + timeline = [row[2] for row in failed if row[0] == event_id] + assert len(timeline) == 3 and timeline[1] - timeline[0] >= 900 and timeline[2] - timeline[1] >= 1900 + print('PASS refused connections capped at two retries with default 1s/2s backoff and isolated healthy targets', flush=True) + + deterministic = copy.deepcopy(isolation_document) + deterministic['default_target'] = 'main' + deterministic['rules'] = [] + determinism_routes = save_routes('determinism', deterministic) + snapshots = [] + for mode in ('ok', 'hang', 'fail', 'reset'): + main_receiver.clear(); main_receiver.mode = mode + computation = root / f'compute-{mode}.sqlite' + run(computation, determinism_routes) + snapshots.append(stored(computation)) + main_port = main_receiver.server.server_port + main_receiver.close() + run(root / 'compute-absent.sqlite', determinism_routes) + snapshots.append(stored(root / 'compute-absent.sqlite')) + main_receiver = Receiver(secrets['MAIN_HMAC'], main_port) + receivers[0] = main_receiver + assert all(snapshot == snapshots[0] for snapshot in snapshots) + print('PASS identical actions, state hashes, ledger inputs and exact payload bytes for fast/slow/failing/absent receivers', flush=True) + + crash_document = copy.deepcopy(isolation_document) + crash_document['targets']['main']['url'] = main_receiver.url + crash_document['rules'] = [] + crash_document['delivery']['max_in_flight'] = 1 + crash_document['delivery']['total_timeout_ms'] = 5000 + crash_document['delivery']['transport_retries'] = 0 + crash_routes = save_routes('crash', crash_document) + crash_ledger = root / 'crash.sqlite' + main_receiver.mode = 'hang' + process = subprocess.Popen([runner] + base + ['--ledger', str(crash_ledger), '--webhook-routes', str(crash_routes)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=environment) + try: + def committed(): + if not crash_ledger.exists(): + return False + try: + return query(crash_ledger, 'SELECT count(*) FROM events')[0][0] == 4 and bool(main_receiver.rows) + except sqlite3.OperationalError: + return False + wait_for(committed) + assert query(crash_ledger, "SELECT count(*) FROM delivery_log WHERE phase='completed'")[0][0] == 0 + assert query(crash_ledger, "SELECT count(*) FROM delivery_log WHERE phase='started'")[0][0] == 1 + before_crash = stored(crash_ledger) + process.kill() + process.communicate(timeout=5) + assert process.returncode == -signal.SIGKILL + finally: + if process.poll() is None: + process.kill(); process.wait(timeout=5) + main_receiver.clear(); main_receiver.mode = 'ok' + run(crash_ledger, crash_routes) + assert [row['action']['sequence'] for row in main_receiver.rows] == [1, 2, 3, 4] + assert stored(crash_ledger) == before_crash + run(crash_ledger, crash_routes) + assert len(main_receiver.rows) == 4 + print('PASS SIGKILL between commit/send and mid-request resumes each unsent action once after replay', flush=True) + + deployment = query(crash_ledger, 'SELECT identity FROM metadata')[0][0] + assert 'requires --deployment' in invoke(['redeliver', '--ledger', str(crash_ledger), '--target', 'main'], + expected=1, auto_deployment=False).stderr + for command in ('redeliver', 'actions', 'status'): + arguments = [command, '--ledger', str(crash_ledger), '--deployment', 'wrong'] + if command == 'redeliver': + arguments += ['--target', 'main'] + assert 'identity mismatch' in invoke(arguments, expected=1).stderr + entries.mode = 'fail' + chosen_environment = dict(environment) + chosen_environment.pop('MAIN_HMAC'); chosen_environment.pop('EXIT_HMAC') + failed_redelivery = json.loads(invoke(['redeliver', '--ledger', str(isolation_ledger), '--target', 'entries'], + expected=2, env=chosen_environment).stdout) + assert failed_redelivery == {'selected': 2, 'delivered': 0, 'failed': 2, 'pending': 0} + entries.mode = 'ok' + print('PASS deployment checks, failed redelivery exit/counts, selected-target secrets and saturation commit order', flush=True) + + stdin_base = list(base) + stdin_base[stdin_base.index('--feed') + 1] = '-' + fatal_document = copy.deepcopy(isolation_document) + fatal_document['delivery'].update(max_in_flight=1, total_timeout_ms=2000) + fatal_routes = save_routes('fatal', fatal_document) + for mode in ('ok', 'hang'): + entries.clear(); exits.clear(); entries.mode = exits.mode = mode + fatal_ledger = root / f'fatal-{mode}.sqlite' + fatal = subprocess.Popen([runner] + stdin_base + ['--ledger', str(fatal_ledger), '--webhook-routes', str(fatal_routes)], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, env=environment) + try: + fatal.stdin.write(feed.read_text()); fatal.stdin.flush() + wait_for(lambda: committed_actions(fatal_ledger)) + started = time.monotonic() + stdout, stderr = fatal.communicate(input='malformed\n', timeout=5) + assert fatal.returncode == 1 and time.monotonic() - started < 2.4, stderr + status = json.loads(invoke(['status', '--ledger', str(fatal_ledger)]).stdout)['targets'] + pending = sum(target['unsent'] for target in status.values()) + completed = query(fatal_ledger, "SELECT count(DISTINCT event_id) FROM delivery_log WHERE phase='completed'")[0][0] + assert completed + pending == 4 + assert f'{pending} actions not sent' in stderr.splitlines()[-1] + if pending: + assert 'pineforge-live redeliver --ledger' in stderr.splitlines()[-1] + assert '--deployment' in stderr.splitlines()[-1] and '--target' in stderr.splitlines()[-1] + assert all(row[0] <= 1 for row in query(fatal_ledger, 'SELECT attempts FROM events')) + finally: + if fatal.poll() is None: + fatal.kill(); fatal.communicate(timeout=5) + print('PASS fatal malformed feed drains within one global timeout without retries and reports every unsent action in status', flush=True) + + entries.mode = exits.mode = 'ok' + main_receiver.clear(); main_receiver.mode = 'hang' + signal_ledger = root / 'signal.sqlite' + active = subprocess.Popen([runner] + base + ['--ledger', str(signal_ledger), '--webhook-routes', str(crash_routes)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=environment) + try: + wait_for(lambda: committed_actions(signal_ledger) and bool(main_receiver.rows)) + refusal = invoke(['redeliver', '--ledger', str(signal_ledger), '--target', 'main'], expected=1) + assert 'the runner is running: stop it first; it resumes from its ledger' in refusal.stderr + started = time.monotonic(); active.send_signal(signal.SIGTERM) + active.communicate(timeout=3) + assert active.returncode == 130 and time.monotonic() - started < 0.75 + finally: + if active.poll() is None: + active.kill(); active.communicate(timeout=5) + main_receiver.clear() + redelivery = subprocess.Popen([runner, 'redeliver', '--ledger', str(signal_ledger), '--target', 'main', + '--deployment', query(signal_ledger, 'SELECT identity FROM metadata')[0][0]], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=environment) + try: + wait_for(lambda: bool(main_receiver.rows)) + started = time.monotonic(); redelivery.send_signal(signal.SIGTERM) + stdout, stderr = redelivery.communicate(timeout=3) + assert redelivery.returncode == 130 and time.monotonic() - started < 0.75 + assert json.loads(stdout)['pending'] == 4 + finally: + if redelivery.poll() is None: + redelivery.kill(); redelivery.communicate(timeout=5) + print('PASS SIGTERM promptly cancels EOF drain and redelivery; live redelivery is explicitly refused', flush=True) + + for receiver in receivers: + assert not receiver.errors, receiver.errors + print('PASS native webhook routing E2E', flush=True) + finally: + for receiver in receivers: + receiver.close() diff --git a/tests/native_live_startup_e2e.py b/tests/native_live_startup_e2e.py index b6b5eba23..d6562739b 100644 --- a/tests/native_live_startup_e2e.py +++ b/tests/native_live_startup_e2e.py @@ -37,7 +37,7 @@ def native_config(path, **clock): 'run': {'session_key': clock.pop('session_key', 'live-1'), 'run_number': clock.pop('run_number', 1)}, 'clock': { - 'input_tf': clock.pop('input_tf', '5'), + 'input_tf': clock.pop('input_tf', '1'), 'script_tf': clock.pop('script_tf', '10'), 'timezone': clock.pop('timezone', 'UTC'), 'session': clock.pop('session', '24x7'), @@ -96,9 +96,9 @@ def base_cmd(strategy, warmup, ledger, config=None, extra=None): warmup1m = root / 'w1m.csv' warmup_csv(warmup1m, [0, 60000, 120000]) warmup5 = root / 'w5.csv' - warmup_csv(warmup5, [0, 300000, 600000, 900000]) + warmup_csv(warmup5, list(range(0, 1200000, 60000))) warmup5_gap = root / 'w5gap.csv' - warmup_csv(warmup5_gap, [0, 300000, 900000]) + warmup_csv(warmup5_gap, [0, 60000, 180000]) cfg = root / 'native.json' native_config(cfg) @@ -144,11 +144,11 @@ def base_cmd(strategy, warmup, ledger, config=None, extra=None): # Explicit CLI contradiction never binds. ledger = root / 'cli.sqlite3' - p = invoke(base_cmd(stub, warmup5, ledger, cfg, extra=['--input-tf', '1']), success=False) + p = invoke(base_cmd(stub, warmup5, ledger, cfg, extra=['--input-tf', '5']), success=False) assert 'contradicts' in p.stderr assert not ledger_bound(ledger) - # Omitted CLI clock values take the file (default 1m is not a contradiction). + # Omitted CLI clock values take the file. ledger = root / 'omit.sqlite3' result = invoke(base_cmd(stub, warmup5, ledger, cfg)) assert ledger_bound(ledger) @@ -182,7 +182,7 @@ def base_cmd(strategy, warmup, ledger, config=None, extra=None): assert identity_of(ledger) == first_identity other_warmup = root / 'w5b.csv' - warmup_csv(other_warmup, [0, 300000, 600000, 900000, 1200000]) + warmup_csv(other_warmup, list(range(0, 1260000, 60000))) p = invoke(base_cmd(stub, other_warmup, ledger, cfg), success=False) assert 'identity' in p.stderr assert identity_of(ledger) == first_identity @@ -193,14 +193,15 @@ def base_cmd(strategy, warmup, ledger, config=None, extra=None): assert ledger_bound(legacy_ledger) ident = identity_of(legacy_ledger) assert len(ident) == 64 - # Changing a legacy input changes identity; a misaligned 1-minute warmup with - # a native config is refused by the native stream preflight. + # Changing a legacy input changes identity; wider native input is refused. p = invoke(base_cmd(absent, warmup1m, legacy_ledger, extra=['--input', 'changed=1']), success=False) assert 'identity' in p.stderr - p = invoke(base_cmd(legacy_library, warmup1m, root / 'legacy-real-nativecfg.sqlite3', cfg), + wider_cfg = root / 'wider-native.json' + native_config(wider_cfg, input_tf='5') + p = invoke(base_cmd(legacy_library, warmup1m, root / 'legacy-real-nativecfg.sqlite3', wider_cfg), success=False) - assert 'native stream refused' in p.stderr, p.stderr + assert 'input-tf currently must be 1 minute' in p.stderr, p.stderr assert not ledger_bound(root / 'legacy-real-nativecfg.sqlite3') # Real native example: nonempty physical actions, durable delivery failure, @@ -237,24 +238,24 @@ def log_message(self, *args): try: example_ledger = root / 'example.sqlite3' example_warmup = root / 'native-example-warmup.csv' - warmup_csv(example_warmup, [0, 300000]) + warmup_csv(example_warmup, list(range(0, 600000, 60000))) example_feed = root / 'native-example.jsonl' feed_rows = [{'type': 'bar', 'bar': { - 'ts_open': 600000 + i * 300000, 'o': 100, 'h': 102, 'l': 99, 'c': 101, 'v': 4, - }} for i in range(8)] + 'ts_open': 600000 + i * 60000, 'o': 100, 'h': 102, 'l': 99, 'c': 101, 'v': 0.8, + }} for i in range(40)] example_feed.write_text(''.join(json.dumps(row) + '\n' for row in feed_rows)) example_cmd = [runner, 'run', '--strategy', native_example, '--warmup', str(example_warmup), '--mode', 'bars', '--ledger', str(example_ledger), '--webhook-url', f'http://127.0.0.1:{server.server_port}/webhook', '--allow-insecure-http', '--feed', str(example_feed), '--native-config', str(cfg), '--name', 'native-example'] - failed_delivery = invoke(example_cmd + ['--max-attempts', '1'], success=False) - assert 'webhook retry limit reached; queued event remains in ledger' in failed_delivery.stderr, failed_delivery.stderr + failed_delivery = invoke(example_cmd + ['--max-attempts', '1']) + assert failed_delivery['webhook_failures'] == 2, failed_delivery failed_rows = ledger_rows(example_ledger) - assert len(received) == 1, received - assert len(failed_rows['events']) == 1, failed_rows - assert failed_rows['events'][0][5:] == (1, 0), failed_rows - assert 0 < len(failed_rows['inputs']) < len(feed_rows), failed_rows + assert len(received) == 2, received + assert len(failed_rows['events']) == 2, failed_rows + assert all(row[5:] == (1, 0) for row in failed_rows['events']), failed_rows + assert len(failed_rows['inputs']) == len(feed_rows), failed_rows assert all(row[2].isdigit() and int(row[2]) != 0 for row in failed_rows['inputs']) failed_payload = received_bytes[0] assert failed_rows['events'][0][4] == failed_payload @@ -263,15 +264,20 @@ def log_message(self, *args): example = invoke(example_cmd) assert example['inputs_committed'] == len(feed_rows), example assert example['prefix_skipped'] == len(failed_rows['inputs']), example + assert len(received) == 2, received + redelivery = invoke([runner, 'redeliver', '--ledger', str(example_ledger), + '--deployment', identity_of(example_ledger), + '--target', 'default', '--failed-only']) + assert redelivery['delivered'] == 2, redelivery committed_rows = ledger_rows(example_ledger) assert len(committed_rows['events']) == 2, committed_rows - assert len(received) == 3, received - assert received_bytes[0] == received_bytes[1] == failed_payload + assert len(received) == 4, received + assert received_bytes[0] == received_bytes[2] == failed_payload assert committed_rows['inputs'][:len(failed_rows['inputs'])] == failed_rows['inputs'] assert committed_rows['events'][0][:5] == failed_rows['events'][0][:5] assert committed_rows['events'][0][5:] == (2, 1), committed_rows - assert committed_rows['events'][1][5:] == (1, 1), committed_rows - assert [row[4] for row in committed_rows['events']] == received_bytes[1:] + assert committed_rows['events'][1][5:] == (2, 1), committed_rows + assert [row[4] for row in committed_rows['events']] == received_bytes[2:] assert len({row[3] for row in committed_rows['events']}) == 2 actions = [json.loads(row[4]) for row in committed_rows['events']] assert [(a['timestamp'], a['order']['action'], a['order']['contracts'], diff --git a/tests/native_live_websocket_e2e.py b/tests/native_live_websocket_e2e.py new file mode 100644 index 000000000..868e96d64 --- /dev/null +++ b/tests/native_live_websocket_e2e.py @@ -0,0 +1,181 @@ +"""WebSocket finality through the real CLI, SQLite ledger and loopback webhook.""" +import base64 +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import sqlite3 +import struct +import subprocess +import sys +import tempfile +import threading +import time + + +runner, library = sys.argv[1:3] +expect_refusal = sys.argv[3:] == ['--expect-refusal'] +assert not sys.argv[3:] or expect_refusal +received = [] +stream_frames = [] +stream_idle = False +final_frame_sent = threading.Event() + + +def frame(opcode, payload, final=True): + header = bytes([opcode | (0x80 if final else 0)]) + if len(payload) < 126: + return header + bytes([len(payload)]) + payload + return header + bytes([126]) + struct.pack('!H', len(payload)) + payload + + +class Receiver(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def do_GET(self): + frames = list(stream_frames) + idle = stream_idle + accept = base64.b64encode(hashlib.sha1( + (self.headers['Sec-WebSocket-Key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11').encode() + ).digest()).decode() + self.send_response(101) + self.send_header('Upgrade', 'websocket') + self.send_header('Connection', 'Upgrade') + self.send_header('Sec-WebSocket-Accept', accept) + self.end_headers() + self.close_connection = True + try: + for data in frames: + if data[0] & 0x80 and data[0] & 0x0f in (0, 1): + final_frame_sent.set() + self.wfile.write(data) + self.wfile.flush() + time.sleep(0.02) + if idle: + self.connection.settimeout(20) + while self.connection.recv(4096): + pass + except OSError: + pass + + def do_POST(self): + payload = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + received.append({'payload': payload, 'before_final': not final_frame_sent.is_set()}) + self.send_response(200) + self.send_header('Content-Length', '0') + self.end_headers() + + def log_message(self, *args): + pass + + +def ledger_counts(path): + if not path.exists(): + return 0, 0 + with sqlite3.connect(path) as database: + tables = {row[0] for row in database.execute('SELECT name FROM sqlite_master WHERE type="table"')} + inputs = database.execute('SELECT count(*) FROM inputs').fetchone()[0] if 'inputs' in tables else 0 + events = database.execute('SELECT count(*) FROM events').fetchone()[0] if 'events' in tables else 0 + return inputs, events + + +server = ThreadingHTTPServer(('127.0.0.1', 0), Receiver) +server.daemon_threads = True +thread = threading.Thread(target=server.serve_forever, daemon=True) +thread.start() +payload = json.dumps({'type': 'bar', 'bar': { + 'ts_open': 120000, 'o': 100, 'h': 102, 'l': 99, 'c': 101, 'v': 4 +}}).encode() +cases = [ + ('fin0-eof', [frame(1, payload, False)], False, False, 'truncated text message'), + ('fin0-close', [frame(1, payload, False), frame(8, b'')], False, False, 'closed during text message'), + ('fin0-idle', [frame(1, payload, False)], True, False, 'idle or message timeout'), + ('unfinished-fragments-ping', [frame(1, payload, False), frame(9, b'alive'), + frame(0, b' ', False), frame(9, b'still alive')], + False, False, 'truncated text message'), + ('binary', [frame(2, payload)], False, False, 'ordered text frames'), + ('new-text-before-final', [frame(1, payload, False), frame(1, b' ')], + False, False, 'truncated text message'), + ('partial-final-frame', [bytes([0x81, 126]) + struct.pack('!H', len(payload) + 10) + payload], + False, False, 'truncated text message'), + ('valid-fragments-ping', [frame(1, payload[:40], False), frame(9, b'alive'), + frame(0, payload[40:80], False), frame(10, b'pong'), + frame(0, payload[80:])], False, True, ''), + ('valid-empty-final', [frame(1, payload, False), frame(9, b'alive'), frame(0, b'')], + False, True, ''), +] +failures = [] +unsupported = False +try: + with tempfile.TemporaryDirectory(prefix='native-ws-finality-') as directory: + root = Path(directory) + warmup = root/'warmup.csv' + warmup.write_text('timestamp,open,high,low,close,volume\n' + '0,100,102,99,101,4\n60000,100,102,99,101,4\n') + base = [runner, 'run', '--strategy', library, '--warmup', str(warmup), + '--script-tf', '1', '--mode', 'bars', '--symbol', 'TEST:ETH', + '--webhook-url', f'http://127.0.0.1:{server.server_port}/actions', + '--allow-insecure-http', '--max-events', '1'] + for name, stream_frames, stream_idle, success, diagnostic in cases: + received.clear() + final_frame_sent.clear() + ledger = root/(name + '.sqlite3') + command = base + ['--ledger', str(ledger), '--feed-url', + f'ws://127.0.0.1:{server.server_port}/stream'] + started = time.monotonic() + result = subprocess.run(command, capture_output=True, text=True, timeout=30) + elapsed = time.monotonic() - started + inputs, events = ledger_counts(ledger) + no_support = 'libcurl build with WS/WSS support enabled' in result.stderr + unsupported = unsupported or no_support + refused = expect_refusal or no_support + print(json.dumps({'case': name, 'returncode': result.returncode, 'inputs': inputs, + 'events': events, 'webhooks': len(received), 'ledger_exists': ledger.exists(), + 'seconds': round(elapsed, 2), 'stderr': result.stderr.strip()}), flush=True) + try: + if refused: + assert result.returncode != 0 + assert ('libcurl 8.14.1 or newer' in result.stderr or no_support) + assert not ledger.exists() + assert inputs == events == len(received) == 0 + elif success: + assert result.returncode == 0, (result.stdout, result.stderr) + assert inputs == events == len(received) == 1 + assert json.loads(result.stdout)['inputs_committed'] == 1 + assert received[0]['payload']['order']['action'] == 'buy' + assert not received[0]['before_final'] + else: + assert result.returncode != 0 + assert inputs == events == len(received) == 0 + assert diagnostic in result.stderr + if stream_idle: + assert elapsed >= 14 + except AssertionError as error: + failures.append(name) + print(f'FAIL {name}: {error}', flush=True) + + if expect_refusal or unsupported: + sentinel = root/'existing.sqlite3' + sentinel.write_bytes(b'ledger must stay byte-identical') + before = sentinel.read_bytes() + for scheme in ('ws', 'wss'): + result = subprocess.run(base + ['--ledger', str(sentinel), '--feed-url', + f'{scheme}://127.0.0.1:{server.server_port}/stream'], + capture_output=True, text=True, timeout=10) + assert result.returncode != 0 + assert 'libcurl' in result.stderr + assert sentinel.read_bytes() == before + assert not Path(str(sentinel) + '-wal').exists() + assert not Path(str(sentinel) + '-shm').exists() + assert not received + print('WebSocket startup refuses ws/wss before touching an existing ledger', flush=True) +finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + +if failures: + raise SystemExit('WebSocket finality failures: ' + ', '.join(failures)) +print('native WebSocket CLI finality: all negative inputs/outbox/webhooks empty; ' + + ('startup refusal verified' if expect_refusal or unsupported else 'both fragmented positives committed once')) +raise SystemExit(77 if unsupported and not expect_refusal else 0) diff --git a/tests/test_live_parser.cpp b/tests/test_live_parser.cpp deleted file mode 100644 index 4fd70825c..000000000 --- a/tests/test_live_parser.cpp +++ /dev/null @@ -1,242 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Build this file normally for the test executable; also build it as two MODULE -// libraries with PF_LIVE_PARSER_TEST_PLUGIN, one additionally defining -// PF_LIVE_PARSER_TEST_ABI=2. argv: demo_plugin fixture_plugin wrong_abi_plugin. -#include - -#include -#include -#include -#include - -#ifdef PF_LIVE_PARSER_TEST_PLUGIN - -#ifndef PF_LIVE_PARSER_TEST_ABI -#define PF_LIVE_PARSER_TEST_ABI PF_LIVE_PARSER_ABI_VERSION -#endif - -extern "C" PF_LIVE_PARSER_API uint32_t pf_live_parser_abi_version(void) { - return PF_LIVE_PARSER_TEST_ABI; -} - -extern "C" PF_LIVE_PARSER_API int pf_live_parse_message( - const char* message, size_t message_size, const char* config, size_t config_size, - pf_live_parser_emit_v1_fn emit, void* user) { - const std::string_view text(message, message_size); - if (text.empty() || text == "heartbeat") return 0; - if (message_size == PF_LIVE_PARSER_MAX_MESSAGE_BYTES) return 0; - if (text == "throw") throw std::runtime_error("provider-secret-not-for-error-output"); - pf_live_parser_event_v1_t tick{}; - tick.kind = PF_LIVE_PARSER_TICK; - tick.timestamp = 60000; - tick.sequence = 18446744073709551615ULL; - tick.price = 100; - tick.quantity = 1; - pf_live_parser_event_v1_t bar{}; - bar.kind = PF_LIVE_PARSER_BAR; - bar.timestamp = 60000; - bar.open = 100; - bar.high = 102; - bar.low = 99; - bar.close = 101; - bar.volume = 3; - pf_live_parser_event_v1_t time{}; - time.kind = PF_LIVE_PARSER_TIME; - time.timestamp = 120000; - if (text == "batch") { - for (const auto* event : {&tick, &bar, &time}) if (emit(event, user)) return -1; - return 0; - } - if (text == "two-trades") { - tick.timestamp=180001;tick.sequence=1; - if(emit(&tick,user)) return -1; - tick.timestamp=180002;tick.sequence=2;tick.price=101; - return emit(&tick,user); - } - if (text == "config") { - if (std::string_view(config, config_size) != "{\"scale\":2}") return -1; - tick.price = 200; - return emit(&tick, user); - } - if (text == "partial-fail") { if (emit(&tick, user)) return -1; return -1; } - if (text == "partial-invalid") { - if (emit(&tick, user)) return -1; - bar.low = 200; - return emit(&bar, user); - } - if (text == "positive-return") { if (emit(&tick, user)) return -1; return 1; } - if (text == "null") return emit(nullptr, user); - if (text == "limit" || text == "overflow") { - const size_t count = PF_LIVE_PARSER_MAX_EVENTS + (text == "overflow" ? 1 : 0); - for (size_t i = 0; i < count; ++i) { - tick.sequence = i + 1; - // Deliberately broken plugin: ignore callback failure. The host - // must retain its refusal even if the plugin claims success. - (void)emit(&tick, user); - } - return 0; - } - if (text == "bad-kind") tick.kind = 999; - else if (text == "reserved") tick.reserved = 1; - else if (text == "negative-time") tick.timestamp = -1; - else if (text == "zero-sequence") tick.sequence = 0; - else if (text == "zero-price") tick.price = 0; - else if (text == "zero-quantity") tick.quantity = 0; - else if (text == "nan-price") tick.price = std::numeric_limits::quiet_NaN(); - else if (text == "inf-quantity") tick.quantity = std::numeric_limits::infinity(); - else if (text == "ignored-fields") { - tick.open = std::numeric_limits::quiet_NaN(); - tick.volume = 999; - } else if (text == "bar-negative-zero") { bar.volume = -0.0; return emit(&bar, user); } - else if (text == "bar-zero-low") { bar.low = 0; return emit(&bar, user); } - else if (text == "bar-inverted") { bar.high = 90; return emit(&bar, user); } - else if (text == "bar-infinite") { bar.volume = std::numeric_limits::infinity(); return emit(&bar, user); } - else if (text == "bar-negative-volume") { bar.volume = -1; return emit(&bar, user); } - else if (text == "bar-unaligned") { bar.timestamp = 1; return emit(&bar, user); } - else if (text == "bar-overflow") { - bar.timestamp = std::numeric_limits::max() / 60000 * 60000; - return emit(&bar, user); - } else return -1; - (void)emit(&tick, user); // Verify host refuses even non-propagated failure. - return 0; -} - -#else - -#include "parser.hpp" -#include -#include -#include -#include -#include -#include - -namespace { -int failures = 0; -#define CHECK(condition) do { if (!(condition)) { \ - std::fprintf(stderr, "FAIL %s:%d %s\n", __FILE__, __LINE__, #condition); ++failures; \ -} } while (0) - -bool refuses(const std::function& action) { - try { action(); } - catch (const std::runtime_error&) { return true; } - return false; -} - -bool same(const pineforge::live::ParsedEvent& a, const pineforge::live::ParsedEvent& b) { - return a.kind == b.kind && a.reserved == b.reserved && a.timestamp == b.timestamp - && a.sequence == b.sequence && a.price == b.price && a.quantity == b.quantity - && a.open == b.open && a.high == b.high && a.low == b.low - && a.close == b.close && a.volume == b.volume; -} - -int reject_event(const pf_live_parser_event_v1_t*, void* count) { - ++*static_cast(count); - return -1; -} -} - -int main(int argc, char** argv) { - using pineforge::live::Parser; - static_assert(std::is_standard_layout::value, "C ABI layout"); - static_assert(std::is_trivial::value, "C ABI POD"); - if (argc != 4) { - std::fprintf(stderr, "usage: test_live_parser demo_plugin fixture_plugin wrong_abi_plugin\n"); - return 2; - } - try { - CHECK(refuses([&] { Parser missing(std::string(argv[1]) + ".missing"); })); - CHECK(refuses([&] { Parser wrong(argv[3]); })); - CHECK(refuses([&] { Parser bad("", "{}"); })); - for (const auto* config : {"", "[]", "null", "{", "{\"secret\":", "{} trailing"}) - CHECK(refuses([&] { Parser bad(argv[1], config); })); - CHECK(refuses([&] { Parser bad(argv[1], std::string(PF_LIVE_PARSER_MAX_MESSAGE_BYTES + 1, ' ')); })); - - Parser demo(argv[1]); - const std::string good = "trade,18446744073709551615,60000,100.25,0.5"; - const auto parsed = demo.parse(good); - CHECK(parsed.size() == 1); - if (parsed.size() == 1) { - CHECK(parsed[0].kind == PF_LIVE_PARSER_TICK); - CHECK(parsed[0].timestamp == 60000); - CHECK(parsed[0].sequence == std::numeric_limits::max()); - CHECK(parsed[0].price == 100.25 && parsed[0].quantity == 0.5); - CHECK(parsed[0].open == 0 && parsed[0].volume == 0 && parsed[0].reserved == 0); - const auto repeated = demo.parse(good); - CHECK(repeated.size() == 1 && same(parsed[0], repeated[0])); - } - CHECK(demo.parse("heartbeat").empty()); - for (const auto* bad : {"", "trade", "trade,1,60000,100", "trade,1,60000,100,1,extra", - "trade,0,60000,100,1", "trade,1,-1,100,1", "trade,1,60000,0,1", - "trade,1,60000,100,-1", "trade,1,60000,nan,1", "trade,1,60000,100,inf", - "trade,18446744073709551616,60000,100,1", "trade,1,9223372036854775808,100,1", - "trade,1,60000,1e999,1", "trade,1,60000,100,1x", "trade,1,60000,100, 1"}) - CHECK(refuses([&] { (void)demo.parse(bad); })); - CHECK(refuses([&] { (void)demo.parse(good + std::string(1, '\0')); })); - CHECK(refuses([&] { (void)demo.parse(std::string(PF_LIVE_PARSER_MAX_MESSAGE_BYTES + 1, 'x')); })); - - // Length-delimited input must not read unrelated trailing bytes. - const auto buffer = good + ",unrelated"; - CHECK(demo.parse(std::string_view(buffer.data(), good.size())).size() == 1); - - Parser fixture(argv[2]); - const auto batch = fixture.parse("batch"); - CHECK(batch.size() == 3); - if (batch.size() == 3) { - CHECK(batch[0].kind == PF_LIVE_PARSER_TICK); - CHECK(batch[1].kind == PF_LIVE_PARSER_BAR && batch[1].high == 102 && batch[1].volume == 3); - CHECK(batch[2].kind == PF_LIVE_PARSER_TIME && batch[2].timestamp == 120000); - } - CHECK(fixture.parse(std::string_view{}).empty()); - CHECK(fixture.parse(std::string(PF_LIVE_PARSER_MAX_MESSAGE_BYTES, 'x')).empty()); - CHECK(fixture.parse("limit").size() == PF_LIVE_PARSER_MAX_EVENTS); - for (const auto* bad : {"partial-fail", "partial-invalid", "positive-return", "null", "overflow", - "bad-kind", "reserved", "negative-time", "zero-sequence", "zero-price", "zero-quantity", - "nan-price", "inf-quantity", "bar-zero-low", "bar-inverted", "bar-infinite", - "bar-negative-volume", "bar-unaligned", "bar-overflow", "config"}) { - CHECK(refuses([&] { (void)fixture.parse(bad); })); - // Rejected staged output cannot leak into a later call. - const auto recovered = fixture.parse("batch"); - CHECK(recovered.size() == batch.size()); - for (size_t i = 0; i < recovered.size() && i < batch.size(); ++i) - CHECK(same(recovered[i], batch[i])); - } - try { (void)fixture.parse("throw"); CHECK(false); } - catch (const std::runtime_error& error) { - CHECK(std::string(error.what()) == "parser plugin threw an exception"); - } - const auto normalized = fixture.parse("ignored-fields"); - CHECK(normalized.size() == 1 && normalized[0].open == 0 && normalized[0].volume == 0); - const auto zero = fixture.parse("bar-negative-zero"); - CHECK(zero.size() == 1 && zero[0].volume == 0 && !std::signbit(zero[0].volume)); - Parser configured(argv[2], "{\"scale\":2}"); - const auto changed = configured.parse("config"); - CHECK(changed.size() == 1 && changed[0].price == 200); - Parser moved(std::move(configured)); - CHECK(refuses([&] { (void)configured.parse("batch"); })); - CHECK(moved.parse("config").size() == 1); - moved = std::move(demo); - CHECK(moved.parse(good).size() == 1); - CHECK(refuses([&] { (void)demo.parse(good); })); - - // The example must itself obey callback rejection, independently of - // the host's defensive failure latch. - void* library = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); - CHECK(library != nullptr); - if (library) { - const auto parse = reinterpret_cast(dlsym(library, "pf_live_parse_message")); - CHECK(parse != nullptr); - if (parse) { - int count = 0; - CHECK(parse(good.data(), good.size(), "{}", 2, reject_event, &count) == -1); - CHECK(count == 1); - } - dlclose(library); - } - } catch (const std::exception& error) { - std::fprintf(stderr, "unexpected parser test failure: %s\n", error.what()); - return 1; - } - return failures ? 1 : 0; -} -#endif diff --git a/tests/test_native_live_routing.cpp b/tests/test_native_live_routing.cpp new file mode 100644 index 000000000..5fe0c271c --- /dev/null +++ b/tests/test_native_live_routing.cpp @@ -0,0 +1,221 @@ +#include "routing.hpp" +#include "store.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +using namespace pineforge::live; + +namespace { +const std::string routes = R"({"schema_version":1,"default_target":"main","targets":{ +"main":{"url":"https://receiver.example/default","secret_env":"TEST_DEFAULT"}, +"entries":{"url":"https://receiver.example/entries","secret_env":"TEST_ENTRIES"}}, +"rules":[{"match":{"order_id":"Long","kind":"entry","side":"long"},"target":"entries"}, +{"match":{"order_id":"Long"},"target":null},{"match":{"side":"short"},"target":"main"}]})"; + +template +void refuses(Callable&& callable, const std::string& expected = "") { + bool refused = false; + try { callable(); } + catch (const std::exception& error) { + refused = true; + assert(expected.empty() || std::string(error.what()).find(expected) != std::string::npos); + } + assert(refused); +} + +void configuration() { + const auto config = parse_routes(routes, false); + assert(config.routed); + assert(config.select("Long", "entry", "long") == "entries"); + assert(!config.select("Long", "exit", "long")); + assert(config.select("Other", "entry", "short") == "main"); + assert(config.select("long", "entry", "long") == "main"); + assert(config.delivery.max_in_flight == 8); + assert(config.delivery.connect_timeout_ms == 2000); + assert(config.delivery.total_timeout_ms == 5000); + assert(config.delivery.transport_retries == 2); + assert((config.delivery.retry_backoff_ms == std::vector{1000, 2000})); + assert(restore_routes(config.stored_document()).stored_document() == config.stored_document()); + assert(!single_target("", "", false).default_target); + assert(single_target("https://receiver.example/", "", false).default_target == "default"); + assert(delivery_identity("event", "main") == sha256_hex("{\"event_id\":\"event\",\"target_id\":\"main\"}")); + assert(delivery_identity("event", "main") != delivery_identity("event", std::nullopt)); + refuses([&] { parse_routes(routes, false, "https://wrong.example"); }, "agree"); + refuses([&] { parse_routes(routes, false, "", "WRONG_ENV"); }, "agree"); + for (const auto& selector : {R"({"alert_message":"message"})", R"({"kind":"close"})"}) { + auto document = parse_json(routes); + document.members["rules"].items[0].members["match"] = parse_json(selector); + refuses([&] { parse_routes(document.dump(), false); }, "strategy-metadata extension"); + } + for (const auto& selector : {R"({"kind":"ENTRY"})", R"({"side":"buy"})", R"({"order_id":3})", R"({"unknown":"x"})"}) { + auto document = parse_json(routes); + document.members["rules"].items[0].members["match"] = parse_json(selector); + refuses([&] { parse_routes(document.dump(), false); }); + } + for (const auto& delivery : {R"({"max_in_flight":0})", R"({"max_in_flight":1025})", + R"({"connect_timeout_ms":0})", R"({"total_timeout_ms":1000})", R"({"transport_retries":3})", + R"({"retry_backoff_ms":[1]})", R"({"retry_backoff_ms":[1,0]})", R"({"unknown":1})"}) { + auto document = parse_json(routes); + document.members["delivery"] = parse_json(delivery); + refuses([&] { parse_routes(document.dump(), false); }); + } + auto document = parse_json(routes); + document.members["default_target"] = Json::string("absent"); + refuses([&] { parse_routes(document.dump(), false); }, "undefined"); + for (const auto& url : {"http://receiver.example/", "https://user:secret@receiver.example/", "https://receiver.example/#fragment", "file:///tmp/feed"}) { + document = parse_json(routes); + document.members["targets"].members["main"].members["url"] = Json::string(url); + refuses([&] { parse_routes(document.dump(), false); }); + } + document = parse_json(routes); + document.members["delivery"] = parse_json(R"({"max_in_flight":3,"connect_timeout_ms":50,"total_timeout_ms":100,"transport_retries":1,"retry_backoff_ms":[10]})"); + const auto custom = parse_routes(document.dump(), false); + assert(custom.delivery.max_in_flight == 3 && custom.delivery.total_timeout_ms == 100); + assert(custom.delivery.transport_retries == 1 && custom.delivery.retry_backoff_ms[0] == 10); +} + +void ledger_audit() { + std::string pattern = (std::filesystem::temp_directory_path() / "pineforge-routing-XXXXXX").string(); + std::vector name(pattern.begin(), pattern.end()); name.push_back('\0'); + const char* directory = mkdtemp(name.data()); + assert(directory); + const auto path = std::string(directory) + "/ledger.sqlite"; + { + Ledger ledger(path, "deployment"); + ledger.bind_routing(parse_routes(routes, false).stored_document()); + ledger.commit_input(0, "{}", 123, {{"first", "{\"sequence\":1}", "main", "delivery-first"}, + {"second", "{\"sequence\":2}", std::nullopt, "delivery-second"}}); + assert(ledger.unsent_count() == 1); + const auto event = ledger.unsent_events(0).front(); + const auto attempt = ledger.start_attempt(event, 100); + assert(attempt.attempt == 1); + assert(ledger.unsent_count() == 1); + ledger.finish_attempt(attempt, 101, 500, false, "http_status"); + assert(ledger.unsent_count() == 0); + LedgerView view(path); + assert(view.actions_after(0).size() == 2); + assert(view.actions_after(1).front().target_id == std::nullopt); + assert(view.redelivery_events("main", 1, true).size() == 1); + const auto retry = ledger.start_attempt(event, 200); + assert(retry.attempt == 2 && retry.event.delivery_id == attempt.event.delivery_id); + ledger.finish_attempt(retry, 201, 204, true, ""); + assert(view.redelivery_events("main", 1, true).empty()); + const auto status = parse_json(view.status_json()).at("targets").at("main"); + assert(status.at("sent").integer() == 1 && status.at("failed").integer() == 1); + std::thread writer([&] { + for (std::uint64_t index = 1; index <= 50; ++index) + ledger.commit_input(index, "{}", index, {}); + }); + for (int iteration = 0; iteration < 50; ++iteration) { + assert(ledger.input(0)->state_hash == "123"); + assert(view.actions_after(0).size() == 2); + } + writer.join(); + assert(ledger.input_count() == 51); + ledger.commit_input(51, "{}", 51, {{"third", "{\"sequence\":3}", "main", "delivery-third"}, + {"fourth", "{\"sequence\":4}", "main", "delivery-fourth"}}); + const auto concurrent = ledger.unsent_events(2); + assert(concurrent.size() == 2); + const auto older = ledger.start_attempt(concurrent[0], 300); + const auto newer = ledger.start_attempt(concurrent[1], 400); + ledger.finish_attempt(newer, 401, 204, true, ""); + ledger.finish_attempt(older, 500, 500, false, "http_status"); + const auto unordered = parse_json(view.status_json()).at("targets").at("main"); + assert(unordered.at("last_attempt").integer() == 400); + assert(unordered.at("last_success").integer() == 401); + assert(unordered.at("last_error").at("at").integer() == 500); + assert(unordered.at("sent").integer() == 2 && unordered.at("failed").integer() == 2); + } + { + Ledger ledger(path, "deployment"); + assert(ledger.unsent_count() == 0); + } + sqlite3* database = nullptr; + assert(sqlite3_open(path.c_str(), &database) == SQLITE_OK); + assert(sqlite3_exec(database, "UPDATE delivery_log SET http_status=200", nullptr, nullptr, nullptr) != SQLITE_OK); + assert(sqlite3_exec(database, "DELETE FROM delivery_log", nullptr, nullptr, nullptr) != SQLITE_OK); + sqlite3_close(database); + std::filesystem::remove_all(directory); +} + +void incremental_scan_and_migration() { + std::string pattern = (std::filesystem::temp_directory_path() / "pineforge-scan-XXXXXX").string(); + std::vector name(pattern.begin(), pattern.end()); name.push_back('\0'); + const char* directory = mkdtemp(name.data()); + assert(directory); + const auto path = std::string(directory) + "/scan.sqlite"; + { + Ledger ledger(path, "scan"); + std::vector events; + for (unsigned index = 0; index < 4096; ++index) + events.push_back({"journal-" + std::to_string(index), "{}", std::nullopt, ""}); + events.push_back({"failed", "{}", "main", "failed-delivery"}); + ledger.commit_input(0, "{}", 1, events); + const auto failed = ledger.unsent_events(0).front(); + const auto attempt = ledger.start_attempt(failed, 1); + ledger.finish_attempt(attempt, 2, 500, false, "http_status"); + std::uint64_t low = 0, visited = 0, steps = 0; + while (const auto next = ledger.next_delivery_event(low, &steps)) { + assert(steps < 150); + assert(!next->unsent); + low = next->event.ordinal; + ++visited; + } + assert(visited == events.size()); + for (unsigned iteration = 0; iteration < 100; ++iteration) { + assert(!ledger.next_delivery_event(low, &steps)); + assert(steps < 25); + } + ledger.commit_input(1, "{}", 2, {{"new", "{}", "main", "new-delivery"}}); + const auto next = ledger.next_delivery_event(low, &steps); + assert(next && next->unsent && next->event.ordinal == low + 1 && steps < 150); + assert(!ledger.next_delivery_event(next->event.ordinal, &steps) && steps < 25); + } + const auto legacy = std::string(directory) + "/phase-a.sqlite"; + sqlite3* database = nullptr; + assert(sqlite3_open(legacy.c_str(), &database) == SQLITE_OK); + assert(sqlite3_exec(database, + "CREATE TABLE metadata(singleton INTEGER PRIMARY KEY,schema_version INTEGER,identity TEXT);" + "CREATE TABLE inputs(input_index INTEGER PRIMARY KEY,canonical_json TEXT,state_hash TEXT);" + "CREATE TABLE events(ordinal INTEGER PRIMARY KEY,input_index INTEGER,input_position INTEGER,event_id TEXT UNIQUE," + "payload TEXT,attempts INTEGER,acknowledged INTEGER,last_error TEXT);" + "INSERT INTO metadata VALUES(1,1,'phase-a'); INSERT INTO inputs VALUES(0,'{}','123');" + "INSERT INTO events VALUES(1,0,0,'ack','{\"sequence\":1}',2,1,'');" + "INSERT INTO events VALUES(2,0,1,'unack','{ \"sequence\" : 2 }',2,0,'timeout');", + nullptr, nullptr, nullptr) == SQLITE_OK); + sqlite3_close(database); + { + Ledger ledger(legacy, "phase-a"); + const auto events = ledger.input(0)->events; + assert(events.size() == 2 && events[0].id == "ack" && events[1].id == "unack"); + assert(events[1].payload == "{ \"sequence\" : 2 }"); + assert(ledger.unsent_count() == 1); + const auto pending = ledger.unsent_events(0).front(); + assert(pending.id == "unack" && pending.delivery_id == "unack" && pending.attempts == 2); + const auto attempt = ledger.start_attempt(pending, 3); + assert(attempt.attempt == 3); + ledger.finish_attempt(attempt, 4, 204, true, ""); + assert(ledger.unsent_count() == 0); + assert(ledger.input(0)->events[1].payload == events[1].payload); + LedgerView view(legacy); + const auto status = parse_json(view.status_json()).at("targets").at("default"); + assert(status.at("sent").integer() == 2 && status.at("unsent").integer() == 0); + } + std::filesystem::remove_all(directory); + std::cout << "PASS incremental delivery scan visits only new rows and phase-A three-table migration preserves bytes and attempts\n"; +} +} + +int main() { + configuration(); + ledger_audit(); + incremental_scan_and_migration(); + std::cout << "PASS native routing configuration, identity, append-only delivery audit and serialized ledger\n"; +} diff --git a/tests/test_native_live_startup.cpp b/tests/test_native_live_startup.cpp index 282156385..b9b39aa52 100644 --- a/tests/test_native_live_startup.cpp +++ b/tests/test_native_live_startup.cpp @@ -108,21 +108,18 @@ void legacy_identity_golden() { const std::string warmup = "timestamp,open,high,low,close,volume\n0,100,102,99,101,4\n"; const std::string library = "legacy-library-bytes"; - const std::string parser; - const std::string parser_config = "{}"; - const auto dump = - identity_document(fields, warmup, library, parser, parser_config); + const auto dump = identity_document(fields, warmup, library); const std::string expected = std::string("{\"chart_timezone\":\"UTC\",\"input_tf\":\"1\",\"inputs\":[],\"library\":\"") + sha256_hex(library) + "\",\"mode\":\"bars\",\"name\":\"strategy\",\"overrides\":[],\"parser\":\"" + - sha256_hex(parser) + "\",\"parser_config\":\"" + sha256_hex(parser_config) + + sha256_hex("") + "\",\"parser_config\":\"" + sha256_hex("{}") + "\",\"schema\":\"pineforge-native-ledger/v1\",\"script_tf\":\"3\",\"session\":\"24x7\"," "\"symbol\":\"TEST:MOCK\",\"syminfo\":{},\"timezone\":\"UTC\",\"warmup\":\"" + sha256_hex(warmup) + "\",\"webhook\":\"https://receiver.example/order-actions\"}"; CHECK(dump == expected); CHECK(dump.find("pineforge-native-run/v1") == std::string::npos); CHECK(dump.find("timezone_dependency") == std::string::npos); - CHECK(identity(fields, warmup, library, parser, parser_config) == sha256_hex(expected)); + CHECK(identity(fields, warmup, library) == sha256_hex(expected)); } void parse_refusals() { @@ -261,39 +258,30 @@ void native_identity_changes() { validate_native_config(n); const std::string warmup = "timestamp,open,high,low,close,volume\n0,100,102,99,101,4\n"; const std::string library = "lib-a"; - const auto base = native_identity(n, "bars", "strategy", "http://example/hook", warmup, library, - "", "{}"); + const auto base = native_identity(n, "bars", "strategy", "http://example/hook", warmup, library); auto other = n; other.run_number = 2; - CHECK(native_identity(other, "bars", "strategy", "http://example/hook", warmup, library, "", - "{}") != base); + CHECK(native_identity(other, "bars", "strategy", "http://example/hook", warmup, library) != base); other = n; other.session_key = "live-2"; - CHECK(native_identity(other, "bars", "strategy", "http://example/hook", warmup, library, "", - "{}") != base); - CHECK(native_identity(n, "ticks", "strategy", "http://example/hook", warmup, library, "", - "{}") != base); - CHECK(native_identity(n, "bars", "strategy", "http://example/hook", warmup, "lib-b", "", - "{}") != base); - CHECK(native_identity(n, "bars", "strategy", "http://example/hook", warmup + "x", library, "", - "{}") != base); + CHECK(native_identity(other, "bars", "strategy", "http://example/hook", warmup, library) != base); + CHECK(native_identity(n, "ticks", "strategy", "http://example/hook", warmup, library) != base); + CHECK(native_identity(n, "bars", "strategy", "http://example/hook", warmup, "lib-b") != base); + CHECK(native_identity(n, "bars", "strategy", "http://example/hook", warmup + "x", library) != base); other = n; other.timezone = "UTC+5"; validate_native_config(other); - CHECK(native_identity(other, "bars", "strategy", "http://example/hook", warmup, library, "", - "{}") != base); + CHECK(native_identity(other, "bars", "strategy", "http://example/hook", warmup, library) != base); other = n; other.timezone = "America/New_York"; validate_native_config(other); - CHECK(native_identity(other, "bars", "strategy", "http://example/hook", warmup, library, "", - "{}") != base); + CHECK(native_identity(other, "bars", "strategy", "http://example/hook", warmup, library) != base); other = n; other.chart_timezone = "UTC"; validate_native_config(other); - CHECK(native_identity(other, "bars", "strategy", "http://example/hook", warmup, library, "", - "{}") != base); + CHECK(native_identity(other, "bars", "strategy", "http://example/hook", warmup, library) != base); const auto dump = native_identity_document(n, "bars", "strategy", "http://example/hook", - warmup, library, "", "{}"); + warmup, library); CHECK(dump.find("pineforge-native-run/v1") != std::string::npos); CHECK(dump.find("unavailable:") == std::string::npos); CHECK(dump.find("\"chart_timezone\":\"\"") != std::string::npos); diff --git a/tests/test_native_live_websocket.cpp b/tests/test_native_live_websocket.cpp index 212bc9639..88aa9ffc5 100644 --- a/tests/test_native_live_websocket.cpp +++ b/tests/test_native_live_websocket.cpp @@ -107,9 +107,9 @@ class WebSocketServer { std::pair pong; WebSocketServer(std::vector frames, bool read_subscription = false, - bool read_pong = false, int delay_ms = 0) + bool read_pong = false, int delay_ms = 0, int hold_ms = 0) : frames_(std::move(frames)), read_subscription_(read_subscription), - read_pong_(read_pong), delay_ms_(delay_ms) { + read_pong_(read_pong), delay_ms_(delay_ms), hold_ms_(hold_ms) { listener_ = socket(AF_INET, SOCK_STREAM, 0); if (listener_ < 0) throw std::runtime_error("WS test socket failed"); sockaddr_in address {}; @@ -138,6 +138,7 @@ class WebSocketServer { bool read_subscription_ = false; bool read_pong_ = false; int delay_ms_ = 0; + int hold_ms_ = 0; std::string error_; void serve() noexcept { @@ -174,6 +175,7 @@ class WebSocketServer { if (delay_ms_) std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms_)); for (const auto& bytes : frames_) send_all(client, bytes); if (read_pong_) pong = read_client_frame(client); + if (hold_ms_) std::this_thread::sleep_for(std::chrono::milliseconds(hold_ms_)); } catch (const std::exception& e) { error_ = e.what(); } if (client >= 0) close(client); } @@ -209,6 +211,13 @@ void websocket_tests() { CHECK(received.size() == 1 && received[0] == large); CHECK(large_message.subscription.first == 0x81 && large_message.subscription.second == large); + WebSocketServer empty_final({frame(1, large, false), frame(9, "alive"), frame(0, "")}); + options.url = empty_final.url(); + received.clear(); + receive_websocket(options, "", capture, running); + empty_final.finish(); + CHECK(received == std::vector{large}); + for (const auto& bad : {frame(2, "binary"), frame(1, std::string("\xc0\x80", 2)), frame(1, large + "x"), frame(1, "unfinished", false), std::string("\x81\x0a", 2) + "abc", frame(8, "")}) { @@ -219,6 +228,21 @@ void websocket_tests() { CHECK(received.empty()); server.finish(); } + WebSocketServer unfinished_ping({frame(1, "unfinished", false), frame(9, "alive"), + frame(0, "still unfinished", false), frame(9, "again")}, + false, false, 0, 100); + options.url = unfinished_ping.url(); + options.connect_timeout_ms = 20; + options.total_timeout_ms = 30; + received.clear(); + bool timeout_error = false; + try { receive_websocket(options, "", capture, running); } + catch (const std::exception& error) { + timeout_error = std::string(error.what()).find("timeout") != std::string::npos; + } + CHECK(timeout_error); + CHECK(received.empty()); + unfinished_ping.finish(); WebSocketServer idle({}, false, false, 100); options.url = idle.url(); options.connect_timeout_ms = 20; @@ -239,6 +263,28 @@ void websocket_tests() { int main() { try { + const auto* info = curl_version_info(CURLVERSION_NOW); + if (!info || info->version_num < 0x080e01) { + for (const auto* url : {"ws://127.0.0.1:1/feed", "wss://127.0.0.1:1/feed"}) { + HttpOptions options; + options.url = url; + options.allow_insecure_http = true; + bool clear_error = false; + try { validate_websocket(options); } + catch (const std::exception& error) { + clear_error = std::string(error.what()).find("libcurl 8.14.1 or newer") != std::string::npos; + } + CHECK(clear_error); + bool called = false; + CHECK(throws([&] { + receive_websocket(options, "", [&](std::string_view) { called = true; return false; }, + [] { return false; }); + })); + CHECK(!called); + } + std::puts("native WebSocket: unqualified-runtime refusal verified; no message callback"); + return failures ? 1 : 0; + } if (!supports_ws()) { HttpOptions options; options.url = "wss://127.0.0.1:1/feed";