diff --git a/native/feed/CMakeLists.txt b/native/feed/CMakeLists.txt index ecfebfb..da33174 100644 --- a/native/feed/CMakeLists.txt +++ b/native/feed/CMakeLists.txt @@ -45,7 +45,8 @@ if(PINEFORGE_FEED_SANITIZER) add_link_options(-fsanitize=${PINEFORGE_FEED_SANITIZER}) endif() -add_library(pineforge_feed_core STATIC src/core.cpp src/state.cpp src/transport.cpp src/binance.cpp) +add_library(pineforge_feed_core STATIC src/core.cpp src/state.cpp src/transport.cpp src/binance.cpp src/okx.cpp + src/bybit.cpp src/session.cpp) target_include_directories(pineforge_feed_core PUBLIC src vendor) target_link_libraries(pineforge_feed_core PUBLIC CURL::libcurl OpenSSL::Crypto Threads::Threads) target_compile_options(pineforge_feed_core PRIVATE -Wall -Wextra -Wpedantic -Werror) @@ -62,5 +63,5 @@ if(BUILD_TESTING) add_test(NAME feed-unit COMMAND feed-unit) find_package(Python3 REQUIRED COMPONENTS Interpreter) add_test(NAME feed-mock COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/mock_venue.py $) - set_tests_properties(feed-mock PROPERTIES TIMEOUT 180) + set_tests_properties(feed-mock PROPERTIES TIMEOUT 420) endif() diff --git a/native/feed/README.md b/native/feed/README.md index 2eff191..80ea19a 100644 --- a/native/feed/README.md +++ b/native/feed/README.md @@ -3,8 +3,28 @@ An independent Apache-2.0 C++17 public-market-data adapter. It produces the native `pineforge-live` runner's JSONL protocol; it does not load strategies, place orders, access accounts, consume fills, or accept API keys. It is not part -of the Python package or wheel. This first release supports **Binance spot**, -`warmup` and `run`, with `bars` and raw `ticks`. There is no `serve` subcommand. +of the Python package or wheel. It has `warmup` and `run`; there is no `serve` +subcommand yet. + +## Venues and modes + +| `--venue` `--market` | Example `--symbol` | `bars` (confirmed 1m) | Ticks | +|---|---|---|---| +| `binance spot` | `BTCUSDT` | `kline_1m`, `x=true` | `ticks`: raw `trade`, ID+1, healed by `historicalTrades` | +| `binance usdm` | `BTCUSDT` | routed `/market` `kline_1m`, `x=true` | `agg-ticks` only: aggregate prints, healed by `aggTrades` within 48 h; the bars built from them can differ from Binance klines at minute edges (use `bars` for kline-exact bars) | +| `okx spot` | `BTC-USDT` | `candle1m`, confirm `"1"` | `ticks`: `trades-all`, ID+1, healed by `history-trades` within 3 months | +| `okx swap` | `BTC-USDT-SWAP` | `candle1m`, confirm `"1"`, base volume | `ticks`: `trades-all`, base quantities | +| `bybit spot`, `bybit linear` | `BTCUSDT` | `kline.1`, `confirm=true` | refused | +| `coinbase` | | refused: deferred | refused: deferred | + +Gates are checked at startup, before any network access. Bybit tick modes stop +with 23: Bybit trade IDs are not a contiguous cursor and its public REST cannot +page back through trades, so no gap can be proven healed. Bybit bars never come +from trades, because trade-built bars disagree with Bybit candles. Binance USD-M +`ticks` stops with 23: its raw trades have no public REST history. Coinbase +stops with 23 as deferred: it has no confirmed one-minute candle stream, and its +REST and WebSocket trade timestamps disagree, so lossless tick paging does not +exist. OKX inverse swaps are refused (their contracts are quote-denominated). ## Build @@ -39,15 +59,24 @@ build-feed/pineforge-feed run --venue binance --market spot --symbol BTCUSDT \ pineforge-live run --strategy strategy.so --warmup warmup.csv --mode bars \ --script-tf 1 --feed - --ledger runner.sqlite3 --symbol BINANCE:BTCUSDT \ --webhook-url https://receiver.example/actions +# OKX swap ticks and USD-M aggregate prints both feed the runner's tick mode: +build-feed/pineforge-feed run --venue okx --market swap --symbol BTC-USDT-SWAP \ + --mode ticks --state-dir okx-state --start 2026-10-03T03:20:00Z | + pineforge-live run --strategy strategy.so --warmup okx-warmup.csv --mode ticks ... +build-feed/pineforge-feed run --venue binance --market usdm --symbol BTCUSDT \ + --mode agg-ticks --state-dir usdm-state --start 2026-10-03T03:20:00Z | ... ``` +Each venue's warmup takes the same flags and writes its own venue's candles: +use a warmup from the venue and market that the live feed reads. + Times are minute-aligned Unix milliseconds or strict UTC RFC3339 seconds. Warmup fetches contiguous REST 1m klines only behind an observed `x=true` WS watermark. Its end is **exclusive** and must be the live start. It writes the runner CSV columns `timestamp,open,high,low,close,volume` and an atomic `warmup.csv.manifest.json` containing source, symbol, units, start, exclusive -cut, confirmation watermark, row count, and the CSV SHA-256. It refuses to -overwrite either output. Keep both files together; an interrupted two-file +cut, confirmation watermark, row count, quantity multiplier, and the CSV +SHA-256. It refuses to overwrite either output. Keep both files together; an interrupted two-file publication without its manifest is not a qualified warmup. ## Protocol and proof policy @@ -64,6 +93,27 @@ Venue decimal string tokens become JSON number tokens unchanged, including trailing zeros. Fixed-point string arithmetic reconciles OHLCV; no price or quantity enters a binary float. Bounds are 96 token characters, 32 fractional digits, and 128 arithmetic digits. Invalid/zero tick quantities fail closed. +Overlap, duplicate and revision checks compare exact decimal **values**, not +lexemes: OKX can render one candle price as `100.0` on its WebSocket and as +`100` over REST, which is not a revision; any change of value still is. +A message carries the token of the source that delivered it, so a bar healed +from REST can differ in bytes from the same bar received on the WebSocket, and +two feeds of one instrument can differ in bytes. The values are equal, a +journal replay is byte-identical, and the feed never canonicalises a token. + +### Units + +Prices are quote per base, times are Unix milliseconds, and every quantity and +volume is in **base** units. OKX swap `sz` and candle `vol` are contracts: the +feed multiplies them exactly (decimal arithmetic, never float) by the +instrument's `ctVal` x `ctMult` from the public instruments endpoint, which must +be a live linear swap with `ctValCcy` equal to the base currency. For +`BTC-USDT-SWAP`, 2.75 contracts are emitted as `0.0275`. A converted quantity is +the canonical product (no trailing fractional zeros); every other token is the +venue's own. The multiplier is bound into the cursor and the warmup manifest; +a resume with a changed multiplier stops with 21. + +### Binance spot ticks Bars mode subscribes to `kline_1m` only; ticks mode adds the raw `trade` stream. Raw `t` is the positive sequence and matched `T` is time, not dispatch @@ -90,23 +140,131 @@ Empty/ambiguous tick minutes deliberately stop rather than invent a fence. Bars heal missing minutes from REST only behind the verified closed watermark. Any discovered already-emitted bar revision stops; history is never rewritten. +### Next-print fence: OKX ticks and USD-M aggregate prints + +OKX candles and USD-M klines name no print-ID range (a USD-M kline's `f..L` +and `n` count raw trades, not aggregates), so a minute `M` closes with +`time = M + 60000` only on this proof: + +1. a contiguous ID chain (`tradeId` or aggregate `a`, each exactly its + predecessor plus one) from the REST-proven predecessor, the last print + strictly before `--start`. It is proven on both sides before it is saved: + OKX walks forward by trade ID from the time lookup until the next print is + at or after the start (REST can trail the newest prints, and prints of one + millisecond need not come back in ID order); USD-M takes the first + aggregate at or after the start within the next hour and the one before it. + Nothing is anchored, and no minute closes, until both sides are visible. A + quiet start waits for the first print (one REST read per closed minute); + once a WebSocket print exists, REST must show it within about 15 seconds, + or the feed stops with 20; +2. the fence: the next print in that chain has a matched time at or after + `M + 60000`, so no further print of `M` can exist (matched time never goes + backwards along the chain; if it does, the feed stops with 23); +3. the minute's closed candle: a confirmed WebSocket candle, or, only behind a + later confirmed WebSocket candle, the REST candle (OKX additionally requires + its per-row confirm `"1"`); +4. for OKX raw prints, exact OHLC and volume equality between the minute's + prints and that candle. USD-M aggregate prints skip this step: an aggregate + is dated by its first fill and never split, and one can hold a fill that + Binance's kline counts in the next minute, so the kline is not their exact + sum (see "Binance USD-M"). Their completeness rests on conditions 1 and 2, + and the kline only marks the minute closed. + +A candle without the fence never closes a minute, and the fence without the +candle waits. A quiet OKX minute closes on the same fence when its candle shows +zero volume; nonzero volume there stops with 21. Prints of later minutes stay +buffered until the proof. Holes heal by ID through REST: OKX `history-trades` +pages of 100 with exclusive `after`/`before` trade-ID bounds, USD-M `aggTrades` +`fromId` pages of 1000. A hole that starts more than the venue's documented +window before the newest venue time on the WebSocket (OKX 89 days, inside its +3-month window; USD-M 48 hours) stops with 20 without asking REST, and a USD-M +`-4166` "recent 2 days" rejection also stops with 20. + +### OKX + +`candle1m` and `trades-all` come from the business endpoint `/ws/v5/business`, +subscribed by message after every handshake. Only confirm `"1"` candles are +bars; a forming candle is never one, and a REST row without confirm `"1"` ends +the healed prefix. Ticks never come from the aggregated `trades` channel: a +push on it, or any print carrying `count`, stops with 23. The feed sends text +`ping` every `--keepalive-seconds` (default 20, at most 25: OKX closes a +connection after 30 seconds without traffic) and treats `pong` as control, +never data; a `notice` event (service upgrade) reconnects with overlap. Every +connection re-reads the instrument: a changed `ctVal x ctMult` stops with 21, +since quantities already emitted were converted with the old one. + +### Bybit + +Bars only, from `kline.1.` on `/v5/public/spot` or `/v5/public/linear`. +A push can carry the minute that just closed (`confirm=true`) and the next +forming one; only the confirmed row is a bar. REST klines arrive newest first +and a range wider than `limit` keeps only its newest rows, so each request asks +for exactly the rows it can hold and the reversed rows must be contiguous. The +feed sends `{"op":"ping"}` every `--keepalive-seconds`. Bybit answers HTTP 403 +for its IP rate ban (about ten minutes); the feed stops with 20 there, so a +supervised restart retries after the ban. + +### Binance USD-M + +Both modes use the routed `/market/stream` endpoint: since 2026-10-02 the +unrouted origin no longer serves `aggTrade` or `kline` streams, and a raw +`trade` event stops with 23. `agg-ticks` maps each aggregate `a` to exactly one +print and never expands `f..l` into imaginary raw trades. **Aggregate prints +are not raw trades**: one print is the sum of same-price, same-side executions, +so a strategy sees fewer, larger intrabar prints than in raw tick mode. The +print quantity is `q`, the aggregate's full executed quantity including RPI +(retail price improvement) executions; `nq` excludes those and would drop real +executions from the print stream. (Every aggregate seen live had `q = nq`.) +USD-M `exchangeInfo` exceeds the 1 MiB JSON +bound, so only its top-level `rateLimits` member is extracted (4 MiB body cap) +and parsed; requests then spend at most half of the published weight ceiling. + +**Bars built from `agg-ticks` are not Binance klines.** An aggregate is dated +by its first fill and never split, so one of its fills can belong to the next +minute's kline. The runner builds its bars from the prints, so at those minute +edges its bars, and the fills and indicators that depend on them, can differ +from Binance klines, from a backtest on klines, and from `--mode bars`. Use +`--mode bars` when bars must equal Binance klines. In 120 consecutive +BTCUSDT minutes (2026-10-03, every aggregate re-read from `aggTrades`), 6 +minutes (5%) differed, always in pairs where one fill moves into the next +minute: that minute's open is off by one tick (0.10) and both volumes by the +fill (0.002-0.011 BTC); close, high and low never differed, and no print lay +outside its own minute's kline range. In one live run, a short entry signalled +at a minute's close filled at the next minute's first print, one tick (0.10) +below that minute's kline open: the kline opened at the price of the fill that +the aggregate dated in the previous minute, so a backtest on klines fills one +tick away. A backtest on the bars built from the same prints equals the runner. +The matching backtest for an `agg-ticks` deployment is therefore a backtest on +prints-built 1m bars; a later feed release adds the export of those bars. + The subscribed data streams are strict: another event type on them, a -malformed trade or kline, or an unknown shape stops with 23. An event outside +malformed trade or kline, or an unknown shape stops with 23 (on every venue). An event outside the data streams with an unknown type (a new venue notice) is logged as a structured `unknown_stream_event` warning and ignored; `serverShutdown` reconnects. -Curl answers server PING with matching-payload PONG. Reconnect occurs on -`serverShutdown`, transport loss/75-second source silence, or at 23h55 (before -the venue's 24-hour limit). The reader buffers during REST healing. Every new +Curl answers server PING with matching-payload PONG; OKX and Bybit get their +text keepalives. Keepalive replies are control messages: they never count as +data, so a silent data stream still reconnects. Reconnect occurs on +`serverShutdown` or an OKX `notice`, transport loss/75 seconds without data, or +at 23h55 (before the 24-hour Binance limit). The reader buffers during REST healing. Every new connection re-fetches up to 1000 raw prints and 32 closed-bar proofs, requires -exact token/ID/time overlap, then continues the same stream epoch and sequence. -An altered duplicate discovered outside that window is checked against the -full retained normalized journal. Older, undiscovered revisions are not claimed -to be continuously polled. - -REST routes are restricted to public `exchangeInfo`, `historicalTrades`, -`aggTrades`, and `klines`. Current public weight/raw-request limits are read +exact value/ID/time overlap, then continues the same stream epoch and sequence. +A venue's REST row for the minute that just closed can trail its WebSocket +close by a second or more (observed on USD-M at a restart): a disagreement must +survive three spaced re-reads (about 7 seconds) before it stops with 21, and +warmup re-reads its newest rows the same way. An altered duplicate discovered +outside that window is checked against the full retained normalized journal. +Older, undiscovered revisions are not claimed to be continuously polled. + +REST routes are allowlisted per venue: Binance spot `exchangeInfo`, +`historicalTrades`, `aggTrades`, `klines`; USD-M `exchangeInfo`, `aggTrades`, +`klines`; OKX `public/instruments`, `market/history-trades`, +`market/history-candles` (one request per 200 ms, half of 20 per 2 s); Bybit +`market/kline`, `market/instruments-info` (one request per 100 ms, OKX `50011` +and Bybit `10006` rate-limit bodies wait and retry like `429`; OKX `50001`, +`50004`, `50013`, `50026` and Bybit `10000`, `10016` mean "try again" and are +retried with backoff like HTTP 5xx, within the same four attempts). Current public weight/raw-request limits are read from `exchangeInfo`, and requests are spaced to spend at most half of each published ceiling (about 50% headroom; 20 ms per weight at 6000 per minute, so 500 ms per 1000-print page). The venue's `X-MBX-USED-WEIGHT-1M` header counts @@ -209,16 +367,19 @@ each bounded by `--max-replay-seconds`. ### Tick mode needs a liquid symbol -75 seconds without any text frame forces a reconnect, and each reconnect costs -a weight-27 overlap check (one `historicalTrades` page and one `klines` call). -A minute without a print cannot be fenced, so tick mode stops with 20 there, -and again on every resume. Use bars mode for symbols that can go quiet for a -minute. +75 seconds without any data message forces a reconnect, and each reconnect +costs an overlap check (on Binance spot a weight-27 `historicalTrades` page and +`klines` call). On Binance spot a minute without a print cannot be fenced, so +tick mode stops with 20 there, and again on every resume. OKX closes a quiet +minute on the next-print fence and its zero-volume candle, USD-M on the fence +alone; the runner then sees a `time` event with no print in that minute. Use bars mode for +symbols that can go quiet for a minute. ### Restart policy Retry budgets are short by design: about 2 minutes of failed WebSocket -connects, or about 7 seconds of failed REST attempts, end with 20 and the +connects (a connection the venue accepts but closes before any frame counts as +failed), or about 7 seconds of failed REST attempts, end with 20 and the verified cursor retained. Run the feed under a supervisor (a systemd unit with `Restart=on-failure`, or a container restart policy) that restarts on 20 with backoff, as `--resume --output-from N` with the runner's committed count. @@ -229,12 +390,18 @@ disk, or a consumer that reads stdout again. ## Qualification and next adapters `tests/unit.cpp` and `tests/mock_venue.py` contain **synthetic** documented -Binance message shapes only: holes, reordering, duplicates, revisions, access, -reconnect overlap, fragmented text, PING/PONG, retirement, budgets, and recovery. -Never commit public exchange captures. `tests/public_e2e.py` explicitly opts -into public BTCUSDT capture, local receiver delivery, a hard-kill restart, -exact REST verification, bar batch action parity through `run_backtest_full`, -and tick same-print replay determinism. Tick actions are compared with the +message shapes only (Binance spot, USD-M, OKX, Bybit): holes, reordering, +duplicates, revisions, access, reconnect overlap, fragmented text, PING/PONG +and text keepalives, retirement, budgets, recovery, and every venue quirk above. +Never commit public exchange captures. `tests/public_e2e.py --venue V --market +M --symbol S --modes bars,ticks|agg-ticks` explicitly opts into a public +capture of one venue, local receiver delivery, a hard-kill restart, exact REST +verification (OKX candle values; lexemes elsewhere), bar batch action parity +through `run_backtest_full`, and tick same-print replay determinism. For tick +modes the batch replays 1m bars built from the feed's own prints (the tape), +not venue candles; for `agg-ticks` the receipt also reports how many minutes +differ from Binance klines (`kline_differs`) and how a batch over those klines +compares with the runner. Tick actions are compared with the batch under the predeclared R-B2 mapping: each action timestamp becomes `floor(ts / script_tf) * script_tf` on both sides, because a tick fill carries its print's time while the OHLC batch carries its modeled segment's clock. @@ -243,9 +410,10 @@ difference fails the run. Its batch observer is linked separately from the read-only engine's `tests/native_live_equivalence_observer.cpp`; it is not a feed dependency. -`src/venue.hpp` is the adapter boundary. Later slices add OKX raw ticks/bars -(contract-unit conversion where needed), Bybit confirmed bars **without ticks**, -and USD-M confirmed bars / separately opted-in aggregate-print ticks. Coinbase -remains deferred. `serve` will add approved MIT CivetWeb only to its server -target, bounded client replay/queues, and cursor-bound fan-out; no server library -is linked in this slice. The engine executable remains `pineforge-live`. +`src/venue.hpp` is the adapter boundary: a venue supplies its WebSocket +connection (path, subscription frames, keepalive, frame classifier), decoding, +REST history and candles, its tick proof and its print history window. +`src/session.cpp` holds the venue-neutral sessions. Coinbase remains deferred. +`serve` will add approved MIT CivetWeb only to its server target, bounded client +replay/queues, and cursor-bound fan-out; no server library is linked yet. The +engine executable remains `pineforge-live`. diff --git a/native/feed/src/binance.cpp b/native/feed/src/binance.cpp index 9f8ab38..f9e5f60 100644 --- a/native/feed/src/binance.cpp +++ b/native/feed/src/binance.cpp @@ -2,7 +2,6 @@ #include "binance.hpp" #include #include -#include #include namespace pineforge::feed { @@ -19,7 +18,18 @@ std::string lower(std::string value) { std::transform(value.begin(), value.end(), value.begin(), [](unsigned char letter) { return static_cast(std::tolower(letter)); }); return value; } -std::size_t trade_bytes(const Trade& trade) { return trade.price.size() + trade.qty.size() + 192; } +std::vector contiguous_klines(const Json& response, std::int64_t start, std::int64_t end) { + std::vector result; + auto expected = start; + for (const auto& row : array(response, 1000)) { + const auto kline = binance_kline(row, false); + if (kline.bar.ts != expected || kline.bar.ts >= end) throw Error(20, "historical klines have a missing or unexpected minute"); + expected += 60000; + result.push_back(kline); + } + if (result.empty()) throw Error(20, "historical minute is unavailable"); + return result; +} } Trade binance_trade(const Json& value, bool websocket) { @@ -52,12 +62,30 @@ Kline binance_kline(const Json& value, bool websocket) { } return kline; } +// USD-M `q` is the aggregate's full executed quantity, RPI (retail price improvement) executions +// included; `nq` excludes those and would drop real executions from the print stream. +Trade usdm_aggregate(const Json& value) { + Trade trade{value.at("a").integer(), value.at("T").integer(), value.at("p").text(), value.at("q").text()}; + const auto first = value.at("f").integer(), last = value.at("l").integer(); + if (!first || last < first) throw Error(23, "invalid aggregate raw-trade range"); + trade.validate(); + return trade; +} +Frame binance_frame(const std::string& message) { + try { + const auto envelope = parse_json(message); + const auto* wrapped = envelope.find("data"); + const auto& data = wrapped ? *wrapped : envelope; + const auto* event = data.find("e"); + return event && event->kind == Json::Kind::String && event->text() == "serverShutdown" ? Frame::Retire : Frame::Data; + } catch (const std::exception&) { throw Error(23, "invalid public WebSocket JSON"); } +} -std::string BinanceSpot::streams() const { +Connection BinanceSpot::connection() const { const auto symbol = lower(config_.symbol); - return config_.mode == "ticks" ? symbol + "@trade/" + symbol + "@kline_1m" : symbol + "@kline_1m"; + return {"/stream?streams=" + (config_.mode == "ticks" ? symbol + "@trade/" + symbol + "@kline_1m" : symbol + "@kline_1m"), {}, {}, &binance_frame}; } -VenueEvent BinanceSpot::decode(const std::string& message) const { +std::vector BinanceSpot::decode(const std::string& message) const { try { const auto envelope = parse_json(message); const auto* wrapped = envelope.find("data"); @@ -73,10 +101,10 @@ VenueEvent BinanceSpot::decode(const std::string& message) const { if (wrapped && !(trades && type == "trade") && !(klines && type == "kline")) throw Error(23, "unsupported public stream event; aggregate prints are not raw trades"); if (data.at("s").text() != config_.symbol) throw Error(23, "public stream symbol changed"); - if (type == "trade") return {VenueEvent::Kind::Trade, binance_trade(data, true), {}}; + if (type == "trade") return {{VenueEvent::Kind::Trade, binance_trade(data, true), {}}}; const auto& row = data.at("k"); if (row.at("s").text() != config_.symbol) throw Error(23, "kline symbol changed"); - return {VenueEvent::Kind::Kline, {}, binance_kline(row, true)}; + return {{VenueEvent::Kind::Kline, {}, binance_kline(row, true)}}; } if (type == "serverShutdown") return {}; // A venue notice outside the data streams carries no market data; a new type must not stop the feed. @@ -124,322 +152,110 @@ std::vector BinanceSpot::klines(std::int64_t start, std::int64_t end) { if (start < 0 || start % 60000 || end <= start || end % 60000) throw Error(23, "invalid exclusive kline range"); const auto response = http_.get("/api/v3/klines?symbol=" + config_.symbol + "&interval=1m&startTime=" + std::to_string(start) + "&endTime=" + std::to_string(end - 1) + "&limit=1000", 2); - std::vector result; - auto expected = start; - for (const auto& row : array(response, 1000)) { - const auto kline = binance_kline(row, false); - if (kline.bar.ts != expected || kline.bar.ts >= end) throw Error(20, "historical klines have a missing or unexpected minute"); - expected += 60000; - result.push_back(kline); - } - if (result.empty()) throw Error(20, "historical minute is unavailable"); - return result; -} - -FeedSession::FeedSession(State& state, Venue& venue, std::function output) - : state_(state), venue_(venue), output_(std::move(output)) { - if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { - const auto event = parse_json(line); - if (event.at("type").text() == "tick") { - const auto trade = normalized_trade(event); - if (trade.ts >= state_.cursor().cut) aggregate_.add(trade); - } - }); + return contiguous_klines(response, start, end); } -void FeedSession::emit(const std::string& line, const std::optional& proof) { - if (stopping) throw Stopped{}; - state_.stage(line, proof); - ++new_messages_; - if (state_.config().max_messages && new_messages_ >= state_.config().max_messages) { - publish(); - throw Stopped{}; - } - if (state_.staged() >= 4096) publish(); +namespace { +RestPolicy usdm_rest() { + RestPolicy policy; + policy.endpoints = {"/fapi/v1/exchangeInfo", "/fapi/v1/aggTrades", "/fapi/v1/klines"}; + // The USD-M exchangeInfo document lists every contract and exceeds the 1 MiB parser bound. + policy.limits_path = "/fapi/v1/exchangeInfo"; + policy.limits_weight = 1; + policy.limits_bytes = 4 * 1024 * 1024; + policy.rejected = [](long status, const std::string& body) { + if (status != 400) return; + if (body.find("\"code\":-4166") != std::string::npos) + throw Error(20, "aggregate history is beyond the venue's 2-day aggTrades window"); + if (body.find("\"code\":-1121") != std::string::npos) throw Error(23, "unknown USD-M symbol"); + }; + return policy; } -void FeedSession::publish() { - for (const auto& line : state_.flush()) output_(line); } -void FeedSession::salvage() noexcept { - try { publish(); } - catch (...) { - try { log("error", "verified_messages_not_committed"); } catch (...) {} - } +unsigned int usdm_kline_weight(std::size_t limit) { return limit < 100 ? 1 : limit < 500 ? 2 : limit <= 1000 ? 5 : 10; } +Connection usdm_connection(const Config& config) { + const auto symbol = lower(config.symbol); + return {"/market/stream?streams=" + (config.mode == "agg-ticks" ? symbol + "@aggTrade/" + symbol + "@kline_1m" : symbol + "@kline_1m"), + {}, {}, &binance_frame}; } -void FeedSession::ingest(const VenueEvent& event) { - try { stage(event); } - catch (const Error&) { salvage(); throw; } - publish(); +BinanceUsdm::BinanceUsdm(const Config& config) : config_(config), http_(config, usdm_rest()) { + // An unknown symbol is refused before any stream is opened (HTTP 400, code -1121). + (void)http_.get("/fapi/v1/klines?symbol=" + config_.symbol + "&interval=1m&limit=1", 1); } -void FeedSession::connected() { - publish(); - const auto& trades = state_.recent_trades(); - if (!trades.empty()) { - const auto fetched = venue_.history(trades.front().id, trades.size()); - if (fetched.size() != trades.size()) throw Error(20, "reconnect raw overlap is unavailable"); - for (std::size_t index = 0; index < trades.size(); ++index) - if (!(trades[index] == fetched[index])) throw Error(21, "reconnect raw overlap changed"); - } else if (state_.cursor().predecessor) { - const auto fetched = venue_.history(state_.cursor().predecessor->id, 1); - if (!(fetched.front() == *state_.cursor().predecessor)) throw Error(21, "initial predecessor overlap changed"); - } - const auto& proofs = state_.config().mode == "bars" ? state_.recent_bars() : state_.cursor().proofs; - if (!proofs.empty()) { - const auto fetched = venue_.klines(proofs.front().ts, proofs.back().ts + 60000); - if (fetched.size() != proofs.size()) throw Error(20, "reconnect closed-bar overlap is unavailable"); - std::map verified; - if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { - const auto event = parse_json(line); - if (event.at("type").text() != "tick") return; - const auto trade = normalized_trade(event); - const auto minute = trade.ts - trade.ts % 60000; - if (minute >= proofs.front().ts && minute <= proofs.back().ts) verified[minute].add(trade); - }); - for (std::size_t index = 0; index < proofs.size(); ++index) { - if (!(proofs[index] == fetched[index].bar)) throw Error(21, "reconnect already-emitted bar changed"); - if (state_.config().mode == "ticks") { - try { verified.at(fetched[index].bar.ts).reconcile(fetched[index], false); } - catch (const std::exception&) { throw Error(21, "reconnect closed-minute count or raw proof changed"); } - } +std::vector BinanceUsdm::decode(const std::string& message) const { + try { + const auto envelope = parse_json(message); + const auto* wrapped = envelope.find("data"); + const auto& data = wrapped ? *wrapped : envelope; + const auto* event = data.find("e"); + if (!event || event->kind != Json::Kind::String) throw Error(23, "unexpected public stream control message"); + const auto type = event->text(); + const auto stream = wrapped ? envelope.at("stream").text() : std::string(); + const auto symbol = lower(config_.symbol); + const bool aggregates = stream == symbol + "@aggTrade", klines = stream == symbol + "@kline_1m"; + // Raw @trade prints reach only unrouted connections and have no public REST history. + if (type == "trade") throw Error(23, "raw USD-M trades are not a supported source; use the routed /market stream"); + if (type == "aggTrade" || type == "kline" || aggregates || klines) { + if (wrapped && !(aggregates && type == "aggTrade" && config_.mode == "agg-ticks") && !(klines && type == "kline")) + throw Error(23, "unsupported USD-M stream event"); + if (data.at("s").text() != config_.symbol) throw Error(23, "public stream symbol changed"); + if (type == "aggTrade") return {{VenueEvent::Kind::Trade, usdm_aggregate(data), {}}}; + const auto& row = data.at("k"); + if (row.at("s").text() != config_.symbol) throw Error(23, "kline symbol changed"); + return {{VenueEvent::Kind::Kline, {}, binance_kline(row, true)}}; } - } - if (!replayed_) { - if (state_.config().resume) state_.visit(state_.config().output_from, output_); - replayed_ = true; - } - log("info", "overlap_verified", state_.status()); -} - -void FeedSession::anchor(std::uint64_t first) { - if (state_.cursor().predecessor) return; - if (first <= 1) throw Error(20, "raw predecessor cannot be proven"); - const auto predecessor = venue_.history(first - 1, 1).front(); - state_.anchor(predecessor); -} -std::uint64_t FeedSession::next_id() const { - const auto previous = state_.cursor().seq ? state_.cursor().seq : state_.cursor().predecessor ? state_.cursor().predecessor->id : 0; - if (!previous || previous == std::numeric_limits::max()) throw Error(20, "raw-trade cursor is not anchored"); - return previous + 1; -} -void FeedSession::put(const Trade& trade) { - trade.validate(); - if (trade.ts < state_.cursor().start) return; - if (state_.cursor().seq && trade.id <= state_.cursor().seq) { - const auto previous = state_.trade(trade.id); - if (!previous || !(*previous == trade)) throw Error(21, "duplicate raw trade conflicts with the immutable prefix"); - return; - } - const auto existing = pending_.find(trade.id); - if (existing != pending_.end()) { - if (!(existing->second == trade)) throw Error(21, "buffered raw duplicate conflicts"); - return; - } - const auto bytes = trade_bytes(trade); - if (bytes > state_.config().max_queue_bytes - std::min(pending_bytes_, state_.config().max_queue_bytes)) - throw Error(22, "unconfirmed raw-trade buffer overrun"); - pending_bytes_ += bytes; - pending_.emplace(trade.id, trade); -} -void FeedSession::emit_trade(const Trade& trade) { - if (trade.ts < state_.cursor().last_tick_ts) throw Error(23, "venue matched time regressed along the contiguous raw-trade chain"); - if (trade.id != next_id() || trade.ts < state_.cursor().cut || trade.ts >= state_.cursor().cut + 60000) - throw Error(20, "raw trade violates the complete chronological minute prefix"); - const auto buffered = pending_.find(trade.id); - if (buffered != pending_.end() && !(buffered->second == trade)) - throw Error(21, "REST healing changed a buffered raw trade"); - aggregate_.add(trade); - emit(trade.wire()); - if (buffered != pending_.end()) { - pending_bytes_ -= trade_bytes(buffered->second); - pending_.erase(buffered); - } + if (type == "serverShutdown") return {}; + log("warn", "unknown_stream_event", Json::object({{"stream", Json::string(stream)}, {"type", Json::string(type)}})); + return {}; + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid Binance USD-M message shape"); } } -void FeedSession::heal_to(std::uint64_t target) { - while (next_id() <= target) { - log("warn", "raw_gap_healing", Json::object({{"from_id", Json::number(std::to_string(next_id()))}, {"through_id", Json::number(std::to_string(target))}})); - const auto page = venue_.history(next_id()); - bool progressed = false; - for (const auto& trade : page) { - if (trade.id > target) break; - const auto buffered = pending_.find(trade.id); - if (buffered != pending_.end() && !(buffered->second == trade)) throw Error(21, "REST raw healing conflicts with WebSocket data"); - emit_trade(trade); - progressed = true; - } - if (!progressed) throw Error(20, "raw gap cannot advance within available history"); +std::vector BinanceUsdm::history(std::uint64_t from, std::size_t limit) { + if (!from || !limit || limit > 1000) throw Error(23, "invalid aggregate-history page request"); + const auto response = http_.get("/fapi/v1/aggTrades?symbol=" + config_.symbol + "&fromId=" + std::to_string(from) + "&limit=" + std::to_string(limit), 20); + std::vector trades; + std::uint64_t expected = from; + for (const auto& row : array(response, limit)) { + const auto trade = usdm_aggregate(row); + if (trade.id != expected) throw Error(20, "aggregate-history page has a missing or nonunit aggregate ID"); + trades.push_back(trade); + if (expected == std::numeric_limits::max()) throw Error(23, "venue aggregate ID exhausted"); + ++expected; } + return trades; } -void FeedSession::drain() { - while (!pending_.empty()) { - const auto trade = pending_.begin()->second; - if (trade.ts >= state_.cursor().cut + 60000) break; - if (trade.id < next_id()) throw Error(21, "buffered cursor fell behind the immutable prefix"); - if (trade.id > next_id()) heal_to(trade.id - 1); - emit_trade(trade); - } +namespace { +// The first aggregate at or after the start, searched over the next hour (the venue's longest time window), +// so a quiet start minute still has a fence; 0 when none is visible yet. +std::uint64_t first_aggregate(HttpClient& http, const std::string& symbol, std::int64_t minute) { + const auto response = http.get("/fapi/v1/aggTrades?symbol=" + symbol + "&startTime=" + std::to_string(minute) + + "&endTime=" + std::to_string(minute + 3599999) + "&limit=1", 20); + const auto& rows = array(response, 1); + if (rows.empty()) return 0; + const auto first = usdm_aggregate(rows.front()); + if (first.ts < minute || first.ts >= minute + 3600000) throw Error(20, "initial aggregate fence is unavailable or ambiguous"); + return first.id; } -// A minute whose own x=true kline was missed closes only on proof: a later x=true watermark, the contiguous -// raw-ID chain from the proven predecessor, the first print of a later minute as the fence, and REST n plus -// exact OHLCV. The REST row has no x=true or f..L, and none is manufactured for it. -void FeedSession::historical_minute() { - if (state_.cursor().cut + 60000 > watermark_) throw Error(20, "historical tick closure lacks a confirmed WebSocket watermark"); - if (!state_.cursor().predecessor) anchor(venue_.first_trade_id(state_.cursor().start)); - const auto minute = state_.cursor().cut; - std::optional fence; - while (!fence) { - const auto page = venue_.history(next_id()); - for (const auto& trade : page) { - const auto buffered = pending_.find(trade.id); - if (buffered != pending_.end() && !(buffered->second == trade)) throw Error(21, "historical raw overlap changed"); - if (trade.ts >= minute + 60000) { - fence = trade; - break; - } - emit_trade(trade); - } - } - if (!aggregate_.count || fence->id != next_id() || fence->id != aggregate_.last + 1) - throw Error(20, "historical minute lacks a contiguous raw prefix and next-minute fence"); - put(*fence); - const auto kline = venue_.klines(minute, minute + 60000).front(); - aggregate_.reconcile(kline, false); - log("info", "historical_tick_fence_verified", Json::object({{"minute", Json::number(std::to_string(minute))}, - {"first_id", Json::number(std::to_string(aggregate_.first))}, {"last_id", Json::number(std::to_string(aggregate_.last))}})); - emit("{\"type\":\"time\",\"ts\":" + std::to_string(minute + 60000) + "}", kline.bar); - aggregate_ = Aggregate{}; } - -void FeedSession::closed(const Kline& kline) { - if (!kline.confirmed) return; - const auto minute = kline.bar.ts; - watermark_ = std::max(watermark_, minute + 60000); - if (minute < state_.cursor().start) return; - if (minute < state_.cursor().cut) { - const auto old = state_.bar(minute); - if (old) { - if (!(*old == kline.bar)) throw Error(21, "already-emitted closed bar was revised"); - } - if (state_.config().mode == "ticks") { - Aggregate reconstructed; - state_.visit(0, [&](const std::string& line) { - const auto event = parse_json(line); - if (event.at("type").text() == "tick") { - const auto trade = normalized_trade(event); - if (trade.ts >= minute && trade.ts < minute + 60000) reconstructed.add(trade); - } - }); - try { reconstructed.reconcile(kline, true); } - catch (const Error& failure) { - if (failure.code == 20 || failure.code == 21) throw Error(21, "already-emitted closed-minute fence was revised"); - throw; - } - } else if (!old) throw Error(21, "closed duplicate cannot be matched to the retained prefix"); - return; - } - if (state_.config().mode == "bars") { - while (state_.cursor().cut < minute) { - const auto page = venue_.klines(state_.cursor().cut, minute); - for (const auto& historical : page) { - if (historical.bar.ts + 60000 > watermark_) throw Error(20, "historical bar exceeds the confirmed watermark"); - emit(historical.bar.wire(), historical.bar); - } - } - emit(kline.bar.wire(), kline.bar); - return; +std::uint64_t BinanceUsdm::first_trade_id(std::int64_t minute) { + for (unsigned int attempt = 0; attempt < 3; ++attempt) { + if (attempt) pause_for(std::chrono::milliseconds(500 * attempt)); + if (const auto first = first_aggregate(http_, config_.symbol, minute)) return first; } - while (state_.cursor().cut < minute) historical_minute(); - if (!kline.count || kline.first <= 0 || kline.last < kline.first || - static_cast(kline.last - kline.first) + 1 != kline.count) - throw Error(20, "closed kline has ambiguous or noncontiguous raw bounds"); - anchor(static_cast(kline.first)); - if (state_.cursor().seq > static_cast(kline.last)) throw Error(21, "closed kline excludes an already-emitted trade"); - heal_to(static_cast(kline.last)); - aggregate_.reconcile(kline, true); - emit("{\"type\":\"time\",\"ts\":" + std::to_string(minute + 60000) + "}", kline.bar); - aggregate_ = Aggregate{}; - drain(); + throw Error(20, "no aggregate after the start minute is available yet"); } -void FeedSession::stage(const VenueEvent& event) { - if (event.kind == VenueEvent::Kind::Kline) { closed(event.kline); return; } - if (event.kind != VenueEvent::Kind::Trade || state_.config().mode != "ticks" || event.trade.ts < state_.cursor().start) return; - if (!state_.cursor().predecessor) anchor(venue_.first_trade_id(state_.cursor().start)); - put(event.trade); - drain(); +std::optional BinanceUsdm::predecessor_if_ready(std::int64_t minute) { + const auto first = first_aggregate(http_, config_.symbol, minute); + if (!first) return std::nullopt; + const auto previous = history(first - 1, 1); + if (first <= 1 || previous.empty() || previous.front().id != first - 1) throw Error(20, "raw predecessor is unavailable"); + return previous.front(); } - -void run_feed(const Config& config) { - State state(config); - try { - BinanceSpot venue(config); - WebSocketPump pump(config, venue.streams()); - FeedSession session(state, venue, &output_line); - for (;;) { - auto message = pump.take(); - try { - // Group commit: verify every source message already queued (bounded), then persist once. - for (std::size_t taken = 1;; ++taken) { - if (message.connected) session.connected(); - else session.stage(venue.decode(message.text)); - if (taken >= 1024 || !pump.try_take(message)) break; - } - } catch (const Error&) { session.salvage(); throw; } - session.publish(); - } - } catch (const Stopped&) { log("info", "stopped", state.status()); } - catch (...) { log("error", "verified_cursor_retained", state.status()); throw; } -} - -void warmup(const Config& config, const std::string& output) { - if (config.start < 0 || config.end <= config.start || config.start % 60000 || config.end % 60000) - throw Error(23, "warmup requires a nonempty minute-aligned exclusive range"); - if ((config.end - config.start) / 60000 > 100000) throw Error(22, "warmup range exceeds 100000 bars"); - if (std::filesystem::exists(output) || std::filesystem::exists(output + ".manifest.json")) throw Error(23, "warmup output already exists"); - BinanceSpot venue(config); - std::int64_t watermark = -1; - std::map confirmed; - { - WebSocketPump pump(config, venue.streams()); - while (watermark < config.end) { - const auto message = pump.take(); - if (message.connected) continue; - const auto event = venue.decode(message.text); - if (event.kind != VenueEvent::Kind::Kline || !event.kline.confirmed) continue; - const auto& bar = event.kline.bar; - watermark = std::max(watermark, bar.ts + 60000); - if (bar.ts < config.start || bar.ts >= config.end) continue; - const auto [found, inserted] = confirmed.emplace(bar.ts, bar); - if (!inserted && !(found->second == bar)) throw Error(21, "confirmed WebSocket bar was revised during warmup"); - } - } - std::string csv = "timestamp,open,high,low,close,volume\n"; - auto next = config.start; - std::uint64_t count = 0, cross_checked = 0; - while (next < config.end) { - for (const auto& kline : venue.klines(next, config.end)) { - if (kline.bar.ts != next || next + 60000 > watermark) throw Error(20, "warmup does not adjoin a verified closed-minute cut"); - const auto& bar = kline.bar; - // The newest rows come from REST moments after the WS close: the x=true bars in hand must match. - const auto observed = confirmed.find(bar.ts); - if (observed != confirmed.end()) { - if (!(observed->second == bar)) throw Error(21, "warmup REST row differs from the confirmed WebSocket bar"); - ++cross_checked; - } - csv += std::to_string(bar.ts) + ',' + bar.open + ',' + bar.high + ',' + bar.low + ',' + bar.close + ',' + bar.volume + '\n'; - if (csv.size() > 16 * 1024 * 1024) throw Error(22, "warmup output exceeds 16 MiB"); - next += 60000; - ++count; - } - } - const auto manifest = Json::object({{"schema", Json::string("pineforge-feed-warmup/v1")}, - {"venue", Json::string(config.venue)}, {"market", Json::string(config.market)}, {"symbol", Json::string(config.symbol)}, - {"source", Json::string(config.rest_url + "/api/v3/klines")}, {"interval", Json::string("1m")}, - {"units", Json::object({{"price", Json::string("quote/base")}, {"volume", Json::string("base")}, {"timestamp", Json::string("unix-ms")}})}, - {"start", Json::number(std::to_string(config.start))}, {"end_exclusive", Json::number(std::to_string(config.end))}, - {"cut", Json::number(std::to_string(config.end))}, {"confirmed_ws_watermark", Json::number(std::to_string(watermark))}, - {"bars", Json::number(std::to_string(count))}, {"ws_cross_checked_bars", Json::number(std::to_string(cross_checked))}, - {"sha256", Json::string(sha256(csv))}}); - atomic_file(output, csv); - atomic_file(output + ".manifest.json", manifest.dump() + '\n'); - log("info", "warmup_verified", manifest); +std::vector BinanceUsdm::klines(std::int64_t start, std::int64_t end) { + if (start < 0 || start % 60000 || end <= start || end % 60000) throw Error(23, "invalid exclusive kline range"); + const auto limit = static_cast(std::min(1000, (end - start) / 60000)); + const auto response = http_.get("/fapi/v1/klines?symbol=" + config_.symbol + "&interval=1m&startTime=" + std::to_string(start) + + "&endTime=" + std::to_string(end - 1) + "&limit=" + std::to_string(limit), usdm_kline_weight(limit)); + return contiguous_klines(response, start, end); } } diff --git a/native/feed/src/binance.hpp b/native/feed/src/binance.hpp index 8f5b06f..aed398e 100644 --- a/native/feed/src/binance.hpp +++ b/native/feed/src/binance.hpp @@ -1,55 +1,48 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once -#include "state.hpp" #include "transport.hpp" #include "venue.hpp" -#include namespace pineforge::feed { Trade binance_trade(const Json& trade, bool websocket); Kline binance_kline(const Json& row, bool websocket); +// One USD-M aggregate print: `a` is the sequence, `q` the quantity. Its raw `f..l` range is validated, +// never expanded into imaginary raw trades. +Trade usdm_aggregate(const Json& row); +Frame binance_frame(const std::string& message); +Connection usdm_connection(const Config& config); +unsigned int usdm_kline_weight(std::size_t limit); class BinanceSpot final : public Venue { Config config_; HttpClient http_; public: - explicit BinanceSpot(const Config& config) : config_(config), http_(config) {} - std::string streams() const override; - VenueEvent decode(const std::string& message) const override; + explicit BinanceSpot(const Config& config) : config_(config), http_(config, binance_spot_rest(config.symbol)) {} + Connection connection() const override; + std::vector decode(const std::string& message) const override; std::vector history(std::uint64_t from, std::size_t limit = 1000) override; std::uint64_t first_trade_id(std::int64_t minute) override; std::vector klines(std::int64_t start, std::int64_t end) override; + std::string kline_source() const override { return "/api/v3/klines"; } }; -class FeedSession { - State& state_; - Venue& venue_; - std::function output_; - Aggregate aggregate_; - std::map pending_; - std::size_t pending_bytes_ = 0; - std::int64_t watermark_ = -1; - std::uint64_t new_messages_ = 0; - bool replayed_ = false; - void emit(const std::string& line, const std::optional& proof = std::nullopt); - void anchor(std::uint64_t first); - std::uint64_t next_id() const; - void put(const Trade& trade); - void emit_trade(const Trade& trade); - void heal_to(std::uint64_t target); - void drain(); - void historical_minute(); - void closed(const Kline& kline); +// Binance USD-M perpetuals: confirmed bars, or the opt-in `agg-ticks` aggregate-print mode. Both use the +// routed /market stream; the unrouted origin no longer serves aggTrade or kline streams. +class BinanceUsdm final : public Venue { + Config config_; + HttpClient http_; public: - FeedSession(State& state, Venue& venue, std::function output); - void connected(); - // stage() verifies an event and stages its messages; publish() group-commits them, then writes them - // out; salvage() publishes what was already verified before a stop. ingest() is all three for one event. - void stage(const VenueEvent& event); - void publish(); - void salvage() noexcept; - void ingest(const VenueEvent& event); + explicit BinanceUsdm(const Config& config); + Connection connection() const override { return usdm_connection(config_); } + std::vector decode(const std::string& message) const override; + std::vector history(std::uint64_t from, std::size_t limit = 1000) override; + std::uint64_t first_trade_id(std::int64_t minute) override; + std::optional predecessor_if_ready(std::int64_t minute) override; + std::vector klines(std::int64_t start, std::int64_t end) override; + TickProof tick_proof() const override { return TickProof::NextPrintFence; } + std::int64_t retention_ms() const override { return 48LL * 3600000; } + // Observed live: an aggregate dated in one minute held a fill the next minute's kline counts. + bool candle_is_print_sum() const override { return false; } + std::string kline_source() const override { return "/fapi/v1/klines"; } }; -void run_feed(const Config& config); -void warmup(const Config& config, const std::string& output); } diff --git a/native/feed/src/bybit.cpp b/native/feed/src/bybit.cpp new file mode 100644 index 0000000..4cded5f --- /dev/null +++ b/native/feed/src/bybit.cpp @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "bybit.hpp" +#include +#include + +namespace pineforge::feed { +namespace { +std::int64_t milliseconds(const Json& value) { + // WebSocket rows carry integers, REST rows decimal strings. + const auto text = value.kind == Json::Kind::String ? value.text() : value.value; + if (value.kind != Json::Kind::String && value.kind != Json::Kind::Number) throw Error(23, "invalid Bybit time"); + std::int64_t result = 0; + const auto parsed = std::from_chars(text.data(), text.data() + text.size(), result); + if (text.empty() || text.find_first_not_of("0123456789") != std::string::npos || parsed.ec != std::errc{} || + parsed.ptr != text.data() + text.size()) throw Error(23, "invalid Bybit time"); + return result; +} +RestPolicy bybit_rest() { + RestPolicy policy; + policy.endpoints = {"/v5/market/kline", "/v5/market/instruments-info"}; + // 600 requests per 5 seconds per IP; this spends well under a fifth of it. + policy.milliseconds_per_request = 100; + policy.throttled = [](const std::string& body) { return body.find("\"retCode\":10006") != std::string::npos; }; + // Bybit answers HTTP 403 for its roughly ten-minute IP rate ban: a restart after the ban recovers. + policy.rejected = [](long status, const std::string&) { + if (status == 403) throw Error(20, "Bybit IP rate ban (HTTP 403); restart after the venue's ban window"); + }; + // Server timeout and server error: the venue asks to retry. + policy.transient = [](long, const std::string& body) { + for (const char* code : {"10000", "10016"}) + for (const char* end : {",", "}"}) + if (body.find(std::string("\"retCode\":") + code + end) != std::string::npos) return true; + return false; + }; + return policy; +} +std::string category(const Config& config) { return config.market == "linear" ? "linear" : "spot"; } +} + +Kline bybit_kline(const Json& row) { + if (row.at("interval").text() != "1") throw Error(23, "only one-minute Bybit klines are supported"); + Kline kline; + kline.bar = {milliseconds(row.at("start")), row.at("open").text(), row.at("high").text(), row.at("low").text(), + row.at("close").text(), row.at("volume").text()}; + kline.bar.validate(); + if (milliseconds(row.at("end")) != kline.bar.ts + 59999) throw Error(23, "invalid inclusive one-minute Bybit kline end"); + const auto& confirm = row.at("confirm"); + if (confirm.kind != Json::Kind::Bool) throw Error(23, "confirmation must be a JSON boolean"); + kline.confirmed = confirm.value == "true"; + return kline; +} +std::vector bybit_rows(const Json& list, std::int64_t start, std::size_t count) { + if (list.kind != Json::Kind::Array || list.items.size() > 1000) throw Error(23, "invalid Bybit kline list"); + std::vector result; + auto expected = start; + for (auto item = list.items.rbegin(); item != list.items.rend(); ++item) { + if (item->kind != Json::Kind::Array || item->items.size() != 7) throw Error(23, "Bybit REST kline must have seven fields"); + const auto& row = item->items; + Kline kline; + kline.bar = {milliseconds(row[0]), row[1].text(), row[2].text(), row[3].text(), row[4].text(), row[5].text()}; + kline.bar.validate(); + if (kline.bar.ts != expected) throw Error(20, "historical klines have a missing, unordered or unexpected minute"); + expected += 60000; + result.push_back(kline); + } + if (result.size() != count) throw Error(20, "historical klines have a missing minute"); + return result; +} +Frame bybit_frame(const std::string& message) { + try { + const auto envelope = parse_json(message); + const auto* operation = envelope.find("op"); + if (!operation) return Frame::Data; + if (operation->text() == "subscribe") { + const auto* success = envelope.find("success"); + if (!success || success->kind != Json::Kind::Bool || success->value != "true") throw Error(23, "Bybit rejected the subscription"); + } + return Frame::Control; + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid Bybit WebSocket JSON"); } +} + +Bybit::Bybit(const Config& config) : config_(config), http_(config, bybit_rest()) { + const auto instruments = result("/v5/market/instruments-info?category=" + category(config_) + "&symbol=" + config_.symbol); + try { + const auto& list = instruments.at("list"); + if (list.kind != Json::Kind::Array || list.items.size() != 1) throw Error(23, "unknown Bybit symbol"); + const auto& row = list.items.front(); + if (row.at("symbol").text() != config_.symbol) throw Error(23, "Bybit instrument metadata does not match the symbol"); + if (row.at("status").text() != "Trading") throw Error(23, "Bybit instrument is not trading"); + if (config_.market == "linear") { + const auto type = row.at("contractType").text(); + if (type != "LinearPerpetual" && type != "LinearFutures") throw Error(23, "Bybit instrument is not a linear contract"); + } + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid Bybit instrument metadata"); } +} +Json Bybit::result(const std::string& path) { + const auto response = http_.get(path, 1); + try { + const auto& code = response.at("retCode"); + if (code.kind != Json::Kind::Number || code.value != "0") throw Error(23, "Bybit rejected the public request (retCode " + code.value + ")"); + return response.at("result"); + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid Bybit REST envelope"); } +} +Connection Bybit::connection() const { + return {"/v5/public/" + category(config_), {"{\"op\":\"subscribe\",\"args\":[\"kline.1." + config_.symbol + "\"]}"}, + "{\"op\":\"ping\"}", &bybit_frame}; +} +std::vector Bybit::decode(const std::string& message) const { + try { + const auto envelope = parse_json(message); + if (envelope.at("topic").text() != "kline.1." + config_.symbol) throw Error(23, "unsupported Bybit topic"); + const auto& data = envelope.at("data"); + if (data.kind != Json::Kind::Array || data.items.size() > 1024) throw Error(23, "invalid Bybit data array"); + // A push can carry the minute that just closed (confirm=true) and the next forming one. + std::vector events; + for (const auto& row : data.items) events.push_back({VenueEvent::Kind::Kline, {}, bybit_kline(row)}); + return events; + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid Bybit message shape"); } +} +std::vector Bybit::klines(std::int64_t start, std::int64_t end) { + if (start < 0 || start % 60000 || end <= start || end % 60000) throw Error(23, "invalid exclusive kline range"); + // `start` and `end` are inclusive candle starts and rows arrive newest first: a wider range than the + // limit would silently drop its oldest rows, so each request asks for exactly the rows it can hold. + const auto count = static_cast(std::min(1000, (end - start) / 60000)); + const auto list = result("/v5/market/kline?category=" + category(config_) + "&symbol=" + config_.symbol + "&interval=1&start=" + + std::to_string(start) + "&end=" + std::to_string(start + static_cast(count - 1) * 60000) + "&limit=" + std::to_string(count)); + try { return bybit_rows(list.at("list"), start, count); } + catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid Bybit kline result"); } +} +} diff --git a/native/feed/src/bybit.hpp b/native/feed/src/bybit.hpp new file mode 100644 index 0000000..fbb1be4 --- /dev/null +++ b/native/feed/src/bybit.hpp @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "transport.hpp" +#include "venue.hpp" + +namespace pineforge::feed { +// {start, end, interval, open, close, high, low, volume, turnover, confirm, timestamp}; only confirm=true closes. +Kline bybit_kline(const Json& row); +// REST rows arrive newest first: they are reversed and must cover [start, start + count minutes) exactly. +std::vector bybit_rows(const Json& list, std::int64_t start, std::size_t count); +Frame bybit_frame(const std::string& message); + +// Bybit v5 spot and linear: confirmed one-minute bars only. Ticks are refused at startup: trade IDs are +// not a contiguous cursor and the public REST cannot page back through trades. Bars never come from +// trades, because trade-built bars disagree with Bybit candles. +class Bybit final : public Venue { + Config config_; + HttpClient http_; + Json result(const std::string& path); +public: + explicit Bybit(const Config& config); + Connection connection() const override; + std::vector decode(const std::string& message) const override; + std::vector history(std::uint64_t, std::size_t) override { throw Error(23, "Bybit has no pageable trade history"); } + std::vector klines(std::int64_t start, std::int64_t end) override; + std::string kline_source() const override { return "/v5/market/kline"; } +}; +} diff --git a/native/feed/src/core.cpp b/native/feed/src/core.cpp index a0dd869..666df49 100644 --- a/native/feed/src/core.cpp +++ b/native/feed/src/core.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -238,6 +239,85 @@ Decimal Decimal::add(const Decimal& other) const { return Decimal(sum, scale); } +Decimal Decimal::multiply(const Decimal& other) const { + if (zero() || other.zero()) return Decimal("0", 0); + // The constructor bounds the product: at most 128 coefficient digits and 32 fractional digits. + std::string product(coefficient_.size() + other.coefficient_.size(), '0'); + for (std::size_t left = coefficient_.size(); left > 0; --left) { + int carry = 0; + for (std::size_t right = other.coefficient_.size(); right > 0; --right) { + auto& digit = product[left + right - 1]; + const int value = (digit - '0') + (coefficient_[left - 1] - '0') * (other.coefficient_[right - 1] - '0') + carry; + digit = static_cast('0' + value % 10); + carry = value / 10; + } + product[left - 1] = static_cast(product[left - 1] + carry); + } + return Decimal(product, scale_ + other.scale_); +} + +std::string Decimal::str() const { + if (!scale_) return coefficient_; + auto digits = coefficient_; + if (digits.size() <= scale_) digits.insert(0, scale_ - digits.size() + 1, '0'); + digits.insert(digits.size() - scale_, 1, '.'); + return digits; +} + +std::string scaled_quantity(const std::string& token, const std::string& multiplier) { + const Decimal amount(token); + if (multiplier == "1") return token; + return amount.multiply(Decimal(multiplier)).str(); +} + +std::string json_member(const std::string& document, const std::string& name) { + std::size_t index = 0; + const auto space = [&] { while (index < document.size() && std::isspace(static_cast(document[index]))) ++index; }; + const auto string_end = [&](std::size_t start) { + for (auto cursor = start + 1; cursor < document.size(); ++cursor) { + if (document[cursor] == '\\') ++cursor; + else if (document[cursor] == '"') return cursor + 1; + } + throw Error(23, "unterminated JSON string"); + }; + const auto value_end = [&](std::size_t start) { + if (start >= document.size()) throw Error(23, "truncated JSON member"); + if (document[start] == '"') return string_end(start); + std::size_t depth = 0, cursor = start; + while (cursor < document.size()) { + const char letter = document[cursor]; + if (letter == '"') { cursor = string_end(cursor); continue; } + if (letter == '{' || letter == '[') ++depth; + else if (letter == '}' || letter == ']') { + if (!depth) return cursor; + if (!--depth) return cursor + 1; + } else if (letter == ',' && !depth) return cursor; + ++cursor; + } + throw Error(23, "truncated JSON member"); + }; + space(); + if (index >= document.size() || document[index] != '{') throw Error(23, "expected a JSON object document"); + ++index; + for (;;) { + space(); + if (index >= document.size() || document[index] != '"') throw Error(23, "JSON member not found: " + name); + const auto key_end = string_end(index); + const auto key = document.substr(index + 1, key_end - index - 2); + index = key_end; + space(); + if (index >= document.size() || document[index] != ':') throw Error(23, "invalid JSON object member"); + ++index; + space(); + const auto start = index; + index = value_end(start); + if (key == name) return document.substr(start, index - start); + space(); + if (index >= document.size() || document[index] != ',') throw Error(23, "JSON member not found: " + name); + ++index; + } +} + void Trade::validate() const { if (!id || ts < 0 || ts > std::numeric_limits::max() - 60000 || Decimal(price).zero() || Decimal(qty).zero()) throw Error(23, "invalid positive venue trade"); @@ -247,8 +327,10 @@ std::string Trade::wire() const { return "{\"type\":\"tick\",\"ts\":" + std::to_string(ts) + ",\"seq\":" + std::to_string(id) + ",\"price\":" + price + ",\"qty\":" + qty + "}"; } +// Exact decimal values, not lexemes: a venue may render one number as "100.0" on its WebSocket and as +// "100" over REST. Emitted tokens stay verbatim; only the comparison is by value. bool Trade::operator==(const Trade& other) const { - return id == other.id && ts == other.ts && price == other.price && qty == other.qty; + return id == other.id && ts == other.ts && Decimal(price) == Decimal(other.price) && Decimal(qty) == Decimal(other.qty); } void Bar::validate() const { if (ts < 0 || ts % 60000 || ts > std::numeric_limits::max() - 60000) @@ -265,7 +347,8 @@ std::string Bar::wire() const { ",\"h\":" + high + ",\"l\":" + low + ",\"c\":" + close + ",\"v\":" + volume + "}}"; } bool Bar::operator==(const Bar& other) const { - return ts == other.ts && open == other.open && high == other.high && low == other.low && close == other.close && volume == other.volume; + return ts == other.ts && Decimal(open) == Decimal(other.open) && Decimal(high) == Decimal(other.high) && + Decimal(low) == Decimal(other.low) && Decimal(close) == Decimal(other.close) && Decimal(volume) == Decimal(other.volume); } void Aggregate::add(const Trade& trade) { trade.validate(); diff --git a/native/feed/src/core.hpp b/native/feed/src/core.hpp index 69b4439..c35d229 100644 --- a/native/feed/src/core.hpp +++ b/native/feed/src/core.hpp @@ -37,6 +37,9 @@ class Decimal { bool zero() const { return coefficient_ == "0"; } int compare(const Decimal& other) const; Decimal add(const Decimal& other) const; + Decimal multiply(const Decimal& other) const; + // Canonical fixed-point token: no sign, no exponent, no trailing fractional zeros. + std::string str() const; bool operator==(const Decimal& other) const { return compare(other) == 0; } }; @@ -71,6 +74,10 @@ struct Aggregate { void add(const Trade& trade); void reconcile(const Kline& kline, bool require_ids) const; }; +// Contract-to-base conversion for a venue quantity token; the token is kept verbatim when the multiplier is 1. +std::string scaled_quantity(const std::string& token, const std::string& multiplier); +// The JSON text of one top-level member, found by an exact scan; for documents above the parser bound. +std::string json_member(const std::string& document, const std::string& name); Trade normalized_trade(const Json& event); Bar normalized_bar(const Json& event); @@ -78,6 +85,7 @@ struct Config { std::string venue = "binance", market = "spot", symbol, mode = "bars", state_dir; std::string rest_url = "https://api.binance.com"; std::string ws_url = "wss://stream.binance.com:443"; + std::string qty_multiplier = "1"; std::int64_t start = -1, end = -1; bool resume = false, allow_insecure = false; std::uint64_t output_from = 0, max_messages = 0; @@ -85,5 +93,8 @@ struct Config { std::uint64_t max_replay_seconds = 60; std::size_t max_queue_bytes = 16 * 1024 * 1024; std::uint64_t reconnect_seconds = 23 * 3600 + 55 * 60; + std::uint64_t keepalive_seconds = 20; }; +// Raw ticks and USD-M aggregate prints share the tick/time journal grammar. +inline bool tick_mode(const std::string& mode) { return mode == "ticks" || mode == "agg-ticks"; } } diff --git a/native/feed/src/main.cpp b/native/feed/src/main.cpp index bfb53e7..0bb6c1b 100644 --- a/native/feed/src/main.cpp +++ b/native/feed/src/main.cpp @@ -1,5 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 -#include "binance.hpp" +#include "session.hpp" +#include "transport.hpp" +#include #include #include #include @@ -22,11 +24,58 @@ struct StdoutFlags { ~StdoutFlags() { if (saved >= 0) ::fcntl(STDOUT_FILENO, F_SETFL, saved); } }; void help() { - std::cout << "pineforge-feed warmup --venue binance --market spot --symbol SYMBOL --start UTC|MS --end UTC|MS --output FILE\n" - "pineforge-feed run --venue binance --market spot --symbol SYMBOL --mode bars|ticks --state-dir DIR [--start UTC|MS | --resume] [--output-from INDEX]\n" - "Limits: --max-messages N --max-log-bytes N --max-replay-seconds N --max-queue-bytes N --reconnect-seconds N\n" + std::cout << "pineforge-feed warmup --venue VENUE --market MARKET --symbol SYMBOL --start UTC|MS --end UTC|MS --output FILE\n" + "pineforge-feed run --venue VENUE --market MARKET --symbol SYMBOL --mode MODE --state-dir DIR [--start UTC|MS | --resume] [--output-from INDEX]\n" + "Venues and modes:\n" + " binance spot BTCUSDT bars | ticks\n" + " binance usdm BTCUSDT bars | agg-ticks (aggregate prints, not raw trades; bars can\n" + " differ from klines at minute edges: use bars for kline-exact bars)\n" + " okx spot BTC-USDT bars | ticks\n" + " okx swap BTC-USDT-SWAP bars | ticks (linear swaps; base quantities)\n" + " bybit spot|linear BTCUSDT bars\n" + "Limits: --max-messages N --max-log-bytes N --max-replay-seconds N --max-queue-bytes N --reconnect-seconds N --keepalive-seconds N\n" "Testing/public origins: --rest-url ORIGIN --ws-url ORIGIN [--allow-insecure-http (loopback only)]\n"; } +// Venue, market and mode gates are checked before any network access. +void gate(pineforge::feed::Config& config, bool warmup, bool rest_given, bool ws_given) { + using pineforge::feed::Error; + const auto& venue = config.venue; + const auto& market = config.market; + const auto& mode = config.mode; + std::string rest, ws, grammar = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + if (venue == "coinbase") + throw Error(23, "Coinbase is deferred: it has no confirmed one-minute candle stream, and its REST and WebSocket trade " + "timestamps disagree, so no lossless tick paging exists"); + if (venue == "binance" && market == "spot") { + rest = "https://api.binance.com", ws = "wss://stream.binance.com:443"; + if (!warmup && mode != "bars" && mode != "ticks") throw Error(23, "Binance spot modes are bars and ticks"); + } else if (venue == "binance" && market == "usdm") { + rest = "https://fapi.binance.com", ws = "wss://fstream.binance.com"; + if (!warmup && mode == "ticks") + throw Error(23, "Binance USD-M raw trades have no public REST history; use --mode agg-ticks for aggregate prints"); + if (!warmup && mode != "bars" && mode != "agg-ticks") throw Error(23, "Binance USD-M modes are bars and agg-ticks"); + } else if (venue == "okx" && (market == "spot" || market == "swap")) { + rest = "https://www.okx.com", ws = "wss://ws.okx.com:8443"; + grammar += '-'; + if (!warmup && mode != "bars" && mode != "ticks") throw Error(23, "OKX modes are bars and ticks (trades-all raw prints)"); + } else if (venue == "bybit" && (market == "spot" || market == "linear")) { + rest = "https://api.bybit.com", ws = "wss://stream.bybit.com"; + if (!warmup && (mode == "ticks" || mode == "agg-ticks")) + throw Error(23, "Bybit tick mode is refused: Bybit trade IDs are not a contiguous cursor and its REST cannot page back " + "through trades; use --mode bars (confirmed kline.1 candles)"); + if (!warmup && mode != "bars") throw Error(23, "Bybit supports bars only"); + } else throw Error(23, "unsupported venue/market: binance spot|usdm, okx spot|swap, bybit spot|linear"); + if (config.symbol.empty() || config.symbol.size() > 32 || config.symbol.find_first_not_of(grammar) != std::string::npos) + throw Error(23, venue == "okx" ? "use an uppercase OKX instrument ID such as BTC-USDT or BTC-USDT-SWAP" : "use an uppercase ASCII symbol"); + if (venue == "okx") { + const auto dashes = std::count(config.symbol.begin(), config.symbol.end(), '-'); + const bool swap = config.symbol.size() > 5 && config.symbol.compare(config.symbol.size() - 5, 5, "-SWAP") == 0; + if (market == "spot" ? dashes != 1 || swap : dashes != 2 || !swap || config.symbol.front() == '-') + throw Error(23, market == "spot" ? "OKX spot instruments look like BTC-USDT" : "OKX perpetual swaps look like BTC-USDT-SWAP"); + } + if (!rest_given) config.rest_url = rest; + if (!ws_given) config.ws_url = ws; +} } int run(int argc, char** argv) { using namespace pineforge::feed; @@ -64,17 +113,17 @@ int run(int argc, char** argv) { else if (key == "--max-replay-seconds") config.max_replay_seconds = unsigned_value(value); else if (key == "--max-queue-bytes") config.max_queue_bytes = unsigned_value(value); else if (key == "--reconnect-seconds") config.reconnect_seconds = unsigned_value(value); + else if (key == "--keepalive-seconds") config.keepalive_seconds = unsigned_value(value); else if (key == "--rest-url") config.rest_url = value; else if (key == "--ws-url") config.ws_url = value; else throw Error(23, "unknown CLI option: " + key); } - if (config.venue != "binance" || config.market != "spot") throw Error(23, "this release supports public Binance spot only"); - if (config.symbol.empty() || config.symbol.size() > 32 || config.symbol.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") != std::string::npos) - throw Error(23, "use an uppercase ASCII Binance symbol"); - if (config.mode != "bars" && config.mode != "ticks") throw Error(23, "mode must be bars or ticks"); + gate(config, command == "warmup", seen.count("--rest-url") != 0, seen.count("--ws-url") != 0); if (!config.max_log_bytes || !config.max_queue_bytes || !config.max_replay_seconds || config.max_replay_seconds > 3600 || !config.reconnect_seconds || config.reconnect_seconds > 86100) throw Error(23, "budgets must be positive; reconnect must precede the 24-hour connection limit"); + // OKX closes a connection after 30 s without traffic; Bybit recommends a ping every 20 s. + if (!config.keepalive_seconds || config.keepalive_seconds > 25) throw Error(23, "--keepalive-seconds must be 1..25"); check_runtime_curl(); validate_origin(config.rest_url, false, config.allow_insecure); validate_origin(config.ws_url, true, config.allow_insecure); diff --git a/native/feed/src/okx.cpp b/native/feed/src/okx.cpp new file mode 100644 index 0000000..4c3a379 --- /dev/null +++ b/native/feed/src/okx.cpp @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "okx.hpp" +#include + +namespace pineforge::feed { +namespace { +// OKX sends identifiers and times as decimal strings. +template Integer digits(const Json& value) { + const auto text = value.text(); + Integer result{}; + const auto parsed = std::from_chars(text.data(), text.data() + text.size(), result); + if (text.empty() || (text.size() > 1 && text.front() == '0') || text.find_first_not_of("0123456789") != std::string::npos || + parsed.ec != std::errc{} || parsed.ptr != text.data() + text.size()) + throw Error(23, "invalid OKX decimal identifier or time"); + return result; +} +const std::vector& rows(const Json& value, std::size_t maximum) { + if (value.kind != Json::Kind::Array || value.items.size() > maximum) throw Error(23, "invalid OKX data array"); + return value.items; +} +RestPolicy okx_rest() { + RestPolicy policy; + policy.endpoints = {"/api/v5/public/instruments", "/api/v5/market/history-trades", "/api/v5/market/history-candles"}; + // Each endpoint allows 20 requests per 2 seconds per IP: spend at most half. + policy.milliseconds_per_request = 200; + policy.throttled = [](const std::string& body) { return body.find("\"code\":\"50011\"") != std::string::npos; }; + // Service unavailable, endpoint timeout, system busy, system error: the venue asks to retry. + policy.transient = [](long, const std::string& body) { + for (const char* code : {"50001", "50004", "50013", "50026"}) + if (body.find(std::string("\"code\":\"") + code + "\"") != std::string::npos) return true; + return false; + }; + return policy; +} +} + +Trade okx_trade(const Json& row, const std::string& symbol, const std::string& multiplier) { + if (row.find("count")) throw Error(23, "aggregated OKX trades (count) are never raw prints"); + if (row.at("instId").text() != symbol) throw Error(23, "OKX instrument changed"); + Trade trade{digits(row.at("tradeId")), digits(row.at("ts")), row.at("px").text(), + scaled_quantity(row.at("sz").text(), multiplier)}; + trade.validate(); + return trade; +} +Kline okx_candle(const Json& row, const std::string& multiplier) { + const auto& fields = rows(row, 9); + if (fields.size() != 9) throw Error(23, "OKX candle must have nine fields"); + Kline kline; + kline.bar = {digits(fields[0]), fields[1].text(), fields[2].text(), fields[3].text(), fields[4].text(), + scaled_quantity(fields[5].text(), multiplier)}; + kline.bar.validate(); + const auto confirm = fields[8].text(); + if (confirm != "0" && confirm != "1") throw Error(23, "OKX candle confirm must be \"0\" or \"1\""); + kline.confirmed = confirm == "1"; + return kline; +} +std::string okx_multiplier(const Json& row, const std::string& market, const std::string& symbol) { + if (row.at("instId").text() != symbol) throw Error(23, "OKX instrument metadata does not match the symbol"); + if (row.at("state").text() != "live") throw Error(23, "OKX instrument is not live"); + const auto type = row.at("instType").text(); + if (market == "spot") { + if (type != "SPOT") throw Error(23, "OKX instrument is not a spot pair"); + return "1"; + } + if (type != "SWAP") throw Error(23, "OKX instrument is not a perpetual swap"); + if (row.at("ctType").text() != "linear") + throw Error(23, "inverse OKX swaps are refused: their contracts are quote-denominated, not base quantities"); + if (row.at("ctValCcy").text() != symbol.substr(0, symbol.find('-'))) throw Error(23, "OKX contract value is not in the base currency"); + const Decimal value(row.at("ctVal").text()), multiple(row.at("ctMult").text()); + if (value.zero() || multiple.zero()) throw Error(23, "OKX contract value or multiplier is zero"); + return value.multiply(multiple).str(); +} +Frame okx_frame(const std::string& message) { + if (message == "pong") return Frame::Control; + try { + const auto envelope = parse_json(message); + const auto* event = envelope.find("event"); + if (!event) return Frame::Data; + const auto name = event->text(); + if (name == "subscribe" || name == "unsubscribe") return Frame::Control; + if (name == "error") { + const auto* code = envelope.find("code"); + throw Error(23, "OKX rejected the subscription (code " + (code && code->kind == Json::Kind::String ? code->text() : std::string("?")) + ")"); + } + // 64008: the venue closes this connection for an upgrade; reconnect with overlap now. + if (name == "notice") return Frame::Retire; + log("warn", "unknown_stream_event", Json::object({{"type", Json::string(name)}})); + return Frame::Control; + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid OKX WebSocket JSON"); } +} + +Okx::Okx(const Config& config) : config_(config), http_(config, okx_rest()) { + multiplier_ = instrument_multiplier(); + log("info", "instrument_verified", Json::object({{"symbol", Json::string(config_.symbol)}, {"qty_multiplier", Json::string(multiplier_)}})); +} +std::string Okx::instrument_multiplier() { + const auto& instruments = data("/api/v5/public/instruments?instType=" + std::string(config_.market == "swap" ? "SWAP" : "SPOT") + + "&instId=" + config_.symbol).items; + if (instruments.size() != 1) throw Error(23, "unknown OKX instrument"); + return okx_multiplier(instruments.front(), config_.market, config_.symbol); +} +// Every reconnect re-reads ctVal x ctMult: quantities already emitted were converted with the old one. +void Okx::reverify() { + if (instrument_multiplier() != multiplier_) throw Error(21, "OKX contract multiplier changed during the run"); +} +Json Okx::data(const std::string& path) { + const auto response = http_.get(path, 1); + try { + if (response.at("code").text() != "0") throw Error(23, "OKX rejected the public request (code " + response.at("code").text() + ")"); + auto result = response.at("data"); + (void)rows(result, 1000); + return result; + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid OKX REST envelope"); } +} +Connection Okx::connection() const { + const auto argument = [&](const char* channel) { return std::string("{\"channel\":\"") + channel + "\",\"instId\":\"" + config_.symbol + "\"}"; }; + const auto subscribe = "{\"op\":\"subscribe\",\"args\":[" + argument("candle1m") + + (config_.mode == "ticks" ? "," + argument("trades-all") : std::string()) + "]}"; + // Business endpoint: trades-all and candles live there. The text keepalive must precede 30 s idle. + return {"/ws/v5/business", {subscribe}, "ping", &okx_frame}; +} +std::vector Okx::decode(const std::string& message) const { + try { + const auto envelope = parse_json(message); + if (const auto* event = envelope.find("event")) { + log("info", "source_notice", Json::object({{"type", Json::string(event->text())}})); + return {}; + } + const auto& argument = envelope.at("arg"); + const auto channel = argument.at("channel").text(); + if (argument.at("instId").text() != config_.symbol) throw Error(23, "public stream instrument changed"); + if (channel == "trades") throw Error(23, "the aggregated OKX trades channel is never a tick source"); + if (channel != "candle1m" && channel != "trades-all") throw Error(23, "unsupported OKX channel"); + std::vector events; + for (const auto& row : rows(envelope.at("data"), 1024)) { + if (channel == "candle1m") events.push_back({VenueEvent::Kind::Kline, {}, okx_candle(row, multiplier_)}); + else events.push_back({VenueEvent::Kind::Trade, okx_trade(row, config_.symbol, multiplier_), {}}); + } + return events; + } catch (const Error&) { throw; } + catch (const std::exception&) { throw Error(23, "invalid OKX message shape"); } +} +std::vector Okx::history(std::uint64_t from, std::size_t limit) { + if (!from || !limit || limit > 1000) throw Error(23, "invalid raw-history page request"); + std::vector result; + auto next = from; + while (result.size() < limit) { + const auto count = std::min(100, limit - result.size()); + // `after` and `before` are exclusive trade-ID bounds; rows arrive newest first. + const auto page = data("/api/v5/market/history-trades?instId=" + config_.symbol + "&type=1&after=" + + std::to_string(next + count) + "&before=" + std::to_string(next - 1) + "&limit=" + std::to_string(count)); + const auto& items = rows(page, count); + for (auto row = items.rbegin(); row != items.rend(); ++row) { + const auto trade = okx_trade(*row, config_.symbol, multiplier_); + if (trade.id != next) throw Error(20, "trade-id page has a missing or nonunit venue trade ID"); + result.push_back(trade); + ++next; + } + if (items.size() < count) break; + } + return result; +} +// The time lookup alone is not proof: REST can trail the newest prints, and prints of one millisecond need +// not come back in ID order. Walk forward by ID until the next print is at or after the minute; only then +// is the candidate the last print before it. Nothing is anchored until that is seen. +std::optional Okx::walk_to(std::int64_t minute) { + // type=2 pages by time: `after` returns prints strictly earlier than the minute, newest first. + const auto page = data("/api/v5/market/history-trades?instId=" + config_.symbol + "&type=2&after=" + std::to_string(minute) + "&limit=1"); + const auto& items = rows(page, 1); + if (items.empty()) throw Error(20, "no print precedes the start minute within the venue history"); + auto candidate = okx_trade(items.front(), config_.symbol, multiplier_); + if (candidate.ts >= minute) throw Error(20, "predecessor lookup returned a print at or after the start minute"); + for (;;) { + const auto later = history(candidate.id + 1, 100); + for (const auto& trade : later) { + if (trade.ts >= minute) return candidate; + candidate = trade; + } + if (later.size() < 100) return std::nullopt; + } +} +// A WebSocket print at or after the minute exists: REST must show its successor within about 15 seconds. +Trade Okx::predecessor(std::int64_t minute) { + for (unsigned int wait = 0; wait < 4; ++wait) { + if (const auto found = walk_to(minute)) return *found; + pause_for(std::chrono::milliseconds((config_.allow_insecure ? 100 : 1000) << wait)); + } + throw Error(20, "the print after the start predecessor is not yet available over REST"); +} +std::optional Okx::predecessor_if_ready(std::int64_t minute) { return walk_to(minute); } +std::vector Okx::klines(std::int64_t start, std::int64_t end) { + if (start <= 0 || start % 60000 || end <= start || end % 60000) throw Error(23, "invalid exclusive kline range"); + std::vector result; + auto next = start; + while (next < end) { + const auto window = std::min(end, next + 100 * 60000); + const auto page = data("/api/v5/market/history-candles?instId=" + config_.symbol + "&bar=1m&after=" + std::to_string(window) + + "&before=" + std::to_string(next - 1) + "&limit=100"); + const auto& items = rows(page, 100); + bool forming = false; + for (auto row = items.rbegin(); row != items.rend(); ++row) { + const auto kline = okx_candle(*row, multiplier_); + if (kline.bar.ts != next || kline.bar.ts >= window) throw Error(20, "historical candles have a missing or unexpected minute"); + // A REST row is a closed candle only with confirm "1"; the confirmed prefix ends at a forming one. + if (!kline.confirmed) { forming = true; break; } + result.push_back(kline); + next += 60000; + } + if (forming || next < window) break; + } + if (result.empty()) throw Error(20, "historical minute is unavailable or not yet confirmed"); + return result; +} +} diff --git a/native/feed/src/okx.hpp b/native/feed/src/okx.hpp new file mode 100644 index 0000000..eec90bd --- /dev/null +++ b/native/feed/src/okx.hpp @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "transport.hpp" +#include "venue.hpp" + +namespace pineforge::feed { +// One trades-all print; `sz` becomes base units through the instrument multiplier. An aggregated +// `trades` row (it carries `count`) is refused: it is never a raw print. +Trade okx_trade(const Json& row, const std::string& symbol, const std::string& multiplier); +// [ts, o, h, l, c, vol, volCcy, volCcyQuote, confirm]; only confirm "1" is a closed candle. +Kline okx_candle(const Json& row, const std::string& multiplier); +// Validates one public instrument row and returns base units per contract (ctVal x ctMult), "1" for spot. +std::string okx_multiplier(const Json& row, const std::string& market, const std::string& symbol); +Frame okx_frame(const std::string& message); + +class Okx final : public Venue { + Config config_; + HttpClient http_; + std::string multiplier_ = "1"; + Json data(const std::string& path); + std::string instrument_multiplier(); + // The time lookup plus the walk by ID; nullopt when no print at or after the minute is visible yet. + std::optional walk_to(std::int64_t minute); +public: + explicit Okx(const Config& config); + Connection connection() const override; + std::vector decode(const std::string& message) const override; + std::vector history(std::uint64_t from, std::size_t limit = 1000) override; + Trade predecessor(std::int64_t minute) override; + std::optional predecessor_if_ready(std::int64_t minute) override; + void reverify() override; + std::vector klines(std::int64_t start, std::int64_t end) override; + TickProof tick_proof() const override { return TickProof::NextPrintFence; } + // history-trades covers the last three months; 89 days stays inside every three-month span. + std::int64_t retention_ms() const override { return 89LL * 86400000; } + std::string qty_multiplier() const override { return multiplier_; } + std::string kline_source() const override { return "/api/v5/market/history-candles"; } +}; +} diff --git a/native/feed/src/session.cpp b/native/feed/src/session.cpp new file mode 100644 index 0000000..415b76e --- /dev/null +++ b/native/feed/src/session.cpp @@ -0,0 +1,626 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "session.hpp" +#include "binance.hpp" +#include "bybit.hpp" +#include "okx.hpp" +#include "transport.hpp" +#include +#include +#include +#include + +namespace pineforge::feed { +namespace { +std::size_t trade_bytes(const Trade& trade) { return trade.price.size() + trade.qty.size() + 192; } +std::string time_line(std::int64_t ts) { return "{\"type\":\"time\",\"ts\":" + std::to_string(ts) + "}"; } +bool same_prints(const std::vector& fetched, const std::deque& trades) { + if (fetched.size() != trades.size()) return false; + for (std::size_t index = 0; index < trades.size(); ++index) + if (!(trades[index] == fetched[index])) return false; + return true; +} +bool same_bars(const std::vector& fetched, const std::deque& bars) { + if (fetched.size() != bars.size()) return false; + for (std::size_t index = 0; index < bars.size(); ++index) + if (!(bars[index] == fetched[index].bar)) return false; + return true; +} +} + +// A venue's REST row for the minute that just closed can trail its WebSocket close by a moment: a +// disagreement must survive spaced re-reads (about 7 seconds) before it counts as a revision (21). +std::vector settled_klines(Venue& venue, const Config& config, std::int64_t start, std::int64_t end, + const std::function&)>& agrees) { + for (unsigned int attempt = 0;; ++attempt) { + try { + auto fetched = venue.klines(start, end); + if (agrees(fetched) || attempt == 3) return fetched; + } catch (const Error& failure) { + // A missing or still-unconfirmed newest row (20) is the same lag; it persists only after re-reads. + if (failure.code != 20 || attempt == 3) throw; + } + log("warn", "rest_overlap_trailing", Json::object({{"attempt", Json::number(std::to_string(attempt + 1))}})); + pause_for(std::chrono::milliseconds((config.allow_insecure ? 100 : 1000) << attempt)); + } +} +// The same spaced re-read for the newest emitted prints, which REST can also trail. +std::vector settled_prints(Venue& venue, const Config& config, const std::deque& trades) { + for (unsigned int attempt = 0;; ++attempt) { + auto fetched = venue.history(trades.front().id, trades.size()); + if (same_prints(fetched, trades) || attempt == 3) return fetched; + log("warn", "rest_overlap_trailing", Json::object({{"attempt", Json::number(std::to_string(attempt + 1))}})); + pause_for(std::chrono::milliseconds((config.allow_insecure ? 100 : 1000) << attempt)); + } +} + +std::unique_ptr make_venue(const Config& config) { + if (config.venue == "binance" && config.market == "spot") return std::make_unique(config); + if (config.venue == "binance" && config.market == "usdm") return std::make_unique(config); + if (config.venue == "okx" && (config.market == "spot" || config.market == "swap")) return std::make_unique(config); + if (config.venue == "bybit" && (config.market == "spot" || config.market == "linear")) return std::make_unique(config); + throw Error(23, "unsupported venue or market"); +} + +Session::Session(State& state, Venue& venue, std::function output) + : state_(state), venue_(venue), output_(std::move(output)) {} +void Session::emit(const std::string& line, const std::optional& proof) { + if (stopping) throw Stopped{}; + state_.stage(line, proof); + ++new_messages_; + if (state_.config().max_messages && new_messages_ >= state_.config().max_messages) { + publish(); + throw Stopped{}; + } + if (state_.staged() >= 4096) publish(); +} +void Session::publish() { + for (const auto& line : state_.flush()) output_(line); +} +void Session::salvage() noexcept { + try { publish(); } + catch (...) { + try { log("error", "verified_messages_not_committed"); } catch (...) {} + } +} +void Session::ingest(const VenueEvent& event) { + try { stage(event); } + catch (const Error&) { salvage(); throw; } + publish(); +} +void Session::replay() { + if (!replayed_) { + if (state_.config().resume) state_.visit(state_.config().output_from, output_); + replayed_ = true; + } + log("info", "overlap_verified", state_.status()); +} + +FeedSession::FeedSession(State& state, Venue& venue, std::function output) + : Session(state, venue, std::move(output)) { + if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() == "tick") { + const auto trade = normalized_trade(event); + if (trade.ts >= state_.cursor().cut) aggregate_.add(trade); + } + }); +} + +void FeedSession::connected() { + publish(); + venue_.reverify(); + const auto& trades = state_.recent_trades(); + if (!trades.empty()) { + const auto fetched = settled_prints(venue_, state_.config(), trades); + if (fetched.size() != trades.size()) throw Error(20, "reconnect raw overlap is unavailable"); + for (std::size_t index = 0; index < trades.size(); ++index) + if (!(trades[index] == fetched[index])) throw Error(21, "reconnect raw overlap changed"); + } else if (state_.cursor().predecessor) { + const auto fetched = venue_.history(state_.cursor().predecessor->id, 1); + if (!(fetched.front() == *state_.cursor().predecessor)) throw Error(21, "initial predecessor overlap changed"); + } + const auto& proofs = state_.config().mode == "bars" ? state_.recent_bars() : state_.cursor().proofs; + if (!proofs.empty()) { + const auto fetched = settled_klines(venue_, state_.config(), proofs.front().ts, proofs.back().ts + 60000, + [&](const auto& rows) { return same_bars(rows, proofs); }); + if (fetched.size() != proofs.size()) throw Error(20, "reconnect closed-bar overlap is unavailable"); + std::map verified; + if (state_.config().mode == "ticks") state_.visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() != "tick") return; + const auto trade = normalized_trade(event); + const auto minute = trade.ts - trade.ts % 60000; + if (minute >= proofs.front().ts && minute <= proofs.back().ts) verified[minute].add(trade); + }); + for (std::size_t index = 0; index < proofs.size(); ++index) { + if (!(proofs[index] == fetched[index].bar)) throw Error(21, "reconnect already-emitted bar changed"); + if (state_.config().mode == "ticks") { + try { verified.at(fetched[index].bar.ts).reconcile(fetched[index], false); } + catch (const std::exception&) { throw Error(21, "reconnect closed-minute count or raw proof changed"); } + } + } + } + replay(); +} + +void FeedSession::anchor(std::uint64_t first) { + if (state_.cursor().predecessor) return; + if (first <= 1) throw Error(20, "raw predecessor cannot be proven"); + const auto predecessor = venue_.history(first - 1, 1).front(); + state_.anchor(predecessor); +} +std::uint64_t FeedSession::next_id() const { + const auto previous = state_.cursor().seq ? state_.cursor().seq : state_.cursor().predecessor ? state_.cursor().predecessor->id : 0; + if (!previous || previous == std::numeric_limits::max()) throw Error(20, "raw-trade cursor is not anchored"); + return previous + 1; +} +void FeedSession::put(const Trade& trade) { + trade.validate(); + if (trade.ts < state_.cursor().start) return; + if (state_.cursor().seq && trade.id <= state_.cursor().seq) { + const auto previous = state_.trade(trade.id); + if (!previous || !(*previous == trade)) throw Error(21, "duplicate raw trade conflicts with the immutable prefix"); + return; + } + const auto existing = pending_.find(trade.id); + if (existing != pending_.end()) { + if (!(existing->second == trade)) throw Error(21, "buffered raw duplicate conflicts"); + return; + } + const auto bytes = trade_bytes(trade); + if (bytes > state_.config().max_queue_bytes - std::min(pending_bytes_, state_.config().max_queue_bytes)) + throw Error(22, "unconfirmed raw-trade buffer overrun"); + pending_bytes_ += bytes; + pending_.emplace(trade.id, trade); +} +void FeedSession::emit_trade(const Trade& trade) { + if (trade.ts < state_.cursor().last_tick_ts) throw Error(23, "venue matched time regressed along the contiguous raw-trade chain"); + if (trade.id != next_id() || trade.ts < state_.cursor().cut || trade.ts >= state_.cursor().cut + 60000) + throw Error(20, "raw trade violates the complete chronological minute prefix"); + const auto buffered = pending_.find(trade.id); + if (buffered != pending_.end() && !(buffered->second == trade)) + throw Error(21, "REST healing changed a buffered raw trade"); + aggregate_.add(trade); + emit(trade.wire()); + if (buffered != pending_.end()) { + pending_bytes_ -= trade_bytes(buffered->second); + pending_.erase(buffered); + } +} +void FeedSession::heal_to(std::uint64_t target) { + while (next_id() <= target) { + log("warn", "raw_gap_healing", Json::object({{"from_id", Json::number(std::to_string(next_id()))}, {"through_id", Json::number(std::to_string(target))}})); + const auto page = venue_.history(next_id()); + bool progressed = false; + for (const auto& trade : page) { + if (trade.id > target) break; + const auto buffered = pending_.find(trade.id); + if (buffered != pending_.end() && !(buffered->second == trade)) throw Error(21, "REST raw healing conflicts with WebSocket data"); + emit_trade(trade); + progressed = true; + } + if (!progressed) throw Error(20, "raw gap cannot advance within available history"); + } +} +void FeedSession::drain() { + while (!pending_.empty()) { + const auto trade = pending_.begin()->second; + if (trade.ts >= state_.cursor().cut + 60000) break; + if (trade.id < next_id()) throw Error(21, "buffered cursor fell behind the immutable prefix"); + if (trade.id > next_id()) heal_to(trade.id - 1); + emit_trade(trade); + } +} +// A minute whose own x=true kline was missed closes only on proof: a later x=true watermark, the contiguous +// raw-ID chain from the proven predecessor, the first print of a later minute as the fence, and REST n plus +// exact OHLCV. The REST row has no x=true or f..L, and none is manufactured for it. +void FeedSession::historical_minute() { + if (state_.cursor().cut + 60000 > watermark_) throw Error(20, "historical tick closure lacks a confirmed WebSocket watermark"); + if (!state_.cursor().predecessor) anchor(venue_.first_trade_id(state_.cursor().start)); + const auto minute = state_.cursor().cut; + std::optional fence; + while (!fence) { + const auto page = venue_.history(next_id()); + for (const auto& trade : page) { + const auto buffered = pending_.find(trade.id); + if (buffered != pending_.end() && !(buffered->second == trade)) throw Error(21, "historical raw overlap changed"); + if (trade.ts >= minute + 60000) { + fence = trade; + break; + } + emit_trade(trade); + } + } + if (!aggregate_.count || fence->id != next_id() || fence->id != aggregate_.last + 1) + throw Error(20, "historical minute lacks a contiguous raw prefix and next-minute fence"); + put(*fence); + const auto kline = venue_.klines(minute, minute + 60000).front(); + aggregate_.reconcile(kline, false); + log("info", "historical_tick_fence_verified", Json::object({{"minute", Json::number(std::to_string(minute))}, + {"first_id", Json::number(std::to_string(aggregate_.first))}, {"last_id", Json::number(std::to_string(aggregate_.last))}})); + emit(time_line(minute + 60000), kline.bar); + aggregate_ = Aggregate{}; +} + +void FeedSession::closed(const Kline& kline) { + if (!kline.confirmed) return; + const auto minute = kline.bar.ts; + watermark_ = std::max(watermark_, minute + 60000); + if (minute < state_.cursor().start) return; + if (minute < state_.cursor().cut) { + const auto old = state_.bar(minute); + if (old) { + if (!(*old == kline.bar)) throw Error(21, "already-emitted closed bar was revised"); + } + if (state_.config().mode == "ticks") { + Aggregate reconstructed; + state_.visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() == "tick") { + const auto trade = normalized_trade(event); + if (trade.ts >= minute && trade.ts < minute + 60000) reconstructed.add(trade); + } + }); + try { reconstructed.reconcile(kline, true); } + catch (const Error& failure) { + if (failure.code == 20 || failure.code == 21) throw Error(21, "already-emitted closed-minute fence was revised"); + throw; + } + } else if (!old) throw Error(21, "closed duplicate cannot be matched to the retained prefix"); + return; + } + if (state_.config().mode == "bars") { + while (state_.cursor().cut < minute) { + const auto page = venue_.klines(state_.cursor().cut, minute); + for (const auto& historical : page) { + if (historical.bar.ts + 60000 > watermark_) throw Error(20, "historical bar exceeds the confirmed watermark"); + emit(historical.bar.wire(), historical.bar); + } + } + emit(kline.bar.wire(), kline.bar); + return; + } + while (state_.cursor().cut < minute) historical_minute(); + if (!kline.count || kline.first <= 0 || kline.last < kline.first || + static_cast(kline.last - kline.first) + 1 != kline.count) + throw Error(20, "closed kline has ambiguous or noncontiguous raw bounds"); + anchor(static_cast(kline.first)); + if (state_.cursor().seq > static_cast(kline.last)) throw Error(21, "closed kline excludes an already-emitted trade"); + heal_to(static_cast(kline.last)); + aggregate_.reconcile(kline, true); + emit(time_line(minute + 60000), kline.bar); + aggregate_ = Aggregate{}; + drain(); +} +void FeedSession::stage(const VenueEvent& event) { + if (event.kind == VenueEvent::Kind::Kline) { closed(event.kline); return; } + if (event.kind != VenueEvent::Kind::Trade || state_.config().mode != "ticks" || event.trade.ts < state_.cursor().start) return; + if (!state_.cursor().predecessor) anchor(venue_.first_trade_id(state_.cursor().start)); + put(event.trade); + drain(); +} + +// Callers pass only closed candles: confirmed WebSocket rows, or REST rows behind a later confirmed one. +void reconcile_fenced(const Aggregate& prints, const Kline& candle) { + if (!prints.count) { + if (!Decimal(candle.bar.volume).zero()) throw Error(21, "confirmed candle volume shows prints the fenced chain does not contain"); + return; + } + if (!(Decimal(prints.open) == Decimal(candle.bar.open)) || !(Decimal(prints.high) == Decimal(candle.bar.high)) || + !(Decimal(prints.low) == Decimal(candle.bar.low)) || !(Decimal(prints.close) == Decimal(candle.bar.close)) || + !(prints.volume == Decimal(candle.bar.volume))) throw Error(21, "fenced minute exact OHLCV reconciliation failed"); +} + +FenceSession::FenceSession(State& state, Venue& venue, std::function output) + : Session(state, venue, std::move(output)) { + state_.visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() == "tick") { + const auto trade = normalized_trade(event); + if (trade.ts >= state_.cursor().cut) aggregate_.add(trade); + } + }); +} +std::uint64_t FenceSession::next_id() const { + const auto previous = state_.cursor().seq ? state_.cursor().seq : state_.cursor().predecessor ? state_.cursor().predecessor->id : 0; + if (!previous || previous == std::numeric_limits::max()) throw Error(20, "print cursor is not anchored"); + return previous + 1; +} +void FenceSession::anchor() { + if (state_.cursor().predecessor) return; + state_.anchor(venue_.predecessor(state_.cursor().start)); +} +void FenceSession::put(const Trade& trade, bool healed) { + trade.validate(); + if (trade.ts < state_.cursor().start) { + if (healed) throw Error(23, "venue matched time regressed: a print after the proven predecessor precedes the start minute"); + return; + } + // The predecessor is the last print before the start minute, and this print is at or after it. + const auto& predecessor = state_.cursor().predecessor; + if (predecessor && trade.id <= predecessor->id) { + if (trade.id == predecessor->id) throw Error(21, "a print changed the proven start predecessor"); + throw Error(23, "venue matched time regressed: a print before the proven predecessor is at or after the start minute"); + } + if (state_.cursor().seq && trade.id <= state_.cursor().seq) { + const auto previous = state_.trade(trade.id); + if (!previous || !(*previous == trade)) throw Error(21, "duplicate print conflicts with the immutable prefix"); + return; + } + const auto existing = pending_.find(trade.id); + if (existing != pending_.end()) { + if (!(existing->second == trade)) throw Error(21, healed ? "REST healing conflicts with WebSocket data" : "buffered duplicate print conflicts"); + return; + } + const auto bytes = trade_bytes(trade); + if (bytes > state_.config().max_queue_bytes - std::min(pending_bytes_, state_.config().max_queue_bytes)) + throw Error(22, "unconfirmed print buffer overrun"); + pending_bytes_ += bytes; + pending_.emplace(trade.id, trade); +} +void FenceSession::emit_trade(const Trade& trade) { + if (trade.ts < state_.cursor().last_tick_ts) throw Error(23, "venue matched time regressed along the contiguous print chain"); + if (trade.id != next_id() || trade.ts < state_.cursor().cut || trade.ts >= state_.cursor().cut + 60000) + throw Error(20, "print violates the complete chronological minute prefix"); + aggregate_.add(trade); + emit(trade.wire()); + const auto buffered = pending_.find(trade.id); + if (buffered != pending_.end()) { + pending_bytes_ -= trade_bytes(buffered->second); + pending_.erase(buffered); + } +} +// The venue documents a finite REST print history: a gap that starts before it is unhealable, so the +// feed stops instead of asking for prints the venue no longer serves. Venue time, not the local clock, +// measures the distance. +void FenceSession::within_retention() const { + const auto window = venue_.retention_ms(); + const auto last = state_.cursor().last_tick_ts >= 0 ? state_.cursor().last_tick_ts : + state_.cursor().predecessor ? state_.cursor().predecessor->ts : -1; + if (window && last >= 0 && newest_ > last && newest_ - last > window) + throw Error(20, "print gap starts beyond the venue's REST history window"); +} +std::vector FenceSession::page(bool patient) { + within_retention(); + // REST may lag the WebSocket print that revealed the gap: about 7 seconds of spaced re-reads, like the + // reconnect overlap, before an empty answer counts. Looking ahead without such a print reads once. + for (unsigned int attempt = 0; attempt < (patient ? 4U : 1U); ++attempt) { + if (attempt) pause_for(std::chrono::milliseconds((state_.config().allow_insecure ? 100 : 1000) << (attempt - 1))); + auto prints = venue_.history(next_id()); + if (!prints.empty()) { + if (prints.front().id != next_id()) throw Error(20, "print history page does not start at the cursor"); + return prints; + } + } + return {}; +} +void FenceSession::heal() { + const auto target = pending_.begin()->first - 1; + log("warn", "print_gap_healing", Json::object({{"from_id", Json::number(std::to_string(next_id()))}, {"through_id", Json::number(std::to_string(target))}})); + const auto prints = page(true); + if (prints.empty()) throw Error(20, "print gap cannot advance within available history"); + for (const auto& trade : prints) { + if (trade.id > target) break; + put(trade, true); + } +} +// A later minute was confirmed while no print beyond the chain is in hand: look for the fence by ID. +// A quiet start waits: nothing is anchored, and no minute closes, until the venue holds a print at or +// after --start. An empty page means no print beyond the chain yet, not a gap. +bool FenceSession::forward() { + if (forward_watermark_ >= watermark_) return false; + forward_watermark_ = watermark_; + if (!state_.cursor().predecessor) { + const auto predecessor = venue_.predecessor_if_ready(state_.cursor().start); + if (!predecessor) return false; + state_.anchor(*predecessor); + } + const auto prints = page(false); + for (const auto& trade : prints) put(trade, true); + return !prints.empty(); +} +bool FenceSession::close_minute() { + const auto minute = state_.cursor().cut; + auto found = candles_.find(minute); + if (found == candles_.end()) { + // REST candles are used only behind a later confirmed WebSocket candle. + if (watermark_ <= minute + 60000) return false; + for (unsigned int attempt = 0;; ++attempt) { + try { + for (const auto& kline : venue_.klines(minute, std::min(watermark_, minute + 100 * 60000))) + candles_.emplace(kline.bar.ts, kline); + break; + } catch (const Error& failure) { + // A REST row can trail the WebSocket confirmation briefly; then the minute is unavailable. + if (failure.code != 20 || attempt == 2) throw; + pause_for(std::chrono::milliseconds(500 * (attempt + 1))); + } + } + found = candles_.find(minute); + if (found == candles_.end()) throw Error(20, "confirmed candle of a fenced minute is unavailable"); + log("info", "historical_candle_verified", Json::object({{"minute", Json::number(std::to_string(minute))}})); + } + const auto candle = found->second; + // Aggregate prints close on the fence and the venue's closed candle alone: the candle is not their sum. + if (venue_.candle_is_print_sum()) reconcile_fenced(aggregate_, candle); + emit(time_line(minute + 60000), candle.bar); + aggregate_ = Aggregate{}; + candles_.erase(candles_.begin(), candles_.upper_bound(minute)); + return true; +} +void FenceSession::candle(const Kline& kline) { + if (!kline.confirmed) return; + const auto minute = kline.bar.ts; + watermark_ = std::max(watermark_, minute + 60000); + if (minute < state_.cursor().start) return; + if (minute < state_.cursor().cut) { + for (const auto& proof : state_.cursor().proofs) + if (proof.ts == minute) { + if (!(proof == kline.bar)) throw Error(21, "already-closed minute's confirmed candle was revised"); + return; + } + // Older than the retained proofs: rebuild that minute from the journal, where the candle is a sum. + if (!venue_.candle_is_print_sum()) return; + Aggregate rebuilt; + state_.visit(0, [&](const std::string& line) { + const auto event = parse_json(line); + if (event.at("type").text() != "tick") return; + const auto trade = normalized_trade(event); + if (trade.ts >= minute && trade.ts < minute + 60000) rebuilt.add(trade); + }); + try { reconcile_fenced(rebuilt, kline); } + catch (const Error&) { throw Error(21, "already-closed minute's confirmed candle was revised"); } + return; + } + if (candles_.size() >= 100000) throw Error(22, "confirmed-candle buffer overrun"); + const auto [found, inserted] = candles_.emplace(minute, kline); + if (!inserted && !(found->second.bar == kline.bar)) throw Error(21, "confirmed candle was revised before its minute closed"); +} +void FenceSession::advance() { + for (;;) { + if (pending_.empty()) { + if (!(watermark_ > state_.cursor().cut + 60000 && forward())) return; + continue; + } + const auto next = pending_.begin()->second; + if (next.id < next_id()) throw Error(21, "buffered cursor fell behind the immutable prefix"); + if (next.id > next_id()) { heal(); continue; } + if (next.ts < state_.cursor().cut + 60000) { emit_trade(next); continue; } + // `next` is the fence: the chain is complete through it, so minute `cut` holds no other print. + if (!close_minute()) return; + } +} +void FenceSession::stage(const VenueEvent& event) { + if (event.kind == VenueEvent::Kind::Kline) { + candle(event.kline); + advance(); + return; + } + if (event.kind != VenueEvent::Kind::Trade) return; + newest_ = std::max(newest_, event.trade.ts); + if (event.trade.ts < state_.cursor().start) return; + anchor(); + put(event.trade, false); + advance(); +} +void FenceSession::connected() { + publish(); + venue_.reverify(); + const auto& trades = state_.recent_trades(); + if (!trades.empty()) { + const auto fetched = settled_prints(venue_, state_.config(), trades); + if (fetched.size() != trades.size()) throw Error(20, "reconnect print overlap is unavailable"); + for (std::size_t index = 0; index < trades.size(); ++index) + if (!(trades[index] == fetched[index])) throw Error(21, "reconnect print overlap changed"); + } else if (state_.cursor().predecessor) { + const auto fetched = venue_.history(state_.cursor().predecessor->id, 1); + if (fetched.empty()) throw Error(20, "initial predecessor overlap is unavailable"); + if (!(fetched.front() == *state_.cursor().predecessor)) throw Error(21, "initial predecessor overlap changed"); + } + const auto& proofs = state_.cursor().proofs; + if (!proofs.empty()) { + const auto fetched = settled_klines(venue_, state_.config(), proofs.front().ts, proofs.back().ts + 60000, + [&](const auto& rows) { return same_bars(rows, proofs); }); + if (fetched.size() != proofs.size()) throw Error(20, "reconnect closed-candle overlap is unavailable"); + for (std::size_t index = 0; index < proofs.size(); ++index) + if (!(proofs[index] == fetched[index].bar)) throw Error(21, "reconnect already-closed candle changed"); + } + replay(); +} + +std::unique_ptr make_session(State& state, Venue& venue, std::function output) { + const auto& mode = state.config().mode; + if (tick_mode(mode) && venue.tick_proof() == TickProof::NextPrintFence) + return std::make_unique(state, venue, std::move(output)); + // FeedSession proves only raw ticks through the kline ID range; any other tick mode would drop prints. + if (tick_mode(mode) && mode != "ticks") throw Error(23, "this venue has no proof for mode " + mode); + return std::make_unique(state, venue, std::move(output)); +} + +void run_feed(Config config) { + // Instrument metadata fixes the quantity units before the cursor binds them. + const auto venue = make_venue(config); + config.qty_multiplier = venue->qty_multiplier(); + State state(config); + try { + WebSocketPump pump(config, venue->connection()); + const auto session = make_session(state, *venue, &output_line); + for (;;) { + auto message = pump.take(); + try { + // Group commit: verify every source message already queued (bounded), then persist once. + for (std::size_t taken = 1;; ++taken) { + if (message.connected) session->connected(); + else for (const auto& event : venue->decode(message.text)) session->stage(event); + if (taken >= 1024 || !pump.try_take(message)) break; + } + } catch (const Error&) { session->salvage(); throw; } + session->publish(); + } + } catch (const Stopped&) { log("info", "stopped", state.status()); } + catch (...) { log("error", "verified_cursor_retained", state.status()); throw; } +} + +void warmup(const Config& config, const std::string& output) { + if (config.start < 0 || config.end <= config.start || config.start % 60000 || config.end % 60000) + throw Error(23, "warmup requires a nonempty minute-aligned exclusive range"); + if ((config.end - config.start) / 60000 > 100000) throw Error(22, "warmup range exceeds 100000 bars"); + if (std::filesystem::exists(output) || std::filesystem::exists(output + ".manifest.json")) throw Error(23, "warmup output already exists"); + const auto venue = make_venue(config); + std::int64_t watermark = -1; + std::map confirmed; + { + WebSocketPump pump(config, venue->connection()); + while (watermark < config.end) { + const auto message = pump.take(); + if (message.connected) continue; + for (const auto& event : venue->decode(message.text)) { + if (event.kind != VenueEvent::Kind::Kline || !event.kline.confirmed) continue; + const auto& bar = event.kline.bar; + watermark = std::max(watermark, bar.ts + 60000); + if (bar.ts < config.start || bar.ts >= config.end) continue; + const auto [found, inserted] = confirmed.emplace(bar.ts, bar); + if (!inserted && !(found->second == bar)) throw Error(21, "confirmed WebSocket bar was revised during warmup"); + } + } + } + std::string csv = "timestamp,open,high,low,close,volume\n"; + auto next = config.start; + std::uint64_t count = 0, cross_checked = 0; + while (next < config.end) { + // The newest rows come from REST moments after the WS close: the confirmed bars in hand must match. + const auto page = settled_klines(*venue, config, next, config.end, [&](const auto& rows) { + for (const auto& row : rows) { + const auto observed = confirmed.find(row.bar.ts); + if (observed != confirmed.end() && !(observed->second == row.bar)) return false; + } + return true; + }); + for (const auto& kline : page) { + if (kline.bar.ts != next || next + 60000 > watermark) throw Error(20, "warmup does not adjoin a verified closed-minute cut"); + const auto& bar = kline.bar; + const auto observed = confirmed.find(bar.ts); + if (observed != confirmed.end()) { + if (!(observed->second == bar)) throw Error(21, "warmup REST row differs from the confirmed WebSocket bar"); + ++cross_checked; + } + csv += std::to_string(bar.ts) + ',' + bar.open + ',' + bar.high + ',' + bar.low + ',' + bar.close + ',' + bar.volume + '\n'; + if (csv.size() > 16 * 1024 * 1024) throw Error(22, "warmup output exceeds 16 MiB"); + next += 60000; + ++count; + } + } + const auto manifest = Json::object({{"schema", Json::string("pineforge-feed-warmup/v1")}, + {"venue", Json::string(config.venue)}, {"market", Json::string(config.market)}, {"symbol", Json::string(config.symbol)}, + {"source", Json::string(config.rest_url + venue->kline_source())}, {"interval", Json::string("1m")}, + {"units", Json::object({{"price", Json::string("quote/base")}, {"volume", Json::string("base")}, {"timestamp", Json::string("unix-ms")}})}, + {"qty_multiplier", Json::string(venue->qty_multiplier())}, + {"start", Json::number(std::to_string(config.start))}, {"end_exclusive", Json::number(std::to_string(config.end))}, + {"cut", Json::number(std::to_string(config.end))}, {"confirmed_ws_watermark", Json::number(std::to_string(watermark))}, + {"bars", Json::number(std::to_string(count))}, {"ws_cross_checked_bars", Json::number(std::to_string(cross_checked))}, + {"sha256", Json::string(sha256(csv))}}); + atomic_file(output, csv); + atomic_file(output + ".manifest.json", manifest.dump() + '\n'); + log("info", "warmup_verified", manifest); +} +} diff --git a/native/feed/src/session.hpp b/native/feed/src/session.hpp new file mode 100644 index 0000000..3833ae0 --- /dev/null +++ b/native/feed/src/session.hpp @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "state.hpp" +#include "venue.hpp" +#include + +namespace pineforge::feed { +class Session { +protected: + State& state_; + Venue& venue_; + std::function output_; + std::uint64_t new_messages_ = 0; + bool replayed_ = false; + void emit(const std::string& line, const std::optional& proof = std::nullopt); + void replay(); +public: + Session(State& state, Venue& venue, std::function output); + virtual ~Session() = default; + Session(const Session&) = delete; + Session& operator=(const Session&) = delete; + virtual void connected() = 0; + // stage() verifies an event and stages its messages; publish() group-commits them, then writes them + // out; salvage() publishes what was already verified before a stop. ingest() is all three for one event. + virtual void stage(const VenueEvent& event) = 0; + void publish(); + void salvage() noexcept; + void ingest(const VenueEvent& event); +}; + +// Confirmed bars for every venue, and Binance spot raw ticks proven by the closed kline's ID range. +class FeedSession final : public Session { + Aggregate aggregate_; + std::map pending_; + std::size_t pending_bytes_ = 0; + std::int64_t watermark_ = -1; + void anchor(std::uint64_t first); + std::uint64_t next_id() const; + void put(const Trade& trade); + void emit_trade(const Trade& trade); + void heal_to(std::uint64_t target); + void drain(); + void historical_minute(); + void closed(const Kline& kline); +public: + FeedSession(State& state, Venue& venue, std::function output); + void connected() override; + void stage(const VenueEvent& event) override; +}; + +// Ticks proven by a next-print fence (OKX trades-all, Binance USD-M aggregate prints). A minute closes +// only when the contiguous ID chain from the proven predecessor reaches a print of a later minute and the +// venue has closed the minute (a confirmed WebSocket candle, or a REST candle behind a later confirmed +// one). For raw prints (OKX) that candle must also equal the prints exactly in OHLC and volume, and a +// quiet minute needs a zero-volume candle; aggregate prints (USD-M) can straddle a minute boundary, so +// their candle is not compared. Prints beyond the closed minute stay buffered until that proof. +class FenceSession final : public Session { + Aggregate aggregate_; + std::map pending_; + std::size_t pending_bytes_ = 0; + std::map candles_; + std::int64_t watermark_ = -1, newest_ = -1, forward_watermark_ = -1; + std::uint64_t next_id() const; + void anchor(); + void put(const Trade& trade, bool healed); + void emit_trade(const Trade& trade); + void within_retention() const; + std::vector page(bool patient); + void heal(); + bool forward(); + bool close_minute(); + void candle(const Kline& kline); + void advance(); +public: + FenceSession(State& state, Venue& venue, std::function output); + void connected() override; + void stage(const VenueEvent& event) override; +}; +// REST closed candles for [start, end), re-read with spaced pauses while `agrees` rejects them: a row for +// the minute that just closed can trail the WebSocket close. The caller decides what persists. +std::vector settled_klines(Venue& venue, const Config& config, std::int64_t start, std::int64_t end, + const std::function&)>& agrees); +std::vector settled_prints(Venue& venue, const Config& config, const std::deque& trades); +// Exact OHLC and volume of a fenced minute's prints against its confirmed candle; zero volume when empty. +void reconcile_fenced(const Aggregate& prints, const Kline& candle); + +std::unique_ptr make_session(State& state, Venue& venue, std::function output); +void run_feed(Config config); +void warmup(const Config& config, const std::string& output); +} diff --git a/native/feed/src/state.cpp b/native/feed/src/state.cpp index 4ec6681..571d1b4 100644 --- a/native/feed/src/state.cpp +++ b/native/feed/src/state.cpp @@ -25,7 +25,7 @@ bool digest_token(const std::string& token) { } void advance(Cursor& cursor, const Json& event, const std::string& mode) { const auto type = event.at("type").text(); - if (type == "tick" && mode == "ticks") { + if (type == "tick" && tick_mode(mode)) { const auto trade = normalized_trade(event); const auto previous = cursor.seq ? cursor.seq : cursor.predecessor ? cursor.predecessor->id : 0; if (!previous || previous == std::numeric_limits::max() || trade.id != previous + 1 || @@ -38,9 +38,10 @@ void advance(Cursor& cursor, const Json& event, const std::string& mode) { if (bar.ts != cursor.cut) throw Error(20, "bar does not adjoin the verified cut"); cursor.last_bar = bar.ts; cursor.cut += 60000; - } else if (type == "time" && mode == "ticks") { + } else if (type == "time" && tick_mode(mode)) { const auto cut = event.at("ts").integer(); - if (!cursor.seq || cut != cursor.cut + 60000 || cut <= cursor.last_tick_ts) + // A fenced quiet minute may close before any print: the anchored predecessor bounds it. + if ((!cursor.seq && !cursor.predecessor) || cut != cursor.cut + 60000 || cut <= cursor.last_tick_ts) throw Error(20, "time does not advance exactly one proven closed minute"); cursor.cut = cut; } else throw Error(23, "journal contains an unsupported normalized message"); @@ -59,8 +60,9 @@ Json State::serialize(const Cursor& cursor) const { {"symbol", Json::string(config_.symbol)}, {"mode", Json::string(config_.mode)}, {"rest_origin", Json::string(config_.rest_url)}, {"ws_origin", Json::string(config_.ws_url)}, {"units", Json::object({{"price", Json::string("quote/base")}, {"qty", Json::string("base")}, {"ts", Json::string("unix-ms")}})}, + {"qty_multiplier", Json::string(config_.qty_multiplier)}, {"start", signed_number(cursor.start)}, {"verified_cut", signed_number(cursor.cut)}, - {"venue_id", config_.mode == "ticks" ? number(cursor.seq) : signed_number(cursor.last_bar)}, + {"venue_id", tick_mode(config_.mode) ? number(cursor.seq) : signed_number(cursor.last_bar)}, {"emitted_seq", number(cursor.seq)}, {"last_tick_ts", signed_number(cursor.last_tick_ts)}, {"last_bar", signed_number(cursor.last_bar)}, {"message_index", number(cursor.message_index)}, {"log_bytes", number(cursor.log_bytes)}, {"prefix_hash", Json::string(cursor.prefix_hash)}, @@ -125,6 +127,10 @@ void State::recover() { const auto& units = document.at("units"); if (units.at("price").text() != "quote/base" || units.at("qty").text() != "base" || units.at("ts").text() != "unix-ms") throw Error(21, "resume units changed"); + // Contract quantities were converted with this instrument multiplier; a change would mix units. + const auto* multiplier = document.find("qty_multiplier"); + if ((multiplier ? multiplier->text() : std::string("1")) != config_.qty_multiplier) + throw Error(21, "resume contract quantity multiplier changed"); cursor_.epoch = document.at("epoch").text(); cursor_.prefix_hash = document.at("prefix_hash").text(); cursor_.overlap_hash = document.at("overlap_hash").text(); @@ -135,7 +141,7 @@ void State::recover() { cursor_.seq = document.at("emitted_seq").integer(); cursor_.last_tick_ts = document.at("last_tick_ts").integer(); cursor_.last_bar = document.at("last_bar").integer(); - if ((config_.mode == "ticks" && document.at("venue_id").integer() != cursor_.seq) || + if ((tick_mode(config_.mode) && document.at("venue_id").integer() != cursor_.seq) || (config_.mode == "bars" && document.at("venue_id").integer() != cursor_.last_bar)) throw Error(21, "venue ID does not match the emitted cursor"); cursor_.message_index = document.at("message_index").integer(); diff --git a/native/feed/src/transport.cpp b/native/feed/src/transport.cpp index 6119619..7a1997d 100644 --- a/native/feed/src/transport.cpp +++ b/native/feed/src/transport.cpp @@ -62,14 +62,16 @@ void pause_until(std::chrono::steady_clock::time_point deadline) { } struct Response { std::string body, retry_after; + std::size_t limit = 1024 * 1024; std::uint64_t used_weight = 0; bool overflow = false; }; std::size_t body_callback(char* data, std::size_t size, std::size_t count, void* context) noexcept { auto* response = static_cast(context); const auto bytes = size * count; - // The JSON parser refuses documents above 1 MiB, so a larger body is refused here. - if (bytes > 1024 * 1024 - response->body.size()) { response->overflow = true; return 0; } + // The JSON parser refuses documents above 1 MiB, so a larger body is refused here unless the caller + // extracts one bounded member from it. + if (bytes > response->limit - response->body.size()) { response->overflow = true; return 0; } try { response->body.append(data, bytes); return bytes; } catch (...) { response->overflow = true; return 0; } } @@ -94,6 +96,18 @@ std::size_t header_callback(char* data, std::size_t size, std::size_t count, voi } catch (...) {} return bytes; } +// A venue frame on a CONNECT_ONLY handle: control-size text, retried while the socket would block. +bool send_text(CURL* handle, curl_socket_t socket, const std::string& text) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + for (;;) { + std::size_t sent = 0; + const auto result = curl_ws_send(handle, text.data(), text.size(), &sent, 0, CURLWS_TEXT); + if (result == CURLE_OK && sent == text.size()) return true; + if (result != CURLE_AGAIN || sent || std::chrono::steady_clock::now() >= deadline) return false; + pollfd descriptor{socket, POLLOUT, 0}; + ::poll(&descriptor, 1, 100); + } +} } void pause_for(std::chrono::milliseconds duration) { pause_until(std::chrono::steady_clock::now() + duration); } @@ -148,12 +162,24 @@ void validate_origin(const std::string& origin, bool websocket, bool allow_insec !field(CURLUPART_QUERY).empty() || field(CURLUPART_PATH) != "/") throw Error(23, "endpoint must be a credential-free origin without a path or query"); } -HttpClient::HttpClient(const Config& config) : config_(config) { +RestPolicy binance_spot_rest(const std::string& symbol) { + RestPolicy policy; + policy.endpoints = {"/api/v3/exchangeInfo", "/api/v3/historicalTrades", "/api/v3/aggTrades", "/api/v3/klines"}; + policy.limits_path = "/api/v3/exchangeInfo?symbol=" + symbol; + policy.limits_weight = 20; + return policy; +} + +HttpClient::HttpClient(const Config& config, RestPolicy policy) : config_(config), policy_(std::move(policy)) { check_runtime_curl(); validate_origin(config_.rest_url, false, config_.allow_insecure); + milliseconds_per_request_ = policy_.milliseconds_per_request; + if (policy_.endpoints.empty()) throw Error(23, "public REST allowlist is empty"); + if (policy_.limits_path.empty()) return; try { - const auto metadata = get("/api/v3/exchangeInfo?symbol=" + config_.symbol, 20); - const auto& limits = metadata.at("rateLimits"); + // The USD-M document exceeds the parser bound: only its top-level rateLimits member is parsed. + const auto document = body(policy_.limits_path, policy_.limits_weight, policy_.limits_bytes); + const auto limits = parse_json(json_member(document, "rateLimits")); if (limits.kind != Json::Kind::Array || limits.items.size() > 64) throw Error(23, "invalid public rate-limit metadata"); bool weighted = false; for (const auto& limit : limits.items) { @@ -181,15 +207,21 @@ HttpClient::HttpClient(const Config& config) : config_(config) { catch (const std::exception&) { throw Error(23, "invalid public exchange rate-limit metadata"); } } Json HttpClient::get(const std::string& path, unsigned int weight) { + const auto text = body(path, weight); + try { return parse_json(text); } + catch (const std::exception&) { throw Error(23, "public market-data response is invalid JSON"); } +} +std::string HttpClient::body(const std::string& path, unsigned int weight, std::size_t limit) { const auto endpoint = path.substr(0, path.find('?')); - if (endpoint != "/api/v3/exchangeInfo" && endpoint != "/api/v3/historicalTrades" && - endpoint != "/api/v3/aggTrades" && endpoint != "/api/v3/klines") + if (std::find(policy_.endpoints.begin(), policy_.endpoints.end(), endpoint) == policy_.endpoints.end()) throw Error(23, "REST request is outside the public market-data allowlist"); + if (!limit || limit > 4 * 1024 * 1024) throw Error(22, "REST body bound exceeds 4 MiB"); for (unsigned int attempt = 0; attempt < 4; ++attempt) { pause_until(next_request_); Progress state; auto handle = handle_for(config_.rest_url + path, false, config_.allow_insecure, &state); Response response; + response.limit = limit; option(handle.get(), CURLOPT_WRITEFUNCTION, &body_callback); option(handle.get(), CURLOPT_WRITEDATA, &response); option(handle.get(), CURLOPT_HEADERFUNCTION, &header_callback); @@ -202,9 +234,11 @@ Json HttpClient::get(const std::string& path, unsigned int weight) { next_request_ = std::chrono::steady_clock::now() + std::chrono::milliseconds(config_.allow_insecure ? 0 : pacing); if (stopping || code == CURLE_ABORTED_BY_CALLBACK) throw Stopped{}; if (response.overflow) throw Error(22, "REST response exceeds the bounded buffer"); + // A venue's own rejection meaning comes first (Bybit's 403 is a rate ban, USD-M's 400 codes). + if (code == CURLE_OK && status != 200 && status != 429 && policy_.rejected) policy_.rejected(status, response.body); if (status == 401 || status == 403 || status == 418 || status == 451 || (status >= 300 && status < 400)) throw Error(23, "public market-data access denied or redirected (HTTP " + std::to_string(status) + ")"); - if (status == 429) { + if (status == 429 || (status == 200 && policy_.throttled && policy_.throttled(response.body))) { const auto wait = response.retry_after.empty() ? 1 : retry_after_seconds(response.retry_after, std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count()); log("warn", "rest_rate_limited", Json::object({{"retry_after", Json::number(std::to_string(wait))}})); @@ -212,6 +246,11 @@ Json HttpClient::get(const std::string& path, unsigned int weight) { next_request_ = std::chrono::steady_clock::now() + std::chrono::seconds(wait); continue; } + if (code == CURLE_OK && policy_.transient && policy_.transient(status, response.body)) { + log("warn", "rest_transient_retry", Json::object({{"status", Json::number(std::to_string(status))}})); + next_request_ = std::chrono::steady_clock::now() + std::chrono::seconds(1U << attempt); + continue; + } if (status == 404) throw Error(20, "public history is unavailable"); if (code != CURLE_OK || status >= 500) { next_request_ = std::chrono::steady_clock::now() + std::chrono::seconds(1U << attempt); @@ -225,16 +264,18 @@ Json HttpClient::get(const std::string& path, unsigned int weight) { next_request_ = std::max(next_request_, std::chrono::steady_clock::now() + std::chrono::milliseconds(60500 - elapsed)); log("warn", "rest_shared_quota_pause"); } - try { return parse_json(response.body); } - catch (const std::exception&) { throw Error(23, "public market-data response is invalid JSON"); } + return std::move(response.body); } throw Error(20, "public history retries exhausted; verified prefix retained"); } bool WebSocketPump::stopped() const { return stopped_.load() || stopping.load(); } -WebSocketPump::WebSocketPump(const Config& config, const std::string& streams) : config_(config), url_(config.ws_url + "/stream?streams=" + streams) { +WebSocketPump::WebSocketPump(const Config& config, Connection connection) + : config_(config), connection_(std::move(connection)), url_(config.ws_url + connection_.path) { check_runtime_curl(); validate_origin(config_.ws_url, true, config_.allow_insecure); + if (connection_.path.empty() || connection_.path.front() != '/' || !connection_.classify) + throw Error(23, "invalid venue WebSocket connection"); worker_ = std::thread([this] { run(); }); } WebSocketPump::~WebSocketPump() { @@ -275,6 +316,8 @@ SourceMessage WebSocketPump::take() { void WebSocketPump::run() { try { unsigned int failures = 0; + // Reconnect backoff doubles from 2 units up to 30; loopback test origins use 50 ms units. + const auto unit = std::chrono::milliseconds(config_.allow_insecure ? 50 : 1000); auto next_attempt = std::chrono::steady_clock::now(); while (!stopped()) { while (!stopped() && std::chrono::steady_clock::now() < next_attempt) @@ -293,24 +336,32 @@ void WebSocketPump::run() { throw Error(23, "public WebSocket access denied or redirected"); if (handshake != CURLE_OK || status != 101) { if (++failures >= 8) throw Error(20, "public WebSocket reconnect attempts exhausted"); - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(std::min(30U, 1U << failures)); + const auto deadline = std::chrono::steady_clock::now() + unit * std::min(30U, 1U << failures); while (!stopped() && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(50)); continue; } - failures = 0; + curl_socket_t socket = CURL_SOCKET_BAD; + curl_easy_getinfo(handle.get(), CURLINFO_ACTIVESOCKET, &socket); + // The marker precedes every message of this connection: the session verifies its REST + // overlap while the subscribed data is buffered behind it. push({true, {}}); - log("info", "websocket_connected"); + bool subscribed = true; + for (const auto& request : connection_.subscribe) subscribed = subscribed && send_text(handle.get(), socket, request); + log(subscribed ? "info" : "warn", subscribed ? "websocket_connected" : "websocket_subscribe_failed"); const auto birth = std::chrono::steady_clock::now(); - auto last_text = birth, message_birth = birth; + auto last_data = birth, message_birth = birth, last_ping = birth; std::string message; - bool assembling = false; + bool assembling = false, heard = false; std::uint64_t frame_offset = 0; - curl_socket_t socket = CURL_SOCKET_BAD; - curl_easy_getinfo(handle.get(), CURLINFO_ACTIVESOCKET, &socket); - while (!stopped()) { + while (subscribed && !stopped()) { const auto now = std::chrono::steady_clock::now(); - if (now - birth >= std::chrono::seconds(config_.reconnect_seconds) || now - last_text > std::chrono::seconds(75) || + // Keepalive replies are control messages: they cannot hide a silent data stream. + if (now - birth >= std::chrono::seconds(config_.reconnect_seconds) || now - last_data > std::chrono::seconds(75) || (assembling && now - message_birth > std::chrono::seconds(30))) break; + if (!connection_.ping.empty() && now - last_ping >= std::chrono::seconds(config_.keepalive_seconds)) { + if (!send_text(handle.get(), socket, connection_.ping)) break; + last_ping = now; + } std::array buffer{}; std::size_t received = 0; const curl_ws_frame* metadata = nullptr; @@ -329,26 +380,28 @@ void WebSocketPump::run() { throw Error(22, "WebSocket message exceeds 1 MiB"); message.append(buffer.data(), received); frame_offset += received; - last_text = now; if (metadata->bytesleft) continue; frame_offset = 0; if (metadata->flags & CURLWS_CONT) continue; - bool retiring = false; - try { - const auto envelope = parse_json(message); - const auto* wrapped = envelope.find("data"); - const auto& data = wrapped ? *wrapped : envelope; - const auto* event = data.find("e"); - retiring = event && event->kind == Json::Kind::String && event->text() == "serverShutdown"; - } catch (const std::exception&) { throw Error(23, "invalid public WebSocket JSON"); } + const auto kind = connection_.classify(message); + assembling = false; + // A connection counts as established only once the venue answers on it. + if (!heard) { heard = true; failures = 0; } + if (kind == Frame::Control) { message.clear(); continue; } + last_data = now; push({false, std::move(message)}); message.clear(); - assembling = false; - if (retiring) { log("info", "source_retiring_reconnect"); break; } + if (kind == Frame::Retire) { log("info", "source_retiring_reconnect"); break; } } if (!stopped()) { log("warn", "websocket_reconnect"); std::this_thread::sleep_for(std::chrono::milliseconds(250)); + // Accepted, then closed or refused before any frame: that spends the reconnect budget too. + if (!heard) { + if (++failures >= 8) throw Error(20, "public WebSocket reconnect attempts exhausted"); + const auto deadline = std::chrono::steady_clock::now() + unit * std::min(30U, 1U << failures); + while (!stopped() && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } } } } catch (const Stopped&) {} diff --git a/native/feed/src/transport.hpp b/native/feed/src/transport.hpp index 2ced947..d4b41c2 100644 --- a/native/feed/src/transport.hpp +++ b/native/feed/src/transport.hpp @@ -1,12 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once #include "core.hpp" +#include "venue.hpp" #include #include #include #include #include #include +#include namespace pineforge::feed { void check_runtime_curl(); @@ -14,12 +16,31 @@ void validate_origin(const std::string& origin, bool websocket, bool allow_insec void pause_for(std::chrono::milliseconds duration); std::uint64_t retry_after_seconds(const std::string& value, std::int64_t now); +// A venue's public REST contract: allowlisted paths and how requests are paced. +struct RestPolicy { + std::vector endpoints; + // Binance publishes its ceilings in a rateLimits document; other venues use fixed documented pacing. + std::string limits_path; + unsigned int limits_weight = 0; + std::size_t limits_bytes = 1024 * 1024; + std::uint64_t milliseconds_per_request = 0; + // Maps a venue's rejection body to a specific stop before the generic HTTP status policy applies. + std::function rejected; + // A venue that signals rate limiting inside an HTTP 200 body. + std::function throttled; + // A venue code that means "try again" (timeout, busy): retried with backoff like HTTP 5xx. + std::function transient; +}; +RestPolicy binance_spot_rest(const std::string& symbol); + class HttpClient { Config config_; + RestPolicy policy_; std::chrono::steady_clock::time_point next_request_{}; std::uint64_t weight_limit_ = 0, milliseconds_per_weight_ = 0, milliseconds_per_request_ = 0; public: - explicit HttpClient(const Config& config); + HttpClient(const Config& config, RestPolicy policy); + std::string body(const std::string& path, unsigned int weight, std::size_t limit = 1024 * 1024); Json get(const std::string& path, unsigned int weight); }; struct SourceMessage { @@ -28,6 +49,7 @@ struct SourceMessage { }; class WebSocketPump { Config config_; + Connection connection_; std::string url_; std::atomic stopped_{false}; std::mutex mutex_; @@ -40,7 +62,7 @@ class WebSocketPump { void push(SourceMessage message); void run(); public: - WebSocketPump(const Config& config, const std::string& streams); + WebSocketPump(const Config& config, Connection connection); ~WebSocketPump(); WebSocketPump(const WebSocketPump&) = delete; WebSocketPump& operator=(const WebSocketPump&) = delete; diff --git a/native/feed/src/venue.hpp b/native/feed/src/venue.hpp index 8887913..f1a25ff 100644 --- a/native/feed/src/venue.hpp +++ b/native/feed/src/venue.hpp @@ -1,6 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once #include "core.hpp" +#include +#include +#include #include namespace pineforge::feed { @@ -9,13 +12,58 @@ struct VenueEvent { Trade trade; Kline kline; }; +// How the WebSocket reader treats one complete text message. Control messages (subscription +// acknowledgements, keepalive replies) prove liveness but are not queued; Retire queues the message, +// then reconnects. +enum class Frame { Data, Control, Retire }; +struct Connection { + std::string path; // appended to the WebSocket origin + std::vector subscribe; // text frames sent after every handshake + std::string ping; // venue text keepalive; empty when curl's PONG suffices + std::function classify; // runs on the reader thread: no shared state +}; +// How a tick minute is proven complete before its `time` event. +// KlineIdRange: the closed kline names the minute's raw-ID range and count (Binance spot). +// NextPrintFence: the contiguous ID chain reaches a print of a later minute and the venue has closed the +// minute (OKX trades-all, Binance USD-M aggregate prints); raw prints must also equal that candle exactly. +enum class TickProof { KlineIdRange, NextPrintFence }; + class Venue { public: virtual ~Venue() = default; - virtual std::string streams() const = 0; - virtual VenueEvent decode(const std::string& message) const = 0; + virtual Connection connection() const = 0; + // One source message can carry several events (OKX and Bybit batch their data arrays). + virtual std::vector decode(const std::string& message) const = 0; + // Up to `limit` contiguous prints from `from`, ascending. Fence venues may return fewer or none. virtual std::vector history(std::uint64_t from, std::size_t limit = 1000) = 0; - virtual std::uint64_t first_trade_id(std::int64_t minute) = 0; + virtual std::uint64_t first_trade_id(std::int64_t) { throw Error(23, "this venue has no raw-trade start fence lookup"); } + // The last print strictly before `minute`. + virtual Trade predecessor(std::int64_t minute) { + const auto first = first_trade_id(minute); + if (first <= 1) throw Error(20, "raw predecessor cannot be proven"); + const auto previous = history(first - 1, 1); + if (previous.empty() || previous.front().id != first - 1) throw Error(20, "raw predecessor is unavailable"); + return previous.front(); + } + // The same predecessor, but only once the venue holds a print at or after `minute`: nullopt while it + // holds none, without waiting or stopping (a quiet start). Fence venues override it. + virtual std::optional predecessor_if_ready(std::int64_t minute) { + try { return predecessor(minute); } + catch (const Error& failure) { if (failure.code == 20) return std::nullopt; throw; } + } + // Re-checks instrument metadata that fixes the stream's units; a change stops 21. Called on connect. + virtual void reverify() {} + // Contiguous closed one-minute candles in [start, end), ascending. virtual std::vector klines(std::int64_t start, std::int64_t end) = 0; + virtual TickProof tick_proof() const { return TickProof::KlineIdRange; } + // Documented REST print history window in milliseconds; 0 when none is documented. + virtual std::int64_t retention_ms() const { return 0; } + // Whether a closed candle is the exact sum of the venue's prints. Raw prints are; aggregate prints are + // not, because one aggregate can straddle a minute boundary and is never split. + virtual bool candle_is_print_sum() const { return true; } + // Base units per venue quantity unit (contract value times multiplier), "1" for base quantities. + virtual std::string qty_multiplier() const { return "1"; } + virtual std::string kline_source() const = 0; }; +std::unique_ptr make_venue(const Config& config); } diff --git a/native/feed/tests/mock_venue.py b/native/feed/tests/mock_venue.py index a4b9c8f..4705376 100644 --- a/native/feed/tests/mock_venue.py +++ b/native/feed/tests/mock_venue.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""Synthetic public Binance shapes; standard-library HTTP/WS fault injection.""" +"""Synthetic public Binance, Binance USD-M, OKX and Bybit shapes; standard-library HTTP/WS fault +injection.""" import base64 import copy @@ -90,6 +91,114 @@ def retirement(): return {"stream": "!serverShutdown", "data": {"e": "serverShutdown", "E": 180000}} +# USD-M aggregate prints: id -> (matched time, price, q, nq, first raw id, last raw id). `q` counts +# every execution and reconciles with the kline volume; `nq` excludes RPI executions. Raw ranges are +# not contiguous across aggregates (1005 is absent) and are never expanded. +AGGREGATES = { + 499: (119999, "10.10000000", "0.10000000", "0.10000000", 998, 999), + 500: (120001, "10.10000000", "0.10000000", "0.05000000", 1000, 1004), + 501: (120050, "11.20000000", "0.20000000", "0.20000000", 1006, 1006), + 502: (179999, "9.90000000", "0.30000000", "0.30000000", 1007, 1009), + 503: (180001, "12.00000000", "0.40000000", "0.40000000", 1010, 1012), + 504: (180050, "11.00000000", "0.10000000", "0.10000000", 1013, 1013), + 505: (240001, "12.00000000", "0.10000000", "0.10000000", 1014, 1014), +} + + +def aggregate(identifier, **changes): + matched, price, quantity, net, first, last = AGGREGATES[identifier] + data = { + "e": "aggTrade", + "E": matched + 1, + "a": identifier, + "s": "TESTUSDT", + "p": price, + "q": quantity, + "nq": net, + "f": first, + "l": last, + "T": matched, + "m": True, + "st": 1, + } + data.update(changes) + return {"stream": "testusdt@aggTrade", "data": data} + + +# OKX trades-all prints in contracts (ctVal 0.01): id -> (ts, px, sz). Candle volumes are contracts. +OKX_TRADES = { + 99: (119999, "10.1", "10"), + 100: (120001, "10.1", "10"), + 101: (120050, "11.2", "20"), + 102: (179999, "9.9", "30"), + 103: (180001, "12", "40"), + 104: (180050, "11", "10"), + 105: (240001, "12", "10"), +} +OKX_CANDLES = { + 120000: ("10.1", "11.2", "9.9", "9.9", "60"), + 180000: ("12", "12", "11", "11", "50"), + 240000: ("12", "12", "12", "12", "10"), +} +OKX_SWAP = "TEST-USDT-SWAP" + + +def okx_trades(*identifiers, channel="trades-all", **changes): + rows = [] + for identifier in identifiers: + matched, price, size = OKX_TRADES[identifier] + row = { + "instId": OKX_SWAP, + "tradeId": str(identifier), + "px": price, + "sz": size, + "side": "buy", + "ts": str(matched), + "source": "0", + } + row.update(changes) + rows.append(row) + return {"arg": {"channel": channel, "instId": OKX_SWAP}, "data": rows} + + +def okx_candle(minute, confirm="1", candles=None): + opening, high, low, close, volume = (candles or OKX_CANDLES)[minute] + row = [str(minute), opening, high, low, close, volume, "0", "0", confirm] + return {"arg": {"channel": "candle1m", "instId": OKX_SWAP}, "data": [row]} + + +# Bybit linear bars: base quantities already. +BYBIT_BARS = { + 120000: ("10.1", "11.2", "9.9", "9.9", "0.6"), + 180000: ("12", "12", "11", "11", "0.5"), + 240000: ("12", "12", "12", "12", "0.1"), + 300000: ("12", "12.5", "12", "12.5", "0.2"), +} + + +def bybit_row(minute, confirm, **changes): + opening, high, low, close, volume = BYBIT_BARS[minute] + row = { + "start": minute, + "end": minute + 59999, + "interval": "1", + "open": opening, + "close": close, + "high": high, + "low": low, + "volume": volume, + "turnover": "0", + "confirm": confirm, + "timestamp": minute + 30000, + } + row.update(changes) + return row + + +def bybit_push(*rows): + return {"topic": "kline.1.TESTUSDT", "data": list(rows), "ts": 1, "type": "snapshot"} + + def frame(connection, payload, opcode=1, final=True): if isinstance(payload, dict): payload = json.dumps(payload, separators=(",", ":")).encode() @@ -119,7 +228,7 @@ def exact(size): size = second & 127 if size == 126: size = struct.unpack("!H", exact(2))[0] - if size == 127: + elif size == 127: size = struct.unpack("!Q", exact(8))[0] if not second & 128: raise AssertionError("client frames must be masked") @@ -132,9 +241,34 @@ class MockServer(http.server.ThreadingHTTPServer): daemon_threads = True block_on_close = False - def __init__(self, scripts): + def __init__(self, scripts, venue="binance"): super().__init__(("127.0.0.1", 0), Handler) + self.venue = venue self.scripts = scripts + self.aggregates = copy.deepcopy(AGGREGATES) + self.okx_trades = copy.deepcopy(OKX_TRADES) + self.okx_candles = copy.deepcopy(OKX_CANDLES) + self.okx_unconfirmed = set() + self.instrument = { + "instId": OKX_SWAP, + "instType": "SWAP", + "state": "live", + "ctType": "linear", + "ctValCcy": "TEST", + "ctVal": "0.01", + "ctMult": "1", + } + self.bybit_bars = copy.deepcopy(BYBIT_BARS) + self.bybit_ascending = False + self.agg_window_error = False + self.queries = [] + self.subscriptions = [] + self.ws_paths = [] + self.exchange_info_bytes = 0 + self.pings = [] + self.okx_time_lookup_skip = 0 + self.refuse_once = None + self.usdm_unknown_symbol = False self.connections = 0 self.trades = copy.deepcopy(TRADES) self.bars = copy.deepcopy(BARS) @@ -168,12 +302,15 @@ def do_GET(self): query = urllib.parse.parse_qs(parsed.query) with self.server.lock: self.server.requests.append(parsed.path) + self.server.queries.append((parsed.path, query)) self.server.request_times.append((parsed.path, time.monotonic())) self.server.auth_headers.extend( - name for name in ("Authorization", "X-MBX-APIKEY") if self.headers.get(name) + name + for name in ("Authorization", "X-MBX-APIKEY", "OK-ACCESS-KEY", "X-BAPI-API-KEY") + if self.headers.get(name) ) - if parsed.path == "/stream": - self.websocket(query) + if self.headers.get("Upgrade", "").lower() == "websocket": + self.websocket(parsed.path, query) return with self.server.lock: if self.server.fail_status: @@ -183,6 +320,15 @@ def do_GET(self): self.server.rate_once = False self.respond(429, {}, retry=True) return + if self.server.refuse_once: + status, value = self.server.refuse_once + self.server.refuse_once = None + self.respond(status, value) + return + if self.server.venue != "binance": + status, rows = getattr(self, "rest_" + self.server.venue)(parsed.path, query) + self.respond(status, rows) + return if parsed.path == "/api/v3/exchangeInfo": rows = { "rateLimits": [ @@ -276,6 +422,151 @@ def do_GET(self): return self.respond(200, rows) + def rest_usdm(self, path, query): + server = self.server + if path == "/fapi/v1/exchangeInfo": + # The real document lists every contract and exceeds the 1 MiB parser bound. + document = { + "timezone": "UTC", + "assets": [ + {"asset": "PAD" + str(index), "note": "x" * 96} for index in range(12000) + ], + "rateLimits": [ + { + "rateLimitType": "REQUEST_WEIGHT", + "interval": "MINUTE", + "intervalNum": 1, + "limit": 2400, + } + ], + "symbols": [{"symbol": "TESTUSDT", "contractType": "PERPETUAL"}], + } + server.exchange_info_bytes = len(json.dumps(document, separators=(",", ":"))) + return 200, document + if path == "/fapi/v1/aggTrades": + if "fromId" in query: + start = int(query["fromId"][0]) + if server.agg_window_error and start > 499: + return 400, { + "code": -4166, + "msg": "Search window is restricted to recent 2 days only.", + } + identifiers = [key for key in sorted(server.aggregates) if key >= start] + else: + begin, end = int(query["startTime"][0]), int(query["endTime"][0]) + identifiers = [ + key + for key in sorted(server.aggregates) + if begin <= server.aggregates[key][0] <= end + ] + rows = [] + for identifier in identifiers[: int(query["limit"][0])]: + matched, price, quantity, net, first, last = server.aggregates[identifier] + rows.append( + { + "a": identifier, + "p": price, + "q": quantity, + "nq": net, + "f": first, + "l": last, + "T": matched, + "m": True, + } + ) + return 200, rows + if path == "/fapi/v1/klines": + if server.usdm_unknown_symbol: + return 400, {"code": -1121, "msg": "Invalid symbol."} + if "startTime" not in query: + return 200, [] + start, end = int(query["startTime"][0]), int(query["endTime"][0]) + rows = [] + for minute in sorted(server.bars): + if start <= minute <= end: + opening, high, low, close, volume, _first, _last, count = server.bars[minute] + rows.append( + [ + minute, + opening, + high, + low, + close, + volume, + minute + 59999, + "0", + count, + "0", + "0", + "0", + ] + ) + return 200, rows[: int(query["limit"][0])] + return 404, {} + + def rest_okx(self, path, query): + server = self.server + if path == "/api/v5/public/instruments": + match = query["instId"][0] == server.instrument["instId"] + return 200, {"code": "0", "msg": "", "data": [server.instrument] if match else []} + if path == "/api/v5/market/history-trades": + limit = int(query["limit"][0]) + if query.get("type") == ["2"]: + before_time = int(query["after"][0]) + keys = [ + key + for key in sorted(server.okx_trades) + if server.okx_trades[key][0] < before_time + ] + # The time lookup can answer an older print first (lag, or one millisecond's order). + keys = keys[: len(keys) - server.okx_time_lookup_skip] + else: + upper = int(query["after"][0]) + lower = int(query["before"][0]) + keys = [key for key in sorted(server.okx_trades) if lower < key < upper] + rows = [] + for identifier in sorted(keys, reverse=True)[:limit]: + matched, price, size = server.okx_trades[identifier] + rows.append( + { + "instId": OKX_SWAP, + "side": "buy", + "sz": size, + "px": price, + "source": "0", + "tradeId": str(identifier), + "ts": str(matched), + } + ) + return 200, {"code": "0", "msg": "", "data": rows} + if path == "/api/v5/market/history-candles": + upper, lower = int(query["after"][0]), int(query["before"][0]) + rows = [] + for minute in sorted(server.okx_candles, reverse=True): + if lower < minute < upper: + opening, high, low, close, volume = server.okx_candles[minute] + confirm = "0" if minute in server.okx_unconfirmed else "1" + rows.append([str(minute), opening, high, low, close, volume, "0", "0", confirm]) + return 200, {"code": "0", "msg": "", "data": rows[: int(query["limit"][0])]} + return 404, {} + + def rest_bybit(self, path, query): + server = self.server + if path == "/v5/market/instruments-info": + row = {"symbol": "TESTUSDT", "contractType": "LinearPerpetual", "status": "Trading"} + return 200, {"retCode": 0, "retMsg": "OK", "result": {"list": [row]}} + if path == "/v5/market/kline": + start, end, limit = (int(query[key][0]) for key in ("start", "end", "limit")) + # Rows newest first; a range wider than the limit keeps only its newest rows. + minutes = [minute for minute in sorted(server.bybit_bars) if start <= minute <= end] + minutes = sorted(minutes, reverse=True)[:limit] + if server.bybit_ascending: + minutes.reverse() + rows = [[str(minute), *server.bybit_bars[minute]] for minute in minutes] + rows = [[*row, "0"] for row in rows] + return 200, {"retCode": 0, "retMsg": "OK", "result": {"list": rows}} + return 404, {} + def respond(self, status, value, retry=False): payload = json.dumps(value, separators=(",", ":")).encode() self.send_response(status) @@ -289,10 +580,27 @@ def respond(self, status, value, retry=False): self.wfile.write(payload) self.close_connection = True - def websocket(self, query): - assert query["streams"] in (["testusdt@trade/testusdt@kline_1m"], ["testusdt@kline_1m"]) + def websocket(self, path, query): with self.server.lock: - self.server.streams.append(query["streams"][0]) + self.server.ws_paths.append(self.path) + if self.server.venue == "binance": + assert path == "/stream" + assert query["streams"] in (["testusdt@trade/testusdt@kline_1m"], ["testusdt@kline_1m"]) + elif self.server.venue == "usdm": + # Only the routed /market endpoint serves aggTrade and kline; an unrouted connection + # gets legacy raw prints only. + assert path in ("/market/stream", "/stream") + assert query["streams"] in ( + ["testusdt@aggTrade/testusdt@kline_1m"], + ["testusdt@kline_1m"], + ) + elif self.server.venue == "okx": + assert path == "/ws/v5/business" + else: + assert path == "/v5/public/linear" + if "streams" in query: + with self.server.lock: + self.server.streams.append(query["streams"][0]) key = self.headers["Sec-WebSocket-Key"] accept = base64.b64encode( hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest() @@ -306,6 +614,11 @@ def websocket(self, query): connection_index = self.server.connections self.server.connections += 1 script = self.server.scripts[min(connection_index, len(self.server.scripts) - 1)] + if self.server.venue == "usdm" and path == "/stream": + script = [trade(100)] + if self.server.venue in ("okx", "bybit"): + self.subscribed(script) + return self.connection.settimeout(3) for action in script: if isinstance(action, dict): @@ -336,6 +649,17 @@ def websocket(self, query): row = list(self.server.bars[action[1]]) row[4] = "0.70000000" self.server.bars[action[1]] = tuple(row) + elif action[0] == "lag_rest": + # The REST row trails the WebSocket close, then catches up. + with self.server.lock: + original = self.server.bars[action[1]] + self.server.bars[action[1]] = (*original[:4], "0.70000000", *original[5:]) + + def restore(minute=action[1], row=original): + with self.server.lock: + self.server.bars[minute] = row + + threading.Timer(action[2], restore).start() elif action[0] == "close": frame(self.connection, b"", opcode=8) self.close_connection = True @@ -350,12 +674,109 @@ def websocket(self, query): break self.close_connection = True + def subscribed(self, script): + """OKX/Bybit: the client subscribes by message and keeps the socket alive with text pings, + which a reader thread answers while the script runs.""" + lock = threading.Lock() + okx = self.server.venue == "okx" + + def send(payload, opcode=1): + with lock: + frame(self.connection, payload, opcode) + + self.connection.settimeout(3) + opcode, payload = receive(self.connection) + request = json.loads(payload) + assert opcode == 1 and request["op"] == "subscribe" + with self.server.lock: + self.server.subscriptions.append(request) + for argument in request["args"]: + if okx: + send({"event": "subscribe", "arg": argument, "connId": "mock"}) + else: + send( + { + "success": True, + "ret_msg": "", + "conn_id": "mock", + "req_id": "", + "op": "subscribe", + } + ) + closed = threading.Event() + + def read(): + self.connection.settimeout(0.1) + while not closed.is_set(): + try: + opcode, payload = receive(self.connection) + except TimeoutError: + continue + except (EOFError, ConnectionResetError, OSError): + break + if opcode == 8: + break + text = payload.decode() + with self.server.lock: + self.server.pings.append(text) + if okx and text == "ping": + send("pong") + elif not okx and json.loads(text) == {"op": "ping"}: + send({"success": True, "ret_msg": "pong", "conn_id": "mock", "op": "ping"}) + closed.set() + + reader = threading.Thread(target=read, daemon=True) + reader.start() + for action in script: + if isinstance(action, dict): + send(action) + elif action[0] == "pause": + time.sleep(action[1]) + elif action[0] == "raw": + send(action[1]) + elif action[0] == "add_okx_trade": + with self.server.lock: + self.server.okx_trades[action[1]] = action[2] + elif action[0] == "set_instrument": + with self.server.lock: + self.server.instrument = dict(self.server.instrument, **action[1]) + elif action[0] == "okx_unconfirm": + # REST trails the WebSocket confirm of this minute for a moment. + with self.server.lock: + self.server.okx_unconfirmed.add(action[1]) + threading.Timer( + action[2], self.server.okx_unconfirmed.discard, (action[1],) + ).start() + elif action[0] == "revise_okx_trade": + with self.server.lock: + matched, price, _size = self.server.okx_trades[action[1]] + self.server.okx_trades[action[1]] = (matched, price, "11") + elif action[0] == "close": + send(b"", opcode=8) + closed.set() + break + reader.join(timeout=15) + closed.set() + self.close_connection = True + + +VENUES = { + "binance": ("binance", "spot", "TESTUSDT"), + "usdm": ("binance", "usdm", "TESTUSDT"), + "okx": ("okx", "swap", OKX_SWAP), + "bybit": ("bybit", "linear", "TESTUSDT"), +} + + +def sequence(output): + return [event["seq"] if event["type"] == "tick" else ("time", event["ts"]) for event in output] + class FeedMockTests(unittest.TestCase): def test_public_qualification_fails_on_any_action_difference(self): for ticks_equal, expected in ((False, 1), (True, 0)): soak = Soak.__new__(Soak) - soak.options = SimpleNamespace(bar_minutes=21, tick_minutes=21) + soak.options = SimpleNamespace(bar_minutes=21, tick_minutes=21, modes="bars,ticks") soak.directory = self.directory / str(ticks_equal) soak.directory.mkdir() soak.results = {} @@ -368,34 +789,35 @@ def test_public_qualification_fails_on_any_action_difference(self): def test_r_b2_maps_only_tick_action_timestamps(self): record = { - "timestamp": 1791037920021, + "timestamp": 120021, "bar_index": 7, "order": { "id": "long", "action": "entry", "leg": "long", "contracts": 1.0, - "price": 84816.0, + "price": 10.1, "reduce_only": False, "entry_incarnation": 1, }, } - batch = dict(record, timestamp=1791037920000) + batch = dict(record, timestamp=120000) self.assertEqual(action_key(record, "ticks"), action_key(batch, "ticks")) + self.assertEqual(action_key(record, "agg-ticks"), action_key(batch, "agg-ticks")) self.assertNotEqual(action_key(record, "bars"), action_key(batch, "bars")) - moved = dict(record, order=dict(record["order"], price=84816.5)) + moved = dict(record, order=dict(record["order"], price=10.15)) self.assertNotEqual(action_key(moved, "ticks"), action_key(batch, "ticks")) def test_only_actions_on_proven_bars_are_compared(self): record = { - "timestamp": 1791051480451, + "timestamp": 180451, "bar_index": 221, "order": { "id": "S", "action": "sell", "leg": "entry", "contracts": 1.0, - "price": 84980.0, + "price": 11.2, "reduce_only": False, "entry_incarnation": 15, }, @@ -410,8 +832,8 @@ def setUp(self): self.addCleanup(self.temporary.cleanup) self.directory = pathlib.Path(self.temporary.name) - def server(self, scripts): - server = MockServer(scripts) + def server(self, scripts, venue="binance"): + server = MockServer(scripts, venue) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() self.addCleanup(server.server_close) @@ -420,15 +842,16 @@ def server(self, scripts): def command(self, server, mode="bars", extra=(), resume=False): port = server.server_address[1] + venue, market, symbol = VENUES[server.venue] arguments = [ BINARY, "run", "--venue", - "binance", + venue, "--market", - "spot", + market, "--symbol", - "TESTUSDT", + symbol, "--mode", mode, "--state-dir", @@ -542,6 +965,14 @@ def test_reconnect_changed_rest_overlap_stops_21(self): output, _ = self.run_feed(server, expected=21) self.assertEqual(len(output), 1) + def test_trailing_rest_row_is_reread_before_a_revision_stop(self): + server = self.server( + [[candle(120000), ("lag_rest", 120000, 0.45), retirement()], [candle(180000)]] + ) + output, result = self.run_feed(server, extra=["--max-messages", "2"]) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000, 180000]) + self.assertIn("rest_overlap_trailing", result.stderr) + def test_planned_24_hour_rotation_uses_overlap(self): server = self.server([[candle(120000)], [candle(180000)]]) output, _ = self.run_feed(server, extra=["--max-messages", "2", "--reconnect-seconds", "1"]) @@ -858,6 +1289,613 @@ def test_changed_committed_journal_is_refused(self): output, _ = self.run_feed(server, resume=True, expected=21) self.assertEqual(output, []) + def reset_state(self): + import shutil + + shutil.rmtree(self.directory / "state", ignore_errors=True) + + def history_pages(self, server): + return [ + query + for path, query in server.queries + if path == "/api/v5/market/history-trades" and query.get("type") == ["1"] + ] + + def test_okx_swap_bars_confirm_flag_and_contract_units(self): + server = self.server([[okx_candle(120000, "0"), okx_candle(180000)]], "okx") + output, result = self.run_feed(server, extra=["--max-messages", "2"]) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000, 180000]) + # 60 and 50 contracts of ctVal 0.01: base volumes 0.6 and 0.5; prices stay verbatim. + self.assertIn('{"ts_open":120000,"o":10.1,"h":11.2,"l":9.9,"c":9.9,"v":0.6}', result.stdout) + self.assertIn('"v":0.5}', result.stdout) + self.assertEqual( + server.subscriptions, + [{"op": "subscribe", "args": [{"channel": "candle1m", "instId": OKX_SWAP}]}], + ) + self.assertEqual(server.ws_paths, ["/ws/v5/business"]) + saved = json.loads((self.directory / "state" / "cursor.json").read_text())["cursor"] + self.assertEqual(saved["qty_multiplier"], "0.01") + + def test_okx_start_predecessor_is_proven_by_the_next_print(self): + server = self.server([[okx_trades(100), okx_trades(101)]], "okx") + server.okx_trades[98] = (119998, "10.1", "10") + server.okx_time_lookup_skip = 1 + output, _ = self.run_feed(server, "ticks", ["--max-messages", "2"]) + self.assertEqual(sequence(output), [100, 101]) + saved = json.loads((self.directory / "state" / "cursor.json").read_text())["cursor"] + self.assertEqual(saved["predecessor"]["seq"], 99) + # No print at or after the start is in REST yet: nothing is anchored and the stop is 20. + self.reset_state() + server = self.server([[okx_trades(100)]], "okx") + for identifier in range(100, 106): + del server.okx_trades[identifier] + output, result = self.run_feed(server, "ticks", expected=20) + self.assertEqual(output, []) + self.assertIn("not yet available", result.stderr) + saved = json.loads((self.directory / "state" / "cursor.json").read_text())["cursor"] + self.assertIsNone(saved["predecessor"]) + + def test_okx_busy_and_rate_limit_codes_are_retried(self): + for status, code, event in ( + (400, "50004", "rest_transient_retry"), + (200, "50011", "rest_rate_limited"), + ): + with self.subTest(code=code): + self.reset_state() + server = self.server([[okx_candle(120000)]], "okx") + server.refuse_once = (status, {"code": code, "msg": "synthetic", "data": []}) + output, result = self.run_feed(server, extra=["--max-messages", "1"]) + self.assertEqual(len(output), 1) + self.assertIn(event, result.stderr) + + def test_bybit_rate_limit_and_server_error_codes_are_retried(self): + for code, event in ((10006, "rest_rate_limited"), (10016, "rest_transient_retry")): + with self.subTest(code=code): + self.reset_state() + server = self.server([[bybit_push(bybit_row(120000, True))]], "bybit") + server.refuse_once = (200, {"retCode": code, "retMsg": "synthetic", "result": {}}) + output, result = self.run_feed(server, extra=["--max-messages", "1"]) + self.assertEqual(len(output), 1) + self.assertIn(event, result.stderr) + + def test_usdm_unknown_symbol_is_refused_at_startup(self): + server = self.server([[candle(120000)]], "usdm") + server.usdm_unknown_symbol = True + output, result = self.run_feed(server, expected=23) + self.assertEqual(output, []) + self.assertIn("unknown USD-M symbol", result.stderr) + self.assertEqual(server.connections, 0) + + def test_usdm_quiet_start_minute_closes_on_the_fence(self): + rows = { + 500: {"T": 180001, "p": "12.00000000", "q": "0.40000000", "f": 1010, "l": 1012}, + 501: {"T": 180050, "p": "11.00000000", "q": "0.10000000", "f": 1013, "l": 1013}, + 502: {"T": 240001, "p": "12.00000000", "q": "0.10000000", "f": 1014, "l": 1014}, + } + server = self.server( + [ + [ + candle(120000, v="0.00000000", n=0, f=-1, L=-1), + aggregate(500, nq=rows[500]["q"], **rows[500]), + aggregate(501, nq=rows[501]["q"], **rows[501]), + candle(180000), + aggregate(502, nq=rows[502]["q"], **rows[502]), + ] + ], + "usdm", + ) + server.aggregates = {499: AGGREGATES[499]} + for identifier, row in rows.items(): + server.aggregates[identifier] = ( + row["T"], + row["p"], + row["q"], + row["q"], + row["f"], + row["l"], + ) + output, _ = self.run_feed(server, "agg-ticks", ["--max-messages", "5"]) + self.assertEqual(sequence(output), [("time", 180000), 500, 501, ("time", 240000), 502]) + + def test_websocket_closed_before_any_frame_spends_the_reconnect_budget(self): + server = self.server([[("close",)]]) + result = subprocess.run(self.command(server), capture_output=True, text=True, timeout=40) + self.assertEqual(result.returncode, 20, result.stderr) + self.assertIn("reconnect attempts exhausted", result.stderr) + self.assertGreaterEqual(server.connections, 8) + + def test_usdm_forming_kline_never_closes_a_fenced_minute(self): + forming = candle(120000, confirmed=False) + server = self.server( + [ + [ + aggregate(500), + aggregate(501), + aggregate(502), + forming, + aggregate(503), + ("pause", 0.3), + ("raw", "{"), + ] + ], + "usdm", + ) + output, _ = self.run_feed(server, "agg-ticks", expected=23) + self.assertEqual(sequence(output), [500, 501, 502]) + + def test_usdm_conflicting_duplicate_aggregate_stops_21(self): + for script, emitted in ( + ([aggregate(500), aggregate(500, q="0.20000000")], [500]), + ( + [ + aggregate(500), + aggregate(501), + aggregate(502), + aggregate(503), + aggregate(503, p="12.10000000"), + ], + [500, 501, 502], + ), + ): + with self.subTest(emitted=emitted): + self.reset_state() + server = self.server([script], "usdm") + output, result = self.run_feed(server, "agg-ticks", expected=21) + self.assertEqual(sequence(output), emitted) + self.assertIn("duplicate print conflicts", result.stderr) + + def test_usdm_reconnect_changed_kline_overlap_stops_21(self): + server = self.server( + [ + [ + aggregate(500), + aggregate(501), + aggregate(502), + candle(120000), + aggregate(503), + ("revise_rest", 120000), + retirement(), + ], + [aggregate(504)], + ], + "usdm", + ) + output, result = self.run_feed(server, "agg-ticks", expected=21) + self.assertEqual(sequence(output), [500, 501, 502, ("time", 180000), 503]) + self.assertIn("reconnect already-closed candle changed", result.stderr) + + def test_okx_rest_page_with_a_nonunit_id_stops_20(self): + # The hole sits inside a heal page, past the 100 IDs the start predecessor's proof reads. + server = self.server([[]], "okx") + server.okx_trades = {99: OKX_TRADES[99]} + for identifier in range(100, 261): + server.okx_trades[identifier] = (120001 + (identifier - 100) * 100, "10", "1") + del server.okx_trades[230] + + def push(identifier): + matched, price, size = server.okx_trades[identifier] + row = { + "instId": OKX_SWAP, + "tradeId": str(identifier), + "px": price, + "sz": size, + "side": "buy", + "ts": str(matched), + "source": "0", + } + return {"arg": {"channel": "trades-all", "instId": OKX_SWAP}, "data": [row]} + + server.scripts = [[push(100), push(260)]] + output, result = self.run_feed(server, "ticks", expected=20) + self.assertEqual(sequence(output), [100]) + self.assertIn("trade-id page has a missing or nonunit venue trade ID", result.stderr) + + def test_usdm_rest_page_with_a_nonunit_id_stops_20(self): + server = self.server([[aggregate(500), aggregate(504)]], "usdm") + del server.aggregates[502] + output, result = self.run_feed(server, "agg-ticks", expected=20) + self.assertEqual(sequence(output), [500]) + self.assertIn("aggregate-history page has a missing or nonunit aggregate ID", result.stderr) + + def test_okx_quiet_start_waits_for_the_first_print(self): + quiet = {minute: ("10.1", "10.1", "10.1", "10.1", "0") for minute in (120000, 180000)} + quiet[240000] = ("12", "12", "12", "12", "10") + server = self.server( + [ + [ + okx_candle(120000, candles=quiet), + okx_candle(180000, candles=quiet), + # Longer than the venue's 1.5 s predecessor wait: a blind anchor would stop 20. + ("pause", 3.0), + ("add_okx_trade", 100, (240001, "12", "10")), + ("add_okx_trade", 101, (300001, "12", "10")), + okx_trades(100, ts="240001", px="12", sz="10"), + okx_candle(240000, candles=quiet), + okx_trades(101, ts="300001", px="12", sz="10"), + ] + ], + "okx", + ) + server.okx_trades = {99: OKX_TRADES[99]} + server.okx_candles = quiet + output, result = self.run_feed(server, "ticks", ["--max-messages", "4"]) + self.assertEqual( + sequence(output), [("time", 180000), ("time", 240000), 100, ("time", 300000)] + ) + self.assertNotIn('"code":20', result.stderr) + + def test_bybit_http_403_is_a_retryable_rate_ban(self): + server = self.server([[bybit_push(bybit_row(120000, True))]], "bybit") + server.fail_status = 403 + output, result = self.run_feed(server, expected=20) + self.assertEqual(output, []) + self.assertIn("Bybit IP rate ban", result.stderr) + self.assertEqual(server.auth_headers, []) + + def test_okx_changed_contract_multiplier_stops_21_on_reconnect(self): + notice = {"event": "notice", "code": "64008", "msg": "upgrade", "connId": "mock"} + server = self.server( + [ + # The pause lets the first connection's instrument check and its bar finish first. + [okx_candle(120000), ("pause", 1.0), ("set_instrument", {"ctVal": "0.1"}), notice], + [okx_candle(180000)], + ], + "okx", + ) + output, result = self.run_feed(server, expected=21) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000]) + self.assertIn("contract multiplier changed", result.stderr) + + def test_okx_websocket_and_rest_lexemes_differ_but_overlap_values_match(self): + notice = {"event": "notice", "code": "64008", "msg": "upgrade", "connId": "mock"} + rendered = {120000: ("10.10", "11.20", "9.90", "9.9", "60.0")} + server = self.server( + [[okx_candle(120000, candles=rendered), notice], [okx_candle(180000)]], "okx" + ) + output, result = self.run_feed(server, extra=["--max-messages", "2"]) + self.assertEqual([event["bar"]["ts_open"] for event in output], [120000, 180000]) + # The WebSocket lexemes are emitted verbatim; the REST overlap compares exact values. + self.assertIn('"o":10.10,"h":11.20,"l":9.90,"c":9.9,"v":0.6}', result.stdout) + self.assertGreaterEqual(server.connections, 2) + + def test_okx_warmup_rereads_a_rest_row_that_trails_the_websocket_confirm(self): + server = self.server([[("okx_unconfirm", 180000, 0.35), okx_candle(180000)]], "okx") + port = server.server_address[1] + output = self.directory / "warmup.csv" + command = [ + BINARY, + "warmup", + "--venue", + "okx", + "--market", + "swap", + "--symbol", + OKX_SWAP, + "--start", + "120000", + "--end", + "240000", + "--output", + str(output), + "--rest-url", + f"http://127.0.0.1:{port}", + "--ws-url", + f"ws://127.0.0.1:{port}", + "--allow-insecure-http", + ] + result = subprocess.run(command, capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("rest_overlap_trailing", result.stderr) + self.assertEqual(len(output.read_text().splitlines()), 3) + + def test_okx_unconfirmed_rest_candle_is_never_a_bar(self): + server = self.server([[okx_candle(180000)]], "okx") + server.okx_unconfirmed = {120000} + output, _ = self.run_feed(server, expected=20) + self.assertEqual(output, []) + + def test_okx_ticks_fence_closes_minute_on_exact_candle(self): + server = self.server( + [ + [ + okx_trades(100, 101, 102), + okx_candle(120000), + okx_trades(103), + okx_candle(180000), + okx_trades(104), + okx_trades(105), + ] + ], + "okx", + ) + output, result = self.run_feed(server, "ticks", ["--max-messages", "8"]) + self.assertEqual( + sequence(output), [100, 101, 102, ("time", 180000), 103, 104, ("time", 240000), 105] + ) + self.assertIn('{"type":"tick","ts":120001,"seq":100,"price":10.1,"qty":0.1}', result.stdout) + self.assertEqual( + server.subscriptions[0]["args"][1], {"channel": "trades-all", "instId": OKX_SWAP} + ) + + def test_okx_count_aggregated_push_is_rejected_on_ticks(self): + for payload in (okx_trades(100, count="3"), okx_trades(100, channel="trades")): + with self.subTest(payload=payload["arg"]["channel"]): + self.reset_state() + server = self.server([[payload]], "okx") + output, result = self.run_feed(server, "ticks", expected=23) + self.assertEqual(output, []) + self.assertRegex(result.stderr, "aggregated OKX trades") + + def test_okx_text_ping_is_answered_and_never_data(self): + server = self.server([[("pause", 2.5), okx_candle(120000)]], "okx") + output, _ = self.run_feed(server, extra=["--max-messages", "1", "--keepalive-seconds", "1"]) + self.assertEqual(len(output), 1) + self.assertGreaterEqual(server.pings.count("ping"), 2) + + def test_okx_tick_gap_heals_by_trade_id_pages(self): + server = self.server([[]], "okx") + server.okx_trades = {99: OKX_TRADES[99]} + for offset in range(250): + server.okx_trades[100 + offset] = (120001 + offset * 100, "10", "1") + server.okx_trades[350] = (180001, "10", "1") + server.okx_candles = {120000: ("10", "10", "10", "10", "250")} + + def push(identifier): + matched, price, size = server.okx_trades[identifier] + row = { + "instId": OKX_SWAP, + "tradeId": str(identifier), + "px": price, + "sz": size, + "side": "buy", + "ts": str(matched), + "source": "0", + } + return {"arg": {"channel": "trades-all", "instId": OKX_SWAP}, "data": [row]} + + server.scripts = [[push(349), okx_candle(120000, candles=server.okx_candles), push(350)]] + output, _ = self.run_feed(server, "ticks", ["--max-messages", "251"]) + self.assertEqual([event["seq"] for event in output[:-1]], list(range(100, 350))) + self.assertEqual(output[-1], {"type": "time", "ts": 180000}) + self.assertGreaterEqual(len(self.history_pages(server)), 3) + self.assertTrue(all(page["limit"] == ["100"] for page in self.history_pages(server))) + + def test_okx_inverse_swap_is_refused_before_streaming(self): + server = self.server([[okx_candle(120000)]], "okx") + server.instrument = dict(server.instrument, ctType="inverse", ctValCcy="USD") + output, result = self.run_feed(server, expected=23) + self.assertEqual(output, []) + self.assertIn("inverse OKX swaps are refused", result.stderr) + self.assertEqual(server.connections, 0) + + def test_okx_notice_reconnects_and_changed_overlap_stops_21(self): + notice = {"event": "notice", "code": "64008", "msg": "upgrade", "connId": "mock"} + server = self.server( + [[okx_trades(100, 101), ("revise_okx_trade", 100), notice], [okx_trades(102)]], "okx" + ) + output, result = self.run_feed(server, "ticks", expected=21) + self.assertEqual([event["seq"] for event in output], [100, 101]) + self.assertIn("reconnect print overlap changed", result.stderr) + self.assertGreaterEqual(server.connections, 2) + + def test_okx_gap_beyond_three_month_window_stops_20(self): + far = 120001 + 90 * 86400000 + server = self.server([[okx_trades(100), okx_trades(105, ts=str(far))]], "okx") + server.okx_trades[105] = (far, "12", "10") + output, result = self.run_feed(server, "ticks", expected=20) + self.assertEqual([event["seq"] for event in output], [100]) + self.assertIn("beyond the venue's REST history window", result.stderr) + # Only the start predecessor's proof read prints by ID; the gap from 101 was never read. + self.assertEqual( + [page for page in self.history_pages(server) if int(page["before"][0]) >= 100], [] + ) + + def test_okx_warmup_converts_contract_volume(self): + server = self.server([[okx_candle(240000)]], "okx") + port = server.server_address[1] + output = self.directory / "warmup.csv" + command = [ + BINARY, + "warmup", + "--venue", + "okx", + "--market", + "swap", + "--symbol", + OKX_SWAP, + "--start", + "120000", + "--end", + "240000", + "--output", + str(output), + "--rest-url", + f"http://127.0.0.1:{port}", + "--ws-url", + f"ws://127.0.0.1:{port}", + "--allow-insecure-http", + ] + result = subprocess.run(command, capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + output.read_text().splitlines()[1:], + ["120000,10.1,11.2,9.9,9.9,0.6", "180000,12,12,11,11,0.5"], + ) + manifest = json.loads(pathlib.Path(str(output) + ".manifest.json").read_text()) + self.assertEqual(manifest["qty_multiplier"], "0.01") + + def test_bybit_bars_two_entry_push_and_confirm_flag(self): + server = self.server( + [ + [ + bybit_push(bybit_row(120000, False, close="10.5", volume="0.3")), + bybit_push( + bybit_row(120000, True), + bybit_row(180000, False, close="11.5", volume="0.05"), + ), + bybit_push(bybit_row(180000, True), bybit_row(240000, False)), + ] + ], + "bybit", + ) + output, _ = self.run_feed(server, extra=["--max-messages", "2"]) + # The forming rows carry other closes and volumes: only the confirmed rows become bars. + self.assertEqual( + [event["bar"] for event in output], + [ + {"ts_open": 120000, "o": 10.1, "h": 11.2, "l": 9.9, "c": 9.9, "v": 0.6}, + {"ts_open": 180000, "o": 12, "h": 12, "l": 11, "c": 11, "v": 0.5}, + ], + ) + self.assertEqual(server.subscriptions, [{"op": "subscribe", "args": ["kline.1.TESTUSDT"]}]) + self.assertEqual(server.ws_paths, ["/v5/public/linear"]) + + def test_bybit_tick_modes_are_refused_at_startup(self): + server = self.server([[]], "bybit") + for mode in ("ticks", "agg-ticks"): + with self.subTest(mode=mode): + output, result = self.run_feed(server, mode, expected=23) + self.assertEqual(output, []) + self.assertIn("Bybit tick mode is refused", result.stderr) + self.assertIn("not a contiguous cursor", result.stderr) + self.assertEqual(server.requests, []) + + def test_bybit_reverse_order_rest_paging_heals_hole(self): + server = self.server([[bybit_push(bybit_row(300000, True))]], "bybit") + output, _ = self.run_feed(server, extra=["--max-messages", "4"]) + self.assertEqual( + [event["bar"]["ts_open"] for event in output], [120000, 180000, 240000, 300000] + ) + windows = [query for path, query in server.queries if path == "/v5/market/kline"] + self.assertTrue(windows) + for query in windows: + start, end, limit = (int(query[key][0]) for key in ("start", "end", "limit")) + self.assertEqual((end - start) // 60000 + 1, limit) + self.reset_state() + server = self.server([[bybit_push(bybit_row(300000, True))]], "bybit") + server.bybit_ascending = True + output, _ = self.run_feed(server, expected=20) + self.assertEqual(output, []) + + def test_bybit_json_ping_keepalive(self): + server = self.server([[("pause", 2.5), bybit_push(bybit_row(120000, True))]], "bybit") + output, _ = self.run_feed(server, extra=["--max-messages", "1", "--keepalive-seconds", "1"]) + self.assertEqual(len(output), 1) + self.assertGreaterEqual(server.pings.count('{"op":"ping"}'), 2) + + def test_usdm_bars_use_the_routed_market_stream(self): + server = self.server([[candle(120000)]], "usdm") + output, _ = self.run_feed(server, extra=["--max-messages", "1"]) + self.assertEqual(len(output), 1) + self.assertEqual(server.ws_paths, ["/market/stream?streams=testusdt@kline_1m"]) + self.assertGreater(server.exchange_info_bytes, 1024 * 1024) + + def test_usdm_raw_trade_event_stops_23(self): + server = self.server([[trade(100)]], "usdm") + output, result = self.run_feed(server, "agg-ticks", expected=23) + self.assertEqual(output, []) + self.assertIn("raw USD-M trades are not a supported source", result.stderr) + + def test_usdm_agg_ticks_one_print_per_aggregate_with_q(self): + server = self.server( + [[aggregate(500), aggregate(501), aggregate(502), candle(120000), aggregate(503)]], + "usdm", + ) + output, result = self.run_feed(server, "agg-ticks", ["--max-messages", "5"]) + self.assertEqual(sequence(output), [500, 501, 502, ("time", 180000), 503]) + self.assertIn('"seq":500,"price":10.10000000,"qty":0.10000000}', result.stdout) + self.assertEqual( + server.ws_paths, ["/market/stream?streams=testusdt@aggTrade/testusdt@kline_1m"] + ) + + def test_usdm_straddling_aggregate_closes_on_the_fence(self): + # Aggregate 502 is dated in minute 120000 but one of its fills belongs to the next kline. + server = self.server( + [ + [ + aggregate(500), + aggregate(501), + aggregate(502), + candle(120000, v="0.59700000"), + aggregate(503), + aggregate(504), + candle(180000, v="0.50300000"), + aggregate(505), + ] + ], + "usdm", + ) + output, _ = self.run_feed(server, "agg-ticks", ["--max-messages", "8"]) + self.assertEqual( + sequence(output), [500, 501, 502, ("time", 180000), 503, 504, ("time", 240000), 505] + ) + + def test_usdm_agg_gap_heals_by_from_id(self): + server = self.server([[aggregate(502), candle(120000), aggregate(503)]], "usdm") + output, _ = self.run_feed(server, "agg-ticks", ["--max-messages", "5"]) + self.assertEqual(sequence(output), [500, 501, 502, ("time", 180000), 503]) + starts = [ + int(query["fromId"][0]) + for path, query in server.queries + if path == "/fapi/v1/aggTrades" and "fromId" in query + ] + self.assertIn(500, starts) + + def test_usdm_agg_gap_beyond_48_hours_stops_20(self): + far = 120001 + 49 * 3600000 + server = self.server([[aggregate(500), aggregate(505, T=far)]], "usdm") + server.aggregates[505] = (far, "12.00000000", "0.10000000", "0.10000000", 1014, 1014) + output, result = self.run_feed(server, "agg-ticks", expected=20) + self.assertEqual([event["seq"] for event in output], [500]) + self.assertIn("beyond the venue's REST history window", result.stderr) + starts = [ + int(query["fromId"][0]) + for path, query in server.queries + if path == "/fapi/v1/aggTrades" and "fromId" in query + ] + self.assertNotIn(501, starts) + + def test_usdm_two_day_window_rejection_maps_to_20(self): + server = self.server([[aggregate(500), aggregate(503)]], "usdm") + server.agg_window_error = True + output, result = self.run_feed(server, "agg-ticks", expected=20) + self.assertEqual([event["seq"] for event in output], [500]) + self.assertIn("2-day aggTrades window", result.stderr) + + def test_usdm_raw_tick_mode_is_refused(self): + server = self.server([[]], "usdm") + output, result = self.run_feed(server, "ticks", expected=23) + self.assertEqual(output, []) + self.assertIn("use --mode agg-ticks", result.stderr) + self.assertEqual(server.requests, []) + + def test_coinbase_is_deferred(self): + result = subprocess.run( + [ + BINARY, + "run", + "--venue", + "coinbase", + "--market", + "spot", + "--symbol", + "BTCUSD", + "--mode", + "bars", + "--state-dir", + str(self.directory / "state"), + "--start", + "120000", + ], + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 23) + self.assertEqual(result.stdout, "") + self.assertIn("Coinbase is deferred", result.stderr) + self.assertFalse((self.directory / "state").exists()) + def warmup(self, server): port = server.server_address[1] output = self.directory / "warmup.csv" diff --git a/native/feed/tests/public_e2e.py b/native/feed/tests/public_e2e.py index 4142d8f..94ac685 100644 --- a/native/feed/tests/public_e2e.py +++ b/native/feed/tests/public_e2e.py @@ -38,7 +38,7 @@ def action_key(record, mode): stays exact.""" order = record["order"] timestamp = record["timestamp"] - if mode == "ticks": + if mode != "bars": timestamp = timestamp // SCRIPT_TF_MS * SCRIPT_TF_MS return ( timestamp, @@ -61,6 +61,140 @@ def proven_actions(records, mode, window): return inside, len(records) - len(inside) +AGENT = {"User-Agent": "pineforge-feed-qualification/0.1"} + + +def canonical(value): + """The feed's contract-to-base token: fixed point, no exponent, no trailing fractional zeros.""" + return format(value.normalize(), "f") + + +# (venue, market) -> public REST origin, the feed's tick mode (none for Bybit), runner label +PROFILES = { + ("binance", "spot"): ("https://api.binance.com", "ticks", "BINANCE:{}"), + ("binance", "usdm"): ("https://fapi.binance.com", "agg-ticks", "BINANCE:{}.P"), + ("okx", "spot"): ("https://www.okx.com", "ticks", "OKX:{}"), + ("okx", "swap"): ("https://www.okx.com", "ticks", "OKX:{}"), + ("bybit", "spot"): ("https://api.bybit.com", None, "BYBIT:{}"), + ("bybit", "linear"): ("https://api.bybit.com", None, "BYBIT:{}.P"), +} + + +class Venue: + """Public REST truth for one venue and market, in the feed's normalized units.""" + + def __init__(self, venue, market, symbol): + self.venue, self.market, self.symbol = venue, market, symbol + self.rest, self.tick_mode, label = PROFILES[(venue, market)] + self.label = label.format(symbol) + self.lock = threading.Lock() + # USD-M aggregates can straddle a minute boundary: their kline is not their exact sum. + self.candle_is_print_sum = (venue, market) != ("binance", "usdm") + self.multiplier = Decimal(1) + if venue == "okx" and market == "swap": + row = self.fetch("/api/v5/public/instruments", {"instType": "SWAP", "instId": symbol})[ + "data" + ][0] + assert row["ctType"] == "linear" + self.multiplier = Decimal(row["ctVal"]) * Decimal(row["ctMult"]) + + def fetch(self, path, query, weight=2): + with self.lock: + time.sleep(weight / 10) + url = self.rest + path + "?" + urllib.parse.urlencode(query) + request = urllib.request.Request(url, headers=AGENT) + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read(), parse_float=Decimal) + + def quantity(self, token): + return token if self.multiplier == 1 else canonical(Decimal(token) * self.multiplier) + + def candles(self, start, end): + """Closed 1m candles in [start, end): ts, the five tokens and the raw-trade count when the + feed's tick proof uses it (Binance spot only).""" + result = [] + while start < end: + if self.venue == "okx": + page = self.fetch( + "/api/v5/market/history-candles", + { + "instId": self.symbol, + "bar": "1m", + "after": min(end, start + 100 * 60000), + "before": start - 1, + "limit": 100, + }, + )["data"][::-1] + assert all(row[8] == "1" for row in page), "unconfirmed REST candle" + page = [(int(row[0]), [*row[1:5], self.quantity(row[5])], None) for row in page] + elif self.venue == "bybit": + count = min(1000, (end - start) // 60000) + page = self.fetch( + "/v5/market/kline", + { + "category": self.market, + "symbol": self.symbol, + "interval": 1, + "start": start, + "end": start + (count - 1) * 60000, + "limit": count, + }, + )["result"]["list"][::-1] + page = [(int(row[0]), row[1:6], None) for row in page] + else: + path = "/fapi/v1/klines" if self.market == "usdm" else "/api/v3/klines" + page = self.fetch( + path, + { + "symbol": self.symbol, + "interval": "1m", + "startTime": start, + "endTime": end - 1, + "limit": 1000, + }, + 5, + ) + spot = self.market == "spot" + page = [(row[0], row[1:6], row[8] if spot else None) for row in page] + assert page and page[0][0] == start, "REST minute missing" + result += page + start = page[-1][0] + 60000 + assert [row[0] for row in result] == list(range(result[0][0], end, 60000)) + return result + + def prints(self, first): + """One REST page of contiguous prints from `first`: id, ts and the price/qty tokens.""" + if self.venue == "okx": + page = self.fetch( + "/api/v5/market/history-trades", + { + "instId": self.symbol, + "type": 1, + "after": first + 100, + "before": first - 1, + "limit": 100, + }, + )["data"][::-1] + page = [ + (int(row["tradeId"]), int(row["ts"]), row["px"], self.quantity(row["sz"])) + for row in page + ] + elif self.market == "usdm": + page = self.fetch( + "/fapi/v1/aggTrades", {"symbol": self.symbol, "fromId": first, "limit": 1000}, 20 + ) + page = [(row["a"], row["T"], row["p"], row["q"]) for row in page] + else: + page = self.fetch( + "/api/v3/historicalTrades", + {"symbol": self.symbol, "fromId": first, "limit": 1000}, + 25, + ) + page = [(row["id"], row["time"], row["price"], row["qty"]) for row in page] + assert page and page[0][0] == first, "REST print page missing" + return page + + class Receiver(http.server.ThreadingHTTPServer): daemon_threads = True @@ -94,6 +228,7 @@ def log_message(self, *_): class Soak: def __init__(self, options): self.options = options + self.venue = Venue(options.venue, options.market, options.symbol) self.directory = pathlib.Path(options.directory).resolve() self.directory.mkdir(parents=True, exist_ok=False) self.receiver = Receiver(self.directory) @@ -118,13 +253,13 @@ def runner_command(self, mode, name, ledger, source, cursor=0): "--feed", source, "--mode", - mode, + "bars" if mode == "bars" else "ticks", "--script-tf", "1", "--ledger", str(ledger), "--symbol", - "BINANCE:BTCUSDT", + self.venue.label, "--name", "public-rsi-qualification", "--webhook-url", @@ -140,11 +275,11 @@ def warmup(self): self.options.feed, "warmup", "--venue", - "binance", + self.venue.venue, "--market", - "spot", + self.venue.market, "--symbol", - "BTCUSDT", + self.venue.symbol, "--start", str(self.cut - 200 * 60000), "--end", @@ -167,11 +302,11 @@ def generation(self, mode, generation, cursor): self.options.feed, "run", "--venue", - "binance", + self.venue.venue, "--market", - "spot", + self.venue.market, "--symbol", - "BTCUSDT", + self.venue.symbol, "--mode", mode, "--state-dir", @@ -232,30 +367,17 @@ def finish(self, mode, generation, processes, crash=False): summary = json.loads((self.directory / f"{mode}-runner-{generation}.stdout").read_text()) return summary["inputs_committed"] - def fetch(self, path, query, weight=2): - time.sleep(weight / 10) - url = "https://api.binance.com" + path + "?" + urllib.parse.urlencode(query) - with urllib.request.urlopen(url, timeout=30) as response: - return json.loads(response.read(), parse_float=Decimal) - - def candles(self, start, end): - result = [] - while start < end: - page = self.fetch( - "/api/v3/klines", - { - "symbol": "BTCUSDT", - "interval": "1m", - "startTime": start, - "endTime": end - 1, - "limit": 1000, - }, - ) - assert page and page[0][0] == start, "REST minute missing" - result += page - start = page[-1][0] + 60000 - assert len(result) * 60000 == end - result[0][0] - return result + @staticmethod + def settled(check, attempts=6, pause=5): + """REST rows of the newest minutes and prints can trail the feed's WebSocket by seconds: + a REST disagreement must persist across spaced re-reads (about 25 s) before it fails.""" + for attempt in range(attempts): + try: + return check() + except AssertionError: + if attempt == attempts - 1: + raise + time.sleep(pause) def validate(self, mode, duration, cursor, minutes): getcontext().prec = 160 @@ -271,16 +393,24 @@ def validate(self, mode, duration, cursor, minutes): if mode == "bars": normalized = [event["bar"] for event in tape] assert len(normalized) >= minutes - 1 and duration >= (minutes - 1) * 60 - expected = self.candles(self.cut, normalized[-1]["ts_open"] + 60000) - for index, (actual, candle) in enumerate(zip(normalized, expected, strict=True)): - assert [actual["ts_open"], *(actual[key] for key in ("o", "h", "l", "c", "v"))] == [ - candle[0], - *(Decimal(candle[index]) for index in range(1, 6)), - ], "bar REST mismatch" - assert tape[index]["type"] == "bar" - assert [tokens[index]["bar"][key] for key in ("o", "h", "l", "c", "v")] == candle[ - 1:6 - ], "bar decimal tokens changed" + + def compare_bars(): + expected = self.venue.candles(self.cut, normalized[-1]["ts_open"] + 60000) + for index, (actual, candle) in enumerate(zip(normalized, expected, strict=True)): + keys = ("o", "h", "l", "c", "v") + assert [actual["ts_open"], *(actual[key] for key in keys)] == [ + candle[0], + *(Decimal(token) for token in candle[1]), + ], "bar REST mismatch" + assert tape[index]["type"] == "bar" + # OKX renders one candle number differently on its WebSocket ("100.0") and + # REST ("100"); values are compared exactly above, lexemes where they agree. + if self.venue.venue != "okx": + assert [tokens[index]["bar"][key] for key in keys] == candle[1], ( + "bar decimal tokens changed" + ) + + self.settled(compare_bars) self.receipt( f"PASS bars equal REST minutes={len(normalized)} duration_seconds={duration:.3f}" ) @@ -292,67 +422,87 @@ def validate(self, mode, duration, cursor, minutes): assert len(times) >= minutes - 1 and trades for previous, current in itertools.pairwise(trades): assert current["seq"] == previous["seq"] + 1 and current["ts"] >= previous["ts"] - next_id = trades[0]["seq"] - trade_offset = 0 - while trade_offset < len(trades): - page = self.fetch( - "/api/v3/historicalTrades", - {"symbol": "BTCUSDT", "fromId": next_id, "limit": 1000}, - 25, - ) - assert page and page[0]["id"] == next_id - for raw in page: - if trade_offset == len(trades): - break - tick = trades[trade_offset] - assert [tick["seq"], tick["ts"], tick["price"], tick["qty"]] == [ - raw["id"], - raw["time"], - Decimal(raw["price"]), - Decimal(raw["qty"]), - ], "raw REST mismatch" - assert [trade_tokens[trade_offset][key] for key in ("price", "qty")] == [ - raw["price"], - raw["qty"], - ], "raw decimal tokens changed" - trade_offset += 1 - next_id += 1 - expected = self.candles(self.cut, times[-1]) - normalized = [] - minute_trades = [] - minute = self.cut - for event in tape: - if event["type"] == "tick": - assert minute <= event["ts"] < minute + 60000, "time/print ordering violation" - minute_trades.append(event) - else: - assert event["type"] == "time" and event["ts"] == minute + 60000 - raw = expected[len(normalized)] - prices = [tick["price"] for tick in minute_trades] - assert prices and len(prices) == raw[8] - values = [ - prices[0], - max(prices), - min(prices), - prices[-1], - sum(tick["qty"] for tick in minute_trades), - ] - assert values == [Decimal(raw[index]) for index in range(1, 6)], ( - "time OHLCV mismatch" - ) - normalized.append( - dict( - zip( - ("ts_open", "o", "h", "l", "c", "v"), [minute, *values], strict=True + + def compare_prints(): + next_id = trades[0]["seq"] + trade_offset = 0 + while trade_offset < len(trades): + for identifier, matched, price, quantity in self.venue.prints(next_id): + if trade_offset == len(trades): + break + tick = trades[trade_offset] + assert [tick["seq"], tick["ts"], tick["price"], tick["qty"]] == [ + identifier, + matched, + Decimal(price), + Decimal(quantity), + ], "print REST mismatch" + assert [trade_tokens[trade_offset][key] for key in ("price", "qty")] == [ + price, + quantity, + ], "print decimal tokens changed" + trade_offset += 1 + next_id += 1 + + def compare_minutes(): + expected = self.venue.candles(self.cut, times[-1]) + differing.clear() + normalized = [] + minute_trades = [] + minute = self.cut + for event in tape: + if event["type"] == "tick": + assert minute <= event["ts"] < minute + 60000, "time/print ordering" + minute_trades.append(event) + else: + assert event["type"] == "time" and event["ts"] == minute + 60000 + candle = expected[len(normalized)] + prices = [tick["price"] for tick in minute_trades] + if not prices: + # A quiet minute closes on the next-print fence (OKX, USD-M) with a + # zero-volume candle; the batch then sees that flat candle. + assert candle[2] is None, "a proven minute without prints" + assert not self.venue.candle_is_print_sum or ( + Decimal(candle[1][4]) == 0 + ), "a quiet minute with candle volume" + values = [*(Decimal(token) for token in candle[1][:4]), Decimal(0)] + else: + assert candle[2] is None or len(prices) == candle[2], "count mismatch" + values = [ + prices[0], + max(prices), + min(prices), + prices[-1], + sum(tick["qty"] for tick in minute_trades), + ] + if self.venue.candle_is_print_sum: + assert values == [Decimal(token) for token in candle[1]], ( + "time OHLCV mismatch" ) - ) - ) - minute += 60000 - minute_trades = [] - self.receipt( - f"PASS ticks contiguous and time events equal REST ticks={len(trades)} " - f"minutes={len(times)} duration_seconds={duration:.3f}" - ) + elif values != [Decimal(token) for token in candle[1]]: + differing.append(minute) + keys = ("ts_open", "o", "h", "l", "c", "v") + normalized.append(dict(zip(keys, [minute, *values], strict=True))) + minute += 60000 + minute_trades = [] + return normalized + + differing = [] + self.settled(compare_prints) + normalized = self.settled(compare_minutes) + if self.venue.candle_is_print_sum: + self.receipt( + f"PASS {mode} contiguous and time events equal REST ticks={len(trades)} " + f"minutes={len(times)} duration_seconds={duration:.3f}" + ) + else: + # Every print equals REST aggTrades and every time is fenced by the next print; the + # kline is reported, not required, where a straddling aggregate moves a fill. + self.receipt( + f"PASS {mode} contiguous fenced aggregates equal REST ticks={len(trades)} " + f"minutes={len(times)} kline_differs={len(differing)} " + f"duration_seconds={duration:.3f}" + ) combined = self.directory / (mode + "-combined.csv") with open(combined, "w") as output: output.write((self.directory / "warmup.csv").read_text()) @@ -387,34 +537,36 @@ def validate(self, mode, duration, cursor, minutes): ) + "\n" ) - mapping = " (R-B2 timestamps)" if mode == "ticks" else "" + mapping = " (R-B2 timestamps)" if mode != "bars" else "" verdict = "PASS" if equal else "FAIL" self.receipt( f"{verdict} {mode} actions equal batch via run_backtest_full{mapping} " f"batch={len(expected_actions)} runner={len(actual_actions)} " f"after_last_proven_minute={trailing}" ) - if mode == "ticks": - name = "ticks-replay" + if mode != "bars" and not self.venue.candle_is_print_sum: + self.kline_batch(mode, normalized, actual_actions, len(differing)) + if mode != "bars": + name = mode + "-replay" completed = subprocess.run( self.runner_command( mode, name, - self.directory / "ticks-replay.sqlite3", - str(self.directory / "ticks-feed.jsonl"), + self.directory / (name + ".sqlite3"), + str(self.directory / (mode + "-feed.jsonl")), ), capture_output=True, text=True, timeout=600, ) - (self.directory / "ticks-replay.stdout").write_text(completed.stdout) - (self.directory / "ticks-replay.stderr").write_text(completed.stderr) + (self.directory / (name + ".stdout")).write_text(completed.stdout) + (self.directory / (name + ".stderr")).write_text(completed.stderr) assert completed.returncode == 0, "tick replay runner failed" raw = [action_key(record, "bars") for record in self.receiver.payloads.get(mode, [])] assert raw == [ action_key(record, "bars") for record in self.receiver.payloads.get(name, []) ] - self.receipt(f"PASS ticks same-print runner replay actions equal actions={len(raw)}") + self.receipt(f"PASS {mode} same-print runner replay actions equal actions={len(raw)}") return { "duration_seconds": duration, "messages": cursor, @@ -424,12 +576,43 @@ def validate(self, mode, duration, cursor, minutes): "batch_actions_equal": equal, } + def kline_batch(self, mode, normalized, actual_actions, kline_differs): + """Information, not a gate: the same strategy over Binance klines instead of the bars built + from the prints. Aggregates can straddle a minute edge, so fills there can differ.""" + candles = self.venue.candles(self.cut, normalized[-1]["ts_open"] + 60000) + combined = self.directory / (mode + "-kline-combined.csv") + with open(combined, "w") as output: + output.write((self.directory / "warmup.csv").read_text()) + for candle in candles: + output.write(",".join([str(candle[0]), *candle[1]]) + "\n") + actions = self.directory / (mode + "-kline-batch-actions.jsonl") + subprocess.run( + [self.options.batch_probe, self.options.observed_strategy, str(combined), str(actions)], + check=True, + ) + expected = [ + action_key(record, mode) + for record in rows(actions) + if record["origin_input_index"] >= 200 + ] + differ = sum( + 1 for left, right in zip(expected, actual_actions, strict=False) if left != right + ) + differ += abs(len(expected) - len(actual_actions)) + self.receipt( + f"INFO {mode} batch over Binance klines: actions={len(expected)} " + f"differ_from_runner={differ} kline_differs={kline_differs}" + ) + def live(self, mode, minutes): processes = None try: started = time.monotonic() processes = self.generation(mode, 0, 0) - self.receipt(f"START {mode} public BTCUSDT cut={self.cut} minimum_minutes={minutes}") + self.receipt( + f"START {mode} public {self.venue.venue} {self.venue.market} {self.venue.symbol} " + f"cut={self.cut} minimum_minutes={minutes}" + ) time.sleep(self.options.restart_seconds) assert processes[0].poll() is None and processes[1].poll() is None, ( "process stopped before restart" @@ -465,8 +648,14 @@ def live(self, mode, minutes): def run(self): self.warmup() threads = [ - threading.Thread(target=self.live, args=("bars", self.options.bar_minutes)), - threading.Thread(target=self.live, args=("ticks", self.options.tick_minutes)), + threading.Thread( + target=self.live, + args=( + mode, + self.options.bar_minutes if mode == "bars" else self.options.tick_minutes, + ), + ) + for mode in self.options.modes.split(",") ] for thread in threads: thread.start() @@ -485,6 +674,12 @@ def run(self): parser = argparse.ArgumentParser() for argument in ("feed", "runner", "strategy", "observed-strategy", "batch-probe", "directory"): parser.add_argument("--" + argument, required=True) + parser.add_argument("--venue", default="binance") + parser.add_argument("--market", default="spot") + parser.add_argument("--symbol", default="BTCUSDT") + parser.add_argument( + "--modes", default="bars,ticks", help="bars, ticks or agg-ticks, comma-separated" + ) parser.add_argument("--bar-minutes", type=int, default=46) parser.add_argument("--tick-minutes", type=int, default=21) parser.add_argument("--restart-seconds", type=int, default=180) diff --git a/native/feed/tests/unit.cpp b/native/feed/tests/unit.cpp index 5fe6967..9ee537f 100644 --- a/native/feed/tests/unit.cpp +++ b/native/feed/tests/unit.cpp @@ -1,5 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 #include "binance.hpp" +#include "bybit.hpp" +#include "okx.hpp" +#include "session.hpp" #include #include #include @@ -34,6 +37,7 @@ Config config(const Temporary& temporary, const std::string& mode) { value.mode = mode; value.start = 120000; value.state_dir = temporary.path; + value.allow_insecure = true; // loopback pacing: REST re-read pauses are 100 ms, not 1 s return value; } struct SyntheticVenue final : Venue { @@ -51,8 +55,9 @@ struct SyntheticVenue final : Venue { {180000, {{180000, "12.00000000", "12.00000000", "11.00000000", "11.00000000", "0.50000000"}, 103, 104, 2, true}}, {240000, {{240000, "12.00000000", "12.00000000", "12.00000000", "12.00000000", "0.10000000"}, 105, 105, 1, true}} }; - std::string streams() const override { return "test@trade/test@kline_1m"; } - VenueEvent decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } + Connection connection() const override { return {"/stream?streams=test@trade/test@kline_1m", {}, {}, &binance_frame}; } + std::vector decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } + std::string kline_source() const override { return "/synthetic"; } std::size_t pages = 0; std::vector history(std::uint64_t from, std::size_t limit) override { ++pages; @@ -81,6 +86,75 @@ struct SyntheticVenue final : Venue { VenueEvent tick(std::uint64_t id) const { return {VenueEvent::Kind::Trade, trades.at(id), {}}; } VenueEvent close(std::int64_t minute) const { return {VenueEvent::Kind::Kline, {}, bars.at(minute)}; } }; +// A next-print-fence venue (OKX trades-all, USD-M aggregates): candles carry no ID range and no count. +struct FenceVenue final : Venue { + std::map trades{ + {99, {99, 119999, "10.10000000", "0.10000000"}}, + {100, {100, 120001, "10.10000000", "0.10000000"}}, + {101, {101, 120050, "11.20000000", "0.20000000"}}, + {102, {102, 179999, "9.90000000", "0.30000000"}}, + {103, {103, 180001, "12.00000000", "0.40000000"}}, + {104, {104, 180050, "11.00000000", "0.10000000"}}, + {105, {105, 240001, "12.00000000", "0.10000000"}}, + {106, {106, 360001, "12.50000000", "0.10000000"}} + }; + std::map candles{ + {120000, {{120000, "10.10000000", "11.20000000", "9.90000000", "9.90000000", "0.60000000"}, -1, -1, 0, true}}, + {180000, {{180000, "12.00000000", "12.00000000", "11.00000000", "11.00000000", "0.50000000"}, -1, -1, 0, true}}, + {240000, {{240000, "12.00000000", "12.00000000", "12.00000000", "12.00000000", "0.10000000"}, -1, -1, 0, true}}, + {300000, {{300000, "12.00000000", "12.00000000", "12.00000000", "12.00000000", "0.00000000"}, -1, -1, 0, true}}, + {360000, {{360000, "12.50000000", "12.50000000", "12.50000000", "12.50000000", "0.10000000"}, -1, -1, 0, true}} + }; + std::int64_t retention = 0; + bool print_sum = true; + std::size_t pages = 0, candle_requests = 0; + // REST lag: the next `empty_reads` history reads return nothing, the next `short_reads` drop their last row. + std::size_t empty_reads = 0, short_reads = 0; + Connection connection() const override { return {"/synthetic", {}, {}, &binance_frame}; } + std::vector decode(const std::string&) const override { throw Error(23, "not used by deterministic unit venue"); } + std::vector history(std::uint64_t from, std::size_t limit) override { + ++pages; + std::vector result; + if (empty_reads) { --empty_reads; return result; } + for (auto found = trades.find(from); found != trades.end() && result.size() < limit && found->first == from + result.size(); ++found) + result.push_back(found->second); + if (short_reads && !result.empty()) { --short_reads; result.pop_back(); } + return result; + } + Trade predecessor(std::int64_t minute) override { + std::optional last; + for (const auto& [id, trade] : trades) if (trade.ts < minute) last = trade; + if (!last) throw Error(20, "synthetic venue has no predecessor"); + return *last; + } + std::optional predecessor_if_ready(std::int64_t minute) override { + for (const auto& [id, trade] : trades) if (trade.ts >= minute) return predecessor(minute); + return std::nullopt; + } + std::vector klines(std::int64_t start, std::int64_t end) override { + ++candle_requests; + std::vector result; + for (auto minute = start; minute < end; minute += 60000) { + if (!candles.count(minute)) throw Error(20, "synthetic missing minute"); + result.push_back(candles.at(minute)); + } + return result; + } + TickProof tick_proof() const override { return TickProof::NextPrintFence; } + std::int64_t retention_ms() const override { return retention; } + bool candle_is_print_sum() const override { return print_sum; } + std::string kline_source() const override { return "/synthetic"; } + VenueEvent tick(std::uint64_t id) const { return {VenueEvent::Kind::Trade, trades.at(id), {}}; } + VenueEvent close(std::int64_t minute) const { return {VenueEvent::Kind::Kline, {}, candles.at(minute)}; } +}; +std::vector kinds(const std::vector& lines) { + std::vector result; + for (const auto& line : lines) { + const auto event = parse_json(line); + result.push_back(event.at("type").text() == "tick" ? event.at("seq").value : "time:" + event.at("ts").value); + } + return result; +} } int main() { @@ -93,6 +167,10 @@ int main() { Trade exact{9007199254740993ULL, 120001, "123.45000000", "0.00000001"}; assert(normalized_trade(parse_json(exact.wire())) == exact); expect(23, [] { Trade{1, 1, "1", "0"}.validate(); }); + // Equality is by exact value: one venue number rendered two ways is not a revision; any change is. + assert((Bar{120000, "100.0", "100.1", "99.5", "99.5", "12.30"} == Bar{120000, "100", "100.1", "99.5", "99.5", "12.3"})); + assert(!(Bar{120000, "100.0", "100.1", "99.5", "99.5", "12.30"} == Bar{120000, "100.01", "100.1", "99.5", "99.5", "12.3"})); + assert((Trade{7, 120001, "10.10", "0.5"} == Trade{7, 120001, "10.1", "0.50"}) && !(Trade{7, 120001, "10.1", "0.5"} == Trade{7, 120001, "10.1", "0.51"})); passed("decimal_tokens_and_uint64_identity"); const auto raw = parse_json(R"({"e":"trade","E":999999,"s":"TESTUSDT","t":100,"p":"10.10000000","q":"0.10000000","T":120001,"m":false})"); @@ -379,6 +457,435 @@ int main() { expect(23, [&] { State state(config(second, "bars")); }); passed("crashed_initialization_without_cursor_is_reinitialized"); } + + { + assert(Decimal("2.75").multiply(Decimal("0.01")).str() == "0.0275"); + assert(Decimal("2").multiply(Decimal("0.01")).str() == "0.02"); + assert(Decimal("100").multiply(Decimal("0.01")).str() == "1"); + assert(Decimal("0.5").multiply(Decimal("0")).str() == "0"); + assert(Decimal("12.30").str() == "12.3" && Decimal("0.00012").str() == "0.00012"); + assert(scaled_quantity("2.750", "1") == "2.750"); + assert(scaled_quantity("2.75", "0.01") == "0.0275"); + // Contract conversion distributes exactly over the candle volume: no float, no rounding. + assert(Decimal(scaled_quantity("0.05", "0.01")).add(Decimal(scaled_quantity("7.20", "0.01"))) == + Decimal(scaled_quantity("7.25", "0.01"))); + expect(23, [] { Decimal("0." + std::string(20, '1')).multiply(Decimal("0." + std::string(20, '1'))); }); + passed("decimal_multiply_exact_contract_units"); + } + { + const std::string document = R"({"timezone":"UTC","note":"x]}\"","rateLimits":[{"limit":2400,"s":"[{"}],"symbols":[{"symbol":"A"}]})"; + assert(json_member(document, "rateLimits") == R"([{"limit":2400,"s":"[{"}])"); + assert(json_member(document, "note") == R"("x]}\"")"); + assert(json_member(document, "symbols") == R"([{"symbol":"A"}])"); + expect(23, [&] { json_member(document, "missing"); }); + expect(23, [] { json_member("[1]", "rateLimits"); }); + passed("json_member_extracts_one_member_beyond_the_parser_bound"); + } + { + auto row = parse_json(R"({"instId":"TEST-USDT-SWAP","tradeId":"100","px":"10.1","sz":"2.75","side":"buy","ts":"120001","source":"0"})"); + const auto swap = okx_trade(row, "TEST-USDT-SWAP", "0.01"); + assert(swap.id == 100 && swap.ts == 120001 && swap.price == "10.1" && swap.qty == "0.0275"); + assert(okx_trade(row, "TEST-USDT-SWAP", "1").qty == "2.75"); + expect(23, [&] { okx_trade(row, "OTHER-USDT-SWAP", "0.01"); }); + auto aggregated = row; + aggregated.members["count"] = Json::string("3"); + expect(23, [&] { okx_trade(aggregated, "TEST-USDT-SWAP", "0.01"); }); + auto padded = row; + padded.members["tradeId"] = Json::string("0100"); + expect(23, [&] { okx_trade(padded, "TEST-USDT-SWAP", "0.01"); }); + const auto candle = okx_candle(parse_json(R"(["120000","10.1","11.2","9.9","9.9","60","0.6","6.06","1"])"), "0.01"); + assert(candle.confirmed && candle.bar.volume == "0.6" && candle.bar.ts == 120000 && candle.count == 0 && candle.first == -1); + assert(!okx_candle(parse_json(R"(["120000","10.1","11.2","9.9","9.9","60","0.6","6.06","0"])"), "0.01").confirmed); + expect(23, [] { okx_candle(parse_json(R"(["120000","10.1","11.2","9.9","9.9","60","0.6","6.06","true"])"), "1"); }); + expect(23, [] { okx_candle(parse_json(R"(["120000","10.1","11.2","9.9","9.9","60","0.6","1"])"), "1"); }); + auto instrument = parse_json(R"({"instId":"TEST-USDT-SWAP","instType":"SWAP","state":"live","ctType":"linear","ctValCcy":"TEST","ctVal":"0.01","ctMult":"1"})"); + assert(okx_multiplier(instrument, "swap", "TEST-USDT-SWAP") == "0.01"); + auto inverse = instrument; + inverse.members["ctType"] = Json::string("inverse"); + expect(23, [&] { okx_multiplier(inverse, "swap", "TEST-USDT-SWAP"); }); + auto quoted = instrument; + quoted.members["ctValCcy"] = Json::string("USD"); + expect(23, [&] { okx_multiplier(quoted, "swap", "TEST-USDT-SWAP"); }); + auto suspended = instrument; + suspended.members["state"] = Json::string("suspend"); + expect(23, [&] { okx_multiplier(suspended, "swap", "TEST-USDT-SWAP"); }); + assert(okx_multiplier(parse_json(R"({"instId":"TEST-USDT","instType":"SPOT","state":"live"})"), "spot", "TEST-USDT") == "1"); + assert(okx_frame("pong") == Frame::Control); + assert(okx_frame(R"({"event":"subscribe","arg":{"channel":"candle1m","instId":"TEST-USDT"},"connId":"a"})") == Frame::Control); + assert(okx_frame(R"({"event":"notice","code":"64008","msg":"upgrade","connId":"a"})") == Frame::Retire); + assert(okx_frame(R"({"arg":{"channel":"candle1m","instId":"TEST-USDT"},"data":[]})") == Frame::Data); + expect(23, [] { okx_frame(R"({"event":"error","code":"60012","msg":"bad"})"); }); + expect(23, [] { okx_frame("ping-not-json"); }); + passed("okx_trades_all_candle_confirm_count_contract_units_and_frames"); + } + { + const auto closing = bybit_kline(parse_json(R"({"start":120000,"end":179999,"interval":"1","open":"10.1","close":"9.9","high":"11.2","low":"9.9","volume":"0.6","turnover":"6","confirm":true,"timestamp":180001})")); + assert(closing.confirmed && closing.bar.ts == 120000 && closing.bar.volume == "0.6"); + assert(!bybit_kline(parse_json(R"({"start":180000,"end":239999,"interval":"1","open":"9.9","close":"9.9","high":"9.9","low":"9.9","volume":"0.1","turnover":"1","confirm":false,"timestamp":180001})")).confirmed); + expect(23, [] { bybit_kline(parse_json(R"({"start":120000,"end":179999,"interval":"1","open":"1","close":"1","high":"1","low":"1","volume":"1","turnover":"1","confirm":"true","timestamp":1})")); }); + expect(23, [] { bybit_kline(parse_json(R"({"start":120000,"end":239999,"interval":"1","open":"1","close":"1","high":"1","low":"1","volume":"1","turnover":"1","confirm":true,"timestamp":1})")); }); + const auto newest_first = parse_json(R"([["180000","12","12","11","11","0.5","6"],["120000","10.1","11.2","9.9","9.9","0.6","6"]])"); + const auto ascending = bybit_rows(newest_first, 120000, 2); + assert(ascending.size() == 2 && ascending.front().bar.ts == 120000 && ascending.back().bar.ts == 180000); + expect(20, [] { bybit_rows(parse_json(R"([["120000","10.1","11.2","9.9","9.9","0.6","6"],["180000","12","12","11","11","0.5","6"]])"), 120000, 2); }); + expect(20, [&] { bybit_rows(newest_first, 120000, 3); }); + assert(bybit_frame(R"({"success":true,"ret_msg":"pong","conn_id":"a","op":"ping"})") == Frame::Control); + assert(bybit_frame(R"({"success":true,"ret_msg":"","conn_id":"a","op":"subscribe"})") == Frame::Control); + assert(bybit_frame(R"({"topic":"kline.1.TESTUSDT","data":[],"ts":1,"type":"snapshot"})") == Frame::Data); + expect(23, [] { bybit_frame(R"({"success":false,"ret_msg":"bad","op":"subscribe"})"); }); + passed("bybit_confirm_flag_two_entry_push_and_reverse_order_rest_rows"); + } + { + Config options; + options.symbol = "TESTUSDT"; + assert(usdm_connection(options).path == "/market/stream?streams=testusdt@kline_1m"); + options.mode = "agg-ticks"; + assert(usdm_connection(options).path == "/market/stream?streams=testusdt@aggTrade/testusdt@kline_1m"); + const auto aggregate = usdm_aggregate(parse_json(R"({"e":"aggTrade","E":1,"s":"TESTUSDT","a":500,"p":"10.1","q":"0.5","nq":"0.4","f":1000,"l":1004,"T":120001,"m":true,"st":1})")); + assert(aggregate.id == 500 && aggregate.ts == 120001 && aggregate.qty == "0.5"); + expect(23, [] { usdm_aggregate(parse_json(R"({"a":500,"p":"10.1","q":"0.5","f":1004,"l":1000,"T":120001})")); }); + assert(usdm_kline_weight(32) == 1 && usdm_kline_weight(100) == 2 && usdm_kline_weight(1000) == 5); + assert(binance_frame(R"({"stream":"!x","data":{"e":"serverShutdown"}})") == Frame::Retire); + passed("usdm_routed_stream_aggregate_q_and_kline_weights"); + } + { + Temporary temporary; + FenceVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.connected(); + for (const std::uint64_t id : {100, 101, 102}) session.ingest(venue.tick(id)); + // The fence alone does not close the minute: its confirmed candle must reconcile first. + session.ingest(venue.tick(103)); + assert(kinds(output) == std::vector({"100", "101", "102"})); + session.ingest(venue.close(120000)); + assert(kinds(output) == std::vector({"100", "101", "102", "time:180000", "103"})); + // The confirmed candle alone does not close the minute either: it waits for the next print. + session.ingest(venue.close(180000)); + assert(output.size() == 5); + session.ingest(venue.tick(104)); + assert(output.size() == 6); + session.ingest(venue.tick(105)); + assert(kinds(output) == std::vector({"100", "101", "102", "time:180000", "103", "104", "time:240000", "105"})); + session.ingest(venue.close(180000)); + auto revised = venue.close(180000); + revised.kline.bar.volume = "0.60000000"; + expect(21, [&] { session.ingest(revised); }); + passed("fence_time_needs_contiguous_ids_next_minute_print_and_exact_candle"); + } + { + Temporary temporary; + FenceVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + for (const std::uint64_t id : {100, 101, 102, 103}) session.ingest(venue.tick(id)); + auto revised = venue.close(120000); + revised.kline.bar.high = "11.30000000"; + expect(21, [&] { session.ingest(revised); }); + assert(kinds(output) == std::vector({"100", "101", "102"}) && state.durable().cut == 120000); + passed("fence_exact_ohlcv_mismatch_stops_21_without_time"); + } + { + Temporary temporary; + FenceVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.tick(103)); + assert(kinds(output) == std::vector({"100", "101", "102"}) && venue.pages == 1); + session.ingest(venue.close(120000)); + session.ingest(venue.close(180000)); + session.ingest(venue.close(240000)); + session.ingest(venue.close(300000)); + session.ingest(venue.tick(106)); + assert(kinds(output) == std::vector({"100", "101", "102", "time:180000", "103", "104", "time:240000", "105", + "time:300000", "time:360000", "106"})); + passed("fence_gap_heals_by_id_pages_and_quiet_minute_closes_on_zero_volume_candle"); + } + { + Temporary temporary; + FenceVenue venue; + venue.candles.at(300000).bar.volume = "0.10000000"; + State state(config(temporary, "ticks")); + FenceSession session(state, venue, [](const auto&) {}); + for (const std::int64_t minute : {120000, 180000, 240000}) session.ingest(venue.close(minute)); + expect(21, [&] { session.ingest(venue.close(300000)); }); + assert(state.durable().cut == 300000); + passed("fence_quiet_minute_with_candle_volume_stops_21"); + } + { + Temporary temporary; + FenceVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + for (const std::uint64_t id : {100, 101, 102, 103}) session.ingest(venue.tick(id)); + assert(output.size() == 3 && venue.candle_requests == 0); + session.ingest(venue.close(180000)); + assert(kinds(output) == std::vector({"100", "101", "102", "time:180000", "103"}) && venue.candle_requests == 1); + passed("fence_rest_candle_only_behind_a_later_confirmed_websocket_candle"); + } + { + Temporary temporary; + FenceVenue venue; + venue.retention = 60000; + State state(config(temporary, "ticks")); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.tick(100)); + const auto before = venue.pages; + expect(20, [&] { session.ingest(venue.tick(105)); }); + assert(venue.pages == before && kinds(output) == std::vector({"100"})); + venue.retention = 0; + Temporary second; + State healing(config(second, "ticks")); + FenceSession healed(healing, venue, [](const auto&) {}); + healed.ingest(venue.tick(100)); + healed.ingest(venue.tick(102)); + assert(healing.durable().seq == 102); + passed("fence_gap_beyond_the_venue_history_window_stops_20_without_rest"); + } + { + Temporary temporary; + FenceVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.tick(100)); + session.ingest(venue.tick(101)); + venue.trades.at(100).qty = "0.10000001"; + expect(21, [&] { session.connected(); }); + assert(output.size() == 2 && state.durable().message_index == 2); + venue.trades.at(100).qty = "0.10000000"; + venue.trades.at(102).ts = 120040; + expect(23, [&] { session.ingest(venue.tick(102)); }); + passed("fence_reconnect_overlap_change_21_and_time_regression_23"); + } + { + Temporary temporary; + FenceVenue venue; + venue.trades = {{99, {99, 119999, "10.10000000", "0.10000000"}}, {100, {100, 180001, "12.00000000", "0.50000000"}}}; + venue.candles.at(120000).bar = {120000, "10.10000000", "10.10000000", "10.10000000", "10.10000000", "0"}; + auto options = config(temporary, "ticks"); + { + State state(options); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.close(120000)); + assert(output.empty()); + session.ingest(venue.tick(100)); + assert(kinds(output) == std::vector({"time:180000", "100"})); + } + options.resume = true; + options.start = -1; + State resumed(options); + assert(resumed.cursor().message_index == 2 && resumed.cursor().seq == 100 && resumed.cursor().cut == 180000); + passed("fence_quiet_first_minute_closes_on_the_anchored_predecessor_and_resumes"); + } + { + // A resumed fence session rebuilds the open minute from the journal before it reconciles. + Temporary temporary; + FenceVenue venue; + auto options = config(temporary, "ticks"); + { + State state(options); + FenceSession session(state, venue, [](const auto&) {}); + session.ingest(venue.tick(100)); + session.ingest(venue.tick(101)); + } + options.resume = true; + options.start = -1; + options.output_from = 2; + State state(options); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.connected(); + session.ingest(venue.tick(102)); + session.ingest(venue.tick(103)); + session.ingest(venue.close(120000)); + assert(kinds(output) == std::vector({"102", "time:180000", "103"})); + passed("fence_resume_rebuilds_the_open_minute_from_the_journal"); + } + { + Temporary temporary; + auto options = config(temporary, "ticks"); + options.qty_multiplier = "0.01"; + { State state(options); } + options.resume = true; + options.start = -1; + options.qty_multiplier = "1"; + expect(21, [&] { State changed(options); }); + options.qty_multiplier = "0.01"; + State same(options); + assert(same.cursor().message_index == 0); + passed("resume_refuses_a_changed_contract_multiplier"); + } + { + Temporary temporary; + FenceVenue venue; + State state(config(temporary, "ticks")); + FenceSession session(state, venue, [](const auto&) {}); + session.ingest(venue.tick(100)); + const Trade earlier{98, 120002, "10.10000000", "0.10000000"}, changed{99, 120003, "10.10000000", "0.10000000"}; + expect(23, [&] { session.ingest({VenueEvent::Kind::Trade, earlier, {}}); }); + expect(21, [&] { session.ingest({VenueEvent::Kind::Trade, changed, {}}); }); + assert(state.durable().seq == 100); + passed("fence_print_at_or_before_the_start_predecessor_stops"); + } + { + // Aggregate prints: an aggregate dated in one minute can hold a fill the next minute's candle + // counts, so the candle is not their sum and the fence alone closes the minute. + Temporary temporary; + FenceVenue venue; + venue.print_sum = false; + venue.candles.at(120000).bar.volume = "0.60300000"; + venue.candles.at(180000).bar = {180000, "12.00000000", "12.00000000", "10.90000000", "11.00000000", "0.49700000"}; + State state(config(temporary, "agg-ticks")); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + for (const std::uint64_t id : {100, 101, 102, 103, 104}) session.ingest(venue.tick(id)); + session.ingest(venue.close(120000)); + session.ingest(venue.close(180000)); + session.ingest(venue.tick(105)); + assert(kinds(output) == std::vector({"100", "101", "102", "time:180000", "103", "104", "time:240000", "105"})); + venue.print_sum = true; + Temporary second; + State strict(config(second, "ticks")); + FenceSession raw(strict, venue, [](const auto&) {}); + for (const std::uint64_t id : {100, 101, 102, 103}) raw.ingest(venue.tick(id)); + expect(21, [&] { raw.ingest(venue.close(120000)); }); + passed("aggregate_prints_close_on_the_fence_without_candle_equality"); + } + { + // A forming candle never closes a fenced minute and never licenses a REST candle. + Temporary temporary; + FenceVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + for (const std::uint64_t id : {100, 101, 102, 103}) session.ingest(venue.tick(id)); + for (const std::int64_t minute : {120000, 180000, 240000}) { + auto forming = venue.close(minute); + forming.kline.confirmed = false; + session.ingest(forming); + } + assert(output.size() == 3 && venue.candle_requests == 0); + session.ingest(venue.close(120000)); + assert(kinds(output).back() == "103" && kinds(output)[3] == "time:180000"); + passed("fence_forming_candle_never_closes_a_minute"); + } + { + // A duplicate print with changed content stops 21, already emitted or still buffered. + Temporary temporary; + FenceVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.tick(100)); + session.ingest(venue.tick(101)); + auto emitted = venue.tick(100); + emitted.trade.qty = "0.20000000"; + expect(21, [&] { session.ingest(emitted); }); + Temporary second; + State buffering(config(second, "ticks")); + FenceSession fence(buffering, venue, [](const auto&) {}); + for (const std::uint64_t id : {100, 101, 102, 103}) fence.ingest(venue.tick(id)); + auto buffered = venue.tick(103); + buffered.trade.price = "12.10000000"; + expect(21, [&] { fence.ingest(buffered); }); + assert(buffering.durable().seq == 102 && output.size() == 2); + passed("fence_conflicting_duplicate_print_stops_21_emitted_or_buffered"); + } + { + // Reconnect: a changed closed candle, a changed start predecessor, or a candle revised before its + // minute closed each stop 21. + Temporary temporary; + FenceVenue venue; + State state(config(temporary, "ticks")); + FenceSession session(state, venue, [](const auto&) {}); + for (const std::uint64_t id : {100, 101, 102, 103}) session.ingest(venue.tick(id)); + session.ingest(venue.close(120000)); + venue.candles.at(120000).bar.volume = "0.70000000"; + expect(21, [&] { session.connected(); }); + FenceVenue quiet; + quiet.trades = {{99, {99, 119999, "10.10000000", "0.10000000"}}, {100, {100, 180001, "12.00000000", "0.50000000"}}}; + Temporary second; + State anchored(config(second, "ticks")); + FenceSession waiting(anchored, quiet, [](const auto&) {}); + waiting.ingest(quiet.tick(100)); + assert(anchored.durable().predecessor && anchored.durable().message_index == 0); + quiet.trades.at(99).qty = "0.20000000"; + expect(21, [&] { waiting.connected(); }); + Temporary third; + State open(config(third, "ticks")); + FenceSession revising(open, venue, [](const auto&) {}); + for (const std::uint64_t id : {100, 101, 102}) revising.ingest(venue.tick(id)); + revising.ingest(venue.close(120000)); + auto revised = venue.close(120000); + revised.kline.bar.close = "11.10000000"; + expect(21, [&] { revising.ingest(revised); }); + assert(open.durable().cut == 120000); + passed("fence_reconnect_candle_and_predecessor_overlap_and_open_candle_revision_stop_21"); + } + { + // The unconfirmed-print buffer is bounded by --max-queue-bytes on a fence venue too. + Temporary temporary; + FenceVenue venue; + auto options = config(temporary, "ticks"); + options.max_queue_bytes = 450; + State state(options); + FenceSession session(state, venue, [](const auto&) {}); + for (const std::uint64_t id : {100, 101, 102, 103, 104}) session.ingest(venue.tick(id)); + expect(22, [&] { session.ingest(venue.tick(105)); }); + assert(state.durable().seq == 102); + passed("fence_unconfirmed_print_buffer_is_bounded"); + } + { + // REST trailing the newest prints is re-read: the reconnect overlap and a heal page. + Temporary temporary; + FenceVenue venue; + State state(config(temporary, "ticks")); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.tick(100)); + session.ingest(venue.tick(101)); + venue.short_reads = 1; + const auto before = venue.pages; + session.connected(); + assert(venue.pages == before + 2); + venue.empty_reads = 1; + session.ingest(venue.tick(103)); + assert(kinds(output) == std::vector({"100", "101", "102"})); + passed("fence_reconnect_overlap_and_heal_page_reread_trailing_rest"); + } + { + // A quiet start waits: no print after --start means nothing anchored, nothing emitted, no stop. + Temporary temporary; + FenceVenue venue; + venue.trades = {{99, {99, 119999, "10.10000000", "0.10000000"}}}; + for (const std::int64_t minute : {120000, 180000}) + venue.candles.at(minute).bar = {minute, "10.10000000", "10.10000000", "10.10000000", "10.10000000", "0"}; + venue.candles.at(240000).bar = {240000, "12.00000000", "12.00000000", "12.00000000", "12.00000000", "0.10000000"}; + State state(config(temporary, "ticks")); + std::vector output; + FenceSession session(state, venue, [&](const auto& line) { output.push_back(line); }); + session.ingest(venue.close(120000)); + session.ingest(venue.close(180000)); + assert(output.empty() && !state.durable().predecessor); + venue.trades[100] = {100, 240001, "12.00000000", "0.10000000"}; + session.ingest(venue.tick(100)); + assert(kinds(output) == std::vector({"time:180000", "time:240000", "100"})); + passed("fence_quiet_start_waits_for_the_first_print"); + } + { + Temporary temporary; + SyntheticVenue venue; + State state(config(temporary, "agg-ticks")); + expect(23, [&] { make_session(state, venue, [](const auto&) {}); }); + passed("aggregate_mode_needs_a_next_print_fence_venue"); + } assert(retry_after_seconds("120", 0) == 120); assert(retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT", 1445412475) == 5); assert(retry_after_seconds("Wed, 21 Oct 2015 07:28:00 GMT", 1445412490) == 0);