diff --git a/pineforge_codegen/__init__.py b/pineforge_codegen/__init__.py index 6bd2489..7684e2b 100644 --- a/pineforge_codegen/__init__.py +++ b/pineforge_codegen/__init__.py @@ -15,6 +15,7 @@ from .limits import TimeBudget, check_ast_depth, check_source_size, ensure_recursion_headroom from .pragmas import extract_pf_trace_pragmas from .request_discovery import discover_requests, request_sites +from .request_feed_inventory import build_request_feed_inventory from .security_contexts import specialize_security_contexts from .block_locals import rename_block_locals from .support_checker import check_support as _support_diagnostics @@ -201,3 +202,58 @@ def transpile_full(pine_source: str, *, check_support: bool = True, if d.level in (Level.WARNING, Level.NOTE)], "requests": discover_requests(gen, ctx, sites), } + + +def transpile_with_request_inventory(pine_source: str, *, check_support: bool = True, + filename: str = "", + libraries: Mapping[str, str] | None = None, + primary_chart_timeframe: str | None = None) -> dict: + """Transpile like :func:`transpile_full`, plus the request-feed inventory + bound to the C++ it returns (``pineforge_codegen.request_feed_inventory``). + + Runs the pipeline once (``_generate``) and returns six keys, the five + :func:`transpile_full` returns computed from that one generation by the + same expressions, and the inventory: + + - ``cpp``: the generated C++ source (identical to :func:`transpile`). + - ``inputs``: the input manifest, as :func:`transpile_full` lists it. + - ``strategyParams``: the literal ``strategy(...)`` kwargs, as + :func:`transpile_full` lists them. + - ``diagnostics``: the warnings and notes, as :func:`transpile_full` lists + them. + - ``requests``: the other-symbol feed projection of + :func:`transpile_full` (``request_discovery.discover_requests``). It + leaves out the same-ticker requests, so it is not the strategy's + complete request list. + - ``request_feed_inventory``: the request-feed inventory bound to ``cpp`` + (``build_request_feed_inventory``). It lists every bar request the + compiled strategy can make, same ticker included, from the same + registration context as ``requests``. Its ``artifact_sha256`` is null + until the trusted compile pipeline binds it after linking with + :func:`pineforge_codegen.request_feed_inventory.bind_request_feed_inventory`. + + :func:`transpile` and :func:`transpile_full` are unchanged. + + Args mirror :func:`transpile_full`, plus: + primary_chart_timeframe: The chart timeframe the compiled artifact is + bound to, as a wire token (a request that reads the chart's own + timeframe is then that token; without it, ``unknown``). A value + that is no wire token raises ``RequestFeedInventoryError``. + + Returns: + ``{"cpp": str, "inputs": list[dict], "strategyParams": dict, + "diagnostics": list[Diagnostic], "requests": list[dict], + "request_feed_inventory": dict}``. + """ + gen, ctx, cpp, support_diagnostics, sites = _generate( + pine_source, check_support, filename, libraries) + return { + "cpp": cpp, + "inputs": gen.extract_input_manifest(), + "strategyParams": dict(ctx.strategy_params), + "diagnostics": [d for d in (*support_diagnostics, *ctx.diagnostics) + if d.level in (Level.WARNING, Level.NOTE)], + "requests": discover_requests(gen, ctx, sites), + "request_feed_inventory": build_request_feed_inventory( + gen, ctx, sites, cpp, primary_chart_timeframe=primary_chart_timeframe), + } diff --git a/pineforge_codegen/request_feed_inventory.py b/pineforge_codegen/request_feed_inventory.py new file mode 100644 index 0000000..d3e183a --- /dev/null +++ b/pineforge_codegen/request_feed_inventory.py @@ -0,0 +1,311 @@ +"""The request-feed inventory a compiled strategy carries (interface v1). + +A selected-window run decides, before it opens a feed or loads the strategy +library, whether every timeframe the strategy requests lies inside the build's +policy. ``build_request_feed_inventory`` is the producer half: for the C++ the +pinned transpiler generated it lists every bar request the compiled strategy +can make, and ``bind_request_feed_inventory`` binds that list to the linked +artifact. One JSON object, exactly these keys:: + + {"schema": "pineforge-request-feed-inventory/v1", + "source_sha256": "<64 lowercase hex of the generated C++'s UTF-8 bytes>", + "artifact_sha256": null, # filled after linking, by the bind helper + "primary_chart_timeframe": null, # the compile caller's chart binding + "entries": [{"kind": "token", "timeframe": "D"}, + {"kind": "input", "key": "HTF", "default": "240"}, + {"kind": "unknown"}]} + +``entries`` holds one member per request site, in source order (line, column, +then registration order). An empty list is complete; an ``unknown`` entry makes +the inventory incomplete. No expression is executed to resolve an inventory: a +timeframe is a token only where it is written, or where the codegen already +folds it at compile time to a string constant, and an input reference names the +manifest key (``codegen.input._get_input_title``) and the declaration's string +default; anything else, a computed expression and a default folded from an +input-dependent computation included, is ``unknown``. + +What is listed, so that none is dropped silently: + +* every registered request context that is not flagged dead + (``CodeGen._security_eval_info``): the chart's own symbol, another symbol's + feed and ``request.security_lower_tf`` alike, one per context a helper's call + paths give it (``security_contexts``) and per call-site clone. A helper no + top-level statement reaches whose request registers all the same (it is not + flagged dead) is listed too, where ``request_discovery`` leaves it out: a + superset only makes the inventory refuse more; +* every deferred refusal ``request_discovery.request_sites`` kept (another + symbol's request registration cannot key before the first bar), by the + timeframe as written; +* every other reachable ``request.security`` / ``request.security_lower_tf`` + call in the analyzed program that neither of those accounts for, as + ``unknown`` (a request the pipeline did not classify is incomplete, never + omitted). + +Not listed: a request lowered to ``na`` (its value reaches display sinks only: +the C++ makes no request), a request of a helper nothing reaches that +registration flags dead (it registers the chart's own timeframe and is never +read) and the recorded fundamentals requests (``request.financial`` / +``earnings`` / ``dividends`` / ``splits``: series recorded per chart bar, no +timeframe feed). + +A request that depends on the chart's own timeframe (an empty string, +``timeframe.period``) is a token only when the compile caller passes +``primary_chart_timeframe``: the inventory records that binding and the +admission requires it to equal the request's primary token. Without it the +entry is ``unknown``. + +Wire tokens (:func:`canonical_wire_timeframe`): a positive whole-minute decimal +without leading zeros; ``D`` / ``1D`` is ``D`` and ``W`` / ``1W`` is ``W``; +bare ``M`` / ``S`` are ``1M`` / ``1S``; any other positive ``nD`` / ``nW`` / +``nM`` / ``nS`` stays as written. Whitespace, an empty string, a leading zero, +a decimal and every other spelling (``1H`` included: the interface gives it no +wire spelling) are no token and the entry is ``unknown``. +""" + +from __future__ import annotations + +import hashlib +import re + +from .external_requests import LOWERING_ANNOTATION +from .request_discovery import ( + _UNPINNED_LOWERING, _guarded, _registered_timeframe, _request_args, _timeframe, _within, +) +from .security_contexts import ScriptIndex, _reached, _request_payload + +SCHEMA = "pineforge-request-feed-inventory/v1" + +_KEYS = frozenset({"schema", "source_sha256", "artifact_sha256", + "primary_chart_timeframe", "entries"}) +_KEY_ORDER = ("schema", "source_sha256", "artifact_sha256", + "primary_chart_timeframe", "entries") +# ``[0-9]`` and no flags: a fullwidth digit, a trailing newline or a lowercase +# unit is no token. +_WIRE_TOKEN = re.compile(r"(?:([1-9][0-9]*)([DWMS])?|([DWMS]))") +_HEX64 = re.compile(r"[0-9a-f]{64}") +_BYTES_LIKE = (bytes, bytearray, memoryview) + + +class RequestFeedInventoryError(ValueError): + """An inventory input the producer refuses: a primary chart timeframe that + is no wire token, source text that is not text, or an inventory the bind + helper cannot validate against the bytes it is given.""" + + +def canonical_wire_timeframe(text) -> str | None: + """``text`` as the inventory's wire token, or None when it is none.""" + if type(text) is not str: + return None + matched = _WIRE_TOKEN.fullmatch(text) + if matched is None: + return None + digits, unit, bare = matched.groups() + if bare is not None: + return bare if bare in ("D", "W") else "1" + bare + if unit is None: + return digits + if digits == "1" and unit in ("D", "W"): + return unit + return digits + unit + + +def build_request_feed_inventory(gen, ctx, sites, cpp, *, primary_chart_timeframe=None) -> dict: + """The unbound inventory (``artifact_sha256`` null) of the strategy ``gen`` + generated ``cpp`` for (module docstring). ``sites`` are the requests the + support checker kept (``_generate``'s fifth result); ``ctx`` and ``gen`` + are the analyzer's and the codegen's, read and left as they were.""" + if type(cpp) is not str: + raise RequestFeedInventoryError("cpp is not the generated C++ source text") + chart = None + if primary_chart_timeframe is not None: + chart = canonical_wire_timeframe(primary_chart_timeframe) + if chart is None: + raise RequestFeedInventoryError( + f"primary_chart_timeframe {primary_chart_timeframe!r} is not a wire timeframe token") + return { + "schema": SCHEMA, + "source_sha256": hashlib.sha256(cpp.encode("utf-8")).hexdigest(), + "artifact_sha256": None, + "primary_chart_timeframe": chart, + "entries": _inventory_entries(gen, ctx, list(sites or ()), chart), + } + + +def bind_request_feed_inventory(inventory, cpp_bytes, artifact_bytes) -> dict: + """A copy of the unbound ``inventory`` with ``artifact_sha256`` filled from + the linked artifact's bytes, once its shape is exact and its + ``source_sha256`` is that of ``cpp_bytes``. Reads no file and loads no + library; the trusted compile pipeline calls it right after linking.""" + _check_shape(inventory) + if not isinstance(cpp_bytes, _BYTES_LIKE) or not isinstance(artifact_bytes, _BYTES_LIKE): + raise RequestFeedInventoryError("cpp_bytes and artifact_bytes must be bytes") + if inventory["artifact_sha256"] is not None: + raise RequestFeedInventoryError( + "inventory artifact_sha256 is already set; only an unbound inventory can be bound") + if hashlib.sha256(cpp_bytes).hexdigest() != inventory["source_sha256"]: + raise RequestFeedInventoryError( + "inventory source_sha256 does not match the supplied C++ bytes") + return { + "schema": inventory["schema"], + "source_sha256": inventory["source_sha256"], + "artifact_sha256": hashlib.sha256(artifact_bytes).hexdigest(), + "primary_chart_timeframe": inventory["primary_chart_timeframe"], + "entries": [_copy_entry(entry) for entry in inventory["entries"]], + } + + +# --------------------------------------------------------------------------- +# Binding: exact shape +# --------------------------------------------------------------------------- + +def _is_hex64(value) -> bool: + return type(value) is str and _HEX64.fullmatch(value) is not None + + +def _is_wire_token(value) -> bool: + return type(value) is str and canonical_wire_timeframe(value) == value + + +def _check_shape(inventory) -> None: + if not isinstance(inventory, dict) or set(inventory) != _KEYS: + raise RequestFeedInventoryError( + "inventory is not an object with exactly the keys " + ", ".join(_KEY_ORDER)) + if type(inventory["schema"]) is not str or inventory["schema"] != SCHEMA: + raise RequestFeedInventoryError(f"inventory schema is not {SCHEMA}") + if not _is_hex64(inventory["source_sha256"]): + raise RequestFeedInventoryError("inventory source_sha256 is not 64 lowercase hex digits") + artifact = inventory["artifact_sha256"] + if artifact is not None and not _is_hex64(artifact): + raise RequestFeedInventoryError( + "inventory artifact_sha256 is neither null nor 64 lowercase hex digits") + primary = inventory["primary_chart_timeframe"] + if primary is not None and not _is_wire_token(primary): + raise RequestFeedInventoryError( + "inventory primary_chart_timeframe is neither null nor a canonical wire token") + entries = inventory["entries"] + if not isinstance(entries, list): + raise RequestFeedInventoryError("inventory entries is not a list") + for position, entry in enumerate(entries): + if not _entry_has_shape(entry): + raise RequestFeedInventoryError( + f"inventory entries[{position}] is not exactly one of token, input or unknown") + + +def _entry_has_shape(entry) -> bool: + if not isinstance(entry, dict): + return False + kind = entry.get("kind") + if kind == "token": + return set(entry) == {"kind", "timeframe"} and _is_wire_token(entry["timeframe"]) + if kind == "input": + return (set(entry) == {"kind", "key", "default"} + and type(entry["key"]) is str and type(entry["default"]) is str) + if kind == "unknown": + return set(entry) == {"kind"} + return False + + +def _copy_entry(entry: dict) -> dict: + if entry["kind"] == "token": + return {"kind": "token", "timeframe": entry["timeframe"]} + if entry["kind"] == "input": + return {"kind": "input", "key": entry["key"], "default": entry["default"]} + return {"kind": "unknown"} + + +# --------------------------------------------------------------------------- +# Entries +# --------------------------------------------------------------------------- + +def _located(loc): + """``(line, column)`` of a located node, else None.""" + if loc is None or loc.line is None or loc.col is None: + return None + return loc.line, loc.col + + +def _entry(classified, chart) -> dict: + """One entry from ``request_discovery``'s classification of a timeframe: + only a literal, a chart dependency under the caller's binding and an input + whose default is a string are anything but ``unknown``.""" + kind = classified.get("kind") if isinstance(classified, dict) else None + if kind == "literal": + token = canonical_wire_timeframe(classified.get("value")) + if token is not None: + return {"kind": "token", "timeframe": token} + elif kind == "chart": + if chart is not None: + return {"kind": "token", "timeframe": chart} + elif kind == "input": + key, default = classified.get("title"), classified.get("default") + if type(key) is str and type(default) is str: + return {"kind": "input", "key": key, "default": default} + return {"kind": "unknown"} + + +def _inventory_entries(gen, ctx, sites: list, chart) -> list[dict]: + """Every request site's entry, in source order (module docstring).""" + found: list[tuple[tuple, dict]] = [] + + def add(loc, entry: dict) -> None: + line, col = _located(loc) or (0, 0) + found.append(((line, col, len(found)), entry)) + + calls = {call.sec_id: call for call in ctx.security_calls} + dead = {item["sec_id"] for item in gen._security_calls if item.get("dead")} + live = [info for info in gen._security_eval_info if info["sec_id"] not in dead] + deferred = [site for site in sites + if (site.annotations or {}).get(LOWERING_ANNOTATION) == _UNPINNED_LOWERING] + budget = getattr(gen, "_budget", None) + # Expanding a name registers a reassigned global's read with the first-bar + # replay; generation is over, so leave its record as it was (as + # ``discover_requests`` does). + mutable_reads = set(gen._security_tf_mutable_reads) + try: + for info in live: + # Past the time budget a timeframe is classified as written, + # unexpanded: an expression stays computed, so unknown. + classified = _guarded(_registered_timeframe, gen, info, expand=_within(budget)) + add(getattr(calls.get(info["sec_id"]), "loc", None), _entry(classified, chart)) + for site in deferred: + # The C++ never registered it, so nothing expands its names here. + _symbol, timeframe = _request_args(site) + classified = _guarded(_timeframe, gen, timeframe, expand=False) + add(site.loc, _entry(classified, chart)) + try: + for node in _unaccounted_requests(ctx, calls, live, deferred): + add(node.loc, {"kind": "unknown"}) + except Exception: # noqa: BLE001 -- a sweep that fails is an unclassified site + add(None, {"kind": "unknown"}) + finally: + gen._security_tf_mutable_reads = mutable_reads + return [entry for _, entry in sorted(found, key=lambda item: item[0])] + + +def _unaccounted_requests(ctx, calls: dict, live: list, deferred: list) -> list: + """The reachable ``request.security`` / ``request.security_lower_tf`` calls + of the analyzed program that no live context and no deferred site accounts + for, by the payload node the analyzer recorded for them, else by their + source position and kind. A call in a helper nothing reaches (a method's + is reached) is no site the strategy can make.""" + index = ScriptIndex(ctx.ast) + reached = _reached(index) + live_calls = [calls[info["sec_id"]] for info in live if info["sec_id"] in calls] + payloads = {id(call.expression) for call in live_calls} + places = {(_located(call.loc), bool(call.is_lower_tf_array)) for call in live_calls} + deferred_ids = {id(site) for site in deferred} + deferred_places = {_located(site.loc) for site in deferred} + found = [] + for owner, requests in index.requests.items(): + if owner in index.funcs and owner not in reached: + continue + for node in requests: + payload = _request_payload(node) + place = _located(node.loc) + lower = node.callee.member == "security_lower_tf" + if id(node) in deferred_ids or (payload is not None and id(payload) in payloads): + continue + if place is not None and ((place, lower) in places or place in deferred_places): + continue + found.append(node) + return found diff --git a/pineforge_codegen/timeframe_token_cli.py b/pineforge_codegen/timeframe_token_cli.py new file mode 100644 index 0000000..9de1d8c --- /dev/null +++ b/pineforge_codegen/timeframe_token_cli.py @@ -0,0 +1,133 @@ +"""Batch timeframe-token transport for the native admission's canonicalizer. + +``python -m pineforge_codegen.timeframe_token_cli`` reads one request on stdin +and writes one result on stdout:: + + request: {"version": 1, "values": [, ...]} + result: {"version": 1, "tokens": [, ...]} + +The result keeps the request's order and cardinality and ends with one LF. +Each element goes once, in order, to +``request_feed_inventory.canonical_wire_timeframe``, which decides every token: +a value that is no string, or no wire token, gives ``null``, and an empty batch +gives an empty list. + +Exit status: 0 for a result (``null`` elements included). 2 for a malformed +request: not UTF-8, not strict JSON, a repeated key, a key other than +``version`` and ``values``, a ``version`` that is not the integer 1, or a +``values`` that is not a list. 1 for any other failure. Both failures write +nothing to stdout and one generic line to stderr, which never echoes the +request. + +The transport reads no source, strategy, feed or artifact, runs no code from +its input, converts no number and caps no input size. +""" + +import json +import sys + +from .request_feed_inventory import canonical_wire_timeframe + +PROTOCOL_VERSION = 1 +EXIT_OK = 0 +EXIT_INTERNAL = 1 +EXIT_MALFORMED = 2 +_REQUEST_KEYS = frozenset({"version", "values"}) +_MALFORMED_MESSAGE = "malformed timeframe token request" +_INTERNAL_MESSAGE = "internal failure in the timeframe token transport" + + +class _Malformed(Exception): + """The request is malformed (exit 2).""" + + +class _JsonNumber: + """A JSON number, kept as its literal: nothing converts it, so no digit + limit applies either. It is no string, hence no token; the version must be + the integer literal ``1``.""" + + __slots__ = ("literal", "is_integer") + + def __init__(self, literal: str, is_integer: bool) -> None: + self.literal = literal + self.is_integer = is_integer + + +def _object(pairs: list) -> dict: + """A JSON object's members; a repeated key is refused (strict JSON).""" + members = {} + for key, value in pairs: + if key in members: + raise ValueError("duplicate key") + members[key] = value + return members + + +def _refuse_constant(name: str): + """NaN, Infinity and -Infinity are not JSON.""" + raise ValueError("not strict JSON") + + +def _decode(raw: bytes): + """The request's JSON value; bytes that are no UTF-8 or no strict JSON are + malformed.""" + try: + return json.loads( + raw.decode("utf-8"), + object_pairs_hook=_object, + parse_int=lambda literal: _JsonNumber(literal, True), + parse_float=lambda literal: _JsonNumber(literal, False), + parse_constant=_refuse_constant, + ) + except (ValueError, RecursionError): + # UnicodeDecodeError is a ValueError. A document nested past the + # parser's recursion limit is JSON the parser cannot read. + raise _Malformed from None + + +def _values(request) -> list: + """The batch's values, once the envelope is exact.""" + if type(request) is not dict or set(request) != _REQUEST_KEYS: + raise _Malformed + version = request["version"] + if not (isinstance(version, _JsonNumber) and version.is_integer + and version.literal == "1"): + raise _Malformed + if type(request["values"]) is not list: + raise _Malformed + return request["values"] + + +def _render(raw: bytes) -> bytes: + """The whole result, as the bytes to write at once.""" + values = _values(_decode(raw)) + # One direct call per value, in order: nothing is deduplicated or reused. + tokens = [canonical_wire_timeframe(value) for value in values] + text = json.dumps({"version": PROTOCOL_VERSION, "tokens": tokens}, + separators=(",", ":")) + return (text + "\n").encode("ascii") + + +def _refuse(code: int, message: str) -> int: + sys.stderr.write(message + "\n") + return code + + +def main() -> int: + """Run one request and return the exit status (module docstring).""" + try: + output = _render(sys.stdin.buffer.read()) + except _Malformed: + return _refuse(EXIT_MALFORMED, _MALFORMED_MESSAGE) + except Exception: # noqa: BLE001 -- every other failure is internal + return _refuse(EXIT_INTERNAL, _INTERNAL_MESSAGE) + try: + sys.stdout.buffer.write(output) + sys.stdout.buffer.flush() + except OSError: + return _refuse(EXIT_INTERNAL, _INTERNAL_MESSAGE) + return EXIT_OK + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_request_feed_inventory.py b/tests/test_request_feed_inventory.py new file mode 100644 index 0000000..d35ee83 --- /dev/null +++ b/tests/test_request_feed_inventory.py @@ -0,0 +1,402 @@ +"""The request-feed inventory (``pineforge_codegen.request_feed_inventory``, +interface v1): every bar request a compiled strategy can make, as a wire token, +an input reference or ``unknown``, bound to the C++ it was generated from. + +The inventory is read before a selected-window run opens a feed or loads the +strategy library, so each test states what an admission may rely on: an empty +list is complete, ``unknown`` is incomplete, a token is exactly a timeframe the +script writes (or the caller's chart binding), an input reference names the +manifest key and its string default, and binding refuses bytes the inventory +was not made from. ``transpile`` and ``transpile_full`` stay as they were. +""" + +from __future__ import annotations + +import hashlib + +import pytest + +import pineforge_codegen +from pineforge_codegen import ( + _generate, transpile, transpile_full, transpile_with_request_inventory, +) +from pineforge_codegen.errors import Level +from pineforge_codegen.request_feed_inventory import ( + SCHEMA, RequestFeedInventoryError, bind_request_feed_inventory, + build_request_feed_inventory, canonical_wire_timeframe, +) + +HEAD = '//@version=6\nstrategy("inventory", overlay = true)\n' +TRADE = 'if c > 0\n strategy.entry("L", strategy.long)\n' +UNKNOWN = {"kind": "unknown"} + + +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _inventory(body: str, **options) -> dict: + return transpile_with_request_inventory(HEAD + body, **options)["request_feed_inventory"] + + +def _entries(body: str, **options) -> list[dict]: + return _inventory(body, **options)["entries"] + + +def _token(timeframe: str) -> dict: + return {"kind": "token", "timeframe": timeframe} + + +# --------------------------------------------------------------------------- +# What the inventory holds +# --------------------------------------------------------------------------- + +def test_no_requests_is_a_complete_empty_inventory(): + result = transpile_with_request_inventory(HEAD + "c = close\n" + TRADE) + assert set(result) == {"cpp", "inputs", "strategyParams", "diagnostics", "requests", + "request_feed_inventory"} + assert result["request_feed_inventory"] == { + "schema": SCHEMA, "source_sha256": _sha(result["cpp"].encode("utf-8")), + "artifact_sha256": None, "primary_chart_timeframe": None, "entries": [], + } + assert SCHEMA == "pineforge-request-feed-inventory/v1" + + +@pytest.mark.parametrize("tf, token", [("60", "60"), ("D", "D")]) +def test_same_ticker_literal_timeframe_is_a_token(tf, token): + """The chart's own symbol reads no feed of another symbol, but its request + registers a timeframe all the same: it is listed (``discover_requests`` + leaves it out).""" + assert _entries(f'c = request.security(syminfo.tickerid, "{tf}", close)\n' + TRADE) == [ + _token(token)] + + +@pytest.mark.parametrize("tf, token", [ + ("240", "240"), ("D", "D"), ("1D", "D"), ("W", "W"), ("1W", "W"), ("M", "1M"), + ("30S", "30S"), +]) +def test_other_symbol_literal_timeframe_is_a_token(tf, token): + assert _entries(f'c = request.security("BINANCE:ETHUSDT", "{tf}", close)\n' + TRADE) == [ + _token(token)] + + +def test_entries_follow_the_source_order(): + entries = _entries( + 'a = request.security("BINANCE:ETHUSDT", "240", close)\n' + 'b = request.security(syminfo.tickerid, "D", close)\n' + 'd = request.security("BYBIT:SOLUSDT.P", "1W", close)\n' + 'c = a + b + d\n' + TRADE) + assert entries == [_token("240"), _token("D"), _token("W")] + + +@pytest.mark.parametrize("title_args, key", [ + ('"D", "HTF"', "HTF"), + ('"240", title = "Higher timeframe"', "Higher timeframe"), + ('"D"', "htf"), +]) +def test_input_reference_names_the_manifest_key_and_the_string_default(title_args, key): + """The key is the one an override is keyed by (the manifest's ``title``: the + title argument, else the declaration's name) and the default is the + declaration's own string, as written.""" + source = HEAD + (f'htf = input.timeframe({title_args})\n' + 'c = request.security("BINANCE:ETHUSDT", htf, close)\n' + TRADE) + default = title_args.split(",")[0].strip().strip('"') + result = transpile_with_request_inventory(source) + assert result["request_feed_inventory"]["entries"] == [ + {"kind": "input", "key": key, "default": default}] + manifest = {entry["title"]: entry for entry in transpile_full(source)["inputs"]} + assert manifest[key]["default"] == default + + +def test_input_string_reference_and_its_override_key(): + source = HEAD + ('mode = input.string("60", "Request timeframe")\n' + 'c = request.security(syminfo.tickerid, mode, close)\n' + TRADE) + inventory = transpile_with_request_inventory(source)["request_feed_inventory"] + assert inventory["entries"] == [ + {"kind": "input", "key": "Request timeframe", "default": "60"}] + assert "Request timeframe" in {e["title"] for e in transpile_full(source)["inputs"]} + + +def test_helper_contexts_are_one_entry_per_call_path(): + """A helper's request registers once per symbol and timeframe its call + paths pass (``security_contexts``): each context is an entry.""" + entries = _entries( + 'o = input.symbol("BINANCE:ETHUSDT", "Other")\n' + 'g(s, tf) => request.security(s, tf, close)\n' + 'h(s) => g(s, "60") + g(s, "D")\n' + 'c = h(o) + g("BINANCE:SOLUSDT", "240")\n' + TRADE) + # One line: the order within it is the order the C++ registers them in. + assert sorted(entry["timeframe"] for entry in entries) == ["240", "60", "D"] + assert all(entry["kind"] == "token" for entry in entries) + + +@pytest.mark.parametrize("body", [ + 'useD = input.bool(true, "Use D")\ntf = useD ? "D" : timeframe.period\n' + 'c = request.security("BINANCE:ETHUSDT", tf, close)\n', + 'mode = input.string("A", "Mode")\ntf = switch mode\n "A" => "60"\n "B" => "D"\n' + 'c = request.security("BINANCE:ETHUSDT", tf, close)\n', +]) +def test_computed_timeframe_is_unknown_never_a_guessed_token(body): + """``discover_requests`` folds these at the inputs' defaults ("1D", "60"); + an inventory states no default it computed, whoever binds the chart.""" + assert _entries(body + TRADE) == [UNKNOWN] + assert _entries(body + TRADE, primary_chart_timeframe="15") == [UNKNOWN] + + +def test_deferred_refusal_is_listed_by_its_timeframe_as_written(): + """Registration cannot key the reassigned symbol, so the run stops where + the value is read; the site is still a request the script makes.""" + source = ('var string s = "BINANCE:ETHUSDT"\n' + 's := close > open ? "BINANCE:ETHUSDT" : "BINANCE:BTCUSDT"\n' + 'c = request.security(s, "60", close)\n' + TRADE) + assert _entries(source) == [_token("60")] + deferred_tf = ('var string tf = "60"\nif bar_index == 0\n tf := "240"\n' + 'c = request.security("BINANCE:ETHUSDT", tf, close)\n' + TRADE) + assert _entries(deferred_tf) == [UNKNOWN] + + +def test_lower_timeframe_request_is_listed(): + entries = _entries( + 'c = array.size(request.security_lower_tf("BINANCE:ETHUSDT", "1", close))\n' + TRADE) + assert entries == [_token("1")] + + +def test_a_request_lowered_to_na_is_not_a_request(): + """Its value reaches plots only: the C++ makes no request, so a complete + empty list is true of it.""" + source = ('c = request.security("BINANCE:ETHUSDT", "60", close)\nplot(c)\n' + 'if close > open\n strategy.entry("L", strategy.long)\n') + assert _entries(source) == [] + + +# --------------------------------------------------------------------------- +# The chart binding +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("body", [ + 'c = request.security("BINANCE:ETHUSDT", timeframe.period, close)\n', + 'c = request.security("BINANCE:ETHUSDT", timeframe.main_period, close)\n', + 'tf = ""\nc = request.security("BINANCE:ETHUSDT", tf, close)\n', +]) +def test_chart_dependency_is_a_token_only_under_the_callers_binding(body): + unbound = _inventory(body + TRADE) + assert (unbound["primary_chart_timeframe"], unbound["entries"]) == (None, [UNKNOWN]) + bound = _inventory(body + TRADE, primary_chart_timeframe="15") + assert (bound["primary_chart_timeframe"], bound["entries"]) == ("15", [_token("15")]) + # The binding is a wire token like any other: spelled 1D, it is D. + daily = _inventory(body + TRADE, primary_chart_timeframe="1D") + assert (daily["primary_chart_timeframe"], daily["entries"]) == ("D", [_token("D")]) + + +def test_a_binding_changes_no_literal_entry(): + body = 'c = request.security("BINANCE:ETHUSDT", "D", close)\n' + TRADE + assert _entries(body, primary_chart_timeframe="15") == [_token("D")] + + +@pytest.mark.parametrize("binding", ["", " 15", "15 ", "05", "1H", "abc", 15, b"15"]) +def test_a_binding_that_is_no_wire_token_is_refused(binding): + with pytest.raises(RequestFeedInventoryError, match="is not a wire timeframe token"): + transpile_with_request_inventory(HEAD + "c = close\n" + TRADE, + primary_chart_timeframe=binding) + + +@pytest.mark.parametrize("text, token", [ + ("1", "1"), ("5", "5"), ("60", "60"), ("240", "240"), ("1440", "1440"), + ("D", "D"), ("1D", "D"), ("2D", "2D"), ("W", "W"), ("1W", "W"), ("3W", "3W"), + ("M", "1M"), ("1M", "1M"), ("12M", "12M"), ("S", "1S"), ("1S", "1S"), ("30S", "30S"), +]) +def test_wire_tokens(text, token): + assert canonical_wire_timeframe(text) == token + + +@pytest.mark.parametrize("text", [ + "", " 5", "5 ", "05", "0", "00", "0D", "-5", "1.5", "5.0", "1H", "4H", "H", "d", "1d", + "2w", "60m", "１５", "5\n", "D1", "1DD", None, 15, b"5", +]) +def test_not_wire_tokens(text): + assert canonical_wire_timeframe(text) is None + + +# --------------------------------------------------------------------------- +# The generated source and the generation state are left alone +# --------------------------------------------------------------------------- + +def test_the_cpp_is_exactly_the_one_transpile_returns(): + source = HEAD + ('other = input.symbol("BINANCE:ETHUSDT", "Other symbol")\n' + 'c = request.security(other, "D", close)\n' + TRADE) + plain = transpile(source) + result = transpile_with_request_inventory(source) + assert result["cpp"] == plain == transpile_full(source)["cpp"] + assert result["request_feed_inventory"]["source_sha256"] == _sha(plain.encode("utf-8")) + assert set(transpile_full(source)) == { + "cpp", "inputs", "strategyParams", "diagnostics", "requests"} + assert transpile_with_request_inventory(source) == result + + +def test_building_leaves_the_generation_state_as_it_was(): + source = HEAD + ('htf = input.timeframe("D", "HTF")\n' + 'c = request.security("BINANCE:ETHUSDT", htf, close)\n' + TRADE) + gen, ctx, cpp, _support, sites = _generate(source, True, "") + before = (set(gen._security_tf_mutable_reads), len(ctx.diagnostics), len(sites), + [info["sec_id"] for info in gen._security_eval_info]) + build_request_feed_inventory(gen, ctx, sites, cpp) + assert before == (set(gen._security_tf_mutable_reads), len(ctx.diagnostics), len(sites), + [info["sec_id"] for info in gen._security_eval_info]) + + +# --------------------------------------------------------------------------- +# Binding +# --------------------------------------------------------------------------- + +CPP = b"// generated source\n" + + +def _valid() -> dict: + return {"schema": SCHEMA, "source_sha256": _sha(CPP), "artifact_sha256": None, + "primary_chart_timeframe": None, + "entries": [_token("D"), {"kind": "input", "key": "HTF", "default": "240"}, + dict(UNKNOWN)]} + + +def test_binding_fills_the_artifact_digest_into_a_copy(): + artifact = b"\x7fELF linked artifact" + original = _valid() + bound = bind_request_feed_inventory(original, CPP, artifact) + assert bound == {**_valid(), "artifact_sha256": _sha(artifact)} + assert list(bound) == ["schema", "source_sha256", "artifact_sha256", + "primary_chart_timeframe", "entries"] + assert original == _valid() + bound["entries"][0]["timeframe"] = "W" + assert original["entries"][0] == _token("D") + + +def test_binding_a_transpiled_inventory_to_its_exact_utf8_bytes(): + result = transpile_with_request_inventory( + HEAD + 'c = request.security("BINANCE:ETHUSDT", "D", close)\n' + TRADE) + inventory, cpp = result["request_feed_inventory"], result["cpp"] + artifact = b"linked" + bound = bind_request_feed_inventory(inventory, cpp.encode("utf-8"), artifact) + assert bound["artifact_sha256"] == _sha(artifact) + assert bound["entries"] == [_token("D")] + assert inventory["artifact_sha256"] is None + + +def test_binding_refuses_source_bytes_the_inventory_was_not_made_from(): + result = transpile_with_request_inventory( + HEAD + 'c = request.security("BINANCE:ETHUSDT", "D", close)\n' + TRADE) + inventory, cpp = result["request_feed_inventory"], result["cpp"] + other = transpile(HEAD + 'c = request.security("BINANCE:ETHUSDT", "W", close)\n' + TRADE) + for wrong in (cpp.encode("utf-8") + b"\n", cpp.replace("\n", "\r\n").encode("utf-8"), + other.encode("utf-8"), b""): + with pytest.raises(RequestFeedInventoryError, + match="source_sha256 does not match the supplied C\\+\\+ bytes"): + bind_request_feed_inventory(inventory, wrong, b"linked") + + +def test_binding_refuses_an_inventory_that_is_already_bound(): + inventory = _valid() + inventory["artifact_sha256"] = "0" * 64 + with pytest.raises(RequestFeedInventoryError, match="artifact_sha256 is already set"): + bind_request_feed_inventory(inventory, CPP, b"linked") + + +@pytest.mark.parametrize("cpp_bytes, artifact_bytes", [ + (CPP.decode(), b"linked"), (CPP, "linked"), (None, b"linked"), (CPP, None), +]) +def test_binding_takes_bytes_only(cpp_bytes, artifact_bytes): + with pytest.raises(RequestFeedInventoryError, match="must be bytes"): + bind_request_feed_inventory(_valid(), cpp_bytes, artifact_bytes) + + +@pytest.mark.parametrize("mutate", [ + lambda inv: inv.pop("entries"), + lambda inv: inv.update(extra=1), + lambda inv: inv.update(schema="pineforge-request-feed-inventory/v2"), + lambda inv: inv.update(source_sha256="A" * 64), + lambda inv: inv.update(source_sha256="abc"), + lambda inv: inv.update(artifact_sha256="abc"), + lambda inv: inv.update(primary_chart_timeframe="1D"), + lambda inv: inv.update(primary_chart_timeframe=15), + lambda inv: inv.update(entries={}), + lambda inv: inv["entries"].append({"kind": "token"}), + lambda inv: inv["entries"].append({"kind": "token", "timeframe": "05"}), + lambda inv: inv["entries"].append({"kind": "token", "timeframe": "1D"}), + lambda inv: inv["entries"].append({"kind": "token", "timeframe": ""}), + lambda inv: inv["entries"].append({"kind": "input", "key": "K", "default": 5}), + lambda inv: inv["entries"].append({"kind": "input", "key": "K"}), + lambda inv: inv["entries"].append({"kind": "unknown", "timeframe": "D"}), + lambda inv: inv["entries"].append({"kind": "other"}), + lambda inv: inv["entries"].append("D"), +]) +def test_binding_refuses_an_inexact_shape(mutate): + inventory = _valid() + mutate(inventory) + with pytest.raises(RequestFeedInventoryError): + bind_request_feed_inventory(inventory, CPP, b"linked") + + +@pytest.mark.parametrize("inventory", [None, [], "inventory", 7]) +def test_binding_refuses_what_is_no_object(inventory): + with pytest.raises(RequestFeedInventoryError, match="is not an object with exactly the keys"): + bind_request_feed_inventory(inventory, CPP, b"linked") + + +# --------------------------------------------------------------------------- +# transpile_with_request_inventory against transpile_full +# --------------------------------------------------------------------------- + +# Input metadata of three kinds (string, int, symbol), strategy() parameters, a +# same-ticker daily request, a feed read by a trade, and the two diagnostics +# this script raises: ta.ema's warmup note and bar_index's window warning. +METADATA = ( + '//@version=6\n' + 'strategy("metadata", overlay = true, initial_capital = 10000, pyramiding = 2)\n' + 'mode = input.string("fast", "Mode", options = ["fast", "slow"])\n' + 'length = input.int(14, "Length", options = [14, 21])\n' + 'other = input.symbol("BINANCE:ETHUSDT", "Other symbol")\n' + 'e = ta.ema(close, length)\n' + 'daily = request.security(syminfo.tickerid, "D", close)\n' + 'feed = request.security(other, "240", close)\n' + 'if mode == "fast" and e > daily and feed > 0 and bar_index > 0\n' + ' strategy.entry("L", strategy.long)\n' +) + + +def test_new_api_returns_the_transpile_full_fields_from_one_generation(): + legacy = transpile_full(METADATA) + result = transpile_with_request_inventory(METADATA) + assert set(result) == {"cpp", "inputs", "strategyParams", "diagnostics", "requests", + "request_feed_inventory"} + assert result["cpp"] == transpile(METADATA) == legacy["cpp"] + for key in ("inputs", "strategyParams", "diagnostics", "requests"): + assert result[key] == legacy[key], key + assert [entry["title"] for entry in result["inputs"]] == ["Mode", "Length", "Other symbol"] + assert {"initial_capital", "pyramiding"} <= set(result["strategyParams"]) + assert {Level.WARNING, Level.NOTE} <= {d.level for d in result["diagnostics"]} + assert result["requests"] == [{ + "line": 8, "fn": "request.security", + "symbol": {"kind": "input", "title": "Other symbol", "default": "BINANCE:ETHUSDT"}, + "timeframe": {"kind": "literal", "value": "240"}, + "lookahead": False, "gaps": False, "ignore_invalid_symbol": False, + }] + assert result["request_feed_inventory"]["entries"] == [_token("D"), _token("240")] + + +def test_new_api_runs_generation_exactly_once(monkeypatch): + generate = pineforge_codegen._generate + calls = [] + + def counted(*args, **kwargs): + calls.append(args) + return generate(*args, **kwargs) + + monkeypatch.setattr(pineforge_codegen, "_generate", counted) + transpile_with_request_inventory(METADATA) + assert len(calls) == 1 + + +def test_same_ticker_daily_request_is_inventoried_though_requests_leaves_it_out(): + source = HEAD + 'c = request.security(syminfo.tickerid, "D", close)\n' + TRADE + result = transpile_with_request_inventory(source) + assert transpile_full(source)["requests"] == result["requests"] == [] + assert result["request_feed_inventory"]["entries"] == [_token("D")] diff --git a/tests/test_timeframe_token_cli.py b/tests/test_timeframe_token_cli.py new file mode 100644 index 0000000..5485a77 --- /dev/null +++ b/tests/test_timeframe_token_cli.py @@ -0,0 +1,205 @@ +"""The batch transport ``python -m pineforge_codegen.timeframe_token_cli``. + +The in-process cases call ``main()`` with its streams replaced and count the calls +to the shared ``canonical_wire_timeframe``; the subprocess cases check the exit +status and the bytes on stdout. No case reads a source, a strategy or a feed. +""" + +import io +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from pineforge_codegen import request_feed_inventory, timeframe_token_cli + +REPO_ROOT = Path(__file__).resolve().parents[1] +MALFORMED_TEXT = "malformed timeframe token request\n" +INTERNAL_TEXT = "internal failure in the timeframe token transport\n" + + +def _result(tokens: list) -> bytes: + text = json.dumps({"version": 1, "tokens": tokens}, separators=(",", ":")) + return (text + "\n").encode("ascii") + + +def _request(values: list) -> bytes: + return json.dumps({"version": 1, "values": values}).encode("utf-8") + + +def _main(monkeypatch, raw: bytes): + """``main()`` on ``raw`` with stdin, stdout and stderr replaced: returns + (exit status, stdout bytes, stderr text).""" + stdout = SimpleNamespace(buffer=io.BytesIO()) + stderr = io.StringIO() + monkeypatch.setattr(sys, "stdin", SimpleNamespace(buffer=io.BytesIO(raw))) + monkeypatch.setattr(sys, "stdout", stdout) + monkeypatch.setattr(sys, "stderr", stderr) + code = timeframe_token_cli.main() + return code, stdout.buffer.getvalue(), stderr.getvalue() + + +def _spy(monkeypatch) -> list: + """Route the module's ``canonical_wire_timeframe`` through a recorder of + its arguments that calls the shared function; returns the record.""" + calls: list = [] + shared = timeframe_token_cli.canonical_wire_timeframe + + def recorder(text): + calls.append(text) + return shared(text) + + monkeypatch.setattr(timeframe_token_cli, "canonical_wire_timeframe", recorder) + return calls + + +def _subprocess(raw: bytes): + return subprocess.run( + [sys.executable, "-m", "pineforge_codegen.timeframe_token_cli"], + input=raw, capture_output=True, cwd=REPO_ROOT, check=False) + + +def test_the_batch_uses_the_shared_function_itself(): + assert (timeframe_token_cli.canonical_wire_timeframe + is request_feed_inventory.canonical_wire_timeframe) + + +def test_a_result_is_one_compact_line(monkeypatch): + code, out, err = _main(monkeypatch, b'{"version":1,"values":["240","1D",null]}') + assert (code, err) == (0, "") + assert out == b'{"version":1,"tokens":["240","D",null]}\n' + + +@pytest.mark.parametrize("value, token", [ + ("1", "1"), ("240", "240"), ("D", "D"), ("1D", "D"), ("2D", "2D"), + ("W", "W"), ("1W", "W"), ("2W", "2W"), ("M", "1M"), ("1M", "1M"), + ("12M", "12M"), ("S", "1S"), ("1S", "1S"), ("30S", "30S"), +]) +def test_aliases_and_spellings_give_the_canonical_token(monkeypatch, value, token): + code, out, err = _main(monkeypatch, _request([value])) + assert (code, err) == (0, "") + assert out == _result([token]) + + +@pytest.mark.parametrize("value", [ + "", " 5", "5 ", "05", "0", "00", "0D", "-5", "1.5", "5.0", "1H", "4H", "H", + "d", "2w", "60m", "D1", "1DD", "１５", "5\n", + 15, 1.5, True, False, None, [], {}, ["D"], {"timeframe": "D"}, +]) +def test_a_string_that_is_no_token_and_any_nonstring_give_null(monkeypatch, value): + code, out, err = _main(monkeypatch, _request([value])) + assert (code, err) == (0, "") + assert out == _result([None]) + + +def test_a_long_known_token_comes_back_unchanged(monkeypatch): + token = "1" + "0" * 5000 + "D" + code, out, err = _main(monkeypatch, _request([token])) + assert (code, err) == (0, "") + assert out == _result([token]) + + +def test_an_empty_batch_gives_an_empty_list(monkeypatch): + code, out, err = _main(monkeypatch, b'{"version":1,"values":[]}') + assert (code, err) == (0, "") + assert out == b'{"version":1,"tokens":[]}\n' + + +def test_order_and_cardinality_are_kept_with_no_deduplication(monkeypatch): + values = ["240", "D", "bogus", "240", None, 5, "W", "D"] + code, out, _ = _main(monkeypatch, _request(values)) + assert code == 0 + assert out == _result(["240", "D", None, "240", None, None, "W", "D"]) + + +def test_each_value_gets_one_direct_call_in_order(monkeypatch): + calls = _spy(monkeypatch) + values = ["D", "240", "bogus", "D", None, ""] + code, out, _ = _main(monkeypatch, _request(values)) + assert code == 0 + assert calls == values + assert out == _result(["D", "240", None, "D", None, None]) + + +def test_a_number_and_a_bool_are_one_call_each(monkeypatch): + calls = _spy(monkeypatch) + code, out, _ = _main(monkeypatch, b'{"version":1,"values":[5,true,"D"]}') + assert code == 0 + assert len(calls) == 3 + assert out == _result([None, None, "D"]) + + +@pytest.mark.parametrize("raw", [ + b'{"version":1,"version":1,"values":[]}', + b'{"version":1,"values":[],"values":[]}', + b'{"version":1,"values":[{"a":1,"a":2}]}', + b'{"version":1,"values":[],"extra":0}', + b'{"version":1}', + b'{"values":[]}', + b'{"version":1,"values":"D"}', + b'{"version":1,"values":{}}', + b'{"version":1,"values":null}', + b"[]", + b'"text"', + b"7", + b"null", +]) +def test_a_repeated_extra_or_missing_key_or_a_non_object_exits_2(monkeypatch, raw): + assert _main(monkeypatch, raw) == (2, b"", MALFORMED_TEXT) + + +@pytest.mark.parametrize("version", [ + b"true", b"false", b"null", b"1.0", b"1e0", b"1E0", b'"1"', b"[1]", b"{}", + b"0", b"2", b"-1", +]) +def test_a_version_that_is_not_the_integer_one_exits_2(monkeypatch, version): + raw = b'{"version":' + version + b',"values":[]}' + assert _main(monkeypatch, raw) == (2, b"", MALFORMED_TEXT) + + +@pytest.mark.parametrize("raw", [ + b'{"version":1,"values":["\xff"]}', + b'{"version":1,"values":["\xed\xa0\x80"]}', + b"\xef\xbb\xbf" + b'{"version":1,"values":[]}', + b'{"version":1,"values":[', + b"", + b'{"version":1,"values":[]} x', + b'{"version":1,"values":[NaN]}', + b'{"version":1,"values":[-Infinity]}', + b'{"version":1,"values":[1,]}', + b'{"version":1,"values":["a\x01b"]}', +]) +def test_bytes_that_are_no_utf8_or_no_strict_json_exit_2(monkeypatch, raw): + assert _main(monkeypatch, raw) == (2, b"", MALFORMED_TEXT) + + +def test_a_document_nested_past_the_parser_limit_exits_2(monkeypatch): + assert _main(monkeypatch, b"[" * 100000) == (2, b"", MALFORMED_TEXT) + + +def test_an_unexpected_failure_exits_1_with_no_stdout(monkeypatch): + def broken(text): + raise RuntimeError("broken on purpose") + + monkeypatch.setattr(timeframe_token_cli, "canonical_wire_timeframe", broken) + assert _main(monkeypatch, _request(["D"])) == (1, b"", INTERNAL_TEXT) + + +def test_subprocess_exits_0_with_the_result_line(): + proc = _subprocess(b'{"version":1,"values":["240","1D",null]}') + assert (proc.returncode, proc.stdout, proc.stderr) == ( + 0, b'{"version":1,"tokens":["240","D",null]}\n', b"") + + +@pytest.mark.parametrize("raw", [ + b'{"version":1,"version":1,"values":[]}', + b"\xff", + b'{"version":true,"values":[]}', +]) +def test_subprocess_refuses_a_malformed_request_with_exit_2_and_no_stdout(raw): + proc = _subprocess(raw) + assert (proc.returncode, proc.stdout, proc.stderr) == ( + 2, b"", MALFORMED_TEXT.encode("ascii"))