diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a1ae9ada..48ef68c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - python-version: ['3.10', '3.11'] + python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] runs-on: ${{ matrix.os }} name: ${{ matrix.os }} Unit test ${{ matrix.python-version }} steps: @@ -32,3 +32,41 @@ jobs: run: poetry install - name: Unit tests run: make test + test-free-threaded: + strategy: + fail-fast: false + matrix: + python-version: ['3.13t', '3.14t'] + runs-on: ubuntu-latest + name: ubuntu-latest Unit test ${{ matrix.python-version }} (PYTHON_GIL=0) + steps: + - uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Setup Poetry + # pipx is Poetry's recommended installation method, and it keeps Poetry + # on the runner's regular Python, independent of the interpreter under test + run: pipx install poetry + - name: Install pypi deps + # the dev group (linters, docs, orjson for the examples) is not needed + # to run the tests, and orjson does not support free-threaded Python + run: | + poetry env use python${{ matrix.python-version }} + poetry install --only main,test + - name: Unit tests + env: + PYTHON_GIL: '0' + run: | + poetry run python - <<'EOF' + import sys, sysconfig + state = ( + f"version={sys.version!r} " + f"Py_GIL_DISABLED={sysconfig.get_config_var('Py_GIL_DISABLED')!r} " + f"_is_gil_enabled={sys._is_gil_enabled()!r}" + ) + print(state) + assert not sys._is_gil_enabled(), f"the GIL is enabled: {state}" + EOF + make test diff --git a/poetry.lock b/poetry.lock index 64a7a03e..1c5e5a7b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -85,7 +85,7 @@ version = "22.1.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.5" -groups = ["dev"] +groups = ["test"] files = [ {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, @@ -214,7 +214,7 @@ description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" groups = ["dev"] -markers = "sys_platform == \"win32\" or platform_system == \"Windows\"" +markers = "platform_system == \"Windows\" or sys_platform == \"win32\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -238,7 +238,7 @@ version = "2.1.3" description = "URL manipulation made simple." optional = false python-versions = "*" -groups = ["dev"] +groups = ["test"] files = [ {file = "furl-2.1.3-py2.py3-none-any.whl", hash = "sha256:9ab425062c4217f9802508e45feb4a83e54324273ac4b202f1850363309666c0"}, {file = "furl-2.1.3.tar.gz", hash = "sha256:5a6188fe2666c484a12159c18be97a1977a71d632ef5bb867ef15f54af39cc4e"}, @@ -299,7 +299,7 @@ version = "4.17.1" description = "An implementation of JSON Schema validation for Python" optional = false python-versions = ">=3.7" -groups = ["dev"] +groups = ["test"] files = [ {file = "jsonschema-4.17.1-py3-none-any.whl", hash = "sha256:410ef23dcdbca4eaedc08b850079179883c2ed09378bd1f760d4af4aacfa28d7"}, {file = "jsonschema-4.17.1.tar.gz", hash = "sha256:05b2d22c83640cde0b7e0aa329ca7754fbd98ea66ad8ae24aa61328dfe057fa3"}, @@ -609,7 +609,7 @@ version = "1.0.1" description = "Ordered Multivalue Dictionary" optional = false python-versions = "*" -groups = ["dev"] +groups = ["test"] files = [ {file = "orderedmultidict-1.0.1-py2.py3-none-any.whl", hash = "sha256:43c839a17ee3cdd62234c47deca1a8508a3f2ca1d0678a3bf791c87cf84adbf3"}, {file = "orderedmultidict-1.0.1.tar.gz", hash = "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad"}, @@ -738,7 +738,7 @@ version = "2.1.6" description = "HTTP traffic mocking and expectations made easy" optional = false python-versions = ">=3.10" -groups = ["dev"] +groups = ["test"] files = [ {file = "pook-2.1.6-py3-none-any.whl", hash = "sha256:e14e710118a9193cb0e08bce2908309b9e0aa1b213b4a151c235c1f549f6eb55"}, {file = "pook-2.1.6.tar.gz", hash = "sha256:7cfcc98ffd1452d7174d0025fef7949549d8f2604a6f11de4d738647fa0ae9ec"}, @@ -770,7 +770,7 @@ version = "0.19.2" description = "Persistent/Functional/Immutable data structures" optional = false python-versions = ">=3.7" -groups = ["dev"] +groups = ["test"] files = [ {file = "pyrsistent-0.19.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d6982b5a0237e1b7d876b60265564648a69b14017f3b5f908c5be2de3f9abb7a"}, {file = "pyrsistent-0.19.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:187d5730b0507d9285a96fca9716310d572e5464cadd19f22b63a6976254d77a"}, @@ -879,7 +879,7 @@ version = "1.16.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -groups = ["dev"] +groups = ["test"] files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -1137,19 +1137,6 @@ files = [ {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, ] -[[package]] -name = "typing-extensions" -version = "4.12.2" -description = "Backported and Experimental Type Hints for Python 3.8+" -optional = false -python-versions = ">=3.8" -groups = ["dev"] -markers = "python_version < \"3.15\"" -files = [ - {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, - {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, -] - [[package]] name = "typing-extensions" version = "4.16.0" @@ -1157,7 +1144,6 @@ description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version >= \"3.15\"" files = [ {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, @@ -1266,7 +1252,7 @@ version = "0.13.0" description = "Makes working with XML feel like you are working with JSON" optional = false python-versions = ">=3.4" -groups = ["dev"] +groups = ["test"] files = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, @@ -1275,4 +1261,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "610b6cf2d6f3cc2100ae3173b2af859dc7e399e8263d67c6586db113e766ca3c" +content-hash = "50ca3ffb193a876b67e29ff447d8bbab038cd5487e0b6bafb2b0cbe7b14aa2e0" diff --git a/pyproject.toml b/pyproject.toml index bb58f3da..4483e735 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,10 @@ urllib3 = ">=1.26.9" websockets = ">=14.0" certifi = ">=2022.5.18,<2027.0.0" -[tool.poetry.dev-dependencies] +[tool.poetry.group.test.dependencies] +pook = "^2.1.6" + +[tool.poetry.group.dev.dependencies] black = "^26.3.1" mypy = "^2.4" types-urllib3 = "^1.26.25" @@ -39,7 +42,6 @@ sphinx-rtd-theme = "^3.1.0" sphinx-autodoc-typehints = "^2.3.0" types-certifi = "^2021.10.8" types-setuptools = "^84.0.0" -pook = "^2.1.6" orjson = "^3.11.6" [build-system] diff --git a/test_rest/base.py b/test_rest/base.py index 40854146..bf3d1555 100644 --- a/test_rest/base.py +++ b/test_rest/base.py @@ -3,6 +3,7 @@ import os import re import unittest +from typing import Set import pook # type: ignore # mocks are stored in file tree @@ -32,6 +33,12 @@ class BaseTest(unittest.TestCase): + # Paths (as listed in `mocks`) whose mock answers any number of requests, + # for tests that repeat a request. Every other mock answers once: the + # pagination tests rely on that, because a first-page mock also matches + # the request for the next page. + persistent_mocks: Set[str] = set() + @classmethod def setUpClass(cls): cls.maxDiff = None @@ -40,4 +47,15 @@ def setUpClass(cls): for m in mocks: url = cls.c.BASE + m[0] # print("register", url) - pook.get(url, reply=200, response_body=m[1]) + pook.get( + url, + reply=200, + response_body=m[1], + persist=m[0] in cls.persistent_mocks, + ) + + @classmethod + def tearDownClass(cls): + # stop intercepting urllib3 and drop the mocks, so they don't leak into + # later test classes + pook.off() diff --git a/test_rest/test_concurrency.py b/test_rest/test_concurrency.py new file mode 100644 index 00000000..c7def92b --- /dev/null +++ b/test_rest/test_concurrency.py @@ -0,0 +1,145 @@ +import json +import threading +import unittest +from concurrent.futures import ThreadPoolExecutor +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import parse_qsl, urlparse + +from massive import RESTClient +from massive.rest.models.request import ( + RequestOptionBuilder, + X_MASSIVE_EDGE_ID, + X_MASSIVE_EDGE_IP_ADDRESS, +) + + +class EchoServer(ThreadingHTTPServer): + """Answers every GET with what it received. + + It counts the requests and records the most requests it had in flight at + once, which shows whether the client's threads really overlapped. + """ + + daemon_threads = True + + def __init__(self): + super().__init__(("127.0.0.1", 0), EchoHandler) + self.lock = threading.Lock() + self.count = 0 + self.in_flight = 0 + self.peak_in_flight = 0 + + +class EchoHandler(BaseHTTPRequestHandler): + # keep-alive, so the client reuses pooled connections across threads + protocol_version = "HTTP/1.1" + server: EchoServer + + def do_GET(self): + with self.server.lock: + self.server.count += 1 + self.server.in_flight += 1 + self.server.peak_in_flight = max( + self.server.peak_in_flight, self.server.in_flight + ) + try: + url = urlparse(self.path) + body = json.dumps( + { + "path": url.path, + "query": dict(parse_qsl(url.query)), + "edge_id": self.headers.get(X_MASSIVE_EDGE_ID), + "edge_ip": self.headers.get(X_MASSIVE_EDGE_IP_ADDRESS), + "authorization": self.headers.get("Authorization"), + } + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + finally: + with self.server.lock: + self.server.in_flight -= 1 + + def log_message(self, format, *args): + pass + + +class ConcurrencyTest(unittest.TestCase): + """One RESTClient shared by many threads against a local HTTP server.""" + + threads = 16 + calls = 200 + + def setUp(self): + self.httpd = EchoServer() + thread = threading.Thread(target=self.httpd.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(self.httpd.server_close) + self.addCleanup(self.httpd.shutdown) + host, port = self.httpd.server_address[:2] + self.base = f"http://{host!s}:{port}" + + def test_shared_client(self): + c = RESTClient("key", base=self.base, retries=0, maxsize=self.threads) + headers_before = list(c.headers.items()) + barrier = threading.Barrier(self.threads, timeout=10) + + def run(t: int) -> List[Tuple[Dict[str, Any], Dict[str, Any]]]: + # start every thread together, so their requests overlap + barrier.wait() + results = [] + for i in range(self.calls): + ticker = f"T{t}N{i}" + date = f"2024-01-{t + 1:02}" + edge_id: Optional[str] = None + edge_ip: Optional[str] = None + if i % 2: + # Odd calls send per-request headers. They go through _get, + # the method every endpoint calls, because the endpoint + # methods also copy `options` into the query string. + edge_id, edge_ip = f"edge-{t}-{i}", f"10.0.{t}.{i % 256}" + resp = c._get( + path=f"/v3/reference/tickers/{ticker}", + params={"ticker": ticker, "date": date}, + raw=True, + options=RequestOptionBuilder( + edge_id=edge_id, edge_ip_address=edge_ip + ), + ) + else: + # raw=True returns the response, so the echoed request can + # be checked instead of a deserialized TickerDetails + resp = c.get_ticker_details(ticker, date=date, raw=True) + expected = { + "path": f"/v3/reference/tickers/{ticker}", + "query": {"ticker": ticker, "date": date}, + "edge_id": edge_id, + "edge_ip": edge_ip, + "authorization": "Bearer key", + } + results.append((expected, json.loads(resp.data))) + return results + + with ThreadPoolExecutor(self.threads) as pool: + futures = [pool.submit(run, t) for t in range(self.threads)] + results = [r for f in futures for r in f.result()] + + # every response answers its own request: the path and query name the + # thread and call, and the edge headers are the ones that call sent + self.assertEqual(len(results), self.threads * self.calls) + for expected, got in results: + self.assertEqual(got, expected) + self.assertEqual(self.httpd.count, self.threads * self.calls) + # the threads really shared the client at the same time + self.assertGreater(self.httpd.peak_in_flight, 1) + # per-request headers must not leak into the client's default headers: + # same keys, same values, same order + self.assertEqual(list(c.headers.items()), headers_before) + + +if __name__ == "__main__": + unittest.main() diff --git a/test_rest/test_dividends.py b/test_rest/test_dividends.py index 7b6575d0..8549df9d 100644 --- a/test_rest/test_dividends.py +++ b/test_rest/test_dividends.py @@ -3,6 +3,9 @@ class DividendsTest(BaseTest): + # test_list_stocks_dividends_ticker_any_of requests this twice + persistent_mocks = {"/stocks/v1/dividends?ticker.any_of=AAPL%2CVZ"} + def test_list_dividends(self): dividends = [d for d in self.c.list_dividends()] expected = [ diff --git a/test_rest/test_snapshots.py b/test_rest/test_snapshots.py index 3ead4e15..4472970a 100644 --- a/test_rest/test_snapshots.py +++ b/test_rest/test_snapshots.py @@ -26,6 +26,9 @@ class SnapshotsTest(BaseTest): + # test_get_snapshot_indices requests this twice + persistent_mocks = {"/v3/snapshot/indices?ticker.any_of=SPX%2CAPx%2CAPy"} + def test_list_universal_snapshots(self): expected = [ UniversalSnapshot(