diff --git a/src/hip/parsers/__init__.py b/src/hip/parsers/__init__.py index 9b403da..aecac5f 100644 --- a/src/hip/parsers/__init__.py +++ b/src/hip/parsers/__init__.py @@ -1,6 +1,10 @@ from .http import HTTPParser, ProtocolError +from .sansio import BytesParser, HTTPClient, ParseError __all__ = [ + "BytesParser", "HTTPParser", + "HTTPClient", + "ParseError", "ProtocolError", ] diff --git a/src/hip/parsers/sansio.py b/src/hip/parsers/sansio.py new file mode 100644 index 0000000..037651f --- /dev/null +++ b/src/hip/parsers/sansio.py @@ -0,0 +1,319 @@ +class ParseError(Exception): + pass + + +class BytesParser: + """ + A parser following the Sans-IO design pattern. + + Provides parsing operations for "read some data" and + "read some data up until a fixed delimiter". + + Used by the HTTP parsing to handle parsing the different + sections of the request and response. + """ + + def __init__(self): + self._buffer = bytearray() + self._closed = False + self._cursor = 0 + + def read_some(self, max_size: int) -> bytes | None: + """ + Returns up to `max_size` bytes. + Returns b'' to indicate the end of the stream. + Removes the returned data from the buffer. + + Returns `None` to indicate feed required. + """ + if not self._buffer: + if self._closed: + return b"" + return None + + read, unread = self._buffer[:max_size], self._buffer[max_size:] + self._buffer = bytearray(unread) + return bytes(read) + + def read_until(self, marker: bytes, max_size: int) -> bytes | None: + """ + Returns data up to and excluding the designated `marker`. + Returns at most `max_size` bytes. + Removes both the returned data and the marker from the buffer. + + Returns `None` to indicate feed required. + + Raises a `ParseError` if the marker cannot be found. + """ + if self._cursor >= len(self._buffer): + if self._closed: + raise ParseError() + return None + + start_point = self._cursor + end_point = max_size + len(marker) + idx = self._buffer.find(marker, start_point, end_point) + if idx < 0: + self._cursor = max(0, len(self._buffer) - len(marker)) + return None + + read, unread = self._buffer[:idx], self._buffer[idx + len(marker) :] + self._buffer = bytearray(unread) + self._cursor = 0 + return bytes(read) + + def feed(self, data: bytes) -> None: + """ + Feed `data` bytes to the parser. + Feed b'' to indicate the end of the stream. + """ + if data: + if self._closed: + raise ValueError("Attempted to feed data to a closed parser.") + self._buffer.extend(data) + else: + self._closed = True + + # def read_is_eof(self) -> bool | None: + # """ + # Returns True to indicate the end of the stream. + # Returns False to indicate the st + # Does not remove any data from the buffer. + + # Returns `None` to indicate feed required. + # """ + +# --- + +class ProtocolError(Exception): + pass + + +WAIT = 0 +SEND_START = 1 +SEND_HEADERS = 2 +SEND_BODY = 3 +SEND_COMPLETE = 4 + +MAX_STATUS_LINE_SIZE = 1024 +MAX_METHOD_LINE_SIZE = 16384 +MAX_HEADERS_SIZE = 65536 +MAX_READ = 65536 + + +# def _validate_request_line(data: bytes) -> tuple[bytes, bytes, bytes]: +# try: +# method, url, protocol = data.split(b" ") +# except ValueError: +# raise ProtocolError("Invalid request line") + +# if method.strip() == b"" or len(method) > 20: +# raise ProtocolError("Invalid method") +# if url.strip() == b"" or len(url) > 16384: +# raise ProtocolError("Invalid url") +# if protocol != b"HTTP/1.1" and protocol != b"HTTP/1.0": +# raise ProtocolError("Invalid protocol") + +# return method, url, protocol + + +def _validate_status_line(data: bytes) -> tuple[bytes, int, bytes]: + try: + protocol, status_code, reason = data.split(b" ", 3) + except ValueError: + raise ProtocolError("Invalid status line") + + if protocol != b"HTTP/1.1" and protocol != b"HTTP/1.0": + raise ProtocolError("Invalid protocol") + + if len(status_code) != 3: + raise ProtocolError("Invalid status code") + try: + status = int(status_code) + except ValueError: + raise ProtocolError("Invalid status code") + if not 100 <= status <= 999: + raise ProtocolError("Invalid status code") + + return protocol, status, reason + + +def _validate_headers(data: bytes) -> list[tuple[bytes, bytes]]: + headers = [] + for line in data.split(b"\r\n"): + try: + name, value = line.split(b":", 1) + except ValueError: + raise ProtocolError("Invalid header") + headers.append((name, value.strip())) + return headers + + +# TODO: + +# * Content-Length validation +# * Connection: keep-alive support +# * Transfer-Encoding: chunked support +# * Validate Host header + + + +class HTTPClient: + def __init__(self): + self._parser = BytesParser() + self._client = SEND_START + self._server = WAIT + + def send_request_line(self, method: bytes, target: bytes, protocol: bytes) -> bytes: + if self._client != SEND_START: + raise RuntimeError("Called 'send_method_line' out of sequence.") + + data = b"%b %b %b\r\n" % (method, target, protocol) + + # Request method line too long + # Invalid request method line + # Protocol not supported + + self._client = SEND_HEADERS + self._server = SEND_START + + return data + + def send_headers(self, headers: list[tuple[bytes, bytes]]) -> bytes: + if self._client != SEND_HEADERS: + raise RuntimeError("Called 'send_headers' out of sequence.") + + lines = [b"%b: %b\r\n" % (name, value) for name, value in headers] + lines.append(b"\r\n") + data = b"".join(lines) + + # Headers too long + # Too many headers + # Invalid request header + + self._client = SEND_BODY + + return data + + def send_body(self, body: bytes) -> bytes: + if self._client != SEND_BODY: + raise RuntimeError("Called 'send_body' out of sequence.") + + # Too little body for declared Content-Length + # Invalid framing for Transfer-Encoding + + if body == b'': + self._client = SEND_COMPLETE + + # if self._server == SEND_COMPLETE: + # self._client = SEND_START + # self._server = WAIT + + return body + + def recv_status_line(self) -> tuple[bytes, int, bytes] | None: + if self._server != SEND_START: + raise RuntimeError("Called 'recv_status_line' out of sequence.") + + if (data := self._parser.read_until(b"\r\n", MAX_STATUS_LINE_SIZE)) is None: + return None + + protocol, status, reason = _validate_status_line(data) + + self._server = SEND_HEADERS + + return protocol, status, reason + + def recv_headers(self) -> list[tuple[bytes, bytes]] | None: + if self._server != SEND_HEADERS: + raise RuntimeError("Called 'recv_headers' out of sequence.") + + if (data := self._parser.read_until(b"\r\n\r\n", MAX_HEADERS_SIZE)) is None: + return None + + headers = _validate_headers(data) + + self._server = SEND_BODY + + return headers + + def recv_body(self) -> bytes | None: + if self._server != SEND_BODY: + raise RuntimeError("Called 'recv_body' out of sequence.") + + if (body := self._parser.read_some(max_size=MAX_READ)) is None: + return None + + if body == b'': + self._server = SEND_COMPLETE + # if self._client == SEND_COMPLETE: + # self._client = SEND_START + # self._server = WAIT + + return body + + def feed(self, data: bytes) -> None: + """ + Feed `data` bytes to the parser. + Feeding b'' indicates EOF. + """ + self._parser.feed(data) + + # @property + # def is_keep_alive(self) -> bool: + # ... + + +# class HTTPServerParser: +# def __init__(self): +# self._parser = BytesParser() + +# def recv_method(self) -> tuple[bytes, bytes, bytes] | None: +# if (data := self._parser.read_until("\r\n", _MAX_REQUEST_LINE)) is None: +# return None +# # Request method line too long +# # Invalid request method line +# # Protocol not supported + +# def recv_headers(self) -> list[tuple[bytes, bytes]] | None: +# if (data := self._parser.read_until("\r\n\r\n", _MAX_HEADERS_SIZE)) is None: +# return None +# ... +# # Headers too long +# # Too many headers +# # Invalid request header + +# def recv_body(self) -> bytes | None: +# if (body := self._parser.read_some(max_size=...)) is None: +# return None +# # Too little body for declared Content-Length +# # Invalid framing for Transfer-Encoding + +# def send_response(self, protocol: bytes, status: int, reason: bytes) -> bytes: +# # Protocol not supported +# # Invalid status code +# # Invalid reason +# ... + +# def send_headers(self, headers: list[tuple[bytes, bytes]]) -> bytes: +# # Headers too long +# # Too many headers +# # Missing Host header +# # Conflicting Content-Length / Transfer-Encoding headers +# ... + +# def send_body(self, body: bytes) -> bytes: +# # Too much body for declared Content-Length +# # Too little body for declared Content-Length +# ... + +# def feed(self, data: bytes) -> None: +# """ +# Feed `data` bytes to the parser. +# Feeding b'' indicates EOF. +# """ +# self._parser.feed(data) + +# @property +# def is_keep_alive(self) -> bool: +# ... diff --git a/tests/test_parsers.py b/tests/test_parsers.py index b6a608f..11c381d 100644 --- a/tests/test_parsers.py +++ b/tests/test_parsers.py @@ -874,3 +874,205 @@ def test_request_missing_host_header(): msg = "Request missing 'Host' header" with pytest.raises(hip.parsers.ProtocolError, match=msg): p.recv_headers() + + +# Test Sans-IO parsers... + + +def test_bytes_parser(): + p = hip.parsers.BytesParser() + + # Feed some data, then half-close the buffer. + p.feed(b"abc") + p.feed(b"defg") + p.feed(b"") + + # Test a sequence of reads. + assert p.read_some(10) == b"abcdefg" + assert p.read_some(10) == b"" + assert p.read_some(10) == b"" + + +def test_bytes_parser_partial_reads(): + p = hip.parsers.BytesParser() + + # Feed some data, then half-close the buffer. + p.feed(b"abc") + p.feed(b"defg") + p.feed(b"") + + # Test a sequence of reads. + assert p.read_some(1) == b"a" + assert p.read_some(2) == b"bc" + assert p.read_some(3) == b"def" + assert p.read_some(4) == b"g" + assert p.read_some(5) == b"" + assert p.read_some(6) == b"" + + +def test_bytes_parser_feed_after_close(): + p = hip.parsers.BytesParser() + + # Feed some data, then half-close the buffer. + p.feed(b"abc") + p.feed(b"defg") + p.feed(b"") + + # Buffer is half-closed, cannot feed any more data. + with pytest.raises(ValueError): + assert p.feed(b"...") + + +def test_bytes_parser_need_feeds(): + p = hip.parsers.BytesParser() + + # Parser needs some I/O feeding to the buffer + assert p.read_some(1) is None + p.feed(b"abc") + assert p.read_some(1) == b"a" + assert p.read_some(2) == b"bc" + + # Parser needs some I/O feeding to the buffer + assert p.read_some(3) is None + p.feed(b"defg") + + assert p.read_some(3) == b"def" + assert p.read_some(4) == b"g" + + # Parser needs some I/O feeding to the buffer + assert p.read_some(5) is None + p.feed(b"") + assert p.read_some(5) == b"" + assert p.read_some(6) == b"" + + +def test_bytes_parser_read_until(): + p = hip.parsers.BytesParser() + + # Feed some data, then half-close the buffer. + p.feed(b"GET / HTTP/1.0\r\n") + p.feed(b"Host: example.com\r\n") + p.feed(b"Connection: close\r\n") + p.feed(b"\r\n") + p.feed(b"") + + # Parse the different components of the requst + request = p.read_until(b"\r\n", max_size=5_000) + headers = p.read_until(b"\r\n\r\n", max_size=10_000) + content = p.read_some(max_size=10_000) + + assert request == b"GET / HTTP/1.0" + assert headers == b"Host: example.com\r\nConnection: close" + assert content == p.read_some(max_size=10_000) + + +def test_bytes_parser_read_until_needs_feeds(): + p = hip.parsers.BytesParser() + + # Parser needs some I/O feeding to the buffer + assert p.read_until(b"\r\n", max_size=5_000) is None + p.feed(b"GET / HTTP/1.0\r\n") + # Request line is complete + assert p.read_until(b"\r\n", max_size=5_000) == b"GET / HTTP/1.0" + + # Parser needs some I/O feeding to the buffer + assert p.read_until(b"\r\n\r\n", max_size=10_000) is None + p.feed(b"Host: example.com\r\n") + # Parser needs some I/O feeding to the buffer + assert p.read_until(b"\r\n\r\n", max_size=10_000) is None + p.feed(b"Connection: close\r\n") + # Parser needs some I/O feeding to the buffer + assert p.read_until(b"\r\n\r\n", max_size=10_000) is None + p.feed(b"\r\n") + # Request headers are complete + assert ( + p.read_until(b"\r\n\r\n", max_size=10_000) + == b"Host: example.com\r\nConnection: close" + ) + + # Parser needs some I/O feeding to the buffer + assert p.read_some(max_size=10_000) is None + p.feed(b"") + # Reques is complete + assert p.read_some(max_size=10_000) == b"" + + +def test_bytes_parser_read_until_exception(): + p = hip.parsers.BytesParser() + p.feed(b"") + + with pytest.raises(hip.parsers.ParseError): + assert p.read_until(b"\r\n", max_size=5_000) + + +# Test HTTP client parser... + +def test_http_client_parser(): + parser = hip.parsers.HTTPClient() + + # Write the request... + output = b'' + output += parser.send_request_line(b"GET", b"/", b"HTTP/1.1") + output += parser.send_headers([ + (b"Host", b"www.example.com"), + (b"User-Agent", b"hip/dev") + ]) + output += parser.send_body(b'') + + assert output == ( + b"GET / HTTP/1.1\r\n" + b"Host: www.example.com\r\n" + b"User-Agent: hip/dev\r\n" + b"\r\n" + ) + + # Read the response... + input = ( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/html\r\n" + b"Content-Length: 25\r\n" + b"\r\n" + b"hello, world" + ) + parser.feed(input) + + protocol, status, reason = parser.recv_status_line() + headers = parser.recv_headers() + body = parser.recv_body() + + assert protocol == b'HTTP/1.1' + assert status == 200 + assert reason == b'OK' + assert headers == [ + (b'Content-Type', b'text/html'), + (b'Content-Length', b'25'), + ] + assert body == b'hello, world' + + +def test_http_client_parser_invalid_transitions(): + # Ensure calling parser methods cannot be called in an invalid order. + with pytest.raises(RuntimeError): + parser = hip.parsers.HTTPClient() + parser.send_request_line(b'GET', b'/', b'HTTP/1.1') + parser.send_request_line(b'GET', b'/', b'HTTP/1.1') + + with pytest.raises(RuntimeError): + parser = hip.parsers.HTTPClient() + parser.send_headers([]) + + with pytest.raises(RuntimeError): + parser = hip.parsers.HTTPClient() + parser.send_body(b'') + + with pytest.raises(RuntimeError): + parser = hip.parsers.HTTPClient() + parser.recv_status_line() + + with pytest.raises(RuntimeError): + parser = hip.parsers.HTTPClient() + parser.recv_headers() + + with pytest.raises(RuntimeError): + parser = hip.parsers.HTTPClient() + parser.recv_body()