pineforge-feed: serve fan-out, segmented journal, REST catch-up and backtest bar export

.github/workflows/native-feed.yml

@@ -55,7 +55,7 @@ jobs:
         run: |
           export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
           cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-            -DCMAKE_CXX_COMPILER=clang++-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \
+            -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \
             -DPINEFORGE_FEED_SANITIZER=address,undefined
           cmake --build build-feed -j4
           ctest --test-dir build-feed --output-on-failure
@@ -68,13 +68,13 @@ jobs:
       - uses: actions/checkout@v7
       - run: sudo apt-get update && sudo apt-get install -y ninja-build libssl-dev clang-18
       - run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl"
-      - name: Threaded receive, healing and termination
+      - name: Threaded receive, healing, termination and serve fan-out
         env:
           TSAN_OPTIONS: halt_on_error=1
         run: |
           export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}"
           cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-            -DCMAKE_CXX_COMPILER=clang++-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \
+            -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \
             -DPINEFORGE_FEED_SANITIZER=thread
           cmake --build build-feed -j4
           ctest --test-dir build-feed --output-on-failure

native/feed/CMakeLists.txt

@@ -46,7 +46,7 @@ if(PINEFORGE_FEED_SANITIZER)
 endif()
 
 add_library(pineforge_feed_core STATIC src/core.cpp src/state.cpp src/transport.cpp src/binance.cpp src/okx.cpp
-  src/bybit.cpp src/session.cpp)
+  src/bybit.cpp src/session.cpp src/export.cpp)
 target_include_directories(pineforge_feed_core PUBLIC src vendor)
 target_link_libraries(pineforge_feed_core PUBLIC CURL::libcurl OpenSSL::Crypto Threads::Threads)
 target_compile_options(pineforge_feed_core PRIVATE -Wall -Wextra -Wpedantic -Werror)
@@ -55,13 +55,53 @@ target_link_libraries(pineforge-feed PRIVATE pineforge_feed_core)
 target_compile_options(pineforge-feed PRIVATE -Wall -Wextra -Wpedantic -Werror)
 install(TARGETS pineforge-feed RUNTIME DESTINATION bin)
 
+# serve: the only code path that links an HTTP/WebSocket server. CivetWeb 1.16 (MIT) is fetched by release
+# archive and pinned by SHA-256; only its core is compiled: no TLS, CGI, Lua, Duktape, WebDAV, SSI or files.
+option(PINEFORGE_FEED_SERVE "Build the serve subcommand with the pinned CivetWeb release" ON)
+if(PINEFORGE_FEED_SERVE)
+  enable_language(C)
+  include(FetchContent)
+  set(PINEFORGE_FEED_CIVETWEB_ARCHIVE "https://github.com/civetweb/civetweb/archive/refs/tags/v1.16.tar.gz"
+      CACHE STRING "CivetWeb 1.16 release archive, a URL or a local path; its SHA-256 is pinned")
+  FetchContent_Declare(civetweb URL "${PINEFORGE_FEED_CIVETWEB_ARCHIVE}"
+    URL_HASH SHA256=f0e471c1bf4e7804a6cfb41ea9d13e7d623b2bcc7bc1e2a4dd54951a24d60285
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE SOURCE_SUBDIR do-not-add-civetweb-cmake)
+  FetchContent_MakeAvailable(civetweb)
+  add_library(pineforge_feed_civetweb STATIC "${civetweb_SOURCE_DIR}/src/civetweb.c")
+  target_include_directories(pineforge_feed_civetweb SYSTEM PUBLIC "${civetweb_SOURCE_DIR}/include")
+  target_compile_definitions(pineforge_feed_civetweb PRIVATE NO_SSL NO_CGI NO_FILES NO_CACHING STOP_FLAG_NEEDS_LOCK
+    USE_IPV6 REENTRANT_TIME PUBLIC USE_WEBSOCKET)
+  set_target_properties(pineforge_feed_civetweb PROPERTIES C_STANDARD 99)
+  target_compile_options(pineforge_feed_civetweb PRIVATE -w)
+  target_link_libraries(pineforge_feed_civetweb PUBLIC Threads::Threads ${CMAKE_DL_LIBS})
+  add_library(pineforge_feed_serve STATIC src/serve.cpp)
+  target_link_libraries(pineforge_feed_serve PUBLIC pineforge_feed_core PRIVATE pineforge_feed_civetweb)
+  target_compile_options(pineforge_feed_serve PRIVATE -Wall -Wextra -Wpedantic -Werror)
+  target_link_libraries(pineforge-feed PRIVATE pineforge_feed_serve)
+  target_compile_definitions(pineforge-feed PRIVATE PINEFORGE_FEED_SERVE=1)
+endif()
+
 include(CTest)
 if(BUILD_TESTING)
   add_executable(feed-unit tests/unit.cpp)
   target_link_libraries(feed-unit PRIVATE pineforge_feed_core)
+  if(PINEFORGE_FEED_SERVE)
+    target_link_libraries(feed-unit PRIVATE pineforge_feed_serve)
+    target_compile_definitions(feed-unit PRIVATE PINEFORGE_FEED_SERVE=1)
+  endif()
   target_compile_options(feed-unit PRIVATE -Wall -Wextra -Wpedantic -Werror -UNDEBUG)
   add_test(NAME feed-unit COMMAND feed-unit)
+  add_executable(feed-export-unit tests/export_unit.cpp)
+  target_link_libraries(feed-export-unit PRIVATE pineforge_feed_core)
+  target_compile_options(feed-export-unit PRIVATE -Wall -Wextra -Wpedantic -Werror -UNDEBUG)
+  add_test(NAME feed-export-unit COMMAND feed-export-unit)
   find_package(Python3 REQUIRED COMPONENTS Interpreter)
+  add_test(NAME feed-mock-export COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/mock_export.py $<TARGET_FILE:pineforge-feed>)
+  set_tests_properties(feed-mock-export PROPERTIES TIMEOUT 300)
   add_test(NAME feed-mock COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/mock_venue.py $<TARGET_FILE:pineforge-feed>)
   set_tests_properties(feed-mock PROPERTIES TIMEOUT 420)
+  if(PINEFORGE_FEED_SANITIZER STREQUAL "thread")
+    set_tests_properties(feed-unit feed-export-unit feed-mock feed-mock-export PROPERTIES ENVIRONMENT
+      "TSAN_OPTIONS=halt_on_error=1 suppressions=${CMAKE_CURRENT_SOURCE_DIR}/tests/tsan.supp")
+  endif()
 endif()

native/feed/NOTICE

@@ -18,6 +18,17 @@ Linked dependencies retain their upstream terms:
   OpenSSL 3.x: Apache License, Version 2.0, https://openssl.org
 Preserve upstream licenses/notices when distributing bundled libraries.
 
+The serve subcommand (build option PINEFORGE_FEED_SERVE, on by default) compiles CivetWeb:
+  CivetWeb 1.16, MIT License. Copyright (c) 2013-2021 The CivetWeb developers;
+  Copyright (c) 2004-2013 Sergey Lyubka; Copyright (c) 2013 No Face Press, LLC (Thomas Davis);
+  Copyright (c) 2013 F-Secure Corporation. https://github.com/civetweb/civetweb
+  Fetched at configure time from the release archive
+  https://github.com/civetweb/civetweb/archive/refs/tags/v1.16.tar.gz
+  Archive SHA-256: f0e471c1bf4e7804a6cfb41ea9d13e7d623b2bcc7bc1e2a4dd54951a24d60285
+  Only src/civetweb.c is compiled, with NO_SSL, NO_CGI, NO_FILES and NO_CACHING; Lua and Duktape
+  are not enabled. Its MIT license text (LICENSE.md in the archive) must accompany binary
+  distributions that include serve.
+
 CI builds libcurl 8.14.1 from curl-8.14.1.tar.xz.
 Archive SHA-256: f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd
 Source: https://curl.se/download/curl-8.14.1.tar.xz