WIP: Reproduce offline disconnect crash

.github/workflows/tests.yml

@@ -348,7 +348,9 @@ jobs:
             --gtest_output="xml:${{ env.BUILD_DIR }}\unit-test-results.xml"
 
       - name: Start livekit-server
-        if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        # The offline-disconnect tester below always needs a local server, even
+        # when a manual run disables integration and stress tests.
+        if: matrix.e2e-testing
         id: livekit_server
         uses: livekit/dev-server-action@5d4d5337a875e2d1afd37bed03c601d159dab002 # v1.1.1
         with:
@@ -359,7 +361,7 @@ jobs:
 
       # Needed by token helper script
       - name: Install livekit-cli
-        if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        if: matrix.e2e-testing
         shell: bash
         env:
           # Windows installs lk via `gh api` / `gh release download`, which need this env var
@@ -431,8 +433,25 @@ jobs:
             --gtest_recreate_environments_when_repeating=1 \
             --gtest_output=xml:${{ env.BUILD_DIR }}/stress-test-results.xml
 
+      # Keep this last: it is a standalone regression reproducer rather than
+      # part of the unit, integration, or stress suites. Git Bash is present
+      # on GitHub's Windows image and lets all matrix entries share the token
+      # helper and invocation.
+      - name: Run offline disconnect tester
+        if: matrix.e2e-testing
+        timeout-minutes: 3
+        shell: bash
+        run: |
+          set -euo pipefail
+          source scripts/set-test-tokens.sh
+          tester="${{ env.BUILD_DIR }}/bin/livekit_disconnect_offline_tester"
+          if [[ "$RUNNER_OS" == "Windows" ]]; then
+            tester+=".exe"
+          fi
+          bash scripts/run-with-backtrace.sh "$tester" --offline-duration-ms 5000
+
       - name: Dump livekit-server log on failure
-        if: failure() && matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        if: failure() && matrix.e2e-testing
         shell: bash
         run: tail -n 500 "${{ steps.livekit_server.outputs.log-path }}" || true
 

docs/testing.md

@@ -45,6 +45,52 @@ __Note:__ The tests require tokens and a running LiveKit server. See the section
 | `livekit_integration_tests` | Quick tests (~1-2 minutes) for SDK functionality |
 | `livekit_stress_tests` | Long-running tests (configurable, default 1 hour) |
 
+## Offline room-operation reproducer
+
+`livekit_disconnect_offline_tester` is a standalone cross-platform tester for
+an offline `Room::disconnect()` or `LocalParticipant::unpublishTrack()` call. It
+publishes local audio and video tracks, captures media for ten seconds, stops
+capture, then pauses traffic through a loopback TCP fault proxy in front of a
+`ws://` LiveKit server. By default it calls the selected operation immediately,
+before the room observes the failure; this matches the timing in issue #222.
+Forwarding resumes after the chosen observation period.
+
+The tester releases application-held audio/video sources before explicit
+disconnect and keeps its `RoomDelegate` alive until after disconnect returns.
+This follows the corrected shutdown ordering from issue #222.
+
+Build it with the normal test build:
+
+```bash
+./build.sh debug-tests
+```
+
+Supply a non-TLS (`ws://`) server URL and token, either directly or through the
+normal test environment:
+
+```bash
+export LIVEKIT_URL=ws://localhost:7880
+export LIVEKIT_TOKEN_A='<token with room-join permission>'
+./build-debug/bin/livekit_disconnect_offline_tester \
+  --operation disconnect --offline-duration-ms 10000
+
+# Exercise the related unpublishTrack wait reported in the follow-up.
+./build-debug/bin/livekit_disconnect_offline_tester \
+  --operation unpublish-track --offline-duration-ms 10000
+
+# Compare the separate case where LiveKit has already reported Reconnecting.
+./build-debug/bin/livekit_disconnect_offline_tester \
+  --operation disconnect --disconnect-timing after-reconnecting --offline-duration-ms 30000
+```
+
+The proxy cannot be used with `wss://`: it tunnels raw TCP through
+`127.0.0.1`, which does not preserve the server hostname required for TLS
+certificate validation. The proxy interrupts signaling traffic only; it does
+not disable direct UDP media transport. A healthy implementation should return
+promptly without waiting for proxy forwarding to resume. With the current Rust
+FFI, the unpublish variant can remain blocked even after forwarding resumes and
+may need to be terminated manually.
+
 ## Running a local LiveKit server for tests
 
 The integration and stress suites need a running LiveKit server. The easiest

src/tests/CMakeLists.txt

@@ -248,6 +248,59 @@ if(INTEGRATION_TEST_SOURCES)
   )
 endif()
 
+# ============================================================================
+# Connection Fault Testers
+# ============================================================================
+
+# Standalone reproducer for offline disconnect and track-unpublish waits.
+add_executable(livekit_disconnect_offline_tester
+  "${CMAKE_CURRENT_SOURCE_DIR}/connection/disconnect_offline_tester.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/common/tcp_fault_proxy.h"
+)
+target_link_libraries(livekit_disconnect_offline_tester PRIVATE
+  livekit
+  $<$<PLATFORM_ID:Windows>:ws2_32>
+)
+target_include_directories(livekit_disconnect_offline_tester PRIVATE
+  ${LIVEKIT_ROOT_DIR}/include
+  ${CMAKE_CURRENT_SOURCE_DIR}/common
+)
+target_compile_definitions(livekit_disconnect_offline_tester PRIVATE
+  $<$<PLATFORM_ID:Windows>:_USE_MATH_DEFINES>
+)
+
+if(WIN32)
+  add_custom_command(TARGET livekit_disconnect_offline_tester POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_FILE:livekit>
+      $<TARGET_FILE_DIR:livekit_disconnect_offline_tester>
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      "$<TARGET_FILE_DIR:livekit>/livekit_ffi.dll"
+      $<TARGET_FILE_DIR:livekit_disconnect_offline_tester>
+    COMMENT "Copying tester DLLs"
+  )
+elseif(APPLE)
+  add_custom_command(TARGET livekit_disconnect_offline_tester POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_FILE:livekit>
+      $<TARGET_FILE_DIR:livekit_disconnect_offline_tester>
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      "$<TARGET_FILE_DIR:livekit>/liblivekit_ffi.dylib"
+      $<TARGET_FILE_DIR:livekit_disconnect_offline_tester>
+    COMMENT "Copying tester shared libraries"
+  )
+else()
+  add_custom_command(TARGET livekit_disconnect_offline_tester POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_FILE:livekit>
+      $<TARGET_FILE_DIR:livekit_disconnect_offline_tester>
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      "$<TARGET_FILE_DIR:livekit>/liblivekit_ffi.so"
+      $<TARGET_FILE_DIR:livekit_disconnect_offline_tester>
+    COMMENT "Copying tester shared libraries"
+  )
+endif()
+
 # ============================================================================
 # Stress Tests
 # ============================================================================