diff --git a/.clang-format-ignore b/.clang-format-ignore new file mode 100644 index 000000000..55013b41e --- /dev/null +++ b/.clang-format-ignore @@ -0,0 +1 @@ +third_party/protobuf/** \ No newline at end of file diff --git a/.gitignore b/.gitignore index d1a1ab4f2..2560dc7fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,46 @@ +# Build Outputs & Directories /out*/ +LASTCHANGE +LASTCHANGE.committime build/ +build/mac_files/ +gclient_args.gni + +# Tools & Dependencies +*sysroot buildtools/ clang-format +third_party/rust-toolchain/ +tools/clang/scripts/update.py +yajsv +yajsv.exe + +# Certificates & Keys osp/impl/quic/certificates/agent_certificate.crt osp/impl/quic/certificates/private_key.key -tools/clang/scripts/update.py -*sysroot -*.profraw + +# Generated, Temp, and Trace Files +*.pftrace *.profdata +*.profraw +*.pyc +__pycache__/ generated_root_cast_receiver* -yajsv -yajsv.exe -__pycache__ + +# IDEs & Editors .vs/ -*.pyc .vscode/ -build/mac_files/ -third_party/rust-toolchain/ -LASTCHANGE -LASTCHANGE.committime -gclient_args.gni + +# AI Assistants & Agents +*copilot* +.aider* +.claude/ +.cline/ +.cline_history/ +.cursor/ +.cursorrules +.gemini/ +GEMINI.md +claude.json +tools/licenses/ +tools/protoc_wrapper/ diff --git a/.gitmodules b/.gitmodules index 8959db46d..98e194d77 100644 --- a/.gitmodules +++ b/.gitmodules @@ -72,3 +72,7 @@ path = third_party/googleurl/src url = https://quiche.googlesource.com/googleurl gclient-condition = not build_with_chromium +[submodule "third_party/perfetto/src"] + path = third_party/perfetto/src + url = https://chromium.googlesource.com/external/github.com/google/perfetto + gclient-condition = not build_with_chromium diff --git a/.gn b/.gn index 43d522bb0..7c127b717 100644 --- a/.gn +++ b/.gn @@ -16,3 +16,19 @@ check_targets = [ "//tools/*", "//util/*", ] + +default_args = { + # Disable js dependencies like the closure compiler. + enable_js_protobuf = false + + # Disable rust dependencies. Would be cool to potentially eventually support + # rust in Open Screen, but not yet! + enable_rust = false + + # Needed only for std::atomic_ref for large Ts http://crbug.com/402171653 + use_llvm_libatomic = false + + # Openscreen generally has the same consumers as WebRTC. So while WebRTC + # stays in C++20, so should openscreen. + use_cxx23 = false +} diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 000000000..557fa7bf8 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,2 @@ +[style] +based_on_style = pep8 diff --git a/.vpython3 b/.vpython3 index baf522027..f1485e758 100644 --- a/.vpython3 +++ b/.vpython3 @@ -22,20 +22,20 @@ # Read more about `vpython` and how to modify this file here: # https://chromium.googlesource.com/infra/infra/+/main/doc/users/vpython.md -python_version: "3.8" +python_version: "3.11" # The default set of platforms vpython checks does not yet include mac-arm64. # Setting `verify_pep425_tag` to the list of platforms we explicitly must support # allows us to ensure that vpython specs stay mac-arm64-friendly verify_pep425_tag: [ - {python: "cp38", abi: "cp38", platform: "manylinux1_x86_64"}, - {python: "cp38", abi: "cp38", platform: "linux_arm64"}, + {python: "cp311", abi: "cp311", platform: "manylinux1_x86_64"}, + {python: "cp311", abi: "cp311", platform: "linux_arm64"}, - {python: "cp38", abi: "cp38", platform: "macosx_10_10_intel"}, - {python: "cp38", abi: "cp38", platform: "macosx_11_0_arm64"}, + {python: "cp311", abi: "cp311", platform: "macosx_10_10_intel"}, + {python: "cp311", abi: "cp311", platform: "macosx_11_0_arm64"}, - {python: "cp38", abi: "cp38", platform: "win32"}, - {python: "cp38", abi: "cp38", platform: "win_amd64"} + {python: "cp311", abi: "cp311", platform: "win32"}, + {python: "cp311", abi: "cp311", platform: "win_amd64"} ] wheel: < diff --git a/BUILD.gn b/BUILD.gn index 021fcba11..4a55fce01 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -23,6 +23,9 @@ config("features") { root = rebase_path("./", "//") defines = [ "OPENSCREEN_TEST_DATA_DIR=\"$root/test/data/\"" ] + if (is_clang) { + cflags = [ "-Werror=exit-time-destructors" ] + } } # All compilable non-test targets in the repository (both executables and @@ -74,6 +77,8 @@ group("gn_all") { if (is_linux || is_mac || is_chromeos || is_android) { deps += [ "cast/standalone_receiver:cast_receiver" ] } + + visibility = [] } openscreen_source_set("openscreen_unittests_all") { @@ -85,7 +90,6 @@ openscreen_source_set("openscreen_unittests_all") { "cast/streaming:unittests", "cast/test:unittests", "platform:unittests", - "third_party/abseil", "util:unittests", ] @@ -97,9 +101,11 @@ openscreen_source_set("openscreen_unittests_all") { "discovery:unittests", ] } - public_deps += [ "osp/msgs:unittests" ] - if (!is_win) { - public_deps += [ "osp:unittests" ] + if (!build_with_chromium) { + public_deps += [ "osp/msgs:unittests" ] + if (!is_win) { + public_deps += [ "osp:unittests" ] + } } } if (!build_with_chromium) { diff --git a/DEPS b/DEPS index d72b2508b..7f9b4087d 100644 --- a/DEPS +++ b/DEPS @@ -46,34 +46,44 @@ vars = { 'checkout_instrumented_libraries': 'checkout_linux and checkout_configuration == "default"', # GN CIPD package version. - 'gn_version': 'git_revision:c97a86a72105f3328a540f5a5ab17d11989ab7dd', + 'gn_version': 'git_revision:c5a0003bcc2ac3f8d128aaffd700def6068e9a76', 'clang_format_revision': '37f6e68a107df43b7d7e044fd36a13cbae3413f2', + # Chrome version to pull clang update.py script from. This is necessary + # because this script does experience breaking changes, such as removing + # command line arguments, that need to be handled intentionally by a roll. + 'chrome_version': '4a1c93eb7da3e438ea5cb677c783379a282ed75d', + # 'magic' text to tell depot_tools that git submodules should be accepted # but parity with DEPS file is expected. 'SUBMODULE_MIGRATION': 'True', + # condition to allowlist deps to be synced in Cider. Allowlisting is needed + # because not all deps are compatible with Cider. Once we migrate everything + # to be compatible we can get rid of this allowlisting mecahnism and remove + # this condition. Tracking bug for removing this condition: b/349365433 + 'non_git_source': 'True', + # This can be overridden, e.g. with custom_vars, to build clang from HEAD # instead of downloading the prebuilt pinned revision. 'llvm_force_head_revision': False, } deps = { - # NOTE: These commit hashes here reference a repository/branch that is a - # mirror of the commits in the corresponding Chromium repository directory, - # and should be regularly updated with the tip of the MIRRORED master branch, - # found here: + # A mirror of the corresponding folder in Chromium maintained here: # https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/main + # + # IMPORTANT: Read the instructions at docs/roll_deps.md 'buildtools': { 'url': Var('chromium_git') + '/chromium/src/buildtools' + - '@' + '00459762409cb29cecf398a23cdb0cae918b7515', + '@' + 'eca5f0685c48ed59ff06077cb18cee00934249dd', }, - # and here: + # A mirror of the corresponding folder in Chromium maintained here: # https://chromium.googlesource.com/chromium/src/build/+/refs/heads/main 'build': { 'url': Var('chromium_git') + '/chromium/src/build' + - '@' + '043f0ac1c5fc7a29960fda36ce6689a96bdc11ee', + '@' + 'c53d22a398b881e70e53a972e285a925337a2494', 'condition': 'not build_with_chromium', }, @@ -143,7 +153,7 @@ deps = { 'third_party/jsoncpp/src': { 'url': Var('chromium_git') + '/external/github.com/open-source-parsers/jsoncpp.git' + - '@' + '89e2973c754a9c02a49974d839779b151e95afd6', # version 1.9.6 + '@' + '9af09c4a4abe5928d1f7a6e7ec1c73a565bb362e', 'condition': 'not build_with_chromium', }, @@ -152,30 +162,30 @@ deps = { 'third_party/googletest/src': { 'url': Var('chromium_git') + '/external/github.com/google/googletest.git' + - '@' + 'b514bdc898e2951020cbdca1304b75f5950d1f59', # 2023-01-25 + '@' + 'eb2d85edd0bff7a712b6aff147cd9f789f0d7d0b', # 2025-08-28 'condition': 'not build_with_chromium', }, - # Note about updating BoringSSL: after changing this hash, run the update - # script in BoringSSL's util folder for generating build files from the - # /third_party/boringssl directory: - # python3 ./src/util/generate_build_files.py --embed_test_data=false gn + # Make sure to also update ./third_party/boringssl/README.chromium's + # `Revision:` field when updating this dependency. 'third_party/boringssl/src': { 'url' : Var('boringssl_git') + '/boringssl.git' + - '@' + '8d19c850d4dbde4bd7ece463c3b3f3685571a779', + '@' + '26e8a8acb91a0cfbd2f95bf7245e2eb87d533a2f', 'condition': 'not build_with_chromium', }, - # To roll forward, use quiche_revision from chromium/src/DEPS. + # To roll forward, typically it is best to match Chrome's version by using + # quiche_revision from chromium/src/DEPS. Coordination with the QUICHE + # maintainers may be needed for some breaking changes. 'third_party/quiche/src': { 'url': Var('quiche_git') + '/quiche.git' + - '@' + '5a433bd7de22c23700d046346bd3d3afe5c9cd07', # 2025-02-10 + '@' + 'b8a4aa531a029737bcd741f85314e89775b923b2', # 2026-05-07 'condition': 'not build_with_chromium', }, 'third_party/instrumented_libs': { 'url': Var('chromium_git') + '/chromium/third_party/instrumented_libraries.git' + - '@' + '3cc43119a29158bcde39d288a8def4b8ec49baf8', + '@' + '69015643b3f68dbd438c010439c59adc52cac808', 'condition': 'not build_with_chromium', }, @@ -189,32 +199,196 @@ deps = { 'third_party/abseil/src': { 'url': Var('chromium_git') + '/external/github.com/abseil/abseil-cpp.git' + '@' + - 'dd4c89bd657f1e247ce5111a5c89ffe6ccfd0c92', # 2025-01-30 + '8a6b6ae902ac1eb7d9d8db810ecabc9c3cf88cf5', # 2026-04-21 'condition': 'not build_with_chromium', }, 'third_party/libfuzzer/src': { 'url': Var('chromium_git') + '/external/github.com/llvm/llvm-project/compiler-rt/lib/fuzzer.git' + - '@' + 'e31b99917861f891308269c36a32363b120126bb', + '@' + 'bea408a6e01f0f7e6c82a43121fe3af4506c932e', 'condition': 'not build_with_chromium', }, + # IMPORTANT: Read the instructions at docs/roll_deps.md 'third_party/libc++/src': { 'url': Var('chromium_git') + - '/external/github.com/llvm/llvm-project/libcxx.git' + '@' + '11c38d901d29bc91aee3efb53652f7141f72f47f', + '/external/github.com/llvm/llvm-project/libcxx.git' + '@' + '07572e7b169225ef3a999584cba9d9004631ae66', 'condition': 'not build_with_chromium', }, + # IMPORTANT: Read the instructions at docs/roll_deps.md 'third_party/libc++abi/src': { 'url': Var('chromium_git') + - '/external/github.com/llvm/llvm-project/libcxxabi.git' + '@' + '83dfa1f5bfce32d5f75695542468e37ead8163b8', + '/external/github.com/llvm/llvm-project/libcxxabi.git' + '@' + '83a852080747b9a362e8f9e361366b7a601f302c', 'condition': 'not build_with_chromium', }, + 'third_party/llvm-build/Release+Asserts': { + 'dep_type': 'gcs', + 'bucket': 'chromium-browser-clang', + 'objects': [ + { + # The Android libclang_rt.builtins libraries are currently only included in the Linux clang package. + 'object_name': 'Linux_x64/clang-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '1ef7b1d60fb433100c27b4552b44577ab86ef5394531d1fbebc237db64a893fd', + 'size_bytes': 56552908, + 'generation': 1762971374100697, + 'condition': '(host_os == "linux" or checkout_android) and non_git_source', + }, + { + 'object_name': 'Linux_x64/clang-tidy-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '505f0fa190dc3266f36f7908f46d4e2514b7b5edab02a25dbd721fb7f28dffd8', + 'size_bytes': 14268616, + 'generation': 1762971374302563, + 'condition': 'host_os == "linux" and checkout_clang_tidy and non_git_source', + }, + { + 'object_name': 'Linux_x64/clangd-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': 'cc0fe5e6f78a6d70234aa5fc9010761e63f283b8ad24e8194529c4677f723fdd', + 'size_bytes': 14443332, + 'generation': 1762971374370609, + 'condition': 'host_os == "linux" and checkout_clangd and non_git_source', + }, + { + 'object_name': 'Linux_x64/llvm-code-coverage-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': 'd5e60668fe312a345637b6c7918715ea54bb7078aa1bed1115dc382f955979d6', + 'size_bytes': 2304960, + 'generation': 1762971374620627, + 'condition': 'host_os == "linux" and checkout_clang_coverage_tools and non_git_source', + }, + { + 'object_name': 'Linux_x64/llvmobjdump-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': 'ec1d88867045b8348659f7a8f677d12aa91d7d61a68603a82bad1926bf57c3b0', + 'size_bytes': 5723188, + 'generation': 1762971374436694, + 'condition': '((checkout_linux or checkout_mac or checkout_android) and host_os == "linux") and non_git_source', + }, + { + 'object_name': 'Mac/clang-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': 'f266b79576d4fc0075e9380b68b8879ec2bc9617c973e7bdea694ec006f43636', + 'size_bytes': 54056416, + 'generation': 1762971376161293, + 'condition': 'host_os == "mac" and host_cpu == "x64"', + }, + { + 'object_name': 'Mac/clang-mac-runtime-library-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '6f2d61383a3c0ab28286e5a57b7e755eb14726bb9a73a7737b685488eae18b90', + 'size_bytes': 1010052, + 'generation': 1762971385382392, + 'condition': 'checkout_mac and not host_os == "mac"', + }, + { + 'object_name': 'Mac/clang-tidy-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '8e5157522a2557e14d8a456a1c227ebc522f383738f498c1451af3a98f361f99', + 'size_bytes': 14299120, + 'generation': 1762971376313425, + 'condition': 'host_os == "mac" and host_cpu == "x64" and checkout_clang_tidy', + }, + { + 'object_name': 'Mac/clangd-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '399d8930899c2f9bfb9bbcf841b9ca237d876e818a54b04256c790e6a2cb14c2', + 'size_bytes': 15832668, + 'generation': 1762971376411558, + 'condition': 'host_os == "mac" and host_cpu == "x64" and checkout_clangd', + }, + { + 'object_name': 'Mac/llvm-code-coverage-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '5b4d56a772e6128c98e7f880de5a052869334f39b59b81fee7079d56cd6bcfd4', + 'size_bytes': 2338512, + 'generation': 1762971376592644, + 'condition': 'host_os == "mac" and host_cpu == "x64" and checkout_clang_coverage_tools', + }, + { + 'object_name': 'Mac/llvmobjdump-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '9a282bf252e0c7ac88152844f347428e02970aa22941fb583439ce72134f0161', + 'size_bytes': 5607404, + 'generation': 1762971376526568, + 'condition': 'host_os == "mac" and host_cpu == "x64"', + }, + { + 'object_name': 'Mac_arm64/clang-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': 'a7b7caf53f4e722234e85aecfdbb3eeb94608c37394672bebd074d6b2f300362', + 'size_bytes': 45184380, + 'generation': 1762971386895625, + 'condition': 'host_os == "mac" and host_cpu == "arm64"', + }, + { + 'object_name': 'Mac_arm64/clang-tidy-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': 'bb3750fb501048c7ec2d145e69236b87bfe016bd3b81251c0e12f220c00d5875', + 'size_bytes': 12313940, + 'generation': 1762971387031271, + 'condition': 'host_os == "mac" and host_cpu == "arm64" and checkout_clang_tidy', + }, + { + 'object_name': 'Mac_arm64/clangd-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '5b585c910a8eb3f251a1efc76bc27fd63bcb4ebe99671f434f5d7fbfe76604c3', + 'size_bytes': 12690748, + 'generation': 1762971387200930, + 'condition': 'host_os == "mac" and host_cpu == "arm64" and checkout_clangd', + }, + { + 'object_name': 'Mac_arm64/llvm-code-coverage-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '044fec98aa72c1f4aebdc454a2bcc8d19735357e9f255d6fc01aae25c1369d41', + 'size_bytes': 1970340, + 'generation': 1762971387351744, + 'condition': 'host_os == "mac" and host_cpu == "arm64" and checkout_clang_coverage_tools', + }, + { + 'object_name': 'Mac_arm64/llvmobjdump-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': 'c5ee70e78ae5aa7a0d9b613ea5a8e21629438f12acb50bca0f7e18fae6abfe0a', + 'size_bytes': 5353832, + 'generation': 1762971387217357, + 'condition': 'host_os == "mac" and host_cpu == "arm64"', + }, + { + 'object_name': 'Win/clang-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '483b9b2809c3f53b9640e77d83ca6ab3017a0974979d242198abf23d99639e62', + 'size_bytes': 48337640, + 'generation': 1762971401378315, + 'condition': 'host_os == "win"', + }, + { + 'object_name': 'Win/clang-tidy-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': 'd40723233f6d59b1ba64cd7600d4da7b67a7433d81c32be3806ff1c47c9794aa', + 'size_bytes': 14255432, + 'generation': 1762971401522927, + 'condition': 'host_os == "win" and checkout_clang_tidy', + }, + { + 'object_name': 'Win/clang-win-runtime-library-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': 'd8b3310760c3a8f5dac4801583f7872601f4ba312742b0bf530f043ce6b6f36f', + 'size_bytes': 2520664, + 'generation': 1762971410370409, + 'condition': 'checkout_win and not host_os == "win"', + }, + { + 'object_name': 'Win/clangd-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '563b9f1a82980634657b89bb61ae5c6d386c8199acc01b84fb57cdcd0a53e1d1', + 'size_bytes': 14641972, + 'generation': 1762971401646458, + 'condition': 'host_os == "win" and checkout_clangd', + }, + { + 'object_name': 'Win/llvm-code-coverage-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '7caa4ecfbc320bf993640dcaa58882433a0adcc266adf26798f45d28d6d73af8', + 'size_bytes': 2385732, + 'generation': 1762971401865919, + 'condition': 'host_os == "win" and checkout_clang_coverage_tools', + }, + { + 'object_name': 'Win/llvmobjdump-llvmorg-22-init-14273-gea10026b-1.tar.xz', + 'sha256sum': '00c4dab7747534548e2111b3adbdbf9ef561887e18c7d6de4c7e273af799c190', + 'size_bytes': 5742908, + 'generation': 1762971401692156, + 'condition': '(checkout_linux or checkout_mac or checkout_android) and host_os == "win"', + }, + ] + }, + 'third_party/llvm-libc/src': { 'url': Var('chromium_git') + - '/external/github.com/llvm/llvm-project/libc.git' + '@' + '2222607a3ea3d5f65338d3b36a4cc5fb563169ab', + '/external/github.com/llvm/llvm-project/libc.git' + '@' + '74b25173cba70124bff5da97cc339d90c516c5f6', 'condition': 'not build_with_chromium', }, @@ -233,12 +407,19 @@ deps = { # Googleurl recommends living at head. This is a copy of Chrome's URL parsing # library. It is meant to be used by QUICHE. # - # NOTE: Pin to the current revision as newer versions use C++20 features. + # Make sure to also update ./third_party/googleurl/README.chromium's + # `Revision:` field when updating this dependency. 'third_party/googleurl/src': { 'url': Var('quiche_git') + '/googleurl.git' + - '@' + 'dfe8ef6164f8b4e3e9a9cbe8521bb81359918393', #2023-08-01 + '@' + '94ff147fe0b96b4cca5d6d316b9af6210c0b8051', #2025-11-11 + 'condition': 'not build_with_chromium', + }, + + 'third_party/perfetto/src': { + 'url': Var('chromium_git') + '/external/github.com/google/perfetto.git' + + '@' + '1d9994a93c6ada2fb261dc72984fa07683a6c86e', 'condition': 'not build_with_chromium', - } + }, } hooks = [ @@ -246,23 +427,39 @@ hooks = [ 'name': 'clang_update_script', 'pattern': '.', 'condition': 'not build_with_chromium', - 'action': [ 'python3', 'tools/download-clang-update-script.py', + 'action': [ 'python3', 'tools/download-chromium-file.py', + '--revision', Var('chrome_version'), + '--path', 'tools/clang/scripts/update.py', '--output', 'tools/clang/scripts/update.py' ], # NOTE: This file appears in .gitignore, as it is not a part of the # openscreen repo. }, { - 'name': 'update_clang', + 'name': 'licenses_script', 'pattern': '.', 'condition': 'not build_with_chromium', - 'action': [ 'python3', 'tools/clang/scripts/update.py' ], + 'action': [ 'python3', 'tools/download-chromium-file.py', + '--revision', Var('chrome_version'), + '--path', 'tools/licenses/licenses.py', + '--output', 'tools/licenses/licenses.py' ], }, { - 'name': 'clang_coverage_tools', + 'name': 'licenses_spdx_writer', 'pattern': '.', - 'condition': 'not build_with_chromium and checkout_clang_coverage_tools', - 'action': ['python3', 'tools/clang/scripts/update.py', - '--package=coverage_tools'], + 'condition': 'not build_with_chromium', + 'action': [ 'python3', 'tools/download-chromium-file.py', + '--revision', Var('chrome_version'), + '--path', 'tools/licenses/spdx_writer.py', + '--output', 'tools/licenses/spdx_writer.py' ], + }, + { + 'name': 'protoc_wrapper_script', + 'pattern': '.', + 'condition': 'not build_with_chromium', + 'action': [ 'python3', 'tools/download-chromium-file.py', + '--revision', Var('chrome_version'), + '--path', 'tools/protoc_wrapper/protoc_wrapper.py', + '--output', 'tools/protoc_wrapper/protoc_wrapper.py' ], }, ] @@ -290,14 +487,9 @@ include_rules = [ '+discovery/mdns/public', '+discovery/public', - # Don't include abseil from the root so the path can change via include_dirs - # rules when in Chromium. + # Don't include Abseil. '-third_party/abseil', - - # Abseil allowed headers. - # IMPORTANT: Do not add new entries; abseil is being removed from the library. - # See https://issuetracker.google.com/158433927 - '+absl/types/variant.h', + '-absl', # Similar to abseil, don't include boringssl using root path. Instead, # explicitly allow 'openssl' where needed. diff --git a/OWNERS b/OWNERS index 3b0bc99ca..eec71fc9c 100644 --- a/OWNERS +++ b/OWNERS @@ -1,12 +1,13 @@ # Primary owners. mfoltz@chromium.org -jopbha@chromium.org +jophba@chromium.org # Additional reviewers as needed. muyaoxu@google.com -takumif@chromium.org +taesuny@google.com # Former OWNERS # btolsch@chromium.org # rwkeane@google.com # cliffordcheng@chromium.org +# takumif@chromium.org diff --git a/PRESUBMIT.py b/PRESUBMIT.py index ed7da8e88..7a6842abf 100755 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -14,7 +14,11 @@ _REPO_PATH = os.path.dirname(os.path.realpath('__file__')) -_IMPORT_SUBFOLDERS = ['tools', os.path.join('buildtools', 'checkdeps')] +_IMPORT_SUBFOLDERS = [ + 'tools', + os.path.join('tools', 'licenses'), + os.path.join('buildtools', 'checkdeps') +] # git-cl upload is not compatible with __init__.py based subfolder imports, so # we extend the system path instead. @@ -27,29 +31,74 @@ from checkdeps import DepsChecker # pylint: disable=wrong-import-position import licenses # pylint: disable=wrong-import-position -def _CheckLicenses(input_api, output_api): + +class _LicensesArgs: + extra_third_party_dirs = None + extra_allowed_dirs = None + exclude_dirs = None + scan_root = _REPO_PATH + target_os = None + gn_out_dir = None + gn_target = None + enable_warnings = True + shipped_only = False + verbose = False + +def _check_licenses(input_api, output_api): """Checks third party licenses and returns a list of violations.""" # NOTE: the licenses check is confused by the fact that we don't actually - # check ou the libraries in buildtools/third_party, so explicitly exclude + # check out the libraries in buildtools/third_party, so explicitly exclude # that folder. See https://crbug.com/1215335 for more info. licenses.PRUNE_PATHS.update([ os.path.join('buildtools', 'third_party'), os.path.join('third_party', 'libc++'), os.path.join('third_party', 'libc++abi'), os.path.join('third_party', 'rust-toolchain'), - os.path.join('third_party', 'depot_tools') - ]) + os.path.join('third_party', 'depot_tools'), + os.path.join('third_party', 'boringssl', 'src', 'third_party'), + os.path.join('third_party', 'getopt'), + os.path.join('third_party', 'googleurl', 'src', 'polyfills', + 'third_party'), + os.path.join('third_party', 'instrumented_libs', 'binaries'), + os.path.join('third_party', 'perfetto', 'src', 'protos', + 'third_party'), + os.path.join('third_party', 'protobuf', 'third_party'), + os.path.join('third_party', 'valijson', 'src', 'thirdparty'), + ]) if any(s.LocalPath().startswith('third_party') for s in input_api.change.AffectedFiles()): + try: + _, had_errors = licenses._DiscoverMetadatas(_LicensesArgs()) + if had_errors: + return [ + output_api.PresubmitError( + "Third party license scan failed. Please check your " + + "third_party metadata or licenses.py SPECIAL_CASES." + ) + ] + except Exception as e: + return [ + output_api.PresubmitError( + "License check failed with exception: %s" % e) + ] + return [] + + +def _check_generated_infra_files(input_api, output_api): + files = input_api.UnixLocalPaths() + if (any(f.endswith('.star') for f in files) + and all(not f.endswith('.cfg') for f in files)): return [ - output_api.PresubmitError(v) - for v in licenses.ScanThirdPartyDirs() + output_api.PresubmitPromptWarning( + 'You changed .star files, but didn\'t run `lucicfg generate ' + 'infra/config/global/main.star`') ] + return [] -def _CheckDeps(input_api, output_api): +def _check_deps(input_api, output_api): """Checks DEPS rules and returns a list of violations.""" deps_checker = DepsChecker(input_api.PresubmitLocalPath()) deps_checker.CheckDirectory(input_api.PresubmitLocalPath()) @@ -64,7 +113,10 @@ def _CheckDeps(input_api, output_api): Error = namedtuple("Error", "type message") -def _CheckNoRegexMatches(regex, cpplint_args, error, include_cpp_files=True): +def _check_no_regex_matches(regex, + cpplint_args, + error, + include_cpp_files=True): """Checks that there are no matches for a specific regex. Args: @@ -89,7 +141,7 @@ def _CheckNoRegexMatches(regex, cpplint_args, error, include_cpp_files=True): r'\s*OSP_D?CHECK\([^)]*\.is_value\(\)\);\s*') -def _CheckNoValueDchecks(filename, clean_lines, linenum, error): +def _check_no_value_dchecks(filename, clean_lines, linenum, error): """Checks that there are no OSP_DCHECK(foo.is_value()) instances. filename: The name of the current file. @@ -101,8 +153,8 @@ def _CheckNoValueDchecks(filename, clean_lines, linenum, error): error_to_return = Error('runtime/is_value_dchecks', 'Unnecessary CHECK for ErrorOr::is_value()') - _CheckNoRegexMatches(_RE_PATTERN_VALUE_CHECK, cpplint_args, - error_to_return) + _check_no_regex_matches(_RE_PATTERN_VALUE_CHECK, cpplint_args, + error_to_return) # Matches Foo(Foo&&) when not followed by noexcept. @@ -110,7 +162,7 @@ def _CheckNoValueDchecks(filename, clean_lines, linenum, error): r'\s*(?P\w+)\((?P=classname)&&[^)]*\)\s*(?!noexcept)\s*[{;=]') -def _CheckNoexceptOnMove(filename, clean_lines, linenum, error): +def _check_noexcept_on_move(filename, clean_lines, linenum, error): """Checks that move constructors are declared with 'noexcept'. filename: The name of the current file. @@ -124,8 +176,8 @@ def _CheckNoexceptOnMove(filename, clean_lines, linenum, error): # We only check headers as noexcept is meaningful on declarations, not # definitions. This may skip some definitions in .cc files though. - _CheckNoRegexMatches(_RE_PATTERN_MOVE_WITHOUT_NOEXCEPT, cpplint_args, - error_to_return, False) + _check_no_regex_matches(_RE_PATTERN_MOVE_WITHOUT_NOEXCEPT, cpplint_args, + error_to_return, False) # Matches "namespace {". Since we only check one line at a time, we @@ -134,7 +186,7 @@ def _CheckNoexceptOnMove(filename, clean_lines, linenum, error): r'namespace +\w+ +\{') -def _CheckUnnestedNamespaces(filename, clean_lines, linenum, error): +def _check_unnested_namespaces(filename, clean_lines, linenum, error): """Checks that nestable namespaces are nested. filename: The name of the current file. @@ -150,7 +202,7 @@ def _CheckUnnestedNamespaces(filename, clean_lines, linenum, error): cpplint_args = CpplintArgs(filename, clean_lines, linenum + 1, error) error_to_return = Error('runtime/nested_namespace', 'Please nest namespaces when possible.') - _CheckNoRegexMatches(re, cpplint_args, error_to_return) + _check_no_regex_matches(re, cpplint_args, error_to_return) # Gives additional debug information whenever a linting error occurs. @@ -162,7 +214,7 @@ def _CheckUnnestedNamespaces(filename, clean_lines, linenum, error): # - There are some false positives with 'explicit' checks, but it's useful # enough to keep. # - We add a custom check for 'noexcept' usage. -def _CheckChangeLintsClean(input_api, output_api): +def _check_change_lints_clean(input_api, output_api): """Checks that all '.cc' and '.h' files pass cpplint.py.""" cpplint = input_api.cpplint # Directive that allows access to a protected member _XX of a client class. @@ -176,8 +228,8 @@ def _CheckChangeLintsClean(input_api, output_api): for file_name in files: cpplint.ProcessFile(file_name, _CPPLINT_VERBOSE_LEVEL, [ - _CheckNoexceptOnMove, _CheckNoValueDchecks, - _CheckUnnestedNamespaces + _check_noexcept_on_move, _check_no_value_dchecks, + _check_unnested_namespaces ]) if cpplint._cpplint_state.error_count: @@ -190,7 +242,7 @@ def _CheckChangeLintsClean(input_api, output_api): return [] -def _CheckLuciCfgLint(input_api, output_api): +def _check_luci_cfg_lint(input_api, output_api): """Check that the luci configs pass the linter.""" path = os.path.join('infra', 'config', 'global', 'main.star') if not input_api.AffectedSourceFiles( @@ -213,7 +265,7 @@ def _CheckLuciCfgLint(input_api, output_api): return result -def _CommonChecks(input_api, output_api): +def _common_checks(input_api, output_api): """Performs a list of checks that should be used for both presubmission and upload validation. """ @@ -245,7 +297,7 @@ def _CommonChecks(input_api, output_api): input_api.canned_checks.CheckChangeTodoHasOwner(input_api, output_api)) # Ensure code change passes linter cleanly. - results.extend(_CheckChangeLintsClean(input_api, output_api)) + results.extend(_check_change_lints_clean(input_api, output_api)) # Ensure code change has already had clang-format ran. results.extend( @@ -258,13 +310,15 @@ def _CommonChecks(input_api, output_api): input_api.canned_checks.CheckGNFormatted(input_api, output_api)) # Run buildtools/checkdeps on code change. - results.extend(_CheckDeps(input_api, output_api)) + results.extend(_check_deps(input_api, output_api)) # Ensure the LUCI configs pass the linter. - results.extend(_CheckLuciCfgLint(input_api, output_api)) + results.extend(_check_luci_cfg_lint(input_api, output_api)) # Run tools/licenses on code change. - results.extend(_CheckLicenses(input_api, output_api)) + results.extend(_check_licenses(input_api, output_api)) + + results.extend(_check_generated_infra_files(input_api, output_api)) return results @@ -280,4 +334,4 @@ def CheckChangeOnUpload(input_api, output_api): def CheckChangeOnCommit(input_api, output_api): """Checks the changelist whenever there is commit (`git cl commit`)""" - return _CommonChecks(input_api, output_api) + return _common_checks(input_api, output_api) diff --git a/README.md b/README.md index e3c937b30..9e2e37de5 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,28 @@ # Open Screen Library The Open Screen Library implements the Open Screen Protocol, Multicast DNS and -DNS-SD, and the Chromecast protocols (discovery, application control, and media +DNS-SD, and the Cast protocols (discovery, application control, and media streaming). -The library consists of feature modules that share a [common platform -API](platform/README.md) that must be implemented and linked by the embedding -application. +The library consists of feature modules that share a [common platform API](platform/README.md) +that must be implemented and linked by the embedding application. The major feature modules in the library can be used independently and have their own documentation: - * [Cast protocols](cast/README.md) (aka `libcast`) - * [Open Screen Protocol](osp/README.md) - * [Multicast DNS and DNS-SD](discovery/README.md) +- [Cast protocols](cast/README.md) (aka `libcast`) +- [Open Screen Protocol](osp/README.md) +- [Multicast DNS and DNS-SD](discovery/README.md) -# Getting the code +## Getting the code -## Installing depot_tools +### Installing depot_tools Library dependencies are managed using `gclient`, from the [depot_tools](https://www.chromium.org/developers/how-tos/depottools) repo. To get gclient, run the following command in your terminal: + ```bash git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ``` @@ -33,25 +33,20 @@ Note that openscreen does not use other features of `depot_tools` like `repo` or `drover`. However, some `git-cl` functions *do* work, like `git cl try`, `git cl format`, `git cl lint`, and `git cl upload.` -## Checking out code +### Checking out code From the parent directory of where you want the openscreen checkout (e.g., -`~/my_project_dir`), configure `gclient` and check out openscreen with the -following commands: +`~/my_project_dir`), check out openscreen with the following command: ```bash - cd ~/my_project_dir - gclient config https://chromium.googlesource.com/openscreen - gclient sync + fetch openscreen ``` -The first `gclient` command will create a default .gclient file in -`~/my_project_dir` that describes how to pull down the `openscreen` repository. -The second command creates an `openscreen/` subdirectory, downloads the source +This command will create an `openscreen/` subdirectory, downloads the source code, all third-party dependencies, and the toolchain needed to build things; and at their appropriate revisions. -## Syncing your local checkout +### Syncing your local checkout To update your local checkout from the openscreen reference repository, just run @@ -64,62 +59,34 @@ To update your local checkout from the openscreen reference repository, just run This will rebase any local commits on the remote top-of-tree, and update any dependencies that have changed. -# Build setup +## Build setup -The following are the main tools are required for development/builds. +The main tools required for development are listed below. The `gclient` tool, +which you installed as part of "Getting the code," will automatically download +and install a pre-built toolchain that includes `gn`, `clang-format`, `ninja`, +and `clang`. -- Installed by gclient automatically - - Build file generator: `gn` (installed into `buildtools/`) - - Code formatter: `clang-format` (installed into `buildtools/`) +- Installed by gclient automatically: + - Build file generator: `gn` (in `buildtools/`) + - Code formatter: `clang-format` (in `buildtools/`) - Builder: `ninja` - Compiler/Linker: `clang` -- Installed by you - - JSON validator: `yajsv` - - `libstdc++` - - `gcc` - - XCode - -## yajsv installation - -1. Install `go` from [https://golang.org](https://golang.org) or your Linux package manager. -2. `go install github.com/neilpa/yajsv@latest` - -## libstdc++ (Linux only) - -Ensure that libstdc++ 8 is installed, as clang depends on the system -instance of it. On Debian flavors, you can run: - -```bash - sudo apt-get install libstdc++-8-dev libstdc++6-8-dbg -``` - -## XCode (Mac only) - -You can install the XCode command-line tools only or the full version of -[XCode](https://apps.apple.com/us/app/xcode/id497799835?mt=12). - -```bash -xcode-select --install -``` - -## gcc (optional, Linux only) +- Installed by you: + - System-specific libraries and compilers. -Setting the `gn` argument `is_clang=false` on Linux enables building using gcc -instead. +### System Dependencies -```bash - mkdir out/debug-gcc - gn gen out/debug-gcc --args="is_clang=false" -``` +Building the library and its executables requires certain system-specific +dependencies (such as `libstdc++` on Linux or `XCode` on macOS). For certain +targets, like the standalone `cast_sender` and `cast_receiver`, additional +external media libraries (e.g., FFmpeg, SDL2) are also required. -Note that g++ version 9 or newer must be installed. On Debian flavors you can -run: +For a comprehensive guide on setting up these system-specific and external +library dependencies for Linux and macOS, please refer to: -```bash - sudo apt-get install gcc-9 -``` +**[→ cast/docs/external_libraries.md](cast/docs/external_libraries.md)** -## Debug build +### Debug build Setting the `gn` argument `is_debug=true` enables debug build. @@ -127,7 +94,7 @@ Setting the `gn` argument `is_debug=true` enables debug build. gn gen out/debug --args="is_debug=true" ``` -## gn configuration +### GN configuration Running `gn args` opens an editor that allows to create a list of arguments passed to every invocation of `gn gen`. `gn args --list` will list all of the @@ -137,7 +104,18 @@ possible arguments you can set. gn args out/debug ``` -# Building targets +### Providing compilation data to LSP tools (like clangd) + +In order for Language Server Protocol (LSP) tools like clangd to properly +interpret the codebase, they need access to a `compile_commands.json` file. + +This file can be generated by `GN` by using the following command: + +```sh +gn gen out/ --add-export-compile-commands="*" +``` + +## Building targets We use the Open Screen Protocol demo application as an example, however, the instructions are essentially the same for all executable targets. @@ -196,20 +174,20 @@ the build flags available. ./out/debug/openscreen_unittests ``` -# Contributing changes +## Contributing changes -Open Screen library code should follow the [Open Screen Library Style -Guide](docs/style_guide.md). +Open Screen library code should follow the [Open Screen Library Style Guide](docs/style_guide.md). -This library uses [Chromium Gerrit](https://chromium-review.googlesource.com/) for -patch management and code review (for better or worse). You will need to register -for an account at `chromium-review.googlesource.com` to upload patches for review. +This library uses [Chromium Gerrit](https://chromium-review.googlesource.com/) +for patch management and code review (for better or worse). You will need to +register for an account at `chromium-review.googlesource.com` to upload patches +for review. The following sections contain some tips about dealing with Gerrit for code reviews, specifically when pushing patches for review, getting patches reviewed, and committing patches. -# Uploading a patch for review +## Uploading a patch for review The `git cl` tool handles details of interacting with Gerrit (the Chromium code review tool) and is recommended for pushing patches for review. Once you have @@ -233,41 +211,38 @@ It's simplest to create a local git branch for each patch you want reviewed separately. `git cl` keeps track of review status separately for each local branch. -## Addressing merge conflicts +### Addressing merge conflicts If conflicting commits have been landed in the repository for a patch in review, Gerrit will flag the patch as having a merge conflict. In that case, use the instructions above to rebase your commits on top-of-tree and upload a new patchset with the merge conflicts resolved. -## Tryjobs +### Tryjobs -Clicking the `CQ DRY RUN` button (also, confusingly, labeled `COMMIT QUEUE +1`) +Clicking the `Cq Dry Run` button (also, confusingly, labeled `Commit Queue +1`) will run the current patchset through all LUCI builders and report the results. It is always a good idea get a green tryjob on a patch before sending it for review to avoid extra back-and-forth. You can also run `git cl try` from the commandline to submit a tryjob. -## Code reviews +### Code reviews -Send your patch to one or more committers in the -[COMMITTERS](https://chromium.googlesource.com/openscreen/+/refs/heads/master/COMMITTERS) +Send your patch to one or more owners from the +[OWNERS](https://chromium.googlesource.com/openscreen/+/refs/heads/main/OWNERS) file for code review. All patches must receive at least one LGTM by a committer +and endorsement by two Google employees to pass the `Review-Enforcement` check before it can be submitted. -## Submitting patches +### Submitting patches -After your patch has received one or more LGTM commit it by clicking the -`SUBMIT` button (or, confusingly, `COMMIT QUEUE +2`) in Gerrit. This will run +After your patch has received one or more LGTMs, commit it by clicking the +`Submit` button (or, confusingly, `Commit Queue +2`) in Gerrit. This will run your patch through the builders again before committing to the main openscreen repository. -# Additional resources - -* [Continuous builders](docs/continuous_build.md) -* [Building and running fuzz tests](docs/fuzzing.md) -* [Running on a Raspberry PI](docs/raspberry_pi.md) -* [Unit test code coverage](docs/code_coverage.md) -* [Rolling dependencies](docs/roll_deps.md) +## Additional resources +- [Main documentation folder](docs/) +- [Cast-specific documentation](cast/docs/) diff --git a/build b/build index 043f0ac1c..c53d22a39 160000 --- a/build +++ b/build @@ -1 +1 @@ -Subproject commit 043f0ac1c5fc7a29960fda36ce6689a96bdc11ee +Subproject commit c53d22a398b881e70e53a972e285a925337a2494 diff --git a/build_overrides/BUILD.gn b/build_overrides/BUILD.gn deleted file mode 100644 index 7f0b1bebf..000000000 --- a/build_overrides/BUILD.gn +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2024 The Chromium Authors -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import("//build_overrides/build.gni") - -#TODO(jophba): delete once we use the new ../:include_dirs target in Chrome. -# This config must be set manually in build targets, but can be used both -# in standalone and embedded builds. Setting default_include_dirs doesn't work -# in embedded builds, because GN does not allow for multiple BUILDCONFIG.gn -# files to be included, or for multiple sets of default configs to be toggled -# on a single target type (e.g. source_set, static_library) -config("openscreen_include_dirs") { - openscreen_root = rebase_path("../", "//") - - include_dirs = [ - "//$openscreen_root", - "$root_gen_dir/$openscreen_root", - ] -} diff --git a/build_overrides/build.gni b/build_overrides/build.gni index 8d0d7c9ca..520189888 100644 --- a/build_overrides/build.gni +++ b/build_overrides/build.gni @@ -17,15 +17,6 @@ build_with_chromium = false # `use_custom_libcxx = true`. enable_safe_libcxx = true -declare_args() { - # Allows googletest to pretty-print various absl types. Disabled for nacl due - # to lack of toolchain support. - gtest_enable_absl_printers = true - - # Allow projects that wish to stay on C++17 to override Chromium's default. - use_cxx17 = true -} - if (host_os == "mac" || is_apple) { # Needed for is_apple when targeting macOS or iOS, independent of host. # Needed for host_os=="mac" for running host tool such as gperf in blink @@ -42,3 +33,5 @@ if (host_os == "mac" || is_apple) { assert(current_os != "ios" || use_system_xcode) assert(host_os == "mac" || !use_system_xcode) } + +import("//build_overrides/perfetto.gni") diff --git a/build_overrides/perfetto.gni b/build_overrides/perfetto.gni new file mode 100644 index 000000000..277382780 --- /dev/null +++ b/build_overrides/perfetto.gni @@ -0,0 +1,34 @@ +# Copyright 2026 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This file is imported by Perfetto's BUILD.gn files. + +# The location of the Perfetto checkout. +perfetto_root = "//third_party/perfetto/src" +perfetto_root_path = "//third_party/perfetto/src/" + +# Prevent Perfetto from trying to use its own standalone build setup. +perfetto_build_with_embedder = true + +# We don't want to build the trace processor, just the tracing SDK. +enable_perfetto_trace_processor = false + +# We don't need the IPC layer. +enable_perfetto_ipc = false + +# We want the client library. +enable_perfetto_platform_services = false + +# Other flags to minimize build size/deps. +enable_perfetto_heapprofd = false +enable_perfetto_traced_probes = false +enable_perfetto_tools = false +enable_perfetto_unittests = false +enable_perfetto_benchmarks = false +enable_perfetto_fuzzers = false +enable_perfetto_integration_tests = false + +# Map dependencies. +perfetto_protobuf_target = "//third_party/protobuf:protobuf_lite" +perfetto_protobuf_config = "//third_party/protobuf:protobuf_config" diff --git a/build_overrides/protobuf.gni b/build_overrides/protobuf.gni new file mode 100644 index 000000000..188021bc1 --- /dev/null +++ b/build_overrides/protobuf.gni @@ -0,0 +1,5 @@ +# Copyright 2025 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +protobuf_abseil_dir = "//third_party/abseil" diff --git a/buildtools b/buildtools index 004597624..eca5f0685 160000 --- a/buildtools +++ b/buildtools @@ -1 +1 @@ -Subproject commit 00459762409cb29cecf398a23cdb0cae918b7515 +Subproject commit eca5f0685c48ed59ff06077cb18cee00934249dd diff --git a/cast/README.md b/cast/README.md index 122eebf5f..479664357 100644 --- a/cast/README.md +++ b/cast/README.md @@ -4,14 +4,38 @@ libcast is an open source implementation of the Cast protocols that allow Cast senders to launch Cast applications and stream real-time media to Cast-compatible devices (aka "receivers"). -Submodules include: +Included are two applications, `cast_sender` and `cast_receiver` that +demonstrate how to send and receive media using a Cast Streaming session. - * [cast/streaming/](streaming/README.md) - Cast Streaming (both sending and receiving media). - * [receiver/public/](receiver/public/README.md) - Cast server socket and a demonstration - server (agent). - * [sender/public/](sender/public/README.md) - Cast client socket and supporting APIs to - launch Cast applications. - * [test/](test/README.md) - Integration tests. +## Components +Libcast is roughly broken into components by folder, with a non-exhaustive list +of the most important listed here: + +* [streaming/](streaming/README.md) - Cast Streaming (both sending and receiving + media). + +* [receiver/public/](receiver/public/README.md) - Cast server socket and a + demonstration server (agent). + +* [sender/public/](sender/public/README.md) - Cast client socket and supporting + APIs to launch Cast applications. + +* standalone_receiver/ - A reference implementation of a receiver application. + Supports `--enable-input-events` to send SDL mouse events to the sender. + +* standalone_sender/ - A reference implementation of a sender application. + Supports `--enable-input-events` to receive and log input events from the receiver. + +* [docs/input_demo.md](docs/input_demo.md) - A guide on how to use the Input Event API demo. + +* [test/](test/README.md) - Integration tests. + +With all of the documentation for this implementation in the [docs](docs/) +folder. See the [architecture.md](docs/architecture.md) as a potential jumping +off point. + +*** aside The `streaming` module can be used independently of the `sender` and `receiver` modules. +*** diff --git a/cast/common/BUILD.gn b/cast/common/BUILD.gn index 1b3321849..cca52ce60 100644 --- a/cast/common/BUILD.gn +++ b/cast/common/BUILD.gn @@ -38,7 +38,6 @@ openscreen_source_set("certificate") { deps = [ "../../platform", - "../../third_party/abseil", "../../util", "certificate/proto:certificate_proto", ] @@ -57,7 +56,7 @@ openscreen_source_set("certificate_boringssl") { "../standalone_receiver:cast_receiver", "../standalone_sender:cast_sender", "../test:*", - "//components/media_router/common/providers/cast/channel:openscreen_cast_auth_util_fuzzer", + "//components/media_router/common/providers/cast/channel:*", ] public = [] sources = [ @@ -76,7 +75,6 @@ openscreen_source_set("certificate_boringssl") { deps = [ ":certificate", - "../../third_party/abseil", "../../util", ] } @@ -111,7 +109,6 @@ openscreen_source_set("channel") { public_deps = [ ":public", "../../platform", - "../../third_party/abseil", "../../util", "channel/proto:channel_proto", ] @@ -141,7 +138,6 @@ openscreen_source_set("public") { public_deps = [ "../../discovery:public", "../../platform", - "../../third_party/abseil", "../../util", ] } @@ -187,7 +183,6 @@ openscreen_source_set("test_helpers") { "../../discovery:public", "../../platform:test", "../../testing/util", - "../../third_party/abseil", "../../third_party/boringssl", "../../third_party/googletest:gmock", "../../third_party/googletest:gtest", diff --git a/cast/common/certificate/boringssl_trust_store.cc b/cast/common/certificate/boringssl_trust_store.cc index 5c6e21514..7d37a01ea 100644 --- a/cast/common/certificate/boringssl_trust_store.cc +++ b/cast/common/certificate/boringssl_trust_store.cc @@ -383,8 +383,8 @@ BoringSSLTrustStore::FindCertificatePath( intermediate_certs.emplace_back(ParseX509Der(der_certs[i])); if (!intermediate_certs.back()) { return Error(Error::Code::kErrCertsParse, - StringPrintf("FindCertificatePath: Failed to parse " - "intermediate certificate %zu of %zu", + StringFormat("FindCertificatePath: Failed to parse " + "intermediate certificate {} of {}", i, der_certs.size())); } } diff --git a/cast/common/certificate/cast_cert_validator.h b/cast/common/certificate/cast_cert_validator.h index 1aacc80fb..e4a8338cc 100644 --- a/cast/common/certificate/cast_cert_validator.h +++ b/cast/common/certificate/cast_cert_validator.h @@ -11,7 +11,6 @@ #include "cast/common/public/certificate_types.h" #include "platform/base/error.h" -#include "platform/base/macros.h" namespace openscreen::cast { diff --git a/cast/common/certificate/cast_cert_validator_unittest.cc b/cast/common/certificate/cast_cert_validator_unittest.cc index 09db72bc0..399835984 100644 --- a/cast/common/certificate/cast_cert_validator_unittest.cc +++ b/cast/common/certificate/cast_cert_validator_unittest.cc @@ -12,9 +12,9 @@ #include "cast/common/public/trust_store.h" #include "gtest/gtest.h" #include "openssl/pem.h" -#include "platform/test/byte_view_test_util.h" #include "platform/test/paths.h" #include "util/crypto/pem_helpers.h" +#include "util/no_destructor.h" namespace openscreen::cast { namespace { @@ -89,10 +89,10 @@ void RunTest(Error::Code expected_result, // Test verification of some invalid signatures. EXPECT_FALSE(target_cert->VerifySignedData( - DigestAlgorithm::kSha256, ByteViewFromLiteral("bogus data"), - ByteViewFromLiteral("bogus signature"))); + DigestAlgorithm::kSha256, ByteViewFromString("bogus data"), + ByteViewFromString("bogus signature"))); EXPECT_FALSE(target_cert->VerifySignedData( - DigestAlgorithm::kSha256, ByteViewFromLiteral("bogus data"), ByteView())); + DigestAlgorithm::kSha256, ByteViewFromString("bogus data"), ByteView())); EXPECT_FALSE(target_cert->VerifySignedData(DigestAlgorithm::kSha256, ByteView(), ByteView())); @@ -145,9 +145,9 @@ DateTime MarchFirst2037() { } const std::string& GetSpecificTestDataPath() { - static std::string data_path = - GetTestDataPath() + "/cast/common/certificate/"; - return data_path; + static const NoDestructor data_path(GetTestDataPath() + + "/cast/common/certificate/"); + return *data_path; } // Tests verifying a valid certificate chain of length 2: diff --git a/cast/common/certificate/cast_crl.cc b/cast/common/certificate/cast_crl.cc index 7cdbdab65..6a1058a9c 100644 --- a/cast/common/certificate/cast_crl.cc +++ b/cast/common/certificate/cast_crl.cc @@ -11,10 +11,8 @@ #include "cast/common/certificate/date_time.h" #include "cast/common/public/parsed_certificate.h" #include "cast/common/public/trust_store.h" -#include "platform/base/macros.h" #include "util/crypto/sha2.h" #include "util/osp_logging.h" -#include "util/span_util.h" namespace openscreen::cast { namespace { @@ -117,7 +115,9 @@ CastCRL::CastCRL(const proto::TbsCrl& tbs_crl, } } -CastCRL::~CastCRL() {} +CastCRL::CastCRL(CastCRL&&) noexcept = default; +CastCRL& CastCRL::operator=(CastCRL&&) = default; +CastCRL::~CastCRL() = default; // Verifies the revocation status of the certificate chain, at the specified // time. diff --git a/cast/common/certificate/cast_crl.h b/cast/common/certificate/cast_crl.h index 936224e6b..f181c56a6 100644 --- a/cast/common/certificate/cast_crl.h +++ b/cast/common/certificate/cast_crl.h @@ -13,7 +13,6 @@ #include "cast/common/certificate/cast_cert_validator.h" #include "cast/common/certificate/proto/revocation.pb.h" -#include "platform/base/macros.h" namespace openscreen::cast { @@ -25,6 +24,10 @@ class TrustStore; class CastCRL { public: CastCRL(const proto::TbsCrl& tbs_crl, const DateTime& overall_not_after); + CastCRL(const CastCRL&) = delete; + CastCRL(CastCRL&&) noexcept; + CastCRL& operator=(const CastCRL&) = delete; + CastCRL& operator=(CastCRL&&); ~CastCRL(); // Verifies the revocation status of a cast device certificate given a chain @@ -61,8 +64,6 @@ class CastCRL { // The value is a list of revoked serial number ranges. std::unordered_map> revoked_serial_numbers_; - - OSP_DISALLOW_COPY_AND_ASSIGN(CastCRL); }; // Parses and verifies the CRL used to verify the revocation status of diff --git a/cast/common/certificate/cast_crl_unittest.cc b/cast/common/certificate/cast_crl_unittest.cc index fb02ab7c5..514c31bb4 100644 --- a/cast/common/certificate/cast_crl_unittest.cc +++ b/cast/common/certificate/cast_crl_unittest.cc @@ -11,6 +11,7 @@ #include "cast/common/public/trust_store.h" #include "gtest/gtest.h" #include "platform/test/paths.h" +#include "util/no_destructor.h" #include "util/osp_logging.h" #include "util/read_file.h" @@ -86,8 +87,9 @@ bool TestVerifyRevocation(Error::Code expected_result, } const std::string& GetSpecificTestDataPath() { - static std::string data_path = GetTestDataPath() + "cast/common/certificate/"; - return data_path; + static const NoDestructor data_path(GetTestDataPath() + + "cast/common/certificate/"); + return *data_path; } bool RunTest(const proto::DeviceCertTest& test_case) { diff --git a/cast/common/channel/cast_socket.cc b/cast/common/channel/cast_socket.cc index 7647acbc5..80a7bcd0a 100644 --- a/cast/common/channel/cast_socket.cc +++ b/cast/common/channel/cast_socket.cc @@ -19,8 +19,7 @@ using proto::CastMessage; CastSocket::Client::~Client() = default; -CastSocket::CastSocket(std::unique_ptr connection, - Client* client) +CastSocket::CastSocket(std::unique_ptr connection, Client* client) : connection_(std::move(connection)), client_(client), socket_id_(g_next_socket_id_++) { @@ -56,27 +55,19 @@ void CastSocket::SetClient(Client* client) { } std::array CastSocket::GetSanitizedIpAddress() { - IPEndpoint remote = connection_->GetRemoteEndpoint(); std::array result; - uint8_t bytes[16]; - if (remote.address.IsV4()) { - remote.address.CopyToV4(bytes); - result[0] = bytes[2]; - result[1] = bytes[3]; - } else { - remote.address.CopyToV6(bytes); - result[0] = bytes[14]; - result[1] = bytes[15]; - } + IPEndpoint remote = connection_->GetRemoteEndpoint(); + std::span bytes = remote.address.bytes().last<2>(); + std::copy(bytes.begin(), bytes.end(), result.begin()); return result; } -void CastSocket::OnError(TlsConnection* connection, const Error& error) { +void CastSocket::OnError(Connection* connection, const Error& error) { state_ = State::kError; client_->OnError(this, error); } -void CastSocket::OnRead(TlsConnection* connection, std::vector block) { +void CastSocket::OnRead(Connection* connection, std::vector block) { read_buffer_.insert(read_buffer_.end(), block.begin(), block.end()); // NOTE: Read as many messages as possible out of `read_buffer_` since we only // get one callback opportunity for this. @@ -86,7 +77,7 @@ void CastSocket::OnRead(TlsConnection* connection, std::vector block) { ByteBuffer(&read_buffer_[0], read_buffer_.size())); if (!message_or_error) { OSP_DLOG_ERROR << __func__ << ": failed to deserialize a message. " - << message_or_error.error().ToString(); + << message_or_error.error(); return; } OSP_DVLOG << __func__ << ": read a message. " diff --git a/cast/common/channel/cast_socket_unittest.cc b/cast/common/channel/cast_socket_unittest.cc index 211977e52..1eec55ce8 100644 --- a/cast/common/channel/cast_socket_unittest.cc +++ b/cast/common/channel/cast_socket_unittest.cc @@ -17,7 +17,6 @@ using proto::CastMessage; namespace { using ::testing::_; -using ::testing::Invoke; using ::testing::Return; class CastSocketTest : public ::testing::Test { @@ -48,21 +47,21 @@ class CastSocketTest : public ::testing::Test { } // namespace TEST_F(CastSocketTest, SendMessage) { - EXPECT_CALL(connection(), Send(_)).WillOnce(Invoke([this](ByteView data) { - EXPECT_EQ(frame_serial_, std::vector(data.cbegin(), data.cend())); + EXPECT_CALL(connection(), Send(_)).WillOnce([this](ByteView data) { + EXPECT_EQ(frame_serial_, std::vector(data.begin(), data.end())); return true; - })); + }); ASSERT_TRUE(socket().Send(message_).ok()); } TEST_F(CastSocketTest, SendMessageEventuallyBlocks) { EXPECT_CALL(connection(), Send(_)) .Times(3) - .WillRepeatedly(Invoke([this](ByteView data) { + .WillRepeatedly([this](ByteView data) { EXPECT_EQ(frame_serial_, - std::vector(data.cbegin(), data.cend())); + std::vector(data.begin(), data.end())); return true; - })) + }) .RetiresOnSaturation(); ASSERT_TRUE(socket().Send(message_).ok()); ASSERT_TRUE(socket().Send(message_).ok()); @@ -75,9 +74,9 @@ TEST_F(CastSocketTest, SendMessageEventuallyBlocks) { TEST_F(CastSocketTest, ReadCompleteMessage) { const uint8_t* data = frame_serial_.data(); EXPECT_CALL(mock_client(), OnMessage(_, _)) - .WillOnce(Invoke([this](CastSocket* socket, CastMessage message) { + .WillOnce([this](CastSocket* socket, CastMessage message) { EXPECT_EQ(message_.SerializeAsString(), message.SerializeAsString()); - })); + }); connection().OnRead(std::vector(data, data + frame_serial_.size())); } @@ -87,9 +86,9 @@ TEST_F(CastSocketTest, ReadChunkedMessage) { connection().OnRead(std::vector(data, data + 10)); EXPECT_CALL(mock_client(), OnMessage(_, _)) - .WillOnce(Invoke([this](CastSocket* socket, CastMessage message) { + .WillOnce([this](CastSocket* socket, CastMessage message) { EXPECT_EQ(message_.SerializeAsString(), message.SerializeAsString()); - })); + }); connection().OnRead( std::vector(data + 10, data + frame_serial_.size())); @@ -100,16 +99,16 @@ TEST_F(CastSocketTest, ReadChunkedMessage) { frame_serial_.end()); data = double_message.data(); EXPECT_CALL(mock_client(), OnMessage(_, _)) - .WillOnce(Invoke([this](CastSocket* socket, CastMessage message) { + .WillOnce([this](CastSocket* socket, CastMessage message) { EXPECT_EQ(message_.SerializeAsString(), message.SerializeAsString()); - })); + }); connection().OnRead( std::vector(data, data + frame_serial_.size() + 10)); EXPECT_CALL(mock_client(), OnMessage(_, _)) - .WillOnce(Invoke([this](CastSocket* socket, CastMessage message) { + .WillOnce([this](CastSocket* socket, CastMessage message) { EXPECT_EQ(message_.SerializeAsString(), message.SerializeAsString()); - })); + }); connection().OnRead(std::vector(data + frame_serial_.size() + 10, data + double_message.size())); } @@ -135,9 +134,9 @@ TEST_F(CastSocketTest, ReadMultipleMessagesPerBlock) { send_data.insert(send_data.end(), frame_serial_.begin(), frame_serial_.end()); send_data.insert(send_data.end(), frame_serial2.begin(), frame_serial2.end()); EXPECT_CALL(mock_client(), OnMessage(_, _)) - .WillOnce(Invoke([this](CastSocket* socket, CastMessage message) { + .WillOnce([this](CastSocket* socket, CastMessage message) { EXPECT_EQ(message_.SerializeAsString(), message.SerializeAsString()); - })) + }) .WillOnce([message2](CastSocket* socket, CastMessage message) { EXPECT_EQ(message2.SerializeAsString(), message.SerializeAsString()); }); diff --git a/cast/common/channel/connection_namespace_handler_unittest.cc b/cast/common/channel/connection_namespace_handler_unittest.cc index fbba05aa8..b8106eab9 100644 --- a/cast/common/channel/connection_namespace_handler_unittest.cc +++ b/cast/common/channel/connection_namespace_handler_unittest.cc @@ -26,7 +26,6 @@ namespace openscreen::cast { namespace { using ::testing::_; -using ::testing::Invoke; using ::testing::NiceMock; using proto::CastMessage; @@ -93,7 +92,7 @@ class ConnectionNamespaceHandlerTest : public ::testing::Test { ON_CALL(vc_policy_, IsConnectionAllowed(_)) .WillByDefault( - Invoke([](const VirtualConnection& virtual_conn) { return true; })); + [](const VirtualConnection& virtual_conn) { return true; }); } protected: @@ -101,15 +100,15 @@ class ConnectionNamespaceHandlerTest : public ::testing::Test { const std::string& source_id, const std::string& destination_id) { EXPECT_CALL(*mock_client, OnMessage(_, _)) - .WillOnce(Invoke([&source_id, &destination_id](CastSocket* socket, - CastMessage message) { + .WillOnce([&source_id, &destination_id](CastSocket* socket, + CastMessage message) { VerifyConnectionMessage(message, source_id, destination_id); Json::Value value = ParseConnectionMessage(message); std::optional type = MaybeGetString( value, JSON_EXPAND_FIND_CONSTANT_ARGS(kMessageKeyType)); ASSERT_TRUE(type) << message.payload_utf8(); EXPECT_EQ(type.value(), kMessageTypeClose) << message.payload_utf8(); - })); + }); } void ExpectConnectedMessage( @@ -118,8 +117,8 @@ class ConnectionNamespaceHandlerTest : public ::testing::Test { const std::string& destination_id, std::optional version = std::nullopt) { EXPECT_CALL(*mock_client, OnMessage(_, _)) - .WillOnce(Invoke([&source_id, &destination_id, version]( - CastSocket* socket, CastMessage message) { + .WillOnce([&source_id, &destination_id, version](CastSocket* socket, + CastMessage message) { VerifyConnectionMessage(message, source_id, destination_id); Json::Value value = ParseConnectionMessage(message); std::optional type = MaybeGetString( @@ -134,7 +133,7 @@ class ConnectionNamespaceHandlerTest : public ::testing::Test { ASSERT_TRUE(message_version) << message.payload_utf8(); EXPECT_EQ(message_version.value(), version.value()); } - })); + }); } FakeCastSocketPair fake_cast_socket_pair_; @@ -161,8 +160,7 @@ TEST_F(ConnectionNamespaceHandlerTest, Connect) { TEST_F(ConnectionNamespaceHandlerTest, PolicyDeniesConnection) { EXPECT_CALL(vc_policy_, IsConnectionAllowed(_)) - .WillOnce( - Invoke([](const VirtualConnection& virtual_conn) { return false; })); + .WillOnce([](const VirtualConnection& virtual_conn) { return false; }); ExpectCloseMessage(&fake_cast_socket_pair_.mock_peer_client, receiver_id_, sender_id_); connection_namespace_handler_.OnMessage( diff --git a/cast/common/channel/message_util.cc b/cast/common/channel/message_util.cc index 53590d343..0e5eeb351 100644 --- a/cast/common/channel/message_util.cc +++ b/cast/common/channel/message_util.cc @@ -169,32 +169,36 @@ std::string ToString(const CastMessage& message) { return ss.str(); } -constexpr EnumNameTable kCastMessageTypeNames{ - {{"PING", CastMessageType::kPing}, +constexpr EnumNameTable kCastMessageTypeNames{ + {{"UNKNOWN", CastMessageType::kUnknown}, + {"PING", CastMessageType::kPing}, {"PONG", CastMessageType::kPong}, {"RPC", CastMessageType::kRpc}, - {"GET_APP_AVAILABILITY", CastMessageType::kGetAppAvailability}, - {"GET_STATUS", CastMessageType::kGetStatus}, {"CONNECT", CastMessageType::kConnect}, {"CLOSE", CastMessageType::kCloseConnection}, {"APPLICATION_BROADCAST", CastMessageType::kBroadcast}, {"LAUNCH", CastMessageType::kLaunch}, + {"LAUNCH_STATUS", CastMessageType::kLaunchStatus}, + {"LAUNCH_ERROR", CastMessageType::kLaunchError}, {"STOP", CastMessageType::kStop}, + {"INVALID_REQUEST", CastMessageType::kInvalidRequest}, + {"GET_APP_AVAILABILITY", CastMessageType::kGetAppAvailability}, {"RECEIVER_STATUS", CastMessageType::kReceiverStatus}, {"MEDIA_STATUS", CastMessageType::kMediaStatus}, - {"LAUNCH_ERROR", CastMessageType::kLaunchError}, {"OFFER", CastMessageType::kOffer}, {"ANSWER", CastMessageType::kAnswer}, + {"GET_CAPABILITIES", CastMessageType::kGetCapabilities}, {"CAPABILITIES_RESPONSE", CastMessageType::kCapabilitiesResponse}, + {"GET_STATUS", CastMessageType::kGetStatus}, {"STATUS_RESPONSE", CastMessageType::kStatusResponse}, - {"MULTIZONE_STATUS", CastMessageType::kMultizoneStatus}, {"INVALID_PLAYER_STATE", CastMessageType::kInvalidPlayerState}, - {"LOAD_FAILED", CastMessageType::kLoadFailed}, {"LOAD_CANCELLED", CastMessageType::kLoadCancelled}, - {"INVALID_REQUEST", CastMessageType::kInvalidRequest}, + {"LOAD_FAILED", CastMessageType::kLoadFailed}, + {"MULTIZONE_STATUS", CastMessageType::kMultizoneStatus}, {"PRESENTATION", CastMessageType::kPresentation}, - {"GET_CAPABILITIES", CastMessageType::kGetCapabilities}, - {"OTHER", CastMessageType::kOther}}}; + {"GET_DEVICE_INFO", CastMessageType::kGetDeviceInfo}, + {"eureka_info", CastMessageType::kEurekaInfo}, + {"INPUT", CastMessageType::kInput}}}; const char* CastMessageTypeToString(CastMessageType type) { return GetEnumName(kCastMessageTypeNames, type).value("OTHER"); diff --git a/cast/common/channel/message_util.h b/cast/common/channel/message_util.h index 4e71ac366..81d40b9f9 100644 --- a/cast/common/channel/message_util.h +++ b/cast/common/channel/message_util.h @@ -32,6 +32,9 @@ inline constexpr char kReceiverNamespace[] = inline constexpr char kBroadcastNamespace[] = "urn:x-cast:com.google.cast.broadcast"; inline constexpr char kMediaNamespace[] = "urn:x-cast:com.google.cast.media"; +static constexpr char kSetupNamespace[] = "urn:x-cast:com.google.cast.setup"; +static constexpr char kDiscoveryNamespace[] = + "urn:x-cast:com.google.cast.receiver.discovery"; // Sender and receiver IDs to use for platform messages. inline constexpr char kPlatformSenderId[] = "sender-0"; @@ -43,6 +46,14 @@ inline constexpr proto::CastMessage_ProtocolVersion kDefaultOutgoingMessageVersion = proto::CastMessage_ProtocolVersion_CASTV2_1_0; +// Default device capabilities reported in DeviceInfo messages. +// This value is a bitmask representing: +// CAP_VIDEO_OUT (0x1) | CAP_AUDIO_OUT (0x4) | CAP_MASTER_OR_FIXED_VOLUME +// (0x800) | CAP_ATTENUATION_OR_FIXED_VOLUME (0x1000) +// See +// https://developers.google.com/android/reference/com/google/android/gms/cast/CastDevice#constants +constexpr int kDefaultDeviceCapabilities = 6149; + // JSON message key strings. inline constexpr char kMessageKeyType[] = "type"; inline constexpr char kMessageKeyProtocolVersion[] = "protocolVersion"; @@ -90,6 +101,7 @@ inline constexpr char kMessageKeyStepInterval[] = "stepInterval"; inline constexpr char kMessageKeyUniversalAppId[] = "universalAppId"; inline constexpr char kMessageKeyUserEq[] = "userEq"; inline constexpr char kMessageKeyVolume[] = "volume"; +inline constexpr char kMessageKeyLaunchRequestId[] = "launchRequestId"; // JSON message field value strings specific to application control messages. inline constexpr char kMessageValueAttenuation[] = "attenuation"; @@ -98,56 +110,101 @@ inline constexpr char kMessageValueInvalidSessionId[] = "INVALID_SESSION_ID"; inline constexpr char kMessageValueInvalidCommand[] = "INVALID_COMMAND"; inline constexpr char kMessageValueNotFound[] = "NOT_FOUND"; inline constexpr char kMessageValueSystemError[] = "SYSTEM_ERROR"; - +inline constexpr char kMessageValueUserAllowed[] = "USER_ALLOWED"; + +// JSON message key strings specific to DEVICE_INFO messages. +inline constexpr char kMessageKeyControlNotifications[] = + "controlNotifications"; +inline constexpr char kMessageKeyDeviceCapabilities[] = "deviceCapabilities"; +inline constexpr char kMessageKeyDeviceId[] = "deviceId"; +inline constexpr char kMessageKeyDeviceModel[] = "deviceModel"; +inline constexpr char kMessageKeyFriendlyName[] = "friendlyName"; + +// JSON message key strings specific to eureka_info messages. +inline constexpr char kMessageKeyEurekaInfoRequestId[] = "request_id"; +inline constexpr char kMessageKeyData[] = "data"; +inline constexpr char kMessageKeyDeviceInfo[] = "device_info"; +inline constexpr char kMessageKeyManufacturer[] = "manufacturer"; +inline constexpr char kMessageKeyProductName[] = "product_name"; +inline constexpr char kMessageKeySsdpUdn[] = "ssdp_udn"; +inline constexpr char kMessageKeyBuildInfo[] = "build_info"; +inline constexpr char kMessageKeyBuildType[] = "build_type"; +inline constexpr char kMessageKeyCastBuildRevision[] = "cast_build_revision"; +inline constexpr char kMessageKeySystemBuildNumber[] = "system_build_number"; +inline constexpr char kMessageKeyResponseCode[] = "response_code"; +inline constexpr char kMessageKeyResponseString[] = "response_string"; + +// Represents known types of cast messages, intended to be used on the wire in +// JSON messages as a serialized string. Values are subject to change and should +// not be depended upon statically. +// +// For more information, see the section on "Messages and Namespaces" in +// ../../docs/streaming_session_protocol.md. enum class CastMessageType { + kUnknown = 0, + // Heartbeat messages. - kPing, - kPong, + kPing = 1, + kPong = 2, // RPC control/status messages used by Media Remoting. These occur at high // frequency, up to dozens per second at times, and should not be logged. - kRpc, - - kGetAppAvailability, - kGetStatus, + kRpc = 3, - // Virtual connection request. - kConnect, - - // Close virtual connection. - kCloseConnection, + // Virtual connection open and close requests. + kConnect = 4, + kCloseConnection = 5, // Application broadcast / precache. - kBroadcast, + kBroadcast = 6, // Session launch request. - kLaunch, + kLaunch = 7, + + // Launch request succeeded. + kLaunchStatus = 8, + + // Indicates that the launch request failed. + kLaunchError = 9, // Session stop request. - kStop, - - kReceiverStatus, - kMediaStatus, - - // Error from receiver. - kLaunchError, - - kOffer, - kAnswer, - kCapabilitiesResponse, - kStatusResponse, - - // The following values are part of the protocol but are not currently used. - kMultizoneStatus, - kInvalidPlayerState, - kLoadFailed, - kLoadCancelled, - kInvalidRequest, - kPresentation, - kGetCapabilities, - - kOther, // Add new types above `kOther`. - kMaxValue = kOther, + kStop = 10, + + // Sent by the receiver whenever an invalid request is received. + kInvalidRequest = 11, + + // Request and reply for information about what applications are available. + kGetAppAvailability = 12, + + // Reply with information about the current state of the receiver. + kReceiverStatus = 13, + kMediaStatus = 14, + + // Used for starting and negotiating a streaming session. + kOffer = 15, + kAnswer = 16, + kGetCapabilities = 17, + kCapabilitiesResponse = 18, + kGetStatus = 19, + kStatusResponse = 20, + + // The following values are part of the protocol but are not currently + // supported anywhere. + kInvalidPlayerState = 21, + kLoadCancelled = 22, + kLoadFailed = 23, + kMultizoneStatus = 24, + kPresentation = 25, + + // Necessary for setup, request for device information. + kGetDeviceInfo = 26, + kEurekaInfo = 27, + + // Input event messages. + kInput = 28, + + // Max value should be updated to the highest value of the enum. + kMaxValue = kInput, }; enum class AppAvailabilityResult { diff --git a/cast/common/channel/namespace_router_unittest.cc b/cast/common/channel/namespace_router_unittest.cc index f14baaea3..059792014 100644 --- a/cast/common/channel/namespace_router_unittest.cc +++ b/cast/common/channel/namespace_router_unittest.cc @@ -19,7 +19,6 @@ namespace { using proto::CastMessage; using ::testing::_; -using ::testing::Invoke; class NamespaceRouterTest : public ::testing::Test { public: @@ -50,15 +49,14 @@ TEST_F(NamespaceRouterTest, MultipleHandlers) { EXPECT_CALL(media_handler, OnMessage(_, _, _)).Times(0); EXPECT_CALL(auth_handler, OnMessage(_, _, _)) - .WillOnce(Invoke([](VirtualConnectionRouter* router, CastSocket*, - CastMessage message) { - EXPECT_EQ(message.namespace_(), "auth"); - })); + .WillOnce( + [](VirtualConnectionRouter* router, CastSocket*, + CastMessage message) { EXPECT_EQ(message.namespace_(), "auth"); }); EXPECT_CALL(connection_handler, OnMessage(_, _, _)) - .WillOnce(Invoke([](VirtualConnectionRouter* router, CastSocket*, - CastMessage message) { + .WillOnce([](VirtualConnectionRouter* router, CastSocket*, + CastMessage message) { EXPECT_EQ(message.namespace_(), "connection"); - })); + }); CastMessage auth_message; auth_message.set_namespace_("auth"); @@ -80,9 +78,9 @@ TEST_F(NamespaceRouterTest, RemoveHandler) { EXPECT_CALL(handler1, OnMessage(_, _, _)).Times(0); EXPECT_CALL(handler2, OnMessage(_, _, _)) - .WillOnce(Invoke( + .WillOnce( [](VirtualConnectionRouter* router, CastSocket* socket, - CastMessage message) { EXPECT_EQ("two", message.namespace_()); })); + CastMessage message) { EXPECT_EQ("two", message.namespace_()); }); CastMessage message1; message1.set_namespace_("one"); diff --git a/cast/common/channel/testing/fake_cast_socket.h b/cast/common/channel/testing/fake_cast_socket.h index edf726358..37d3f479b 100644 --- a/cast/common/channel/testing/fake_cast_socket.h +++ b/cast/common/channel/testing/fake_cast_socket.h @@ -62,7 +62,6 @@ struct FakeCastSocketPair { const IPEndpoint& remote_endpoint) : local_endpoint(local_endpoint), remote_endpoint(remote_endpoint) { using ::testing::_; - using ::testing::Invoke; auto moved_connection = std::make_unique<::testing::NiceMock>( @@ -77,15 +76,14 @@ struct FakeCastSocketPair { peer_socket = std::make_unique(std::move(moved_peer), &mock_peer_client); - ON_CALL(*connection, Send(_)).WillByDefault(Invoke([this](ByteView data) { - peer_connection->OnRead(std::vector(data.cbegin(), data.cend())); + ON_CALL(*connection, Send(_)).WillByDefault([this](ByteView data) { + peer_connection->OnRead(std::vector(data.begin(), data.end())); return true; - })); - ON_CALL(*peer_connection, Send(_)) - .WillByDefault(Invoke([this](ByteView data) { - connection->OnRead(std::vector(data.cbegin(), data.cend())); - return true; - })); + }); + ON_CALL(*peer_connection, Send(_)).WillByDefault([this](ByteView data) { + connection->OnRead(std::vector(data.begin(), data.end())); + return true; + }); } ~FakeCastSocketPair() = default; diff --git a/cast/common/channel/virtual_connection_router_unittest.cc b/cast/common/channel/virtual_connection_router_unittest.cc index a21cb8c71..03c372087 100644 --- a/cast/common/channel/virtual_connection_router_unittest.cc +++ b/cast/common/channel/virtual_connection_router_unittest.cc @@ -33,7 +33,6 @@ static_assert(proto::CastMessage_ProtocolVersion_CASTV2_1_3 == using proto::CastMessage; using ::testing::_; -using ::testing::Invoke; using ::testing::SaveArg; using ::testing::WithArg; @@ -232,12 +231,11 @@ TEST_F(VirtualConnectionRouterTest, SendMessage) { ASSERT_TRUE(message.IsInitialized()); EXPECT_CALL(destination, OnMessage(&remote_router_, remote_socket_, _)) - .WillOnce( - WithArg<2>(Invoke([&message](CastMessage message_at_destination) { - ASSERT_TRUE(message_at_destination.IsInitialized()); - EXPECT_EQ(message.SerializeAsString(), - message_at_destination.SerializeAsString()); - }))); + .WillOnce(WithArg<2>([&message](CastMessage message_at_destination) { + ASSERT_TRUE(message_at_destination.IsInitialized()); + EXPECT_EQ(message.SerializeAsString(), + message_at_destination.SerializeAsString()); + })); local_router_.Send(VirtualConnection{"receiver-1234", "sender-4321", local_socket_->socket_id()}, message); diff --git a/cast/common/public/cast_socket.h b/cast/common/public/cast_socket.h index 663eb2e1b..2d0877ec6 100644 --- a/cast/common/public/cast_socket.h +++ b/cast/common/public/cast_socket.h @@ -20,7 +20,7 @@ class CastMessage; // Represents a simple message-oriented socket for communicating with the Cast // V2 protocol. It isn't thread-safe, so it should only be used on the same // TaskRunner thread as its TlsConnection. -class CastSocket : public TlsConnection::Client { +class CastSocket : public Connection::Client { public: class Client { public: @@ -33,7 +33,7 @@ class CastSocket : public TlsConnection::Client { virtual ~Client(); }; - CastSocket(std::unique_ptr connection, Client* client); + CastSocket(std::unique_ptr connection, Client* client); ~CastSocket(); // Sends `message` immediately unless the underlying TLS connection is @@ -56,9 +56,9 @@ class CastSocket : public TlsConnection::Client { void set_audio_only(bool audio_only) { audio_only_ = audio_only; } bool audio_only() const { return audio_only_; } - // TlsConnection::Client overrides. - void OnError(TlsConnection* connection, const Error& error) override; - void OnRead(TlsConnection* connection, std::vector block) override; + // Connection::Client overrides. + void OnError(Connection* connection, const Error& error) override; + void OnRead(Connection* connection, std::vector block) override; WeakPtr GetWeakPtr() const { return weak_factory_.GetWeakPtr(); } @@ -70,7 +70,7 @@ class CastSocket : public TlsConnection::Client { static int g_next_socket_id_; - const std::unique_ptr connection_; + const std::unique_ptr connection_; Client* client_; // May never be null. const int socket_id_; bool audio_only_ = false; diff --git a/cast/common/public/cast_streaming_app_ids.cc b/cast/common/public/cast_streaming_app_ids.cc index 44f0c61d7..9fa7bce7e 100644 --- a/cast/common/public/cast_streaming_app_ids.cc +++ b/cast/common/public/cast_streaming_app_ids.cc @@ -5,7 +5,6 @@ #include "cast/common/public/cast_streaming_app_ids.h" #include - #include "util/std_util.h" #include "util/string_util.h" @@ -110,22 +109,22 @@ constexpr std::array } // namespace -bool IsCastStreamingAppId(const std::string& app_id) { +bool IsCastStreamingAppId(std::string_view app_id) { return IsCastStreamingAudioOnlyAppId(app_id) || IsCastStreamingAudioVideoAppId(app_id); } -bool IsCastStreamingAudioVideoAppId(const std::string& app_id) { +bool IsCastStreamingAudioVideoAppId(std::string_view app_id) { return string_util::EqualsIgnoreCase(app_id, GetCastStreamingAudioVideoAppId()); } -bool IsCastStreamingAudioOnlyAppId(const std::string& app_id) { +bool IsCastStreamingAudioOnlyAppId(std::string_view app_id) { return string_util::EqualsIgnoreCase(app_id, GetCastStreamingAudioOnlyAppId()); } -bool IsCastStreamingReceiverAppId(const std::string& app_id) { +bool IsCastStreamingReceiverAppId(std::string_view app_id) { if (string_util::EqualsIgnoreCase(app_id, GetCastStreamingAudioVideoAppId()) || string_util::EqualsIgnoreCase(app_id, GetCastStreamingAudioOnlyAppId()) || @@ -141,7 +140,7 @@ bool IsCastStreamingReceiverAppId(const std::string& app_id) { } return ContainsIf(kRemoteDisplayAppStreamingAudioVideoAppIds, - [app_id](const std::string& id) { + [app_id](std::string_view id) { return string_util::EqualsIgnoreCase(id, app_id); }); } diff --git a/cast/common/public/cast_streaming_app_ids.h b/cast/common/public/cast_streaming_app_ids.h index 235f90954..8dd8a3429 100644 --- a/cast/common/public/cast_streaming_app_ids.h +++ b/cast/common/public/cast_streaming_app_ids.h @@ -6,21 +6,19 @@ #define CAST_COMMON_PUBLIC_CAST_STREAMING_APP_IDS_H_ #include +#include #include namespace openscreen::cast { // Returns true only if `app_id` matches the Cast application ID for the // corresponding Chromium Cast Streaming receiver application. -// -// TODO(b/204583004): Use std::string_view instead of std::string following the -// move to C++17 so that these comparisons can be made constexpr. -bool IsCastStreamingAppId(const std::string& app_id); -bool IsCastStreamingAudioVideoAppId(const std::string& app_id); -bool IsCastStreamingAudioOnlyAppId(const std::string& app_id); +bool IsCastStreamingAppId(std::string_view app_id); +bool IsCastStreamingAudioVideoAppId(std::string_view app_id); +bool IsCastStreamingAudioOnlyAppId(std::string_view app_id); // Returns true only if `app_id` matches any Cast Streaming app ID. -bool IsCastStreamingReceiverAppId(const std::string& app_id); +bool IsCastStreamingReceiverAppId(std::string_view app_id); // Returns all app IDs for Cast Streaming receivers. std::vector GetCastStreamingAppIds(); diff --git a/cast/common/public/receiver_info.cc b/cast/common/public/receiver_info.cc index 54a7f6fa6..e98df1546 100644 --- a/cast/common/public/receiver_info.cc +++ b/cast/common/public/receiver_info.cc @@ -12,7 +12,6 @@ #include "discovery/mdns/public/mdns_constants.h" #include "util/osp_logging.h" -#include "util/span_util.h" #include "util/string_parse.h" namespace openscreen::cast { diff --git a/cast/docs/USING.md b/cast/docs/USING.md new file mode 100644 index 000000000..00211edda --- /dev/null +++ b/cast/docs/USING.md @@ -0,0 +1,186 @@ +# Using the Standalone Sender and Receiver + +## 1. Configure your machine + +First, follow the setup steps in the root [README.md](../../README.md). + +If you want to actually play back content on the Open Screen receiver, you +must make sure the FFmpeg and libSDL2 dependencies are properly configured. +Otherwise, the dummy player will be used and you will only get log statements. + +If you want to use the cast sender binary at *all*, `libopus` and `libvpx` at +minimum are required to encode frames. + +Instructions for setting up these dependencies are provided in +[external_libraries.md](external_libraries.md). + +## 2. Build your binaries + +Once your GN configuration is happy, you just need to compile your binaries. +For example, if you used `out/Default` as your build directory: + +```sh +autoninja -C out/Default cast_sender cast_receiver +``` + +> Both the `cast_sender` and `cast_receiver` binaries have reasonably thorough +> help documentation available by invoking them with the `-h` flag. + +## 3. Generate a developer certificate + +To use the sender and receiver application together, a valid Cast certificate is +required. The easiest way to generate the certificate is to just run the +`cast_receiver` with `-g`, and both should be written out to files: + +```sh +$ ./out/Default/cast_receiver -g + [INFO:../../cast/receiver/channel/static_credentials.cc(161):T0] Generated new private key for session: ./generated_root_cast_receiver.key + [INFO:../../cast/receiver/channel/static_credentials.cc(169):T0] Generated new root certificate for session: ./generated_root_cast_receiver.crt +``` + +## 4. Start a `cast_receiver` session + +After you have a successful build working, next step is to start a test receiver +session. + +First, you need to find a valid interface address to bind the `cast_receiver` +to. There are many tools available for this task, such as `ifconfig`: + +```sh +$ ifconfig +eth0: flags=4163 mtu 1500 + inet 192.168.1.10 netmask 255.255.255.0 broadcast 192.168.1.255 + inet6 fe80::abcd:ef01:2345:6789%eth0 prefixlen 64 scopeid 0x20 + ether 00:1a:2b:3c:4d:5e txqueuelen 1000 (Ethernet) + RX packets 92800 errors 0 dropped 0 overruns 0 frame 0 + TX packets 93810 errors 0 dropped 0 overruns 0 carrier 0 collisions 0 + collisions 0 + +lo: flags=73 mtu 65536 + inet 127.0.0.1 netmask 255.0.0.0 + inet6 ::1 prefixlen 128 scopeid 0x10 + loop txqueuelen 1000 (Local Loopback) + RX packets 100 errors 0 dropped 0 overruns 0 frame 0 + TX packets 100 errors 0 dropped 0 overruns 0 carrier 0 collisions 0 + collisions 0 +``` + +Pay attention to the `inet` address associated with your preferred interface +if you are connecting with the `cast_sender` binary -- you may need it to +connect later. + +Once you have your key, certificate, and network interface, you should be ready +to start your `cast_receiver` binary. That should look something like this: + +```sh +./out/Default/cast_receiver -d generated_root_cast_receiver.crt -p generated_root_cast_receiver.key eth0 +``` + +Pay attention to the command output as it runs -- it should prompt with any +breaking errors. + +### Note on discovery and macOS + +Currently, the discovery component only works on Linux, as we have not written +a Bonjour integration for macOS. When running the `cast_receiver` on macOS, +you must disable mDNS using the `-x` flag. Note that this means that you +**cannot connect to `cast_receiver` instances on macOS using discovery**. + +## 5. Connecting with a libcast sender + +### 5a. Option 1: Using the discovery component + +Assuming your `cast_receiver` instance has discovery enabled (true by default), +you can connect to it using the cast sender and specifying the network interface +and media file. It should then prompt you with a list of discovered receivers, +with indexes that allow you to connect to a specific receiver. + +Example invocation: + +```sh +$ ./out/Default/cast_sender -d generated_root_cast_receiver.crt eth0 ~/file_example_MP4_1920_18MG.mp4 +[INFO:../../cast/standalone_sender/main.cc(198):T0] using cast trust store generated from: generated_root_cast_receiver.crt +[INFO:../../cast/standalone_sender/receiver_chooser.cc(41):T0] Starting discovery. Note that it can take dozens of seconds to detect anything on some networks! +[INFO:../../cast/standalone_sender/receiver_chooser.cc(71):T0] Discovered: Cast Standalone Receiver (id: cast_standalone_rece-4201ac134af2) + +[0]: Cast Standalone Receiver @ 172.19.74.242:8010 + +Enter choice, or 'n' to wait longer: 0 +[INFO:../../cast/standalone_sender/looping_file_cast_agent.cc(84):T0] Launching Mirroring App on the Cast Receiver... +[INFO:../../cast/standalone_sender/looping_file_cast_agent.cc(235):T0] Starting-up message routing to the Cast Receiver's Mirroring App (sessionId=streaming_receiver-10000)... +[INFO:../../cast/standalone_sender/looping_file_cast_agent.cc(249):T0] Starting streaming session... +... +``` + +### 5b. Option 2: Using a direct connection + +If you want to connect directly to the receiver instead of using discovery, you +can pass the receiver's network address instead of a network interface: + +```sh +./out/Default/cast_sender -d generated_root_cast_receiver.crt -c vp9 172.19.74.242:8010 ~/file_example_MP4_1920_18MG.mp4 +``` + +### Specifying the video codec + +To determine which video codec to use, the `-c` or `--codec` flag should be +passed. Currently supported values include `vp8`, `vp9`, and `av1`: + +```bash +./out/Default/cast_sender -d generated_root_cast_receiver.crt -r 127.0.0.1 -c av1 ~/video-1080-mp4.mp4 +``` + +## 6. Connecting with a Chrome sender + +In some cases, it is desirable to directly test the Open Screen receiver using +Chrome as a receiver. This is a basic starter guide on this workflow. + +### Motivation + +Frequently, it can be difficult to test directly against a real, live Chromecast +device. For example, developing on a cloud-based machine or on a network with +tight access controls. Using Open Screen as a receiver allows development to +occur in these environments. + +Also, sometimes you just wanna test the Open Screen receiver and make sure all +of the flows are working. + +### Warning: **Linux required ahead** + +You **must be running the cast receiver on a Linux device**. The libcast +standalone receiver does not have the ability to do discovery on macOS due to +the discovery code not being configured to work with Bonjour (yet). Other +platforms, like Windows, have the potential for support in the future but are +currently missing dependencies. + +### Connecting with Chrome + +Assuming you have followed the above steps and your `cast_receiver` binary +is happily awaiting connections on a TLS server socket, all you have to do now +is connect with Chrome. + +The necessary flag is present on Chrome stable and does not require a debug +build. Any recent build of Chrome should work out of the box. In order to +connect, you just need to start Chrome from the command line, specifying the +**fully qualified path** to the developer certificate generated by the test +receiver like so: + +```sh +/path/to/chrome/chrome --cast-developer-certificate-path=/fully/qualified/path/to/cert/generated_root_cast_receiver.crt +``` + +**Note on finding Chrome:** The path to the Chrome executable varies by platform +and channel. On Linux, you can often use `which google-chrome-stable`. On macOS, +the path is typically +`/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`. + +> Chrome may fail to resolve the test certificate if you do not specify the +> **fully qualified path**! +> + +## The `standalone_e2e.py` script: a helpful reference + +If you have any issues with the above documentation, or just want to see an +example usage of it, the [`standalone_e2e.py`](../standalone_e2e.py) script +exercises the Cast Sender sender and receiver through some basic end to +end test scenarios, using the same steps as above. diff --git a/cast/docs/architecture.md b/cast/docs/architecture.md new file mode 100644 index 000000000..0c82e4ddb --- /dev/null +++ b/cast/docs/architecture.md @@ -0,0 +1,154 @@ +# libcast High-Level Architecture + +## Introduction + +The `libcast` library implements the core Cast protocols, enabling discovery, +application control, and real-time media streaming between sender and receiver +devices. The library is designed to be modular and portable, relying on a +platform abstraction layer for easy integration into various applications and +operating systems. + +[TOC] + +## `libcast` vs. Host Application Responsibilities + +One of the more complex elements of working with libcast is the division of +responsibilities between libcast and the hosting application. While the host +application is the consumer of libcast, taking outputted frames and rendering +media in its own user interface, it also plays the role of provider, being +responsible for implementing the platform API so that libcast can access lower +level functionality, such as logs, the network stack, and tracing features. + +![libcast vs. Host Application Responsibilities](images/architecture_responsibilities.png) + +- **Host Application:** This is the code you write. It includes the main + business logic of your application (e.g., the UI of a receiver app) and the + code that handles media playback (e.g., an SDL-based video player on a + receiver). Crucially, it provides the concrete _implementations_ for the + Platform API (like networking) and is responsible for publishing the receiver + to the network. + +- **`libcast` Library:** This is the core Open Screen component. It manages the + CastV2 protocol, session negotiation, and streaming transport. It defines the + _interfaces_ for the Platform API that the host application must implement. + +## Core Components + +The libcast library is primarily composed of four major components: the Sender, +the Receiver, the Streaming component. Note that libcast take some dependencies +on other areas of Open Screen, such as the [Platform](../../platform/README.md) +API, utilities in the `util/` folder, etc. + +### Sender Component + +The Sender is the client component that initiates communication. Its primary +roles are: + +- **Session Management:** Establishing a secure connection to a receiver, + launching applications, and managing the application lifecycle. + +- **Media Control:** `libcast` does not directly implement media playback + control. Support for controlling playback during remoting sessions is handled + by the application sending messages from the + [remoting.proto](../streaming/remoting.proto) Protobuf schema. Playback + control of streaming sessions is not currently defined, and play, pause, and + seek messages are ignored. Client applications desiring to send custom media + control messages can implement this using `libcast`'s + [`CastSocket`](../common/public/cast_socket.h) API. + +### Receiver Component + +The `libcast` Receiver is the server component running on a Cast device. It is +not responsible for media playback or network discovery, but rather for managing +the Cast session and the lifecycle of the Receiver Application. Its primary +roles are: + +- **Connection Handling:** Accepting connections from senders and managing the + secure CastV2 channel. + +- **Application Management:** Launching and terminating the Receiver Application + based on requests from the sender. It acts as a bridge between the sender and + the Receiver Application. + +*** note +One of the more confusing things about the layout of the `cast/` +folder is that the `receiver/` and `sender/` subfolders do not actually contain +the Cast `Receiver` and `Sender` classes, since these classes are actually part +of the `streaming/` component. + +The receiver component actually contains a pretty small subset of the castv2 +logic, most notably the [`ApplicationAgent`](../receiver/application_agent.h) +and some channel code. +*** + +### Streaming Component + +The Streaming component manages the real-time transport of media. It is used by +both the Sender (for packetizing and sending) and the Receiver (for receiving +and reconstructing the media stream). + +- **Session Negotiation:** Implements the offer/answer protocol (similar to + WebRTC) to negotiate media formats, codecs, and network parameters. + +- **Media Transport:** Manages the RTP/RTCP sessions for efficient and + synchronized delivery of audio and video frames. + +- **Encoding/Decoding Negotiation:** Provides the framework for negotiating + hardware or software codecs, but does not implement the codecs themselves. + +*** aside +For more information, see the streaming [README.md](../streaming/README.md). +*** + +## Extended Topics + +### Libcast and Its Use of The Platform API + +The Platform API is a crucial abstraction layer that decouples the core +`libcast` logic from any specific operating system or hardware. It defines a set +of interfaces that the embedding application must implement. + +- **Responsibilities:** Provides platform-specific implementations for + networking (sockets, mDNS), threading (clocks, tasks), and logging. + +- **Portability:** By requiring the host application to provide these + implementations, `libcast` remains highly portable across different + environments, from embedded devices to desktop applications. + +### Hosting the Library + +The `libcast` Sender and Receiver are library components, not standalone +executables. A developer must write a host application that integrates and +"hosts" the library. This host application is what gets compiled into the final +`sender` or `receiver` binary. The host application's primary responsibilities +from libcast's perspective are: + +1. **Platform Initialization:** Setting up the platform-specific implementations + (networking, threading, etc.) required by the Platform API. In the Open + Screen codebase, this is handled by creating an `Environment`. + +2. **Component Instantiation:** Creating an instance of the `libcast` Sender or + Receiver component. + +3. **Execution:** Starting the component and running the main event loop (or + task runner) that drives all asynchronous operations. + +4. **Service Publication (for Receivers):** For a receiver to be discoverable, + the host application must publish its network presence. This involves + creating a `ReceiverInfo` record with the device's details (e.g., friendly + name, port), converting it to a `DnsSdInstance`, and registering it with the + DNS-SD service. The `standalone_receiver/cast_service.cc` file is the + reference example of this application flow. + +5. **Discovery (for Senders)** Finding Cast-enabled receivers on the local + network using mDNS/DNS-SD. Libcast does not directly depend on discovery, and + the host application is responsible for implementing it, potentially by using + Open Screen's [discovery/](../../discovery/README.md) implementation. The + standalone sender and receiver both integrate this discovery functionality. + +For a receiver, the host application instantiates the `libcast` Receiver, which +then listens for connections. When a LAUNCH request arrives, the `libcast` +Receiver then launches the separate Receiver Application logic. + +The `standalone_sender/main.cc` and `standalone_receiver/main.cc` files are +reference examples of how to write such a host application. diff --git a/cast/docs/architecture_responsibilities.dot b/cast/docs/architecture_responsibilities.dot new file mode 100644 index 000000000..4dd3a0170 --- /dev/null +++ b/cast/docs/architecture_responsibilities.dot @@ -0,0 +1,79 @@ +digraph { + graph [ + bgcolor=transparent, + fontname="sans-serif", + fontsize=15, + ]; + + node [shape=box,fontname="sans-serif"] + edge [fontname="sans-serif", fontsize=9]; + + // Use a single, invisible cluster to force vertical alignment + subgraph cluster_main_stack { + style=invis; + + // Sender components at the top + subgraph cluster_host_sender { + label="Host Sender Application"; + style=dashed; + bgcolor="#FBBC04BB"; + + sender_app [label="Sender Application\n(UI, etc.)"]; + + subgraph cluster_libcast_sender { + label="libcast Library"; + style=solid; + bgcolor="#4285F4CC"; + + libcast_sender [label="libcast Sender"]; + streaming_sender [label="Streaming Component"]; + platform_api_sender [label="Platform API"]; + } + + platform_impl_sender [label="Platform API Implementation"]; + } + + // Receiver components at the bottom + subgraph cluster_host_receiver { + label="Host Receiver Application"; + style=dashed; + bgcolor="#34A853BB"; + + { rank=min; receiver_app; } // Force receiver_app to the top of this subgraph + receiver_app [label="Receiver Application\n(Media Player)"]; + + subgraph cluster_libcast_receiver { + label="libcast Library"; + style=solid; + bgcolor="#4285F4CC"; + + libcast_receiver [label="libcast Receiver"]; + streaming_receiver [label="Streaming Component"]; + platform_api_receiver [label="Platform API"]; + } + // Put dns_sd and platform_impl_receiver on the same rank below the libcast cluster + { rank=same; dns_sd; platform_impl_receiver; } + dns_sd [label="DNS-SD Publisher"]; + platform_impl_receiver [label="Platform API Implementation"]; + } + } + + // Sender Side + sender_app -> libcast_sender [label=" Owns & Calls "]; + platform_api_sender -> platform_impl_sender [dir=back, style=dashed, label="Implemented By"]; + libcast_sender -> platform_api_sender [label="Uses", color="#333333"]; + libcast_sender -> streaming_sender [label="Uses", color="#333333"]; + streaming_sender -> platform_api_sender [label="Uses", color="#333333"]; + + // Receiver Side + dns_sd -> libcast_receiver [label="Publishes"]; + libcast_receiver -> receiver_app [label="Launches & Forwards Stream"]; + platform_api_receiver -> platform_impl_receiver [dir=back, style=dashed, label="Implemented By"]; + libcast_receiver -> platform_api_receiver [label="Uses", color="#333333"]; + receiver_app -> streaming_receiver [dir=back, label="Receives From"]; + libcast_receiver -> streaming_receiver [label="Uses", color="#333333"]; + streaming_receiver -> platform_api_receiver [label="Uses", color="#333333"]; + + // Network Connection + libcast_sender -> libcast_receiver [style=dotted, label="Network"]; +} diff --git a/cast/docs/dscp.md b/cast/docs/dscp.md new file mode 100644 index 000000000..b41a1936d --- /dev/null +++ b/cast/docs/dscp.md @@ -0,0 +1,69 @@ +# Differentiated Services Code Point (DSCP) for Cast Streaming + +This document describes how to use Differentiated Services Code Point (DSCP) +support in the Open Screen Library. + +## Overview + +DSCP is a mechanism for classifying network traffic to provide Quality of +Service (QoS). By marking packets with a DSCP value, network routers can +prioritize traffic, which is useful for latency-sensitive applications like +video streaming. + +The Open Screen Library supports setting DSCP values on RTP and RTCP packets for +Cast Streaming sessions. This is based on recommendations from [RFC +8837](https://datatracker.ietf.org/doc/html/rfc8837), where video streams are +marked with `AF41` and audio-only streams with `EF`. + +DSCP may improve performance in some network environments, as long as the DSCP +field is not stripped by the operating system or router. This is in line with +other network streaming implementations, like WebRTC's +[RTCRtpEncodingParameters](https://www.w3.org/TR/webrtc/#dom-rtcrtpencodingparameters) +concept of `priority` and `networkPriority`. + +DSCP support in libcast is part of a larger effort to prioritize Cast traffic +on congested networks and improve performance even in the worst conditions +for streaming. + +## How to Enable DSCP + +DSCP is disabled by default. To enable it, set the `enable_dscp` flag to `true` +in the session configuration for both the sender and the receiver. + +- **Sender:** Set `enable_dscp` in `SenderSession::Configuration`. +- **Receiver:** Set `enable_dscp` in `ReceiverConstraints`. + +If both the sender and receiver have DSCP enabled, the sender session will +set the spec-recommended DSCP value in the OFFER message, and then both +the sender and receiver should set the DSCP value on their respective +UdpSockets. + +## Setting DSCP Values + +The `UdpSocket::SetDscp()` method is used to set the DSCP value for a socket. It +takes a `DscpMode` enum value, backed by a `uint8_t` that is valid for values +of 0-63. + +```cpp +void UdpSocket::SetDscp(DscpMode mode)); +``` + +While any valid 6-bit integer can be passed by using a `static_cast`, the +`UdpSocket::DscpMode` enum in `platform/api/udp_socket.h` provides a convenient +way to use some of the more common DSCP values: + +```cpp +enum class UdpSocket::DscpMode : uint8_t { + kBestEffort = 0, + kAF11 = 10, + // ... + kEF = 46 +}; + +// Example usage: +// Non-standard DSCP value. +socket->SetDscp(static_cast(55)); + +// Standard value. +socket->SetDscp(UdpSocket::DscpMode::kEF); +``` diff --git a/cast/docs/external_libraries.md b/cast/docs/external_libraries.md new file mode 100644 index 000000000..2a09c56fd --- /dev/null +++ b/cast/docs/external_libraries.md @@ -0,0 +1,149 @@ +# Standalone Sender and Receiver Dependencies + +Currently, external libraries are used exclusively by the standalone sender and +receiver applications, for compiling in dependencies used for video decoding and +playback. + +The decision to link external libraries is made manually by setting the GN args. + +> NOTE: The build currently defaults to using a sysroot on some platforms, such +> as Linux. You will likely find it helpful to disable the `use_sysroot` GN arg +> in order to properly use system libraries and avoid linker issues. + +For example, a developer wanting to link the necessary libraries for the +standalone sender and receiver executables might add the following to `gn args +out/Default`: + +```python +is_debug=true +have_ffmpeg=true +have_libsdl2=true +have_libopus=true +have_libvpx=true +use_sysroot=false +``` + +Or on the command line as: + +```bash +gn gen --args="is_debug=true have_ffmpeg=true have_libsdl2=true have_libopus=true have_libvpx=true use_sysroot=false" out/Default +``` + +## Linux + +As of ==December 22nd, 2025==, the below command installs all of the needed +external libraries on gLinux Rodete. Your mileage may vary on other Debian +based OSes. + +```sh +sudo apt install libsdl2-2.0-0 libsdl2-dev libavcodec61 libavcodec-dev libavformat61 libavformat-dev libavutil59 libavutil-dev libswresample5 libswresample-dev libopus0 libopus-dev libvpx11 libvpx-dev +``` + +Note: release of these operating systems may require slightly different +packages, so these `sh` commands are merely a potential starting point. + +Also note that generally the headers for packages must also be installed. +In Debian Linux flavors, this usually means that the `*-dev` version of each +package must also be installed. In the example above, this looks like having +both `libavcodec61` and `libavcodec-dev`. + +Finally, sometimes header resolution can fail. If that occurs for you, please +specific the header include dirs. For example, if your build complains about +`SDL2/SDL.h` and related headers missing, the GN argument to fix it may look +something like this (adjusted for your specific system): + +```sh +libsdl2_include_dirs = [ "/usr/include", "/usr/include/x64_64-linux-gnu" ] +``` + +A more extensive example is provided in the below +[Library specific include paths](#library-specific-include-paths) section. For a +full list of potential GN arguments, see +[external_libaries.gni](../streaming/external_libraries.gni). + +## MacOS (Homebrew) + +You can use [Homebrew](https://brew.sh/) to install the libraries needed to compile the +standalone sender and receiver applications. + +```sh +brew install ffmpeg sdl2 opus libvpx aom +``` + +To compile and link against these libraries, set the path arguments as follows +in your `gn args`. + +### Library specific include paths + +**Important**: Using Homebrew's top-level include directory +(`/opt/homebrew/include`) can cause header conflicts with the project's internal +BoringSSL. You must use the specific include paths for each library from +Homebrew's `Cellar` directory. + +Homebrew libraries are often built for the latest version of macOS. You may need +to set the `mac_deployment_target` to match the version of macOS you are running +to avoid linker errors. For example, on macOS Sonoma (version 14): + +You will need to replace the `` placeholders with the actual versions +installed on your system. You can find these in `/opt/homebrew/Cellar/`. + +```python +mac_deployment_target="14.0" + +have_ffmpeg=true +have_libsdl2=true +have_libopus=true +have_libvpx=true +have_libaom=true + +# Homebrew on Apple Silicon installs to /opt/homebrew. +# On Intel macs, it's /usr/local. +external_lib_dirs=["/opt/homebrew/lib"] + +ffmpeg_include_dirs=["/opt/homebrew/Cellar/ffmpeg//include"] +libsdl2_include_dirs=["/opt/homebrew/Cellar/sdl2//include"] +libopus_include_dirs=["/opt/homebrew/Cellar/opus//include"] +libvpx_include_dirs=["/opt/homebrew/Cellar/libvpx//include"] +libaom_include_dirs=["/opt/homebrew/Cellar/aom//include"] +``` + +## libaom + +For AV1 support, it is advised that most Linux users compile and install +`libaom` from source, using the instructions at +https://aomedia.googlesource.com/aom/ Older versions found in many package +management systems are not compatible with the Open Screen Library because of +API compatibility and performance issues. + +To to enable AV1 support, also add the following to your GN args: + +```python +have_libaom=true +``` + +Note that AV1 support is configured separately from the other standalone +libraries and the `have_libaom` flag is not necessary to run the standalone +demo. + +Similar to other libraries, you may need to set `libaom_include_dirs` to the +location of the libaom header files and `libaom_lib_dirs` to the location of +the linkable libaom libraries. + +## Standalone Sender + +The standalone sender uses `ffmpeg`, `libopus`, and `libvpx` for encoding video +and audio for sending. When the build has determined that +[have_external_libs](../standalone_sender/BUILD.gn) is set to true, meaning that +all of these libraries are installed, then the VP8 and Opus encoders are enabled +and actual video files can be sent to standalone receiver instances. Without +these dependencies, the standalone sender cannot properly function (contrasted +with the standalone receiver, which can use a dummy player). + +## Standalone Receiver + +The standalone receiver also uses `ffmpeg`, for decoding the video stream +encoded by the sender, and also uses `libsdl2` to create a surface for decoding +video. Unlike the sender, the standalone receiver can work without having its +[have_external_libs](../standalone_receiver/BUILD.gn) set to true, through the +use of its [Dummy Player](../standalone_receiver/dummy_player.h) that does not +perform any actual decoding or playback. diff --git a/cast/docs/frame_flow_tracing.md b/cast/docs/frame_flow_tracing.md new file mode 100644 index 000000000..058c3d43b --- /dev/null +++ b/cast/docs/frame_flow_tracing.md @@ -0,0 +1,129 @@ +# Frame Flow Tracing in Open Screen + +Open Screen utilizes **Flow Events** to track the lifecycle of audio and video +frames as they travel through the system. This allows developers to visualize +the journey of a single frame from generation (at the Sender) to presentation +(at the Receiver) across thread and process boundaries in the tracing tool of +the embedder's choice. + +*** aside +The concept of flows in Open Screen is inspired by Perfetto, which is +the default tracing tool of the Chromium project as well as other consumers of +libcast. Flows as a concept may or may not be applicable to a given tracing +tool. +*** + +## The Concept + +A "Flow" connects distinct trace events that share a common ID. In the context +of Cast Streaming, the natural identifier is the `FrameId`. By associating trace +events with the `FrameId`, we can visualize the latency breakdown, queueing +delays, and network transmission times for every frame. + +## The Flow Lifecycle + +### 1. Origin (Embedder Responsibility) + +The flow typically begins when a frame is captured or encoded. Since this +happens outside the Open Screen library (in the Embedder's code), the Embedder +is responsible for providing the capture timestamps. + +**Mechanism:** Provide `capture_begin_time` in `EncodedFrame`. + +The Open Screen `Sender` automatically emits a `TRACE_FLOW_BEGIN` event using +the provided `capture_begin_time` as the start timestamp. This ensures the flow +visually starts at the correct moment of capture, even if the frame is enqueued +later. + +```cpp +// Example: Inside your Encoder or Capture mechanism +EncodedFrame frame; +frame.capture_begin_time = my_capture_clock.now(); +// ... encode ... +sender->EnqueueFrame(frame); +// The library will emit TRACE_FLOW_BEGIN("Frame.Capture", frame_id, capture_begin_time) automatically. +``` + +### 2. Sender Library + +Once the frame is passed to the `Sender` class, the library emits steps to track +its progress within the sending queue and network stack. + +* **`Frame.Capture`**: The start of the flow, back-dated to `capture_begin_time` + provided by the Embedder. +* **`Frame.Capture.End`**: The end of capture (if `capture_end_time` provided). +* **`Frame.Encode`**: The completion of encoding (logged at Enqueue time). +* **`Frame.Enqueued`**: The frame has entered the Sender's processing queue. +* **`Frame.Acked`**: The Receiver has acknowledged receipt of this frame. +* **`Frame.Cancelled`**: The frame was dropped or replaced before it could be + fully sent/acked. + +### 3. Receiver Library + +When packets arrive at the `Receiver`, the library tracks the reassembly and +availability of the frame. + +* **`Frame.Complete`**: All packets for this frame have been received and + reassembled. The frame is valid and sitting in the network queue. + +* **`Frame.Ready`**: The frame is the next in sequence and has satisfied all + dependencies. It is ready for the application to consume. + +* **`Frame.Consumed`**: The application has popped the frame from the queue. + +### 4. Presentation (Embedder Responsibility) + +After the frame leaves the Receiver library, the Embedder (Player) takes over. +To complete the picture, Embedders should continue the flow through decoding and +rendering. + +**Macro:** `TRACE_FLOW_STEP` and `TRACE_FLOW_END` + +```cpp +// Example: Inside your Player's OnFramesReady() +EncodedFrame frame = receiver.ConsumeNextFrame(buffer); +// 'Frame.Consumed' is emitted by the library here. + +// 1. Embedder acknowledges receipt +TRACE_FLOW_STEP(TraceCategory::kReceiver, "Frame.Received", frame.frame_id); + +// ... Decoding ... + +// 2. Rendering Start +TRACE_FLOW_STEP(TraceCategory::kReceiver, "Frame.Render.Begin", frame.frame_id); + +// ... Rendering ... + +// 3. Rendering End +TRACE_FLOW_STEP(TraceCategory::kReceiver, "Frame.Render.End", frame.frame_id); + +// 4. Playout (Flow End) +// Call ReportPlayoutEvent() to notify the library. The library will emit +// TRACE_FLOW_END_WITH_TIME("Frame.PlayedOut", ..., playout_time). +receiver.ReportPlayoutEvent(frame.frame_id, frame.rtp_timestamp, playout_time); +``` + +## Debugging & Diagnostics + +By visualizing these flows in a tool like +[ui.perfetto.dev](https://ui.perfetto.dev/), you can diagnose: + +* **Sender Blocking**: Long gaps before `Frame.Enqueued`. + +* **Network Latency**: Time between `Frame.Enqueued` and `Frame.Complete` + (approximate). + +* **Receiver Queueing**: Time between `Frame.Complete` and `Frame.Ready`. + * *High duration here suggests the Receiver application is not pulling frames + fast enough or the jitter buffer is holding them.* + +* **Application Latency**: Time between `Frame.Ready` and `Frame.PlayedOut`. + +## Integration Tips + +1. **Use `FrameId`**: Ensure your flow IDs are derived from the `FrameId`. The + library uses `FrameId::value()` (casted to `uint64_t`) as the flow ID. + +2. **Category Consistency**: While flows can span categories, it is helpful to + use `TraceCategory::kSender` for sender-side events and + `TraceCategory::kReceiver` for receiver-side events. diff --git a/cast/docs/generate_images.sh b/cast/docs/generate_images.sh new file mode 100755 index 000000000..ab0eb8009 --- /dev/null +++ b/cast/docs/generate_images.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env sh +# Copyright 2026 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# This script finds all .dot files in the current directory and generates +# PNG images from them using the Graphviz 'dot' tool. Graphviz is generally +# considered in Chromium to be the most commonly used diagram generation tool. +# Other tools, such as Mermaid, could be added here for things like sequence +# diagram creation (which is an absolutely dreadful experience with Graphviz). +# +# Unfortunately, Chromium's (and thus Open Screen's) Gitiles renderer does not +# currently handle rendering markdown-based diagrams natively in documentation. +# See crbug.com/383566360. If this actually changes later, our documentation +# should likely take advantage of it. + +set -e + +DOCS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +OUTPUT_DIR="${DOCS_DIR}/images" + +echo "Generating diagrams..." +echo "Source directory: ${DOCS_DIR}" +echo "Output directory: ${OUTPUT_DIR}" + +# Create the output directory if it doesn't exist. +if [ ! -d "${OUTPUT_DIR}" ]; then + echo "Creating output directory: ${OUTPUT_DIR}" + mkdir -p "${OUTPUT_DIR}" +fi + +# Check if Graphviz is installed. +if ! command -v dot &> /dev/null +then + echo "Graphviz 'dot' command could not be found. Please install Graphviz." + exit 1 +fi + +# Find all .dot files and generate PNGs. +for dot_file in "${DOCS_DIR}"/*.dot; do + # Check if the file exists to avoid errors when no .dot files are found. + if [ -f "$dot_file" ]; then + base_name=$(basename "$dot_file" .dot) + output_file="${OUTPUT_DIR}/${base_name}.png" + echo " • Processing ${dot_file} -> ${output_file}" + dot -Tpng -Gdpi=300 -o "$output_file" "$dot_file" + fi +done + +echo "Diagram generation complete." diff --git a/cast/docs/images/architecture_responsibilities.png b/cast/docs/images/architecture_responsibilities.png new file mode 100644 index 000000000..81d716187 Binary files /dev/null and b/cast/docs/images/architecture_responsibilities.png differ diff --git a/cast/docs/input_demo.md b/cast/docs/input_demo.md new file mode 100644 index 000000000..5d7d2d06e --- /dev/null +++ b/cast/docs/input_demo.md @@ -0,0 +1,77 @@ +# Input Event API Demo Guide + +This document explains how to use the Input Event API demo in the standalone +Cast reference implementations. + +## Overview + +The Input Event API allows a Cast Receiver to send user interactions (mouse +clicks, keyboard events, etc.) back to the Cast Sender. This demo specifically +shows mouse click events being captured on the receiver and visualized on the +sender. + +## Prerequisites + +- **SDL2**: The standalone receiver requires SDL2 for window management and + input event capture. +- **FFmpeg/LibVPX/LibOpus**: Required for media encoding and decoding. + +## Building the Demo + +Ensure your GN args are configured for standalone build with external libraries. +Then build the following targets: + +```bash +autoninja -C out/Default cast_receiver cast_sender +``` + +## Running the Demo + +### 1. Start the Receiver + +Run the receiver on a networked machine (or your local machine using the loopback +interface). You **must** opt-in using the `--enable-input-events` flag. + +```bash +./out/Default/cast_receiver --enable-input-events +``` + +Replace `` with your network interface (e.g., `eth0`, `wlan0`, +or `lo` for local testing). + +### 2. Start the Sender + +Run the sender, pointing it to the receiver's address and a media file. Again, +the `--enable-input-events` flag is required. + +```bash +./out/Default/cast_sender --enable-input-events +``` + +## Observing the Results + +1. A window will open on the Receiver's machine showing the mirrored video. +2. **Click anywhere** inside the Receiver's video window. +3. **Visual Feedback**: An animated "ping" (an expanding white ring) will appear + directly in the video stream at the location where you clicked. This ring is + drawn by the **Sender** on the raw frames before they are encoded. +4. **Console Logs**: Check the Sender's terminal output. You should see logs + indicating the received mouse events: + + ``` + [Input] Received MOUSE_DOWN at (450, 300) button=1 display=1920x1080 + [Input] Received MOUSE_UP at (450, 300) button=1 display=1920x1080 + ``` + +## How it Works + +1. **Capture**: The `SDLEventLoopProcessor` in `cast_receiver` captures SDL + mouse events. +2. **Mapping**: `StreamingPlaybackController` maps window coordinates to the + video's logical coordinate space. +3. **Transport**: The events are sent back to the sender via the `InputProducer` + using the negotiated `input_events` RTP extension. +4. **Consumption**: The `LoopingFileCastAgent` in `cast_sender` receives the + events via `InputConsumer`. +5. **Overlay**: `LoopingFileSender` draws an animated ring directly into the YUV + pixel data of the next video frame before encoding. diff --git a/cast/docs/protocol_flow.md b/cast/docs/protocol_flow.md new file mode 100644 index 000000000..855171ace --- /dev/null +++ b/cast/docs/protocol_flow.md @@ -0,0 +1,126 @@ +# Cast Protocol Flow Overview + +## Introduction + +The Cast protocol involves a sequence of interactions between a Sender device +and a Receiver device to establish a connection, launch an application, and +stream media. This document provides a high-level overview of this flow. + +The key stages are: + +1. **Discovery:** The Sender finds the Receiver on the local network. +2. **Connection Establishment:** A secure channel is created between the two + devices. +3. **Application Launch/Session Establishment:** The Sender requests the + Receiver to launch a specific application. +4. **Streaming Setup:** Media streaming parameters are negotiated. +5. **Media Streaming:** Media is transported from Sender to Receiver. +6. **Teardown:** The session is terminated. + +## Sequence Diagram + +The following diagram illustrates the typical message flow. + +```ascii ++--------------+ +----------------+ +| Sender Device| | Receiver Device| ++--------------+ +----------------+ + | | + |------------- 1. Discovery (mDNS) ----------------->| + | (sends mDNS query for Cast devices) | + | | + |<--------------- (responds with mDNS record) -------| + | | + |------------ 2. Connection Establishment ---------->| + | (establishes TCP connection) | + | | + |--------------- (TLS Handshake) ------------------->| + |<-------------- (TLS Handshake) --------------------| + | | + |---------- (CastV2 Channel Handshake) ------------->| + | (Auth, Device Capabilities) | + | | + |--------- 3. Application Launch ------------------->| + | (sends LAUNCH message) | + | +---| + | (launches app) | + | +-->| + |<----------- (reports App status, session ID) ------| + | | + |------------ 4. Streaming Setup ------------------->| + | (sends OFFER message) | + | | + |<----------- (responds with ANSWER message) --------| + | | + |-------------- 5. Media Streaming ----------------->| + | (streams RTP/RTCP packets via UDP) | + | | + | | + |<-------------- (sends RTCP feedback) --------------| + | | + |------------ (Ongoing Control Messages) ----------->| + | (e.g., PAUSE) | + | | + |----------------- 6. Teardown --------------------->| + | (sends STOP message or disconnects) | + | +---| + | (ends session) | + | +---| +``` + +## Key Messages and States + +### Discovery + +- The Sender broadcasts a Multicast DNS (mDNS) query for services of type + `_googlecast._tcp.local`. +- Cast Receivers on the network respond with their IP address, port, and + device information. + +A minimal mDNS record for a Cast device, named `Living Room`, is shown below. +For more details on how this record is parsed, see +`DnsSdInstanceEndpointToReceiverInfo()` in +`cast/common/public/receiver_info.cc`. + +```bash +_googlecast._tcp.local. 86400 IN PTR LivingRoom.local. +LivingRoom.local. 86400 IN SRV 0 0 8009 LivingRoom.local. +LivingRoom.local. 86400 IN A 192.168.1.100 +LivingRoom.local. 86400 IN TXT "id=" "ve=02" "ca=5" "fn=Living Room" "st=0" "md=Chromecast" +``` + +### Connection Establishment + +- The Sender establishes a TCP connection to the Receiver's IP address and + port. +- A TLS handshake is performed to create a secure channel. +- The CastV2 protocol handshake occurs over this secure channel, where the + sender authenticates that the receiver is an official Cast receiver device. + +### Application Launch + +- The Sender sends a `LAUNCH` message to the Receiver, requesting it to start + a specific application (identified by an App ID). +- The Receiver launches the application (e.g., a web-based media player) in + its own environment. Once ready, the Receiver informs the Sender of the + application's status and provides a unique `sessionId`. + +### Streaming Setup + +- Using the Cast Streaming Control Protocol (CSCP), the Sender and Receiver + negotiate streaming parameters via an `OFFER`/`ANSWER` exchange. +- This negotiation determines codecs, resolutions, bitrates, and the UDP ports + to be used for media transport. + +### Media Streaming + +- The Sender begins encoding media and streaming it to the Receiver as a + sequence of RTP (Real-time Transport Protocol) packets over UDP. +- The Receiver receives, decodes, and renders the media. +- RTCP (RTP Control Protocol) packets are exchanged periodically to provide + feedback on stream quality, synchronization, and to handle packet loss. + +### Teardown + +- When the streaming session is finished, the Sender sends a `STOP` message to + terminate the application on the Receiver, or simply closes the connection. diff --git a/cast/protocol/BUILD.gn b/cast/protocol/BUILD.gn index fd7225117..302c305c3 100644 --- a/cast/protocol/BUILD.gn +++ b/cast/protocol/BUILD.gn @@ -74,7 +74,6 @@ openscreen_source_set("unittests") { ":receiver_examples", ":streaming_examples", "../../platform:base", - "../../third_party/abseil", "../../third_party/googletest:gmock", "../../third_party/googletest:gtest", "../../util", diff --git a/cast/protocol/castv2/README.md b/cast/protocol/castv2/README.md index 22938235a..d010f0014 100644 --- a/cast/protocol/castv2/README.md +++ b/cast/protocol/castv2/README.md @@ -1,63 +1,45 @@ -# [libcast] Cast Streaming Control Protocol (CSCP) - -## What is it? - -CSCP is the standardized and modernized implement of the Castv2 Mirroring -Control Protocol, a legacy API implemented both inside of -[Chrome's Mirroring Service](https://source.chromium.org/chromium/chromium/src/+/master:components/mirroring/service/receiver_response.h;l=75?q=receiverresponse%20&ss=chromium%2Fchromium%2Fsrc) -and other Google products that communicate with -Chrome, such as Chromecast. This API handles session control messaging, such as -managing OFFER/ANSWER exchanges, getting status and capability information from -the receiver, and exchanging RPC messaging for handling media remoting. - -## What's in this folder? +# What's in this folder? The `streaming_schema.json` file in this directory contains a [JsonSchema](https://json-schema.org/) for the validation of control messaging defined in the Cast Streaming Control Protocol. This includes comprehensive rules for message definitions as well as valid values and ranges. Similarly, the `receiver_schema.json` file contains a JsonSchema for validating receiver -control messages, such as `LAUNCH` and `GET_APP_AVAILABILITY`. +control messages, such as `LAUNCH` and `GET_APP_AVAILABILITY`, and their +corresponding receiver response messages. + +## Example Validation The `validate_examples.sh` runs the control protocol against a wide variety of -example files in this directory, one for each type of supported message in CSCP. -In order to see what kind of validation this library provides, you can modify -these example files and see what kind of errors this script presents. +example files in this directory, one for each type of supported message in the +streaming and the receiver schema definitions. In order to see what kind of +validation this library provides, you can modify these example files and see +what kind of errors this script presents. -NOTE: this script uses -[`yajsv`](https://github.com/neilpa/yajsv/releases/tag/v1.4.0), -which needs to be installed. See the [README.md](../../../README.md#). +### yajsv installation -For example, if we modify the launch.json to not have a language field: +NOTE: this script requires [`yajsv`](https://github.com/neilpa/yajsv), a JSON +schema validator, in order to run. Follow the link to see latest installation +instructions. -``` --> % ./validate_examples.sh -1- -/usr/local/src/openscreen/cast/protocol/castv2/streaming_examples/answer.json: pass -/usr/local/src/openscreen/cast/protocol/castv2/streaming_examples/capabilities_response.json: pass -/usr/local/src/openscreen/cast/protocol/castv2/streaming_examples/get_capabilities.json: pass -/usr/local/src/openscreen/cast/protocol/castv2/streaming_examples/get_status.json: pass -/usr/local/src/openscreen/cast/protocol/castv2/streaming_examples/offer.json: pass -/usr/local/src/openscreen/cast/protocol/castv2/streaming_examples/rpc.json: pass -/usr/local/src/openscreen/cast/protocol/castv2/streaming_examples/status_response.json: pass -/usr/local/src/openscreen/cast/protocol/castv2/receiver_examples/get_app_availability.json: pass -/usr/local/src/openscreen/cast/protocol/castv2/receiver_examples/get_app_availability_response.json: pass -/usr/local/src/openscreen/cast/protocol/castv2/receiver_examples/launch.json: fail: (root): Must validate "then" as "if" was valid -/usr/local/src/openscreen/cast/protocol/castv2/receiver_examples/launch.json: fail: (root): language is required -/usr/local/src/openscreen/cast/protocol/castv2/receiver_examples/launch.json: fail: (root): Must validate all the schemas (allOf) -1 of 1 failed validation -/usr/local/src/openscreen/cast/protocol/castv2/receiver_examples/launch.json: fail: (root): Must validate "then" as "if" was valid -/usr/local/src/openscreen/cast/protocol/castv2/receiver_examples/launch.json: fail: (root): language is required -/usr/local/src/openscreen/cast/protocol/castv2/receiver_examples/launch.json: fail: (root): Must validate all the schemas (allOf) -/usr/local/src/openscreen/cast/protocol/castv2/receiver_examples/stop.json: pass -``` +### Example validation + +For example, if we modify the launch.json to not have a language field, the +script will fail and print an error message indicating that the `language` field +is required. ## Updating schemas -When updating the JSON schema files, take care to ensure consistant formatting. +When updating the JSON schema files, take care to ensure consistent formatting. Since `clang-format` doesn't support JSON files (currently only Python, C++, and JavaScript), the JSON files here are instead formatted using -(json-stringify-pretty-compact)[https://github.com/lydell/json-stringify-pretty-compact] +[json-stringify-pretty-compact](https://github.com/lydell/json-stringify-pretty-compact) with a max line length of 80 and a 2-character indent. Many IDEs have an extension for this, such as VSCode's -(json-compact-prettifier)[https://marketplace.visualstudio.com/items?itemName=inadarei.json-compact-prettifier]. +[json-compact-prettifier](https://marketplace.visualstudio.com/items?itemName=inadarei.json-compact-prettifier). +You can also use `npx` to run the formatter from the command line: + +```bash +npx json-stringify-pretty-compact --maxLength 80 --indent 2 my_schema.json +``` diff --git a/cast/protocol/castv2/receiver_examples/eureka_info.json b/cast/protocol/castv2/receiver_examples/eureka_info.json new file mode 100644 index 000000000..0fc46d13c --- /dev/null +++ b/cast/protocol/castv2/receiver_examples/eureka_info.json @@ -0,0 +1,20 @@ +{ + "type": "eureka_info", + "request_id": 789, + "response_code": 200, + "response_string": "OK", + "data": { + "name": "Living Room TV", + "version": 12, + "device_info": { + "manufacturer": "Google Inc.", + "product_name": "Chromecast", + "ssdp_udn": "some-unique-device-name" + }, + "build_info": { + "build_type": 2, + "cast_build_revision": "1.56.275994", + "system_build_number": "275994" + } + } +} diff --git a/cast/protocol/castv2/receiver_examples/get_device_info_response.json b/cast/protocol/castv2/receiver_examples/get_device_info_response.json new file mode 100644 index 000000000..f2409cf96 --- /dev/null +++ b/cast/protocol/castv2/receiver_examples/get_device_info_response.json @@ -0,0 +1,9 @@ +{ + "type": "GET_DEVICE_INFO", + "requestId": 789, + "deviceId": "some-unique-device-id", + "friendlyName": "Living Room TV", + "deviceModel": "Chromecast", + "capabilities": 6149, + "controlNotifications": 1 +} \ No newline at end of file diff --git a/cast/protocol/castv2/receiver_examples/invalid_request.json b/cast/protocol/castv2/receiver_examples/invalid_request.json new file mode 100644 index 000000000..93c1caab6 --- /dev/null +++ b/cast/protocol/castv2/receiver_examples/invalid_request.json @@ -0,0 +1,5 @@ +{ + "responseType": "INVALID_REQUEST", + "requestId": 456, + "reason": "INVALID_COMMAND" +} diff --git a/cast/protocol/castv2/receiver_examples/launch_error.json b/cast/protocol/castv2/receiver_examples/launch_error.json new file mode 100644 index 000000000..58f83676f --- /dev/null +++ b/cast/protocol/castv2/receiver_examples/launch_error.json @@ -0,0 +1,5 @@ +{ + "responseType": "LAUNCH_ERROR", + "requestId": 123, + "reason": "NOT_FOUND" +} diff --git a/cast/protocol/castv2/receiver_examples/launch_status.json b/cast/protocol/castv2/receiver_examples/launch_status.json new file mode 100644 index 000000000..9bb965519 --- /dev/null +++ b/cast/protocol/castv2/receiver_examples/launch_status.json @@ -0,0 +1,5 @@ +{ + "responseType": "LAUNCH_STATUS", + "launchRequestId": 17, + "status": "USER_ALLOWED" +} \ No newline at end of file diff --git a/cast/protocol/castv2/receiver_examples/media_status.json b/cast/protocol/castv2/receiver_examples/media_status.json new file mode 100644 index 000000000..e28792205 --- /dev/null +++ b/cast/protocol/castv2/receiver_examples/media_status.json @@ -0,0 +1,24 @@ +{ + "responseType": "MEDIA_STATUS", + "requestId": 101, + "media": [ + { + "mediaSessionId": 0, + "playbackRate": 1.0, + "playerState": "PLAYING", + "currentTime": 0.0, + "supportedMediaCommands": 0, + "volume": { + "level": 1.0, + "muted": false, + "controlType": "attenuation", + "stepInterval": 0.05 + }, + "media": { + "contentId": "some-content-id", + "streamType": "LIVE", + "contentType": "video/webm" + } + } + ] +} diff --git a/cast/protocol/castv2/receiver_examples/receiver_status.json b/cast/protocol/castv2/receiver_examples/receiver_status.json new file mode 100644 index 000000000..36070cefa --- /dev/null +++ b/cast/protocol/castv2/receiver_examples/receiver_status.json @@ -0,0 +1,27 @@ +{ + "responseType": "RECEIVER_STATUS", + "requestId": 1, + "status": { + "applications": [ + { + "appId": "E8C28D3C", + "displayName": "Backdrop", + "isIdleScreen": true, + "launchedFromCloud": false, + "namespaces": [ + {"name": "urn:x-cast:com.google.cast.debugoverlay"} + ], + "sessionId": "2b6e3d57-5d7f-407c-9a0b-9df07dba98e9", + "statusText": "Backdrop is running", + "transportId": "2b6e3d57-5d7f-407c-9a0b-9df07dba98e9" + } + ], + "volume": { + "controlType": "attenuation", + "level": 0.8, + "muted": false, + "stepInterval": 0.05 + }, + "userEq": {} + } +} diff --git a/cast/protocol/castv2/receiver_schema.json b/cast/protocol/castv2/receiver_schema.json index 9ca943577..e8051be5c 100644 --- a/cast/protocol/castv2/receiver_schema.json +++ b/cast/protocol/castv2/receiver_schema.json @@ -2,61 +2,205 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://something/app_schema.json", "definitions": { - "app_id": {"type": "string", "enum": ["0F5096E8", "85CDB22F"]} - }, - "type": "object", - "properties": { - "availability": { + "app_id": {"type": "string", "pattern": "[0-9a-fA-F]{8}"}, + "volume": { + "type": "object", + "properties": { + "controlType": {"type": "string", "enum": ["attenuation"]}, + "level": {"type": "number", "minimum": 0, "maximum": 1}, + "muted": {"type": "boolean"}, + "stepInterval": {"type": "number"} + }, + "required": ["controlType", "level", "muted"] + }, + "application": { "type": "object", - "patternProperties": { - "[0-9a-fA-F]": { - "type": "string", - "enum": ["APP_AVAILABLE", "APP_UNAVAILABLE"] + "properties": { + "appId": {"$ref": "#/definitions/app_id"}, + "displayName": {"type": "string"}, + "sessionId": {"type": "string"}, + "statusText": {"type": "string"}, + "transportId": {"type": "string"}, + "isIdleScreen": {"type": "boolean"}, + "launchedFromCloud": {"type": "boolean"}, + "namespaces": { + "type": "array", + "items": { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"] + } } }, - "additionalProperties": false + "required": ["appId", "displayName", "sessionId", "transportId"] }, - "type": { - "type": "string", - "enum": ["LAUNCH", "STOP", "GET_APP_AVAILABILITY"] + "status": { + "type": "object", + "properties": { + "applications": { + "type": "array", + "items": {"$ref": "#/definitions/application"} + }, + "volume": {"$ref": "#/definitions/volume"}, + "userEq": {"type": "object"} + }, + "required": ["volume"] }, - "responseType": {"type": "string", "enum": ["GET_APP_AVAILABILITY"]}, - "requestId": {"type": "integer", "minimum": 0}, - "language": {"type": "string", "minLength": 2}, - "supportedAppTypes": { - "type": "array", - "items": {"type": "string", "enum": ["ANDROID_TV", "WEB"]} + "media_status_item": { + "type": "object", + "properties": { + "mediaSessionId": {"type": "integer"}, + "playbackRate": {"type": "number"}, + "playerState": {"type": "string", "enum": ["IDLE", "PLAYING", "PAUSED", "BUFFERING"]}, + "currentTime": {"type": "number"}, + "supportedMediaCommands": {"type": "integer"}, + "volume": {"$ref": "#/definitions/volume"}, + "media": { + "type": "object", + "properties": { + "contentId": {"type": "string"}, + "streamType": {"type": "string", "enum": ["BUFFERED", "LIVE"]}, + "contentType": {"type": "string"} + }, + "required": ["contentId", "streamType", "contentType"] + } + }, + "required": ["mediaSessionId", "playerState", "currentTime", "supportedMediaCommands", "volume", "media"] } }, - "required": ["requestId"], - "allOf": [ + "oneOf": [ { - "if": { - "properties": {"type": {"const": "GET_APP_AVAILABILITY"}}, - "required": ["type"] + "title": "LAUNCH", + "properties": { + "type": {"const": "LAUNCH"}, + "requestId": {"type": "integer"}, + "appId": {"$ref": "#/definitions/app_id"} }, - "then": { - "properties": { - "appId": {"type": "array", "items": {"$ref": "#/definitions/app_id"}} - }, - "required": ["appId"] + "required": ["type", "requestId", "appId"] + }, + { + "title": "STOP", + "properties": { + "type": {"const": "STOP"}, + "requestId": {"type": "integer"}, + "sessionId": {"type": "string"} }, - "else": {"properties": {"appId": {"$ref": "#/definitions/app_id"}}} + "required": ["type", "requestId", "sessionId"] }, { - "if": { - "properties": {"responseType": {"const": "GET_APP_AVAILABILITY"}}, - "required": ["responseType"] + "title": "GET_APP_AVAILABILITY Request", + "properties": { + "type": {"const": "GET_APP_AVAILABILITY"}, + "requestId": {"type": "integer"}, + "appId": {"type": "array", "items": {"$ref": "#/definitions/app_id"}} }, - "then": {"required": ["availability"]} + "required": ["type", "requestId", "appId"] }, { - "if": {"properties": {"type": {"const": "LAUNCH"}}, "required": ["type"]}, - "then": {"required": ["supportedAppTypes", "language", "appId"]} + "title": "GET_APP_AVAILABILITY Response", + "properties": { + "responseType": {"const": "GET_APP_AVAILABILITY"}, + "requestId": {"type": "integer"}, + "availability": { + "type": "object", + "patternProperties": { + "^[0-9a-fA-F]{8}$": {"type": "string", "enum": ["APP_AVAILABLE", "APP_UNAVAILABLE"]} + } + } + }, + "required": ["responseType", "requestId", "availability"] + }, + { + "title": "GET_STATUS", + "properties": { + "type": {"const": "GET_STATUS"}, + "requestId": {"type": "integer"} + }, + "required": ["type", "requestId"] + }, + { + "title": "RECEIVER_STATUS", + "properties": { + "responseType": {"const": "RECEIVER_STATUS"}, + "requestId": {"type": "integer"}, + "status": {"$ref": "#/definitions/status"} + }, + "required": ["responseType", "requestId", "status"] }, { - "if": {"properties": {"type": {"const": "STOP"}}, "required": ["type"]}, - "then": {"required": ["sessionId"]} + "title": "LAUNCH_STATUS", + "properties": { + "responseType": {"const": "LAUNCH_STATUS"}, + "launchRequestId": {"type": "integer"}, + "status": {"type": "string", "const": "USER_ALLOWED"} + }, + "required": ["responseType", "launchRequestId", "status"] + }, + { + "title": "LAUNCH_ERROR", + "properties": { + "responseType": {"const": "LAUNCH_ERROR"}, + "requestId": {"type": "integer"}, + "reason": {"type": "string"} + }, + "required": ["responseType", "requestId", "reason"] + }, + { + "title": "INVALID_REQUEST", + "properties": { + "responseType": {"const": "INVALID_REQUEST"}, + "requestId": {"type": "integer"}, + "reason": {"type": "string"} + }, + "required": ["responseType", "requestId", "reason"] + }, + { + "title": "MEDIA_STATUS", + "properties": { + "responseType": {"const": "MEDIA_STATUS"}, + "requestId": {"type": "integer"}, + "media": {"type": "array", "items": {"$ref": "#/definitions/media_status_item"}} + }, + "required": ["responseType", "requestId", "media"] + }, + { + "title": "GET_DEVICE_INFO Request", + "properties": { + "type": {"const": "GET_DEVICE_INFO"}, + "requestId": {"type": "integer"} + }, + "required": ["type", "requestId"], + "additionalProperties": false + }, + { + "title": "GET_DEVICE_INFO Response", + "properties": { + "type": {"const": "GET_DEVICE_INFO"}, + "requestId": {"type": "integer"}, + "deviceId": {"type": "string"}, + "friendlyName": {"type": "string"}, + "deviceModel": {"type": "string"}, + "capabilities": {"type": "integer"}, + "controlNotifications": {"type": "integer"} + }, + "required": ["type", "requestId", "deviceId", "friendlyName", "deviceModel", "capabilities", "controlNotifications"] + }, + { + "title": "EUREKA_INFO", + "properties": { + "type": {"const": "eureka_info"}, + "request_id": {"type": "integer"}, + "response_code": {"type": "integer"}, + "response_string": {"type": "string"}, + "data": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "version": {"type": "integer"} + } + } + }, + "required": ["type", "request_id", "response_code", "response_string", "data"] } ] -} \ No newline at end of file +} diff --git a/cast/protocol/castv2/streaming_examples/answer.json b/cast/protocol/castv2/streaming_examples/answer.json index 0e115fc15..9c3efe22d 100644 --- a/cast/protocol/castv2/streaming_examples/answer.json +++ b/cast/protocol/castv2/streaming_examples/answer.json @@ -29,7 +29,7 @@ "scaling": "sender" }, "receiverRtcpEventLog": [0, 1], - "receiverRtcpDscp": [234, 567], - "rtpExtensions": ["adaptive_playout_delay"] + "receiverRtcpDscp": [1, 3], + "rtpExtensions": [["adaptive_playout_delay", "input_events"], ["adaptive_playout_delay"]] } } \ No newline at end of file diff --git a/cast/protocol/castv2/streaming_schema.json b/cast/protocol/castv2/streaming_schema.json index 4c78d5264..743e3e7c5 100644 --- a/cast/protocol/castv2/streaming_schema.json +++ b/cast/protocol/castv2/streaming_schema.json @@ -21,7 +21,10 @@ }, "rtp_extensions": { "type": "array", - "items": {"type": "string", "enum": ["adaptive_playout_delay"]} + "items": { + "type": "string", + "enum": ["adaptive_playout_delay", "input_events"] + } }, "stream": { "properties": { @@ -36,7 +39,7 @@ "aesKey": {"type": "string", "pattern": "[0-9a-fA-F]{32}"}, "aesIvMask": {"type": "string", "pattern": "[0-9a-fA-F]{32}"}, "receiverRtcpEventLog": {"type": "boolean"}, - "receiverRtcpDscp": {"type": "integer", "minimum": 0, "default": 46}, + "receiverRtcpDscp": {"type": "integer", "minimum": 0, "default": 46, "maximum": 63}, "rtpExtensions": {"$ref": "#/definitions/rtp_extensions"}, "timeBase": { "type": "string", @@ -166,7 +169,12 @@ "type": "array", "items": {"type": "integer", "minimum": 0} }, - "rtpExtensions": {"$ref": "#/definitions/rtp_extensions"} + "rtpExtensions": { + "type": "array", + "items": { + "$ref": "#/definitions/rtp_extensions" + } + } }, "required": ["udpPort", "sendIndexes", "ssrcs"] }, @@ -214,7 +222,8 @@ "STATUS_RESPONSE", "GET_CAPABILITIES", "CAPABILITIES_RESPONSE", - "RPC" + "RPC", + "INPUT" ] } }, @@ -271,6 +280,12 @@ "properties": {"type": {"const": "RPC"}, "result": {"const": "ok"}} }, "then": {"required": ["rpc"]} + }, + { + "if": { + "properties": {"type": {"const": "INPUT"}, "result": {"const": "ok"}} + }, + "then": {"required": ["input"]} } ] } diff --git a/cast/protocol/castv2/validate_examples.sh b/cast/protocol/castv2/validate_examples.sh index 2ac54658d..1316708fb 100755 --- a/cast/protocol/castv2/validate_examples.sh +++ b/cast/protocol/castv2/validate_examples.sh @@ -6,16 +6,23 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +DEFAULT_YAJSV_PATH=~/go/bin/yajsv + YAJSV_BIN=$(which yajsv) if [ "$YAJSV_BIN" == "" ]; then - echo "Could not find yajsv, see the top-level README.md" - exit 1 + if [[ -f "$DEFAULT_YAJSV_PATH" ]]; then + echo "Found yajsv at the default path ($DEFAULT_YAJSV_PATH), using." + YAJSV_BIN=$DEFAULT_YAJSV_PATH + else + echo "Could not find yajsv, see //cast/protocol/castv2/README.md" + exit 1 + fi fi for filename in $SCRIPT_DIR/streaming_examples/*.json; do -"$YAJSV_BIN" -s "$SCRIPT_DIR/streaming_schema.json" "$filename" + "$YAJSV_BIN" -s "$SCRIPT_DIR/streaming_schema.json" "$filename" done for filename in $SCRIPT_DIR/receiver_examples/*.json; do -"$YAJSV_BIN" -s "$SCRIPT_DIR/receiver_schema.json" "$filename" + "$YAJSV_BIN" -s "$SCRIPT_DIR/receiver_schema.json" "$filename" done diff --git a/cast/protocol/castv2/validation.cc b/cast/protocol/castv2/validation.cc index 2a736b2aa..ab31a5791 100644 --- a/cast/protocol/castv2/validation.cc +++ b/cast/protocol/castv2/validation.cc @@ -9,11 +9,23 @@ #include "cast/protocol/castv2/receiver_schema_data.h" #include "cast/protocol/castv2/streaming_schema_data.h" + +// NOTE: this is preferred over a public_configs entry in the BUILD.gn so that +// the rest of this file / component gets checked for exit time destructors. +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wexit-time-destructors" +#endif + #include "third_party/valijson/src/include/valijson/adapters/jsoncpp_adapter.hpp" #include "third_party/valijson/src/include/valijson/schema.hpp" #include "third_party/valijson/src/include/valijson/schema_parser.hpp" #include "third_party/valijson/src/include/valijson/utils/jsoncpp_utils.hpp" #include "third_party/valijson/src/include/valijson/validator.hpp" + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif #include "util/json/json_serialization.h" #include "util/osp_logging.h" #include "util/std_util.h" @@ -31,7 +43,7 @@ std::vector MapErrors(const valijson::ValidationResults& results) { const std::string context = string_util::Join(result.context.cbegin(), result.context.cend(), ", "); errors.emplace_back(Error::Code::kJsonParseError, - StringPrintf("Node: %s, Message: %s", context.c_str(), + StringFormat("Node: {}, Message: {}", context.c_str(), result.description.c_str())); } return errors; @@ -67,17 +79,17 @@ std::vector Validate(const Json::Value& document, } std::vector ValidateStreamingMessage(const Json::Value& message) { - static valijson::Schema schema; + static valijson::Schema* const schema = new valijson::Schema(); static std::once_flag flag; - std::call_once(flag, [] { LoadSchema(kStreamingSchema, &schema); }); - return Validate(message, schema); + std::call_once(flag, [] { LoadSchema(kStreamingSchema, schema); }); + return Validate(message, *schema); } std::vector ValidateReceiverMessage(const Json::Value& message) { - static valijson::Schema schema; + static valijson::Schema* const schema = new valijson::Schema(); static std::once_flag flag; - std::call_once(flag, [] { LoadSchema(kReceiverSchema, &schema); }); - return Validate(message, schema); + std::call_once(flag, [] { LoadSchema(kReceiverSchema, schema); }); + return Validate(message, *schema); } } // namespace openscreen::cast diff --git a/cast/protocol/castv2/validation_unittest.cc b/cast/protocol/castv2/validation_unittest.cc index 0a095abc4..494aa445a 100644 --- a/cast/protocol/castv2/validation_unittest.cc +++ b/cast/protocol/castv2/validation_unittest.cc @@ -25,6 +25,7 @@ #include "json/value.h" #include "platform/base/error.h" #include "util/json/json_serialization.h" +#include "util/no_destructor.h" #include "util/osp_logging.h" #include "util/std_util.h" #include "util/stringprintf.h" @@ -37,18 +38,18 @@ constexpr char kEmptyJson[] = "{}"; // Schema format string, that allows for specifying definitions, // properties, and required fields. -constexpr char kSchemaFormat[] = R"({ +static constexpr char kSchemaFormat[] = R"({{ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://something/app_schema_data.h", - "definitions": { - %s - }, + "definitions": {{ + {} + }}, "type": "object", - "properties": { - %s - }, - "required": [%s] -})"; + "properties": {{ + {} + }}, + "required": [{}] +}})"; // Fields used for an appId containing schema constexpr char kAppIdDefinition[] = R"("app_id": { @@ -66,7 +67,7 @@ constexpr char kInvalidAppIdDocument[] = R"({ "appId": "FooBar" })"; std::string BuildSchema(const char* definitions, const char* properties, const char* required) { - return StringPrintf(kSchemaFormat, definitions, properties, required); + return StringFormat(kSchemaFormat, definitions, properties, required); } bool TestValidate(std::string_view document, std::string_view schema) { @@ -81,14 +82,14 @@ bool TestValidate(std::string_view document, std::string_view schema) { } const std::string& GetEmptySchema() { - static const std::string kEmptySchema = BuildSchema("", "", ""); - return kEmptySchema; + static const NoDestructor kEmptySchema(BuildSchema("", "", "")); + return *kEmptySchema; } const std::string& GetAppSchema() { - static const std::string kAppIdSchema = - BuildSchema(kAppIdDefinition, kAppIdProperty, kAppIdName); - return kAppIdSchema; + static const NoDestructor kAppIdSchema( + BuildSchema(kAppIdDefinition, kAppIdProperty, kAppIdName)); + return *kAppIdSchema; } class StreamingValidationTest : public testing::TestWithParam {}; diff --git a/cast/protocol/streaming_session_protocol.md b/cast/protocol/streaming_session_protocol.md new file mode 100644 index 000000000..a2043f055 --- /dev/null +++ b/cast/protocol/streaming_session_protocol.md @@ -0,0 +1,913 @@ +# Streaming Session Protocol + +**Owner:** [Jordan Bayles](mailto:jophba@google.com) + +--- + +*** promo +This document is formatted to follow [Gitiles](https://gerrit.googlesource.com/gitiles/+/HEAD/Documentation/markdown.md) +syntax, which includes several extensions (but *not* GitHub Extended Markdown). +*** + +[TOC] + +## Objective and Requirements + +This document is the reference specification for the libcast Streaming Session +Protocol. This spec was originally developed as part of the Google Cast v2 +project over a decade ago, and has slowly evolved over time to meet the needs of +various Cast devices and scenarios. + +With the implementation of [libcast](#libcast) and its ever increasing adoption +rate across the Cast ecosystem, this document will continue to evolve as the +official standard for the Cast Streaming protocol. + +### Goals + +1. Provide a reference specification for consumers and implementers of the + libcast library, so that proposed features can be properly discussed, + designed, implemented, and maintained. + +2. Define interfaces for establishing, controlling, and terminating Cast + streaming (mirroring, remoting, and otherwise) sessions. + +3. Prioritize interoperability across the Cast ecosystem, so that Cast devices + using the latest and greatest version of libcast are still, to the best of + our ability, compatible with legacy devices no longer receiving updates. + **This specification will attempt to maintain compatibility with + legacy Cast v2 devices, but the protocol is expected to grow and change over + time.** + +### Non-Goals + +Unfortunately, some important Cast APIs are still defined by closed source +specifications, and are thus out of scope of this document. Defining the +following APIs is thus out of scope: + +* **Application control messaging**: Some application control messages, such as + `LAUNCH` and `STOP`, are included. However, it is likely that there are other, + unspecified messages used to manage higher level app state. A functional + implementation is provided for the receiver ( + [openscreen::cast::ApplicationAgent](../receiver/application_agent.h)) and a + reference implementation for the sender + ([openscreen::cast::LoopingFileAgent](../standalone_sender/looping_file_cast_agent.h)). + +* **Authentication messaging**: On top of TLS, Cast provides additional + authentication through the use of certificates and private keys. The API for + authentication is implemented in several directories of cast, especially the + [//cast/common/channel](../common/channel/) and [//cast/common/certificate](../common/certificate/) + folders. + +* **Keep-alive behavior**: Although the control channel may (or may not) have + more sophisticated behavior for keeping a session alive, in this specific + protocol it is limited to a simple timeout, with [deprecated PING/PONG messages](#deprecated-messages) + kept in the specification. + +* **Flinging messaging**: there is a rich suite of messages in the `media` + namespace used for controlling flinging messages, i.e. sessions where the + receiver is responsible for fetching content and controlling playback. This + document is focused on streaming, both mirroring and remoting, and leaves + flinging for closed source documentation and closed source APIs. + +## Background + +### Libcast + +See the [cast/README.md](../README.md) for more information about the libcast +project. + +## Overview + +The streaming session protocol defines how a streaming session interacts with +standard Cast messages (in the `com.google.cast` namespace), +mirroring-specific messages (in the `urn:x-cast:com.google.cast.webrtc` +namespace), and remoting-specific messages (in the +`urn:x-cast:com.google.cast.remoting` namespace). + +In this context, "mirroring" refers to sending a real-time encoded stream of the +sender's screen to the receiver. "Remoting" refers to an optimization where, if +the sender's screen is primarily composed of a video that the receiver is +capable of performantly decoding, instead of transcoding the video and streaming +it to the receiver, it is instead sent to the receiver still encoded and decoded +directly by the receiver. + +* **Launch and Termination Request from Sender**: streaming is initiated using a + standard com.google.cast `LAUNCH` request with the `appId` parameter set to a + pre-defined 8-digit alphanumeric app identifier. The full set of application + IDs known to libcast at this time are defined in the [cast_streaming_app_ids.h](../common/public/cast_streaming_app_ids.h) + header. Termination is via a standard `STOP` request. + +* **Transport Session Negotiation**: a streaming-specific `OFFER` event is + defined, and used by the sender to generate an offer for the receiver. The + `ANSWER` response to this event contains the answer object. The `OFFER` must + be sent immediately after launch, and can be sent again at any time. + +* **Presentation Request from Sender**: a `PRESENTATION` request allows sender + control over rendering transformations applied on the receiver. The + transformation defines zoom, offset, and rotation; it enables the current + letterboxing-elimination feature, overscan compensation (for flexibility, if + needed), and rotation (intended for Android, not used by Chrome). + +* **Keep Alive**: both sender and receiver are expected to use media-level + activity and/or transport layer activity for keeping the connection alive. + There's no separate application-level keep alive message. + +* **Application Protocol & Other Control Messages:** the minimum set of control + messages required for streaming is `LAUNCH`, `STOP`, and `RECEIVER_STATUS`. + The full list is documented in the [Session Control Messages](#session-control-messages-comgooglecast) + section. + +## Detailed Design + +### Protocol Messages + +This section defines the JSON payloads for various messages used in the Cast +protocol. + +#### Notes on Types + +Message field types generally one of the following three classes: primitives, +structures, or collections of primitives or structures. The most common type +primitives are `string` and `int`. Although this spec is generally written in +C++, there is no technical reason one could not produce an implementation in +other languages. This specification's assumptions about primitive types are +defined in the below table: + +| Name | Definition | +|----------|-------------------------------------------------------------------| +| `int` | A signed at-least-32-bit integer value (`int` in C++) | +| `uint32` | An unsigned 32 bit integer value (`uint32_t` in C++) | +| `string` | An array of ASCII characters (`char*` or `std::string` in C++) | + +#### Common Message Fields + +It is assumed that messages generally have all of the following properties: + +| Name | Type | Value/description | +|-----------|----------|-------------------------------------------------------| +| sessionId | `int` | Unique identifier of the session | +| seqNum | `int` | Request sequence number | +| type | `string` | Represents the specific kind of message | + +#### Session Control Messages (`com.google.cast`) + +Basic streaming session control occurs via `com.google.cast` messages (as +currently defined with `version=2`), as follows: + +* `LAUNCH` (*request*): initiates the streaming session. For v2 streaming, the + appId parameter must be set to `0F5096E8` if the session is audio and video, + or `85CDB22F` if the session is audio only. For this app name, it is expected + that the Cast receiver will run a specific built-in streaming receiver that + implements this specification. + +* `LAUNCH_STATUS` (*reply*): sent from the Cast receiver to indicate that the + launch request succeeded. + +* `LAUNCH_ERROR` (*reply*): sent from the Cast receiver to indicate that the + launch request failed. + +* `GET_APP_AVAILABILITY` (*request*, *reply*): name of both the request and + response message for getting information about what applications are + available. + +* `GET_STATUS` (*request*): requests the status of the receiver. + +* `STATUS_RESPONSE` (*reply*): response to a `GET_STATUS` request. + +* `STOP` (*request*): terminates the streaming session, and must terminate all underlying + media streams from the sender to the receiver. + +* `INVALID_REQUEST` (*reply*): Optional message sent by the receiver whenever an + invalid command is received. + +* `RECEIVER_STATUS` (*reply*): response to a `GET_STATUS` request. + +##### LAUNCH + +This message is sent from the sender to the receiver to initiate a streaming +session. + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `type` | `string` | Must be `LAUNCH`. | +| `requestId` | `int` | A unique identifier for the request. | +| `appId` | `string` | The ID of the application to launch. For streaming, this is typically `0F5096E8` for A/V or `85CDB22F` for audio-only. | +| `appParams` | `object` (optional) | An optional object containing application-specific parameters. | +| `language` | `string` (optional) | The preferred language for the application (e.g., "en-US"). | +| `supportedAppTypes` | array of `string` (optional) | A list of application types supported by the sender (e.g., "WEB", "ANDROID_TV"). | + +##### LAUNCH_STATUS + +This message is sent from the receiver to the sender to indicate that the +application launch was successful. Note that a full `RECEIVER_STATUS` message is typically sent immediately after this. + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `responseType` | `string` | Must be `LAUNCH_STATUS`. | +| `launchRequestId` | `int` | The `requestId` from the original `LAUNCH` request. | +| `status` | `string` | A status string, must be `USER_ALLOWED`. | + +##### LAUNCH_ERROR + +This message is sent from the receiver to the sender if the application launch +failed. + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `responseType` | `string` | Must be `LAUNCH_ERROR`. | +| `requestId` | `int` | The `requestId` from the original `LAUNCH` request. | +| `reason` | `string` | A string code indicating the reason for the failure (e.g., `NOT_FOUND`, `SYSTEM_ERROR`). | + +##### GET_APP_AVAILABILITY + +A request from the sender to check if specific applications can be launched on +the receiver. The response uses the same message name in its `responseType` field. + +**Request:** + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `type` | `string` | Must be `GET_APP_AVAILABILITY`. | +| `requestId` | `int` | A unique identifier for the request. | +| `appId` | array of `string` | An array of application IDs to check. | + +**Response:** + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `responseType` | `string` | Must be `GET_APP_AVAILABILITY`. | +| `requestId` | `int` | The `requestId` from the request. | +| `availability` | `object` | An object where keys are `appId`s and values are `APP_AVAILABLE` or `APP_UNAVAILABLE`. | + +##### GET_STATUS + +A request from the sender to get the receiver's current status. It has no +payload other than the common `type` and `requestId` fields. + +##### RECEIVER_STATUS + +A response sent by the receiver containing its current status. This can be in +response to a `GET_STATUS` request or sent unsolicited when the receiver's state +changes. + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `responseType` | `string` | Must be `RECEIVER_STATUS`. | +| `requestId` | `int` | The `requestId` from the `GET_STATUS` request, or 0 if unsolicited. | +| `status` | `object` | The main status object. | +| `status.applications` | array of `object` | A list of running applications. Each object contains `appId`, `displayName`, `sessionId`, `transportId`, `isIdleScreen`, etc. | +| `status.volume` | `object` | An object describing the device's volume state, with fields like `level`, `muted`, and `controlType`. | + +##### STOP + +This message is sent from the sender to the receiver to terminate a running +application. + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `type` | `string` | Must be `STOP`. | +| `requestId` | `int` | A unique identifier for the request. | +| `sessionId` | `string` | The ID of the session to be terminated. | + +##### INVALID_REQUEST + +Sent by the receiver when it receives a malformed or invalid request. + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `responseType` | `string` | Must be `INVALID_REQUEST`. | +| `requestId` | `int` | The `requestId` from the invalid request, if it could be parsed. | +| `reason` | `string` | A string code for the error (e.g., `INVALID_COMMAND`). | + +#### Discovery Messages (`google.cast.receiver.discovery`) + +##### GET_DEVICE_INFO + +A request from the sender for detailed information about the receiver device. +The response unexpectedly uses the same message `type` and is not a `responseType`. + +**Request:** + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `type` | `string` | Must be `GET_DEVICE_INFO`. | +| `requestId` | `int` | A unique identifier for the request. | + +**Response:** + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `type` | `string` | Must be `GET_DEVICE_INFO`. | +| `requestId` | `int` | The `requestId` from the request. | +| `deviceId` | `string` | A unique identifier for the receiver device. | +| `friendlyName` | `string` | The user-configured device name. | +| `deviceModel` | `string` | The product model name. | +| `capabilities` | `int` | A bitmask of device capabilities. | +| `controlNotifications` | `int` | A flag for control notifications. | + +#### Setup Messages (`com.google.cast.setup`) + +##### eureka_info + +A response message providing detailed product and build information about the +receiver hardware. The request for this message is not clearly defined in the +code, but this response also uses a `type` field instead of a `responseType`. + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `type` | `string` | Must be `eureka_info`. | +| `request_id` | `int` | The `request_id` from the original request. Note the underscore. | +| `response_code` | `int` | A status code (e.g., 200 for OK). | +| `response_string` | `string` | A status string (e.g., "OK"). | +| `data` | `object` | An object containing the device details. | +| `data.name` | `string` | The friendly name of the device. | +| `data.version` | `int` | The version of this info structure. | +| `data.device_info` | `object` | An object containing device hardware details like `manufacturer` and `product_name`. | +| `data.build_info` | `object` | An object containing software build details like `cast_build_revision`. | + +#### Media Transport Messages (`com.google.cast.webrtc`) + +A number of streaming-specific features are defined via the +`com.google.cast.webrtc` namespace, which defines the following additional +messages: + +##### OFFER + +This message is sent from the sender to the receiver to initiate a streaming +session. + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `type` | `string` | Must be `OFFER`. | +| `offer` | `object` | [Offer object](#offer-object-definition) | + +The `OFFER` request can be sent by the sender at any point in time during the +session to renegotiate parameters of the session. + +If the receiver generates an error response to the initial offer, the sender +should immediately terminate the session and inform the receiver unless it's +able to generate a fallback offer. + +A subsequent offer after a session is successfully established is only effective +once an "ok" response is generated by the receiver. If an "error" response is +generated, the already-established session *should* remain in effect. + +###### Example `OFFER` Message + +For a full example `OFFER` message, see [castv2/streaming_examples/offer.json](./castv2/streaming_examples/offer.json). + +##### ANSWER + +This message is sent from the receiver to the sender in response to an `OFFER`. + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `type` | `string` | Must be `ANSWER`. | +| `result` | `string` | Must be `ok` or `error`. | +| `error` | `object` (optional) | Only populated if `result` is `error`. [Error object](#error-object-definition) | +| `answer` | `object` (optional) | Only populated if `result` is `ok`. [Answer object](#answer-object-definition) | + +###### Example ANSWER message + +For a full example `ANSWER` message, see [castv2/streaming_examples/answer.json](./castv2/streaming_examples/answer.json). + +##### GET_CAPABILITIES + +The "type" must be set to `GET_CAPABILITIES`, with no message body. + +##### CAPABILITIES_RESPONSE + +The "type" must be set to `CAPABILITIES_RESPONSE`, with the message body in a +"capabilities" object. Note that the `key_systems` field has been deprecated and +removed here. + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| type | `string` | Must be set to `CAPABILITIES_RESPONSE` | +| capabilities | `object` | A [Capabilities object](#capabilities-object-definition) | + +### Object Definitions + +#### Offer Object Definition + +The "type" must be set to "OFFER", with the message body in an "offer" object. +For a living reference, see [libcast's offer_messages.h](../cast/streaming/public/offer_messages.h). + +NOTE: the libcast implementation separates out `supportedStreams` into strongly +typed `audio_streams` and `video_streams` arrays, but they are collated together +when serialized to JSON. + +| Name | Type | Value/description | +|------------------|---------------------------|-------------------------------| +| supportedStreams | array of `Stream` objects | An array of stream objects describing all acceptable stream formats that this endpoint supports. Sender only includes codecs it supports and the order of the stream objects shows the sender's preference. Receiver can choose any stream it prefers, or the first stream it supports if it doesn't have any preferences. Receiver informs sender about the selected stream objects in the sendIndexes of the `ANSWER` object. | +| castMode | `string` | Indicates whether the offer is for "mirroring" or "remoting". See `CastMode` in [`cast/streaming/public/constants.h`](../streaming/public/constants.h). | + +##### Stream object + +The stream object contains a generic section common for both audio and video; it also contains an audio or video specific section based on the type specified in the generic section. + +*** aside +A note on codecs: +The set of codec profiles supported for Cast playback / remoting is notably +larger than the codec profiles supported for mirroring streams. This is due to +the practical limitation that implementing decoding support for a given codec is +generally easier, more likely to have hardware support, and less likely to run +into licensing issues. +*** + +| Name | Type | Value/description | +|----------------------|----------|--------------------------------------------| +| index | `int` | An identifier established by the initiator that MUST contain a Number. The index of the first stream object must start with 0 and each following index MUST be the previous index +1. | +| type | `string` | A String specifying the type of stream offered. Supported values are defined in `Stream::Type` in [`cast/streaming/public/offer_messages.h`](../streaming/public/offer_messages.h). | +| codecName | `string` | A String specifying the codec. Supported values are defined in `AudioCodec` and `VideoCodec` in [`cast/streaming/public/constants.h`](../streaming/public/constants.h). To be compliant, cast receivers must at least implement `opus`, if they support audio, and `vp8` if they support video. Senders must implement at least the baseline codecs (`h264` or `vp8`, and `aac` or `opus`). | +| codecParameter | `string` | A string specifying the codec parameter, in accordance with [RFC.6381](https://datatracker.ietf.org/doc/html/rfc6381#section-3.2). Also known as the "media type string" as in [**Supported Media for Google Cast**](https://developers.google.com/cast/docs/media). Examples include `avc1.64002A` for H.264 level 4.2, and `hev1.1.6.L150.B0` for H.265 main 5.0. | +| rtpProfile | `string` | A String specifying supported RTP profile. Currently only `cast` is supported, with `codec` reserved for future interop with the intention of being used to indicate that codec-defined RTP profiles (defined by their respective RFCs) shall be used. | +| rtpPayloadType | `int` | A Number specifying the RTP payload type used for this stream. *Valid values are in the range [96, 127]. See [RtpPayloadType](../streaming/impl/rtp_defines.h)* | +| ssrc | `uint32` | A Number specifying the RTP SSRC used for this stream. Values must be unique between all streams for this sender. All values are valid. | +| targetDelay | `int` | Indicates the desired total end-to-end latency. | +| aesKey | `string` | A String specifying which AES key to use. Must consist of exactly 32 hex digits. Both an AES key and initialization vector are required: if either field is missing, this stream is invalid. | +| aesIvMask | `string` | A String specifying which initialization vector mask to use. Must consist of exactly 32 hex digits. Must be provided. | +| receiverRtcpEventLog | `boolean` (optional) | True to request receiver to send event log via RTCP. False otherwise. | +| receiverRtcpDscp | `int` (optional) | Request receiver to send RTCP packets using DSCP value indicated. Typically this value is 46. | +| rtpExtensions | `Array of string` (optional) | RTP extensions supported by the Sender. Receivers can then reply with a list of rtpExtensions from this list that it also supports. | +| timeBase | `string` (optional) | Number specifying the time base used by this "rtpPayloadType". Default value is `1/90000`. Valid values are "1/\" where sample rate is strictly positive. | + +##### Audio Stream object + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| bitRate | `int` | A Number specifying the average bitrate in bits per second used by this "rtpPayloadType". | +| channels | `int` | A Number specifying the number of audio channels used by this "rtpPayloadType". | + +##### Video Stream object + +Note that additional video codec information such as codec profile and level, and video stream protection, are not implemented by any current senders or receivers. If these features need to be used in the future, they should be reimplemented.>TODO(crbug.com/471102790): The implementation includes `profile` and `level` fields, which contradicts the spec's claim that they are not implemented. The spec should be updated to reflect their presence. + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| maxFrameRate | `string` | Max number of frames per second used by this "rtpPayloadType". Note: Receivers may ignore this field when providing constraints in the `ANSWER` message. In this case, the sender must respect those constraints. | +| maxBitRate | `int` | Max bitrate in bits per second used by this "rtpPayloadType". Note: Receivers may ignore this field when providing constraints in the `ANSWER` message. In this case, the sender must respect those constraints. | +| resolutions | array of Video resolution objects | An array of resolutions supported by this "rtpPayloadType". Note: Receivers may ignore this field when providing constraints in the `ANSWER` message. In this case, the sender must respect those constraints. | +| errorRecoveryMode | `string` (optional) | String to indicate how video stream is encoded. Default value is **castv2.** "**castv2**" means that the receiver cannot drop any video packets. There is no key frame or intra refresh mode after the first video frame in the session. "**intra_mb_refresh**" means that frames are encoded using intra macroblock refresh mode. The receiver can drop a video frame and recover later on after receiving new key frames or intra refresh macroblocks. | + +##### Video resolution object + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| width | `int` | Width in pixels. | +| height | `int` | Height in pixels. | + +#### Error Object Definition + +The "type" may be set to anything, but the "result" field must be present and +set to "error", with the message body in an "error" object. + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| code | `int32` | A code indicating what class of error occurred. | +| description | `string` | Description of the error. | + +For a comprehensive and up-to-date list of error codes, refer to the +`openscreen::Error::Code` enum in [`platform/base/error.h`](../platform/base/error.h). + +#### Answer Object Definition + +The "type" must be set to "ANSWER", with the message body in an "answer" object. For a living reference, see [libcast's answer_messages.h](../cast/streaming/public/answer_messages.h). + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| udpPort | `int` | A Number specifying the UDP port used for all streams (RTP and RTCP) in this session. *Note: values 1 to 65535 are valid.* | +| sendIndexes | `Array of int` | Numbers specifying the indexes chosen from the `OFFER` message. | +| ssrcs | `Array of uint32` | Numbers specifying the RTP SSRC used to send the RTCP feedback of the stream indicated by the "sendIndexes" above. Order must match `sendIndexes` exactly. *Note: values 0 to 2^32 are valid.* | +| constraints | `receiver constraints object` (optional, but highly recommended) | Provides detailed maximum capabilities of the receiver for processing the streams selected in "sendIndexes" above; including audio sampling rate and number of channels, video dimensions and rates, encoded bit rates, and target latency. A sender may alter video resolution or frame rate throughout a session. The constraints here restrict how much data volume is allowed before the sender must subsample (e.g., downscale and/or reduce frame rate). | +| display | `display description object` (optional, but highly recommended) | Provides details about the display on the receiver, including dimensions (aspect ratio implied), scaling behavior, color profile, etc. | +| receiverRtcpEventLog | `Array of int` (optional) | Numbers specifying the indexes of streams that will send event log via RTCP. If this field is not present then the receiver does not support sending an event log via RTCP. | +| receiverRtcpDscp | `Array of int` (optional) | Numbers specifying the indexes of streams that will use DSCP values specified in the `OFFER` message for RTCP packets. If this field is not present then the receiver does not support DSCP. | +| rtpExtensions | `Array of Array of string` (optional) | Arrays specifying the RTP extensions enabled for each stream, in the same order as `sendIndexes`. If this field is not present then the receiver does not support any RTP extensions. | + +##### Receiver Constraints Object Definition + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| audio | `audio receiver constraints object` | Audio constraints. See below. | +| video | `video receiver constraints object` | Video constraints. See below. | + +##### Audio Receiver Constraints Object Definition + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| codecName | `string` | Audio codec name. See `AudioCodec` in [`cast/streaming/public/constants.h`](../streaming/public/constants.h). | +| maxSampleRate | `int` | Maximum supported sampling frequency (not necessarily the ideal sampling frequency). | +| maxChannels | `int` | Maximum number of audio channels supported. The number here is interpreted to relate to a standard speaker layout (e.g., 2 for left-and-right stereo, 5 for a left+center+right+left_surround+right_surround). | +| minBitRate | `int` (optional) | Minimum encoded audio data bits per second. If not specified, the sender will assume 32 kbps. Note: A receiver should never restrict the minBitRate to try to improve quality. This should reflect the true operational minimum. | +| maxBitRate | `int` | Maximum encoded audio data bits per second. This is the lower of: 1\) The maximum capability of the decoder; or 2\) The maximum sustained data transfer rate (e.g., could be limited by the CPU, RAM bandwidth, etc.). If not specified, the sender will assume no greater than 320kbps. | +| maxDelay | `int` (optional) | Maximum supported end-to-end latency, in milliseconds, for audio. This is proportional to the size of the data buffers in the receiver. Meaning, assume a very low-latency link between sender and receiver, and this value would indicate the amount of buffering that can be maintained (due to RAM capacity, etc.). If not provided, a default of 1200ms should be used. | + +##### Video Receiver Constraints Object Definition + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| codecName | `string` (optional) | Video codec name. See `VideoCodec` in [`cast/streaming/public/constants.h`](../streaming/public/constants.h). If omitted, these constraints apply to all video codecs. | +| maxPixelsPerSecond | `double` (optional) | Maximum pixel rate (width x height x framerate). Note that this value can, and often will be, much less than multiplying the fields in maxDimensions. The purpose of this field is to limit the overall maximum processing rate. A sender will use this, in conjunction with the fields below, to trade-off between higher/lower resolution and lower/higher frame rate. *Example: A device may be capable of 62208000 pixels per second, which allows a sender to send* 1280x720@60 or 1920x1080@30*. In this example, the maxDimensions might specify {width:1920, height:1080, frameRate:60}.* | +| minResolution | `resolution object` (optional) | Minimum width and height in pixels. If not specified, the sender will assume a reasonable minimum having the same aspect ratio as maxDimensions, with an area as close to 320x180 as possible. Note: A receiver should never restrict the minResolution in an effort to improve quality. This should reflect the true operational minimum. | +| maxDimensions | `dimensions object` | Maximum width and height in pixels (not necessarily the ideal width or height), and the maximum frame rate (not necessarily the ideal frame rate). | +| minBitRate | `int` (optional) | Minimum encoded video data bits per second. If not specified, the sender will assume 300 kbps. Note: A receiver should never restrict the minBitRate in an effort to improve quality. This should reflect the true operational minimum. | +| maxBitRate | `int` | Maximum encoded video data bits per second. This is the lower of: 1\) The maximum capability of the decoder; or 2\) The maximum sustained data transfer rate (e.g., could be limited by the CPU, RAM bandwidth, etc.). | +| maxDelay | `int` (optional) | Maximum supported end-to-end latency, in milliseconds, for video. This is proportional to the size of the data buffers in the receiver. Meaning, assume a very low-latency link between sender and receiver, and this value would indicate the amount of buffering that can be maintained (due to RAM capacity, etc.). If not provided, a default of 1200ms should be used. | + +##### Resolution Object Definition + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| width | `int` | Width, in pixels. | +| height | `int` | Height, in pixels. | + +##### Dimensions Object Definition + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| width | `int` | Width, in pixels. | +| height | `int` | Height, in pixels. | +| frameRate | `string` | Frame rate. This should be specified as a rational decimal number (e.g., "30" or "30000/1001"). | + +##### Display Description Object Definition + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| dimensions | `dimensions object` (optional) | If present, the receiver is attached to a fixed display having the given dimensions and frame rate (vsync) configuration. These dimensions may exceed, be the same, or be less than those mentioned in the constraints. If undefined, the receiver display is assumed to be fixed (e.g., a panel in a Hangouts UI). The sender uses this to decide the best way to sample, capture, and encode the content to optimize the user viewing experience. | +| aspectRatio | `string` (optional) | The aspect ratio, in "#:#" format, when the receiver is attached to a fixed display. When missing and dimensions are specified, the sender will assume pixels are square, and the dimensions imply the aspect ratio of the fixed display. When present and dimensions are also specified, this implies the display pixels are not square. | +| scaling | `string` (optional) | One of: "sender" The sender must scale and letterbox the content and provide video frames of a fixed aspect ratio. "receiver" The sender may send arbitrarily sized frames, and the receiver will handle the scaling and letterboxing as necessary for proper display. | + +*** aside +Support for color balance profile, bit depth, and other properties has been +discussed in the past, but never added to the spec or finalized. +*** + +##### Capabilities Object Definition + +| Name | Type | Value/description | +| :---- | :---- | :---- | +| mediaCaps | `array of string` | List of media capabilities of the receiver. See `AudioCapability` and `VideoCapability` in [`cast/streaming/remoting_capabilities.h`](../streaming/remoting_capabilities.h). `video` is deprecated and not used. | +| remoting | `int` | Remoting version of the receiver. | +| result | `string` | Indicates whether getting capabilities succeeded, must be either "ok" or "error." | + +*** promo +`audio` is a special value that indicates support for an set of codecs that have +been defined as a "baseline" set. The current baseline set is defined in the +Chrome [RendererController](https://source.chromium.org/chromium/chromium/src/+/main:media/remoting/renderer_controller.cc;drc=7cc84add564ab18554cefa903004afca74849fe3;l=460) +as the following list: + +1. **MP3** +2. **PCM** (Baseline, S16BE, S24BE, ALAW) +3. **Vorbis** +4. **FLAC** (Free Lossless Audio codec) +5. **AMR** (both narrow band and wide band) +6. **GSM MS** (a special Microsoft version of GSM Full Rate) +7. **Enhanced AC-3** (Dolby Digital Plus) +8. **ALAC** (Apple Lossless Audio Codec) +9. **AC3** (also known as Dolby Digital) +10. **DTS-HD** Master Audio +11. **DTS:X** (Profile 2, lossy) +12. **DTS** Extended Surround + +*** + +*** aside +Some legacy receivers may report `vp9` and `hevc` in their `mediaCaps` response, +even if they cannot remote these codecs. +*** + +#### Remoting Messages (`com.google.cast.remoting`) + +Finally, in the `com.google.cast.remoting` namespace contains the following +remoting specific messages: + +##### RPC + +The "type" must be set to "RPC", with the base64-encoded protobuf message stored +as a string under the "rpc" key. + +Protobuf messages are complex, and defined in the [remoting.proto](../streaming/remoting.proto) +file. + +#### Input (Draft) + +The `com.google.cast.remoting` namespace also supports sending input events from +receiver to sender. + +##### INPUT + +The "type" must be set to "INPUT", with the base64-encoded protobuf message +stored as a string under the "input" key. + +Protobuf messages are complex, and defined in the [input.proto](../streaming/input.proto) +file. + +#### Media Status Messages (`com.google.cast.media`) + +Most of these messages are not supported in libcast, and are instead used for +controlling flinging sessions with the Cast SDKs. + +*** aside +TODO(crbug.com/471102790): evaluate including more media message types in +libcast. +*** + +##### MEDIA_STATUS + +Sent by a media application on the receiver to update the sender on the state +of media playback. + +| Name | Type | Value/description | +| :--- | :--- | :--- | +| `responseType` | `string` | Must be `MEDIA_STATUS`. | +| `requestId` | `int` | An identifier for the request, or 0 if unsolicited. | +| `media` | array of `object` | An array containing one or more media status objects. | +| `media[n].mediaSessionId`| `int` | The ID of the media session. | +| `media[n].playerState` | `string` | The state of playback (e.g., `PLAYING`, `PAUSED`, `IDLE`). | +| `media[n].currentTime` | `double` | The current playback time in seconds. | +| `media[n].media` | `object` | An object with metadata about the content, such as `contentId` and `title`. | + +### Reference Schemas and Examples + +*** aside +TODO(crbug.com/471102790): rename castv2 folder to reflect its evergreen nature. +*** + +This specification is backed by two JSON Schemas: + +1. [receiver_schema.json](./castv2/receiver_examples/): containing core receiver + control and status messages, including `LAUNCH`, `STOP`, + `GET_APP_AVAILABILITY`, `LAUNCH_STATUS`, `LAUNCH_ERROR`, `GET_STATUS`, + `RECEIVER_STATUS`, `INVALID_REQUEST`, `GET_DEVICE_INFO`, `eureka_info`, and + `MEDIA_STATUS`. + +2. [streaming_schema.json](./castv2/streaming_schema.json): containing messages + specific to the streaming session, such as `OFFER`, `ANSWER`, + `GET_CAPABILITIES`, `CAPABILITIES_RESPONSE`, and `RPC`. + +Examples are provided in the [castv2/receiver_examples](./castv2/receiver_examples/) +(for receiver control and media status messages) and +[castv2/streaming_examples](./castv2/streaming_examples/) (for streaming +specific messages) folders, with a C++ validation component defined in +[castv2/validation.h](./castv2/validation.h). + +*** note +When adding or modifying messages in this specification, the corresponding +schema and examples should be **updated concurrently**. The syntax of these +files can be validated using `yajsv` -- see the +[castv2/README.md](./castv2/README.md) for more information. +*** + +### Discovering Receiver Capabilities + +Prior to the offer/answer exchange, the sender may desire information about the +receiver in order to create an optimal offer. This discovery of capabilities is +currently limited to the DNS-SD [ca bit-field](https://docs.google.com/document/d/1d1wuxHioJ9cBVBQ6UwqBFVMrn48Nb3Yp5k1a_eTXsHE/edit?usp=sharing), +which indicates whether the receiver supports audio or video. Remoting specific +capabilities may be discovered by the sender using a `GET_CAPABILITIES` call, as +defined below. + +### Keep Alive + +There is **no** official control/application-level keep alive, and the +`PING` and `PONG` messages originally included in the protocol are now +[deprecated](#deprecated-messages). The sender and receiver are both expected to +either disconnect by inferring the session has ended through status messages, or +independently based on media-level timeouts (the media layer must thus send an +event to the application level to do the appropriate cleanup, but the exact +mechanism to this is specific to particular sender and receiver +implementations). + +The default timeout is **15 seconds**. If no media packets (`RTP`, `ACK`, +`NACK`, etc.) are received from the remote peer within this duration, the sender +or receiver that failed to receive data should disconnect. + +*** aside +In legacy Cast devices, an application-level "keep-alive" message was used both +by the sender and the receiver to terminate a streaming session. This was used +to handle a variety of scenarios, with the most common being the desire to +quickly end a session when a user closes their laptop (for example). + +This approach was abandoned in favor of relying on ACKs, status messages, and +timeouts due to these mechanisms resulting in more robust streaming sessions, +as well as reducing unnecessary network traffic and battery drain. +*** + +### Security Considerations + +The Cast v2 protocol includes several security mechanisms to protect the +streaming session. + +#### Encryption + +Media streams *must* be encrypted using [AES-128](https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.197-upd1.pdf). +The `aesKey` and `aesIvMask` fields in the `Stream` object of the `OFFER` +message are used to provide the encryption key and initialization vector mask. +Both are 32-digit hex strings. If these fields are absent, the `OFFER` message +shall be considered invalid and the session request rejected. + +#### Authentication + +Device authentication is performed as the first step of the connection, as shown +in [Appendix A](#appendix-a---sample-cast-channel-message-flow). This is handled +by the `com.google.cast.tp.deviceauth` namespace. The details of the +authentication protocol are outside the scope of this document. + +### Deprecated Messages + +Several message types were originally included in the Cast v2 specification, +but have become deprecated and may or may not be implemented on modern Cast +devices and are not required for devices to be compliant Cast senders or +receivers. + +These message types are called out in the [CastMessageType](../common/channel/message_util.h) +enum in the libcast implementation, as well as listed here: + +* `APPLICATION_BROADCAST` (*reply*): context is *unknown* -- lost to time. + +* `INVALID_PLAYER_STATE` (*reply*): indicates that the player is in an invalid state. + +* `LOAD_FAILED` (*reply*): indicates that loading a media has failed. + +* `LOAD_CANCELLED` (*reply*): indicates that loading a media was cancelled. + +* `MULTIZONE_STATUS` (*request*): context is *unknown* -- lost to time. + +* `PRESENTATION` (*request*): controls zooming, panning, rotation. + +* `PING` (*request*): sent to ask the receiver if it is currently alive. + +* `PONG` (*response*): sent to inform the sender that this receiver is alive. +* +## Appendix A - Sample Cast Channel Message Flow + +This appendix describes the typical sequence of JSON messages exchanged between +a Cast sender and receiver to establish, run, and terminate a streaming session. + +### 1. Discovery and Handshake + +Before a streaming session can be launched, the sender first discovers the +receiver on the network (e.g., via mDNS) and establishes a secure channel. The +following message exchange then occurs: + +1. **Device Authentication**: The sender and receiver exchange authentication + messages to verify their identities. This is a prerequisite for all further + communication. + * **Namespace**: `urn:x-cast:com.google.cast.tp.deviceauth` + +2. **Virtual Connection**: The sender establishes a "virtual connection" to the + main receiver process. This acts as the initial control channel. + * **Sender → Receiver**: `urn:x-cast:com.google.cast.tp.connection`, type + `CONNECT` + +3. **Receiver Status Check**: The sender queries the receiver's current status. + * **Sender → Receiver**: `urn:x-cast:com.google.cast.receiver`, type + `GET_STATUS` + * **Receiver → Sender**: `urn:x-cast:com.google.cast.receiver`, type + `RECEIVER_STATUS`. The reply indicates the currently running application + (typically the "Backdrop" idle screen) and other device state like + volume. + +4. **Application Availability**: The sender checks if the receiver supports the + Cast streaming application. + * **Sender → Receiver**: `urn:x-cast:com.google.cast.receiver`, type + `GET_APP_AVAILABILITY`. The sender sends one request for the standard + audio/video streaming app (`0F5096E8`) and another for the audio-only + app (`85CDB22F`). + * **Receiver → Sender**: `urn:x-cast:com.google.cast.receiver`, type + `GET_APP_AVAILABILITY` response. The receiver confirms whether each app + is available or not. + +### 2. Streaming Session Launch + +Once the user initiates streaming (e.g., by selecting "Cast..." in Chrome), the following message flow begins: + +1. **Launch Request**: The sender requests the receiver to launch the streaming + application. + * **Sender → Receiver**: `urn:x-cast:com.google.cast.receiver`, type `LAUNCH` + + ```json + { + "type": "LAUNCH", + "appId": "0F5096E8", + "requestId": 17 + } + ``` + +2. **Launch Confirmation**: The receiver confirms the launch by sending a + `RECEIVER_STATUS` update. This crucial message contains the `transportId` and + `sessionId` for the newly created application instance. This `transportId` + will be used as the `destination_id` for all subsequent messages to the + streaming app. + + ```json + { + "type": "RECEIVER_STATUS", + "requestId": 17, + "status": { + "applications": [ + { + "appId": "0F5096E8", + "displayName": "Chrome Mirroring", + "sessionId": "154d8823-...", + "transportId": "154d8823-...", + "isIdleScreen": false + } + ] + } + } + ``` + +3. **Connect to Streaming App**: The sender establishes a new virtual + connection, this time directly to the streaming application instance using + its `transportId`. + * **Sender → Streaming App**: `urn:x-cast:com.google.cast.tp.connection`, + type `CONNECT` + +4. **Media Negotiation (Offer/Answer)**: The sender and receiver negotiate + the media format for streaming. + * **Sender → Streaming App**: `urn:x-cast:com.google.cast.webrtc`, type + `OFFER`. The sender proposes a set of supported audio and video streams. + + ```json + { + "type": "OFFER", + "seqNum": 820263768, + "offer": { + "castMode": "mirroring", + "supportedStreams": [ + { + "index": 0, + "type": "audio_source", + "codecName": "opus", + "rtpPayloadType": 127, + "ssrc": 264890, + "targetDelay": 400, + "channels": 2, + ... + }, + { + "index": 1, + "type": "video_source", + "codecName": "vp8", + "rtpPayloadType": 96, + "ssrc": 748229, + "maxFrameRate": "30", + "resolutions": [{"width": 1920, "height": 1080}], + ... + } + ] + } + } + ``` + + * **Streaming App → Sender**: `urn:x-cast:com.google.cast.webrtc`, type + `ANSWER`. The receiver accepts the offer, selects which streams it will + use (via `sendIndexes`), and specifies the UDP port for media transport. + + ```json + { + "type": "ANSWER", + "seqNum": 820263768, + "result": "ok", + "answer": { + "udpPort": 33533, + "sendIndexes": [0, 1], + "ssrcs": [264891, 748230] + } + } + ``` + +5. **Streaming Active**: With the negotiation complete, media begins to flow + over UDP. The streaming app sends a `MEDIA_STATUS` message to confirm that + playback has started. + * **Streaming App → Sender**: `urn:x-cast:com.google.cast.media`, + type `MEDIA_STATUS` + +### 3. Streaming Session Termination + +When the user stops the session: + +1. **Stop Request**: The sender sends a `STOP` message to the main receiver + process, referencing the `sessionId` of the streaming application. + * **Sender → Receiver**: `urn:x-cast:com.google.cast.receiver`, + type `STOP` + + ```json + { + "type": "STOP", + "sessionId": "154d8823-...", + "requestId": 22 + } + ``` + +2. **Close Connection**: The streaming application, upon termination, sends a + `CLOSE` message to tear down its virtual connection with the sender. + * **Streaming App → Sender**: `urn:x-cast:com.google.cast.tp.connection`, + type `CLOSE` + +3. **Return to Idle**: The receiver returns to the idle screen and broadcasts a + final `RECEIVER_STATUS` update, showing that "Backdrop" is now the active + application. + diff --git a/cast/receiver/BUILD.gn b/cast/receiver/BUILD.gn index b87d547db..0b891c3cf 100644 --- a/cast/receiver/BUILD.gn +++ b/cast/receiver/BUILD.gn @@ -25,7 +25,6 @@ openscreen_source_set("channel") { public_deps = [ "../../platform", - "../../third_party/abseil", "../../third_party/boringssl", "../common:channel", "../common/channel/proto:channel_proto", diff --git a/cast/receiver/application_agent.cc b/cast/receiver/application_agent.cc index 852a60b44..960154033 100644 --- a/cast/receiver/application_agent.cc +++ b/cast/receiver/application_agent.cc @@ -9,6 +9,7 @@ #include "cast/common/channel/message_util.h" #include "cast/common/channel/virtual_connection.h" #include "cast/common/public/cast_socket.h" +#include "cast/common/public/receiver_info.h" #include "platform/base/tls_credentials.h" #include "platform/base/tls_listen_options.h" #include "util/json/json_helpers.h" @@ -29,11 +30,13 @@ std::string GetFirstAppId(ApplicationAgent::Application* app) { ApplicationAgent::ApplicationAgent( TaskRunner& task_runner, - DeviceAuthNamespaceHandler::CredentialsProvider& credentials_provider) + DeviceAuthNamespaceHandler::CredentialsProvider& credentials_provider, + const std::string& device_uuid) : task_runner_(task_runner), auth_handler_(credentials_provider), connection_handler_(router_, *this), - message_port_(router_) { + message_port_(router_), + device_id_(device_uuid) { router_.AddHandlerForLocalId(kPlatformReceiverId, this); } @@ -136,6 +139,8 @@ void ApplicationAgent::OnMessage(VirtualConnectionRouter* router, response = HandlePing(); } } else if (ns == kReceiverNamespace) { + const Json::Value& value = + request.value().get(kMessageKeyType, Json::Value::nullSingleton()); if (request.value()[kMessageKeyRequestId].isNull()) { response = HandleInvalidCommand(request.value()); } else if (HasType(request.value(), CastMessageType::kGetAppAvailability)) { @@ -149,6 +154,22 @@ void ApplicationAgent::OnMessage(VirtualConnectionRouter* router, } else { response = HandleInvalidCommand(request.value()); } + } else if (ns == kSetupNamespace) { + const Json::Value& value = + request.value().get(kMessageKeyType, Json::Value::nullSingleton()); + if (HasType(request.value(), CastMessageType::kEurekaInfo)) { + response = HandleEurekaInfo(request.value()); + } else { + response = HandleInvalidCommand(request.value()); + } + } else if (ns == kDiscoveryNamespace) { + const Json::Value& value = + request.value().get(kMessageKeyType, Json::Value::nullSingleton()); + if (HasType(request.value(), CastMessageType::kGetDeviceInfo)) { + response = HandleDeviceInfo(request.value()); + } else { + response = HandleInvalidCommand(request.value()); + } } else { // Ignore messages for all other namespaces. } @@ -158,6 +179,15 @@ void ApplicationAgent::OnMessage(VirtualConnectionRouter* router, message.source_id(), ToCastSocketId(socket)}, MakeSimpleUTF8Message(ns, json::Stringify(response).value())); } + + // Send another RECEIVER_STATUS if LAUNCH succeeds. + if (HasType(request.value(), CastMessageType::kLaunch) && + HasType(response, CastMessageType::kLaunchStatus)) { + response = HandleGetStatus(request.value()); + router_.Send(VirtualConnection{message.destination_id(), + message.source_id(), ToCastSocketId(socket)}, + MakeSimpleUTF8Message(ns, json::Stringify(response).value())); + } } bool ApplicationAgent::IsConnectionAllowed( @@ -220,6 +250,21 @@ Json::Value ApplicationAgent::HandleGetStatus(const Json::Value& request) { return response; } +Json::Value ApplicationAgent::HandleDeviceInfo(const Json::Value& request) { + Json::Value response; + PopulateDeviceInfo(&response); + response[kMessageKeyRequestId] = request[kMessageKeyRequestId]; + return response; +} + +Json::Value ApplicationAgent::HandleEurekaInfo(const Json::Value& request) { + Json::Value response; + PopulateEurekaInfo(&response); + response[kMessageKeyEurekaInfoRequestId] = + request[kMessageKeyEurekaInfoRequestId]; + return response; +} + Json::Value ApplicationAgent::HandleLaunch(const Json::Value& request, CastSocket* socket) { const Json::Value& app_id = request[kMessageKeyAppId]; @@ -230,19 +275,19 @@ Json::Value ApplicationAgent::HandleLaunch(const Json::Value& request, } else { error = Error(Error::Code::kParameterInvalid, kMessageValueBadParameter); } + Json::Value response; if (!error.ok()) { - Json::Value response; response[kMessageKeyRequestId] = request[kMessageKeyRequestId]; response[kMessageKeyType] = CastMessageTypeToString(CastMessageType::kLaunchError); response[kMessageKeyReason] = error.message(); - return response; + } else { + response[kMessageKeyType] = + CastMessageTypeToString(CastMessageType::kLaunchStatus); + response[kMessageKeyLaunchRequestId] = request[kMessageKeyRequestId]; + response[kMessageKeyStatus] = kMessageValueUserAllowed; } - - // Note: No reply is sent. Instead, the requestor will get a RECEIVER_STATUS - // broadcast message from SwitchToApplication(), which is how it will see that - // the launch succeeded. - return {}; + return response; } Json::Value ApplicationAgent::HandleStop(const Json::Value& request) { @@ -392,6 +437,45 @@ void ApplicationAgent::BroadcastReceiverStatus() { json::Stringify(message).value())); } +void ApplicationAgent::SetReceiverInfo(ReceiverInfo receiver_info) { + receiver_info_ = receiver_info; +} + +void ApplicationAgent::PopulateDeviceInfo(Json::Value* out) { + Json::Value& message = *out; + + message[kMessageKeyControlNotifications] = 1; + message[kMessageKeyDeviceCapabilities] = kDefaultDeviceCapabilities; + message[kMessageKeyDeviceId] = receiver_info_.GetInstanceId(); + message[kMessageKeyDeviceModel] = receiver_info_.model_name; + message[kMessageKeyFriendlyName] = receiver_info_.friendly_name; + message[kMessageKeyType] = + CastMessageTypeToString(CastMessageType::kGetDeviceInfo); +} + +void ApplicationAgent::PopulateEurekaInfo(Json::Value* out) { + Json::Value& message = *out; + + Json::Value& data = message[kMessageKeyData]; + data[kMessageKeyVersion] = 12; + data[kMessageKeyName] = receiver_info_.friendly_name; + + Json::Value& device_info = data[kMessageKeyDeviceInfo]; + device_info[kMessageKeyManufacturer] = "google"; + device_info[kMessageKeyProductName] = receiver_info_.model_name; + device_info[kMessageKeySsdpUdn] = receiver_info_.GetInstanceId(); + + Json::Value& build_info = data[kMessageKeyBuildInfo]; + build_info[kMessageKeyBuildType] = 2; + build_info[kMessageKeyCastBuildRevision] = "1.0"; + build_info[kMessageKeySystemBuildNumber] = "BUILD_NUMBER"; + + message[kMessageKeyResponseCode] = 200; + message[kMessageKeyResponseString] = "OK"; + message[kMessageKeyType] = + CastMessageTypeToString(CastMessageType::kEurekaInfo); +} + ApplicationAgent::Application::~Application() = default; } // namespace openscreen::cast diff --git a/cast/receiver/application_agent.h b/cast/receiver/application_agent.h index 99c35043e..cbe0c0631 100644 --- a/cast/receiver/application_agent.h +++ b/cast/receiver/application_agent.h @@ -14,6 +14,7 @@ #include "cast/common/channel/connection_namespace_handler.h" #include "cast/common/channel/virtual_connection_router.h" #include "cast/common/public/cast_socket.h" +#include "cast/common/public/receiver_info.h" #include "cast/receiver/channel/device_auth_namespace_handler.h" #include "cast/receiver/public/receiver_socket_factory.h" #include "platform/api/task_runner.h" @@ -58,7 +59,7 @@ class ApplicationAgent final // Launches the application and returns true if successful. `app_id` is the // specific ID that was used to launch the app, and `app_params` is a - // pass-through for any arbitrary app-specfic structure (or null if not + // pass-through for any arbitrary app-specific structure (or null if not // provided). If the Application wishes to send/receive messages, it uses // the provided `message_port` and must call MessagePort::SetClient() before // any flow will occur. @@ -81,7 +82,8 @@ class ApplicationAgent final ApplicationAgent( TaskRunner& task_runner, - DeviceAuthNamespaceHandler::CredentialsProvider& credentials_provider); + DeviceAuthNamespaceHandler::CredentialsProvider& credentials_provider, + const std::string& device_uuid); ~ApplicationAgent() final; @@ -101,6 +103,8 @@ class ApplicationAgent final // errors). void StopApplicationIfRunning(Application* app); + void SetReceiverInfo(ReceiverInfo receiver_info); + private: // ReceiverSocketFactory::Client overrides. void OnConnected(ReceiverSocketFactory* factory, @@ -126,6 +130,8 @@ class ApplicationAgent final Json::Value HandlePing(); Json::Value HandleGetAppAvailability(const Json::Value& request); Json::Value HandleGetStatus(const Json::Value& request); + Json::Value HandleDeviceInfo(const Json::Value& request); + Json::Value HandleEurekaInfo(const Json::Value& request); Json::Value HandleLaunch(const Json::Value& request, CastSocket* socket); Json::Value HandleStop(const Json::Value& request); Json::Value HandleInvalidCommand(const Json::Value& request); @@ -142,10 +148,20 @@ class ApplicationAgent final // Stops the currently-running Application and launches the "idle screen." void GoIdle(); - // Populates the given `message` object with the RECEIVER_STATUS fields, + // Populates the given `out` object with the RECEIVER_STATUS fields, + // reflecting the currently-launched app (if any), and a fake volume level + // status. + void PopulateReceiverStatus(Json::Value* out); + + // Populates the given `out` object with the DEVICE_INFO fields, + // reflecting the currently-launched app (if any), and a fake volume level + // status. + void PopulateDeviceInfo(Json::Value* out); + + // Populates the given `out` object with the eureka_info fields, // reflecting the currently-launched app (if any), and a fake volume level // status. - void PopulateReceiverStatus(Json::Value* message); + void PopulateEurekaInfo(Json::Value* out); // Broadcasts new RECEIVER_STATUS to all endpoints. This is called after an // Application LAUNCH or STOP. @@ -162,6 +178,8 @@ class ApplicationAgent final CastSocketMessagePort message_port_; Application* launched_app_ = nullptr; std::string launched_via_app_id_; + std::string device_id_; + ReceiverInfo receiver_info_; }; } // namespace openscreen::cast diff --git a/cast/receiver/application_agent_unittest.cc b/cast/receiver/application_agent_unittest.cc index 78dcf80ab..f7b74781a 100644 --- a/cast/receiver/application_agent_unittest.cc +++ b/cast/receiver/application_agent_unittest.cc @@ -30,7 +30,6 @@ namespace { using proto::CastMessage; using ::testing::_; -using ::testing::Invoke; using ::testing::Mock; using ::testing::Ne; using ::testing::NiceMock; @@ -207,10 +206,10 @@ class ApplicationAgentTest : public ::testing::Test { // The remote will send the auth challenge message and get a reply. EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) - .WillOnce(Invoke([](CastSocket*, CastMessage message) { + .WillOnce([](CastSocket*, CastMessage message) { EXPECT_EQ(kAuthNamespace, message.namespace_()); EXPECT_FALSE(message.payload_binary().empty()); - })); + }); const auto result = sender_outbound()->Send(TestAuthChallengeMessage()); ASSERT_TRUE(result.ok()) << result; Mock::VerifyAndClearExpectations(sender_inbound()); @@ -220,7 +219,7 @@ class ApplicationAgentTest : public ::testing::Test { // The ApplicationAgent should send a final "no apps running" // RECEIVER_STATUS broadcast to the sender at destruction time. EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) - .WillOnce(Invoke([](CastSocket*, CastMessage message) { + .WillOnce([](CastSocket*, CastMessage message) { constexpr char kExpectedJson[] = R"({ "requestId":0, "type":"RECEIVER_STATUS", @@ -237,7 +236,7 @@ class ApplicationAgentTest : public ::testing::Test { const Json::Value payload = ValidateAndParseMessage( message, kPlatformReceiverId, kBroadcastId, kReceiverNamespace); EXPECT_EQ(json::Parse(kExpectedJson).value(), payload); - })); + }); } FakeClock clock_{Clock::time_point() + std::chrono::hours(1)}; @@ -245,7 +244,7 @@ class ApplicationAgentTest : public ::testing::Test { FakeCastSocketPair socket_pair_; StrictMock idle_app_{"E8C28D3C", "Backdrop"}; TestCredentialsProvider creds_; - ApplicationAgent agent_{task_runner_, creds_}; + ApplicationAgent agent_{task_runner_, creds_, "fakeId"}; }; TEST_F(ApplicationAgentTest, JustConnectsWithoutDoingAnything) {} @@ -273,7 +272,7 @@ TEST_F(ApplicationAgentTest, IgnoresGarbageMessages) { TEST_F(ApplicationAgentTest, HandlesInvalidCommands) { EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) - .WillOnce(Invoke([&](CastSocket*, CastMessage message) { + .WillOnce([&](CastSocket*, CastMessage message) { constexpr char kExpectedJson[] = R"({ "requestId":3, "type":"INVALID_REQUEST", @@ -283,7 +282,7 @@ TEST_F(ApplicationAgentTest, HandlesInvalidCommands) { ValidateAndParseMessage(message, kPlatformReceiverId, kPlatformSenderId, kReceiverNamespace); EXPECT_EQ(json::Parse(kExpectedJson).value(), payload); - })); + }); auto result = sender_outbound()->Send(MakeCastMessage( kPlatformSenderId, kPlatformReceiverId, kReceiverNamespace, R"({ "requestId":3, @@ -298,13 +297,13 @@ TEST_F(ApplicationAgentTest, HandlesPings) { int num_pongs = 0; EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) .Times(kNumPings) - .WillRepeatedly(Invoke([&num_pongs](CastSocket*, CastMessage message) { + .WillRepeatedly([&num_pongs](CastSocket*, CastMessage message) { const Json::Value payload = ValidateAndParseMessage(message, kPlatformReceiverId, kPlatformSenderId, kHeartbeatNamespace); EXPECT_EQ(json::Parse(R"({"type":"PONG"})").value(), payload); ++num_pongs; - })); + }); const CastMessage message = MakeCastMessage(kPlatformSenderId, kPlatformReceiverId, @@ -320,7 +319,7 @@ TEST_F(ApplicationAgentTest, HandlesGetAppAvailability) { // Send the request before any apps have been registered. Expect an // "unavailable" response. EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) - .WillOnce(Invoke([&](CastSocket*, CastMessage message) { + .WillOnce([&](CastSocket*, CastMessage message) { constexpr char kExpectedJson[] = R"({ "requestId":548, "responseType":"GET_APP_AVAILABILITY", @@ -330,7 +329,7 @@ TEST_F(ApplicationAgentTest, HandlesGetAppAvailability) { ValidateAndParseMessage(message, kPlatformReceiverId, kPlatformSenderId, kReceiverNamespace); EXPECT_EQ(json::Parse(kExpectedJson).value(), payload); - })); + }); auto result = sender_outbound()->Send(MakeCastMessage( kPlatformSenderId, kPlatformReceiverId, kReceiverNamespace, R"({ "requestId":548, @@ -345,7 +344,7 @@ TEST_F(ApplicationAgentTest, HandlesGetAppAvailability) { // Send another request, and expect the application to be available. EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) - .WillOnce(Invoke([&](CastSocket*, CastMessage message) { + .WillOnce([&](CastSocket*, CastMessage message) { constexpr char kExpectedJson[] = R"({ "requestId":549, "responseType":"GET_APP_AVAILABILITY", @@ -355,7 +354,7 @@ TEST_F(ApplicationAgentTest, HandlesGetAppAvailability) { ValidateAndParseMessage(message, kPlatformReceiverId, kPlatformSenderId, kReceiverNamespace); EXPECT_EQ(json::Parse(kExpectedJson).value(), payload); - })); + }); result = sender_outbound()->Send(MakeCastMessage( kPlatformSenderId, kPlatformReceiverId, kReceiverNamespace, R"({ "requestId":549, @@ -369,7 +368,7 @@ TEST_F(ApplicationAgentTest, HandlesGetAppAvailability) { TEST_F(ApplicationAgentTest, HandlesGetStatus) { EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) - .WillOnce(Invoke([&](CastSocket*, CastMessage message) { + .WillOnce([&](CastSocket*, CastMessage message) { // NOTE: These appIDs and the displayName come from `idle_app_`. constexpr char kExpectedJson[] = R"({ "requestId":123, @@ -399,7 +398,7 @@ TEST_F(ApplicationAgentTest, HandlesGetStatus) { ValidateAndParseMessage(message, kPlatformReceiverId, kPlatformSenderId, kReceiverNamespace); EXPECT_EQ(json::Parse(kExpectedJson).value(), payload); - })); + }); auto result = sender_outbound()->Send(MakeCastMessage( kPlatformSenderId, kPlatformReceiverId, kReceiverNamespace, R"({ "requestId":123, @@ -410,7 +409,7 @@ TEST_F(ApplicationAgentTest, HandlesGetStatus) { TEST_F(ApplicationAgentTest, FailsLaunchRequestWithBadAppID) { EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) - .WillOnce(Invoke([&](CastSocket*, CastMessage message) { + .WillOnce([&](CastSocket*, CastMessage message) { constexpr char kExpectedJson[] = R"({ "requestId":1, "type":"LAUNCH_ERROR", @@ -420,7 +419,7 @@ TEST_F(ApplicationAgentTest, FailsLaunchRequestWithBadAppID) { ValidateAndParseMessage(message, kPlatformReceiverId, kPlatformSenderId, kReceiverNamespace); EXPECT_EQ(json::Parse(kExpectedJson).value(), payload); - })); + }); auto launch_result = sender_outbound()->Send(MakeCastMessage( kPlatformSenderId, kPlatformReceiverId, kReceiverNamespace, R"({ "requestId":1, @@ -445,11 +444,11 @@ TEST_F(ApplicationAgentTest, LaunchesApp_PassesMessages_ThenStopsApp) { EXPECT_CALL(*idle_app(), DidStop()).InSequence(phase1); EXPECT_CALL(some_app, DidLaunch(_, NotNull())) .InSequence(phase1) - .WillOnce(Invoke([&](Json::Value params, MessagePort* port) { + .WillOnce([&](Json::Value params, MessagePort* port) { EXPECT_EQ(json::Parse(R"({"a":1,"b":2})").value(), params); port_for_app = port; port->SetClient(some_app); - })); + }); // Notes: // - requestId is 0 for broadcast (no requestor). // - These appIDs and the displayName come from `some_app`. @@ -480,11 +479,38 @@ TEST_F(ApplicationAgentTest, LaunchesApp_PassesMessages_ThenStopsApp) { })"; EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) .InSequence(phase1) - .WillOnce(Invoke([&](CastSocket*, CastMessage message) { + .WillOnce([&](CastSocket*, CastMessage message) { const Json::Value payload = ValidateAndParseMessage( message, kPlatformReceiverId, kBroadcastId, kReceiverNamespace); EXPECT_EQ(json::Parse(kRunningAppReceiverStatus).value(), payload); - })); + }); + + EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) + .InSequence(phase1) + .WillOnce([&](CastSocket*, CastMessage message) { + constexpr char kExpectedJson[] = R"({ + "type":"LAUNCH_STATUS", + "launchRequestId":17, + "status":"USER_ALLOWED" + })"; + const Json::Value payload = + ValidateAndParseMessage(message, kPlatformReceiverId, + kPlatformSenderId, kReceiverNamespace); + EXPECT_EQ(json::Parse(kExpectedJson).value(), payload); + }); + + EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) + .InSequence(phase1) + .WillOnce([&](CastSocket*, CastMessage message) { + const Json::Value payload = + ValidateAndParseMessage(message, kPlatformReceiverId, + kPlatformSenderId, kReceiverNamespace); + ErrorOr status = json::Parse(kRunningAppReceiverStatus); + Json::Value expected = status.value(); + expected["requestId"] = 17; + EXPECT_EQ(expected, payload); + }); + auto launch_result = sender_outbound()->Send(MakeCastMessage( kPlatformSenderId, kPlatformReceiverId, kReceiverNamespace, R"({ "requestId":17, @@ -508,9 +534,9 @@ TEST_F(ApplicationAgentTest, LaunchesApp_PassesMessages_ThenStopsApp) { Sequence phase2; EXPECT_CALL(some_app, OnMessage(_, _, _)) .InSequence(phase2) - .WillOnce(Invoke([&](const std::string& source_id, - const std::string& the_namespace, - const std::string& message) { + .WillOnce([&](const std::string& source_id, + const std::string& the_namespace, + const std::string& message) { EXPECT_EQ(kSenderTransportId, source_id); EXPECT_EQ(kAppNamespace, the_namespace); const auto parsed = json::Parse(message); @@ -522,15 +548,15 @@ TEST_F(ApplicationAgentTest, LaunchesApp_PassesMessages_ThenStopsApp) { kReplyMessage); } } - })); + }); EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) .InSequence(phase2) - .WillOnce(Invoke([&](CastSocket*, CastMessage message) { + .WillOnce([&](CastSocket*, CastMessage message) { const Json::Value payload = ValidateAndParseMessage(message, some_app.GetSessionId(), kSenderTransportId, kAppNamespace); EXPECT_EQ(json::Parse(kReplyMessage).value(), payload); - })); + }); auto message_send_result = sender_outbound()->Send(MakeCastMessage( kSenderTransportId, some_app.GetSessionId(), kAppNamespace, kMessage)); ASSERT_TRUE(message_send_result.ok()) << message_send_result; @@ -549,7 +575,7 @@ TEST_F(ApplicationAgentTest, LaunchesApp_PassesMessages_ThenStopsApp) { // - These appIDs and the displayName come from `idle_app_`. EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) .InSequence(phase3) - .WillOnce(Invoke([&](CastSocket*, CastMessage message) { + .WillOnce([&](CastSocket*, CastMessage message) { const std::string kExpectedJson = R"({ "requestId":0, "type":"RECEIVER_STATUS", @@ -577,7 +603,7 @@ TEST_F(ApplicationAgentTest, LaunchesApp_PassesMessages_ThenStopsApp) { const Json::Value payload = ValidateAndParseMessage( message, kPlatformReceiverId, kBroadcastId, kReceiverNamespace); EXPECT_EQ(json::Parse(kExpectedJson).value(), payload); - })); + }); auto stop_result = sender_outbound()->Send(MakeCastMessage( kPlatformSenderId, kPlatformReceiverId, kReceiverNamespace, R"({ "requestId":18, @@ -601,19 +627,28 @@ TEST_F(ApplicationAgentTest, AllowsVirtualConnectionsToApp) { // of the app. EXPECT_CALL(*idle_app(), DidStop()); EXPECT_CALL(some_app, DidLaunch(_, NotNull())) - .WillOnce(Invoke([&](Json::Value params, MessagePort* port) { + .WillOnce([&](Json::Value params, MessagePort* port) { port->SetClient(some_app); - })); + }); std::string transport_id; + Sequence sequence; EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) - .WillRepeatedly(Invoke([&](CastSocket*, CastMessage message) { + .InSequence(sequence) + .WillOnce([&](CastSocket*, CastMessage message) { const Json::Value payload = ValidateAndParseMessage( message, kPlatformReceiverId, kBroadcastId, kReceiverNamespace); if (payload["type"].asString() == "RECEIVER_STATUS") { transport_id = payload["status"]["applications"][0]["transportId"].asString(); } - })); + }); + EXPECT_CALL(*sender_inbound(), OnMessage(_, _)) + .InSequence(sequence) + .WillRepeatedly([&](CastSocket*, CastMessage message) { + const Json::Value payload = + ValidateAndParseMessage(message, kPlatformReceiverId, + kPlatformSenderId, kReceiverNamespace); + }); auto launch_result = sender_outbound()->Send(MakeCastMessage( kPlatformSenderId, kPlatformReceiverId, kReceiverNamespace, R"({ "requestId":1, diff --git a/cast/receiver/channel/device_auth_namespace_handler.cc b/cast/receiver/channel/device_auth_namespace_handler.cc index 3d5d6ff58..dd2625519 100644 --- a/cast/receiver/channel/device_auth_namespace_handler.cc +++ b/cast/receiver/channel/device_auth_namespace_handler.cc @@ -66,7 +66,7 @@ void DeviceAuthNamespaceHandler::OnMessage(VirtualConnectionRouter* router, } const std::string& payload = message.payload_binary(); DeviceAuthMessage device_auth_message; - if (!device_auth_message.ParseFromArray(payload.data(), payload.length())) { + if (!device_auth_message.ParseFromString(payload)) { // TODO(btolsch): Consider all of these cases for future error reporting // mechanism. return; diff --git a/cast/receiver/channel/device_auth_namespace_handler_unittest.cc b/cast/receiver/channel/device_auth_namespace_handler_unittest.cc index 4ff7d8c2e..c35242070 100644 --- a/cast/receiver/channel/device_auth_namespace_handler_unittest.cc +++ b/cast/receiver/channel/device_auth_namespace_handler_unittest.cc @@ -18,6 +18,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "platform/test/paths.h" +#include "util/no_destructor.h" #include "util/read_file.h" namespace openscreen::cast { @@ -30,11 +31,11 @@ using proto::SignatureAlgorithm; using ::testing::_; using ::testing::ElementsAreArray; -using ::testing::Invoke; const std::string& GetSpecificTestDataPath() { - static std::string data_path = GetTestDataPath() + "cast/receiver/channel/"; - return data_path; + static const NoDestructor data_path(GetTestDataPath() + + "cast/receiver/channel/"); + return *data_path; } class DeviceAuthNamespaceHandlerTest : public ::testing::Test { @@ -87,10 +88,9 @@ TEST_F(DeviceAuthNamespaceHandlerTest, AuthResponse) { CastMessage challenge_reply; EXPECT_CALL(fake_cast_socket_pair_.mock_peer_client, OnMessage(_, _)) - .WillOnce( - Invoke([&challenge_reply](CastSocket* socket, CastMessage message) { - challenge_reply = std::move(message); - })); + .WillOnce([&challenge_reply](CastSocket* socket, CastMessage message) { + challenge_reply = std::move(message); + }); ASSERT_TRUE( fake_cast_socket_pair_.peer_socket->Send(std::move(auth_challenge)).ok()); @@ -150,10 +150,9 @@ TEST_F(DeviceAuthNamespaceHandlerTest, BadNonce) { CastMessage challenge_reply; EXPECT_CALL(fake_cast_socket_pair_.mock_peer_client, OnMessage(_, _)) - .WillOnce( - Invoke([&challenge_reply](CastSocket* socket, CastMessage message) { - challenge_reply = std::move(message); - })); + .WillOnce([&challenge_reply](CastSocket* socket, CastMessage message) { + challenge_reply = std::move(message); + }); ASSERT_TRUE( fake_cast_socket_pair_.peer_socket->Send(std::move(auth_challenge)).ok()); @@ -201,10 +200,9 @@ TEST_F(DeviceAuthNamespaceHandlerTest, UnsupportedSignatureAlgorithm) { CastMessage challenge_reply; EXPECT_CALL(fake_cast_socket_pair_.mock_peer_client, OnMessage(_, _)) - .WillOnce( - Invoke([&challenge_reply](CastSocket* socket, CastMessage message) { - challenge_reply = std::move(message); - })); + .WillOnce([&challenge_reply](CastSocket* socket, CastMessage message) { + challenge_reply = std::move(message); + }); ASSERT_TRUE( fake_cast_socket_pair_.peer_socket->Send(std::move(auth_challenge)).ok()); diff --git a/cast/receiver/channel/receiver_socket_factory.cc b/cast/receiver/channel/receiver_socket_factory.cc index 231273d06..f72459e3a 100644 --- a/cast/receiver/channel/receiver_socket_factory.cc +++ b/cast/receiver/channel/receiver_socket_factory.cc @@ -4,6 +4,7 @@ #include "cast/receiver/public/receiver_socket_factory.h" +#include "platform/api/tls_connection.h" #include "util/osp_logging.h" namespace openscreen::cast { @@ -20,10 +21,7 @@ void ReceiverSocketFactory::OnAccepted( TlsConnectionFactory* factory, std::vector der_x509_peer_cert, std::unique_ptr connection) { - IPEndpoint endpoint = connection->GetRemoteEndpoint(); - auto socket = - std::make_unique(std::move(connection), &socket_client_); - client_.OnConnected(this, endpoint, std::move(socket)); + CreateSocket(std::move(connection)); } void ReceiverSocketFactory::OnConnected( @@ -45,4 +43,12 @@ void ReceiverSocketFactory::OnError(TlsConnectionFactory* factory, client_.OnError(this, error); } +void ReceiverSocketFactory::CreateSocket( + std::unique_ptr connection) { + IPEndpoint endpoint = connection->GetRemoteEndpoint(); + auto socket = + std::make_unique(std::move(connection), &socket_client_); + client_.OnConnected(this, endpoint, std::move(socket)); +} + } // namespace openscreen::cast diff --git a/cast/receiver/channel/static_credentials.cc b/cast/receiver/channel/static_credentials.cc index 13d7b7ab5..ef04057d5 100644 --- a/cast/receiver/channel/static_credentials.cc +++ b/cast/receiver/channel/static_credentials.cc @@ -15,6 +15,7 @@ #include "platform/base/tls_credentials.h" #include "util/crypto/certificate_utils.h" +#include "util/crypto/openssl_util.h" #include "util/osp_logging.h" namespace openscreen::cast { @@ -132,6 +133,46 @@ bssl::UniquePtr GenerateRootCert(const EVP_PKEY& root_key) { OSP_CHECK(root_cert_or_error); return std::move(root_cert_or_error.value()); } + +ErrorOr GenerateCredentialsInternal( + const std::string& device_certificate_id, + const std::string& private_key_path, + const std::string& server_certificate_path) { + if (private_key_path.empty() || server_certificate_path.empty()) { + return Error(Error::Code::kParameterInvalid, + "Missing either private key or server certificate"); + } + + FileUniquePtr key_file(fopen(private_key_path.c_str(), "r"), &fclose); + if (!key_file) { + return Error(Error::Code::kParameterInvalid, + "Missing private key file path"); + } + + bssl::UniquePtr root_key = + bssl::UniquePtr(PEM_read_PrivateKey( + key_file.get(), nullptr /* x */, nullptr /* cb */, nullptr /* u */)); + if (!root_key) { + return Error(Error::Code::kParseError, "Failed to parse private key file"); + } + + FileUniquePtr cert_file(fopen(server_certificate_path.c_str(), "r"), &fclose); + if (!cert_file) { + return Error(Error::Code::kParameterInvalid, + "Missing server certificate file path"); + } + + bssl::UniquePtr root_cert = bssl::UniquePtr(PEM_read_X509( + cert_file.get(), nullptr /* x */, nullptr /* cb */, nullptr /* u */)); + if (!root_cert) { + return Error(Error::Code::kParseError, + "Failed to parse server certificate"); + } + + return GenerateCredentials(device_certificate_id, root_key.get(), + root_cert.get()); +} + } // namespace StaticCredentialsProvider::StaticCredentialsProvider() = default; @@ -181,30 +222,12 @@ ErrorOr GenerateCredentials( const std::string& device_certificate_id, const std::string& private_key_path, const std::string& server_certificate_path) { - if (private_key_path.empty() || server_certificate_path.empty()) { - return Error(Error::Code::kParameterInvalid, - "Missing either private key or server certificate"); - } - - FileUniquePtr key_file(fopen(private_key_path.c_str(), "r"), &fclose); - if (!key_file) { - return Error(Error::Code::kParameterInvalid, - "Missing private key file path"); + ErrorOr creds = GenerateCredentialsInternal( + device_certificate_id, private_key_path, server_certificate_path); + if (!creds) { + ClearOpenSSLERRStack(CURRENT_LOCATION); } - bssl::UniquePtr root_key = - bssl::UniquePtr(PEM_read_PrivateKey( - key_file.get(), nullptr /* x */, nullptr /* cb */, nullptr /* u */)); - - FileUniquePtr cert_file(fopen(server_certificate_path.c_str(), "r"), &fclose); - if (!cert_file) { - return Error(Error::Code::kParameterInvalid, - "Missing server certificate file path"); - } - bssl::UniquePtr root_cert = bssl::UniquePtr(PEM_read_X509( - cert_file.get(), nullptr /* x */, nullptr /* cb */, nullptr /* u */)); - - return GenerateCredentials(device_certificate_id, root_key.get(), - root_cert.get()); + return creds; } } // namespace openscreen::cast diff --git a/cast/receiver/public/receiver_socket_factory.h b/cast/receiver/public/receiver_socket_factory.h index 95bb1c124..278d7bf53 100644 --- a/cast/receiver/public/receiver_socket_factory.h +++ b/cast/receiver/public/receiver_socket_factory.h @@ -44,6 +44,9 @@ class ReceiverSocketFactory final : public TlsConnectionFactory::Client { const IPEndpoint& remote_address) override; void OnError(TlsConnectionFactory* factory, const Error& error) override; + // Accepts a generic Connection. + void CreateSocket(std::unique_ptr connection); + private: Client& client_; CastSocket::Client& socket_client_; diff --git a/cast/sender/BUILD.gn b/cast/sender/BUILD.gn index 77e052d85..d789b179a 100644 --- a/cast/sender/BUILD.gn +++ b/cast/sender/BUILD.gn @@ -21,7 +21,6 @@ openscreen_source_set("channel") { ] deps = [ - "../../third_party/abseil", "../common:channel", "../common/certificate/proto:certificate_proto", "../common/channel/proto:channel_proto", @@ -58,7 +57,6 @@ openscreen_source_set("sender") { public_deps = [ ":channel", "../../platform", - "../../third_party/abseil", "../../util", "../common:channel", "../common:public", diff --git a/cast/sender/cast_platform_client.cc b/cast/sender/cast_platform_client.cc index 5d644d45a..83ea092c3 100644 --- a/cast/sender/cast_platform_client.cc +++ b/cast/sender/cast_platform_client.cc @@ -126,7 +126,7 @@ void CastPlatformClient::OnMessage(VirtualConnectionRouter* router, return; } ErrorOr dict_or_error = json::Parse(GetPayload(message)); - if (dict_or_error.is_error()) { + if (dict_or_error.is_error() || !dict_or_error.value().isObject()) { return; } diff --git a/cast/sender/channel/cast_auth_util.cc b/cast/sender/channel/cast_auth_util.cc index 373655f6f..1cfaefe02 100644 --- a/cast/sender/channel/cast_auth_util.cc +++ b/cast/sender/channel/cast_auth_util.cc @@ -18,8 +18,8 @@ #include "platform/api/time.h" #include "platform/base/error.h" #include "util/osp_logging.h" -#include "util/span_util.h" #include "util/string_util.h" +#include "util/no_destructor.h" namespace openscreen::cast { @@ -74,8 +74,8 @@ Error ParseAuthMessage(const CastMessage& challenge_reply, class CastNonce { public: static CastNonce* GetInstance() { - static CastNonce* cast_nonce = new CastNonce(); - return cast_nonce; + static openscreen::NoDestructor cast_nonce; + return cast_nonce.get(); } static const std::string& Get() { @@ -83,8 +83,9 @@ class CastNonce { return GetInstance()->nonce_; } - private: CastNonce() : nonce_(kNonceSizeInBytes, 0) { GenerateNonce(); } + + private: void GenerateNonce() { OSP_CHECK_EQ( RAND_bytes(reinterpret_cast(&nonce_[0]), kNonceSizeInBytes), diff --git a/cast/sender/channel/cast_auth_util_unittest.cc b/cast/sender/channel/cast_auth_util_unittest.cc index b746dc3df..5af9e1a50 100644 --- a/cast/sender/channel/cast_auth_util_unittest.cc +++ b/cast/sender/channel/cast_auth_util_unittest.cc @@ -21,9 +21,9 @@ #include "platform/api/time.h" #include "platform/test/paths.h" #include "util/crypto/pem_helpers.h" +#include "util/no_destructor.h" #include "util/osp_logging.h" #include "util/read_file.h" -#include "util/span_util.h" namespace openscreen::cast { @@ -117,8 +117,9 @@ bool ConvertTimeSeconds(const DateTime& time, uint64_t* seconds) { } const std::string& GetSpecificTestDataPath() { - static std::string data_path = GetTestDataPath() + "cast/common/certificate/"; - return data_path; + static const NoDestructor data_path(GetTestDataPath() + + "cast/common/certificate/"); + return *data_path; } class CastAuthUtilTest : public ::testing::Test { @@ -155,7 +156,7 @@ class CastAuthUtilTest : public ::testing::Test { response.set_signature(ByteViewToString(signatures.sha256)); break; } - signed_data->assign(signatures.message.cbegin(), signatures.message.cend()); + signed_data->assign(signatures.message.begin(), signatures.message.end()); return response; } diff --git a/cast/standalone_common/BUILD.gn b/cast/standalone_common/BUILD.gn new file mode 100644 index 000000000..5641855c0 --- /dev/null +++ b/cast/standalone_common/BUILD.gn @@ -0,0 +1,22 @@ +# Copyright 2026 The Chromium Authors +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//build_overrides/build.gni") +import("../../gni/openscreen.gni") +import("../streaming/external_libraries.gni") + +# This component should only be built if FFMPEG is enabled. +assert(have_ffmpeg, "FFMPEG must be enabled to build this target") + +openscreen_source_set("ffmpeg_glue") { + testonly = true + visibility = [ "../*" ] + + public = [ "ffmpeg_glue.h" ] + sources = [ "ffmpeg_glue.cc" ] + include_dirs = ffmpeg_include_dirs + lib_dirs = external_lib_dirs + ffmpeg_lib_dirs + libs = ffmpeg_libs + deps = [ "../../util" ] +} diff --git a/cast/standalone_sender/ffmpeg_glue.cc b/cast/standalone_common/ffmpeg_glue.cc similarity index 93% rename from cast/standalone_sender/ffmpeg_glue.cc rename to cast/standalone_common/ffmpeg_glue.cc index 95fb3a72a..5a9e4f4ce 100644 --- a/cast/standalone_sender/ffmpeg_glue.cc +++ b/cast/standalone_common/ffmpeg_glue.cc @@ -1,8 +1,8 @@ -// Copyright 2020 The Chromium Authors +// Copyright 2026 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "cast/standalone_sender/ffmpeg_glue.h" +#include "cast/standalone_common/ffmpeg_glue.h" #include diff --git a/cast/standalone_sender/ffmpeg_glue.h b/cast/standalone_common/ffmpeg_glue.h similarity index 79% rename from cast/standalone_sender/ffmpeg_glue.h rename to cast/standalone_common/ffmpeg_glue.h index 789c2b324..a7eafc6ab 100644 --- a/cast/standalone_sender/ffmpeg_glue.h +++ b/cast/standalone_common/ffmpeg_glue.h @@ -1,9 +1,9 @@ -// Copyright 2020 The Chromium Authors +// Copyright 2026 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef CAST_STANDALONE_SENDER_FFMPEG_GLUE_H_ -#define CAST_STANDALONE_SENDER_FFMPEG_GLUE_H_ +#ifndef CAST_STANDALONE_COMMON_FFMPEG_GLUE_H_ +#define CAST_STANDALONE_COMMON_FFMPEG_GLUE_H_ extern "C" { #include @@ -35,12 +35,12 @@ AVFormatContext* CreateAVFormatContextForFile(const char* path); // // using FooUniquePtr = std::unique_ptr; // FooUniquePtr MakeUniqueFoo(...args...); -#define DEFINE_AV_UNIQUE_PTR(name, create_func, free_func) \ +#define DEFINE_AV_UNIQUE_PTR(name, create_func, free_statement) \ namespace internal { \ struct name##Freer { \ void operator()(name* obj) const { \ if (obj) { \ - free_func(&obj); \ + free_statement; \ } \ } \ }; \ @@ -55,20 +55,19 @@ AVFormatContext* CreateAVFormatContextForFile(const char* path); DEFINE_AV_UNIQUE_PTR(AVFormatContext, ::openscreen::cast::internal::CreateAVFormatContextForFile, - avformat_close_input); + avformat_close_input(&obj)) DEFINE_AV_UNIQUE_PTR(AVCodecContext, avcodec_alloc_context3, - avcodec_free_context); -DEFINE_AV_UNIQUE_PTR(AVPacket, av_packet_alloc, av_packet_free); -DEFINE_AV_UNIQUE_PTR(AVFrame, av_frame_alloc, av_frame_free); -DEFINE_AV_UNIQUE_PTR(SwrContext, swr_alloc, swr_free); + avcodec_free_context(&obj)) +DEFINE_AV_UNIQUE_PTR(AVPacket, av_packet_alloc, av_packet_free(&obj)) +DEFINE_AV_UNIQUE_PTR(AVFrame, av_frame_alloc, av_frame_free(&obj)) +DEFINE_AV_UNIQUE_PTR(SwrContext, swr_alloc, swr_free(&obj)) +DEFINE_AV_UNIQUE_PTR(AVCodecParserContext, av_parser_init, av_parser_close(obj)) #undef DEFINE_AV_UNIQUE_PTR // The av_err2str macro uses a compound literal, which is a C99-only feature. // So instead, we roll our own here. -// TODO(issuetracker.google.com/224642520): dedup with standalone -// receiver. std::string AvErrorToString(int error_num); // Macros to enable backwards compability codepaths for older versions of @@ -79,4 +78,4 @@ std::string AvErrorToString(int error_num); } // namespace openscreen::cast -#endif // CAST_STANDALONE_SENDER_FFMPEG_GLUE_H_ +#endif // CAST_STANDALONE_COMMON_FFMPEG_GLUE_H_ diff --git a/cast/standalone_e2e.py b/cast/standalone_e2e.py index 9d07cc187..a69913a83 100755 --- a/cast/standalone_e2e.py +++ b/cast/standalone_e2e.py @@ -2,37 +2,37 @@ # Copyright 2021 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" -This script is intended to cover end to end testing for the standalone sender -and receiver executables in cast. This ensures that the basic functionality of -these executables is not impaired, such as the TLS/UDP connections and encoding -and decoding video. +"""End-to-end tests for the standalone Cast sender and receiver executables. + +This script ensures that the basic functionality of these executables is not +impaired, such as the TLS/UDP connections and encoding and decoding video. """ import argparse +from collections import namedtuple +from enum import IntEnum +from enum import IntFlag +import logging import os import pathlib -import logging +import ssl import subprocess +import shutil import sys import time import unittest -import ssl -from collections import namedtuple -from enum import IntEnum, IntFlag -from urllib import request # Environment variables that can be overridden to set test properties. ROOT_ENVVAR = 'OPENSCREEN_ROOT_DIR' BUILD_ENVVAR = 'OPENSCREEN_BUILD_DIR' LIBAOM_ENVVAR = 'OPENSCREEN_HAVE_LIBAOM' -TEST_VIDEO_NAME = 'Contador_Glam.mp4' -TEST_VIDEO_URL = ('https://storage.googleapis.com/openscreen_standalone/' + - TEST_VIDEO_NAME) +# 10 second long example video. +TEST_VIDEO_NAME = 'bbb_sunflower_2160p_60fps_normal.mp4' -PROCESS_TIMEOUT = 15 # seconds +# Plenty of padding for playing out the video. +PROCESS_TIMEOUT = 60 # seconds # Open Screen test certificates expire after 3 days. We crop this slightly (by # 8 hours) to account for potential errors in time calculations. @@ -46,31 +46,34 @@ RECEIVER_BINARY_NAME = 'cast_receiver' EXPECTED_RECEIVER_MESSAGES = [ - "CastService is running.", "Found codec: opus (known to FFMPEG as opus)", - "Successfully negotiated a session, creating SDL players.", - "Receivers are currently destroying, resetting SDL players." + 'CastService is running.', + 'Found codec: opus (known to FFMPEG as opus)', + 'Successfully negotiated a session, creating SDL players.', ] + class VideoCodec(IntEnum): """An enumeration of the video codecs that should be sent by the sender.""" + VP8 = 0 VP9 = 1 AV1 = 2 VIDEO_CODEC_SPECIFIC_RECEIVER_MESSAGES = [ - "Found codec: vp8 (known to FFMPEG as vp8)", - "Found codec: vp9 (known to FFMPEG as vp9)", - "Found codec: libaom-av1 (known to FFMPEG as av1)" + 'Found codec: vp8 (known to FFMPEG as vp8)', + 'Found codec: vp9 (known to FFMPEG as vp9)', + 'Found codec: libaom-av1 (known to FFMPEG as av1)', ] EXPECTED_SENDER_MESSAGES = [ - "Launching Mirroring App on the Cast Receiver", - "Max allowed media bitrate (audio + video) will be", - "Contador_Glam.mp4 (starts in one second)...", - "The video capturer has reached the end of the media stream.", - "The audio capturer has reached the end of the media stream.", - "Video complete. Exiting...", "Shutting down..." + 'Launching Mirroring App on the Cast Receiver', + 'Max allowed media bitrate (audio + video) will be', + TEST_VIDEO_NAME + ' (starts in one second)...', + 'The video capturer has reached the end of the media stream.', + 'The audio capturer has reached the end of the media stream.', + 'Video complete. Exiting...', + 'Shutting down...', ] MISSING_LOG_MESSAGE = """Missing an expected message from either the sender @@ -118,69 +121,74 @@ def _get_file_age_in_seconds(path): def _get_build_paths(): """Gets the root and build paths (either default or from the environment - variables), and sets related paths to binaries and files.""" + + variables), and sets related paths to binaries and files. + """ root_path = pathlib.Path( - os.environ[ROOT_ENVVAR] if os.getenv(ROOT_ENVVAR) else subprocess. - getoutput('git rev-parse --show-toplevel')) + os.environ[ROOT_ENVVAR] if os.getenv(ROOT_ENVVAR) else subprocess. + getoutput('git rev-parse --show-toplevel')) assert root_path.exists(), 'Could not find openscreen root!' - build_path = pathlib.Path(os.environ[BUILD_ENVVAR]) if os.getenv( - BUILD_ENVVAR) else root_path.joinpath('out', - 'Default').resolve() + build_path = (pathlib.Path(os.environ[BUILD_ENVVAR]) + if os.getenv(BUILD_ENVVAR) else root_path.joinpath( + 'out', 'Default').resolve()) assert build_path.exists(), 'Could not find openscreen build!' - BuildPaths = namedtuple("BuildPaths", - "root build test_video cast_receiver cast_sender") - return BuildPaths(root = root_path, - build = build_path, - test_video = build_path.joinpath(TEST_VIDEO_NAME).resolve(), - cast_receiver = build_path.joinpath(RECEIVER_BINARY_NAME).resolve(), - cast_sender = build_path.joinpath(SENDER_BINARY_NAME).resolve() - ) + BuildPaths = namedtuple('BuildPaths', + 'root build test_video cast_receiver cast_sender') + return BuildPaths( + root=root_path, + build=build_path, + test_video=build_path.joinpath(TEST_VIDEO_NAME).resolve(), + cast_receiver=build_path.joinpath(RECEIVER_BINARY_NAME).resolve(), + cast_sender=build_path.joinpath(SENDER_BINARY_NAME).resolve(), + ) class TestFlags(IntFlag): - """ - Test flags, primarily used to control sender and receiver configuration - to test different features of the standalone libraries. - """ + """Test flags, primarily used to control sender and receiver configuration + + to test different features of the standalone libraries. + """ + USE_REMOTING = 1 USE_ANDROID_HACK = 2 class StandaloneCastTest(unittest.TestCase): - """ - Test class for setting up and running end to end tests on the - standalone sender and receiver binaries. This class uses the unittest - package, so methods that are executed as tests all have named prefixed - with "test_". + """Test class for setting up and running end to end tests on the - This suite sets the current working directory to the root of the Open - Screen repository, and references all files from the root directory. - Generated certificates should always be in |cls.build_paths.root|. - """ + standalone sender and receiver binaries. This class uses the unittest + package, so methods that are executed as tests all have named prefixed + with "test_". + + This suite sets the current working directory to the root of the Open + Screen repository, and references all files from the root directory. + Generated certificates should always be in |cls.build_paths.root|. + """ @classmethod def setUpClass(cls): """Shared setup method for all tests, handles one-time updates.""" cls.build_paths = _get_build_paths() os.chdir(cls.build_paths.root) - cls.download_video() + cls.setup_video() cls.generate_certificates() @classmethod - def download_video(cls): - """Downloads the test video from Google storage.""" + def setup_video(cls): + """Copies the test video from the test data folder.""" if os.path.exists(cls.build_paths.test_video): - logging.debug('Video already exists, skipping download...') + logging.debug('Video already exists, skipping copy...') return - logging.debug('Downloading video from %s', TEST_VIDEO_URL) - with request.urlopen(TEST_VIDEO_URL, - context=ssl.SSLContext( - ssl.PROTOCOL_TLS_CLIENT)) as url: - with open(cls.build_paths.test_video, 'wb') as file: - file.write(url.read()) + source_path = cls.build_paths.root / 'test/data/cast' / TEST_VIDEO_NAME + if not source_path.exists(): + raise FileNotFoundError( + f'Could not find test video at {source_path}') + + logging.debug('Copying video from %s', source_path) + shutil.copy(source_path, cls.build_paths.test_video) @classmethod def generate_certificates(cls): @@ -195,7 +203,7 @@ def generate_certificates(cls): command = [ cls.build_paths.cast_receiver, '-g', # Generate certificate and private key. - '-v' # Enable verbose logging. + '-v', # Enable verbose logging. ] try: subprocess.check_output(command, stderr=subprocess.STDOUT) @@ -217,7 +225,8 @@ def launch_receiver(self): TEST_KEY_NAME, '-x', # Skip discovery, only necessary on Mac OS X. '-v', # Enable verbose logging. - loopback + '-P', # enable Perfetto based performance logging. + loopback, ] return subprocess.Popen(command, stdout=subprocess.PIPE, @@ -232,7 +241,8 @@ def launch_sender(self, flags, codec=None): self.build_paths.test_video, '-d', TEST_CERT_NAME, - '-n' # Only play the video once, and then exit. + '-n', # Only play the video once, and then exit. + '-P', # enable Perfetto based performance logging. ] if TestFlags.USE_ANDROID_HACK in flags: command.append('-a') @@ -251,7 +261,7 @@ def launch_sender(self, flags, codec=None): self.assertTrue(codec == VideoCodec.AV1) command.append('av1') - #pylint: disable = consider-using-with + # pylint: disable = consider-using-with return subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -264,17 +274,29 @@ def check_logs(self, logs, codec=None): if codec is None: codec = VideoCodec.VP8 - for message in (EXPECTED_RECEIVER_MESSAGES + - [VIDEO_CODEC_SPECIFIC_RECEIVER_MESSAGES[codec]]): + for message in EXPECTED_RECEIVER_MESSAGES + [ + VIDEO_CODEC_SPECIFIC_RECEIVER_MESSAGES[codec] + ]: self.assertTrue( message in logs[0], - f'Missing log message: {message}.\n{MISSING_LOG_MESSAGE}') + f'Missing log message: {message}.\n{MISSING_LOG_MESSAGE}', + ) for message in EXPECTED_SENDER_MESSAGES: self.assertTrue( message in logs[1], - f'Missing log message: {message}.\n{MISSING_LOG_MESSAGE}') - for log, prefix in logs, ["[ERROR:", "[FATAL:"]: - self.assertTrue(prefix not in log, "Logs contained an error") + f'Missing log message: {message}.\n{MISSING_LOG_MESSAGE}', + ) + for log in logs: + for prefix in ['[ERROR:', '[FATAL:']: + if prefix in log: + lines = log.splitlines() + for i, line in enumerate(lines): + if prefix in line: + print(f'Found error line: {line}') + for j in range(1, 10): + if i + j < len(lines): print(lines[i + j]) + self.assertTrue(prefix not in log, + f'Logs contained an error: {prefix}') logging.debug('Finished validating log output') def get_output(self, flags, codec=None): @@ -284,20 +306,54 @@ def get_output(self, flags, codec=None): time.sleep(3) sender_process = self.launch_sender(flags, codec) - logging.debug('Launched sender PID %i and receiver PID %i...', - sender_process.pid, receiver_process.pid) + logging.debug( + 'Launched sender PID %i and receiver PID %i...', + sender_process.pid, + receiver_process.pid, + ) logging.debug('collating output...') - output = (receiver_process.communicate( - timeout=PROCESS_TIMEOUT)[1].decode('utf-8'), - sender_process.communicate( - timeout=PROCESS_TIMEOUT)[1].decode('utf-8')) - - # TODO(issuetracker.google.com/194292855): standalones should exit zero. - # Remoting causes the sender to exit with code -4. - if not TestFlags.USE_REMOTING in flags: - self.assertEqual(sender_process.returncode, 0, - 'sender had non-zero exit code') - return output + + try: + # We wait for the sender to complete, as it drives the session. + sender_out, sender_err = sender_process.communicate( + timeout=PROCESS_TIMEOUT) + + # Give the receiver a moment to process any final messages / settle. + time.sleep(2) + + # The receiver might not exit automatically, so we terminate it. + receiver_process.terminate() + try: + receiver_process.wait(timeout=5) + except subprocess.TimeoutExpired: + receiver_process.kill() + + receiver_out, receiver_err = receiver_process.communicate() + + if TestFlags.USE_REMOTING not in flags: + self.assertEqual(sender_process.returncode, 0, + 'sender had non-zero exit code') + + return ( + receiver_err.decode('utf-8', errors='replace'), + sender_err.decode('utf-8', errors='replace'), + ) + except subprocess.TimeoutExpired: + logging.error('Test timed out, killing processes...') + receiver_process.kill() + sender_process.kill() + receiver_out, receiver_err = receiver_process.communicate() + sender_out, sender_err = sender_process.communicate() + logging.error('Receiver Stderr: %s', + receiver_err.decode('utf-8', errors='replace')) + logging.error('Sender Stderr: %s', + sender_err.decode('utf-8', errors='replace')) + raise + finally: + if receiver_process.poll() is None: + receiver_process.kill() + if sender_process.poll() is None: + sender_process.kill() def test_golden_case(self): """Tests that when settings are normal, things work end to end.""" @@ -325,14 +381,14 @@ def test_vp9_flag(self): self.check_logs(output, VideoCodec.VP9) @unittest.skipUnless(os.getenv(LIBAOM_ENVVAR), - 'Skipping AV1 test since LibAOM not installed.') + 'Skipping AV1 test since LibAOM not installed.') def test_av1_flag(self): """Tests that the AV1 flag works with standard settings.""" output = self.get_output([], VideoCodec.AV1) self.check_logs(output, VideoCodec.AV1) -def parse_args(): +def _parse_args(): """Parses the command line arguments and sets up the logging module.""" # NOTE for future developers: the `unittest` module will complain if it is # passed any args that it doesn't understand. If any Open Screen-specific @@ -344,10 +400,14 @@ def parse_args(): help='enable debug logging', action='store_true') - parsed_args = parser.parse_args(sys.argv[1:]) + parsed_args, remaining_args = parser.parse_known_args(sys.argv[1:]) _set_log_level(parsed_args.verbose) + # Crop Open Screen-specific command line arguments from sys.argv before + # calling unittest.main(). + sys.argv = [sys.argv[0]] + remaining_args + if __name__ == '__main__': - parse_args() + _parse_args() unittest.main() diff --git a/cast/standalone_receiver/BUILD.gn b/cast/standalone_receiver/BUILD.gn index 93d0bf4c8..0b2248895 100644 --- a/cast/standalone_receiver/BUILD.gn +++ b/cast/standalone_receiver/BUILD.gn @@ -41,7 +41,6 @@ openscreen_source_set("cast_receiver_lib") { if (have_ffmpeg && have_libsdl2) { defines = [ "CAST_STANDALONE_RECEIVER_HAVE_EXTERNAL_LIBS" ] sources += [ - "avcodec_glue.h", "decoder.cc", "decoder.h", "sdl_audio_player.cc", @@ -53,8 +52,9 @@ openscreen_source_set("cast_receiver_lib") { "sdl_video_player.cc", "sdl_video_player.h", ] + deps += [ "../standalone_common:ffmpeg_glue" ] include_dirs = ffmpeg_include_dirs + libsdl2_include_dirs - lib_dirs = ffmpeg_lib_dirs + libsdl2_lib_dirs + lib_dirs = external_lib_dirs + ffmpeg_lib_dirs + libsdl2_lib_dirs libs = ffmpeg_libs + libsdl2_libs } else { sources += [ diff --git a/cast/standalone_receiver/DEPS b/cast/standalone_receiver/DEPS index 2a365f905..480405dd3 100644 --- a/cast/standalone_receiver/DEPS +++ b/cast/standalone_receiver/DEPS @@ -12,6 +12,7 @@ include_rules = [ '+cast/streaming/public', '+cast/streaming/message_fields.h', '+cast/streaming/remoting.pb.h', + '+cast/standalone_common', '+platform/impl', '+discovery/common', '+discovery/public', diff --git a/cast/standalone_receiver/avcodec_glue.h b/cast/standalone_receiver/avcodec_glue.h deleted file mode 100644 index ce1e1d122..000000000 --- a/cast/standalone_receiver/avcodec_glue.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2019 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CAST_STANDALONE_RECEIVER_AVCODEC_GLUE_H_ -#define CAST_STANDALONE_RECEIVER_AVCODEC_GLUE_H_ - -extern "C" { -#include -#include -#include -#include -} - -#include -#include - -namespace openscreen::cast { - -// Macro that, for an AVFoo, generates code for: -// -// using FooUniquePtr = std::unique_ptr; -// FooUniquePtr MakeUniqueFoo(...args...); -#define DEFINE_AV_UNIQUE_PTR(name, create_func, free_statement) \ - namespace internal { \ - struct name##Freer { \ - void operator()(name* obj) const { \ - if (obj) { \ - free_statement; \ - } \ - } \ - }; \ - } \ - \ - using name##UniquePtr = std::unique_ptr; \ - \ - template \ - name##UniquePtr MakeUnique##name(Args&&... args) { \ - return name##UniquePtr(create_func(std::forward(args)...)); \ - } - -DEFINE_AV_UNIQUE_PTR(AVCodecParserContext, - av_parser_init, - av_parser_close(obj)); -DEFINE_AV_UNIQUE_PTR(AVCodecContext, - avcodec_alloc_context3, - avcodec_free_context(&obj)); -DEFINE_AV_UNIQUE_PTR(AVPacket, av_packet_alloc, av_packet_free(&obj)); -DEFINE_AV_UNIQUE_PTR(AVFrame, av_frame_alloc, av_frame_free(&obj)); - -#undef DEFINE_AV_UNIQUE_PTR - -// Macros to enable backwards compability codepaths for older versions of -// ffmpeg, where newer versions have deprecated APIs. Note that ffmpeg defines -// its own FF_API* macros that are related to removing APIs (not deprecating -// them). -// -// TODO(issuetracker.google.com/224642520): dedup with standalone -// sender. -#define _LIBAVUTIL_OLD_CHANNEL_LAYOUT (LIBAVUTIL_VERSION_MAJOR < 57) - -} // namespace openscreen::cast - -#endif // CAST_STANDALONE_RECEIVER_AVCODEC_GLUE_H_ diff --git a/cast/standalone_receiver/cast_service.cc b/cast/standalone_receiver/cast_service.cc index e08b06e64..43970caed 100644 --- a/cast/standalone_receiver/cast_service.cc +++ b/cast/standalone_receiver/cast_service.cc @@ -44,10 +44,12 @@ discovery::Config MakeDiscoveryConfig(const InterfaceInfo& interface) { CastService::CastService(CastService::Configuration config) : local_endpoint_(DetermineEndpoint(config.interface)), credentials_(std::move(config.credentials)), - agent_(config.task_runner, *credentials_.provider), + agent_(config.task_runner, *credentials_.provider, config.device_uuid), mirroring_application_(config.task_runner, local_endpoint_.address, - agent_), + agent_, + config.enable_dscp, + config.enable_input_events), socket_factory_(agent_, *agent_.cast_socket_client()), connection_factory_( TlsConnectionFactory::CreateFactory(socket_factory_, @@ -80,8 +82,8 @@ CastService::CastService(CastService::Configuration config) OSP_LOG_WARN << "Hardware address for interface " << config.interface.name << " is empty. Generating a random unique_id."; std::array random_bytes; - GenerateRandomBytes(random_bytes.data(), kCastUniqueIdLength); - info.unique_id = HexEncode(random_bytes.data(), kCastUniqueIdLength); + GenerateRandomBytes(random_bytes); + info.unique_id = HexEncode(random_bytes); } info.friendly_name = config.friendly_name; info.model_name = config.model_name; @@ -99,6 +101,19 @@ CastService::~CastService() { } } +void CastService::AddApplicationNamespace( + std::string_view namespace_, + MirroringApplication::CustomMessageCallback handler) { + mirroring_application_.AddCustomNamespace(namespace_); + mirroring_application_.SetCustomMessageHandler(namespace_, + std::move(handler)); +} + +void CastService::RemoveApplicationNamespace(std::string_view namespace_) { + mirroring_application_.RemoveCustomNamespace(namespace_); + mirroring_application_.SetCustomMessageHandler(namespace_, nullptr); +} + void CastService::OnFatalError(const Error& error) { OSP_LOG_FATAL << "Encountered fatal discovery error: " << error; } diff --git a/cast/standalone_receiver/cast_service.h b/cast/standalone_receiver/cast_service.h index 31c05c020..2d4e4c2a8 100644 --- a/cast/standalone_receiver/cast_service.h +++ b/cast/standalone_receiver/cast_service.h @@ -7,6 +7,7 @@ #include #include +#include #include "cast/common/public/receiver_info.h" #include "cast/receiver/application_agent.h" @@ -51,6 +52,9 @@ class CastService final : public discovery::ReportingClient { // The credentials that the cast service should use for TLS. GeneratedCredentials credentials; + // Device UUID + std::string device_uuid; + // The friendly name to be used for broadcasting. std::string friendly_name; @@ -59,11 +63,23 @@ class CastService final : public discovery::ReportingClient { // Whether we should broadcast over mDNS/DNS-SD. bool enable_discovery = true; + + // Whether we should enable DSCP packet prioritization for UDP sockets. + bool enable_dscp = true; + + // Whether input event API should be enabled for this session. + bool enable_input_events = false; }; explicit CastService(Configuration config); ~CastService() final; + // Registers a namespace handler for the mirroring application. + void AddApplicationNamespace( + std::string_view namespace_, + MirroringApplication::CustomMessageCallback handler); + void RemoveApplicationNamespace(std::string_view namespace_); + private: using LazyDeletedDiscoveryPublisher = std::unique_ptr, diff --git a/cast/standalone_receiver/decoder.cc b/cast/standalone_receiver/decoder.cc index f6fc8ea98..0b5319dc7 100644 --- a/cast/standalone_receiver/decoder.cc +++ b/cast/standalone_receiver/decoder.cc @@ -17,18 +17,6 @@ namespace openscreen::cast { -namespace { -// The av_err2str macro uses a compound literal, which is a C99-only feature. -// So instead, we roll our own here. -// TODO(issuetracker.google.com/224642520): dedup with standalone -// sender. -std::string AvErrorToString(int error_num) { - std::string out(AV_ERROR_MAX_STRING_SIZE, '\0'); - av_make_error_string(data(out), out.length(), error_num); - return out; -} -} // namespace - Decoder::Buffer::Buffer() { Resize(0); } @@ -71,7 +59,8 @@ Decoder::Decoder(const std::string& codec_name) : codec_name_(codec_name) { Decoder::~Decoder() = default; void Decoder::Decode(FrameId frame_id, const Decoder::Buffer& buffer) { - TRACE_DEFAULT_SCOPED(TraceCategory::kStandaloneReceiver); + TRACE_FLOW_STEP(TraceCategory::kStandaloneReceiver, "Frame.Decode.Begin", + frame_id); if (!codec_ && !Initialize()) { return; } @@ -116,6 +105,8 @@ void Decoder::Decode(FrameId frame_id, const Decoder::Buffer& buffer) { OnError("avcodec_receive_frame", receive_frame_result, decoded_frame_id); return; } + TRACE_FLOW_STEP(TraceCategory::kStandaloneReceiver, "Frame.Decode.End", + decoded_frame_id); if (client_) { client_->OnFrameDecoded(decoded_frame_id, *decoded_frame_); } @@ -221,10 +212,7 @@ void Decoder::OnError(const char* what, int av_errnum, FrameId frame_id) { error << "frame: " << frame_id << "; "; } - char human_readable_error[AV_ERROR_MAX_STRING_SIZE]{0}; - av_make_error_string(human_readable_error, AV_ERROR_MAX_STRING_SIZE, - av_errnum); - error << "what: " << what << "; error: " << human_readable_error; + error << "what: " << what << "; error: " << AvErrorToString(av_errnum); // Dispatch to either the fatal error handler, or the one for decode errors, // as appropriate. diff --git a/cast/standalone_receiver/decoder.h b/cast/standalone_receiver/decoder.h index 3b27d2cc3..b39ead535 100644 --- a/cast/standalone_receiver/decoder.h +++ b/cast/standalone_receiver/decoder.h @@ -10,7 +10,7 @@ #include #include -#include "cast/standalone_receiver/avcodec_glue.h" +#include "cast/standalone_common/ffmpeg_glue.h" #include "cast/streaming/public/frame_id.h" #include "platform/base/span.h" diff --git a/cast/standalone_receiver/dummy_player.cc b/cast/standalone_receiver/dummy_player.cc index cb831aa57..3374802bf 100644 --- a/cast/standalone_receiver/dummy_player.cc +++ b/cast/standalone_receiver/dummy_player.cc @@ -24,7 +24,7 @@ DummyPlayer::~DummyPlayer() { receiver_.SetConsumer(nullptr); } -void DummyPlayer::OnFramesReady(int buffer_size) { +void DummyPlayer::OnFramesReady(size_t buffer_size) { // Consume the next frame. buffer_.resize(buffer_size); const EncodedFrame frame = receiver_.ConsumeNextFrame(buffer_); @@ -33,8 +33,8 @@ void DummyPlayer::OnFramesReady(int buffer_size) { // some short information about the frame. const auto media_timestamp = frame.rtp_timestamp.ToTimeSinceOrigin( - receiver_.rtp_timebase()); - OSP_LOG_INFO << "[SSRC " << receiver_.ssrc() << "] " + receiver_.config().rtp_timebase); + OSP_LOG_INFO << "[SSRC " << receiver_.config().receiver_ssrc << "] " << (frame.dependency == EncodedFrame::Dependency::kKeyFrame ? "KEY " : "") diff --git a/cast/standalone_receiver/dummy_player.h b/cast/standalone_receiver/dummy_player.h index ab7b772c9..2035b5b25 100644 --- a/cast/standalone_receiver/dummy_player.h +++ b/cast/standalone_receiver/dummy_player.h @@ -27,7 +27,7 @@ class DummyPlayer final : public Receiver::Consumer { private: // Receiver::Consumer implementation. - void OnFramesReady(int next_frame_buffer_size) final; + void OnFramesReady(size_t next_frame_buffer_size) final; Receiver& receiver_; std::vector buffer_; diff --git a/cast/standalone_receiver/main.cc b/cast/standalone_receiver/main.cc index 314f0b322..3632f0197 100644 --- a/cast/standalone_receiver/main.cc +++ b/cast/standalone_receiver/main.cc @@ -3,8 +3,11 @@ // found in the LICENSE file. #include +#include #include #include +#include +#include #include #include #include @@ -24,13 +27,18 @@ #include "util/string_util.h" #include "util/stringprintf.h" #include "util/trace_logging.h" +#include "util/uuid.h" + +#if defined(USE_PERFETTO) +#include "platform/impl/perfetto_trace_logging_platform.h" +#endif namespace openscreen::cast { namespace { void LogUsage(const char* argv0) { - constexpr char kTemplate[] = R"( -usage: %s + static constexpr char kTemplate[] = R"( +usage: {} interface Specifies the network interface to bind to. The interface is @@ -38,11 +46,6 @@ usage: %s Mandatory, as it must be known for publishing discovery. options: - -p, --private-key=path-to-key: Path to OpenSSL-generated private key to be - used for TLS authentication. If a private key is not - provided, a randomly generated one will be used for this - session. - -d, --developer-certificate=path-to-cert: Path to PEM file containing a developer generated server root TLS certificate. If a root server certificate is not provided, one @@ -50,29 +53,39 @@ usage: %s private key. Note that if a certificate path is passed, the private key path is a mandatory field. + -f, --friendly-name: Friendly name to be used for receiver discovery. + -g, --generate-credentials: Instructs the binary to generate a private key and self-signed root certificate with the CA bit set to true, and then exit. The resulting private key and certificate can then be used as values for the -p and -s flags. - -f, --friendly-name: Friendly name to be used for receiver discovery. + -h, --help: Show this help message. -m, --model-name: Model name to be used for receiver discovery. - -t, --tracing: Enable performance tracing logging. + -p, --private-key=path-to-key: Path to OpenSSL-generated private key to be + used for TLS authentication. If a private key is not + provided, a randomly generated one will be used for this + session. + + -q, --disable-dscp: Disable DSCP packet prioritization, used for QoS over + the UDP socket connection. + + -t, --tracing: Enable text based performance trace logging. -v, --verbose: Enable verbose logging. - -h, --help: Show this help message. + -x, --disable-discovery: Disable discovery. - -x, --disable-discovery: Disable discovery, useful for platforms like Mac OS - where our implementation is incompatible with - the native Bonjour service. + -P, --perfetto: Enable Perfetto based performance trace logging. + + -i, --enable-input-events: Enable the Input event API (receiver-to-sender). )"; - std::cerr << StringPrintf(kTemplate, argv0); + std::cerr << StringFormat(kTemplate, argv0); } InterfaceInfo GetInterfaceInfoFromName(const char* name) { @@ -109,110 +122,147 @@ void RunCastService(TaskRunnerImpl* runner, CastService::Configuration config) { OSP_LOG_INFO << "Bye!"; } -int RunStandaloneReceiver(int argc, char* argv[]) { -#if !defined(CAST_STANDALONE_RECEIVER_HAVE_EXTERNAL_LIBS) - OSP_LOG_INFO - << "Note: compiled without external libs. The dummy player will " - "be linked and no video decoding will occur. If this is not desired, " - "install the required external libraries. For more information, see: " - "[external_libraries.md](../streaming/external_libraries.md)."; -#endif +struct Arguments { + // Required positional arguments + const char* interface_name = nullptr; + + // Optional arguments + std::string developer_certificate_path; + bool enable_discovery = true; + bool enable_dscp = true; + bool enable_input_events = false; + std::string friendly_name = "Cast Standalone Receiver"; + bool should_generate_credentials = false; + std::string model_name = "cast_standalone_receiver"; + std::string private_key_path; + std::unique_ptr trace_logger; + bool is_verbose = false; +}; +std::optional ParseArgs(int argc, char* argv[]) { // A note about modifying command line arguments: consider uniformity // between all Open Screen executables. If it is a platform feature // being exposed, consider if it applies to the standalone receiver, // standalone sender, osp demo, and test_main argument options. const get_opt::option kArgumentOptions[] = { - {"private-key", required_argument, nullptr, 'p'}, {"developer-certificate", required_argument, nullptr, 'd'}, - {"generate-credentials", no_argument, nullptr, 'g'}, + {"disable-discovery", no_argument, nullptr, 'x'}, + {"disable-dscp", no_argument, nullptr, 'q'}, + {"enable-input-events", no_argument, nullptr, 'i'}, {"friendly-name", required_argument, nullptr, 'f'}, + {"generate-credentials", no_argument, nullptr, 'g'}, + {"help", no_argument, nullptr, 'h'}, {"model-name", required_argument, nullptr, 'm'}, + {"private-key", required_argument, nullptr, 'p'}, {"tracing", no_argument, nullptr, 't'}, {"verbose", no_argument, nullptr, 'v'}, - {"help", no_argument, nullptr, 'h'}, - - // Discovery is enabled by default, however there are cases where it - // needs to be disabled, such as on Mac OS X. - {"disable-discovery", no_argument, nullptr, 'x'}, +#if defined(USE_PERFETTO) + {"perfetto", no_argument, nullptr, 'P'}, +#endif {nullptr, 0, nullptr, 0}}; - bool is_verbose = false; - bool enable_discovery = true; - std::string private_key_path; - std::string developer_certificate_path; - std::string friendly_name = "Cast Standalone Receiver"; - std::string model_name = "cast_standalone_receiver"; - bool should_generate_credentials = false; - std::unique_ptr trace_logger; + Arguments args; int ch = -1; - while ((ch = getopt_long(argc, argv, "p:d:f:m:grtvhx", kArgumentOptions, + while ((ch = getopt_long(argc, argv, "d:f:ghim:p:qtvxP", kArgumentOptions, nullptr)) != -1) { switch (ch) { - case 'p': - private_key_path = get_opt::optarg; - break; case 'd': - developer_certificate_path = get_opt::optarg; + args.developer_certificate_path = get_opt::optarg; break; case 'f': - friendly_name = get_opt::optarg; + args.friendly_name = get_opt::optarg; + break; + case 'g': + args.should_generate_credentials = true; + break; + case 'h': + return {}; + case 'i': + args.enable_input_events = true; break; case 'm': - model_name = get_opt::optarg; + args.model_name = get_opt::optarg; break; - case 'g': - should_generate_credentials = true; + case 'p': + args.private_key_path = get_opt::optarg; + break; + case 'q': + args.enable_dscp = false; break; case 't': - trace_logger = std::make_unique(); + args.trace_logger = std::make_unique(); break; case 'v': - is_verbose = true; + args.is_verbose = true; break; case 'x': - enable_discovery = false; + args.enable_discovery = false; break; - case 'h': - LogUsage(argv[0]); - return 1; +#if defined(USE_PERFETTO) + case 'P': + args.trace_logger = std::make_unique(); + break; +#endif } } - SetLogLevel(is_verbose ? LogLevel::kVerbose : LogLevel::kInfo); + args.interface_name = argv[get_opt::optind]; + if (!args.should_generate_credentials) { + OSP_CHECK(args.interface_name && strlen(args.interface_name) > 0) + << "No interface name provided."; + } + return args; +} + +int RunStandaloneReceiver(int argc, char* argv[]) { +#if !defined(CAST_STANDALONE_RECEIVER_HAVE_EXTERNAL_LIBS) + OSP_LOG_INFO + << "Note: compiled without external libs. The dummy player will " + "be linked and no video decoding will occur. If this is not desired, " + "install the required external libraries. For more information, see: " + "[external_libraries.md](../streaming/external_libraries.md)."; +#endif + const std::optional args = ParseArgs(argc, argv); + if (!args) { + LogUsage(argv[0]); + return 1; + } + SetLogLevel(args->is_verbose ? LogLevel::kVerbose : LogLevel::kInfo); // Either -g is required, or both -p and -d. - if (should_generate_credentials) { + if (args->should_generate_credentials) { GenerateDeveloperCredentialsToFile(); return 0; } - if (private_key_path.empty() || developer_certificate_path.empty()) { + if (args->private_key_path.empty() || + args->developer_certificate_path.empty()) { OSP_LOG_FATAL << "You must either invoke with -g to generate credentials, " "or provide both a private key path and root certificate " "using -p and -d"; return 1; } - const char* interface_name = argv[get_opt::optind]; - OSP_CHECK(interface_name && strlen(interface_name) > 0) - << "No interface name provided."; - - std::string receiver_id = - string_util::StrCat({"Standalone Receiver on ", interface_name}); + const std::string receiver_id = + string_util::StrCat({"Standalone Receiver on ", args->interface_name}); ErrorOr creds = GenerateCredentials( - receiver_id, private_key_path, developer_certificate_path); + receiver_id, args->private_key_path, args->developer_certificate_path); OSP_CHECK(creds.is_value()) << creds.error(); - const InterfaceInfo interface = GetInterfaceInfoFromName(interface_name); + const InterfaceInfo interface = + GetInterfaceInfoFromName(args->interface_name); OSP_CHECK(interface.GetIpAddressV4() || interface.GetIpAddressV6()); auto* const task_runner = new TaskRunnerImpl(&Clock::now); PlatformClientPosix::Create(milliseconds(50), std::unique_ptr(task_runner)); - RunCastService(task_runner, - CastService::Configuration{ - *task_runner, interface, std::move(creds.value()), - friendly_name, model_name, enable_discovery}); + + RunCastService( + task_runner, + CastService::Configuration{ + *task_runner, interface, std::move(creds.value()), + Uuid::GenerateRandomV4().AsLowercaseString(), args->friendly_name, + args->model_name, args->enable_discovery, args->enable_dscp, + args->enable_input_events}); PlatformClientPosix::ShutDown(); return 0; @@ -222,5 +272,10 @@ int RunStandaloneReceiver(int argc, char* argv[]) { } // namespace openscreen::cast int main(int argc, char* argv[]) { + // Ignore SIGPIPE events at the application level -- tearing down the network + // interface will close a TLS or UDP socket connection, which will result + // in a more graceful exit than terminating on the SIGPIPE call. + std::signal(SIGPIPE, SIG_IGN); + return openscreen::cast::RunStandaloneReceiver(argc, argv); } diff --git a/cast/standalone_receiver/mirroring_application.cc b/cast/standalone_receiver/mirroring_application.cc index 56b917e53..bd80a36f5 100644 --- a/cast/standalone_receiver/mirroring_application.cc +++ b/cast/standalone_receiver/mirroring_application.cc @@ -4,6 +4,7 @@ #include "cast/standalone_receiver/mirroring_application.h" +#include #include #include "build/build_config.h" @@ -26,11 +27,15 @@ constexpr char kRemotingRpcNamespace[] = "urn:x-cast:com.google.cast.remoting"; MirroringApplication::MirroringApplication(TaskRunner& task_runner, const IPAddress& interface_address, - ApplicationAgent& agent) + ApplicationAgent& agent, + bool enable_dscp, + bool enable_input_events) : task_runner_(task_runner), interface_address_(interface_address), app_ids_(GetCastStreamingAppIds()), - agent_(agent) { + agent_(agent), + enable_dscp_(enable_dscp), + enable_input_events_(enable_input_events) { agent_.RegisterApplication(this); } @@ -57,17 +62,25 @@ bool MirroringApplication::Launch(const std::string& app_id, &Clock::now, task_runner_, IPEndpoint{interface_address_, kDefaultCastStreamingPort}); #if defined(CAST_STANDALONE_RECEIVER_HAVE_EXTERNAL_LIBS) - controller_ = - std::make_unique(task_runner_, this); + controller_ = std::make_unique( + task_runner_, this, enable_input_events_); #else - controller_ = std::make_unique(this); + controller_ = + std::make_unique(this, enable_input_events_); #endif // defined(CAST_STANDALONE_RECEIVER_HAVE_EXTERNAL_LIBS) ReceiverConstraints constraints; constraints.video_codecs.insert(constraints.video_codecs.begin(), {VideoCodec::kAv1, VideoCodec::kVp9}); constraints.remoting = std::make_unique(); + constraints.enable_dscp = enable_dscp_; + constraints.supports_input_events = enable_input_events_; current_session_ = std::make_unique( *controller_, *environment_, *message_port, std::move(constraints)); + + for (auto const& [ns, handler] : custom_message_handlers_) { + current_session_->SetCustomMessageHandler(ns, handler); + } + return true; } @@ -80,10 +93,19 @@ std::string MirroringApplication::GetDisplayName() { } std::vector MirroringApplication::GetSupportedNamespaces() { - return {kCastWebrtcNamespace, kRemotingRpcNamespace}; + std::vector namespaces = {kCastWebrtcNamespace, + kRemotingRpcNamespace}; + namespaces.insert(namespaces.end(), custom_namespaces_.begin(), + custom_namespaces_.end()); + return namespaces; } void MirroringApplication::Stop() { + if (current_session_) { + for (auto const& [ns, handler] : custom_message_handlers_) { + current_session_->SetCustomMessageHandler(ns, nullptr); + } + } current_session_.reset(); controller_.reset(); environment_.reset(); @@ -96,4 +118,68 @@ void MirroringApplication::OnPlaybackError(StreamingPlaybackController*, agent_.StopApplicationIfRunning(this); // ApplicationAgent calls Stop(). } +void MirroringApplication::AddCustomNamespace( + std::string_view message_namespace) { + if (std::find(custom_namespaces_.begin(), custom_namespaces_.end(), + message_namespace) == custom_namespaces_.end()) { + custom_namespaces_.push_back(std::string(message_namespace)); + } +} + +void MirroringApplication::RemoveCustomNamespace( + std::string_view message_namespace) { + auto it = std::find(custom_namespaces_.begin(), custom_namespaces_.end(), + message_namespace); + if (it != custom_namespaces_.end()) { + custom_namespaces_.erase(it); + } +} + +void MirroringApplication::SetCustomMessageHandler( + std::string_view message_namespace, + CustomMessageCallback cb) { + auto it = std::find_if(custom_message_handlers_.begin(), + custom_message_handlers_.end(), + [&message_namespace](const auto& pair) { + return pair.first == message_namespace; + }); + + if (!cb) { + if (it != custom_message_handlers_.end()) { + custom_message_handlers_.erase(it); + } + if (current_session_) { + current_session_->SetCustomMessageHandler(message_namespace, nullptr); + } + return; + } + + if (it != custom_message_handlers_.end()) { + OSP_LOG_ERROR << "Handler already exists for namespace: " + << message_namespace; + return; + } else { + custom_message_handlers_.emplace_back(std::string(message_namespace), cb); + } + + if (current_session_) { + current_session_->SetCustomMessageHandler(message_namespace, std::move(cb)); + } +} + +void MirroringApplication::SendMessage(std::string_view destination_id, + std::string_view message_namespace, + std::string_view message) { + if (!current_session_ || !current_session_->messenger()) { + OSP_LOG_ERROR << "Cannot send message: session or messenger is null"; + return; + } + + const Error error = current_session_->messenger()->SendMessage( + destination_id, message_namespace, message); + if (!error.ok()) { + OSP_LOG_ERROR << "Failed to send message: " << error; + } +} + } // namespace openscreen::cast diff --git a/cast/standalone_receiver/mirroring_application.h b/cast/standalone_receiver/mirroring_application.h index 845d4b934..147ae8b12 100644 --- a/cast/standalone_receiver/mirroring_application.h +++ b/cast/standalone_receiver/mirroring_application.h @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include "cast/receiver/application_agent.h" @@ -32,7 +34,9 @@ class MirroringApplication final : public ApplicationAgent::Application, public: MirroringApplication(TaskRunner& task_runner, const IPAddress& interface_address, - ApplicationAgent& agent); + ApplicationAgent& agent, + bool enable_dscp, + bool enable_input_events); ~MirroringApplication() final; @@ -50,11 +54,31 @@ class MirroringApplication final : public ApplicationAgent::Application, void OnPlaybackError(StreamingPlaybackController* controller, const Error& error) final; + void AddCustomNamespace(std::string_view message_namespace); + void RemoveCustomNamespace(std::string_view message_namespace); + + using CustomMessageCallback = + std::function; + void SetCustomMessageHandler(std::string_view message_namespace, + CustomMessageCallback cb); + + void SendMessage(std::string_view destination_id, + std::string_view message_namespace, + std::string_view message); + private: TaskRunner& task_runner_; const IPAddress interface_address_; const std::vector app_ids_; ApplicationAgent& agent_; + const bool enable_dscp_; + const bool enable_input_events_; + + std::vector custom_namespaces_; + std::vector> + custom_message_handlers_; ScopedWakeLockPtr wake_lock_; std::unique_ptr environment_; diff --git a/cast/standalone_receiver/sdl_audio_player.cc b/cast/standalone_receiver/sdl_audio_player.cc index 358d18df9..3907ce741 100644 --- a/cast/standalone_receiver/sdl_audio_player.cc +++ b/cast/standalone_receiver/sdl_audio_player.cc @@ -4,11 +4,12 @@ #include "cast/standalone_receiver/sdl_audio_player.h" +#include #include #include #include -#include "cast/standalone_receiver/avcodec_glue.h" +#include "cast/standalone_common/ffmpeg_glue.h" #include "platform/base/span.h" #include "util/big_endian.h" #include "util/chrono_helpers.h" diff --git a/cast/standalone_receiver/sdl_glue.cc b/cast/standalone_receiver/sdl_glue.cc index ef622bcc6..f771b0a7b 100644 --- a/cast/standalone_receiver/sdl_glue.cc +++ b/cast/standalone_receiver/sdl_glue.cc @@ -27,6 +27,11 @@ void SDLEventLoopProcessor::RegisterForKeyboardEvent( keyboard_callbacks_.push_back(std::move(cb)); } +void SDLEventLoopProcessor::RegisterForMouseButtonEvent( + SDLEventLoopProcessor::MouseButtonEventCallback cb) { + mouse_button_callbacks_.push_back(std::move(cb)); +} + void SDLEventLoopProcessor::ProcessPendingEvents() { // Process all pending events. SDL_Event event; @@ -40,6 +45,11 @@ void SDLEventLoopProcessor::ProcessPendingEvents() { for (auto& cb : keyboard_callbacks_) { cb(event.key); } + } else if (event.type == SDL_MOUSEBUTTONDOWN || + event.type == SDL_MOUSEBUTTONUP) { + for (auto& cb : mouse_button_callbacks_) { + cb(event.button); + } } } diff --git a/cast/standalone_receiver/sdl_glue.h b/cast/standalone_receiver/sdl_glue.h index eba30829c..5e3b17b7f 100644 --- a/cast/standalone_receiver/sdl_glue.h +++ b/cast/standalone_receiver/sdl_glue.h @@ -52,9 +52,9 @@ class ScopedSDLSubSystem { SDL_Create##name(std::forward(args)...)); \ } -DEFINE_SDL_UNIQUE_PTR(Window); -DEFINE_SDL_UNIQUE_PTR(Renderer); -DEFINE_SDL_UNIQUE_PTR(Texture); +DEFINE_SDL_UNIQUE_PTR(Window) +DEFINE_SDL_UNIQUE_PTR(Renderer) +DEFINE_SDL_UNIQUE_PTR(Texture) #undef DEFINE_SDL_UNIQUE_PTR @@ -71,12 +71,17 @@ class SDLEventLoopProcessor { using KeyboardEventCallback = std::function; void RegisterForKeyboardEvent(KeyboardEventCallback cb); + using MouseButtonEventCallback = + std::function; + void RegisterForMouseButtonEvent(MouseButtonEventCallback cb); + private: void ProcessPendingEvents(); Alarm alarm_; std::function quit_callback_; std::vector keyboard_callbacks_; + std::vector mouse_button_callbacks_; }; } // namespace cast diff --git a/cast/standalone_receiver/sdl_player_base.cc b/cast/standalone_receiver/sdl_player_base.cc index 8b8c82425..bcee6c069 100644 --- a/cast/standalone_receiver/sdl_player_base.cc +++ b/cast/standalone_receiver/sdl_player_base.cc @@ -8,7 +8,7 @@ #include #include -#include "cast/standalone_receiver/avcodec_glue.h" +#include "cast/standalone_common/ffmpeg_glue.h" #include "cast/streaming/public/constants.h" #include "cast/streaming/public/encoded_frame.h" #include "util/big_endian.h" @@ -68,7 +68,7 @@ Clock::time_point SDLPlayerBase::ResyncAndDeterminePresentationTime( constexpr auto kMaxPlayoutDrift = milliseconds(100); const auto media_time_since_last_sync = (frame.rtp_timestamp - last_sync_rtp_timestamp_) - .ToDuration(receiver_.rtp_timebase()); + .ToDuration(receiver_.config().rtp_timebase); Clock::time_point presentation_time = last_sync_reference_time_ + media_time_since_last_sync; const auto drift = to_milliseconds(frame.reference_time - presentation_time); @@ -88,8 +88,7 @@ Clock::time_point SDLPlayerBase::ResyncAndDeterminePresentationTime( return presentation_time; } -void SDLPlayerBase::OnFramesReady(int buffer_size) { - TRACE_DEFAULT_SCOPED(TraceCategory::kStandaloneReceiver); +void SDLPlayerBase::OnFramesReady(size_t buffer_size) { // Do not consume anything if there are too many frames in the pipeline // already. if (static_cast(frames_to_render_.size()) > kMaxFramesInPipeline) { @@ -101,10 +100,14 @@ void SDLPlayerBase::OnFramesReady(int buffer_size) { buffer_.Resize(buffer_size); EncodedFrame frame = receiver_.ConsumeNextFrame(buffer_.AsByteBuffer()); + TRACE_FLOW_STEP(TraceCategory::kStandaloneReceiver, "Frame.Received", + frame.frame_id); + // Create the tracking state for the frame in the player pipeline. OSP_CHECK_EQ(frames_to_render_.count(frame.frame_id), 0); PendingFrame& pending_frame = frames_to_render_[frame.frame_id]; pending_frame.start_time = start_time; + pending_frame.rtp_timestamp = frame.rtp_timestamp; pending_frame.presentation_time = ResyncAndDeterminePresentationTime(frame); @@ -114,7 +117,6 @@ void SDLPlayerBase::OnFramesReady(int buffer_size) { } void SDLPlayerBase::OnFrameDecoded(FrameId frame_id, const AVFrame& frame) { - TRACE_DEFAULT_SCOPED(TraceCategory::kStandaloneReceiver); const auto it = frames_to_render_.find(frame_id); if (it == frames_to_render_.end()) { return; @@ -182,21 +184,28 @@ void SDLPlayerBase::RenderAndSchedulePresentation() { // Remove the frame from the queue, making it the `current_frame_`. Then, // render it and, if successful, schedule its presentation. + const FrameId frame_id = it->first; current_frame_ = std::move(it->second); frames_to_render_.erase(it); + + TRACE_FLOW_STEP(TraceCategory::kStandaloneReceiver, "Frame.Render.Begin", + frame_id); const ErrorOr presentation_time = RenderNextFrame(current_frame_); + TRACE_FLOW_STEP(TraceCategory::kStandaloneReceiver, "Frame.Render.End", + frame_id); if (!presentation_time) { OnFatalError(presentation_time.error().message()); return; } state_ = kScheduledToPresent; presentation_alarm_.Schedule( - [this] { + [this, frame_id, rtp_timestamp = current_frame_.rtp_timestamp] { Present(); if (state_ == kScheduledToPresent) { state_ = kPresented; } + receiver_.ReportPlayoutEvent(frame_id, rtp_timestamp, now_()); ResumeRendering(); }, presentation_time.value()); @@ -221,9 +230,10 @@ void SDLPlayerBase::RenderAndSchedulePresentation() { void SDLPlayerBase::ResumeDecoding() { decode_alarm_.Schedule( [this] { - const int buffer_size = receiver_.AdvanceToNextFrame(); - if (buffer_size != Receiver::kNoFramesReady) { - OnFramesReady(buffer_size); + const std::optional buffer_size = + receiver_.AdvanceToNextFrame(); + if (buffer_size) { + OnFramesReady(*buffer_size); } }, Alarm::kImmediately); diff --git a/cast/standalone_receiver/sdl_player_base.h b/cast/standalone_receiver/sdl_player_base.h index 888cf9c96..9cbb2d70a 100644 --- a/cast/standalone_receiver/sdl_player_base.h +++ b/cast/standalone_receiver/sdl_player_base.h @@ -85,6 +85,7 @@ class SDLPlayerBase : public Receiver::Consumer, public Decoder::Client { private: struct PendingFrame : public PresentableFrame { Clock::time_point start_time; + RtpTimeTicks rtp_timestamp; PendingFrame(); ~PendingFrame(); @@ -93,7 +94,7 @@ class SDLPlayerBase : public Receiver::Consumer, public Decoder::Client { }; // Receiver::Consumer implementation. - void OnFramesReady(int next_frame_buffer_size) final; + void OnFramesReady(size_t next_frame_buffer_size) final; // Determine the presentation time of the frame. Ideally, this will occur // based on the time progression of the media, given by the RTP timestamps. diff --git a/cast/standalone_receiver/sdl_video_player.cc b/cast/standalone_receiver/sdl_video_player.cc index 600d227de..f6a0e2b6f 100644 --- a/cast/standalone_receiver/sdl_video_player.cc +++ b/cast/standalone_receiver/sdl_video_player.cc @@ -7,7 +7,7 @@ #include #include -#include "cast/standalone_receiver/avcodec_glue.h" +#include "cast/standalone_common/ffmpeg_glue.h" #include "util/enum_name_table.h" #include "util/osp_logging.h" #include "util/trace_logging.h" diff --git a/cast/standalone_receiver/streaming_playback_controller.cc b/cast/standalone_receiver/streaming_playback_controller.cc index 2c35b7c76..1ccfec2fb 100644 --- a/cast/standalone_receiver/streaming_playback_controller.cc +++ b/cast/standalone_receiver/streaming_playback_controller.cc @@ -23,9 +23,11 @@ StreamingPlaybackController::Client::~Client() = default; #if defined(CAST_STANDALONE_RECEIVER_HAVE_EXTERNAL_LIBS) StreamingPlaybackController::StreamingPlaybackController( TaskRunner& task_runner, - StreamingPlaybackController::Client* client) + StreamingPlaybackController::Client* client, + bool enable_input_events) : client_(client), task_runner_(task_runner), + enable_input_events_(enable_input_events), sdl_event_loop_(task_runner_, [this] { client_->OnPlaybackError(this, Error{Error::Code::kOperationCancelled, @@ -47,10 +49,18 @@ StreamingPlaybackController::StreamingPlaybackController( [this](const SDL_KeyboardEvent& event) { this->HandleKeyboardEvent(event); }); + + if (enable_input_events_) { + sdl_event_loop_.RegisterForMouseButtonEvent( + [this](const SDL_MouseButtonEvent& event) { + this->HandleMouseButtonEvent(event); + }); + } } #else StreamingPlaybackController::StreamingPlaybackController( - StreamingPlaybackController::Client* client) + StreamingPlaybackController::Client* client, + bool enable_input_events) : client_(client) { OSP_CHECK(client_); } @@ -60,12 +70,14 @@ void StreamingPlaybackController::OnNegotiated( const ReceiverSession* session, ReceiverSession::ConfiguredReceivers receivers) { TRACE_DEFAULT_SCOPED(TraceCategory::kStandaloneReceiver); + session_ = session; Initialize(receivers); } void StreamingPlaybackController::OnRemotingNegotiated( const ReceiverSession* session, ReceiverSession::RemotingNegotiation negotiation) { + session_ = session; remoting_receiver_ = std::make_unique(negotiation.messenger); remoting_receiver_->SendInitializeMessage( @@ -88,6 +100,7 @@ void StreamingPlaybackController::OnReceiversDestroying( OSP_LOG_INFO << "Receivers are currently destroying, resetting SDL players."; audio_player_.reset(); video_player_.reset(); + session_ = nullptr; } void StreamingPlaybackController::OnError(const ReceiverSession* session, @@ -99,6 +112,13 @@ void StreamingPlaybackController::Initialize( ReceiverSession::ConfiguredReceivers receivers) { #if defined(CAST_STANDALONE_RECEIVER_HAVE_EXTERNAL_LIBS) OSP_LOG_INFO << "Successfully negotiated a session, creating SDL players."; + if (receivers.video_receiver && !receivers.video_config.resolutions.empty()) { + const auto& res = receivers.video_config.resolutions[0]; + SDL_RenderSetLogicalSize(renderer_.get(), res.width, res.height); + } else { + // Default to a logical size of 1920x1080. + SDL_RenderSetLogicalSize(renderer_.get(), 1920, 1080); + } if (receivers.audio_receiver) { audio_player_ = std::make_unique( &Clock::now, task_runner_, *receivers.audio_receiver, @@ -141,6 +161,55 @@ void StreamingPlaybackController::HandleKeyboardEvent( break; } } + +void StreamingPlaybackController::HandleMouseButtonEvent( + const SDL_MouseButtonEvent& event) { + if (!session_) { + return; + } + + int w, h; + SDL_RenderGetLogicalSize(renderer_.get(), &w, &h); + if (w <= 0 || h <= 0) { + return; + } + + // If the click is outside the logical area (e.g. in letterboxing), skip it. + // Note: SDL2 automatically scales mouse event coordinates to the logical + // size if it is set on the renderer. + if (event.x < 0 || event.x >= w || event.y < 0 || event.y >= h) { + return; + } + + InputMessage message; + auto* input_event = message.add_events(); + auto* ts = input_event->mutable_timestamp(); + ts->set_seconds(event.timestamp / 1000); + ts->set_nanos((event.timestamp % 1000) * 1000000); + + if (event.type == SDL_MOUSEBUTTONDOWN) { + input_event->set_type(InputMessage::INPUT_TYPE_MOUSE_DOWN); + } else { + input_event->set_type(InputMessage::INPUT_TYPE_MOUSE_UP); + } + + auto* mouse_event = input_event->mutable_mouse_event(); + auto* loc = mouse_event->mutable_location(); + loc->set_x(static_cast(event.x) / static_cast(w)); + loc->set_y(static_cast(event.y) / static_cast(h)); + if (event.button == SDL_BUTTON_LEFT) { + mouse_event->add_buttons(InputMessage::MOUSE_BUTTON_PRIMARY); + } else if (event.button == SDL_BUTTON_RIGHT) { + mouse_event->add_buttons(InputMessage::MOUSE_BUTTON_SECONDARY); + } else if (event.button == SDL_BUTTON_MIDDLE) { + mouse_event->add_buttons(InputMessage::MOUSE_BUTTON_AUXILIARY); + } + + message.set_viewport_width(w); + message.set_viewport_height(h); + + const_cast(session_)->SendInputMessage(message); +} #endif } // namespace openscreen::cast diff --git a/cast/standalone_receiver/streaming_playback_controller.h b/cast/standalone_receiver/streaming_playback_controller.h index f6cfd67c3..1496cd6e0 100644 --- a/cast/standalone_receiver/streaming_playback_controller.h +++ b/cast/standalone_receiver/streaming_playback_controller.h @@ -34,10 +34,11 @@ class StreamingPlaybackController final : public ReceiverSession::Client { #if defined(CAST_STANDALONE_RECEIVER_HAVE_EXTERNAL_LIBS) StreamingPlaybackController(TaskRunner& task_runner, - StreamingPlaybackController::Client* client); + StreamingPlaybackController::Client* client, + bool enable_input_events); #else - explicit StreamingPlaybackController( - StreamingPlaybackController::Client* client); + StreamingPlaybackController(StreamingPlaybackController::Client* client, + bool enable_input_events); #endif // defined(CAST_STANDALONE_RECEIVER_HAVE_EXTERNAL_LIBS) // ReceiverSession::Client overrides. @@ -57,8 +58,10 @@ class StreamingPlaybackController final : public ReceiverSession::Client { #if defined(CAST_STANDALONE_RECEIVER_HAVE_EXTERNAL_LIBS) void HandleKeyboardEvent(const SDL_KeyboardEvent& event); + void HandleMouseButtonEvent(const SDL_MouseButtonEvent& event); TaskRunner& task_runner_; + const bool enable_input_events_; // NOTE: member ordering is important, since the sub systems must be // first-constructed, last-destroyed. Make sure any new SDL related @@ -77,6 +80,7 @@ class StreamingPlaybackController final : public ReceiverSession::Client { std::unique_ptr video_player_; #endif // defined(CAST_STANDALONE_RECEIVER_HAVE_EXTERNAL_LIBS) + const ReceiverSession* session_ = nullptr; std::unique_ptr remoting_receiver_; }; diff --git a/cast/standalone_sender/BUILD.gn b/cast/standalone_sender/BUILD.gn index 4383a9beb..58f65578b 100644 --- a/cast/standalone_sender/BUILD.gn +++ b/cast/standalone_sender/BUILD.gn @@ -1,6 +1,6 @@ # Copyright 2020 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. +# found in the LICENSE file. import("//build_overrides/build.gni") import("../../gni/openscreen.gni") @@ -50,8 +50,6 @@ if (!build_with_chromium) { if (have_external_libs) { sources += [ "connection_settings.h", - "ffmpeg_glue.cc", - "ffmpeg_glue.h", "looping_file_cast_agent.cc", "looping_file_cast_agent.h", "looping_file_sender.cc", @@ -71,10 +69,12 @@ if (!build_with_chromium) { "streaming_vpx_encoder.cc", "streaming_vpx_encoder.h", ] + deps += [ "../standalone_common:ffmpeg_glue" ] include_dirs += ffmpeg_include_dirs + libopus_include_dirs + libvpx_include_dirs - lib_dirs += ffmpeg_lib_dirs + libopus_lib_dirs + libvpx_lib_dirs + lib_dirs += external_lib_dirs + ffmpeg_lib_dirs + libopus_lib_dirs + + libvpx_lib_dirs libs += ffmpeg_libs + libopus_libs + libvpx_libs # LibAOM support currently recommends building from source, so is included diff --git a/cast/standalone_sender/DEPS b/cast/standalone_sender/DEPS index 261da7a4f..0fd550cc9 100644 --- a/cast/standalone_sender/DEPS +++ b/cast/standalone_sender/DEPS @@ -9,6 +9,7 @@ include_rules = [ '+cast/streaming/public', '+cast/streaming/message_fields.h', '+cast/streaming/rtp_time.h', + '+cast/standalone_common', '+discovery', '+platform/impl', ] diff --git a/cast/standalone_sender/connection_settings.h b/cast/standalone_sender/connection_settings.h index 5af6d1054..a4078ce33 100644 --- a/cast/standalone_sender/connection_settings.h +++ b/cast/standalone_sender/connection_settings.h @@ -43,6 +43,12 @@ struct ConnectionSettings { // The codec to use for encoding negotiated video streams. VideoCodec codec; + + // Whether DSCP support should be enabled for Quality of Service. + bool enable_dscp = true; + + // Whether input event API should be enabled for this session. + bool enable_input_events = false; }; } // namespace openscreen::cast diff --git a/cast/standalone_sender/looping_file_cast_agent.cc b/cast/standalone_sender/looping_file_cast_agent.cc index 59f6c3eea..f77a8c493 100644 --- a/cast/standalone_sender/looping_file_cast_agent.cc +++ b/cast/standalone_sender/looping_file_cast_agent.cc @@ -137,6 +137,12 @@ void LoopingFileCastAgent::OnMessage(VirtualConnectionRouter* router, const ErrorOr payload = json::Parse(GetPayload(message)); if (payload.is_error()) { OSP_LOG_ERROR << "Failed to parse message: " << payload.error(); + return; + } + + if (!payload.value().isObject()) { + OSP_LOG_ERROR << "Parsed message is not a JSON object"; + return; } if (HasType(payload.value(), CastMessageType::kReceiverStatus)) { @@ -266,12 +272,12 @@ void LoopingFileCastAgent::OnReceiverMessagingOpened(bool success) { } static constexpr char kLaunchMessageTemplate[] = - R"({"type":"LAUNCH", "requestId":%d, "appId":"%s", "language": "en-US", - "supportedAppTypes":["WEB"]})"; + R"({{"type":"LAUNCH", "requestId":{}, "appId":"{}", "language": "en-US", + "supportedAppTypes":["WEB"]}})"; router_.Send(*platform_remote_connection_, MakeSimpleUTF8Message( kReceiverNamespace, - StringPrintf(kLaunchMessageTemplate, next_request_id_++, + StringFormat(kLaunchMessageTemplate, next_request_id_++, GetStreamingAppId()))); } @@ -289,9 +295,14 @@ void LoopingFileCastAgent::CreateAndStartSession() { &message_port_, remote_connection_->local_id, remote_connection_->peer_id, - connection_settings_->use_android_rtp_hack}; + connection_settings_->use_android_rtp_hack, + connection_settings_->enable_dscp}; current_session_ = std::make_unique(std::move(config)); current_session_->SetStatsClient(this); + if (connection_settings_->enable_input_events) { + current_session_->SetInputCallback( + [this](InputMessage message) { OnInputMessage(std::move(message)); }); + } OSP_CHECK(!message_port_.source_id().empty()); AudioCaptureConfig audio_config; @@ -347,12 +358,46 @@ void LoopingFileCastAgent::OnError(const SenderSession* session, Shutdown(); } +void LoopingFileCastAgent::OnInputMessage(InputMessage message) { + if (file_sender_) { + file_sender_->OnInputMessage(message); + } + for (const auto& event : message.events()) { + std::string type_name; + switch (event.type()) { + case InputMessage::INPUT_TYPE_MOUSE_DOWN: + type_name = "MOUSE_DOWN"; + break; + case InputMessage::INPUT_TYPE_MOUSE_UP: + type_name = "MOUSE_UP"; + break; + case InputMessage::INPUT_TYPE_MOUSE_MOVE: + type_name = "MOUSE_MOVE"; + break; + default: + type_name = "OTHER"; + break; + } + + if (event.has_mouse_event()) { + const auto& mouse = event.mouse_event(); + OSP_LOG_INFO << "[Input] Received " << type_name << " at (" + << mouse.location().x() << ", " << mouse.location().y() + << ") buttons=" << mouse.buttons_size() + << " viewport_size=" << message.viewport_width() << "x" + << message.viewport_height(); + } else { + OSP_LOG_INFO << "[Input] Received event type=" << type_name; + } + } +} + void LoopingFileCastAgent::OnStatisticsUpdated( const SenderStats& updated_stats) { // Only log every 10 times, or roughly every 5 seconds. constexpr int kLoggingInterval = 10; if ((num_times_on_statistics_updated_called_++ % kLoggingInterval) == 0) { - OSP_VLOG << __func__ << ": updated_stats=" << updated_stats.ToString(); + OSP_VLOG << __func__ << ": updated_stats=" << updated_stats; } last_reported_statistics_ = std::make_optional(updated_stats); } @@ -387,8 +432,7 @@ void LoopingFileCastAgent::Shutdown() { current_session_.reset(); if (last_reported_statistics_) { - OSP_LOG_INFO << "Last reported statistics=" - << last_reported_statistics_->ToString(); + OSP_LOG_INFO << "Last reported statistics=" << *last_reported_statistics_; } } OSP_CHECK(message_port_.source_id().empty()); @@ -413,8 +457,8 @@ void LoopingFileCastAgent::Shutdown() { if (!app_session_id_.empty()) { OSP_LOG_INFO << "Stopping the Cast Receiver's Mirroring App..."; static constexpr char kStopMessageTemplate[] = - R"({"type":"STOP", "requestId":%d, "sessionId":"%s"})"; - std::string stop_json = StringPrintf( + R"({{"type":"STOP", "requestId":{}, "sessionId":"{}"}})"; + std::string stop_json = StringFormat( kStopMessageTemplate, next_request_id_++, app_session_id_.c_str()); router_.Send( VirtualConnection{kPlatformSenderId, kPlatformReceiverId, diff --git a/cast/standalone_sender/looping_file_cast_agent.h b/cast/standalone_sender/looping_file_cast_agent.h index cf88a7382..d391bf473 100644 --- a/cast/standalone_sender/looping_file_cast_agent.h +++ b/cast/standalone_sender/looping_file_cast_agent.h @@ -140,6 +140,9 @@ class LoopingFileCastAgent final capture_recommendations) override; void OnError(const SenderSession* session, const Error& error) override; + // Input message handler. + void OnInputMessage(InputMessage message); + // SenderStatsClient overrides. void OnStatisticsUpdated(const SenderStats& updated_stats) override; diff --git a/cast/standalone_sender/looping_file_sender.cc b/cast/standalone_sender/looping_file_sender.cc index 3e656af71..9e4e04254 100644 --- a/cast/standalone_sender/looping_file_sender.cc +++ b/cast/standalone_sender/looping_file_sender.cc @@ -56,6 +56,18 @@ void LoopingFileSender::SetPlaybackRate(double rate) { audio_capturer_->SetPlaybackRate(rate); } +void LoopingFileSender::OnInputMessage(InputMessage message) { + const auto now = env_.now(); + for (const auto& event : message.events()) { + if (event.type() == InputMessage::INPUT_TYPE_MOUSE_DOWN && + event.has_mouse_event()) { + active_clicks_.push_back(Click{event.mouse_event().location().x(), + event.mouse_event().location().y(), now, + now + std::chrono::milliseconds(500)}); + } + } +} + void LoopingFileSender::UpdateEncoderBitrates() { if (bandwidth_being_utilized_ >= kHighBandwidthThreshold) { audio_encoder_.UseHighQuality(); @@ -142,6 +154,9 @@ void LoopingFileSender::OnVideoFrame(const AVFrame& av_frame, StreamingVideoEncoder::VideoFrame frame{}; frame.width = av_frame.width - av_frame.crop_left - av_frame.crop_right; frame.height = av_frame.height - av_frame.crop_top - av_frame.crop_bottom; + + DrawAnimations(av_frame, frame.width, frame.height); + frame.yuv_planes[0] = av_frame.data[0] + av_frame.crop_left + av_frame.linesize[0] * av_frame.crop_top; frame.yuv_planes[1] = av_frame.data[1] + av_frame.crop_left / 2 + @@ -161,24 +176,87 @@ void LoopingFileSender::OnVideoFrame(const AVFrame& av_frame, void LoopingFileSender::UpdateStatusOnConsole() { const Clock::duration elapsed = latest_frame_time_ - capture_begin_time_; - const auto seconds_part = to_seconds(elapsed); - const auto millis_part = to_milliseconds(elapsed - seconds_part); // The control codes here attempt to erase the current line the cursor is // on, and then print out the updated status text. If the terminal does not // support simple ANSI escape codes, the following will still work, but // there might sometimes be old status lines not getting erased (i.e., just // partially overwritten). - fprintf(stdout, - "\r\x1b[2K\rLoopingFileSender: At %01" PRId64 - ".%03ds in file (est. network bandwidth: %d kbps). \n", - static_cast(seconds_part.count()), - static_cast(millis_part.count()), bandwidth_estimate_ / 1024); - fflush(stdout); - + OSP_VLOG << "LoopingFileSender: At " << elapsed + << " in file (est. network bandwidth: " << bandwidth_estimate_ / 1024 + << ")."; console_update_task_.ScheduleFromNow([this] { UpdateStatusOnConsole(); }, kConsoleUpdateInterval); } +void LoopingFileSender::DrawAnimations(const AVFrame& av_frame, + int frame_width, + int frame_height) { + const auto now = env_.now(); + // Remove clicks that have exceeded their 500ms display duration. + active_clicks_.erase( + std::remove_if(active_clicks_.begin(), active_clicks_.end(), + [now](const Click& c) { return c.end_time < now; }), + active_clicks_.end()); + + if (active_clicks_.empty()) { + return; + } + + // We modify the AVFrame data in-place. We use the same base pointer offsets + // as the encoder to account for any cropping or padding in the original file. + uint8_t* const y_plane = av_frame.data[0] + av_frame.crop_left + + av_frame.linesize[0] * av_frame.crop_top; + uint8_t* const u_plane = av_frame.data[1] + av_frame.crop_left / 2 + + av_frame.linesize[1] * av_frame.crop_top / 2; + uint8_t* const v_plane = av_frame.data[2] + av_frame.crop_left / 2 + + av_frame.linesize[2] * av_frame.crop_top / 2; + + for (const auto& click : active_clicks_) { + // Map the click coordinates from the receiver's logical display space + // to the sender's actual video frame resolution. + const float scale_x = static_cast(frame_width); + const float scale_y = static_cast(frame_height); + const int center_x = static_cast(std::round(click.x * scale_x)); + const int center_y = static_cast(std::round(click.y * scale_y)); + + // Calculate animation progress (0.0 to 1.0) and the resulting ring radius. + const double duration = (click.end_time - click.start_time).count(); + const double elapsed = (now - click.start_time).count(); + const double progress = std::clamp(elapsed / duration, 0.0, 1.0); + + const float radius = static_cast(5.0 + 40.0 * progress); + const float thickness = 3.0f; + const float inner_radius_sq = std::pow(radius - thickness, 2); + const float outer_radius_sq = std::pow(radius, 2); + + // Iterate over a bounding box for the ring and draw pixels that fall + // within the calculated thickness. + const int box_size = static_cast(radius) + 1; + for (int dy = -box_size; dy <= box_size; ++dy) { + for (int dx = -box_size; dx <= box_size; ++dx) { + const float dist_sq = static_cast(dx * dx + dy * dy); + if (dist_sq >= inner_radius_sq && dist_sq <= outer_radius_sq) { + const int px = center_x + dx; + const int py = center_y + dy; + + if (px >= 0 && px < frame_width && py >= 0 && py < frame_height) { + // In YUV, pure white is represented by maximum luminance (Y=255) + // and neutral chrominance (U=128, V=128). + y_plane[py * av_frame.linesize[0] + px] = 255; + + // Chrominance (U/V) planes are sub-sampled 2x2 in YUV420p. + if (px % 2 == 0 && py % 2 == 0) { + const int uv_offset = (py / 2) * av_frame.linesize[1] + (px / 2); + u_plane[uv_offset] = 128; + v_plane[uv_offset] = 128; + } + } + } + } + } + } +} + void LoopingFileSender::OnEndOfFile(SimulatedCapturer* capturer) { OSP_LOG_INFO << "The " << ToTrackName(capturer) << " capturer has reached the end of the media stream."; diff --git a/cast/standalone_sender/looping_file_sender.h b/cast/standalone_sender/looping_file_sender.h index d5fb3d9a9..910416041 100644 --- a/cast/standalone_sender/looping_file_sender.h +++ b/cast/standalone_sender/looping_file_sender.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "cast/standalone_sender/connection_settings.h" #include "cast/standalone_sender/constants.h" @@ -36,6 +37,8 @@ class LoopingFileSender final : public SimulatedAudioCapturer::Client, void SetPlaybackRate(double rate); + void OnInputMessage(InputMessage message); + private: void UpdateEncoderBitrates(); void ControlForNetworkCongestion(); @@ -56,6 +59,11 @@ class LoopingFileSender final : public SimulatedAudioCapturer::Client, void UpdateStatusOnConsole(); + // Draws any active animations (like mouse clicks) onto the frame. + void DrawAnimations(const AVFrame& av_frame, + int frame_width, + int frame_height); + // SimulatedCapturer::Client overrides. void OnEndOfFile(SimulatedCapturer* capturer) final; void OnError(SimulatedCapturer* capturer, const std::string& message) final; @@ -93,6 +101,14 @@ class LoopingFileSender final : public SimulatedAudioCapturer::Client, std::optional audio_capturer_; std::optional video_capturer_; + struct Click { + float x; + float y; + Clock::time_point start_time; + Clock::time_point end_time; + }; + std::vector active_clicks_; + Alarm next_task_; Alarm console_update_task_; }; diff --git a/cast/standalone_sender/main.cc b/cast/standalone_sender/main.cc index 3e80e558e..d1716dc6c 100644 --- a/cast/standalone_sender/main.cc +++ b/cast/standalone_sender/main.cc @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include + #include "platform/impl/logging.h" #if defined(CAST_STANDALONE_SENDER_HAVE_EXTERNAL_LIBS) @@ -27,18 +29,24 @@ #include "platform/impl/text_trace_logging_platform.h" #include "third_party/getopt/getopt.h" #include "util/chrono_helpers.h" +#include "util/string_parse.h" +#include "util/string_util.h" #include "util/stringprintf.h" +#if defined(USE_PERFETTO) +#include "platform/impl/perfetto_trace_logging_platform.h" +#endif + namespace openscreen::cast { namespace { void LogUsage(const char* argv0) { - constexpr char kTemplate[] = R"( -usage: %s network_interface media_file + static constexpr char kTemplate[] = R"( +usage: {} network_interface media_file or -usage: %s addr[:port] media_file +usage: {} addr[:port] media_file The first form runs this application in discovery+interactive mode. It will scan for Cast Receivers on the LAN reachable from the given network @@ -49,37 +57,46 @@ usage: %s addr[:port] media_file discover Cast Receivers, and instead connect directly to the Cast Receiver at addr:[port] (e.g., 192.168.1.22, 192.168.1.22:%d or [::1]:%d). - -m, --max-bitrate=N - Specifies the maximum bits per second for the media streams. +options: + -a, --android-hack: + Use the wrong RTP payload types, for compatibility with older Android + TV receivers. See https://crbug.com/631828. + + -c, --codec: Specifies the video codec to be used. Can be one of: + vp8, vp9, av1. Defaults to vp8 if not specified. + + -d, --developer-certificate=path-to-cert + Specifies the path to a self-signed developer certificate that will + be permitted for use as a root CA certificate for receivers that + this sender instance will connect to. If omitted, only connections to + receivers using an official Google-signed cast certificate chain will + be permitted. + + -h, --help: Show this help message. - Default if not set: %d + -m, --max-bitrate=N + Specifies the maximum bits per second for the media streams. + Default if not set: %d - -n, --no-looping - Disable looping the passed in video after it finishes playing. + -n, --no-looping + Disable looping the passed in video after it finishes playing. - -d, --developer-certificate=path-to-cert - Specifies the path to a self-signed developer certificate that will - be permitted for use as a root CA certificate for receivers that - this sender instance will connect to. If omitted, only connections to - receivers using an official Google-signed cast certificate chain will - be permitted. - -a, --android-hack: - Use the wrong RTP payload types, for compatibility with older Android - TV receivers. See https://crbug.com/631828. + -q, --disable-dscp: Disable DSCP packet prioritization, used for QoS over + the UDP socket connection. - -r, --remoting: Enable remoting content instead of mirroring. + -r, --remoting: Enable remoting content instead of mirroring. - -t, --tracing: Enable performance tracing logging. + -t, --tracing: Enable text based performance trace logging. - -v, --verbose: Enable verbose logging. + -v, --verbose: Enable verbose logging. - -h, --help: Show this help message. + -P, --perfetto: Enable perfetto based performance trace logging. + + -i, --enable-input-events: Enable receiving input events from the receiver. - -c, --codec: Specifies the video codec to be used. Can be one of: - vp8, vp9, av1. Defaults to vp8 if not specified. )"; - std::cerr << StringPrintf(kTemplate, argv0, argv0, kDefaultCastPort, + std::cerr << StringFormat(kTemplate, argv0, argv0, kDefaultCastPort, kDefaultCastPort, kDefaultMaxBitrate); } @@ -102,101 +119,143 @@ IPEndpoint ParseAsEndpoint(const char* string_form) { return result; } -int StandaloneSenderMain(int argc, char* argv[]) { +std::optional ParseCodec(std::string_view arg) { + // We can only support codecs that have a corresponding encoder library. + static constexpr std::array kSupportedCodecs = { + {VideoCodec::kVp8, VideoCodec::kVp9, VideoCodec::kAv1}}; + + const auto parsed = StringToVideoCodec(arg); + if (!parsed || std::ranges::find(kSupportedCodecs, parsed.value()) == + std::ranges::end(kSupportedCodecs)) { + OSP_LOG_ERROR << "Invalid --codec specified: " << arg << " is not one of: " + << string_util::Join(kSupportedCodecs, " "); + return std::nullopt; + } + return parsed.value(); +} + +struct Arguments { + // Required positional arguments + const char* iface_or_endpoint = nullptr; + const char* file_path = nullptr; + + // Optional arguments + int max_bitrate = kDefaultMaxBitrate; + bool should_loop_video = true; + std::string developer_certificate_path; + bool use_android_rtp_hack = false; + bool use_remoting = false; + bool enable_input_events = false; + bool is_verbose = false; + VideoCodec codec = VideoCodec::kVp8; + std::unique_ptr trace_logger; + bool enable_dscp = true; +}; + +std::optional ParseArgs(int argc, char* argv[]) { // A note about modifying command line arguments: consider uniformity // between all Open Screen executables. If it is a platform feature // being exposed, consider if it applies to the standalone receiver, // standalone sender, osp demo, and test_main argument options. const get_opt::option kArgumentOptions[] = { + {"android-hack", no_argument, nullptr, 'a'}, + {"codec", required_argument, nullptr, 'c'}, + {"developer-certificate", required_argument, nullptr, 'd'}, + {"enable-input-events", no_argument, nullptr, 'i'}, + {"help", no_argument, nullptr, 'h'}, {"max-bitrate", required_argument, nullptr, 'm'}, {"no-looping", no_argument, nullptr, 'n'}, - {"developer-certificate", required_argument, nullptr, 'd'}, - {"android-hack", no_argument, nullptr, 'a'}, + {"disable-dscp", no_argument, nullptr, 'q'}, {"remoting", no_argument, nullptr, 'r'}, {"tracing", no_argument, nullptr, 't'}, +#if defined(USE_PERFETTO) + {"perfetto", no_argument, nullptr, 'P'}, +#endif {"verbose", no_argument, nullptr, 'v'}, - {"help", no_argument, nullptr, 'h'}, - {"codec", required_argument, nullptr, 'c'}, {nullptr, 0, nullptr, 0}}; - int max_bitrate = kDefaultMaxBitrate; - bool should_loop_video = true; - std::string developer_certificate_path; - bool use_android_rtp_hack = false; - bool use_remoting = false; - bool is_verbose = false; - VideoCodec codec = VideoCodec::kVp8; - std::unique_ptr trace_logger; + Arguments args; int ch = -1; - while ((ch = getopt_long(argc, argv, "m:nd:artvhc:", kArgumentOptions, + while ((ch = getopt_long(argc, argv, "ac:d:him:nqrtvP", kArgumentOptions, nullptr)) != -1) { switch (ch) { + case 'a': + args.use_android_rtp_hack = true; + break; case 'm': - max_bitrate = atoi(get_opt::optarg); - if (max_bitrate < kMinRequiredBitrate) { + if (!string_parse::ParseAsciiNumber(get_opt::optarg, + args.max_bitrate) || + args.max_bitrate < kMinRequiredBitrate) { OSP_LOG_ERROR << "Invalid --max-bitrate specified: " << get_opt::optarg << " is less than " << kMinRequiredBitrate; - LogUsage(argv[0]); - return 1; + return std::nullopt; } break; case 'n': - should_loop_video = false; + args.should_loop_video = false; break; case 'd': - developer_certificate_path = get_opt::optarg; + args.developer_certificate_path = get_opt::optarg; break; - case 'a': - use_android_rtp_hack = true; + case 'i': + args.enable_input_events = true; + break; + case 'q': + args.enable_dscp = false; break; case 'r': - use_remoting = true; + args.use_remoting = true; break; case 't': - trace_logger = std::make_unique(); + args.trace_logger = std::make_unique(); break; case 'v': - is_verbose = true; + args.is_verbose = true; break; case 'h': - LogUsage(argv[0]); - return 1; + return std::nullopt; case 'c': - auto specified_codec = StringToVideoCodec(get_opt::optarg); - if (specified_codec.is_value() && - (specified_codec.value() == VideoCodec::kVp8 || - specified_codec.value() == VideoCodec::kVp9 || - specified_codec.value() == VideoCodec::kAv1)) { - codec = specified_codec.value(); + if (const auto parsed = ParseCodec(get_opt::optarg)) { + args.codec = *parsed; } else { - OSP_LOG_ERROR << "Invalid --codec specified: " << get_opt::optarg - << " is not one of: vp8, vp9, av1."; - LogUsage(argv[0]); - return 1; + return std::nullopt; } break; +#if defined(USE_PERFETTO) + case 'P': + args.trace_logger = std::make_unique(); + break; +#endif } } - openscreen::SetLogLevel(is_verbose ? openscreen::LogLevel::kVerbose - : openscreen::LogLevel::kInfo); // The second to last command line argument must be one of: 1) the network // interface name or 2) a specific IP address (port is optional). The last // argument must be the path to the file. if (get_opt::optind != (argc - 2)) { + return std::nullopt; + } + args.iface_or_endpoint = argv[get_opt::optind++]; + args.file_path = argv[get_opt::optind]; + return args; +} + +int StandaloneSenderMain(int argc, char* argv[]) { + const std::optional args = ParseArgs(argc, argv); + if (!args) { LogUsage(argv[0]); return 1; } - const char* const iface_or_endpoint = argv[get_opt::optind++]; - const char* const path = argv[get_opt::optind]; + openscreen::SetLogLevel(args->is_verbose ? openscreen::LogLevel::kVerbose + : openscreen::LogLevel::kInfo); std::unique_ptr cast_trust_store; - if (!developer_certificate_path.empty()) { + if (!args->developer_certificate_path.empty()) { cast_trust_store = - TrustStore::CreateInstanceFromPemFile(developer_certificate_path); + TrustStore::CreateInstanceFromPemFile(args->developer_certificate_path); OSP_LOG_INFO << "using cast trust store generated from: " - << developer_certificate_path; + << args->developer_certificate_path; } if (!cast_trust_store) { cast_trust_store = CastTrustStore::Create(); @@ -206,10 +265,10 @@ int StandaloneSenderMain(int argc, char* argv[]) { PlatformClientPosix::Create(milliseconds(50), std::unique_ptr(task_runner)); - IPEndpoint remote_endpoint = ParseAsEndpoint(iface_or_endpoint); + IPEndpoint remote_endpoint = ParseAsEndpoint(args->iface_or_endpoint); if (!remote_endpoint.port) { for (const InterfaceInfo& interface : GetNetworkInterfaces()) { - if (interface.name == iface_or_endpoint) { + if (interface.name == args->iface_or_endpoint) { ReceiverChooser chooser(interface, *task_runner, [&](IPEndpoint endpoint) { remote_endpoint = endpoint; @@ -230,20 +289,22 @@ int StandaloneSenderMain(int argc, char* argv[]) { // `cast_agent` must be constructed and destroyed from a Task run by the // TaskRunner. - LoopingFileCastAgent* cast_agent = nullptr; + std::unique_ptr cast_agent; task_runner->PostTask([&] { - cast_agent = - new LoopingFileCastAgent(*task_runner, std::move(cast_trust_store), - [&] { task_runner->RequestStopSoon(); }); + cast_agent = std::make_unique( + *task_runner, std::move(cast_trust_store), + [&] { task_runner->RequestStopSoon(); }); cast_agent->Connect({.receiver_endpoint = remote_endpoint, - .path_to_file = path, - .max_bitrate = max_bitrate, + .path_to_file = args->file_path, + .max_bitrate = args->max_bitrate, .should_include_video = true, - .use_android_rtp_hack = use_android_rtp_hack, - .use_remoting = use_remoting, - .should_loop_video = should_loop_video, - .codec = codec}); + .use_android_rtp_hack = args->use_android_rtp_hack, + .use_remoting = args->use_remoting, + .should_loop_video = args->should_loop_video, + .codec = args->codec, + .enable_dscp = args->enable_dscp, + .enable_input_events = args->enable_input_events}); }); // Run the event loop until SIGINT (e.g., CTRL-C at the console) or @@ -254,7 +315,7 @@ int StandaloneSenderMain(int argc, char* argv[]) { // destruction/shutdown tasks. OSP_LOG_INFO << "Shutting down..."; task_runner->PostTask([&] { - delete cast_agent; + cast_agent.reset(); task_runner->RequestStopSoon(); }); task_runner->RunUntilStopped(); @@ -269,14 +330,18 @@ int StandaloneSenderMain(int argc, char* argv[]) { #endif int main(int argc, char* argv[]) { + // Ignore SIGPIPE events at the application level -- tearing down the network + // interface will close a TLS or UDP socket connection, which will result + // in a more graceful exit than terminating on the SIGPIPE call. + std::signal(SIGPIPE, SIG_IGN); + #if defined(CAST_STANDALONE_SENDER_HAVE_EXTERNAL_LIBS) return openscreen::cast::StandaloneSenderMain(argc, argv); #else OSP_LOG_ERROR << "It compiled! However, you need to configure the build to point to " "external libraries in order to build a useful app. For more " - "information, see " - "[external_libraries.md](../../build/config/external_libraries.md)."; + "information, please check the docs for external_libraries.md."; return 1; #endif } diff --git a/cast/standalone_sender/receiver_chooser.cc b/cast/standalone_sender/receiver_chooser.cc index d1a5c4f70..789de28ea 100644 --- a/cast/standalone_sender/receiver_chooser.cc +++ b/cast/standalone_sender/receiver_chooser.cc @@ -15,12 +15,6 @@ namespace openscreen::cast { -// NOTE: the compile requires a definition as well as the declaration -// in the header. -// TODO(issuetracker.google.com/174081818): move to inline C++17 feature. -constexpr decltype(ReceiverChooser::kWaitForStragglersDelay) - ReceiverChooser::kWaitForStragglersDelay; - ReceiverChooser::ReceiverChooser(const InterfaceInfo& interface, TaskRunner& task_runner, ResultCallback result_callback) diff --git a/cast/standalone_sender/simulated_capturer.cc b/cast/standalone_sender/simulated_capturer.cc index d209b1f84..4ec675051 100644 --- a/cast/standalone_sender/simulated_capturer.cc +++ b/cast/standalone_sender/simulated_capturer.cc @@ -12,7 +12,7 @@ #include #include -#include "cast/standalone_sender/ffmpeg_glue.h" +#include "cast/standalone_common/ffmpeg_glue.h" #include "cast/streaming/public/environment.h" #include "util/osp_logging.h" @@ -228,7 +228,8 @@ void SimulatedCapturer::ConsumeNextDecodedFrame() { next_task_.Schedule( [this, reference_time] { - DeliverDataToClient(*decoded_frame_, capture_begin_time_, now_(), + DeliverDataToClient(*decoded_frame_, capture_begin_time_, + capture_begin_time_ + std::chrono::milliseconds(10), reference_time); av_frame_unref(decoded_frame_.get()); ConsumeNextDecodedFrame(); diff --git a/cast/standalone_sender/simulated_capturer.h b/cast/standalone_sender/simulated_capturer.h index 03dfce282..5dcb0bcb2 100644 --- a/cast/standalone_sender/simulated_capturer.h +++ b/cast/standalone_sender/simulated_capturer.h @@ -11,7 +11,7 @@ #include #include -#include "cast/standalone_sender/ffmpeg_glue.h" +#include "cast/standalone_common/ffmpeg_glue.h" #include "platform/api/time.h" #include "util/alarm.h" diff --git a/cast/standalone_sender/streaming_av1_encoder.cc b/cast/standalone_sender/streaming_av1_encoder.cc index e7a02c764..66f9e5315 100644 --- a/cast/standalone_sender/streaming_av1_encoder.cc +++ b/cast/standalone_sender/streaming_av1_encoder.cc @@ -85,7 +85,7 @@ StreamingAv1Encoder::StreamingAv1Encoder(const Parameters& params, StreamingAv1Encoder::~StreamingAv1Encoder() { { - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); target_bitrate_ = 0; cv_.notify_one(); } @@ -93,8 +93,7 @@ StreamingAv1Encoder::~StreamingAv1Encoder() { } int StreamingAv1Encoder::GetTargetBitrate() const { - // Note: No need to lock the `mutex_` since this method should be called on - // the same thread as SetTargetBitrate(). + std::lock_guard lock(mutex_); return target_bitrate_; } @@ -103,7 +102,7 @@ void StreamingAv1Encoder::SetTargetBitrate(int new_bitrate) { // bitrate will not be zero. new_bitrate = std::max(new_bitrate, kBytesPerKilobyte); - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); // Only assign the new target bitrate if `target_bitrate_` has not yet been // used to signal the `encode_thread_` to end. if (target_bitrate_ > 0) { @@ -128,9 +127,9 @@ void StreamingAv1Encoder::EncodeAndSend( work_unit.rtp_timestamp = RtpTimeTicks(); } else { work_unit.rtp_timestamp = RtpTimeTicks::FromTimeSinceOrigin( - reference_time - start_time_, sender_->rtp_timebase()); + reference_time - start_time_, sender_->config().rtp_timebase); if (work_unit.rtp_timestamp <= last_enqueued_rtp_timestamp_) { - OSP_LOG_WARN << "VIDEO[" << sender_->ssrc() + OSP_LOG_WARN << "VIDEO[" << sender_->config().sender_ssrc << "] Dropping: RTP timestamp is not monotonically " "increasing from last frame."; return; @@ -138,7 +137,7 @@ void StreamingAv1Encoder::EncodeAndSend( } if (sender_->GetInFlightMediaDuration(work_unit.rtp_timestamp) > sender_->GetMaxInFlightMediaDuration()) { - OSP_LOG_WARN << "VIDEO[" << sender_->ssrc() + OSP_LOG_WARN << "VIDEO[" << sender_->config().sender_ssrc << "] Dropping: In-flight media duration would be too high."; return; } @@ -155,7 +154,7 @@ void StreamingAv1Encoder::EncodeAndSend( // a prediction for the next frame's duration. frame_duration = (work_unit.rtp_timestamp - last_enqueued_rtp_timestamp_) - .ToDuration(sender_->rtp_timebase()); + .ToDuration(sender_->config().rtp_timebase); } } work_unit.duration = @@ -168,7 +167,7 @@ void StreamingAv1Encoder::EncodeAndSend( work_unit.stats_callback = std::move(stats_callback); const bool force_key_frame = sender_->NeedsKeyFrame(); { - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); needs_key_frame_ |= force_key_frame; encode_queue_.push(std::move(work_unit)); cv_.notify_one(); @@ -389,7 +388,7 @@ void StreamingAv1Encoder::SendEncodedFrame(WorkUnitWithResults results) { if (sender_->EnqueueFrame(frame) != Sender::OK) { // Since the frame will not be sent, the encoder's frame dependency chain // has been broken. Force a key frame for the next frame. - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); needs_key_frame_ = true; } diff --git a/cast/standalone_sender/streaming_av1_encoder.h b/cast/standalone_sender/streaming_av1_encoder.h index 756bebf27..f8b68d64e 100644 --- a/cast/standalone_sender/streaming_av1_encoder.h +++ b/cast/standalone_sender/streaming_av1_encoder.h @@ -23,6 +23,7 @@ #include "cast/streaming/rtp_time.h" #include "platform/api/task_runner.h" #include "platform/api/time.h" +#include "util/thread_annotations.h" namespace openscreen { @@ -102,7 +103,7 @@ class StreamingAv1Encoder : public StreamingVideoEncoder { // The procedure for the `encode_thread_` that loops, processing work units // from the `encode_queue_` by calling Encode() until it's time to end the // thread. - void ProcessWorkUnitsUntilTimeToQuit(); + void ProcessWorkUnitsUntilTimeToQuit() OSP_NO_THREAD_SAFETY_ANALYSIS; // If the `encoder_` is live, attempt reconfiguration to allow it to encode // frames at a new frame size or target bitrate. If reconfiguration is not @@ -135,23 +136,22 @@ class StreamingAv1Encoder : public StreamingVideoEncoder { RtpTimeTicks last_enqueued_rtp_timestamp_; // Guards a few members shared by both the main and encode threads. - std::mutex mutex_; + mutable std::mutex mutex_; // Used by the encode thread to sleep until more work is available. - std::condition_variable cv_; // ABSL_GUARDED_BY(mutex_) + std::condition_variable cv_; // These encode parameters not passed in the WorkUnit struct because it is // desirable for them to be applied as soon as possible, with the very next // WorkUnit popped from the `encode_queue_` on the encode thread, and not to // wait until some later WorkUnit is processed. - bool needs_key_frame_ /* ABSL_GUARDED_BY(mutex_) */ = true; - int target_bitrate_ /* ABSL_GUARDED_BY(mutex_) */ = - 2 << 20; // Default: 2 Mbps. + bool needs_key_frame_ OSP_GUARDED_BY(mutex_) = true; + int target_bitrate_ OSP_GUARDED_BY(mutex_) = 2 << 20; // Default: 2 Mbps. // The queue of frame encodes. The size of this queue is implicitly bounded by // EncodeAndSend(), where it checks for the total in-flight media duration and // maybe drops a frame. - std::queue encode_queue_; // ABSL_GUARDED_BY(mutex_) + std::queue encode_queue_ OSP_GUARDED_BY(mutex_); // Current AV1 encoder configuration. Most of the fields are unchanging, and // are populated in the ctor; but thereafter, only the encode thread accesses diff --git a/cast/standalone_sender/streaming_opus_encoder.cc b/cast/standalone_sender/streaming_opus_encoder.cc index ceba58111..7e74df0a4 100644 --- a/cast/standalone_sender/streaming_opus_encoder.cc +++ b/cast/standalone_sender/streaming_opus_encoder.cc @@ -93,10 +93,6 @@ void StreamingOpusEncoder::EncodeAndSend(const float* interleaved_samples, ResolveTimestampsAndMaybeSkip(reference_time); - if (frame_.capture_begin_time == Clock::time_point::min()) { - frame_.capture_begin_time = capture_begin_time; - } - frame_.capture_end_time = capture_end_time; while (num_samples > 0) { const int samples_copied = FillInputBuffer(interleaved_samples, num_samples); @@ -112,7 +108,7 @@ void StreamingOpusEncoder::EncodeAndSend(const float* interleaved_samples, output_.get(), kOpusMaxPayloadSize); num_samples_queued_ = 0; if (packet_size_or_error < 0) { - OSP_LOG_FATAL << "AUDIO[" << sender_->ssrc() + OSP_LOG_FATAL << "AUDIO[" << sender_->config().sender_ssrc << "] Error code from opus_encode_float(): " << packet_size_or_error; return; @@ -124,16 +120,18 @@ void StreamingOpusEncoder::EncodeAndSend(const float* interleaved_samples, // audio frame anyway, to represent the passage of silence and to send other // stream metadata. frame_.data = ByteView(output_.get(), packet_size_or_error); + frame_.capture_begin_time = capture_begin_time; + frame_.capture_end_time = capture_end_time; last_sent_frame_reference_time_ = frame_.reference_time; switch (sender_->EnqueueFrame(frame_)) { case Sender::OK: break; case Sender::REACHED_ID_SPAN_LIMIT: - OSP_LOG_WARN << "AUDIO[" << sender_->ssrc() + OSP_LOG_WARN << "AUDIO[" << sender_->config().sender_ssrc << "] Dropping: FrameId span limit reached."; break; case Sender::MAX_DURATION_IN_FLIGHT: - OSP_LOG_INFO << "AUDIO[" << sender_->ssrc() + OSP_LOG_INFO << "AUDIO[" << sender_->config().sender_ssrc << "] Dropping: In-flight duration would be too high."; break; case Sender::PAYLOAD_TOO_LARGE: diff --git a/cast/standalone_sender/streaming_opus_encoder.h b/cast/standalone_sender/streaming_opus_encoder.h index 265fe25fe..b057ed0f6 100644 --- a/cast/standalone_sender/streaming_opus_encoder.h +++ b/cast/standalone_sender/streaming_opus_encoder.h @@ -28,7 +28,7 @@ class StreamingOpusEncoder { // audio samples up into chunks as determined by the given // `cast_frames_per_second`, and for EncodedFrame output to the given // `sender`. The sample rate of the audio is assumed to be the Sender's fixed - // `rtp_timebase()`. + // `config().rtp_timebase`. StreamingOpusEncoder(int num_channels, int cast_frames_per_second, std::unique_ptr sender); @@ -36,7 +36,7 @@ class StreamingOpusEncoder { ~StreamingOpusEncoder(); int num_channels() const { return num_channels_; } - int sample_rate() const { return sender_->rtp_timebase(); } + int sample_rate() const { return sender_->config().rtp_timebase; } int GetBitrate() const; diff --git a/cast/standalone_sender/streaming_video_encoder.h b/cast/standalone_sender/streaming_video_encoder.h index 00167e9d5..3248bebf0 100644 --- a/cast/standalone_sender/streaming_video_encoder.h +++ b/cast/standalone_sender/streaming_video_encoder.h @@ -90,7 +90,7 @@ class StreamingVideoEncoder { // For full details on how to use these stats in an end-to-end system, see: // https://www.chromium.org/developers/design-documents/ // auto-throttled-screen-capture-and-mirroring - // and https://source.chromium.org/chromium/chromium/src/+/master: + // and https://source.chromium.org/chromium/chromium/src/+/main: // media/cast/sender/performance_metrics_overlay.h struct Stats { // The Cast Streaming ID that was assigned to the frame. diff --git a/cast/standalone_sender/streaming_vpx_encoder.cc b/cast/standalone_sender/streaming_vpx_encoder.cc index ec6693c27..15d470b4a 100644 --- a/cast/standalone_sender/streaming_vpx_encoder.cc +++ b/cast/standalone_sender/streaming_vpx_encoder.cc @@ -96,7 +96,7 @@ StreamingVpxEncoder::StreamingVpxEncoder(const Parameters& params, StreamingVpxEncoder::~StreamingVpxEncoder() { { - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); target_bitrate_ = 0; cv_.notify_one(); } @@ -104,8 +104,7 @@ StreamingVpxEncoder::~StreamingVpxEncoder() { } int StreamingVpxEncoder::GetTargetBitrate() const { - // Note: No need to lock the `mutex_` since this method should be called on - // the same thread as SetTargetBitrate(). + std::lock_guard lock(mutex_); return target_bitrate_; } @@ -114,7 +113,7 @@ void StreamingVpxEncoder::SetTargetBitrate(int new_bitrate) { // bitrate will not be zero. new_bitrate = std::max(new_bitrate, kBytesPerKilobyte); - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); // Only assign the new target bitrate if `target_bitrate_` has not yet been // used to signal the `encode_thread_` to end. if (target_bitrate_ > 0) { @@ -139,9 +138,9 @@ void StreamingVpxEncoder::EncodeAndSend( work_unit.rtp_timestamp = RtpTimeTicks(); } else { work_unit.rtp_timestamp = RtpTimeTicks::FromTimeSinceOrigin( - reference_time - start_time_, sender_->rtp_timebase()); + reference_time - start_time_, sender_->config().rtp_timebase); if (work_unit.rtp_timestamp <= last_enqueued_rtp_timestamp_) { - OSP_LOG_WARN << "VIDEO[" << sender_->ssrc() + OSP_LOG_WARN << "VIDEO[" << sender_->config().sender_ssrc << "] Dropping: RTP timestamp is not monotonically " "increasing from last frame."; return; @@ -149,7 +148,7 @@ void StreamingVpxEncoder::EncodeAndSend( } if (sender_->GetInFlightMediaDuration(work_unit.rtp_timestamp) > sender_->GetMaxInFlightMediaDuration()) { - OSP_LOG_WARN << "VIDEO[" << sender_->ssrc() + OSP_LOG_WARN << "VIDEO[" << sender_->config().sender_ssrc << "] Dropping: In-flight media duration would be too high."; return; } @@ -166,7 +165,7 @@ void StreamingVpxEncoder::EncodeAndSend( // a prediction for the next frame's duration. frame_duration = (work_unit.rtp_timestamp - last_enqueued_rtp_timestamp_) - .ToDuration(sender_->rtp_timebase()); + .ToDuration(sender_->config().rtp_timebase); } } work_unit.duration = @@ -179,7 +178,7 @@ void StreamingVpxEncoder::EncodeAndSend( work_unit.stats_callback = std::move(stats_callback); const bool force_key_frame = sender_->NeedsKeyFrame(); { - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); needs_key_frame_ |= force_key_frame; encode_queue_.push(std::move(work_unit)); cv_.notify_one(); @@ -411,7 +410,7 @@ void StreamingVpxEncoder::SendEncodedFrame(WorkUnitWithResults results) { if (sender_->EnqueueFrame(frame) != Sender::OK) { // Since the frame will not be sent, the encoder's frame dependency chain // has been broken. Force a key frame for the next frame. - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); needs_key_frame_ = true; } diff --git a/cast/standalone_sender/streaming_vpx_encoder.h b/cast/standalone_sender/streaming_vpx_encoder.h index 5b2fb907c..5a00e4928 100644 --- a/cast/standalone_sender/streaming_vpx_encoder.h +++ b/cast/standalone_sender/streaming_vpx_encoder.h @@ -23,6 +23,7 @@ #include "cast/streaming/rtp_time.h" #include "platform/api/task_runner.h" #include "platform/api/time.h" +#include "util/thread_annotations.h" namespace openscreen { @@ -102,7 +103,7 @@ class StreamingVpxEncoder : public StreamingVideoEncoder { // The procedure for the `encode_thread_` that loops, processing work units // from the `encode_queue_` by calling Encode() until it's time to end the // thread. - void ProcessWorkUnitsUntilTimeToQuit(); + void ProcessWorkUnitsUntilTimeToQuit() OSP_NO_THREAD_SAFETY_ANALYSIS; // If the `encoder_` is live, attempt reconfiguration to allow it to encode // frames at a new frame size or target bitrate. If reconfiguration is not @@ -135,23 +136,22 @@ class StreamingVpxEncoder : public StreamingVideoEncoder { RtpTimeTicks last_enqueued_rtp_timestamp_; // Guards a few members shared by both the main and encode threads. - std::mutex mutex_; + mutable std::mutex mutex_; // Used by the encode thread to sleep until more work is available. - std::condition_variable cv_; // ABSL_GUARDED_BY(mutex_) + std::condition_variable cv_; // These encode parameters not passed in the WorkUnit struct because it is // desirable for them to be applied as soon as possible, with the very next // WorkUnit popped from the `encode_queue_` on the encode thread, and not to // wait until some later WorkUnit is processed. - bool needs_key_frame_ /* ABSL_GUARDED_BY(mutex_) */ = true; - int target_bitrate_ /* ABSL_GUARDED_BY(mutex_) */ = - 2 << 20; // Default: 2 Mbps. + bool needs_key_frame_ OSP_GUARDED_BY(mutex_) = true; + int target_bitrate_ OSP_GUARDED_BY(mutex_) = 2 << 20; // Default: 2 Mbps. // The queue of frame encodes. The size of this queue is implicitly bounded by // EncodeAndSend(), where it checks for the total in-flight media duration and // maybe drops a frame. - std::queue encode_queue_; // ABSL_GUARDED_BY(mutex_) + std::queue encode_queue_ OSP_GUARDED_BY(mutex_); // Current VP8 encoder configuration. Most of the fields are unchanging, and // are populated in the ctor; but thereafter, only the encode thread accesses diff --git a/cast/streaming/BUILD.gn b/cast/streaming/BUILD.gn index a3b1ab5a3..ec1923740 100644 --- a/cast/streaming/BUILD.gn +++ b/cast/streaming/BUILD.gn @@ -19,6 +19,11 @@ fuzzable_proto_library("remoting_proto") { sources = [ "remoting.proto" ] } +fuzzable_proto_library("input_proto") { + visibility += [ "*" ] + sources = [ "input.proto" ] +} + fuzzable_proto_library("sender_stats_proto") { visibility += [ "*" ] # Used by Android. sources = [ "sender_stats.proto" ] @@ -30,7 +35,6 @@ openscreen_source_set("streaming_configs") { visibility += [ "*" ] public = [ "capture_configs.h", # FIXME - "constants.h", "message_fields.h", # FIXME "public/constants.h", "resolution.h", # FIXME @@ -41,10 +45,7 @@ openscreen_source_set("streaming_configs") { "resolution.cc", ] - public_deps = [ - "../../third_party/abseil", - "../../third_party/jsoncpp", - ] + public_deps = [ "../../third_party/jsoncpp" ] deps = [ "../../platform:base", @@ -56,26 +57,19 @@ openscreen_source_set("common") { # Used by multiple targets in Chromium. visibility += [ "*" ] public = [ - "answer_messages.h", - "capture_recommendations.h", - "encoded_frame.h", - "environment.h", - "frame_id.h", - "offer_messages.h", "public/answer_messages.h", "public/capture_recommendations.h", "public/encoded_frame.h", "public/environment.h", "public/frame_id.h", "public/offer_messages.h", + "public/protobuf_messenger.h", "public/receiver_message.h", "public/rpc_messenger.h", + "public/session_config.h", "public/session_messenger.h", - "receiver_message.h", - "rpc_messenger.h", "rtp_time.h", # FIXME "sender_message.h", # FIXME: Remove Chromium #include and make private - "session_messenger.h", "ssrc.h", # FIXME: Remove Chromium #include and make private ] sources = [ @@ -84,6 +78,7 @@ openscreen_source_set("common") { "impl/expanded_value_base.h", "impl/frame_crypto.cc", "impl/frame_crypto.h", + "impl/message_constants.h", "impl/ntp_time.cc", "impl/ntp_time.h", "impl/packet_util.cc", @@ -94,12 +89,12 @@ openscreen_source_set("common") { "impl/rtcp_session.h", "impl/rtp_defines.cc", "impl/rtp_defines.h", - "impl/session_config.cc", - "impl/session_config.h", "impl/statistics_collector.cc", "impl/statistics_collector.h", - "impl/statistics_defines.cc", - "impl/statistics_defines.h", + "impl/statistics_common.cc", + "impl/statistics_common.h", + "impl/statistics_dispatcher.cc", + "impl/statistics_dispatcher.h", "public/answer_messages.cc", "public/capture_recommendations.cc", "public/encoded_frame.cc", @@ -108,6 +103,7 @@ openscreen_source_set("common") { "public/offer_messages.cc", "public/receiver_message.cc", "public/rpc_messenger.cc", + "public/session_config.cc", "public/session_messenger.cc", "rtp_time.cc", "sender_message.cc", @@ -115,9 +111,9 @@ openscreen_source_set("common") { ] public_deps = [ + ":input_proto", ":remoting_proto", ":streaming_configs", - "../../third_party/abseil", "../../third_party/boringssl", "../common:channel", "../common:public", @@ -149,9 +145,6 @@ openscreen_source_set("receiver") { "public/receiver.h", "public/receiver_constraints.h", "public/receiver_session.h", - "receiver.h", - "receiver_constraints.h", - "receiver_session.h", ] sources = [ "impl/compound_rtcp_builder.cc", @@ -160,8 +153,8 @@ openscreen_source_set("receiver") { "impl/frame_collector.h", "impl/packet_receive_stats_tracker.cc", "impl/packet_receive_stats_tracker.h", - "impl/receiver_base.cc", - "impl/receiver_base.h", + "impl/receiver_impl.cc", + "impl/receiver_impl.h", "impl/receiver_packet_router.cc", "impl/receiver_packet_router.h", "impl/rtp_packet_parser.cc", @@ -193,10 +186,7 @@ openscreen_source_set("sender") { "public/sender_session.h", "public/statistics.h", "remoting_capabilities.h", # FIXME: Only for Chromium tests. - "sender.h", "sender_packet_router.h", # FIXME: Only for Chromium tests. - "sender_session.h", - "statistics.h", ] sources = [ "impl/bandwidth_estimator.cc", @@ -207,6 +197,8 @@ openscreen_source_set("sender") { "impl/compound_rtcp_parser.cc", "impl/rtp_packetizer.cc", "impl/rtp_packetizer.h", + "impl/sender_impl.cc", + "impl/sender_impl.h", "impl/sender_report_builder.cc", "impl/sender_report_builder.h", "impl/statistics_analyzer.cc", @@ -259,6 +251,7 @@ openscreen_source_set("unittests") { "impl/answer_messages_unittest.cc", "impl/bandwidth_estimator_unittest.cc", "impl/capture_recommendations_unittest.cc", + "impl/clock_drift_smoother_unittest.cc", "impl/clock_offset_estimator_impl_unittest.cc", "impl/compound_rtcp_builder_unittest.cc", "impl/compound_rtcp_parser_unittest.cc", @@ -270,20 +263,23 @@ openscreen_source_set("unittests") { "impl/offer_messages_unittest.cc", "impl/packet_receive_stats_tracker_unittest.cc", "impl/packet_util_unittest.cc", + "impl/protobuf_messenger_unittest.cc", "impl/receiver_constraints_unittest.cc", + "impl/receiver_impl_unittest.cc", "impl/receiver_message_unittest.cc", "impl/receiver_session_unittest.cc", - "impl/receiver_unittest.cc", "impl/rpc_messenger_unittest.cc", "impl/rtcp_common_unittest.cc", "impl/rtp_packet_parser_unittest.cc", "impl/rtp_packetizer_unittest.cc", + "impl/sender_impl_unittest.cc", + "impl/sender_message_unittest.cc", "impl/sender_report_unittest.cc", "impl/sender_session_unittest.cc", - "impl/sender_unittest.cc", "impl/session_messenger_unittest.cc", "impl/statistics_analyzer_unittest.cc", "impl/statistics_collector_unittest.cc", + "impl/statistics_dispatcher_unittest.cc", "impl/statistics_unittest.cc", "message_fields_unittest.cc", "rtp_time_unittest.cc", @@ -296,6 +292,7 @@ openscreen_source_set("unittests") { ":sender", ":testing", "../../platform:test", + "../../testing/util", "../../third_party/googletest:gmock", "../../third_party/googletest:gtest", "../../util", @@ -310,10 +307,7 @@ openscreen_fuzzer_test("compound_rtcp_parser_fuzzer") { public = [] sources = [ "impl/compound_rtcp_parser_fuzzer.cc" ] - deps = [ - ":sender", - "../../third_party/abseil", - ] + deps = [ ":sender" ] seed_corpus = "compound_rtcp_parser_fuzzer_seeds" @@ -325,10 +319,7 @@ openscreen_fuzzer_test("rtp_packet_parser_fuzzer") { public = [] sources = [ "impl/rtp_packet_parser_fuzzer.cc" ] - deps = [ - ":receiver", - "../../third_party/abseil", - ] + deps = [ ":receiver" ] seed_corpus = "rtp_packet_parser_fuzzer_seeds" @@ -340,10 +331,7 @@ openscreen_fuzzer_test("sender_report_parser_fuzzer") { public = [] sources = [ "impl/sender_report_parser_fuzzer.cc" ] - deps = [ - ":receiver", - "../../third_party/abseil", - ] + deps = [ ":receiver" ] seed_corpus = "sender_report_parser_fuzzer_seeds" diff --git a/cast/streaming/DEPS b/cast/streaming/DEPS index de4027bf0..a6ec7b9e9 100644 --- a/cast/streaming/DEPS +++ b/cast/streaming/DEPS @@ -9,3 +9,9 @@ include_rules = [ '+openssl', '+json', ] + +specific_include_rules = { + 'protobuf_messenger.h': [ + '+google/protobuf/message_lite.h', + ], +} diff --git a/cast/streaming/README.md b/cast/streaming/README.md index bb50d04c7..8ac6217c6 100644 --- a/cast/streaming/README.md +++ b/cast/streaming/README.md @@ -3,74 +3,268 @@ This module contains an implementation of Cast Streaming, the real-time media streaming protocol between Cast senders and Cast receivers. -Included are two applications, `cast_sender` and `cast_receiver` that -demonstrate how to send and receive media using a Cast Streaming session. +[TOC] -## Prerequisites +## Directory Structure -To run the `cast_sender` and `cast_receiver`, you first need to [install -external libraries](external_libraries.md). +The `streaming` directory is organized as follows: -## Compilation -Putting this together with the [ -external libraries](external_libraries.md) configuration, a complete GN -configuration might look like: +- `public/`: Contains the public C++ header files that form the primary API for + the streaming library. -```python -is_debug=true -have_ffmpeg=true -have_libsdl2=true -have_libopus=true -have_libvpx=true -``` +- `impl/`: Contains the internal implementation details of the streaming + protocol. The components in this directory are not meant to be used directly + by most applications. -This can be done on the command line as: -```bash -$ mkdir -p out/Default -$ gn gen --args="is_debug=true have_ffmpeg=true have_libsdl2=true have_libopus=true have_libvpx=true" out/Default -$ autoninja -C out/Default cast_sender cast_receiver -``` +- `testing/`: Contains testing-related utilities and mocks. -## Developer certificate generation and use +- The root directory contains build files and protocol buffer definitions, as + well as some implementation and public specific files, although this is + discouraged in favor of using `public/` and `impl/` where possible. -To use the sender and receiver application together, a valid Cast certificate is -required. The easiest way to generate the certificate is to just run the -`cast_receiver` with `-g`, and both should be written out to files: +## API Overview -``` -$ ./out/Default/cast_receiver -g - [INFO:../../cast/receiver/channel/static_credentials.cc(161):T0] Generated new private key for session: ./generated_root_cast_receiver.key - [INFO:../../cast/receiver/channel/static_credentials.cc(169):T0] Generated new root certificate for session: ./generated_root_cast_receiver.crt -``` +This document provides a code-level overview of the Cast Streaming Sender and +Receiver APIs. For a higher-level view of the entire Cast ecosystem, please see +the main [architecture document](../../cast/docs/architecture.md). -## Running +The Cast Streaming library is designed around two primary classes: the `Sender` +and the `Receiver`. -In addition to the certificate, to start a session you need a valid network -address and path to a video for the sender to play. Note that you may need to -enable permissions for the cast receiver to bind to the network socket. +### Core Concepts: Sender and Receiver -```bash -$ ./out/Default/cast_receiver -d generated_root_cast_receiver.crt -p generated_root_cast_receiver.key lo0 -$ ./out/Default/cast_sender -d generated_root_cast_receiver.crt -r 127.0.0.1 ~/video-1080-mp4.mp4 -``` +- [**`cast::Sender`**](./sender.h): This class is used by the client application + that wishes to stream media _to_ a Cast device. Its main responsibility is to + take encoded media frames (e.g., from a video encoder), packetize them into + RTP packets, and send them to a `Receiver`. It also handles feedback from the + `Receiver`, such as acknowledgements and requests for retransmissions. -TODO(https://issuetracker.google.com/197659238): Fix discovery mode for `cast_sender`. +- [**`cast::Receiver`**](./receiver.h): This class runs on the Cast device that + is the target for the media stream. It receives RTP packets, reassembles them + into encoded media frames, and makes them available to a client application + (e.g., a media player) for decoding and rendering. -### Specifying the codec +A typical Cast session involves at least one `Sender` and `Receiver` pair. For +A/V streaming, there are usually two pairs: one for audio and one for video. -To determine which video codec to use, the `-c` or `--codec` flag should be -passed. Currently supported values include `vp8`, `vp9`, and `av1`: +### The Offer/Answer Negotiation Flow -```bash -$ ./out/Default/cast_sender -d generated_root_cast_receiver.crt -r 127.0.0.1 -c av1 ~/video-1080-mp4.mp4 -``` +Before streaming can begin, the `Sender` and `Receiver` must negotiate the +parameters of the stream. This is done using an "Offer/Answer" model, similar to +WebRTC. -### Mac +1. **The Sender Creates an `Offer`**: The sender application constructs an + `Offer` object. This object describes the media stream(s) the sender wishes + to send, including details like: + - Codecs (e.g., VP8, Opus) + - Resolutions and frame rates + - Bit rates + - Encryption parameters (`aes_key`, `aes_iv_mask`) -When running on Mac OS X, also pass the `-x` flag to `cast_receiver` to disable -DNS-SD/mDNS, since Open Screen does not currently integrate with Bonjour. + ```cpp + // Sender-side example + openscreen::cast::Offer offer; + offer.cast_mode = openscreen::cast::CastMode::kMirroring; -# End-to-end tests + openscreen::cast::AudioStream audio_stream; + audio_stream.stream.index = 0; + audio_stream.stream.type = openscreen::cast::Stream::Type::kAudioSource; + audio_stream.codec = openscreen::cast::AudioCodec::kOpus; + // ... and other audio parameters ... + offer.audio_streams.push_back(audio_stream); -The script [`standalone_e2e.py`](../standalone_e2e.py) exercises the Cast Sender -sender and receiver through an end-to-end test. + openscreen::cast::VideoStream video_stream; + video_stream.stream.index = 1; + // ... and other video parameters ... + offer.video_streams.push_back(video_stream); + + // The offer is then typically serialized to JSON and sent to the receiver + // over a separate control channel (e.g., a Cast V2 message). + ``` + +2. **The Receiver Responds with an `Answer`**: The receiver application receives + the `Offer`, parses it, and determines if it can accept the proposed + stream(s). It then constructs an `Answer` object, which specifies the + parameters it has agreed to. Key fields in the `Answer` include: + - `udp_port`: The port on which the `Receiver` will listen for RTP packets. + - `send_indexes`: A list of indexes corresponding to the streams in the + `Offer` that the receiver has chosen to accept. + - `ssrcs`: The synchronization source identifiers the receiver will use. + + The `Answer` is serialized and sent back to the sender application. + +This negotiation happens at the application level, before the `Sender` and +`Receiver` objects are instantiated. The result of this exchange is a +`SessionConfig` object that is used to initialize both sides of the stream. + +### Configuration and Instantiation + +Once the Offer/Answer exchange is complete, both the sender and receiver have +the necessary `SessionConfig`. They can then create the `Sender` and `Receiver` +objects. + +- **Sender Instantiation**: + + ```cpp + // Sender-side example + #include "cast/streaming/public/sender.h" + #include "cast/streaming/public/environment.h" + // ... + + // The SessionConfig is created from the Offer and Answer. + const auto session_config = + SessionConfig::Create(offer, answer); + + // An Environment provides platform dependencies like a clock and task runner. + openscreen::cast::Environment environment(...); + openscreen::cast::Sender sender(environment, + packet_router, // Manages network sockets + session_config, + rtp_payload_type); + ``` + +- **Receiver Instantiation**: + + ```cpp + // Receiver-side example + #include "cast/streaming/public/receiver.h" + #include "cast/streaming/public/environment.h" + // ... + + // The SessionConfig is created from the Offer and Answer. + const auto session_config = + SessionConfig::Create(offer, answer); + + openscreen::cast::Environment environment(...); + openscreen::cast::Receiver receiver(environment, + packet_router, // Manages network sockets + session_config); + ``` + +### The Client Pattern + +Both the `Sender` and `Receiver` are designed to be used with a delegate (or +client) pattern. The application provides an implementation of an interface to +receive asynchronous events and data. + +- **`Sender::Observer`**: The application implements the `Sender::Observer` + interface to handle events from the `Sender`. This is crucial for reacting to + network conditions and receiver status. + + ```cpp + class MySenderClient : public openscreen::cast::Sender::Observer { + public: + void OnPictureLost() override { + // The receiver has indicated it needs a key frame to continue. + // The application should request one from its encoder. + if (sender_->NeedsKeyFrame()) { + RequestNewKeyFrameFromEncoder(); + } + } + + void OnFrameCanceled(openscreen::cast::FrameId frame_id) override { + // The sender is done with this frame (it was either acknowledged or + // skipped). The application can now free any resources associated + // with it. + } + // ... + }; + ``` + +- **`Receiver::Consumer`**: The application implements the `Receiver::Consumer` + interface to be notified when new media frames are ready for consumption. + + ```cpp + class MyPlayer : public openscreen::cast::Receiver::Consumer { + public: + void OnFramesReady(size_t next_frame_buffer_size) override { + // The Receiver has one or more frames ready. + std::vector buffer(next_frame_buffer_size); + openscreen::cast::EncodedFrame frame = + receiver_->ConsumeNextFrame(std::move(buffer)); + + // Pass the frame data to the decoder. + decoder_->Decode(frame.data); + } + // ... + }; + ``` + +### Handling Media Frames + +- **Sending Frames**: The sender application gets encoded frames from its media + source (e.g., a hardware encoder) and passes them to the `Sender`. + + ```cpp + // Sender-side example + openscreen::cast::EncodedFrame frame; + frame.frame_id = sender.GetNextFrameId(); + frame.dependency = is_key_frame ? + openscreen::cast::EncodedFrame::Dependency::kKeyFrame : + openscreen::cast::EncodedFrame::Dependency::kDependent; + frame.rtp_timestamp = ...; + frame.data = ...; // Pointer to encoded frame data + + sender.EnqueueFrame(frame); + ``` + +- **Receiving Frames**: The `Receiver` collects incoming RTP packets and + reassembles them. When a full frame is ready, it notifies the application via + the `OnFramesReady()` callback on the `Receiver::Consumer`. The application + can then call `ConsumeNextFrame()` to retrieve the frame data for decoding and + playback. + +### Input Event API + +The Input Event API allows a `Receiver` to send input events (like mouse or +keyboard events) back to the `Sender`. This is a unidirectional flow from +receiver to sender. + +- [**`cast::InputProducer`**](./input_producer.h): Used by the receiver to send + events. It is available via the `ConfiguredReceivers` struct during session + negotiation. +- [**`cast::InputConsumer`**](./input_consumer.h): Used by the sender to receive + events. Applications set an `InputConsumer::Client` on the `SenderSession`. + +Support for input events is negotiated during the Offer/Answer exchange via the +`rtp_extensions` field ("input_events"). + +## Advanced Topics + +### A Note on Packet Routers + +The `Sender` and `Receiver` classes do not interact directly with the network. +Instead, they are connected to the network via Packet Routers: + +- [`SenderPacketRouter`](./impl/sender_packet_router.h): This class manages + packet transmission for one or more `Sender`s. It paces outbound packets in + bursts to optimize for network conditions (especially WiFi) and can be used to + implement congestion control. `Sender`s are registered with the router and + request transmission slots, rather than sending packets directly. + +- [`ReceiverPacketRouter`](./impl/receiver_packet_router.h): This router is + responsible for handling all incoming network traffic from a sender. It + dispatches packets to the appropriate `Receiver` (e.g., for an audio or video + stream) based on the SSRC of the packet. + +In a typical application, you will have one packet router on each side of the +connection, managing all the `Sender` or `Receiver` instances for that +connection. + +### RTP/RTCP Implementation + +The RTP/RTCP implementation in `libcast` is a **custom, from-scratch +implementation** tailored specifically for the Cast protocol. It does not use +external third-party libraries for RTP/RTCP and does not explicitly reference +specific RFCs like RFC 3550. While it follows the spirit of these standards, it +is optimized for Cast use cases. + +The core implementation of the RTP and RTCP protocols, including packet parsing, +frame construction, and feedback messaging, is located in the `impl` +sub-directory. The public API abstracts away most of these details, but +developers working on the core streaming logic may need to interact with these +components. + +For those looking to understand the implementation, the best places to start are +probably the [`RtpPacketizer`](./impl/rtp_packetizer.h) and the +[`CompoundRtcpBuilder`](./impl/compound_rtcp_builder.h) classes. diff --git a/cast/streaming/answer_messages.h b/cast/streaming/answer_messages.h deleted file mode 100644 index 6f45a2436..000000000 --- a/cast/streaming/answer_messages.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_ANSWER_MESSAGES_H_ -#define CAST_STREAMING_ANSWER_MESSAGES_H_ - -#include "cast/streaming/public/answer_messages.h" - -#endif // CAST_STREAMING_ANSWER_MESSAGES_H_ diff --git a/cast/streaming/capture_recommendations.h b/cast/streaming/capture_recommendations.h deleted file mode 100644 index cdb592376..000000000 --- a/cast/streaming/capture_recommendations.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_CAPTURE_RECOMMENDATIONS_H_ -#define CAST_STREAMING_CAPTURE_RECOMMENDATIONS_H_ - -#include "cast/streaming/public/capture_recommendations.h" - -#endif // CAST_STREAMING_CAPTURE_RECOMMENDATIONS_H_ diff --git a/cast/streaming/constants.h b/cast/streaming/constants.h deleted file mode 100644 index fd1518d16..000000000 --- a/cast/streaming/constants.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_CONSTANTS_H_ -#define CAST_STREAMING_CONSTANTS_H_ - -#include "cast/streaming/public/constants.h" - -#endif // CAST_STREAMING_CONSTANTS_H_ diff --git a/cast/streaming/encoded_frame.h b/cast/streaming/encoded_frame.h deleted file mode 100644 index 1582d049f..000000000 --- a/cast/streaming/encoded_frame.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_ENCODED_FRAME_H_ -#define CAST_STREAMING_ENCODED_FRAME_H_ - -#include "cast/streaming/public/encoded_frame.h" - -#endif // CAST_STREAMING_ENCODED_FRAME_H_ diff --git a/cast/streaming/environment.h b/cast/streaming/environment.h deleted file mode 100644 index f4e215de1..000000000 --- a/cast/streaming/environment.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_ENVIRONMENT_H_ -#define CAST_STREAMING_ENVIRONMENT_H_ - -#include "cast/streaming/public/environment.h" - -#endif // CAST_STREAMING_ENVIRONMENT_H_ diff --git a/cast/streaming/external_libraries.gni b/cast/streaming/external_libraries.gni index bb71376af..622748792 100644 --- a/cast/streaming/external_libraries.gni +++ b/cast/streaming/external_libraries.gni @@ -3,9 +3,11 @@ # found in the LICENSE file. # NOTE: Only add *_dirs declarations if the libraries have been installed at -# non-standard locations. See the [documentation](external_libraries.md) for -# more details. +# non-standard locations. See ../docs/external_libraries.md for more details. declare_args() { + # Common path for external libraries, e.g., Homebrew's /opt/homebrew/lib. + external_lib_dirs = [] + have_ffmpeg = false ffmpeg_libs = [ "avcodec", diff --git a/cast/streaming/external_libraries.md b/cast/streaming/external_libraries.md index 0aa8d935f..11ecb894b 100644 --- a/cast/streaming/external_libraries.md +++ b/cast/streaming/external_libraries.md @@ -28,10 +28,10 @@ On some versions of Debian, the following apt-get command will install all of the necessary external libraries for Open Screen: ```sh -sudo apt-get install libsdl2-2.0 libsdl2-dev libavcodec libavcodec-dev - libavformat libavformat-dev libavutil libavutil-dev - libswresample libswresample-dev libopus0 libopus-dev - libvpx5 libvpx-dev +sudo apt-get install libsdl2-2.0 libsdl2-dev libavcodec libavcodec-dev \ + libavformat libavformat-dev libavutil libavutil-dev \ + libswresample libswresample-dev libopus0 libopus-dev \ + libvpx5 libvpx-dev \ ``` Similarly, on some versions of Raspian, the following command will install the @@ -39,9 +39,9 @@ necessary external libraries, at least for the standalone receiver. Note that this command is based off of the packages linked in the [sysroot](../../build/config/sysroot.gni): ```sh -sudo apt-get install libavcodec58=7:4.1.4* libavcodec-dev=7:4.1.4* - libsdl2-2.0-0=2.0.9* libsdl2-dev=2.0.9* - libavformat-dev=7:4.1.4* +sudo apt-get install libavcodec58=7:4.1.4* libavcodec-dev=7:4.1.4* \ + libsdl2-2.0-0=2.0.9* libsdl2-dev=2.0.9* \ + libavformat-dev=7:4.1.4* \ ``` Note: release of these operating systems may require slightly different @@ -58,29 +58,42 @@ You can use [Homebrew](https://brew.sh/) to install the libraries needed to comp standalone sender and receiver applications. ```sh -brew install ffmpeg SDL2 +brew install ffmpeg sdl2 opus libvpx aom ``` To compile and link against these libraries, set the path arguments as follows -in your `gn.args`. Adjust for your current version of MacOS and the versions of -the brew formulae you installed. +in your `gn args`. +**Important**: Using Homebrew's top-level include directory +(`/opt/homebrew/include`) can cause header conflicts with the project's internal +BoringSSL. You must use the specific include paths for each library from +Homebrew's `Cellar` directory. + +Homebrew libraries are often built for the latest version of macOS. You may need +to set the `mac_deployment_target` to match the version of macOS you are running +to avoid linker errors. For example, on macOS Sonoma (version 14): + +You will need to replace the `` placeholders with the actual versions +installed on your system. You can find these in `/opt/homebrew/Cellar/`. ```python -mac_deployment_target=14 -mac_min_system_version=14 +mac_deployment_target="14.0" + have_ffmpeg=true -ffmpeg_lib_dirs=["/opt/homebrew/lib"] -ffmpeg_include_dirs=["/opt/homebrew/Cellar/ffmpeg/7.0.1/include"] have_libsdl2=true -libsdl2_lib_dirs=["/opt/homebrew/lib"] -libsdl2_include_dirs=["/opt/homebrew/Cellar/sdl2/2.30.5/include"] have_libopus=true -libopus_lib_dirs=["/opt/homebrew/lib"] -libopus_include_dirs=["/opt/homebrew/Cellar/opus/1.5.2/include"] have_libvpx=true -libvpx_lib_dirs=["/opt/homebrew/lib"] -libvpx_include_dirs=["/opt/homebrew/Cellar/libvpx/1.13.1/include"] +have_libaom=true + +# Homebrew on Apple Silicon installs to /opt/homebrew. +# On Intel macs, it's /usr/local. +external_lib_dirs=["/opt/homebrew/lib"] + +ffmpeg_include_dirs=["/opt/homebrew/Cellar/ffmpeg//include"] +libsdl2_include_dirs=["/opt/homebrew/Cellar/sdl2//include"] +libopus_include_dirs=["/opt/homebrew/Cellar/opus//include"] +libvpx_include_dirs=["/opt/homebrew/Cellar/libvpx//include"] +libaom_include_dirs=["/opt/homebrew/Cellar/aom//include"] ``` ## libaom diff --git a/cast/streaming/frame_id.h b/cast/streaming/frame_id.h deleted file mode 100644 index 2c4949061..000000000 --- a/cast/streaming/frame_id.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_FRAME_ID_H_ -#define CAST_STREAMING_FRAME_ID_H_ - -#include "cast/streaming/public/frame_id.h" - -#endif // CAST_STREAMING_FRAME_ID_H_ diff --git a/cast/streaming/impl/answer_messages_unittest.cc b/cast/streaming/impl/answer_messages_unittest.cc index 5fb81f5bd..9010ad689 100644 --- a/cast/streaming/impl/answer_messages_unittest.cc +++ b/cast/streaming/impl/answer_messages_unittest.cc @@ -12,6 +12,7 @@ #include "gtest/gtest.h" #include "util/chrono_helpers.h" #include "util/json/json_serialization.h" +#include "util/no_destructor.h" namespace openscreen::cast { @@ -62,42 +63,47 @@ constexpr char kValidAnswerJson[] = R"({ "scaling": "sender" }, "receiverRtcpEventLog": [0, 1], - "receiverRtcpDscp": [234, 567], - "rtpExtensions": ["adaptive_playout_delay"] + "receiverRtcpDscp": [1, 3], + "rtpExtensions": [["input_events", "adaptive_playout_delay"], + ["adaptive_playout_delay"]] })"; -const Answer kValidAnswer{ - 1234, // udp_port - std::vector{1, 2, 3}, // send_indexes - std::vector{123, 456}, // ssrcs - std::optional(Constraints{ - AudioConstraints{ - 96000, // max_sample_rate - 7, // max_channels - 32000, // min_bit_rate - 96000, // max_bit_rate - milliseconds(2000) // max_delay - }, // audio - VideoConstraints{ - 40000.0, // max_pixels_per_second - std::optional( - Dimensions{320, 480, SimpleFraction{15000, 101}}), - Dimensions{1920, 1080, SimpleFraction{288, 2}}, - 300000, // min_bit_rate - 144000000, // max_bit_rate - milliseconds(3000) // max_delay - } // video - }), // constraints - std::optional(DisplayDescription{ - std::optional(Dimensions{640, 480, SimpleFraction{30, 1}}), - std::optional(AspectRatio{16, 9}), // aspect_ratio - std::optional( - AspectRatioConstraint::kFixed), // scaling - }), - std::vector{7, 8, 9}, // receiver_rtcp_event_log - std::vector{11, 12, 13}, // receiver_rtcp_dscp - std::vector{"foo", "bar"} // rtp_extensions -}; +const Answer& GetValidAnswer() { + static const NoDestructor kValidAnswer( + 1234, // udp_port + std::vector{1, 3}, // send_indexes + std::vector{123, 456}, // ssrcs + std::optional(Constraints{ + AudioConstraints{ + 96000, // max_sample_rate + 7, // max_channels + 32000, // min_bit_rate + 96000, // max_bit_rate + milliseconds(2000) // max_delay + }, // audio + VideoConstraints{ + 40000.0, // max_pixels_per_second + std::optional( + Dimensions{320, 480, SimpleFraction{15000, 101}}), + Dimensions{1920, 1080, SimpleFraction{288, 2}}, + 300000, // min_bit_rate + 144000000, // max_bit_rate + milliseconds(3000) // max_delay + } // video + }), // constraints + std::optional(DisplayDescription{ + std::optional( + Dimensions{640, 480, SimpleFraction{30, 1}}), + std::optional(AspectRatio{16, 9}), // aspect_ratio + std::optional( + AspectRatioConstraint::kFixed), // scaling + }), + std::vector{7, 8, 9}, // receiver_rtcp_event_log + std::vector{1, 3}, // receiver_rtcp_dscp + std::vector>{{"foo"}, {"bar"}} // rtp_extensions + ); + return *kValidAnswer; +} constexpr int kValidMaxPixelsPerSecond = 1920 * 1080 * 30; constexpr Dimensions kValidDimensions{1920, 1080, SimpleFraction{60, 1}}; @@ -146,8 +152,10 @@ void ExpectEqualsValidAnswerJson(const Answer& answer) { display.aspect_ratio_constraint.value()); EXPECT_THAT(answer.receiver_rtcp_event_log, ElementsAre(0, 1)); - EXPECT_THAT(answer.receiver_rtcp_dscp, ElementsAre(234, 567)); - EXPECT_THAT(answer.rtp_extensions, ElementsAre("adaptive_playout_delay")); + EXPECT_THAT(answer.receiver_rtcp_dscp, ElementsAre(1, 3)); + EXPECT_THAT(answer.rtp_extensions, + ElementsAre(ElementsAre("input_events", "adaptive_playout_delay"), + ElementsAre("adaptive_playout_delay"))); } void ExpectFailureOnParse(std::string_view raw_json) { @@ -155,9 +163,8 @@ void ExpectFailureOnParse(std::string_view raw_json) { // Must be a valid JSON object, but not a valid answer. ASSERT_TRUE(root.is_value()); - Answer answer; - EXPECT_FALSE(Answer::TryParse(std::move(root.value()), &answer)); - EXPECT_FALSE(answer.IsValid()); + const auto answer_or_error = Answer::TryParse(std::move(root.value())); + EXPECT_TRUE(answer_or_error.is_error()); } // Functions that use ASSERT_* must return void, so we use an out parameter @@ -167,26 +174,25 @@ void ExpectSuccessOnParse(std::string_view raw_json, Answer* out = nullptr) { // Must be a valid JSON object, but not a valid answer. ASSERT_TRUE(root.is_value()); - Answer answer; - ASSERT_TRUE(Answer::TryParse(std::move(root.value()), &answer)); - EXPECT_TRUE(answer.IsValid()); + const auto answer_or_error = Answer::TryParse(std::move(root.value())); + ASSERT_TRUE(answer_or_error.is_value()); + EXPECT_TRUE(answer_or_error.value().IsValid()); if (out) { - *out = std::move(answer); + *out = std::move(answer_or_error.value()); } } } // anonymous namespace TEST(AnswerMessagesTest, ProperlyPopulatedAnswerSerializesProperly) { - ASSERT_TRUE(kValidAnswer.IsValid()); - Json::Value root = kValidAnswer.ToJson(); + ASSERT_TRUE(GetValidAnswer().IsValid()); + Json::Value root = GetValidAnswer().ToJson(); EXPECT_EQ(root["udpPort"], 1234); Json::Value sendIndexes = std::move(root["sendIndexes"]); EXPECT_EQ(sendIndexes.type(), Json::ValueType::arrayValue); EXPECT_EQ(sendIndexes[0], 1); - EXPECT_EQ(sendIndexes[1], 2); - EXPECT_EQ(sendIndexes[2], 3); + EXPECT_EQ(sendIndexes[1], 3); Json::Value ssrcs = std::move(root["ssrcs"]); EXPECT_EQ(ssrcs.type(), Json::ValueType::arrayValue); @@ -240,30 +246,31 @@ TEST(AnswerMessagesTest, ProperlyPopulatedAnswerSerializesProperly) { Json::Value receiver_rtcp_dscp = std::move(root["receiverRtcpDscp"]); EXPECT_EQ(receiver_rtcp_dscp.type(), Json::ValueType::arrayValue); - EXPECT_EQ(receiver_rtcp_dscp[0], 11); - EXPECT_EQ(receiver_rtcp_dscp[1], 12); - EXPECT_EQ(receiver_rtcp_dscp[2], 13); + EXPECT_EQ(receiver_rtcp_dscp[0], 1); + EXPECT_EQ(receiver_rtcp_dscp[1], 3); Json::Value rtp_extensions = std::move(root["rtpExtensions"]); EXPECT_EQ(rtp_extensions.type(), Json::ValueType::arrayValue); - EXPECT_EQ(rtp_extensions[0], "foo"); - EXPECT_EQ(rtp_extensions[1], "bar"); + EXPECT_EQ(rtp_extensions[0].type(), Json::ValueType::arrayValue); + EXPECT_EQ(rtp_extensions[0][0], "foo"); + EXPECT_EQ(rtp_extensions[1].type(), Json::ValueType::arrayValue); + EXPECT_EQ(rtp_extensions[1][0], "bar"); } TEST(AnswerMessagesTest, EmptyArraysOmitted) { - Answer missing_event_log = kValidAnswer; + Answer missing_event_log = GetValidAnswer(); missing_event_log.receiver_rtcp_event_log.clear(); ASSERT_TRUE(missing_event_log.IsValid()); Json::Value root = missing_event_log.ToJson(); EXPECT_FALSE(root["receiverRtcpEventLog"]); - Answer missing_rtcp_dscp = kValidAnswer; + Answer missing_rtcp_dscp = GetValidAnswer(); missing_rtcp_dscp.receiver_rtcp_dscp.clear(); ASSERT_TRUE(missing_rtcp_dscp.IsValid()); root = missing_rtcp_dscp.ToJson(); EXPECT_FALSE(root["receiverRtcpDscp"]); - Answer missing_extensions = kValidAnswer; + Answer missing_extensions = GetValidAnswer(); missing_extensions.rtp_extensions.clear(); ASSERT_TRUE(missing_extensions.IsValid()); root = missing_extensions.ToJson(); @@ -271,32 +278,32 @@ TEST(AnswerMessagesTest, EmptyArraysOmitted) { } TEST(AnswerMessagesTest, InvalidDimensionsCauseInvalid) { - Answer invalid_dimensions = kValidAnswer; + Answer invalid_dimensions = GetValidAnswer(); invalid_dimensions.display->dimensions->width = -1; EXPECT_FALSE(invalid_dimensions.IsValid()); } TEST(AnswerMessagesTest, InvalidAudioConstraintsCauseError) { - Answer invalid_audio = kValidAnswer; + Answer invalid_audio = GetValidAnswer(); invalid_audio.constraints->audio.max_bit_rate = invalid_audio.constraints->audio.min_bit_rate - 1; EXPECT_FALSE(invalid_audio.IsValid()); } TEST(AnswerMessagesTest, InvalidVideoConstraintsCauseError) { - Answer invalid_video = kValidAnswer; + Answer invalid_video = GetValidAnswer(); invalid_video.constraints->video.max_pixels_per_second = -1.0; EXPECT_FALSE(invalid_video.IsValid()); } TEST(AnswerMessagesTest, InvalidDisplayDescriptionsCauseError) { - Answer invalid_display = kValidAnswer; + Answer invalid_display = GetValidAnswer(); invalid_display.display->aspect_ratio = {0, 0}; EXPECT_FALSE(invalid_display.IsValid()); } TEST(AnswerMessagesTest, InvalidUdpPortsCauseError) { - Answer invalid_port = kValidAnswer; + Answer invalid_port = GetValidAnswer(); invalid_port.udp_port = 65536; EXPECT_FALSE(invalid_port.IsValid()); } @@ -322,6 +329,17 @@ TEST(AnswerMessagesTest, ErrorOnEmptyAnswer) { ExpectFailureOnParse("{}"); } +TEST(AnswerMessagesTest, ErrorOnNonObjectAnswer) { + Json::Value array_val(Json::arrayValue); + EXPECT_TRUE(Answer::TryParse(array_val).is_error()); + + Json::Value string_val("string"); + EXPECT_TRUE(Answer::TryParse(string_val).is_error()); + + Json::Value int_val(42); + EXPECT_TRUE(Answer::TryParse(int_val).is_error()); +} + TEST(AnswerMessagesTest, ErrorOnMissingUdpPort) { ExpectFailureOnParse(R"({ "sendIndexes": [1, 3], @@ -509,22 +527,22 @@ TEST(AnswerMessagesTest, AspectRatioTryParse) { const Json::Value kZeroWidth = "0:9"; const Json::Value kZeroHeight = "16:0"; - AspectRatio out; - EXPECT_TRUE(AspectRatio::TryParse(kValid, &out)); - EXPECT_EQ(out.width, 16); - EXPECT_EQ(out.height, 9); - EXPECT_FALSE(AspectRatio::TryParse(kWrongDelimiter, &out)); - EXPECT_FALSE(AspectRatio::TryParse(kTooManyFields, &out)); - EXPECT_FALSE(AspectRatio::TryParse(kTooFewFields, &out)); - EXPECT_FALSE(AspectRatio::TryParse(kWrongDelimiter, &out)); - EXPECT_FALSE(AspectRatio::TryParse(kNoDelimiter, &out)); - EXPECT_FALSE(AspectRatio::TryParse(kNegativeWidth, &out)); - EXPECT_FALSE(AspectRatio::TryParse(kNegativeHeight, &out)); - EXPECT_FALSE(AspectRatio::TryParse(kNegativeBoth, &out)); - EXPECT_FALSE(AspectRatio::TryParse(kNonNumberWidth, &out)); - EXPECT_FALSE(AspectRatio::TryParse(kNonNumberHeight, &out)); - EXPECT_FALSE(AspectRatio::TryParse(kZeroWidth, &out)); - EXPECT_FALSE(AspectRatio::TryParse(kZeroHeight, &out)); + const auto out = AspectRatio::TryParse(kValid); + ASSERT_TRUE(out.is_value()); + EXPECT_EQ(out.value().width, 16); + EXPECT_EQ(out.value().height, 9); + EXPECT_TRUE(AspectRatio::TryParse(kWrongDelimiter).is_error()); + EXPECT_TRUE(AspectRatio::TryParse(kTooManyFields).is_error()); + EXPECT_TRUE(AspectRatio::TryParse(kTooFewFields).is_error()); + EXPECT_TRUE(AspectRatio::TryParse(kWrongDelimiter).is_error()); + EXPECT_TRUE(AspectRatio::TryParse(kNoDelimiter).is_error()); + EXPECT_TRUE(AspectRatio::TryParse(kNegativeWidth).is_error()); + EXPECT_TRUE(AspectRatio::TryParse(kNegativeHeight).is_error()); + EXPECT_TRUE(AspectRatio::TryParse(kNegativeBoth).is_error()); + EXPECT_TRUE(AspectRatio::TryParse(kNonNumberWidth).is_error()); + EXPECT_TRUE(AspectRatio::TryParse(kNonNumberHeight).is_error()); + EXPECT_TRUE(AspectRatio::TryParse(kZeroWidth).is_error()); + EXPECT_TRUE(AspectRatio::TryParse(kZeroHeight).is_error()); } TEST(AnswerMessagesTest, DisplayDescriptionTryParse) { @@ -552,25 +570,29 @@ TEST(AnswerMessagesTest, DisplayDescriptionTryParse) { aspect_ratio_and_constraint["scaling"] = "sender"; aspect_ratio_and_constraint["aspectRatio"] = "4:3"; - DisplayDescription out; - ASSERT_TRUE(DisplayDescription::TryParse(valid_scaling, &out)); - ASSERT_TRUE(out.aspect_ratio_constraint.has_value()); - EXPECT_EQ(out.aspect_ratio_constraint.value(), + const auto out = DisplayDescription::TryParse(valid_scaling); + ASSERT_TRUE(out.is_value()); + ASSERT_TRUE(out.value().aspect_ratio_constraint.has_value()); + EXPECT_EQ(out.value().aspect_ratio_constraint.value(), AspectRatioConstraint::kVariable); - EXPECT_FALSE(DisplayDescription::TryParse(invalid_scaling, &out)); - EXPECT_TRUE(DisplayDescription::TryParse(invalid_scaling_valid_ratio, &out)); + EXPECT_TRUE(DisplayDescription::TryParse(invalid_scaling).is_error()); + EXPECT_TRUE( + DisplayDescription::TryParse(invalid_scaling_valid_ratio).is_value()); - ASSERT_TRUE(DisplayDescription::TryParse(valid_dimensions, &out)); - ASSERT_TRUE(out.dimensions.has_value()); - EXPECT_EQ(1920, out.dimensions->width); - EXPECT_EQ(1080, out.dimensions->height); - EXPECT_EQ((SimpleFraction{30, 1}), out.dimensions->frame_rate); + const auto out2 = DisplayDescription::TryParse(valid_dimensions); + ASSERT_TRUE(out2.is_value()); + ASSERT_TRUE(out2.value().dimensions.has_value()); + EXPECT_EQ(1920, out2.value().dimensions->width); + EXPECT_EQ(1080, out2.value().dimensions->height); + EXPECT_EQ((SimpleFraction{30, 1}), out2.value().dimensions->frame_rate); - EXPECT_FALSE(DisplayDescription::TryParse(invalid_dimensions, &out)); + EXPECT_TRUE(DisplayDescription::TryParse(invalid_dimensions).is_error()); - ASSERT_TRUE(DisplayDescription::TryParse(aspect_ratio_and_constraint, &out)); - EXPECT_EQ(AspectRatioConstraint::kFixed, out.aspect_ratio_constraint.value()); + const auto out3 = DisplayDescription::TryParse(aspect_ratio_and_constraint); + ASSERT_TRUE(out3.is_value()); + EXPECT_EQ(AspectRatioConstraint::kFixed, + out3.value().aspect_ratio_constraint.value()); } TEST(AnswerMessagesTest, DisplayDescriptionIsValid) { diff --git a/cast/streaming/impl/clock_drift_smoother.cc b/cast/streaming/impl/clock_drift_smoother.cc index 55eba4114..90cd10cd9 100644 --- a/cast/streaming/impl/clock_drift_smoother.cc +++ b/cast/streaming/impl/clock_drift_smoother.cc @@ -27,8 +27,10 @@ ClockDriftSmoother::ClockDriftSmoother(Clock::duration time_constant) ClockDriftSmoother::~ClockDriftSmoother() = default; -Clock::duration ClockDriftSmoother::Current() const { - OSP_CHECK_NE(last_update_time_, kNullTime); +std::optional ClockDriftSmoother::Current() const { + if (last_update_time_ == kNullTime) { + return std::nullopt; + } return Clock::duration( rounded_saturate_cast(estimated_tick_offset_)); } @@ -45,32 +47,30 @@ void ClockDriftSmoother::Update(Clock::time_point now, OSP_CHECK_NE(now, kNullTime); if (last_update_time_ == kNullTime) { Reset(now, measured_offset); - } else if (now < last_update_time_) { + return; + } + + if (now < last_update_time_) { // `now` is not monotonically non-decreasing. OSP_NOTREACHED(); - } else { - const double elapsed_ticks = - static_cast((now - last_update_time_).count()); - last_update_time_ = now; - // Compute a weighted-average between the last estimate and - // `measured_offset`. The more time that has elasped since the last call to - // Update(), the more-heavily `measured_offset` will be weighed. - const double weight = - elapsed_ticks / (elapsed_ticks + time_constant_.count()); - estimated_tick_offset_ = - weight * static_cast(measured_offset.count()) + - (1.0 - weight) * estimated_tick_offset_; - - // If after calculation the current offset is lower than the weighted - // average, we can simply use it and eliminate some of the error due to - // transmission time. - if (measured_offset < Current()) { - Reset(now, measured_offset); - } - - OSP_VLOG << "Local clock is ahead of the remote clock by: measured = " - << measured_offset << ", " << "filtered = " << Current() << "."; } + + const double elapsed_ticks = + static_cast((now - last_update_time_).count()); + last_update_time_ = now; + + // This is a standard exponential moving average (EMA) filter. + // The alpha value is calculated such that the filter has the desired time + // constant. + const double alpha = 1.0 - std::exp(-elapsed_ticks / time_constant_.count()); + estimated_tick_offset_ = + alpha * static_cast(measured_offset.count()) + + (1.0 - alpha) * estimated_tick_offset_; + + const auto current = Current(); + OSP_VLOG << "Local clock is ahead of the remote clock by: measured = " + << measured_offset << ", " + << "filtered = " << (current ? ToString(*current) : "null") << "."; } // static diff --git a/cast/streaming/impl/clock_drift_smoother.h b/cast/streaming/impl/clock_drift_smoother.h index 31339cf62..4ea8bd3e1 100644 --- a/cast/streaming/impl/clock_drift_smoother.h +++ b/cast/streaming/impl/clock_drift_smoother.h @@ -6,6 +6,7 @@ #define CAST_STREAMING_IMPL_CLOCK_DRIFT_SMOOTHER_H_ #include +#include #include "platform/api/time.h" @@ -23,8 +24,9 @@ class ClockDriftSmoother { explicit ClockDriftSmoother(Clock::duration time_constant); ~ClockDriftSmoother(); - // Returns the current offset. - Clock::duration Current() const; + // Returns the current offset. Will be std::nullopt if no values have been + // set yet (via Reset() or Update()). + std::optional Current() const; // Discard all history and reset to exactly `offset`, measured `now`. void Reset(Clock::time_point now, Clock::duration offset); diff --git a/cast/streaming/impl/clock_drift_smoother_unittest.cc b/cast/streaming/impl/clock_drift_smoother_unittest.cc new file mode 100644 index 000000000..934f6a552 --- /dev/null +++ b/cast/streaming/impl/clock_drift_smoother_unittest.cc @@ -0,0 +1,168 @@ +// Copyright 2025 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cast/streaming/impl/clock_drift_smoother.h" + +#include + +#include +#include + +#include "platform/base/trivial_clock_traits.h" +#include "testing/util/chrono_test_helpers.h" + +namespace openscreen::cast { + +TEST(ClockDriftSmootherTest, InitializesToNullopt) { + ClockDriftSmoother smoother(seconds(1)); + EXPECT_EQ(smoother.Current(), std::nullopt); +} + +TEST(ClockDriftSmootherTest, ResetSetsOffset) { + ClockDriftSmoother smoother(seconds(1)); + const Clock::time_point now = Clock::now(); + const Clock::duration offset = milliseconds(100); + smoother.Reset(now, offset); + ASSERT_TRUE(smoother.Current().has_value()); + EXPECT_EQ(smoother.Current().value(), offset); +} + +TEST(ClockDriftSmootherTest, BasicSmoothing) { + ClockDriftSmoother smoother(seconds(1)); + Clock::time_point now = Clock::now(); + smoother.Reset(now, milliseconds(100)); + + now += seconds(1); + smoother.Update(now, milliseconds(200)); + + // After 1 time constant, the value should be a weighted average. + // alpha = 1 - exp(-1/1) = 1 - exp(-1) = 0.632 + // new_value = 0.632 * 200 + (1 - 0.632) * 100 = 126.4 + 36.8 = 163.2 + constexpr auto kExpectedOffset = milliseconds(163); + ASSERT_TRUE(smoother.Current().has_value()); + ExpectDurationNear(smoother.Current().value(), kExpectedOffset, + milliseconds(1)); +} + +TEST(ClockDriftSmootherTest, TimeProgression) { + ClockDriftSmoother smoother(seconds(1)); + Clock::time_point now = Clock::now(); + smoother.Reset(now, milliseconds(100)); + + now += milliseconds(100); + smoother.Update(now, milliseconds(1000)); + ASSERT_TRUE(smoother.Current().has_value()); + const Clock::duration first_update = smoother.Current().value(); + + now += seconds(2); + smoother.Update(now, milliseconds(1000)); + ASSERT_TRUE(smoother.Current().has_value()); + const Clock::duration second_update = smoother.Current().value(); + + // The second update should be closer to the target because more time has + // passed. + EXPECT_GT(second_update, first_update); +} + +TEST(ClockDriftSmootherTest, HandlesZeroOffset) { + ClockDriftSmoother smoother(seconds(1)); + Clock::time_point now = Clock::now(); + smoother.Reset(now, milliseconds(100)); + + now += seconds(1); + smoother.Update(now, Clock::duration::zero()); + ASSERT_TRUE(smoother.Current().has_value()); + EXPECT_LT(smoother.Current().value(), milliseconds(100)); +} + +TEST(ClockDriftSmootherTest, HandlesNegativeOffset) { + ClockDriftSmoother smoother(seconds(1)); + Clock::time_point now = Clock::now(); + smoother.Reset(now, milliseconds(100)); + + now += seconds(1); + smoother.Update(now, milliseconds(-100)); + ASSERT_TRUE(smoother.Current().has_value()); + EXPECT_LT(smoother.Current().value(), milliseconds(100)); +} + +TEST(ClockDriftSmootherTest, StabilityWithJitter) { + ClockDriftSmoother smoother(seconds(5)); + Clock::time_point now = Clock::now(); + smoother.Reset(now, milliseconds(100)); + + for (int i = 0; i < 100; ++i) { + now += milliseconds(100); + const auto offset = (i % 2 == 0) ? milliseconds(105) : milliseconds(95); + smoother.Update(now, offset); + } + + // After many updates, the smoother should converge to the average, despite + // the jitter. + ASSERT_TRUE(smoother.Current().has_value()); + ExpectDurationNear(smoother.Current().value(), milliseconds(100), + milliseconds(5)); +} + +TEST(ClockDriftSmootherTest, ConvergenceAfterSuddenJump) { + ClockDriftSmoother smoother(seconds(1)); + Clock::time_point now = Clock::now(); + smoother.Reset(now, milliseconds(100)); + + now += seconds(1); + smoother.Update(now, milliseconds(1000)); + + // After a jump, the value should be closer to the new value. + ASSERT_TRUE(smoother.Current().has_value()); + EXPECT_GT(smoother.Current().value(), milliseconds(100)); + EXPECT_LT(smoother.Current().value(), milliseconds(1000)); +} + +TEST(ClockDriftSmootherTest, UpdateWithZeroElapsedTime) { + ClockDriftSmoother smoother(seconds(1)); + Clock::time_point now = Clock::now(); + smoother.Reset(now, milliseconds(100)); + ASSERT_TRUE(smoother.Current().has_value()); + const Clock::duration initial_value = smoother.Current().value(); + + smoother.Update(now, milliseconds(1000)); + + // With zero elapsed time, the value should not change. + ASSERT_TRUE(smoother.Current().has_value()); + EXPECT_EQ(smoother.Current().value(), initial_value); +} + +TEST(ClockDriftSmootherTest, HeavyWeightingAfterLongGap) { + ClockDriftSmoother smoother(seconds(1)); + Clock::time_point now = Clock::now(); + smoother.Reset(now, milliseconds(100)); + + now += seconds(100); + smoother.Update(now, milliseconds(1000)); + + // After a long gap, the new value should be very close to the new + // measurement. + ASSERT_TRUE(smoother.Current().has_value()); + ExpectDurationNear(smoother.Current().value(), milliseconds(1000), + milliseconds(1)); +} + +TEST(ClockDriftSmootherTest, Responsiveness) { + ClockDriftSmoother smoother(seconds(1)); + Clock::time_point now = Clock::now(); + smoother.Reset(now, milliseconds(100)); + + now += milliseconds(500); + smoother.Update(now, milliseconds(200)); + + // After 500ms, the value should be a weighted average. + // alpha = 1 - exp(-500/1000) = 1 - exp(-0.5) = 0.393 + // new_value = 0.393 * 200 + (1 - 0.393) * 100 = 78.6 + 60.7 = 139.3 + constexpr auto kExpectedOffset = milliseconds(139); + ASSERT_TRUE(smoother.Current().has_value()); + ExpectDurationNear(smoother.Current().value(), kExpectedOffset, + milliseconds(1)); +} + +} // namespace openscreen::cast diff --git a/cast/streaming/impl/clock_offset_estimator.h b/cast/streaming/impl/clock_offset_estimator.h index 4936869e6..be51abf3b 100644 --- a/cast/streaming/impl/clock_offset_estimator.h +++ b/cast/streaming/impl/clock_offset_estimator.h @@ -8,7 +8,7 @@ #include #include -#include "cast/streaming/impl/statistics_defines.h" +#include "cast/streaming/impl/statistics_common.h" #include "cast/streaming/public/statistics.h" #include "platform/base/trivial_clock_traits.h" @@ -27,8 +27,43 @@ class ClockOffsetEstimator { virtual void OnFrameEvent(const FrameEvent& frame_event) = 0; virtual void OnPacketEvent(const PacketEvent& packet_event) = 0; - // Returns nullopt if not enough data is in yet to produce an estimate. + // Estimates the clock offset between the sender and the receiver. + // + // This is calculated by solving a system of two linear equations with two + // unknowns: the clock offset and the network latency. The two equations are + // derived from two round-trip time measurements. + // + // Let's define: + // - latency: the one-way network latency. + // - offset: the clock offset, where Clock_Receiver(t) = Clock_Sender(t) + + // offset. + // + // The estimator measures two bounds: + // + // 1. packet_bound (sender -> receiver): + // delta = TS_receiver - TS_sender + // = (TS_sender + latency + offset) - TS_sender + // = latency + offset + // + // 2. frame_bound (receiver -> sender): + // delta = TS_sender - TS_receiver + // = (TS_receiver + latency - offset) - TS_receiver + // = latency - offset + // + // The offset is then isolated by the formula: + // (packet_bound - frame_bound) / 2 = + // ( (latency + offset) - (latency - offset) ) / 2 = + // (2 * offset) / 2 = offset virtual std::optional GetEstimatedOffset() const = 0; + + // Estimates the one-way network latency. + // This uses the same bounds as GetEstimatedOffset(). + // + // The latency is isolated by the formula: + // (packet_bound + frame_bound) / 2 = + // ( (latency + offset) + (latency - offset) ) / 2 = (2 * latency) / 2 = + // latency + virtual std::optional GetEstimatedLatency() const = 0; }; } // namespace openscreen::cast diff --git a/cast/streaming/impl/clock_offset_estimator_impl.cc b/cast/streaming/impl/clock_offset_estimator_impl.cc index 34693e7f0..49a7002bc 100644 --- a/cast/streaming/impl/clock_offset_estimator_impl.cc +++ b/cast/streaming/impl/clock_offset_estimator_impl.cc @@ -5,19 +5,16 @@ #include "cast/streaming/impl/clock_offset_estimator_impl.h" #include +#include #include #include #include "platform/base/trivial_clock_traits.h" +#include "util/chrono_helpers.h" namespace openscreen::cast { namespace { -// The lower this is, the faster we adjust to clock drift (but with more -// jitter). Each successful call to BoundCalculator::UpdateBound() uses this as -// the weight of the bound upte. -constexpr size_t kBoundUpdateWeight = 500; - // This should be large enough so that we can collect all 3 events before // the entry gets removed from the map. constexpr size_t kMaxEventTimesMapSize = 500; @@ -53,16 +50,16 @@ ClockOffsetEstimatorImpl::~ClockOffsetEstimatorImpl() = default; void ClockOffsetEstimatorImpl::OnFrameEvent(const FrameEvent& frame_event) { switch (frame_event.type) { - case StatisticsEventType::kFrameAckSent: + case StatisticsEvent::Type::kFrameAckSent: frame_bound_.SetSent( frame_event.rtp_timestamp, 0, - frame_event.media_type == StatisticsEventMediaType::kAudio, + frame_event.media_type == StatisticsEvent::MediaType::kAudio, frame_event.timestamp); break; - case StatisticsEventType::kFrameAckReceived: + case StatisticsEvent::Type::kFrameAckReceived: frame_bound_.SetReceived( frame_event.rtp_timestamp, 0, - frame_event.media_type == StatisticsEventMediaType::kAudio, + frame_event.media_type == StatisticsEvent::MediaType::kAudio, frame_event.timestamp); break; default: @@ -73,16 +70,16 @@ void ClockOffsetEstimatorImpl::OnFrameEvent(const FrameEvent& frame_event) { void ClockOffsetEstimatorImpl::OnPacketEvent(const PacketEvent& packet_event) { switch (packet_event.type) { - case StatisticsEventType::kPacketSentToNetwork: + case StatisticsEvent::Type::kPacketSentToNetwork: packet_bound_.SetSent( packet_event.rtp_timestamp, packet_event.packet_id, - packet_event.media_type == StatisticsEventMediaType::kAudio, + packet_event.media_type == StatisticsEvent::MediaType::kAudio, packet_event.timestamp); break; - case StatisticsEventType::kPacketReceived: + case StatisticsEvent::Type::kPacketReceived: packet_bound_.SetReceived( packet_event.rtp_timestamp, packet_event.packet_id, - packet_event.media_type == StatisticsEventMediaType::kAudio, + packet_event.media_type == StatisticsEvent::MediaType::kAudio, packet_event.timestamp); break; default: @@ -114,7 +111,67 @@ std::optional ClockOffsetEstimatorImpl::GetEstimatedOffset() return (packet_bound + frame_bound) / 2; } -ClockOffsetEstimatorImpl::BoundCalculator::BoundCalculator() = default; +std::optional ClockOffsetEstimatorImpl::GetEstimatedLatency() + const { + Clock::duration frame_bound; + Clock::duration packet_bound; + if (!GetReceiverOffsetBounds(frame_bound, packet_bound)) { + return {}; + } + return (packet_bound - frame_bound) / 2; +} + +ClockOffsetEstimatorImpl::KalmanFilter::KalmanFilter( + Clock::duration process_noise, + Clock::duration measurement_noise) + : q_nanos_squared_( + static_cast(std::chrono::nanoseconds(process_noise).count()) * + std::chrono::nanoseconds(process_noise).count()), + r_nanos_squared_( + static_cast( + std::chrono::nanoseconds(measurement_noise).count()) * + std::chrono::nanoseconds(measurement_noise).count()) {} + +void ClockOffsetEstimatorImpl::KalmanFilter::Update( + Clock::duration measurement) { + if (!has_estimate_) { + // First measurement, initialize the state. + estimated_latency_ = measurement; + error_covariance_nanos_squared_ = r_nanos_squared_; + has_estimate_ = true; + return; + } + + // --- 1. PREDICT --- + // The predicted state is the same as the previous state. + // The uncertainty (covariance) increases by the process noise. + const double predicted_error_covariance = + error_covariance_nanos_squared_ + q_nanos_squared_; + + // --- 2. UPDATE --- + // Calculate Kalman Gain. + const double kalman_gain = predicted_error_covariance / + (predicted_error_covariance + r_nanos_squared_); + + // Update the estimate with the new measurement. + const double measurement_nanos = + static_cast(std::chrono::nanoseconds(measurement).count()); + const double estimate_nanos = + static_cast(std::chrono::nanoseconds(estimated_latency_).count()); + const double new_estimate_nanos = + estimate_nanos + kalman_gain * (measurement_nanos - estimate_nanos); + estimated_latency_ = + std::chrono::duration_cast(std::chrono::nanoseconds( + static_cast(new_estimate_nanos))); + + // Update the error covariance. + error_covariance_nanos_squared_ = + (1.0 - kalman_gain) * predicted_error_covariance; +} + +ClockOffsetEstimatorImpl::BoundCalculator::BoundCalculator() + : filter_(kProcessNoise, kMeasurementNoise) {} + ClockOffsetEstimatorImpl::BoundCalculator::BoundCalculator( BoundCalculator&&) noexcept = default; ClockOffsetEstimatorImpl::BoundCalculator& @@ -144,17 +201,7 @@ void ClockOffsetEstimatorImpl::BoundCalculator::SetReceived( void ClockOffsetEstimatorImpl::BoundCalculator::UpdateBound( Clock::time_point sent, Clock::time_point received) { - Clock::duration delta = received - sent; - if (has_bound_) { - if (delta < bound_) { - bound_ = delta; - } else { - bound_ += (delta - bound_) / kBoundUpdateWeight; - } - } else { - bound_ = delta; - } - has_bound_ = true; + filter_.Update(received - sent); } void ClockOffsetEstimatorImpl::BoundCalculator::CheckUpdate(uint64_t key) { diff --git a/cast/streaming/impl/clock_offset_estimator_impl.h b/cast/streaming/impl/clock_offset_estimator_impl.h index ad45d194a..5bf942252 100644 --- a/cast/streaming/impl/clock_offset_estimator_impl.h +++ b/cast/streaming/impl/clock_offset_estimator_impl.h @@ -13,9 +13,10 @@ #include #include "cast/streaming/impl/clock_offset_estimator.h" -#include "cast/streaming/impl/statistics_defines.h" +#include "cast/streaming/impl/statistics_common.h" #include "cast/streaming/rtp_time.h" #include "platform/base/trivial_clock_traits.h" +#include "util/chrono_helpers.h" namespace openscreen::cast { @@ -41,11 +42,47 @@ class ClockOffsetEstimatorImpl final : public ClockOffsetEstimator { bool GetReceiverOffsetBounds(Clock::duration& frame_bound, Clock::duration& packet_bound) const; - // Returns the average of the offset bounds for frame and packet events. - // Returns nullopt if not enough data is in yet to produce an estimate. + // ClockOffsetEstimator overrides. std::optional GetEstimatedOffset() const final; + std::optional GetEstimatedLatency() const final; private: + // These values are chosen based on common network conditions. + // + // Q (process_noise): We expect latency to drift by up to 5ms between + // measurements. + static constexpr Clock::duration kProcessNoise = milliseconds(5); + // + // R (measurement_noise): We expect jitter of up to 30ms. + static constexpr Clock::duration kMeasurementNoise = milliseconds(30); + + // Simplified 1D Kalman Filter for latency estimation. + class KalmanFilter { + public: + // Q: process_noise - Represents the expected variance of the latency + // itself between time steps. A higher value makes the filter adapt + // more quickly to real changes in latency. + // R: measurement_noise - Represents the variance of the measurement + // noise (jitter). A higher value makes the filter trust its own + // prediction more and smooth out noisy measurements. + KalmanFilter(Clock::duration process_noise, + Clock::duration measurement_noise); + KalmanFilter(KalmanFilter&&) noexcept = default; + KalmanFilter& operator=(KalmanFilter&&) = default; + + Clock::duration GetEstimate() const { return estimated_latency_; } + bool HasEstimate() const { return has_estimate_; } + void Update(Clock::duration measurement); + + private: + double q_nanos_squared_; + double r_nanos_squared_; + + bool has_estimate_ = false; + Clock::duration estimated_latency_{}; + double error_covariance_nanos_squared_ = 0.0; + }; + // This helper uses the difference between sent and received event // to calculate an upper bound on the difference between the clocks // on the sender and receiver. Note that this difference can take @@ -67,8 +104,8 @@ class ClockOffsetEstimatorImpl final : public ClockOffsetEstimator { BoundCalculator& operator=(BoundCalculator&&); BoundCalculator& operator=(const BoundCalculator&) = delete; ~BoundCalculator(); - bool has_bound() const { return has_bound_; } - Clock::duration bound() const { return bound_; } + bool has_bound() const { return filter_.HasEstimate(); } + Clock::duration bound() const { return filter_.GetEstimate(); } void SetSent(RtpTimeTicks rtp, uint16_t packet_id, @@ -86,8 +123,7 @@ class ClockOffsetEstimatorImpl final : public ClockOffsetEstimator { private: EventMap events_; - bool has_bound_ = false; - Clock::duration bound_{}; + KalmanFilter filter_; }; // Fixed size storage to store event times for recent frames and packets. diff --git a/cast/streaming/impl/clock_offset_estimator_impl_unittest.cc b/cast/streaming/impl/clock_offset_estimator_impl_unittest.cc index ab0300b66..a826bf36e 100644 --- a/cast/streaming/impl/clock_offset_estimator_impl_unittest.cc +++ b/cast/streaming/impl/clock_offset_estimator_impl_unittest.cc @@ -7,371 +7,357 @@ #include #include +#include #include +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "platform/base/trivial_clock_traits.h" #include "platform/test/fake_clock.h" +#include "testing/util/chrono_test_helpers.h" #include "util/chrono_helpers.h" namespace openscreen::cast { +namespace { + +FrameEvent CreateFrameEvent(StatisticsEvent::Type type, + Clock::time_point timestamp, + FrameId frame_id, + RtpTimeTicks rtp_timestamp, + StatisticsEvent::MediaType media_type) { + FrameEvent event; + event.type = type; + event.media_type = media_type; + event.timestamp = timestamp; + event.frame_id = frame_id; + event.rtp_timestamp = rtp_timestamp; + return event; +} + +PacketEvent CreatePacketEvent(StatisticsEvent::Type type, + Clock::time_point timestamp, + FrameId frame_id, + RtpTimeTicks rtp_timestamp, + StatisticsEvent::MediaType media_type) { + PacketEvent event; + event.type = type; + event.media_type = media_type; + event.timestamp = timestamp; + event.frame_id = frame_id; + event.rtp_timestamp = rtp_timestamp; + event.packet_id = 0; + event.max_packet_id = 1; + event.size = 1500; + return event; +} + +} // namespace + class ClockOffsetEstimatorImplTest : public ::testing::Test { public: ClockOffsetEstimatorImplTest() - : sender_time_(Clock::now()), receiver_clock_(Clock::now()) {} + : sender_time_(Clock::now()), + receiver_clock_(Clock::now()), + estimator_() {} ~ClockOffsetEstimatorImplTest() override = default; + protected: void AdvanceClocks(Clock::duration time) { receiver_clock_.Advance(time); sender_time_ += time; } + void SendAndReceiveEvents(FrameId frame_id, + RtpTimeTicks rtp, + milliseconds network_latency, + StatisticsEvent::MediaType media_type) { + estimator_.OnFrameEvent( + CreateFrameEvent(StatisticsEvent::Type::kFrameEncoded, sender_time_, + frame_id, rtp, media_type)); + estimator_.OnPacketEvent( + CreatePacketEvent(StatisticsEvent::Type::kPacketSentToNetwork, + sender_time_, frame_id, rtp, media_type)); + AdvanceClocks(network_latency); + estimator_.OnPacketEvent( + CreatePacketEvent(StatisticsEvent::Type::kPacketReceived, + receiver_clock_.now(), frame_id, rtp, media_type)); + estimator_.OnFrameEvent( + CreateFrameEvent(StatisticsEvent::Type::kFrameAckSent, + receiver_clock_.now(), frame_id, rtp, media_type)); + AdvanceClocks(network_latency); + estimator_.OnFrameEvent( + CreateFrameEvent(StatisticsEvent::Type::kFrameAckReceived, sender_time_, + frame_id, rtp, media_type)); + } + Clock::time_point sender_time_; - // Only one fake clock instance is allowed. FakeClock receiver_clock_; ClockOffsetEstimatorImpl estimator_; }; -// Suppose the true offset is 100ms. -// Event A occurred at sender time 20ms. -// Event B occurred at receiver time 130ms. (sender time 30ms) -// Event C occurred at sender time 60ms. -// Then the bound after all 3 events have arrived is [130-60=70, 130-20=110]. -TEST_F(ClockOffsetEstimatorImplTest, EstimateOffset) { - const milliseconds kTrueOffset(100); +TEST_F(ClockOffsetEstimatorImplTest, ReturnsNulloptWhenNoEvents) { + EXPECT_FALSE(estimator_.GetEstimatedOffset()); + EXPECT_FALSE(estimator_.GetEstimatedLatency()); +} + +TEST_F(ClockOffsetEstimatorImplTest, CalculatesOffsetAndLatencyAfterOneTrip) { + constexpr milliseconds kTrueOffset(100); + constexpr milliseconds kNetworkLatency(10); receiver_clock_.Advance(kTrueOffset); - Clock::duration frame_bound; - Clock::duration packet_bound; + SendAndReceiveEvents(FrameId::first(), RtpTimeTicks(), kNetworkLatency, + StatisticsEvent::MediaType::kVideo); - EXPECT_FALSE(estimator_.GetReceiverOffsetBounds(frame_bound, packet_bound)); + ASSERT_TRUE(estimator_.GetEstimatedOffset()); + EXPECT_EQ(kTrueOffset, to_milliseconds(*estimator_.GetEstimatedOffset())); + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_EQ(kNetworkLatency, + to_milliseconds(*estimator_.GetEstimatedLatency())); +} + +TEST_F(ClockOffsetEstimatorImplTest, + CalculatesOffsetAndLatencyWithOutOfOrderEvents) { + constexpr milliseconds kTrueOffset(100); + receiver_clock_.Advance(kTrueOffset); const RtpTimeTicks rtp_timestamp; - FrameId frame_id = FrameId::first(); + const FrameId frame_id = FrameId::first(); AdvanceClocks(milliseconds(20)); - - FrameEvent encode_event; - encode_event.timestamp = sender_time_; - encode_event.type = StatisticsEventType::kFrameEncoded; - encode_event.media_type = StatisticsEventMediaType::kVideo; - encode_event.rtp_timestamp = rtp_timestamp; - encode_event.frame_id = frame_id; - encode_event.size = 1234; - encode_event.key_frame = true; - encode_event.target_bitrate = 5678; - estimator_.OnFrameEvent(encode_event); - - PacketEvent send_event; - send_event.timestamp = sender_time_; - send_event.type = StatisticsEventType::kPacketSentToNetwork; - send_event.media_type = StatisticsEventMediaType::kVideo; - send_event.rtp_timestamp = rtp_timestamp; - send_event.frame_id = frame_id; - send_event.packet_id = 56; - send_event.max_packet_id = 78; - send_event.size = 1500; - estimator_.OnPacketEvent(send_event); - - EXPECT_FALSE(estimator_.GetReceiverOffsetBounds(frame_bound, packet_bound)); + estimator_.OnFrameEvent(CreateFrameEvent( + StatisticsEvent::Type::kFrameEncoded, sender_time_, frame_id, + rtp_timestamp, StatisticsEvent::MediaType::kVideo)); + estimator_.OnPacketEvent(CreatePacketEvent( + StatisticsEvent::Type::kPacketSentToNetwork, sender_time_, frame_id, + rtp_timestamp, StatisticsEvent::MediaType::kVideo)); AdvanceClocks(milliseconds(10)); - FrameEvent ack_sent_event; - ack_sent_event.timestamp = receiver_clock_.now(); - ack_sent_event.type = StatisticsEventType::kFrameAckSent; - ack_sent_event.media_type = StatisticsEventMediaType::kVideo; - ack_sent_event.rtp_timestamp = rtp_timestamp; - ack_sent_event.frame_id = frame_id; - estimator_.OnFrameEvent(ack_sent_event); - - PacketEvent receive_event; - receive_event.timestamp = receiver_clock_.now(); - receive_event.type = StatisticsEventType::kPacketReceived; - receive_event.media_type = StatisticsEventMediaType::kVideo; - receive_event.rtp_timestamp = rtp_timestamp; - receive_event.frame_id = frame_id; - receive_event.packet_id = 56; - receive_event.max_packet_id = 78; - receive_event.size = 1500; - estimator_.OnPacketEvent(receive_event); - - EXPECT_FALSE(estimator_.GetReceiverOffsetBounds(frame_bound, packet_bound)); - + const auto event_b_time = receiver_clock_.now(); AdvanceClocks(milliseconds(30)); - FrameEvent ack_event; - ack_event.timestamp = sender_time_; - ack_event.type = StatisticsEventType::kFrameAckReceived; - ack_event.media_type = StatisticsEventMediaType::kVideo; - ack_event.rtp_timestamp = rtp_timestamp; - ack_event.frame_id = frame_id; - estimator_.OnFrameEvent(ack_event); - - EXPECT_TRUE(estimator_.GetReceiverOffsetBounds(frame_bound, packet_bound)); - - EXPECT_EQ(milliseconds(70), to_milliseconds(frame_bound)); - EXPECT_EQ(milliseconds(110), to_milliseconds(packet_bound)); - EXPECT_GE(kTrueOffset, frame_bound); - EXPECT_LE(kTrueOffset, packet_bound); + const auto event_c_time = sender_time_; + + estimator_.OnFrameEvent(CreateFrameEvent( + StatisticsEvent::Type::kFrameAckReceived, event_c_time, frame_id, + rtp_timestamp, StatisticsEvent::MediaType::kVideo)); + estimator_.OnPacketEvent(CreatePacketEvent( + StatisticsEvent::Type::kPacketReceived, event_b_time, frame_id, + rtp_timestamp, StatisticsEvent::MediaType::kVideo)); + estimator_.OnFrameEvent(CreateFrameEvent( + StatisticsEvent::Type::kFrameAckSent, event_b_time, frame_id, + rtp_timestamp, StatisticsEvent::MediaType::kVideo)); + + ASSERT_TRUE(estimator_.GetEstimatedOffset()); + EXPECT_EQ(milliseconds(90), + to_milliseconds(*estimator_.GetEstimatedOffset())); + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_EQ(milliseconds(20), + to_milliseconds(*estimator_.GetEstimatedLatency())); } -// Same scenario as above, but event C arrives before event B. It doesn't mean -// event C occurred before event B. -TEST_F(ClockOffsetEstimatorImplTest, EventCArrivesBeforeEventB) { +TEST_F(ClockOffsetEstimatorImplTest, + UpdatesOffsetAndLatencyAfterMultipleRoundTrips) { constexpr milliseconds kTrueOffset(100); + constexpr milliseconds kNetworkLatency(5); receiver_clock_.Advance(kTrueOffset); - Clock::duration frame_bound; - Clock::duration packet_bound; + SendAndReceiveEvents(FrameId::first(), RtpTimeTicks(), kNetworkLatency, + StatisticsEvent::MediaType::kVideo); + ASSERT_TRUE(estimator_.GetEstimatedOffset()); + EXPECT_EQ(kTrueOffset, to_milliseconds(*estimator_.GetEstimatedOffset())); + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_EQ(kNetworkLatency, + to_milliseconds(*estimator_.GetEstimatedLatency())); + + AdvanceClocks(milliseconds(100)); + SendAndReceiveEvents(FrameId::first() + 1, + RtpTimeTicks() + RtpTimeDelta::FromTicks(90), + kNetworkLatency, StatisticsEvent::MediaType::kVideo); + ASSERT_TRUE(estimator_.GetEstimatedOffset()); + EXPECT_EQ(kTrueOffset, to_milliseconds(*estimator_.GetEstimatedOffset())); + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_EQ(kNetworkLatency, + to_milliseconds(*estimator_.GetEstimatedLatency())); +} - EXPECT_FALSE(estimator_.GetReceiverOffsetBounds(frame_bound, packet_bound)); +TEST_F(ClockOffsetEstimatorImplTest, + CalculatesLatencyWithVaryingNetworkConditions) { + constexpr milliseconds kTrueOffset(100); + receiver_clock_.Advance(kTrueOffset); - const RtpTimeTicks rtp_timestamp; - FrameId frame_id = FrameId::first(); + // Start with a baseline latency. + constexpr milliseconds kBaselineLatency(10); + SendAndReceiveEvents(FrameId::first(), RtpTimeTicks(), kBaselineLatency, + StatisticsEvent::MediaType::kVideo); + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_THAT(to_milliseconds(*estimator_.GetEstimatedLatency()), + EqualsDuration(kBaselineLatency)); + + // Test with zero latency. + constexpr milliseconds kZeroLatency(0); + for (int i = 0; i < 10; ++i) { + SendAndReceiveEvents(FrameId::first() + 1 + i, + RtpTimeTicks() + RtpTimeDelta::FromTicks(90 * (i + 1)), + kZeroLatency, StatisticsEvent::MediaType::kVideo); + } + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedLatency()).count(), + kZeroLatency.count(), 5); + + // Test with high latency. + constexpr milliseconds kHighLatency(100); + for (int i = 0; i < 10; ++i) { + SendAndReceiveEvents( + FrameId::first() + 11 + i, + RtpTimeTicks() + RtpTimeDelta::FromTicks(90 * (i + 11)), kHighLatency, + StatisticsEvent::MediaType::kVideo); + } + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedLatency()).count(), + kHighLatency.count(), 20); +} - AdvanceClocks(milliseconds(20)); +TEST_F(ClockOffsetEstimatorImplTest, ConvergesToMeanLatencyWithJitter) { + constexpr milliseconds kTrueOffset(100); + constexpr milliseconds kMeanNetworkLatency(50); + constexpr milliseconds kJitter(40); + receiver_clock_.Advance(kTrueOffset); - FrameEvent encode_event; - encode_event.timestamp = sender_time_; - encode_event.type = StatisticsEventType::kFrameEncoded; - encode_event.media_type = StatisticsEventMediaType::kVideo; - encode_event.rtp_timestamp = rtp_timestamp; - encode_event.frame_id = frame_id; - encode_event.size = 1234; - encode_event.key_frame = true; - encode_event.target_bitrate = 5678; - estimator_.OnFrameEvent(encode_event); - - PacketEvent send_event; - send_event.timestamp = sender_time_; - send_event.type = StatisticsEventType::kPacketSentToNetwork; - send_event.media_type = StatisticsEventMediaType::kVideo; - send_event.rtp_timestamp = rtp_timestamp; - send_event.frame_id = frame_id; - send_event.packet_id = 56; - send_event.max_packet_id = 78; - send_event.size = 1500; - estimator_.OnPacketEvent(send_event); - - EXPECT_FALSE(estimator_.GetReceiverOffsetBounds(frame_bound, packet_bound)); + std::minstd_rand prng; + std::uniform_int_distribution jitter_dist(-kJitter.count(), + kJitter.count()); - AdvanceClocks(milliseconds(10)); - Clock::time_point event_b_time = receiver_clock_.now(); - AdvanceClocks(milliseconds(30)); - Clock::time_point event_c_time = sender_time_; - - FrameEvent ack_event; - ack_event.timestamp = event_c_time; - ack_event.type = StatisticsEventType::kFrameAckReceived; - ack_event.media_type = StatisticsEventMediaType::kVideo; - ack_event.rtp_timestamp = rtp_timestamp; - ack_event.frame_id = frame_id; - estimator_.OnFrameEvent(ack_event); - - EXPECT_FALSE(estimator_.GetReceiverOffsetBounds(frame_bound, packet_bound)); - - PacketEvent receive_event; - receive_event.timestamp = event_b_time; - receive_event.type = StatisticsEventType::kPacketReceived; - receive_event.media_type = StatisticsEventMediaType::kVideo; - receive_event.rtp_timestamp = rtp_timestamp; - receive_event.frame_id = frame_id; - receive_event.packet_id = 56; - receive_event.max_packet_id = 78; - receive_event.size = 1500; - estimator_.OnPacketEvent(receive_event); - - FrameEvent ack_sent_event; - ack_sent_event.timestamp = event_b_time; - ack_sent_event.type = StatisticsEventType::kFrameAckSent; - ack_sent_event.media_type = StatisticsEventMediaType::kVideo; - ack_sent_event.rtp_timestamp = rtp_timestamp; - ack_sent_event.frame_id = frame_id; - estimator_.OnFrameEvent(ack_sent_event); - - EXPECT_TRUE(estimator_.GetReceiverOffsetBounds(frame_bound, packet_bound)); - - // Note: although the bounds are measured in microseconds, we round here to - // the nearest millisecond to avoid comparison inaccuracies due to floating - // point representation. - EXPECT_EQ(milliseconds(70), to_milliseconds(frame_bound)); - EXPECT_EQ(milliseconds(110), to_milliseconds(packet_bound)); - EXPECT_GE(kTrueOffset, frame_bound); - EXPECT_LE(kTrueOffset, packet_bound); + for (int i = 0; i < 50; ++i) { + const milliseconds jitter = milliseconds(jitter_dist(prng)); + const milliseconds network_latency = kMeanNetworkLatency + jitter; + SendAndReceiveEvents(FrameId::first() + i, + RtpTimeTicks() + RtpTimeDelta::FromTicks(i * 90), + network_latency, StatisticsEvent::MediaType::kVideo); + } + + // After many measurements, the estimate should be very close to the mean. + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedLatency()).count(), + kMeanNetworkLatency.count(), 20); } -TEST_F(ClockOffsetEstimatorImplTest, MultipleIterations) { +TEST_F(ClockOffsetEstimatorImplTest, TracksClockDrift) { + constexpr milliseconds kInitialOffset(100); + constexpr milliseconds kNetworkLatency(10); + constexpr microseconds kDriftPerFrame(100); + constexpr int kNumFrames = 50; + + receiver_clock_.Advance(kInitialOffset); + + for (int i = 0; i < kNumFrames; ++i) { + SendAndReceiveEvents(FrameId::first() + i, + RtpTimeTicks() + RtpTimeDelta::FromTicks(i * 90), + kNetworkLatency, StatisticsEvent::MediaType::kVideo); + receiver_clock_.Advance(kDriftPerFrame); + } + + const milliseconds kFinalOffset = + kInitialOffset + + std::chrono::duration_cast(kDriftPerFrame * kNumFrames); + ASSERT_TRUE(estimator_.GetEstimatedOffset()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedOffset()).count(), + kFinalOffset.count(), 5); +} + +TEST_F(ClockOffsetEstimatorImplTest, IsStableWithPacketLoss) { constexpr milliseconds kTrueOffset(100); - receiver_clock_.Advance(milliseconds(kTrueOffset)); - - Clock::duration frame_bound; - Clock::duration packet_bound; - - const RtpTimeTicks rtp_timestamp_a; - FrameId frame_id_a = FrameId::first(); - const RtpTimeTicks rtp_timestamp_b = - rtp_timestamp_a + RtpTimeDelta::FromTicks(90); - FrameId frame_id_b = frame_id_a + 1; - const RtpTimeTicks rtp_timestamp_c = - rtp_timestamp_b + RtpTimeDelta::FromTicks(90); - FrameId frame_id_c = frame_id_b + 1; - - // Frame 1 times: [20, 30+100, 60] - // Frame 2 times: [30, 50+100, 55] - // Frame 3 times: [77, 80+100, 110] - // Bound should end up at [95, 103] - // Events times in chronological order: 20, 30 x2, 50, 55, 60, 77, 80, 110 - AdvanceClocks(milliseconds(20)); - FrameEvent encode_event; - encode_event.timestamp = sender_time_; - encode_event.type = StatisticsEventType::kFrameEncoded; - encode_event.media_type = StatisticsEventMediaType::kVideo; - encode_event.rtp_timestamp = rtp_timestamp_a; - encode_event.frame_id = frame_id_a; - encode_event.size = 1234; - encode_event.key_frame = true; - encode_event.target_bitrate = 5678; - estimator_.OnFrameEvent(encode_event); - - PacketEvent send_event; - send_event.timestamp = sender_time_; - send_event.type = StatisticsEventType::kPacketSentToNetwork; - send_event.media_type = StatisticsEventMediaType::kVideo; - send_event.rtp_timestamp = rtp_timestamp_a; - send_event.frame_id = frame_id_a; - send_event.packet_id = 56; - send_event.max_packet_id = 78; - send_event.size = 1500; - estimator_.OnPacketEvent(send_event); + constexpr milliseconds kNetworkLatency(10); + receiver_clock_.Advance(kTrueOffset); - AdvanceClocks(milliseconds(10)); - FrameEvent second_encode_event; - second_encode_event.timestamp = sender_time_; - second_encode_event.type = StatisticsEventType::kFrameEncoded; - second_encode_event.media_type = StatisticsEventMediaType::kVideo; - second_encode_event.rtp_timestamp = rtp_timestamp_b; - second_encode_event.frame_id = frame_id_b; - second_encode_event.size = 1234; - second_encode_event.key_frame = true; - second_encode_event.target_bitrate = 5678; - estimator_.OnFrameEvent(second_encode_event); - - PacketEvent second_send_event; - second_send_event.timestamp = sender_time_; - second_send_event.type = StatisticsEventType::kPacketSentToNetwork; - second_send_event.media_type = StatisticsEventMediaType::kVideo; - second_send_event.rtp_timestamp = rtp_timestamp_b; - second_send_event.frame_id = frame_id_b; - second_send_event.packet_id = 56; - second_send_event.max_packet_id = 78; - second_send_event.size = 1500; - estimator_.OnPacketEvent(second_send_event); - - FrameEvent ack_sent_event; - ack_sent_event.timestamp = receiver_clock_.now(); - ack_sent_event.type = StatisticsEventType::kFrameAckSent; - ack_sent_event.media_type = StatisticsEventMediaType::kVideo; - ack_sent_event.rtp_timestamp = rtp_timestamp_a; - ack_sent_event.frame_id = frame_id_a; - estimator_.OnFrameEvent(ack_sent_event); + // Simulate a burst of 1000 lost packets. + for (int i = 0; i < 1000; ++i) { + estimator_.OnPacketEvent(CreatePacketEvent( + StatisticsEvent::Type::kPacketSentToNetwork, sender_time_, + FrameId::first() + i, RtpTimeTicks() + RtpTimeDelta::FromTicks(i * 90), + StatisticsEvent::MediaType::kVideo)); + AdvanceClocks(milliseconds(1)); + } - AdvanceClocks(milliseconds(20)); + // Send a final, successful round-trip. + SendAndReceiveEvents(FrameId::first() + 1000, + RtpTimeTicks() + RtpTimeDelta::FromTicks(1000 * 90), + kNetworkLatency, StatisticsEvent::MediaType::kVideo); + + // The estimator should still produce a valid estimate. + ASSERT_TRUE(estimator_.GetEstimatedOffset()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedOffset()).count(), + kTrueOffset.count(), 5); + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedLatency()).count(), + kNetworkLatency.count(), 5); +} - PacketEvent receive_event; - receive_event.timestamp = receiver_clock_.now(); - receive_event.type = StatisticsEventType::kPacketReceived; - receive_event.media_type = StatisticsEventMediaType::kVideo; - receive_event.rtp_timestamp = rtp_timestamp_b; - receive_event.frame_id = frame_id_b; - receive_event.packet_id = 56; - receive_event.max_packet_id = 78; - receive_event.size = 1500; - estimator_.OnPacketEvent(receive_event); - - FrameEvent second_ack_sent_event; - second_ack_sent_event.timestamp = receiver_clock_.now(); - second_ack_sent_event.type = StatisticsEventType::kFrameAckSent; - second_ack_sent_event.media_type = StatisticsEventMediaType::kVideo; - second_ack_sent_event.rtp_timestamp = rtp_timestamp_b; - second_ack_sent_event.frame_id = frame_id_b; - estimator_.OnFrameEvent(second_ack_sent_event); - - AdvanceClocks(milliseconds(5)); - FrameEvent ack_event; - ack_event.timestamp = sender_time_; - ack_event.type = StatisticsEventType::kFrameAckReceived; - ack_event.media_type = StatisticsEventMediaType::kVideo; - ack_event.rtp_timestamp = rtp_timestamp_b; - ack_event.frame_id = frame_id_b; - estimator_.OnFrameEvent(ack_event); - - AdvanceClocks(milliseconds(5)); - FrameEvent second_ack_event; - second_ack_event.timestamp = sender_time_; - second_ack_event.type = StatisticsEventType::kFrameAckReceived; - second_ack_event.media_type = StatisticsEventMediaType::kVideo; - second_ack_event.rtp_timestamp = rtp_timestamp_a; - second_ack_event.frame_id = frame_id_a; - estimator_.OnFrameEvent(second_ack_event); - - AdvanceClocks(milliseconds(17)); - FrameEvent third_encode_event; - third_encode_event.timestamp = sender_time_; - third_encode_event.type = StatisticsEventType::kFrameEncoded; - third_encode_event.media_type = StatisticsEventMediaType::kVideo; - third_encode_event.rtp_timestamp = rtp_timestamp_c; - third_encode_event.frame_id = frame_id_c; - third_encode_event.size = 1234; - third_encode_event.key_frame = true; - third_encode_event.target_bitrate = 5678; - estimator_.OnFrameEvent(third_encode_event); - - PacketEvent third_send_event; - third_send_event.timestamp = sender_time_; - third_send_event.type = StatisticsEventType::kPacketSentToNetwork; - third_send_event.media_type = StatisticsEventMediaType::kVideo; - third_send_event.rtp_timestamp = rtp_timestamp_c; - third_send_event.frame_id = frame_id_c; - third_send_event.packet_id = 56; - third_send_event.max_packet_id = 78; - third_send_event.size = 1500; - estimator_.OnPacketEvent(third_send_event); - - AdvanceClocks(milliseconds(3)); - PacketEvent second_receive_event; - second_receive_event.timestamp = receiver_clock_.now(); - second_receive_event.type = StatisticsEventType::kPacketReceived; - second_receive_event.media_type = StatisticsEventMediaType::kVideo; - second_receive_event.rtp_timestamp = rtp_timestamp_c; - second_receive_event.frame_id = frame_id_c; - second_receive_event.packet_id = 56; - second_receive_event.max_packet_id = 78; - second_receive_event.size = 1500; - estimator_.OnPacketEvent(second_receive_event); - - FrameEvent third_ack_sent_event; - third_ack_sent_event.timestamp = receiver_clock_.now(); - third_ack_sent_event.type = StatisticsEventType::kFrameAckSent; - third_ack_sent_event.media_type = StatisticsEventMediaType::kVideo; - third_ack_sent_event.rtp_timestamp = rtp_timestamp_c; - third_ack_sent_event.frame_id = frame_id_c; - estimator_.OnFrameEvent(third_ack_sent_event); +TEST_F(ClockOffsetEstimatorImplTest, RecoversFromLatencySpike) { + constexpr milliseconds kTrueOffset(100); + constexpr milliseconds kBaselineLatency(10); + receiver_clock_.Advance(kTrueOffset); - AdvanceClocks(milliseconds(30)); - FrameEvent third_ack_event; - third_ack_event.timestamp = sender_time_; - third_ack_event.type = StatisticsEventType::kFrameAckReceived; - third_ack_event.media_type = StatisticsEventMediaType::kVideo; - third_ack_event.rtp_timestamp = rtp_timestamp_c; - third_ack_event.frame_id = frame_id_c; - estimator_.OnFrameEvent(third_ack_event); - - EXPECT_TRUE(estimator_.GetReceiverOffsetBounds(frame_bound, packet_bound)); - EXPECT_GT(frame_bound, milliseconds(90)); - EXPECT_LE(frame_bound, kTrueOffset); - EXPECT_LT(packet_bound, milliseconds(150)); - EXPECT_GT(packet_bound, kTrueOffset); + // Establish a baseline estimate. + for (int i = 0; i < 10; ++i) { + SendAndReceiveEvents(FrameId::first() + i, + RtpTimeTicks() + RtpTimeDelta::FromTicks(i * 90), + kBaselineLatency, StatisticsEvent::MediaType::kVideo); + } + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedLatency()).count(), + kBaselineLatency.count(), 5); + + // Introduce a large latency spike. + const auto kSpikeLatency = milliseconds(500); + SendAndReceiveEvents(FrameId::first() + 10, + RtpTimeTicks() + RtpTimeDelta::FromTicks(10 * 90), + kSpikeLatency, StatisticsEvent::MediaType::kVideo); + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + + // Ensure that there is a significant jump in the estimate, but not all the + // way to the entire spike value. + EXPECT_GT(to_milliseconds(*estimator_.GetEstimatedLatency()).count(), + kBaselineLatency.count() * 5); + EXPECT_LT(to_milliseconds(*estimator_.GetEstimatedLatency()).count(), + kSpikeLatency.count() / 2); + + // After several more measurements, the estimate should recover. + for (int i = 11; i < 25; ++i) { + SendAndReceiveEvents(FrameId::first() + i, + RtpTimeTicks() + RtpTimeDelta::FromTicks(i * 90), + kBaselineLatency, StatisticsEvent::MediaType::kVideo); + } + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedLatency()).count(), + kBaselineLatency.count(), 10); +} + +TEST_F(ClockOffsetEstimatorImplTest, HandlesMixedAudioAndVideoEvents) { + constexpr milliseconds kTrueOffset(50); + constexpr milliseconds kNetworkLatency(20); + receiver_clock_.Advance(kTrueOffset); + + // Send a video frame and check the estimate. + SendAndReceiveEvents(FrameId::first(), RtpTimeTicks(), kNetworkLatency, + StatisticsEvent::MediaType::kVideo); + ASSERT_TRUE(estimator_.GetEstimatedOffset()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedOffset()).count(), + kTrueOffset.count(), 5); + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedLatency()).count(), + kNetworkLatency.count(), 5); + + // Now send an audio frame and check that the estimate is updated. + SendAndReceiveEvents(FrameId::first() + 1, + RtpTimeTicks() + RtpTimeDelta::FromTicks(90), + kNetworkLatency, StatisticsEvent::MediaType::kAudio); + ASSERT_TRUE(estimator_.GetEstimatedOffset()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedOffset()).count(), + kTrueOffset.count(), 5); + ASSERT_TRUE(estimator_.GetEstimatedLatency()); + EXPECT_NEAR(to_milliseconds(*estimator_.GetEstimatedLatency()).count(), + kNetworkLatency.count(), 5); } } // namespace openscreen::cast diff --git a/cast/streaming/impl/compound_rtcp_builder.cc b/cast/streaming/impl/compound_rtcp_builder.cc index bf3dbf58e..dc3e302af 100644 --- a/cast/streaming/impl/compound_rtcp_builder.cc +++ b/cast/streaming/impl/compound_rtcp_builder.cc @@ -11,6 +11,7 @@ #include "cast/streaming/impl/packet_util.h" #include "cast/streaming/impl/rtcp_session.h" #include "platform/base/span.h" +#include "util/chrono_helpers.h" #include "util/integer_division.h" #include "util/osp_logging.h" #include "util/std_util.h" @@ -81,6 +82,11 @@ void CompoundRtcpBuilder::IncludeFeedbackInNextPacket( #endif } +void CompoundRtcpBuilder::IncludeReceiverLogsInNextPacket( + std::vector logs) { + logs_for_next_packet_ = std::move(logs); +} + ByteBuffer CompoundRtcpBuilder::BuildPacket(Clock::time_point send_time, ByteBuffer buffer) { OSP_CHECK_GE(buffer.size(), kRequiredBufferSize); @@ -107,6 +113,10 @@ ByteBuffer CompoundRtcpBuilder::BuildPacket(Clock::time_point send_time, // the remaining space available in the buffer will allow for. AppendCastFeedbackPacket(buffer); + // Receiver Log: Add as many receiver logs as the remaining space available + // in the buffer will allow for. + AppendReceiverLogPacket(buffer); + uint8_t* const packet_end = buffer.data(); return ByteBuffer(packet_begin, packet_end - packet_begin); } @@ -161,7 +171,7 @@ void CompoundRtcpBuilder::AppendCastFeedbackPacket(ByteBuffer& buffer) { // Reserve space for the RTCP Common Header. It will be serialized later, // after the total size of the Cast Feedback message is known. ByteBuffer space_for_header = ReserveSpace(kRtcpCommonHeaderSize, buffer); - uint8_t* const feedback_fields_begin = buffer.data(); + const size_t initial_buffer_size = buffer.size(); // Append the mandatory fields. AppendField(session_.receiver_ssrc(), buffer); @@ -192,8 +202,7 @@ void CompoundRtcpBuilder::AppendCastFeedbackPacket(ByteBuffer& buffer) { RtcpCommonHeader header; header.packet_type = RtcpPacketType::kPayloadSpecific; header.with.subtype = RtcpSubtype::kFeedback; - uint8_t* const feedback_fields_end = buffer.data(); - header.payload_size = feedback_fields_end - feedback_fields_begin; + header.payload_size = initial_buffer_size - buffer.size(); header.AppendFields(space_for_header); ++feedback_count_; @@ -316,4 +325,84 @@ void CompoundRtcpBuilder::AppendCastFeedbackAckFields(ByteBuffer& buffer) { acks_for_next_packet_.clear(); } +void CompoundRtcpBuilder::AppendReceiverLogPacket(ByteBuffer& buffer) { + if (logs_for_next_packet_.empty()) { + return; + } + + // Reserve space for the RTCP Common Header. It will be serialized later, + // after the total size of the message is known. + ByteBuffer space_for_header = ReserveSpace(kRtcpCommonHeaderSize, buffer); + const size_t initial_buffer_size = buffer.size(); + + // Append the mandatory fields. + AppendField(session_.receiver_ssrc(), buffer); + AppendField(kCastName, buffer); + + for (const auto& frame_log : logs_for_next_packet_) { + if (buffer.size() < kRtcpReceiverFrameLogMessageHeaderSize) { + break; + } + AppendField(frame_log.rtp_timestamp.lower_32_bits(), buffer); + + const int num_events = frame_log.messages.size(); + // The number of events is encoded as N-1. + const uint8_t num_events_wire = num_events - 1; + + // The event timestamp is a 24-bit field. + const auto event_timestamp_ms = + to_milliseconds(frame_log.messages[0].timestamp - session_.start_time()) + .count(); + const uint32_t event_timestamp_wire = + static_cast(event_timestamp_ms); + + AppendField( + (num_events_wire << 24) | (event_timestamp_wire & 0xFFFFFF), buffer); + + for (const auto& event_log : frame_log.messages) { + if (buffer.size() < kRtcpReceiverFrameLogMessageBlockSize) { + FinalizeReceiverLogPacket(space_for_header, + initial_buffer_size - buffer.size()); + return; + } + + uint16_t delay_delta_or_packet_id = 0; + if (event_log.type == StatisticsEvent::Type::kPacketReceived) { + delay_delta_or_packet_id = event_log.packet_id; + } else { + delay_delta_or_packet_id = + static_cast(to_milliseconds(event_log.delay).count()); + } + AppendField(delay_delta_or_packet_id, buffer); + + // The event type on the wire is a 4-bit field. + const auto wire_type = + static_cast(StatisticsEvent::ToWireType(event_log.type)); + const auto event_timestamp_delta_ms = + to_milliseconds(event_log.timestamp - frame_log.messages[0].timestamp) + .count(); + const uint16_t wire_timestamp = + static_cast(event_timestamp_delta_ms); + + AppendField((wire_type << 12) | (wire_timestamp & 0xFFF), + buffer); + } + } + + FinalizeReceiverLogPacket(space_for_header, + initial_buffer_size - buffer.size()); +} + +void CompoundRtcpBuilder::FinalizeReceiverLogPacket( + ByteBuffer& space_for_header, + size_t payload_size) { + RtcpCommonHeader header; + header.packet_type = RtcpPacketType::kApplicationDefined; + header.with.subtype = RtcpSubtype::kReceiverLog; + header.payload_size = payload_size; + header.AppendFields(space_for_header); + + logs_for_next_packet_.clear(); +} + } // namespace openscreen::cast diff --git a/cast/streaming/impl/compound_rtcp_builder.h b/cast/streaming/impl/compound_rtcp_builder.h index 209b427fe..34619af25 100644 --- a/cast/streaming/impl/compound_rtcp_builder.h +++ b/cast/streaming/impl/compound_rtcp_builder.h @@ -41,7 +41,7 @@ class RtcpSession; class CompoundRtcpBuilder { public: explicit CompoundRtcpBuilder(RtcpSession& session); - ~CompoundRtcpBuilder(); + virtual ~CompoundRtcpBuilder(); // Gets/Sets the checkpoint `frame_id` that will be included in built RTCP // packets. This value indicates to the sender that all of the packets for all @@ -87,6 +87,9 @@ class CompoundRtcpBuilder { void IncludeFeedbackInNextPacket(std::vector packet_nacks, std::vector frame_acks); + virtual void IncludeReceiverLogsInNextPacket( + std::vector logs); + // Builds a compound RTCP packet and returns the portion of the `buffer` that // was used. The buffer's size must be at least kRequiredBufferSize, but // should generally be the maximum packet size (see discussion in @@ -111,9 +114,12 @@ class CompoundRtcpBuilder { void AppendReceiverReferenceTimeReportPacket(Clock::time_point send_time, ByteBuffer& buffer); void AppendPictureLossIndicatorPacket(ByteBuffer& buffer); + void AppendReceiverLogPacket(ByteBuffer& buffer); void AppendCastFeedbackPacket(ByteBuffer& buffer); int AppendCastFeedbackLossFields(ByteBuffer& buffer); void AppendCastFeedbackAckFields(ByteBuffer& buffer); + void FinalizeReceiverLogPacket(ByteBuffer& space_for_header, + size_t payload_size); RtcpSession& session_; @@ -123,6 +129,7 @@ class CompoundRtcpBuilder { std::optional receiver_report_for_next_packet_; std::vector nacks_for_next_packet_; std::vector acks_for_next_packet_; + std::vector logs_for_next_packet_; bool picture_loss_indicator_ = false; // An 8-bit wrap-around counter that tracks how many times Cast Feedback has diff --git a/cast/streaming/impl/compound_rtcp_builder_unittest.cc b/cast/streaming/impl/compound_rtcp_builder_unittest.cc index 07e760461..33befca46 100644 --- a/cast/streaming/impl/compound_rtcp_builder_unittest.cc +++ b/cast/streaming/impl/compound_rtcp_builder_unittest.cc @@ -17,7 +17,6 @@ #include "util/chrono_helpers.h" using testing::_; -using testing::Invoke; using testing::Mock; using testing::SaveArg; using testing::StrictMock; @@ -298,12 +297,12 @@ TEST_F(CompoundRtcpBuilderTest, WithEverythingThatCanFit) { // No ACKs could be included. EXPECT_CALL(*(client()), OnReceiverHasFrames(_)).Times(0); EXPECT_CALL(*(client()), OnReceiverIsMissingPackets(_)) - .WillOnce(Invoke([&](std::vector parsed_nacks) { + .WillOnce([&](std::vector parsed_nacks) { // Some should be dropped. ASSERT_LT(parsed_nacks.size(), nacks.size()); EXPECT_TRUE(std::equal(parsed_nacks.begin(), parsed_nacks.end(), nacks.begin())); - })); + }); ASSERT_TRUE(parser()->Parse(packet, max_feedback_frame_id)); Mock::VerifyAndClearExpectations(client()); @@ -323,19 +322,19 @@ TEST_F(CompoundRtcpBuilderTest, WithEverythingThatCanFit) { EXPECT_CALL(*(client()), OnReceiverReferenceTimeAdvanced(_)); EXPECT_CALL(*(client()), OnReceiverCheckpoint(checkpoint, _)); EXPECT_CALL(*(client()), OnReceiverHasFrames(_)) - .WillOnce(Invoke([&](std::vector parsed_acks) { + .WillOnce([&](std::vector parsed_acks) { // Some of the ACKs should be dropped. ASSERT_LT(parsed_acks.size(), acks.size()); EXPECT_TRUE( std::equal(parsed_acks.begin(), parsed_acks.end(), acks.begin())); - })); + }); EXPECT_CALL(*(client()), OnReceiverIsMissingPackets(_)) - .WillOnce(Invoke([&](Span parsed_nacks) { + .WillOnce([&](Span parsed_nacks) { // All of the 48 NACKs provided should be present. ASSERT_EQ(kFewerNackCount, static_cast(parsed_nacks.size())); EXPECT_TRUE(std::equal(parsed_nacks.begin(), parsed_nacks.end(), nacks.begin())); - })); + }); ASSERT_TRUE(parser()->Parse(second_packet, max_feedback_frame_id)); Mock::VerifyAndClearExpectations(client()); @@ -351,17 +350,17 @@ TEST_F(CompoundRtcpBuilderTest, WithEverythingThatCanFit) { EXPECT_CALL(*(client()), OnReceiverReferenceTimeAdvanced(_)); EXPECT_CALL(*(client()), OnReceiverCheckpoint(checkpoint, _)); EXPECT_CALL(*(client()), OnReceiverHasFrames(_)) - .WillOnce(Invoke([&](std::vector parsed_acks) { + .WillOnce([&](std::vector parsed_acks) { // All acks should be present. EXPECT_EQ(acks, parsed_acks); - })); + }); EXPECT_CALL(*(client()), OnReceiverIsMissingPackets(_)) - .WillOnce(Invoke([&](Span parsed_nacks) { + .WillOnce([&](Span parsed_nacks) { // Only the first 46 NACKs provided should be present. ASSERT_EQ(kEvenFewerNackCount, static_cast(parsed_nacks.size())); EXPECT_TRUE(std::equal(parsed_nacks.begin(), parsed_nacks.end(), nacks.begin())); - })); + }); ASSERT_TRUE(parser()->Parse(third_packet, max_feedback_frame_id)); Mock::VerifyAndClearExpectations(client()); } diff --git a/cast/streaming/impl/compound_rtcp_parser.cc b/cast/streaming/impl/compound_rtcp_parser.cc index 4152c2260..c3b01ab46 100644 --- a/cast/streaming/impl/compound_rtcp_parser.cc +++ b/cast/streaming/impl/compound_rtcp_parser.cc @@ -9,6 +9,7 @@ #include "cast/streaming/impl/packet_util.h" #include "cast/streaming/impl/rtcp_session.h" +#include "cast/streaming/impl/statistics_common.h" #include "util/chrono_helpers.h" #include "util/osp_logging.h" #include "util/std_util.h" @@ -21,8 +22,6 @@ namespace { // time) to represent unset time_point values. constexpr auto kNullTimePoint = Clock::time_point::min(); -constexpr uint32_t kCastName = ('C' << 24) + ('A' << 16) + ('S' << 8) + 'T'; - // Some receivers send time sync requests (that we ignore). constexpr uint32_t kTimeSyncRequestName = ('T' << 24) + ('I' << 16) + ('M' << 8) + 'E'; @@ -80,40 +79,6 @@ void CanonicalizePacketNackVector(std::vector* packets) { } } -// TODO(issuetracker.google.com/298085631): implement the serialization of -// StatisticsEventType to wire type as part of implementing receiver side event -// generation. -// NOTE: the legacy mappings, like AudioAckSent below, may still be in use -// on some legacy receivers. -StatisticsEventType ToEventTypeFromWire(uint8_t wire_event) { - switch (wire_event) { - case 1: // AudioAckSent - case 5: // VideoAckSent - case 11: // Unified - return StatisticsEventType::kFrameAckSent; - - case 2: // AudioPlayoutDelay - case 7: // VideoRenderDelay - case 12: // Unified - return StatisticsEventType::kFramePlayedOut; - - case 3: // AudioFrameDecoded - case 6: // VideoFrameDecoded - case 13: // Unified - return StatisticsEventType::kFrameDecoded; - - case 4: // AudioPacketReceived - case 8: // VideoPacketReceived - case 14: // Unified - return StatisticsEventType::kPacketReceived; - - default: - OSP_VLOG << "Unexpected RTCP log message received: " - << static_cast(wire_event); - return StatisticsEventType::kUnknown; - } -} - } // namespace CompoundRtcpParser::CompoundRtcpParser(RtcpSession& session, @@ -145,12 +110,12 @@ bool CompoundRtcpParser::Parse(ByteView buffer, FrameId max_feedback_frame_id) { if (!header) { return false; } - buffer.remove_prefix(kRtcpCommonHeaderSize); + buffer = buffer.subspan(kRtcpCommonHeaderSize); if (static_cast(buffer.size()) < header->payload_size) { return false; } ByteView payload = buffer.subspan(0, header->payload_size); - buffer.remove_prefix(header->payload_size); + buffer = buffer.subspan(header->payload_size); switch (header->packet_type) { case RtcpPacketType::kReceiverReport: @@ -317,9 +282,10 @@ bool CompoundRtcpParser::ParseFrameLogMessages( ConsumeField(in); // Skip unknown event types, they are not useful. - const StatisticsEventType event_type = ToEventTypeFromWire( - static_cast(event_type_and_timestamp_delta >> 12)); - if (event_type == StatisticsEventType::kUnknown) { + const auto event_type = + StatisticsEvent::FromWireType(static_cast( + event_type_and_timestamp_delta >> 12)); + if (event_type == StatisticsEvent::Type::kUnknown) { continue; } @@ -327,7 +293,7 @@ bool CompoundRtcpParser::ParseFrameLogMessages( .type = event_type, .timestamp = event_timestamp_base + milliseconds(event_type_and_timestamp_delta & 0xFFF)}; - if (event_type == StatisticsEventType::kPacketReceived) { + if (event_type == StatisticsEvent::Type::kPacketReceived) { event_log.packet_id = delay_delta_or_packet_id; } else { event_log.delay = @@ -412,7 +378,7 @@ bool CompoundRtcpParser::ParseFeedback(ByteView in, } // Skip over the "Feedback Count" field. It's currently unused, though it // might be useful for event tracing later... - in.remove_prefix(sizeof(uint8_t)); + in = in.subspan(sizeof(uint8_t)); const int ack_bitvector_octet_count = ConsumeField(in); if (static_cast(in.size()) < ack_bitvector_octet_count) { return false; @@ -454,7 +420,7 @@ bool CompoundRtcpParser::ParseExtendedReports( return false; } const uint8_t block_type = ConsumeField(in); - in.remove_prefix(sizeof(uint8_t)); // Skip the "reserved" byte. + in = in.subspan(sizeof(uint8_t)); // Skip the "reserved" byte. const int block_data_size = static_cast(ConsumeField(in)) * 4; if (static_cast(in.size()) < block_data_size) { @@ -469,7 +435,7 @@ bool CompoundRtcpParser::ParseExtendedReports( } else { // Ignore any other type of extended report. } - in.remove_prefix(block_data_size); + in = in.subspan(block_data_size); } return true; diff --git a/cast/streaming/impl/compound_rtcp_parser_fuzzer.cc b/cast/streaming/impl/compound_rtcp_parser_fuzzer.cc index ce3db8f5a..ec507a7cc 100644 --- a/cast/streaming/impl/compound_rtcp_parser_fuzzer.cc +++ b/cast/streaming/impl/compound_rtcp_parser_fuzzer.cc @@ -28,16 +28,16 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { // also contains a NtpTimeConverter, which samples the system clock at // construction time. There is no reason to re-construct these objects for // each fuzzer test input. -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wexit-time-destructors" - static RtcpSession session(kSenderSsrcInSeedCorpus, kReceiverSsrcInSeedCorpus, - openscreen::Clock::time_point{}); - static ClientThatIgnoresEverything client_that_ignores_everything; - static CompoundRtcpParser parser(session, client_that_ignores_everything); -#pragma clang diagnostic pop - - const auto max_feedback_frame_id = FrameId::first() + 100; - parser.Parse(openscreen::ByteView(data, size), max_feedback_frame_id); + static auto* session = + new RtcpSession(kSenderSsrcInSeedCorpus, kReceiverSsrcInSeedCorpus, + openscreen::Clock::time_point{}); + static auto* client_that_ignores_everything = + new ClientThatIgnoresEverything(); + static auto* parser = + new CompoundRtcpParser(*session, *client_that_ignores_everything); + + static constexpr auto kMaxFeedbackFrameId = FrameId::first() + 100; + parser->Parse(openscreen::ByteView(data, size), kMaxFeedbackFrameId); return 0; } diff --git a/cast/streaming/impl/compound_rtcp_parser_unittest.cc b/cast/streaming/impl/compound_rtcp_parser_unittest.cc index 8195c2afa..a902c6f80 100644 --- a/cast/streaming/impl/compound_rtcp_parser_unittest.cc +++ b/cast/streaming/impl/compound_rtcp_parser_unittest.cc @@ -16,7 +16,6 @@ #include "util/chrono_helpers.h" using testing::_; -using testing::Invoke; using testing::Mock; using testing::SaveArg; using testing::StrictMock; @@ -176,7 +175,7 @@ TEST_F(CompoundRtcpParserTest, OnCastReceiverFrameLogMessages_ValidPacket) { EXPECT_EQ(1u, messages[0].messages.size()); const RtcpReceiverEventLogMessage& log = messages[0].messages[0]; - EXPECT_EQ(StatisticsEventType::kPacketReceived, log.type); + EXPECT_EQ(StatisticsEvent::Type::kPacketReceived, log.type); EXPECT_EQ(session()->start_time() + microseconds{1057321000}, log.timestamp); EXPECT_EQ(milliseconds{}, log.delay); EXPECT_EQ(FramePacketId{7701}, log.packet_id); @@ -222,14 +221,14 @@ TEST_F(CompoundRtcpParserTest, // Note: the first log message is removed due to it being an invalid type. const RtcpReceiverEventLogMessage& second_log = first_message.messages[0]; - EXPECT_EQ(StatisticsEventType::kPacketReceived, second_log.type); + EXPECT_EQ(StatisticsEvent::Type::kPacketReceived, second_log.type); EXPECT_EQ(session()->start_time() + microseconds{1057097000}, second_log.timestamp); EXPECT_EQ(milliseconds{}, second_log.delay); EXPECT_EQ(FramePacketId{277}, second_log.packet_id); const RtcpReceiverEventLogMessage& third_log = first_message.messages[1]; - EXPECT_EQ(StatisticsEventType::kFramePlayedOut, third_log.type); + EXPECT_EQ(StatisticsEvent::Type::kFramePlayedOut, third_log.type); EXPECT_EQ(session()->start_time() + microseconds{1057367000}, third_log.timestamp); EXPECT_EQ(milliseconds{535}, third_log.delay); @@ -242,7 +241,7 @@ TEST_F(CompoundRtcpParserTest, const RtcpReceiverEventLogMessage& second_first_log = second_message.messages[0]; - EXPECT_EQ(StatisticsEventType::kPacketReceived, second_first_log.type); + EXPECT_EQ(StatisticsEvent::Type::kPacketReceived, second_first_log.type); EXPECT_EQ(session()->start_time() + microseconds{4203049000}, second_first_log.timestamp); EXPECT_EQ(milliseconds{}, second_first_log.delay); diff --git a/cast/streaming/impl/frame_collector.cc b/cast/streaming/impl/frame_collector.cc index 53bb4af3b..4e7bc594c 100644 --- a/cast/streaming/impl/frame_collector.cc +++ b/cast/streaming/impl/frame_collector.cc @@ -104,7 +104,7 @@ bool FrameCollector::CollectRtpPacket(const RtpPacketParser::ParseResult& part, void FrameCollector::GetMissingPackets(std::vector* nacks) const { OSP_CHECK(!frame_.frame_id.is_null()); - if (num_missing_packets_ == 0) { + if (is_complete()) { return; } @@ -122,34 +122,33 @@ void FrameCollector::GetMissingPackets(std::vector* nacks) const { } } -const EncryptedFrame& FrameCollector::PeekAtAssembledFrame() { - OSP_CHECK_EQ(num_missing_packets_, 0); - - if (!frame_.data.data()) { - // Allocate the frame's payload buffer once, right-sized to the sum of all - // chunk sizes. - frame_.owned_data_.reserve( - std::accumulate(chunks_.cbegin(), chunks_.cend(), size_t{0}, - [](size_t num_bytes_so_far, const PayloadChunk& chunk) { - return num_bytes_so_far + chunk.payload.size(); - })); - // Now, populate the frame's payload buffer with each chunk of data. - for (const PayloadChunk& chunk : chunks_) { - frame_.owned_data_.insert(frame_.owned_data_.end(), chunk.payload.begin(), - chunk.payload.end()); - } - frame_.data = frame_.owned_data_; - } - +const EncodedFrame& FrameCollector::PeekFrameMetadata() const { + OSP_CHECK(is_complete()); return frame_; } +size_t FrameCollector::GetFramePayloadSize() const { + OSP_CHECK(is_complete()); + return std::accumulate( + chunks_.cbegin(), chunks_.cend(), size_t{0}, + [](size_t num_bytes_so_far, const PayloadChunk& chunk) { + return num_bytes_so_far + chunk.payload.size(); + }); +} + +std::vector FrameCollector::GetPayloadChunks() const { + OSP_CHECK(is_complete()); + std::vector result; + result.reserve(chunks_.size()); + for (const PayloadChunk& chunk : chunks_) { + result.push_back(chunk.payload); + } + return result; +} + void FrameCollector::Reset() { num_missing_packets_ = kUnknownNumberOfPackets; - frame_.frame_id = FrameId(); - frame_.owned_data_.clear(); - frame_.owned_data_.shrink_to_fit(); - frame_.data = ByteView(); + frame_ = EncodedFrame(); chunks_.clear(); } diff --git a/cast/streaming/impl/frame_collector.h b/cast/streaming/impl/frame_collector.h index 2298970a0..cff569fe1 100644 --- a/cast/streaming/impl/frame_collector.h +++ b/cast/streaming/impl/frame_collector.h @@ -45,14 +45,20 @@ class FrameCollector { // packet ID. void GetMissingPackets(std::vector* nacks) const; - // Returns a read-only reference to the completely-collected frame, assembling - // it if necessary. The caller should reset the FrameCollector (see Reset() - // below) to free-up memory once it has finished reading from the returned - // frame. - // + // Returns the metadata for the completely-collected frame. // Precondition: is_complete() must return true before this method can be // called. - const EncryptedFrame& PeekAtAssembledFrame(); + const EncodedFrame& PeekFrameMetadata() const; + + // Returns the total size of the payload for the frame. + // Precondition: is_complete() must return true before this method can be + // called. + size_t GetFramePayloadSize() const; + + // Returns the collected payload chunks in order. + // Precondition: is_complete() must return true before this method can be + // called. + std::vector GetPayloadChunks() const; // Resets the FrameCollector back to its initial state, freeing-up memory. void Reset(); @@ -68,10 +74,8 @@ class FrameCollector { bool has_data() const { return !!payload.data(); } }; - // Storage for frame metadata and data. Once the frame has been completely - // collected and assembled, `frame_.data` is set to non-null, and this is - // exposed externally (read-only). - EncryptedFrame frame_; + // Storage for frame metadata. + EncodedFrame frame_; // The number of packets needed to complete the frame, or the maximum int if // this is not yet known. diff --git a/cast/streaming/impl/frame_collector_unittest.cc b/cast/streaming/impl/frame_collector_unittest.cc index ae957474a..f32054e4e 100644 --- a/cast/streaming/impl/frame_collector_unittest.cc +++ b/cast/streaming/impl/frame_collector_unittest.cc @@ -75,7 +75,7 @@ TEST(FrameCollectorTest, CollectsFrameWithOnlyOnePart) { // Examine the assembled frame, and confirm its metadata and payload match // what was put into the collector via the packet above. - const auto& frame = collector.PeekAtAssembledFrame(); + const auto& frame = collector.PeekFrameMetadata(); if (i == 0) { EXPECT_EQ(EncodedFrame::Dependency::kKeyFrame, frame.dependency); EXPECT_EQ(std::chrono::milliseconds(), frame.new_playout_delay); @@ -86,8 +86,12 @@ TEST(FrameCollectorTest, CollectsFrameWithOnlyOnePart) { EXPECT_EQ(part.frame_id, frame.frame_id); EXPECT_EQ(kSomeFrameId, frame.referenced_frame_id); EXPECT_EQ(part.rtp_timestamp, frame.rtp_timestamp); + + auto chunks = collector.GetPayloadChunks(); + ASSERT_EQ(1u, chunks.size()); + ByteView chunk_data = chunks[0]; for (int j = 0; j < 255; ++j) { - EXPECT_EQ(static_cast(j), frame.data[j]); + EXPECT_EQ(static_cast(j), chunk_data[j]); } collector.Reset(); @@ -151,20 +155,19 @@ TEST(FrameCollectorTest, CollectsFrameWithMultiplePartsArrivingOutOfOrder) { // Examine the assembled frame, and confirm its metadata and payload match // what was put into the collector via the packets above, and that the payload // bytes are in-order. - const auto& frame = collector.PeekAtAssembledFrame(); + const auto& frame = collector.PeekFrameMetadata(); EXPECT_EQ(EncodedFrame::Dependency::kKeyFrame, frame.dependency); EXPECT_EQ(kSomeFrameId, frame.frame_id); EXPECT_EQ(kSomeFrameId, frame.referenced_frame_id); EXPECT_EQ(kSomeRtpTimestamp, frame.rtp_timestamp); - ByteView remaining_data = frame.data; + + auto chunks = collector.GetPayloadChunks(); + ASSERT_EQ(6u, chunks.size()); for (int i = 0; i < 6; ++i) { - ASSERT_LE(kPayloadSizes[i], static_cast(remaining_data.size())); - EXPECT_THAT(remaining_data.subspan(0, kPayloadSizes[i]), - ElementsAreArray(payloads[i])) - << "i=" << i; - remaining_data.remove_prefix(kPayloadSizes[i]); + ByteView chunk_data = chunks[i]; + ASSERT_EQ(kPayloadSizes[i], static_cast(chunk_data.size())); + EXPECT_THAT(chunk_data, ElementsAreArray(payloads[i])) << "i=" << i; } - ASSERT_TRUE(remaining_data.empty()); } TEST(FrameCollectorTest, RejectsInvalidParts) { diff --git a/cast/streaming/impl/frame_crypto.cc b/cast/streaming/impl/frame_crypto.cc index a0c06bb4b..37f64c7ba 100644 --- a/cast/streaming/impl/frame_crypto.cc +++ b/cast/streaming/impl/frame_crypto.cc @@ -65,27 +65,24 @@ EncryptedFrame FrameCrypto::Encrypt(const EncodedFrame& encoded_frame) const { encoded_frame.CopyMetadataTo(&result); result.owned_data_.resize(encoded_frame.data.size()); result.data = result.owned_data_; - EncryptCommon(encoded_frame.frame_id, encoded_frame.data, result.owned_data_); + Crypt(encoded_frame.frame_id, {&encoded_frame.data, 1}, result.owned_data_); return result; } -void FrameCrypto::Decrypt(const EncryptedFrame& encrypted_frame, +void FrameCrypto::Decrypt(FrameId frame_id, + ChunkList chunks, ByteBuffer out) const { - // AES-CTC is symmetric. Thus, decryption back to the plaintext is the same as - // encrypting the ciphertext; and both are the same size. - OSP_CHECK_EQ(encrypted_frame.data.size(), out.size()); - EncryptCommon(encrypted_frame.frame_id, encrypted_frame.data, out); + Crypt(frame_id, chunks, out); } -void FrameCrypto::EncryptCommon(FrameId frame_id, - ByteView in, - ByteBuffer out) const { +void FrameCrypto::Crypt(FrameId frame_id, + ChunkList chunks, + ByteBuffer out) const { OSP_CHECK(!frame_id.is_null()); - OSP_CHECK_EQ(in.size(), out.size()); // Compute the AES nonce for Cast Streaming payload encryption, which is based // on the `frame_id`. - std::array aes_nonce{/* zero initialized */}; + std::array aes_nonce{}; static_assert(AES_BLOCK_SIZE == sizeof(aes_nonce), "AES_BLOCK_SIZE is not 16 bytes."); WriteBigEndian(frame_id.lower_32_bits(), aes_nonce.data() + 8); @@ -93,10 +90,17 @@ void FrameCrypto::EncryptCommon(FrameId frame_id, aes_nonce[i] ^= cast_iv_mask_[i]; } - std::array ecount_buf{/* zero initialized */}; + std::array ecount_buf{}; unsigned int block_offset = 0; - AES_ctr128_encrypt(in.data(), out.data(), in.size(), &aes_key_, - aes_nonce.data(), ecount_buf.data(), &block_offset); + size_t out_offset = 0; + for (ByteView chunk : chunks) { + OSP_CHECK_LE(out_offset + chunk.size(), out.size()); + AES_ctr128_encrypt(chunk.data(), out.data() + out_offset, chunk.size(), + &aes_key_, aes_nonce.data(), ecount_buf.data(), + &block_offset); + out_offset += chunk.size(); + } + OSP_CHECK_EQ(out_offset, out.size()); } } // namespace openscreen::cast diff --git a/cast/streaming/impl/frame_crypto.h b/cast/streaming/impl/frame_crypto.h index 5b9f809ff..4804d5fac 100644 --- a/cast/streaming/impl/frame_crypto.h +++ b/cast/streaming/impl/frame_crypto.h @@ -13,12 +13,10 @@ #include "cast/streaming/public/encoded_frame.h" #include "openssl/aes.h" -#include "platform/base/macros.h" #include "platform/base/span.h" namespace openscreen::cast { -class FrameCollector; class FrameCrypto; // A subclass of EncodedFrame that represents an EncodedFrame with encrypted @@ -32,9 +30,8 @@ struct EncryptedFrame : public EncodedFrame { EncryptedFrame& operator=(EncryptedFrame&&); protected: - // Since only FrameCrypto and FrameCollector are trusted to generate the - // payload data, only they are allowed direct access to the storage. - friend class FrameCollector; + // Since only FrameCrypto is trusted to generate the + // payload data, it is allowed direct access to the storage. friend class FrameCrypto; // Note: EncodedFrame::data must be updated whenever any mutations are @@ -46,6 +43,8 @@ struct EncryptedFrame : public EncodedFrame { // been received. class FrameCrypto { public: + using ChunkList = std::span; + // Construct with the given 16-bytes AES key and IV mask. Both arguments // should be randomly-generated for each new streaming session. // GenerateRandomBytes() can be used to create them. @@ -56,18 +55,9 @@ class FrameCrypto { EncryptedFrame Encrypt(const EncodedFrame& encoded_frame) const; - // Decrypts `encrypted_frame` into `out`. `out` must have a sufficiently-sized - // data buffer (see GetPlaintextSize()). - void Decrypt(const EncryptedFrame& encrypted_frame, ByteBuffer out) const; - - // AES crypto inputs and outputs (for either encrypting or decrypting) are - // always the same size in bytes. The following are just "documentative code." - static int GetEncryptedSize(const EncodedFrame& encoded_frame) { - return encoded_frame.data.size(); - } - static int GetPlaintextSize(const EncryptedFrame& encrypted_frame) { - return encrypted_frame.data.size(); - } + // Decrypts `chunks` into `out`. `out` must have a sufficiently-sized + // data buffer. + void Decrypt(FrameId frame_id, ChunkList chunks, ByteBuffer out) const; private: // The 244-byte AES_KEY struct, derived from the `aes_key` passed to the ctor, @@ -80,7 +70,7 @@ class FrameCrypto { // AES-CTR is symmetric. Thus, the "meat" of both Encrypt() and Decrypt() is // the same. - void EncryptCommon(FrameId frame_id, ByteView in, ByteBuffer out) const; + void Crypt(FrameId frame_id, ChunkList chunks, ByteBuffer out) const; }; } // namespace openscreen::cast diff --git a/cast/streaming/impl/frame_crypto_unittest.cc b/cast/streaming/impl/frame_crypto_unittest.cc index 70fd26be4..81ba2a178 100644 --- a/cast/streaming/impl/frame_crypto_unittest.cc +++ b/cast/streaming/impl/frame_crypto_unittest.cc @@ -8,14 +8,17 @@ #include #include +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "platform/base/span.h" -#include "platform/test/byte_view_test_util.h" #include "util/crypto/random_bytes.h" namespace openscreen::cast { namespace { +using testing::ElementsAreArray; +using testing::Not; + TEST(FrameCryptoTest, EncryptsAndDecryptsFrames) { // Prepare two frames with different FrameIds, but having the same payload // bytes. @@ -39,36 +42,35 @@ TEST(FrameCryptoTest, EncryptsAndDecryptsFrames) { // the plaintext, and that both frames have different encrypted data. const EncryptedFrame encrypted_frame0 = crypto.Encrypt(frame0); EXPECT_EQ(frame0.frame_id, encrypted_frame0.frame_id); - ASSERT_EQ(static_cast(frame0.data.size()), - FrameCrypto::GetPlaintextSize(encrypted_frame0)); - ExpectByteViewsHaveDifferentBytes(frame0.data, encrypted_frame0.data); + ASSERT_EQ(frame0.data.size(), encrypted_frame0.data.size()); + EXPECT_THAT(frame0.data, Not(ElementsAreArray(encrypted_frame0.data))); + const EncryptedFrame encrypted_frame1 = crypto.Encrypt(frame1); EXPECT_EQ(frame1.frame_id, encrypted_frame1.frame_id); - ASSERT_EQ(static_cast(frame1.data.size()), - FrameCrypto::GetPlaintextSize(encrypted_frame1)); - ExpectByteViewsHaveDifferentBytes(frame1.data, encrypted_frame1.data); - ExpectByteViewsHaveDifferentBytes(encrypted_frame0.data, - encrypted_frame1.data); + ASSERT_EQ(frame1.data.size(), encrypted_frame1.data.size()); + EXPECT_THAT(frame1.data, Not(ElementsAreArray(encrypted_frame1.data))); + EXPECT_THAT(encrypted_frame0.data, + Not(ElementsAreArray(encrypted_frame1.data))); // Now, decrypt the encrypted frames, and confirm the original payload // plaintext is retrieved. - std::vector decrypted_frame0_buffer( - FrameCrypto::GetPlaintextSize(encrypted_frame0)); - crypto.Decrypt(encrypted_frame0, decrypted_frame0_buffer); + std::vector decrypted_frame0_buffer(encrypted_frame0.data.size()); + crypto.Decrypt(encrypted_frame0.frame_id, {&encrypted_frame0.data, 1}, + decrypted_frame0_buffer); EncodedFrame decrypted_frame0; encrypted_frame0.CopyMetadataTo(&decrypted_frame0); decrypted_frame0.data = decrypted_frame0_buffer; EXPECT_EQ(frame0.frame_id, decrypted_frame0.frame_id); - ExpectByteViewsHaveSameBytes(frame0.data, decrypted_frame0.data); + EXPECT_THAT(frame0.data, ElementsAreArray(decrypted_frame0.data)); - std::vector decrypted_frame1_buffer( - FrameCrypto::GetPlaintextSize(encrypted_frame1)); - crypto.Decrypt(encrypted_frame1, decrypted_frame1_buffer); + std::vector decrypted_frame1_buffer(encrypted_frame1.data.size()); + crypto.Decrypt(encrypted_frame1.frame_id, {&encrypted_frame1.data, 1}, + decrypted_frame1_buffer); EncodedFrame decrypted_frame1; encrypted_frame1.CopyMetadataTo(&decrypted_frame1); decrypted_frame1.data = decrypted_frame1_buffer; EXPECT_EQ(frame1.frame_id, decrypted_frame1.frame_id); - ExpectByteViewsHaveSameBytes(frame1.data, decrypted_frame1.data); + EXPECT_THAT(frame1.data, ElementsAreArray(decrypted_frame1.data)); } } // namespace diff --git a/cast/streaming/impl/message_constants.h b/cast/streaming/impl/message_constants.h new file mode 100644 index 000000000..8634add78 --- /dev/null +++ b/cast/streaming/impl/message_constants.h @@ -0,0 +1,15 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CAST_STREAMING_IMPL_MESSAGE_CONSTANTS_H_ +#define CAST_STREAMING_IMPL_MESSAGE_CONSTANTS_H_ + +namespace openscreen::cast { + +// RTP extension strings. +inline constexpr char kInputEventsRtpExtension[] = "input_events"; + +} // namespace openscreen::cast + +#endif // CAST_STREAMING_IMPL_MESSAGE_CONSTANTS_H_ diff --git a/cast/streaming/impl/offer_messages_unittest.cc b/cast/streaming/impl/offer_messages_unittest.cc index 1bfa83b8c..6feb1b377 100644 --- a/cast/streaming/impl/offer_messages_unittest.cc +++ b/cast/streaming/impl/offer_messages_unittest.cc @@ -107,11 +107,10 @@ void ExpectFailureOnParse(std::string_view body, ErrorOr root = json::Parse(body); ASSERT_TRUE(root.is_value()) << root.error(); - Offer offer; - Error error = Offer::TryParse(std::move(root.value()), &offer); - EXPECT_FALSE(error.ok()); + const auto offer_or_error = Offer::TryParse(std::move(root.value())); + EXPECT_TRUE(offer_or_error.is_error()); if (expected) { - EXPECT_EQ(expected, error.code()); + EXPECT_EQ(expected, offer_or_error.error().code()); } } @@ -226,6 +225,17 @@ TEST(OfferTest, ErrorOnEmptyOffer) { ExpectFailureOnParse("{}"); } +TEST(OfferTest, ErrorOnNonObjectOffer) { + Json::Value array_val(Json::arrayValue); + EXPECT_TRUE(Offer::TryParse(array_val).is_error()); + + Json::Value string_val("string"); + EXPECT_TRUE(Offer::TryParse(string_val).is_error()); + + Json::Value int_val(42); + EXPECT_TRUE(Offer::TryParse(int_val).is_error()); +} + TEST(OfferTest, ErrorOnMissingMandatoryFields) { // It's okay if castMode is omitted, but if supportedStreams is omitted we // should fail here. @@ -241,8 +251,8 @@ TEST(OfferTest, CanParseValidButStreamlessOffer) { })"); ASSERT_TRUE(root.is_value()) << root.error(); - Offer offer; - EXPECT_TRUE(Offer::TryParse(std::move(root.value()), &offer).ok()); + const auto offer_or_error = Offer::TryParse(std::move(root.value())); + EXPECT_TRUE(offer_or_error.is_value()); } TEST(OfferTest, ErrorOnMissingAudioStreamMandatoryField) { @@ -291,8 +301,8 @@ TEST(OfferTest, CanParseValidButMinimalAudioOffer) { }] })"); ASSERT_TRUE(root.is_value()); - Offer offer; - EXPECT_TRUE(Offer::TryParse(std::move(root.value()), &offer).ok()); + const auto offer_or_error = Offer::TryParse(root.value()); + EXPECT_TRUE(offer_or_error.is_value()); } TEST(OfferTest, CanParseValidZeroBitRateAudioOffer) { @@ -313,8 +323,8 @@ TEST(OfferTest, CanParseValidZeroBitRateAudioOffer) { }] })"); ASSERT_TRUE(root.is_value()) << root.error(); - Offer offer; - EXPECT_TRUE(Offer::TryParse(std::move(root.value()), &offer).ok()); + const auto offer_or_error = Offer::TryParse(root.value()); + EXPECT_TRUE(offer_or_error.is_value()); } TEST(OfferTest, ErrorOnInvalidRtpTimebase) { @@ -556,29 +566,29 @@ TEST(OfferTest, CanParseValidButMinimalVideoOffer) { })"); ASSERT_TRUE(root.is_value()); - Offer offer; - EXPECT_TRUE(Offer::TryParse(std::move(root.value()), &offer).ok()); + const auto offer_or_error = Offer::TryParse(root.value()); + EXPECT_TRUE(offer_or_error.is_value()); } TEST(OfferTest, CanParseValidOffer) { ErrorOr root = json::Parse(kValidOffer); ASSERT_TRUE(root.is_value()); - Offer offer; - EXPECT_TRUE(Offer::TryParse(std::move(root.value()), &offer).ok()); + const auto offer_or_error = Offer::TryParse(root.value()); + ASSERT_TRUE(offer_or_error.is_value()); - ExpectEqualsValidOffer(offer); + ExpectEqualsValidOffer(offer_or_error.value()); } TEST(OfferTest, ParseAndToJsonResultsInSameOffer) { ErrorOr root = json::Parse(kValidOffer); ASSERT_TRUE(root.is_value()); - Offer offer; - EXPECT_TRUE(Offer::TryParse(std::move(root.value()), &offer).ok()); - ExpectEqualsValidOffer(offer); + const auto offer_or_error = Offer::TryParse(root.value()); + ASSERT_TRUE(offer_or_error.is_value()); + ExpectEqualsValidOffer(offer_or_error.value()); - Offer reparsed_offer; - EXPECT_TRUE(Offer::TryParse(std::move(root.value()), &reparsed_offer).ok()); - ExpectEqualsValidOffer(reparsed_offer); + const auto reparsed_offer_or_error = Offer::TryParse(root.value()); + ASSERT_TRUE(reparsed_offer_or_error.is_value()); + ExpectEqualsValidOffer(reparsed_offer_or_error.value()); } // We don't want to enforce that a given offer must have both audio and @@ -586,10 +596,10 @@ TEST(OfferTest, ParseAndToJsonResultsInSameOffer) { TEST(OfferTest, IsValidWithMissingStreams) { ErrorOr root = json::Parse(kValidOffer); ASSERT_TRUE(root.is_value()); - Offer offer; - EXPECT_TRUE(Offer::TryParse(std::move(root.value()), &offer).ok()); - ExpectEqualsValidOffer(offer); - const Offer valid_offer = std::move(offer); + const auto offer_or_error = Offer::TryParse(root.value()); + ASSERT_TRUE(offer_or_error.is_value()); + ExpectEqualsValidOffer(offer_or_error.value()); + const Offer valid_offer = std::move(offer_or_error.value()); Offer missing_audio_streams = valid_offer; missing_audio_streams.audio_streams.clear(); @@ -603,17 +613,17 @@ TEST(OfferTest, IsValidWithMissingStreams) { TEST(OfferTest, InvalidIfInvalidStreams) { ErrorOr root = json::Parse(kValidOffer); ASSERT_TRUE(root.is_value()); - Offer offer; - EXPECT_TRUE(Offer::TryParse(std::move(root.value()), &offer).ok()); - ExpectEqualsValidOffer(offer); + const auto offer_or_error = Offer::TryParse(root.value()); + ASSERT_TRUE(offer_or_error.is_value()); + ExpectEqualsValidOffer(offer_or_error.value()); - Offer video_stream_invalid = offer; + Offer video_stream_invalid = offer_or_error.value(); video_stream_invalid.video_streams[0].max_frame_rate = SimpleFraction{1, 0}; EXPECT_FALSE(video_stream_invalid.IsValid()); - Offer audio_stream_invalid = offer; - video_stream_invalid.audio_streams[0].bit_rate = 0; - EXPECT_FALSE(video_stream_invalid.IsValid()); + Offer audio_stream_invalid = offer_or_error.value(); + audio_stream_invalid.audio_streams[0].bit_rate = -1; + EXPECT_FALSE(audio_stream_invalid.IsValid()); } TEST(OfferTest, FailsIfUnencrypted) { @@ -707,4 +717,41 @@ TEST(OfferTest, FailsIfUnencrypted) { Error::Code::kUnencryptedOffer); } +TEST(OfferTest, ErrorOnMixedDscpValues) { + ExpectFailureOnParse(R"({ + "castMode": "mirroring", + "supportedStreams": [ + { + "index": 0, + "type": "video_source", + "codecName": "h264", + "rtpProfile": "cast", + "rtpPayloadType": 101, + "ssrc": 19088743, + "maxFrameRate": "60000/1000", + "timeBase": "1/90000", + "maxBitRate": 5000000, + "aesKey": "040d756791711fd3adb939066e6d8690", + "aesIvMask": "9ff0f022a959150e70a2d05a6c184aed", + "receiverRtcpDscp": 10 + }, + { + "index": 2, + "type": "audio_source", + "codecName": "opus", + "rtpProfile": "cast", + "rtpPayloadType": 96, + "ssrc": 4294967295, + "bitRate": 124000, + "timeBase": "1/48000", + "channels": 2, + "aesKey": "51027e4e2347cbcb49d57ef10177aebc", + "aesIvMask": "7f12a19be62a36c04ae4116caaeff6d1", + "receiverRtcpDscp": 20 + } + ] + })", + Error::Code::kJsonParseError); +} + } // namespace openscreen::cast diff --git a/cast/streaming/impl/packet_util.h b/cast/streaming/impl/packet_util.h index d27cf2b7b..dd6fe62cc 100644 --- a/cast/streaming/impl/packet_util.h +++ b/cast/streaming/impl/packet_util.h @@ -18,7 +18,7 @@ namespace openscreen::cast { template inline Integer ConsumeField(ByteView& in) { const Integer result = ReadBigEndian(in.data()); - in.remove_prefix(sizeof(Integer)); + in = in.subspan(sizeof(Integer)); return result; } @@ -27,7 +27,7 @@ inline Integer ConsumeField(ByteView& in) { template inline void AppendField(Integer value, ByteBuffer& out) { WriteBigEndian(value, out.data()); - out.remove_prefix(sizeof(Integer)); + out = out.subspan(sizeof(Integer)); } // Returns a bitmask for a field having the given number of bits. For example, @@ -41,7 +41,7 @@ constexpr Integer FieldBitmask(unsigned field_size_in_bits) { // reserved space. inline ByteBuffer ReserveSpace(int num_bytes, ByteBuffer& out) { const ByteBuffer reserved = out.subspan(0, num_bytes); - out.remove_prefix(num_bytes); + out = out.subspan(num_bytes); return reserved; } diff --git a/cast/streaming/impl/protobuf_messenger_unittest.cc b/cast/streaming/impl/protobuf_messenger_unittest.cc new file mode 100644 index 000000000..675e67de5 --- /dev/null +++ b/cast/streaming/impl/protobuf_messenger_unittest.cc @@ -0,0 +1,83 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cast/streaming/public/protobuf_messenger.h" + +#include +#include + +#include "cast/streaming/input.pb.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace openscreen::cast { + +using ::testing::_; + +TEST(ProtobufMessengerTest, ProcessesMessageFromRemote) { + bool called = false; + ProtobufMessenger messenger( + [](std::vector message) {}, + [&called](std::unique_ptr received) { + called = true; + EXPECT_EQ(received->events_size(), 1); + EXPECT_EQ(received->events(0).type(), + InputMessage::INPUT_TYPE_KEY_DOWN); + EXPECT_EQ(received->events(0).key_event().key_value(), "a"); + }); + + InputMessage message; + auto* event = message.add_events(); + event->set_type(InputMessage::INPUT_TYPE_KEY_DOWN); + auto* timestamp = event->mutable_timestamp(); + timestamp->set_seconds(1234); + timestamp->set_nanos(500000000); + auto* key_event = event->mutable_key_event(); + key_event->set_key_code("KeyA"); + key_event->set_key_value("a"); + + std::vector serialized(message.ByteSizeLong()); + message.SerializeToArray(serialized.data(), serialized.size()); + + messenger.ProcessMessageFromRemote(serialized); + EXPECT_TRUE(called); +} + +TEST(ProtobufMessengerTest, SendsMessageToRemote) { + std::vector captured_message; + ProtobufMessenger messenger( + [&captured_message](std::vector message) { + captured_message = std::move(message); + }); + + InputMessage message; + auto* event = message.add_events(); + event->set_type(InputMessage::INPUT_TYPE_MOUSE_MOVE); + auto* mouse_event = event->mutable_mouse_event(); + mouse_event->mutable_location()->set_x(0.5f); + mouse_event->mutable_location()->set_y(0.5f); + + messenger.SendMessageToRemote(message); + + ASSERT_FALSE(captured_message.empty()); + InputMessage parsed; + ASSERT_TRUE( + parsed.ParseFromArray(captured_message.data(), captured_message.size())); + EXPECT_EQ(parsed.events_size(), 1); + EXPECT_EQ(parsed.events(0).type(), InputMessage::INPUT_TYPE_MOUSE_MOVE); + EXPECT_EQ(parsed.events(0).mouse_event().location().x(), 0.5f); +} + +TEST(ProtobufMessengerTest, HandlesInvalidData) { + bool called = false; + ProtobufMessenger messenger( + [](std::vector message) {}, + [&called](std::unique_ptr received) { called = true; }); + + uint8_t invalid_data[] = {0xff, 0x00, 0xff}; + messenger.ProcessMessageFromRemote(invalid_data); + EXPECT_FALSE(called); +} + +} // namespace openscreen::cast diff --git a/cast/streaming/impl/receiver_base.cc b/cast/streaming/impl/receiver_base.cc deleted file mode 100644 index 9eb0f204c..000000000 --- a/cast/streaming/impl/receiver_base.cc +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2021 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "cast/streaming/impl/receiver_base.h" - -namespace openscreen::cast { - -ReceiverBase::Consumer::~Consumer() = default; - -ReceiverBase::ReceiverBase() = default; - -ReceiverBase::~ReceiverBase() = default; - -} // namespace openscreen::cast diff --git a/cast/streaming/impl/receiver_base.h b/cast/streaming/impl/receiver_base.h deleted file mode 100644 index 0105fe3ae..000000000 --- a/cast/streaming/impl/receiver_base.h +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2021 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CAST_STREAMING_IMPL_RECEIVER_BASE_H_ -#define CAST_STREAMING_IMPL_RECEIVER_BASE_H_ - -#include - -#include "cast/streaming/impl/session_config.h" -#include "cast/streaming/public/encoded_frame.h" -#include "cast/streaming/ssrc.h" -#include "platform/api/time.h" -#include "platform/base/span.h" - -namespace openscreen::cast { - -// The Cast Streaming Receiver, a peer corresponding to some Cast Streaming -// Sender at the other end of a network link. -// -// Cast Streaming is a transport protocol which divides up the frames for one -// media stream (e.g., audio or video) into multiple RTP packets containing an -// encrypted payload. The Receiver is the peer responsible for collecting the -// RTP packets, decrypting the payload, and re-assembling a frame that can be -// passed to a decoder and played out. -// -// A Sender ↔ Receiver pair is used to transport each media stream. Typically, -// there are two pairs in a normal system, one for the audio stream and one for -// video stream. A local player is responsible for synchronizing the playout of -// the frames of each stream to achieve lip-sync. See the discussion in -// encoded_frame.h for how the `reference_time` and `rtp_timestamp` of the -// EncodedFrames are used to achieve this. -class ReceiverBase { - public: - class Consumer { - public: - virtual ~Consumer(); - - // Called whenever one or more frames have become ready for consumption. The - // `next_frame_buffer_size` argument is identical to the result of calling - // AdvanceToNextFrame(), and so the Consumer only needs to prepare a buffer - // and call ConsumeNextFrame(). It may then call AdvanceToNextFrame() to - // check whether there are any more frames ready, but this is not mandatory. - // See usage example in SDLPlayerBase::OnFramesReady. - virtual void OnFramesReady(int next_frame_buffer_size) = 0; - }; - - ReceiverBase(); - virtual ~ReceiverBase(); - - virtual const SessionConfig& config() const = 0; - virtual int rtp_timebase() const = 0; - virtual Ssrc ssrc() const = 0; - - // Set the Consumer receiving notifications when new frames are ready for - // consumption. Frames received before this method is called will remain in - // the queue indefinitely. - virtual void SetConsumer(Consumer* consumer) = 0; - - // Sets how much time the consumer will need to decode/buffer/render/etc., and - // otherwise fully process a frame for on-time playback. This information is - // used by the Receiver to decide whether to skip past frames that have - // arrived too late. This method can be called repeatedly to make adjustments - // based on changing environmental conditions. It is HIGHLY recommended that - // consumers of this API provide a proper processing time, otherwise there - // may be significantly larger playout delays. - // - // Default setting: kDefaultPlayerProcessingTime - virtual void SetPlayerProcessingTime(Clock::duration needed_time) = 0; - - // Propagates a "picture loss indicator" notification to the Sender, - // requesting a key frame so that decode/playout can recover. It is safe to - // call this redundantly. The Receiver will clear the picture loss condition - // automatically, once a key frame is received (i.e., before - // ConsumeNextFrame() is called to access it). - virtual void RequestKeyFrame() = 0; - - // Advances to the next frame ready for consumption. This may skip-over - // incomplete frames that will not play out on-time; but only if there are - // completed frames further down the queue that have no dependency - // relationship with them (e.g., key frames). - // - // This method returns kNoFramesReady if there is not currently a frame ready - // for consumption. The caller should wait for a Consumer::OnFramesReady() - // notification before trying again. Otherwise, the number of bytes of encoded - // data is returned, and the caller should use this to ensure the buffer it - // passes to ConsumeNextFrame() is large enough. - virtual int AdvanceToNextFrame() = 0; - - // Returns the next frame, both metadata and payload data. The Consumer calls - // this method after being notified via OnFramesReady(), and it can also call - // this whenever AdvanceToNextFrame() indicates another frame is ready. - // `buffer` must point to a sufficiently-sized buffer that will be populated - // with the frame's payload data. Upon return |frame->data| will be set to the - // portion of the buffer that was populated. - virtual EncodedFrame ConsumeNextFrame(ByteBuffer buffer) = 0; - - // The default "player processing time" amount. See SetPlayerProcessingTime(). - // This value is based on real world experimentation, however may vary - // widely depending on the platform of the receiver and what type of - // decoder is available. - static constexpr std::chrono::milliseconds kDefaultPlayerProcessingTime{50}; - - // Returned by AdvanceToNextFrame() when there are no frames currently ready - // for consumption. - static constexpr int kNoFramesReady = -1; -}; - -} // namespace openscreen::cast - -#endif // CAST_STREAMING_IMPL_RECEIVER_BASE_H_ diff --git a/cast/streaming/impl/receiver_impl.cc b/cast/streaming/impl/receiver_impl.cc new file mode 100644 index 000000000..ec5d26d62 --- /dev/null +++ b/cast/streaming/impl/receiver_impl.cc @@ -0,0 +1,587 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cast/streaming/impl/receiver_impl.h" + +#include +#include + +#include "cast/streaming/impl/receiver_packet_router.h" +#include "cast/streaming/public/constants.h" +#include "cast/streaming/public/session_config.h" +#include "platform/base/span.h" +#include "platform/base/trivial_clock_traits.h" +#include "util/chrono_helpers.h" +#include "util/osp_logging.h" +#include "util/std_util.h" +#include "util/trace_logging.h" + +namespace openscreen::cast { + +using clock_operators::operator<<; + +// Conveniences for ensuring logging output includes the SSRC of the Receiver, +// to help distinguish one out of multiple instances in a Cast Streaming +// session. +// +#define SSRC() "[SSRC:" << config().receiver_ssrc << "] " +#define RECEIVER_DLOG(level) OSP_DLOG_##level << SSRC() +#define RECEIVER_LOG(level) OSP_LOG_##level << SSRC() +#define RECEIVER_VLOG OSP_VLOG << SSRC() +#define RECEIVER_DVLOG OSP_DVLOG << SSRC() + +ReceiverImpl::ReceiverImpl(Environment& environment, + ReceiverPacketRouter& packet_router, + SessionConfig config) + : now_(environment.now_function()), + packet_router_(packet_router), + config_(config), + rtcp_session_(config.sender_ssrc, config.receiver_ssrc, now_()), + rtcp_parser_(rtcp_session_), + rtcp_builder_(std::make_unique(rtcp_session_)), + stats_tracker_(config.rtp_timebase), + rtp_parser_(config.sender_ssrc), + rtp_timebase_(config.rtp_timebase), + crypto_(config.aes_secret_key, config.aes_iv_mask), + is_pli_enabled_(config.is_pli_enabled), + rtcp_alarm_(environment.now_function(), environment.task_runner()), + smoothed_clock_offset_(ClockDriftSmoother::kDefaultTimeConstant), + consumption_alarm_(environment.now_function(), + environment.task_runner()) { + OSP_CHECK_EQ(checkpoint_frame(), FrameId::leader()); + + rtcp_buffer_.assign(environment.GetMaxPacketSize(), 0); + OSP_CHECK_GT(rtcp_buffer_.size(), 0); + + rtcp_builder_->SetPlayoutDelay(config.target_playout_delay); + playout_delay_changes_.emplace_back(FrameId::leader(), + config.target_playout_delay); + + packet_router_.RegisterPacketConsumer(rtcp_session_.sender_ssrc(), this); +} + +ReceiverImpl::~ReceiverImpl() { + packet_router_.DeregisterPacketConsumer(rtcp_session_.sender_ssrc()); +} + +const SessionConfig& ReceiverImpl::config() const { + return config_; +} + +void ReceiverImpl::SetConsumer(Consumer* consumer) { + consumer_ = consumer; + ScheduleFrameReadyCheck(); +} + +void ReceiverImpl::SetPlayerProcessingTime(Clock::duration needed_time) { + RECEIVER_DVLOG << __func__ << ": setting processing time to " << needed_time; + player_processing_time_ = std::max(Clock::duration::zero(), needed_time); +} + +Error ReceiverImpl::ReportPlayoutEvent(FrameId frame_id, + RtpTimeTicks rtp_timestamp, + Clock::time_point playout_time) { + if (!config_.are_receiver_event_logs_enabled) { + return Error(Error::Code::kOperationInvalid, + "receiver event logs are disabled. reports are ignored."); + } + + if (frame_id <= latest_frame_expected_ - kMaxUnackedFrames) { + return Error(Error::Code::kParameterOutOfRange, "frame is too old."); + } + + const PendingFrame& entry = GetQueueEntry(frame_id); + OSP_CHECK(entry.estimated_capture_time); + const Clock::duration playout_delay = + std::max(Clock::duration(), playout_time - *entry.estimated_capture_time); + + if (config_.are_receiver_event_logs_enabled) { + AddEventToPendingLogs( + rtp_timestamp, + RtcpReceiverEventLogMessage{ + .type = StatisticsEvent::Type::kFramePlayedOut, + .timestamp = playout_time, + .delay = std::chrono::duration_cast( + playout_delay)}); + } + + TRACE_FLOW_END_WITH_TIME(TraceCategory::kReceiver, "Frame.PlayedOut", + frame_id, playout_time); + + return Error::None(); +} + +void ReceiverImpl::RequestKeyFrame() { + // If we don't have picture loss indication enabled, we should not request + // any key frames. + if (!is_pli_enabled_) { + RECEIVER_LOG(WARN) << "Should not request any key frames when picture loss " + "indication is not enabled"; + return; + } + + if (!last_key_frame_received_.is_null() && + last_frame_consumed_ >= last_key_frame_received_ && + !rtcp_builder_->is_picture_loss_indicator_set()) { + rtcp_builder_->SetPictureLossIndicator(true); + SendRtcp(); + } +} + +std::optional ReceiverImpl::AdvanceToNextFrame() { + TRACE_DEFAULT_SCOPED(TraceCategory::kReceiver); + const FrameId immediate_next_frame = last_frame_consumed_ + 1; + + // Scan the queue for the next frame that should be consumed. Typically, this + // is the very next frame; but if it is incomplete and already late for + // playout, consider skipping-ahead. + for (FrameId f = immediate_next_frame; f <= latest_frame_expected_; ++f) { + PendingFrame& entry = GetQueueEntry(f); + if (entry.collector.is_complete()) { + const EncodedFrame& metadata = entry.collector.PeekFrameMetadata(); + + const bool is_next_frame = f == immediate_next_frame; + const bool is_independent = + metadata.dependency != EncodedFrame::Dependency::kDependent; + const bool is_ready = is_next_frame || is_independent; + if (is_ready) { + // Found a frame after skipping past some frames. Drop the ones being + // skipped, advancing `last_frame_consumed_` before returning. + // TODO(crbug.com/472513637): we may not always want to drop all + // dependent frames just because we have a complete independent frame. + if (!is_next_frame) { + DropAllFramesBefore(f); + } + TRACE_FLOW_STEP(TraceCategory::kReceiver, "Frame.Ready", f); + return static_cast(entry.collector.GetFramePayloadSize()); + } + } + + // Do not consider skipping past this frame if its estimated capture time is + // unknown. The implication here is that, if `estimated_capture_time` is + // set, the Receiver also knows whether any target playout delay changes + // were communicated from the Sender in the frame's first RTP packet. + if (!entry.estimated_capture_time) { + break; + } + + // If this incomplete frame is not yet late for playout, simply wait for the + // rest of its packets to come in. However, do schedule a check to + // re-examine things at the time it should be processed. + const auto process_time = *entry.estimated_capture_time + + ResolveTargetPlayoutDelay(f) - + player_processing_time_; + if (process_time > now_()) { + ScheduleFrameReadyCheck(process_time); + break; + } + } + + return std::nullopt; +} + +EncodedFrame ReceiverImpl::ConsumeNextFrame(ByteBuffer buffer) { + TRACE_DEFAULT_SCOPED(TraceCategory::kReceiver); + // Assumption: The required call to AdvanceToNextFrame() ensures that + // `last_frame_consumed_` is set to one before the frame to be consumed here. + const FrameId frame_id = last_frame_consumed_ + 1; + OSP_CHECK_LE(frame_id, checkpoint_frame()); + + TRACE_FLOW_STEP(TraceCategory::kReceiver, "Frame.Consumed", frame_id); + + // Decrypt the frame, populating the given output `frame`. + PendingFrame& entry = GetQueueEntry(frame_id); + OSP_CHECK(entry.collector.is_complete()); + OSP_CHECK(entry.estimated_capture_time); + + const EncodedFrame& metadata = entry.collector.PeekFrameMetadata(); + + // `buffer` will contain the decrypted frame contents. + crypto_.Decrypt(metadata.frame_id, entry.collector.GetPayloadChunks(), + buffer); + EncodedFrame frame; + metadata.CopyMetadataTo(&frame); + frame.data = buffer; + frame.reference_time = *entry.estimated_capture_time + + ResolveTargetPlayoutDelay(frame_id) - + player_processing_time_; + + RECEIVER_VLOG << "ConsumeNextFrame → " << frame.frame_id << ": " + << frame.data.size() << " payload bytes, RTP Timestamp " + << frame.rtp_timestamp.ToTimeSinceOrigin( + rtp_timebase_) + << ", to play-out " << (frame.reference_time - now_()) + << " from now."; + + // Reset the collector to free up memory, and leave the estimated_capture_time + // for this entry, as it may still be used if the consumer decides to report + // the playout event. + entry.collector.Reset(); + last_frame_consumed_ = frame_id; + + // Ensure the Consumer is notified if there are already more frames ready for + // consumption, and it hasn't explicitly called AdvanceToNextFrame() to check + // for itself. + ScheduleFrameReadyCheck(); + + return frame; +} + +void ReceiverImpl::OnReceivedRtpPacket(Clock::time_point arrival_time, + std::vector packet) { + const std::optional part = + rtp_parser_.Parse(packet); + if (!part) { + RECEIVER_LOG(WARN) << "Parsing of " << packet.size() + << " bytes as an RTP packet failed."; + return; + } + stats_tracker_.OnReceivedValidRtpPacket(part->sequence_number, + part->rtp_timestamp, arrival_time); + + // Ignore packets for frames the Receiver is no longer interested in. + if (part->frame_id <= checkpoint_frame()) { + RECEIVER_VLOG << "ignoring packet for frame " << part->frame_id + << " as it has been consumed or dropped already."; + return; + } + + // Extend the range of frames known to this Receiver, within the capacity of + // this Receiver's queue. Prepare the FrameCollectors to receive any + // newly-discovered frames. + if (part->frame_id > latest_frame_expected_) { + const FrameId max_allowed_frame_id = + last_frame_consumed_ + kMaxUnackedFrames; + if (part->frame_id > max_allowed_frame_id) { + RECEIVER_VLOG << "ignoring packet for unknown frame " << part->frame_id; + return; + } + do { + ++latest_frame_expected_; + PendingFrame& entry = GetQueueEntry(latest_frame_expected_); + + // The collector was already reset, so just reset the capture time. + entry.estimated_capture_time.reset(); + entry.collector.set_frame_id(latest_frame_expected_); + } while (latest_frame_expected_ < part->frame_id); + } + + // Start-up edge case: Blatantly drop the first packet of all frames until the + // Receiver has processed at least one Sender Report containing the necessary + // clock-drift and lip-sync information (see OnReceivedRtcpPacket()). This is + // an inescapable data dependency. Note that this special case should almost + // never trigger, since a well-behaving Sender will send the first Sender + // Report RTCP packet before any of the RTP packets. + if (!last_sender_report_ && part->packet_id == FramePacketId{0}) { + RECEIVER_LOG(WARN) << "Dropping packet 0 of frame " << part->frame_id + << " because it arrived before the first Sender Report."; + // Note: The Sender will have to re-transmit this dropped packet after the + // Sender Report to allow the Receiver to move forward. + return; + } + + PendingFrame& pending_frame = GetQueueEntry(part->frame_id); + FrameCollector& collector = pending_frame.collector; + if (collector.is_complete()) { + // An extra, redundant `packet` was received. Do nothing since the frame was + // already complete. + RECEIVER_VLOG << "ignoring redundant packet for frame " << part->frame_id; + return; + } + + if (!collector.CollectRtpPacket(*part, &packet)) { + RECEIVER_LOG(WARN) << "bad data in parsed packet for frame " + << part->frame_id; + return; // Bad data in the parsed packet. Ignore it. + } + + // The first packet in a frame contains timing information critical for + // computing this frame's (and all future frames') playout time. Process that, + // but only once. + if (part->packet_id == FramePacketId{0} && + !pending_frame.estimated_capture_time) { + pending_frame.rtp_timestamp = part->rtp_timestamp; + + // Estimate the original capture time of this frame (at the Sender), in + // terms of the Receiver's clock: First, start with a reference time point + // from the Sender's clock (the one from the last Sender Report). Then, + // translate it into the equivalent reference time point in terms of the + // Receiver's clock by applying the measured offset between the two clocks. + // Finally, apply the RTP timestamp difference between the Sender Report and + // this frame to determine what the original capture time of this frame was. + const auto smoothed_offset = smoothed_clock_offset_.Current(); + if (!smoothed_offset) { + return; + } + pending_frame.estimated_capture_time = + last_sender_report_->reference_time + *smoothed_offset + + (part->rtp_timestamp - last_sender_report_->rtp_timestamp) + .ToDuration(rtp_timebase_); + + // If a target playout delay change was included in this packet, record it. + if (part->new_playout_delay > milliseconds::zero()) { + RecordNewTargetPlayoutDelay(part->frame_id, part->new_playout_delay); + } + + // Now that the estimated capture time is known, other frames may have just + // become ready, per the frame-skipping logic in AdvanceToNextFrame(). + ScheduleFrameReadyCheck(); + } + + if (config_.are_receiver_event_logs_enabled) { + AddEventToPendingLogs(part->rtp_timestamp, + RtcpReceiverEventLogMessage{ + .type = StatisticsEvent::Type::kPacketReceived, + .timestamp = arrival_time, + .packet_id = part->packet_id}); + } + + if (!collector.is_complete()) { + return; // Wait for the rest of the packets to come in. + } + TRACE_FLOW_STEP(TraceCategory::kReceiver, "Frame.Complete", part->frame_id); + + const EncodedFrame& metadata = collector.PeekFrameMetadata(); + + // Whenever a key frame has been received, the decoder has what it needs to + // recover. In this case, clear the PLI condition. + if (metadata.dependency == EncryptedFrame::Dependency::kKeyFrame) { + rtcp_builder_->SetPictureLossIndicator(false); + last_key_frame_received_ = part->frame_id; + } + + // If this just-completed frame is the one right after the checkpoint frame, + // advance the checkpoint forward. + if (part->frame_id == (checkpoint_frame() + 1)) { + // Make sure we provide a FrameAckSent event to the sender later. + pending_frame_acks_.push_back(part->rtp_timestamp); + AdvanceCheckpoint(part->frame_id); + } + + // Since a frame has become complete, schedule a check to see whether this or + // any other frames have become ready for consumption. + ScheduleFrameReadyCheck(); +} + +void ReceiverImpl::OnReceivedRtcpPacket(Clock::time_point arrival_time, + std::span packet) { + TRACE_DEFAULT_SCOPED(TraceCategory::kReceiver); + std::optional parsed_report = + rtcp_parser_.Parse(packet); + if (!parsed_report) { + TRACE_SCOPED(TraceCategory::kReceiver, "ReceivedInvalidRtcpReport"); + RECEIVER_LOG(WARN) << "Parsing of " << packet.size() + << " bytes as an RTCP packet failed."; + return; + } + + TRACE_DEFAULT_SCOPED1(TraceCategory::kReceiver, "packet_id", + parsed_report->report_id); + last_sender_report_ = std::move(parsed_report); + last_sender_report_arrival_time_ = arrival_time; + + // Measure the offset between the Sender's clock and the Receiver's Clock. + // This will be used to translate reference timestamps from the Sender into + // timestamps that represent the exact same moment in time at the Receiver. + // + // Note: Due to design limitations in the Cast Streaming spec, the Receiver + // has no way to compute how long it took the Sender Report to travel over the + // network. The calculation here just ignores that, and so the + // `measured_offset` below will be larger than the true value by that amount. + // This will have the effect of a later-than-configured playout delay. + // TODO(crbug.com/496703606): determine network delay by using round trip + // timestamp estimations. + const Clock::duration measured_offset = + arrival_time - last_sender_report_->reference_time; + smoothed_clock_offset_.Update(arrival_time, measured_offset); + + RtcpReportBlock report; + report.ssrc = rtcp_session_.sender_ssrc(); + stats_tracker_.PopulateNextReport(&report); + report.last_status_report_id = last_sender_report_->report_id; + report.SetDelaySinceLastReport(now_() - last_sender_report_arrival_time_); + rtcp_builder_->IncludeReceiverReportInNextPacket(report); + + SendRtcp(); +} + +void ReceiverImpl::SendRtcp() { + // Collect ACK/NACK feedback for all active frames in the queue. + std::vector packet_nacks; + std::vector frame_acks; + for (FrameId f = checkpoint_frame() + 1; f <= latest_frame_expected_; ++f) { + const PendingFrame& entry = GetQueueEntry(f); + if (entry.collector.is_complete()) { + frame_acks.push_back(f); + + if (config_.are_receiver_event_logs_enabled) { + if (entry.rtp_timestamp) { + pending_frame_acks_.push_back(entry.rtp_timestamp.value()); + } + } + } else { + entry.collector.GetMissingPackets(&packet_nacks); + } + } + + // Fire off events for frames that were implicitly ACKed. + if (config_.are_receiver_event_logs_enabled) { + for (auto rtp_timestamp : pending_frame_acks_) { + AddEventToPendingLogs(rtp_timestamp, + RtcpReceiverEventLogMessage{ + .type = StatisticsEvent::Type::kFrameAckSent, + .timestamp = now_(), + }); + } + pending_frame_acks_.clear(); + + rtcp_builder_->IncludeReceiverLogsInNextPacket(std::move(pending_logs_)); + pending_logs_.clear(); + } + + // Build and send a compound RTCP packet. + rtcp_builder_->IncludeFeedbackInNextPacket(std::move(packet_nacks), + std::move(frame_acks)); + last_rtcp_send_time_ = now_(); + packet_router_.SendRtcpPacket( + rtcp_builder_->BuildPacket(last_rtcp_send_time_, rtcp_buffer_)); + + // Schedule the automatic sending of another RTCP packet, if this method is + // not called within some bounded amount of time. While incomplete frames + // exist in the queue, send RTCP packets (with ACK/NACK feedback) frequently. + // When there are no incomplete frames, use a longer "keepalive" interval. + const Clock::duration interval = + (packet_nacks.empty() ? kRtcpReportInterval : kNackFeedbackInterval); + rtcp_alarm_.Schedule([this] { SendRtcp(); }, last_rtcp_send_time_ + interval); +} + +const ReceiverImpl::PendingFrame& ReceiverImpl::GetQueueEntry( + FrameId frame_id) const { + return const_cast(this)->GetQueueEntry(frame_id); +} + +ReceiverImpl::PendingFrame& ReceiverImpl::GetQueueEntry(FrameId frame_id) { + return pending_frames_[(frame_id - FrameId::first()) % + pending_frames_.size()]; +} + +void ReceiverImpl::RecordNewTargetPlayoutDelay(FrameId as_of_frame, + milliseconds delay) { + OSP_CHECK_GT(as_of_frame, checkpoint_frame()); + + // Prune-out entries from `playout_delay_changes_` that are no longer needed. + // At least one entry must always be kept (i.e., there must always be a + // "current" setting). + const FrameId next_frame = last_frame_consumed_ - kMaxUnackedFrames + 1; + const auto keep_one_before_it = std::find_if( + std::next(playout_delay_changes_.begin()), playout_delay_changes_.end(), + [&](const auto& entry) { return entry.first > next_frame; }); + playout_delay_changes_.erase(playout_delay_changes_.begin(), + std::prev(keep_one_before_it)); + + // Insert the delay change entry, maintaining the ascending ordering of the + // vector. + const auto insert_it = std::find_if( + playout_delay_changes_.begin(), playout_delay_changes_.end(), + [&](const auto& entry) { return entry.first > as_of_frame; }); + playout_delay_changes_.emplace(insert_it, as_of_frame, delay); + + OSP_DCHECK(AreElementsSortedAndUnique(playout_delay_changes_)); +} + +milliseconds ReceiverImpl::ResolveTargetPlayoutDelay(FrameId frame_id) const { + const FrameId first_possible = + last_frame_consumed_ > FrameId::first() + kMaxUnackedFrames + ? last_frame_consumed_ - kMaxUnackedFrames + : FrameId::first(); + OSP_CHECK_GE(frame_id, first_possible); + +#if OSP_DCHECK_IS_ON() + // Extra precaution: Ensure all possible playout delay changes are known. In + // other words, every unconsumed frame in the queue, up to (and including) + // `frame_id`, must have an assigned estimated_capture_time. + for (FrameId f = first_possible; f <= frame_id; ++f) { + OSP_CHECK(GetQueueEntry(f).estimated_capture_time) + << " don't know whether there was a playout delay change for frame " + << f; + } +#endif + + const auto it = std::find_if( + playout_delay_changes_.crbegin(), playout_delay_changes_.crend(), + [&](const auto& entry) { return entry.first <= frame_id; }); + OSP_CHECK(it != playout_delay_changes_.crend()); + return it->second; +} + +void ReceiverImpl::AdvanceCheckpoint(FrameId new_checkpoint) { + TRACE_DEFAULT_SCOPED(TraceCategory::kReceiver); + OSP_CHECK_GT(new_checkpoint, checkpoint_frame()); + OSP_CHECK_LE(new_checkpoint, latest_frame_expected_); + + while (new_checkpoint < latest_frame_expected_) { + const FrameId next = new_checkpoint + 1; + if (!GetQueueEntry(next).collector.is_complete()) { + break; + } + new_checkpoint = next; + } + + set_checkpoint_frame(new_checkpoint); + rtcp_builder_->SetPlayoutDelay(ResolveTargetPlayoutDelay(new_checkpoint)); + SendRtcp(); +} + +void ReceiverImpl::DropAllFramesBefore(FrameId first_kept_frame) { + // The following CHECKs are verifying that this method is only being called + // because one or more incomplete frames are being skipped-over. + const FrameId first_to_drop = last_frame_consumed_ + 1; + OSP_CHECK_GT(first_kept_frame, first_to_drop); + OSP_CHECK_GT(first_kept_frame, checkpoint_frame()); + OSP_CHECK_LE(first_kept_frame, latest_frame_expected_); + + // Reset each of the frames being dropped, pretending that they were consumed. + for (FrameId f = first_to_drop; f < first_kept_frame; ++f) { + PendingFrame& entry = GetQueueEntry(f); + // Pedantic sanity-check: Ensure the "target playout delay change" data + // dependency was satisfied. See comments in AdvanceToNextFrame(). + OSP_CHECK(entry.estimated_capture_time); + entry.collector.Reset(); + } + last_frame_consumed_ = first_kept_frame - 1; + + RECEIVER_LOG(INFO) << "Artificially advancing checkpoint after skipping."; + AdvanceCheckpoint(first_kept_frame); +} + +void ReceiverImpl::ScheduleFrameReadyCheck(Clock::time_point when) { + consumption_alarm_.Schedule( + [this] { + if (consumer_) { + const std::optional next_size = AdvanceToNextFrame(); + if (next_size.has_value()) { + consumer_->OnFramesReady(*next_size); + } + } + }, + when); +} + +void ReceiverImpl::AddEventToPendingLogs( + RtpTimeTicks rtp_timestamp, + RtcpReceiverEventLogMessage event_log) { + OSP_CHECK(config_.are_receiver_event_logs_enabled); + + // Find or create a frame log for this RTP timestamp. + auto it = std::find_if( + pending_logs_.begin(), pending_logs_.end(), + [&](const auto& log) { return log.rtp_timestamp == rtp_timestamp; }); + if (it == pending_logs_.end()) { + pending_logs_.push_back({rtp_timestamp, {}}); + it = pending_logs_.end() - 1; + } + it->messages.push_back(event_log); +} + +} // namespace openscreen::cast diff --git a/cast/streaming/impl/receiver_impl.h b/cast/streaming/impl/receiver_impl.h new file mode 100644 index 000000000..50400c15b --- /dev/null +++ b/cast/streaming/impl/receiver_impl.h @@ -0,0 +1,243 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CAST_STREAMING_IMPL_RECEIVER_IMPL_H_ +#define CAST_STREAMING_IMPL_RECEIVER_IMPL_H_ + +#include + +#include +#include +#include +#include +#include +#include + +#include "cast/streaming/impl/clock_drift_smoother.h" +#include "cast/streaming/impl/compound_rtcp_builder.h" +#include "cast/streaming/impl/frame_collector.h" +#include "cast/streaming/impl/packet_receive_stats_tracker.h" +#include "cast/streaming/impl/receiver_packet_router.h" +#include "cast/streaming/impl/rtcp_common.h" +#include "cast/streaming/impl/rtcp_session.h" +#include "cast/streaming/impl/rtp_packet_parser.h" +#include "cast/streaming/impl/sender_report_parser.h" +#include "cast/streaming/public/environment.h" +#include "cast/streaming/public/frame_id.h" +#include "cast/streaming/public/receiver.h" +#include "cast/streaming/public/session_config.h" +#include "cast/streaming/ssrc.h" +#include "platform/api/time.h" +#include "platform/base/span.h" +#include "util/alarm.h" +#include "util/chrono_helpers.h" + +namespace openscreen::cast { + +struct EncodedFrame; +class ReceiverTest; + +class ReceiverImpl : public Receiver, + public ReceiverPacketRouter::PacketConsumer { + public: + using Receiver::Consumer; + + // Constructs a Receiver that attaches to the given `environment` and + // `packet_router`. The config contains the settings that were + // agreed-upon by both sides from the OFFER/ANSWER exchange (i.e., the part of + // the overall end-to-end connection process that occurs before Cast Streaming + // is started). + ReceiverImpl(Environment& environment, + ReceiverPacketRouter& packet_router, + SessionConfig config); + ~ReceiverImpl() override; + + // Receiver overrides. + const SessionConfig& config() const override; + void SetConsumer(Consumer* consumer) override; + void SetPlayerProcessingTime(Clock::duration needed_time) override; + Error ReportPlayoutEvent(FrameId frame_id, + RtpTimeTicks rtp_timestamp, + Clock::time_point playout_time) override; + void RequestKeyFrame() override; + std::optional AdvanceToNextFrame() override; + EncodedFrame ConsumeNextFrame(ByteBuffer buffer) override; + + // The default "player processing time" amount. See SetPlayerProcessingTime(). + using openscreen::cast::Receiver::kDefaultPlayerProcessingTime; + + protected: + // ReceiverPacketRouter::PacketConsumer implementation. + void OnReceivedRtpPacket(Clock::time_point arrival_time, + std::vector packet) override; + void OnReceivedRtcpPacket(Clock::time_point arrival_time, + std::span packet) override; + + private: + // An entry in the circular queue (see `pending_frames_`). + struct PendingFrame { + // NOTE: to free resources, the collector may be Reset(). + FrameCollector collector; + + // The Receiver's [local] Clock time when this frame was originally captured + // at the Sender. This is computed and assigned when the RTP packet with ID + // 0 is processed. Add the target playout delay to this to get the target + // playout time. + std::optional estimated_capture_time; + + // The timestamp associated with the frame. + std::optional rtp_timestamp; + }; + + // Get/Set the checkpoint FrameId. This indicates that all of the packets for + // all frames up to and including this FrameId have been successfully received + // (or otherwise do not need to be re-transmitted). + FrameId checkpoint_frame() const { return rtcp_builder_->checkpoint_frame(); } + void set_checkpoint_frame(FrameId frame_id) { + rtcp_builder_->SetCheckpointFrame(frame_id); + } + + // Send an RTCP packet to the Sender immediately, to acknowledge the complete + // reception of one or more additional frames, to reply to a Sender Report, or + // to request re-transmits. Calling this also schedules additional RTCP + // packets to be sent periodically for the life of this Receiver. + void SendRtcp(); + + // Helpers to map the given `frame_id` to the element in the `pending_frames_` + // circular queue. There are both const and non-const versions, but neither + // mutate any state (i.e., they are just look-ups). + const PendingFrame& GetQueueEntry(FrameId frame_id) const; + PendingFrame& GetQueueEntry(FrameId frame_id); + + // Record that the target playout delay has changed starting with the given + // FrameId. + void RecordNewTargetPlayoutDelay(FrameId as_of_frame, + std::chrono::milliseconds delay); + + // Examine the known target playout delay changes to determine what setting is + // in-effect for the given frame. + std::chrono::milliseconds ResolveTargetPlayoutDelay(FrameId frame_id) const; + + // Called to move the checkpoint forward. This scans the queue, starting from + // `new_checkpoint`, to find the latest in a contiguous sequence of completed + // frames. Then, it records that frame as the new checkpoint, and immediately + // sends a feedback RTCP packet to the Sender. + void AdvanceCheckpoint(FrameId new_checkpoint); + + // Helper to force-drop all frames before `first_kept_frame`, even if they + // were never consumed. This will also auto-cancel frames that were never + // completely received, artificially moving the checkpoint forward, and + // notifying the Sender of that. The caller of this method is responsible for + // making sure that frame data dependencies will not be broken by dropping the + // frames. + void DropAllFramesBefore(FrameId first_kept_frame); + + // Sets the `consumption_alarm_` to check whether any frames are ready, + // including possibly skipping over late frames in order to make not-yet-late + // frames become ready. The default argument value means "without delay." + void ScheduleFrameReadyCheck(Clock::time_point when = Alarm::kImmediately); + + void AddEventToPendingLogs(RtpTimeTicks rtp_timestamp, + RtcpReceiverEventLogMessage event_log); + + const ClockNowFunctionPtr now_; + ReceiverPacketRouter& packet_router_; + const SessionConfig config_; + RtcpSession rtcp_session_; + SenderReportParser rtcp_parser_; + std::unique_ptr rtcp_builder_; + PacketReceiveStatsTracker stats_tracker_; // Tracks transmission stats. + RtpPacketParser rtp_parser_; + const int rtp_timebase_; // RTP timestamp ticks per second. + const FrameCrypto crypto_; // Decrypts assembled frames. + bool is_pli_enabled_; // Whether picture loss indication is enabled. + + // Buffer for serializing/sending RTCP packets. + std::vector rtcp_buffer_; + + // Schedules tasks to ensure RTCP reports are sent within a bounded interval. + // Not scheduled until after this Receiver has processed the first packet from + // the Sender. + Alarm rtcp_alarm_; + Clock::time_point last_rtcp_send_time_ = Clock::time_point::min(); + + // The last Sender Report received and when the packet containing it had + // arrived. This contains lip-sync timestamps used as part of the calculation + // of playout times for the received frames, as well as ping-pong data bounced + // back to the Sender in the Receiver Reports. It is nullopt until the first + // parseable Sender Report is received. + std::optional last_sender_report_; + Clock::time_point last_sender_report_arrival_time_; + + // Tracks the offset between the Receiver's [local] clock and the Sender's + // clock. This is invalid until the first Sender Report has been successfully + // processed (i.e., `last_sender_report_` is not nullopt). + ClockDriftSmoother smoothed_clock_offset_; + + // The ID of the latest frame whose existence is known to this Receiver. This + // value must always be greater than or equal to `checkpoint_frame()`. + FrameId latest_frame_expected_ = FrameId::leader(); + + // The ID of the last frame consumed. This value must always be less than or + // equal to `checkpoint_frame()`, since it's impossible to consume incomplete + // frames! + FrameId last_frame_consumed_ = FrameId::leader(); + + // The ID of the latest key frame known to be in-flight. This is used by + // RequestKeyFrame() to ensure the PLI condition doesn't get set again until + // after the consumer has seen a key frame that would clear the condition. + FrameId last_key_frame_received_; + + // The frame queue (circular), which tracks which frames are in-flight, stores + // data for partially-received frames, and holds onto completed frames until + // the consumer consumes them. After the frame has been consumed, its capture + // time and rtp_timestamp are intentionally left valid so they may be used + // for statistics gathering. The consumer then has until the slot is reused + // to report playout events, after which an error will be thrown. + // + // Use GetQueueEntry() to access a slot. The currently-active slots are those + // for the frames after `last_frame_consumed_` and up-to/including + // `latest_frame_expected_`. + std::array pending_frames_ = {}; + + // A vector containing the RTP timestamps of all of the frames that are + // implicitly ACKed by the checkpoint frame ID advancing. + std::vector pending_frame_acks_; + + // Tracks the recent changes to the target playout delay, which is controlled + // by the Sender. The FrameId indicates the first frame where a new delay + // setting takes effect. This vector is never empty, is kept sorted, and is + // pruned to remain as small as possible. + // + // The target playout delay is the amount of time between a frame's + // capture/recording on the Sender and when it should be played-out at the + // Receiver. + std::vector> + playout_delay_changes_; + + // The consumer to notify when there are one or more frames completed and + // ready to be consumed. + Consumer* consumer_ = nullptr; + + // The additional time needed to decode/play-out each frame after being + // consumed from this Receiver. + Clock::duration player_processing_time_ = kDefaultPlayerProcessingTime; + + // Scheduled to check whether there are frames ready and, if there are, to + // notify the Consumer via OnFramesReady(). + Alarm consumption_alarm_; + + std::vector pending_logs_; + + // The interval between sending ACK/NACK feedback RTCP messages while + // incomplete frames exist in the queue. + // + // TODO(jophba): This should be a function of the current target playout + // delay, similar to the Sender's kickstart interval logic. + static constexpr milliseconds kNackFeedbackInterval = milliseconds(30); +}; + +} // namespace openscreen::cast + +#endif // CAST_STREAMING_IMPL_RECEIVER_IMPL_H_ diff --git a/cast/streaming/impl/receiver_impl_unittest.cc b/cast/streaming/impl/receiver_impl_unittest.cc new file mode 100644 index 000000000..29b39bb63 --- /dev/null +++ b/cast/streaming/impl/receiver_impl_unittest.cc @@ -0,0 +1,1000 @@ +// Copyright 2019 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cast/streaming/impl/receiver_impl.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "cast/streaming/impl/compound_rtcp_builder.h" +#include "cast/streaming/impl/compound_rtcp_parser.h" +#include "cast/streaming/impl/frame_crypto.h" +#include "cast/streaming/impl/receiver_packet_router.h" +#include "cast/streaming/impl/rtcp_common.h" +#include "cast/streaming/impl/rtcp_session.h" +#include "cast/streaming/impl/rtp_defines.h" +#include "cast/streaming/impl/rtp_packetizer.h" +#include "cast/streaming/impl/sender_report_builder.h" +#include "cast/streaming/impl/statistics_common.h" +#include "cast/streaming/public/constants.h" +#include "cast/streaming/public/encoded_frame.h" +#include "cast/streaming/public/environment.h" +#include "cast/streaming/public/session_config.h" +#include "cast/streaming/rtp_time.h" +#include "cast/streaming/ssrc.h" +#include "cast/streaming/testing/mock_environment.h" +#include "cast/streaming/testing/simple_socket_subscriber.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "platform/api/time.h" +#include "platform/api/udp_socket.h" +#include "platform/base/error.h" +#include "platform/base/ip_address.h" +#include "platform/base/span.h" +#include "platform/base/trivial_clock_traits.h" +#include "platform/base/udp_packet.h" +#include "platform/test/fake_clock.h" +#include "platform/test/fake_task_runner.h" +#include "util/chrono_helpers.h" +#include "util/osp_logging.h" + +using testing::_; +using testing::AllOf; +using testing::AtLeast; +using testing::ElementsAre; +using testing::Ge; +using testing::Gt; +using testing::Le; +using testing::NiceMock; +using testing::Return; +using testing::SaveArg; + +namespace openscreen::cast { + +namespace { + +// Receiver configuration. + +constexpr Ssrc kSenderSsrc = 1; +constexpr Ssrc kReceiverSsrc = 2; +constexpr int kRtpTimebase = 48000; +constexpr milliseconds kTargetPlayoutDelay(100); +constexpr auto kAesKey = + std::array{{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}}; +constexpr auto kCastIvMask = + std::array{{0xf0, 0xe0, 0xd0, 0xc0, 0xb0, 0xa0, 0x90, 0x80, + 0x70, 0x60, 0x50, 0x40, 0x30, 0x20, 0x10, 0x00}}; + +constexpr milliseconds kTargetPlayoutDelayChange(800); +// Additional configuration for the Sender. +constexpr RtpPayloadType kRtpPayloadType = RtpPayloadType::kVideoVp8; +constexpr int kMaxRtpPacketSize = 64; + +// A simulated one-way network delay, and round-trip network delay. +constexpr auto kOneWayNetworkDelay = milliseconds(3); +constexpr auto kRoundTripNetworkDelay = 2 * kOneWayNetworkDelay; +static_assert(kRoundTripNetworkDelay < kTargetPlayoutDelay && + kRoundTripNetworkDelay < kTargetPlayoutDelayChange, + "Network delay must be smaller than target playout delay."); + +// An EncodedFrame for unit testing, one of a sequence of simulated frames, each +// of 10 ms duration. The first frame will be a key frame; and any later frames +// will be non-key, dependent on the prior frame. Frame 5 (the 6th frame in the +// zero-based sequence) will include a target playout delay change, an increase +// to 800 ms. Frames with different IDs will contain vary in their payload data +// size, but are always 3 or more packets' worth of data. +struct SimulatedFrame : public EncodedFrame { + static constexpr milliseconds kFrameDuration = milliseconds(10); + static constexpr milliseconds kTargetPlayoutDelayChange = milliseconds(800); + + static constexpr int kPlayoutChangeAtFrame = 5; + + SimulatedFrame(Clock::time_point first_frame_reference_time, int which) { + frame_id = FrameId::first() + which; + if (which == 0) { + dependency = EncodedFrame::Dependency::kKeyFrame; + referenced_frame_id = frame_id; + } else { + dependency = EncodedFrame::Dependency::kDependent; + referenced_frame_id = frame_id - 1; + } + rtp_timestamp = + GetRtpStartTime() + + RtpTimeDelta::FromDuration(kFrameDuration * which, kRtpTimebase); + reference_time = first_frame_reference_time + kFrameDuration * which; + if (which == kPlayoutChangeAtFrame) { + new_playout_delay = kTargetPlayoutDelayChange; + } + constexpr int kAdditionalBytesEachSuccessiveFrame = 3; + buffer_.resize(3 * kMaxRtpPacketSize + + which * kAdditionalBytesEachSuccessiveFrame); + for (size_t i = 0; i < buffer_.size(); ++i) { + buffer_[i] = static_cast(which + static_cast(i)); + } + data = buffer_; + } + + static RtpTimeTicks GetRtpStartTime() { + return RtpTimeTicks::FromTimeSinceOrigin(seconds(0), kRtpTimebase); + } + + static milliseconds GetExpectedPlayoutDelay(int which) { + return (which < kPlayoutChangeAtFrame) ? kTargetPlayoutDelay + : kTargetPlayoutDelayChange; + } + + private: + std::vector buffer_; +}; + +// static +constexpr milliseconds SimulatedFrame::kFrameDuration; +constexpr milliseconds SimulatedFrame::kTargetPlayoutDelayChange; +constexpr int SimulatedFrame::kPlayoutChangeAtFrame; + +template +std::string ToString(T duration) { + std::ostringstream ss; + openscreen::clock_operators::operator<<(ss, duration); + return ss.str(); +} + +// TODO(jophba): this matcher is likely more generally useful and should +// be refactored. +MATCHER_P(EqualsDuration, expected, ToString(expected)) { + if (arg == expected) { + return true; + } + *result_listener << ToString(arg) << " (a difference of " + << ToString(arg - expected) << ")"; + return false; +} + +// Processes packets from the Receiver under test, as a real Sender might, and +// allows the unit tests to set expectations on events of interest to confirm +// proper behavior of the Receiver. +class MockSender : public CompoundRtcpParser::Client { + public: + MockSender(TaskRunner& task_runner, UdpSocket::Client* receiver) + : task_runner_(task_runner), + receiver_(receiver), + sender_endpoint_{ + // Use a random IPv6 address in the range reserved for + // "documentation purposes." Thus, the following is a fake address + // that should be blocked by the OS (and all network packet + // routers). But, these tests don't use real sockets, so... + IPAddress::Parse("2001:db8:0d93:69c2:fd1a:49a6:a7c0:e8a6").value(), + 2344}, + rtcp_session_(kSenderSsrc, kReceiverSsrc, FakeClock::now()), + sender_report_builder_(rtcp_session_), + rtcp_parser_(rtcp_session_, *this), + crypto_(kAesKey, kCastIvMask), + rtp_packetizer_(kRtpPayloadType, kSenderSsrc, kMaxRtpPacketSize) {} + + ~MockSender() override = default; + + void set_max_feedback_frame_id(FrameId f) { max_feedback_frame_id_ = f; } + + // Called by the test procedures to generate a Sender Report containing the + // given lip-sync timestamps, and send it to the Receiver. The caller must + // spin the TaskRunner for the RTCP packet to be delivered to the Receiver. + StatusReportId SendSenderReport(Clock::time_point reference_time, + RtpTimeTicks rtp_timestamp) { + // Generate the Sender Report RTCP packet. + uint8_t buffer[kMaxRtpPacketSizeForIpv4UdpOnEthernet]; + RtcpSenderReport sender_report; + sender_report.reference_time = reference_time; + sender_report.rtp_timestamp = rtp_timestamp; + const auto packet_and_report_id = + sender_report_builder_.BuildPacket(sender_report, buffer); + + // Send the RTCP packet as a UdpPacket directly to the Receiver instance. + UdpPacket packet_to_send(packet_and_report_id.first.begin(), + packet_and_report_id.first.end()); + packet_to_send.set_source(sender_endpoint_); + task_runner_.PostTaskWithDelay( + [receiver = receiver_, packet = std::move(packet_to_send)]() mutable { + receiver->OnRead(nullptr, ErrorOr(std::move(packet))); + }, + kOneWayNetworkDelay); + + return packet_and_report_id.second; + } + + // Sets which frame is currently being sent by this MockSender. Test code must + // call SendRtpPackets() to send the packets. + void SetFrameBeingSent(const EncodedFrame& frame) { + frame_being_sent_ = crypto_.Encrypt(frame); + } + + // Returns a vector containing each packet ID once (of the current frame being + // sent). `permutation` controls the sort order of the vector: zero will + // provide all the packet IDs in order, and greater values will provide them + // in a different, predictable order. + std::vector GetAllPacketIds(int permutation = 0) { + const int num_packets = + rtp_packetizer_.ComputeNumberOfPackets(frame_being_sent_); + OSP_CHECK_GT(num_packets, 0); + std::vector ids; + ids.reserve(num_packets); + const FramePacketId last_packet_id = + static_cast(num_packets - 1); + for (FramePacketId packet_id = 0; packet_id <= last_packet_id; + ++packet_id) { + ids.push_back(packet_id); + } + for (int i = 0; i < permutation; ++i) { + std::next_permutation(ids.begin(), ids.end()); + } + return ids; + } + + // Send the specified packets of the current frame being sent. + void SendRtpPackets(const std::vector& packets_to_send) { + uint8_t buffer[kMaxRtpPacketSize]; + for (FramePacketId packet_id : packets_to_send) { + const auto span = rtp_packetizer_.GeneratePacket( + frame_being_sent_, packet_id, ByteBuffer(buffer, kMaxRtpPacketSize)); + UdpPacket packet_to_send(span.begin(), span.end()); + packet_to_send.set_source(sender_endpoint_); + task_runner_.PostTaskWithDelay( + [receiver = receiver_, packet = std::move(packet_to_send)]() mutable { + receiver->OnRead(nullptr, ErrorOr(std::move(packet))); + }, + kOneWayNetworkDelay); + } + } + + // Called to process a packet from the Receiver. + void OnPacketFromReceiver(ByteView packet) { + EXPECT_TRUE(rtcp_parser_.Parse(packet, max_feedback_frame_id_)); + } + + // CompoundRtcpParser::Client implementation: Tests set expectations on these + // mocks to confirm that the receiver is providing the right data to the + // sender in its RTCP packets. + MOCK_METHOD(void, + OnReceiverReferenceTimeAdvanced, + (Clock::time_point reference_time), + (override)); + MOCK_METHOD(void, + OnReceiverReport, + (const RtcpReportBlock& receiver_report), + (override)); + MOCK_METHOD(void, + OnCastReceiverFrameLogMessages, + (std::vector messages), + (override)); + MOCK_METHOD(void, OnReceiverIndicatesPictureLoss, (), (override)); + MOCK_METHOD(void, + OnReceiverCheckpoint, + (FrameId frame_id, milliseconds playout_delay), + (override)); + MOCK_METHOD(void, + OnReceiverHasFrames, + (std::vector acks), + (override)); + MOCK_METHOD(void, + OnReceiverIsMissingPackets, + (std::vector nacks), + (override)); + + private: + TaskRunner& task_runner_; + UdpSocket::Client* const receiver_; + const IPEndpoint sender_endpoint_; + RtcpSession rtcp_session_; + SenderReportBuilder sender_report_builder_; + CompoundRtcpParser rtcp_parser_; + FrameCrypto crypto_; + RtpPacketizer rtp_packetizer_; + FrameId max_feedback_frame_id_ = FrameId::first() + kMaxUnackedFrames; + + EncryptedFrame frame_being_sent_; +}; + +class MockConsumer : public Receiver::Consumer { + public: + MOCK_METHOD(void, OnFramesReady, (size_t next_frame_buffer_size), (override)); +}; + +class ReceiverTest : public testing::Test { + public: + ReceiverTest() + : clock_(Clock::now()), + task_runner_(clock_), + env_(&FakeClock::now, task_runner_), + packet_router_(env_), + sender_(task_runner_, &env_) { + ConstructReceiver(); + } + + ~ReceiverTest() override = default; + + void ConstructReceiver(bool is_pli_enabled = true) { + receiver_.reset(); + receiver_ = std::make_unique( + env_, packet_router_, + SessionConfig( + kSenderSsrc, kReceiverSsrc, kRtpTimebase, + /* .channels = */ 2, kTargetPlayoutDelay, kAesKey, kCastIvMask, + /* .is_pli_enabled = */ is_pli_enabled, StreamType::kUnknown, + /* .are_receiver_event_logs_enabled = */ true)); + env_.SetSocketSubscriber(&socket_subscriber_); + ON_CALL(env_, SendPacket(_, _)) + .WillByDefault([this](ByteView packet, PacketMetadata metadata) { + task_runner_.PostTaskWithDelay( + [sender = &sender_, copy_of_packet = std::vector( + packet.begin(), packet.end())]() mutable { + sender->OnPacketFromReceiver(std::move(copy_of_packet)); + }, + kOneWayNetworkDelay); + }); + receiver_->SetConsumer(&consumer_); + } + + ReceiverImpl* receiver() { return receiver_.get(); } + MockSender* sender() { return &sender_; } + MockConsumer* consumer() { return &consumer_; } + + Clock::time_point now() const { return clock_.now(); } + void AdvanceClockAndRunTasks(Clock::duration delta) { clock_.Advance(delta); } + void RunTasksUntilIdle() { task_runner_.RunTasksUntilIdle(); } + + // Sends the initial Sender Report with lip-sync timing information to + // "unblock" the Receiver, and confirms the Receiver immediately replies with + // a corresponding Receiver Report. + void ExchangeInitialReportPackets(Clock::time_point start_time) { + sender()->SendSenderReport(start_time, SimulatedFrame::GetRtpStartTime()); + AdvanceClockAndRunTasks( + kOneWayNetworkDelay); // Transmit report to Receiver. + // The Receiver will immediately reply with a Receiver Report. + EXPECT_CALL(*sender(), + OnReceiverCheckpoint(FrameId::leader(), kTargetPlayoutDelay)) + .Times(1); + AdvanceClockAndRunTasks(kOneWayNetworkDelay); // Transmit reply to Sender. + testing::Mock::VerifyAndClearExpectations(sender()); + } + + void ReceiveFrame(int frame_index, + Clock::time_point reference_time, + std::chrono::milliseconds playout_delay, + Clock::duration clock_advancement) { + EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); + EXPECT_CALL(*sender(), OnReceiverCheckpoint(FrameId::first() + frame_index, + playout_delay)) + .Times(1); + sender()->SetFrameBeingSent(SimulatedFrame(reference_time, frame_index)); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(clock_advancement); + testing::Mock::VerifyAndClearExpectations(sender()); + testing::Mock::VerifyAndClearExpectations(consumer()); + } + + // Consume one frame from the Receiver, and verify that it is the same as the + // `sent_frame`. Exception: The `reference_time` is the playout time on the + // Receiver's end, while it refers to the capture time on the Sender's end. + void ConsumeAndVerifyFrame(const SimulatedFrame& sent_frame) { + SCOPED_TRACE(testing::Message() << "for frame " << sent_frame.frame_id); + + const std::optional payload_size = receiver()->AdvanceToNextFrame(); + ASSERT_TRUE(payload_size.has_value()); + std::vector buffer(*payload_size); + EncodedFrame received_frame = receiver()->ConsumeNextFrame(buffer); + + EXPECT_EQ(sent_frame.dependency, received_frame.dependency); + EXPECT_EQ(sent_frame.frame_id, received_frame.frame_id); + EXPECT_EQ(sent_frame.referenced_frame_id, + received_frame.referenced_frame_id); + EXPECT_EQ(sent_frame.rtp_timestamp, received_frame.rtp_timestamp); + EXPECT_THAT(sent_frame.reference_time + kOneWayNetworkDelay + + SimulatedFrame::GetExpectedPlayoutDelay( + sent_frame.frame_id - FrameId::first()) - + expected_player_processing_time_, + EqualsDuration(received_frame.reference_time)); + EXPECT_THAT(sent_frame.new_playout_delay, + EqualsDuration(received_frame.new_playout_delay)); + EXPECT_THAT(sent_frame.data, + testing::ElementsAreArray(received_frame.data)); + } + + // Consume zero or more frames from the Receiver, verifying that they are the + // same as the SimulatedFrame that was sent. + void ConsumeAndVerifyFrames(int first, + int last, + Clock::time_point start_time) { + for (int i = first; i <= last; ++i) { + ConsumeAndVerifyFrame(SimulatedFrame(start_time, i)); + } + } + + protected: + Clock::duration expected_player_processing_time_ = + Receiver::kDefaultPlayerProcessingTime; + + private: + FakeClock clock_; + FakeTaskRunner task_runner_; + testing::NiceMock env_; + ReceiverPacketRouter packet_router_; + std::unique_ptr receiver_; + testing::NiceMock sender_; + testing::NiceMock consumer_; + SimpleSubscriber socket_subscriber_; +}; + +// Tests that the Receiver processes RTCP packets correctly and sends RTCP +// reports at regular intervals. +TEST_F(ReceiverTest, ReceivesAndSendsRtcpPackets) { + // Sender-side expectations, after the Receiver has processed the first Sender + // Report. + Clock::time_point receiver_reference_time{}; + EXPECT_CALL(*sender(), OnReceiverReferenceTimeAdvanced(_)) + .WillOnce(SaveArg<0>(&receiver_reference_time)); + RtcpReportBlock receiver_report; + EXPECT_CALL(*sender(), OnReceiverReport(_)) + .WillOnce(SaveArg<0>(&receiver_report)); + EXPECT_CALL(*sender(), + OnReceiverCheckpoint(FrameId::leader(), kTargetPlayoutDelay)) + .Times(1); + + // Have the MockSender send a Sender Report with lip-sync timing information. + const Clock::time_point sender_reference_time = FakeClock::now(); + const RtpTimeTicks sender_rtp_timestamp = + RtpTimeTicks::FromTimeSinceOrigin(seconds(1), kRtpTimebase); + const StatusReportId sender_report_id = + sender()->SendSenderReport(sender_reference_time, sender_rtp_timestamp); + + AdvanceClockAndRunTasks(kRoundTripNetworkDelay); + + // Expect the MockSender got back a Receiver Report that includes its SSRC and + // the last Sender Report ID. + testing::Mock::VerifyAndClearExpectations(sender()); + EXPECT_EQ(kSenderSsrc, receiver_report.ssrc); + EXPECT_EQ(sender_report_id, receiver_report.last_status_report_id); + + // Confirm the clock offset math: Since the Receiver and MockSender share the + // same underlying FakeClock, the Receiver should be ahead of the Sender, + // which reflects the simulated one-way network packet travel time (of the + // Sender Report). + // + // Note: The offset can be affected by the lossy conversion when going to and + // from the wire-format NtpTimestamps. See the unit tests in + // ntp_time_unittest.cc for further discussion. + constexpr auto kAllowedNtpRoundingError = microseconds(2); + EXPECT_NEAR(to_microseconds(kOneWayNetworkDelay).count(), + static_cast(to_microseconds(receiver_reference_time - + sender_reference_time) + .count()), + kAllowedNtpRoundingError.count()); + + // Without the Sender doing anything, the Receiver should continue providing + // RTCP reports at regular intervals. Simulate three intervals of time, + // verifying that the Receiver did send reports. + Clock::time_point last_receiver_reference_time = receiver_reference_time; + for (int i = 0; i < 3; ++i) { + receiver_reference_time = Clock::time_point(); + EXPECT_CALL(*sender(), OnReceiverReferenceTimeAdvanced(_)) + .WillRepeatedly(SaveArg<0>(&receiver_reference_time)); + AdvanceClockAndRunTasks(kRtcpReportInterval); + testing::Mock::VerifyAndClearExpectations(sender()); + EXPECT_LT(last_receiver_reference_time, receiver_reference_time); + last_receiver_reference_time = receiver_reference_time; + } +} + +// Tests that the Receiver processes RTP packets, which might arrive in-order or +// out of order, but such that each frame is completely received in-order. Also, +// confirms that target playout delay changes are processed/applied correctly. +TEST_F(ReceiverTest, ReceivesFramesInOrder) { + const Clock::time_point start_time = FakeClock::now(); + ExchangeInitialReportPackets(start_time); + + EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(10); + for (int i = 0; i <= 9; ++i) { + EXPECT_CALL(*sender(), OnReceiverCheckpoint( + FrameId::first() + i, + SimulatedFrame::GetExpectedPlayoutDelay(i))) + .Times(1); + EXPECT_CALL(*sender(), OnReceiverIsMissingPackets(_)).Times(0); + + sender()->SetFrameBeingSent(SimulatedFrame(start_time, i)); + // Send the frame's packets in-order half the time, out-of-order the other + // half. + const int permutation = (i % 2) ? i : 0; + sender()->SendRtpPackets(sender()->GetAllPacketIds(permutation)); + + AdvanceClockAndRunTasks(kRoundTripNetworkDelay); + + // The Receiver should immediately ACK once it has received all the RTP + // packets to complete the frame. + testing::Mock::VerifyAndClearExpectations(sender()); + + // Advance to next frame transmission time. + AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration - + kRoundTripNetworkDelay); + } + + // When the Receiver has all of the frames and they are complete, it should + // send out a low-frequency periodic RTCP "ping." Verify that there is one and + // only one "ping" sent when the clock moves forward by one default report + // interval during a period of inactivity. + EXPECT_CALL(*sender(), OnReceiverCheckpoint(FrameId::first() + 9, + kTargetPlayoutDelayChange)) + .Times(1); + AdvanceClockAndRunTasks(kRtcpReportInterval); + testing::Mock::VerifyAndClearExpectations(sender()); + + ConsumeAndVerifyFrames(0, 9, start_time); + EXPECT_FALSE(receiver()->AdvanceToNextFrame().has_value()); +} + +// Tests that the Receiver processes RTP packets, can receive frames out of +// order, and issues the appropriate ACK/NACK feedback to the Sender as it +// realizes what it has and what it's missing. +TEST_F(ReceiverTest, ReceivesFramesOutOfOrder) { + const Clock::time_point start_time = FakeClock::now(); + ExchangeInitialReportPackets(start_time); + + // Send frames 3, 4, and 2. No NACKs expected yet. + sender()->SetFrameBeingSent(SimulatedFrame(start_time, 3)); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(kRoundTripNetworkDelay); + sender()->SetFrameBeingSent(SimulatedFrame(start_time, 4)); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(kRoundTripNetworkDelay); + sender()->SetFrameBeingSent(SimulatedFrame(start_time, 2)); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(kRoundTripNetworkDelay); + + // Send frame 0. Now there's a hole (frame 1) between received frames, so + // expect a NACK. + EXPECT_CALL(*sender(), OnReceiverIsMissingPackets(ElementsAre(PacketNack{ + FrameId::first() + 1, kAllPacketsLost}))) + .Times(1); + sender()->SetFrameBeingSent(SimulatedFrame(start_time, 0)); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(kRoundTripNetworkDelay); + testing::Mock::VerifyAndClearExpectations(sender()); + + // Send frame 1. Now all frames up to 4 are complete. Expect a checkpoint + // advancement, and also that the consumer is notified. + EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(AtLeast(1)); + EXPECT_CALL(*sender(), + OnReceiverCheckpoint(FrameId::first() + 4, kTargetPlayoutDelay)) + .Times(1); + sender()->SetFrameBeingSent(SimulatedFrame(start_time, 1)); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(kRoundTripNetworkDelay); + testing::Mock::VerifyAndClearExpectations(sender()); + + ConsumeAndVerifyFrames(0, 4, start_time); + EXPECT_FALSE(receiver()->AdvanceToNextFrame().has_value()); +} + +// Tests that the Receiver will respond to a key frame request from its client +// by sending a Picture Loss Indicator (PLI) to the Sender, and then will +// automatically stop sending the PLI once a key frame has been received. +TEST_F(ReceiverTest, RequestsKeyFrameToRectifyPictureLoss) { + const Clock::time_point start_time = FakeClock::now(); + ExchangeInitialReportPackets(start_time); + + // Send and Receive three frames in-order, normally. + for (int i = 0; i <= 2; ++i) { + ReceiveFrame(i, start_time, kTargetPlayoutDelay, kRoundTripNetworkDelay); + + // Advance to next frame transmission time. + AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration - + kRoundTripNetworkDelay); + } + ConsumeAndVerifyFrames(0, 2, start_time); + + // Simulate the Consumer requesting a key frame after picture loss (e.g., a + // decoder failure). Ensure the Sender is immediately notified. + EXPECT_CALL(*sender(), OnReceiverIndicatesPictureLoss()).Times(1); + receiver()->RequestKeyFrame(); + AdvanceClockAndRunTasks(kOneWayNetworkDelay); // Propagate request to Sender. + testing::Mock::VerifyAndClearExpectations(sender()); + + // The Sender sends another frame that is not a key frame and, upon receipt, + // the Receiver should repeat its "cry" for a key frame. + ReceiveFrame(3, start_time, kTargetPlayoutDelay, + SimulatedFrame::kFrameDuration - kOneWayNetworkDelay); + ConsumeAndVerifyFrames(3, 3, start_time); + + // Finally, the Sender responds to the PLI condition by sending a key frame. + // Confirm the Receiver has stopped indicating picture loss after having + // received the key frame. + EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); + EXPECT_CALL(*sender(), + OnReceiverCheckpoint(FrameId::first() + 4, kTargetPlayoutDelay)) + .Times(1); + EXPECT_CALL(*sender(), OnReceiverIndicatesPictureLoss()).Times(0); + SimulatedFrame key_frame(start_time, 4); + key_frame.dependency = EncodedFrame::Dependency::kKeyFrame; + key_frame.referenced_frame_id = key_frame.frame_id; + sender()->SetFrameBeingSent(key_frame); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); + testing::Mock::VerifyAndClearExpectations(sender()); + testing::Mock::VerifyAndClearExpectations(consumer()); + + // The client has not yet consumed the key frame, so any calls to + // RequestKeyFrame() should not set the PLI condition again. + EXPECT_CALL(*sender(), OnReceiverIndicatesPictureLoss()).Times(0); + receiver()->RequestKeyFrame(); + AdvanceClockAndRunTasks(kOneWayNetworkDelay); + testing::Mock::VerifyAndClearExpectations(sender()); + + // After consuming the requested key frame, the client should be able to set + // the PLI condition again with another RequestKeyFrame() call. + ConsumeAndVerifyFrame(key_frame); + EXPECT_CALL(*sender(), OnReceiverIndicatesPictureLoss()).Times(1); + receiver()->RequestKeyFrame(); + AdvanceClockAndRunTasks(kOneWayNetworkDelay); + testing::Mock::VerifyAndClearExpectations(sender()); +} + +TEST_F(ReceiverTest, PLICanBeDisabled) { + ConstructReceiver(false); + + EXPECT_CALL(*sender(), OnReceiverIndicatesPictureLoss()).Times(0); + receiver()->RequestKeyFrame(); + AdvanceClockAndRunTasks(kOneWayNetworkDelay); + testing::Mock::VerifyAndClearExpectations(sender()); +} + +// Tests that the Receiver will start dropping packets once its frame queue is +// full (i.e., when the consumer is not pulling them out of the queue). Since +// the Receiver will stop ACK'ing frames, the Sender will become stalled. +TEST_F(ReceiverTest, EatsItsFill) { + const Clock::time_point start_time = FakeClock::now(); + ExchangeInitialReportPackets(start_time); + + // Send and Receive the maximum possible number of frames in-order, normally. + for (int i = 0; i < kMaxUnackedFrames; ++i) { + EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); + EXPECT_CALL(*sender(), OnReceiverCheckpoint( + FrameId::first() + i, + SimulatedFrame::GetExpectedPlayoutDelay(i))) + .Times(1); + sender()->SetFrameBeingSent(SimulatedFrame(start_time, i)); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); + testing::Mock::VerifyAndClearExpectations(sender()); + testing::Mock::VerifyAndClearExpectations(consumer()); + } + + // Sending one more frame should be ignored. Over and over. None of the + // feedback reports from the Receiver should indicate it is collecting packets + // for future frames. + int ignored_frame = kMaxUnackedFrames; + for (int i = 0; i < 5; ++i) { + EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); + EXPECT_CALL(*sender(), + OnReceiverCheckpoint(FrameId::first() + (ignored_frame - 1), + kTargetPlayoutDelayChange)) + .Times(AtLeast(0)); + EXPECT_CALL(*sender(), OnReceiverIsMissingPackets(_)).Times(0); + sender()->SetFrameBeingSent(SimulatedFrame(start_time, ignored_frame)); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); + testing::Mock::VerifyAndClearExpectations(sender()); + testing::Mock::VerifyAndClearExpectations(consumer()); + } + + // Consume only one frame, and confirm the Receiver allows only one frame more + // to be received. + ConsumeAndVerifyFrames(0, 0, start_time); + int no_longer_ignored_frame = ignored_frame; + ++ignored_frame; + EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(AtLeast(1)); + EXPECT_CALL(*sender(), + OnReceiverCheckpoint(FrameId::first() + no_longer_ignored_frame, + kTargetPlayoutDelayChange)) + .Times(AtLeast(1)); + EXPECT_CALL(*sender(), OnReceiverIsMissingPackets(_)).Times(0); + // This frame should be received successfully. + sender()->SetFrameBeingSent( + SimulatedFrame(start_time, no_longer_ignored_frame)); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); + // This second frame should be ignored, however. + sender()->SetFrameBeingSent(SimulatedFrame(start_time, ignored_frame)); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); + testing::Mock::VerifyAndClearExpectations(sender()); + testing::Mock::VerifyAndClearExpectations(consumer()); +} + +// Tests that incomplete frames that would be played-out too late are dropped, +// but only as inter-frame data dependency requirements permit, and only if no +// target playout delay change information would have been missed. +TEST_F(ReceiverTest, DropsLateFrames) { + const Clock::time_point start_time = FakeClock::now(); + ExchangeInitialReportPackets(start_time); + + // Before any packets have been sent/received, the Receiver should indicate no + // frames are ready. + EXPECT_FALSE(receiver()->AdvanceToNextFrame().has_value()); + + // Set a ridiculously-large estimated player processing time so that the logic + // thinks every frame going to play out too late. + receiver()->SetPlayerProcessingTime(seconds(3)); + expected_player_processing_time_ = seconds(3); + + // In this test there are eight frames total: + // - Frame 0: Key frame. + // - Frames 1-4: Non-key frames. + // - Frame 5: Non-key frame that contains a target playout delay change. + // - Frame 6: Key frame. + // - Frame 7: Non-key frame. + ASSERT_EQ(SimulatedFrame::kPlayoutChangeAtFrame, 5); + SimulatedFrame frames[8] = {{start_time, 0}, {start_time, 1}, {start_time, 2}, + {start_time, 3}, {start_time, 4}, {start_time, 5}, + {start_time, 6}, {start_time, 7}}; + frames[6].dependency = EncodedFrame::Dependency::kKeyFrame; + frames[6].referenced_frame_id = frames[6].frame_id; + + // Send just packet 1 (NOT packet 0) of all the frames. The Receiver should + // never notify the consumer via the callback, nor report that any frames are + // ready, because none of the frames have been completely received. + EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); + EXPECT_CALL(*sender(), OnReceiverCheckpoint(_, _)).Times(0); + for (int i = 0; i <= 7; ++i) { + sender()->SetFrameBeingSent(frames[i]); + // Assumption: There are at least three packets in each frame, else the test + // is not exercising the logic meaningfully. + ASSERT_LE(size_t{3}, sender()->GetAllPacketIds().size()); + sender()->SendRtpPackets({FramePacketId{1}}); + AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); + } + testing::Mock::VerifyAndClearExpectations(consumer()); + testing::Mock::VerifyAndClearExpectations(sender()); + EXPECT_FALSE(receiver()->AdvanceToNextFrame().has_value()); + + // Send all the packets of Frame 6 (the second key frame) and Frame 7. The + // Receiver still cannot drop any frames because it has not seen packet 0 of + // every prior frame. In other words, it cannot ignore any possibility of a + // target playout delay change from the Sender. + EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); + EXPECT_CALL(*sender(), OnReceiverCheckpoint(_, _)).Times(0); + for (int i = 6; i <= 7; ++i) { + sender()->SetFrameBeingSent(frames[i]); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + } + AdvanceClockAndRunTasks(kRoundTripNetworkDelay); + testing::Mock::VerifyAndClearExpectations(consumer()); + testing::Mock::VerifyAndClearExpectations(sender()); + EXPECT_FALSE(receiver()->AdvanceToNextFrame().has_value()); + + // Send packet 0 for all but Frame 5, which contains a target playout delay + // change. All but the last two frames will still be incomplete. The Receiver + // still cannot drop any frames because it doesn't know whether Frame 5 had a + // target playout delay change. + EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); + EXPECT_CALL(*sender(), OnReceiverCheckpoint(_, _)).Times(0); + for (int i = 0; i <= 7; ++i) { + if (i == 5) { + continue; + } + sender()->SetFrameBeingSent(frames[i]); + sender()->SendRtpPackets({FramePacketId{0}}); + } + AdvanceClockAndRunTasks(kRoundTripNetworkDelay); + testing::Mock::VerifyAndClearExpectations(consumer()); + testing::Mock::VerifyAndClearExpectations(sender()); + EXPECT_FALSE(receiver()->AdvanceToNextFrame().has_value()); + + // Finally, send packet 0 for Frame 5. Now, the Receiver will drop every frame + // before the completely-received second key frame, as they are all still + // incomplete and will play-out too late. When it drops the frames, it will + // notify the sender of the new checkpoint so that it stops trying to + // re-transmit the dropped frames. + EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); + EXPECT_CALL(*sender(), OnReceiverCheckpoint(FrameId::first() + 7, + kTargetPlayoutDelayChange)) + .Times(1); + sender()->SetFrameBeingSent(frames[5]); + sender()->SendRtpPackets({FramePacketId{0}}); + AdvanceClockAndRunTasks(kRoundTripNetworkDelay); + // Note: Consuming Frame 6 will trigger the checkpoint advancement, since the + // call to AdvanceToNextFrame() contains the frame skipping/dropping logic. + ConsumeAndVerifyFrame(frames[6]); + testing::Mock::VerifyAndClearExpectations(consumer()); + testing::Mock::VerifyAndClearExpectations(sender()); + + // After consuming Frame 6, the Receiver knows Frame 7 is also available and + // should have scheduled an immediate task to notify the Consumer of this. + EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); + AdvanceClockAndRunTasks(kOneWayNetworkDelay); + testing::Mock::VerifyAndClearExpectations(consumer()); + + // Now consume Frame 7. This shouldn't trigger any further checkpoint + // advancement. + EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); + EXPECT_CALL(*sender(), OnReceiverCheckpoint(_, _)).Times(0); + ConsumeAndVerifyFrame(frames[7]); + AdvanceClockAndRunTasks(kOneWayNetworkDelay); + testing::Mock::VerifyAndClearExpectations(consumer()); + testing::Mock::VerifyAndClearExpectations(sender()); +} + +// Verifies that a playout event is correctly reported to the sender. +TEST_F(ReceiverTest, ReportsFrameAckAndPacketReceivedEvents) { + const Clock::time_point start_time = FakeClock::now(); + ExchangeInitialReportPackets(start_time); + + const SimulatedFrame kFrame(start_time, 0); + + // Intercept the playout event logs sent to the sender. + std::vector logs; + EXPECT_CALL(*sender(), OnCastReceiverFrameLogMessages(_)) + .WillRepeatedly([&](std::vector new_logs) { + logs.insert(logs.end(), new_logs.begin(), new_logs.end()); + }); + + // Send a frame, and consume it. + ReceiveFrame(0, start_time, kTargetPlayoutDelay, kRoundTripNetworkDelay); + ConsumeAndVerifyFrame(kFrame); + AdvanceClockAndRunTasks(kRtcpReportInterval); + + ASSERT_EQ(logs.size(), 1u); + const auto& frame_log = logs[0]; + EXPECT_EQ(frame_log.rtp_timestamp, kFrame.rtp_timestamp); + ASSERT_EQ(frame_log.messages.size(), 6u); + + // We should have gotten five valid packet received messages. + for (int i = 0; i < 5; ++i) { + const auto& message = frame_log.messages[i]; + EXPECT_EQ(message.type, StatisticsEvent::Type::kPacketReceived); + EXPECT_THAT(message.delay, EqualsDuration(milliseconds(0))); + EXPECT_THAT(message.timestamp, + EqualsDuration(start_time + milliseconds(9))); + } + + // And one frame acknowledgement. + EXPECT_EQ(frame_log.messages[5].type, StatisticsEvent::Type::kFrameAckSent); + EXPECT_THAT(frame_log.messages[5].delay, EqualsDuration(milliseconds(0))); + EXPECT_THAT(frame_log.messages[5].timestamp, + EqualsDuration(start_time + milliseconds(9))); +} + +// Verifies that a playout event is correctly reported to the sender. +TEST_F(ReceiverTest, ReportPlayoutEvent) { + const Clock::time_point start_time = FakeClock::now(); + ExchangeInitialReportPackets(start_time); + + const SimulatedFrame kPlayedOutFrame(start_time, 0); + + // Send a frame, and consume it. + ReceiveFrame(0, start_time, kTargetPlayoutDelay, kRoundTripNetworkDelay); + ConsumeAndVerifyFrame(kPlayedOutFrame); + + // Intercept the playout event logs sent to the sender. + std::vector logs; + EXPECT_CALL(*sender(), OnCastReceiverFrameLogMessages(_)) + .WillRepeatedly([&](std::vector new_logs) { + logs.insert(logs.end(), new_logs.begin(), new_logs.end()); + }); + + receiver()->ReportPlayoutEvent(kPlayedOutFrame.frame_id, + kPlayedOutFrame.rtp_timestamp, now()); + AdvanceClockAndRunTasks(kRtcpReportInterval + kOneWayNetworkDelay); + + ASSERT_EQ(logs.size(), 1u); + + // Only check out the log for the frame playout event. + const auto& frame_log = logs[0]; + EXPECT_EQ(frame_log.rtp_timestamp, kPlayedOutFrame.rtp_timestamp); + ASSERT_EQ(frame_log.messages.size(), 1u); + const auto& event_log1 = frame_log.messages[0]; + EXPECT_EQ(event_log1.type, StatisticsEvent::Type::kFramePlayedOut); + EXPECT_THAT(event_log1.delay, EqualsDuration(milliseconds(9))); + EXPECT_THAT(event_log1.timestamp, + EqualsDuration(start_time + milliseconds(12))); +} + +// This test ensures that frames played ahead of schedule are reported +// accurately, without causing any errors. +TEST_F(ReceiverTest, ReportPlayoutEventTooEarly) { + const Clock::time_point start_time = FakeClock::now(); + ExchangeInitialReportPackets(start_time); + + const SimulatedFrame kPlayedOutFrame(start_time, 0); + // Send a frame, and consume it. + ReceiveFrame(0, start_time, kTargetPlayoutDelay, kRoundTripNetworkDelay); + ConsumeAndVerifyFrame(kPlayedOutFrame); + + // Intercept the playout event logs sent to the sender. + EXPECT_CALL(*sender(), OnCastReceiverFrameLogMessages(_)) + .WillOnce([&](std::vector logs) { + ASSERT_EQ(logs.size(), 1u); + const auto& frame_log = logs[0]; + EXPECT_EQ(frame_log.rtp_timestamp, kPlayedOutFrame.rtp_timestamp); + ASSERT_EQ(frame_log.messages.size(), 1u); + + EXPECT_EQ(frame_log.messages[0].type, + StatisticsEvent::Type::kFramePlayedOut); + EXPECT_THAT(frame_log.messages[0].delay, + EqualsDuration(milliseconds(59))); + EXPECT_THAT(frame_log.messages[0].timestamp, + EqualsDuration(start_time + milliseconds(62))); + }); + + AdvanceClockAndRunTasks(milliseconds(50)); + receiver()->ReportPlayoutEvent(kPlayedOutFrame.frame_id, + kPlayedOutFrame.rtp_timestamp, now()); + AdvanceClockAndRunTasks(kRtcpReportInterval + kOneWayNetworkDelay); +} + +// Verifies that reporting a playout event for a frame that is too old +// results in a specific error. +TEST_F(ReceiverTest, ReportPlayoutEventForTooOldFrame) { + const Clock::time_point start_time = FakeClock::now(); + ExchangeInitialReportPackets(start_time); + + // Send and consume kMaxUnackedFrames + 1 frames. + for (int i = 0; i < kMaxUnackedFrames + 1; ++i) { + sender()->SetFrameBeingSent(SimulatedFrame(start_time, i)); + sender()->SendRtpPackets(sender()->GetAllPacketIds()); + AdvanceClockAndRunTasks(kRoundTripNetworkDelay); + const std::optional payload_size = receiver()->AdvanceToNextFrame(); + ASSERT_TRUE(payload_size.has_value()); + + std::vector buffer(*payload_size); + receiver()->ConsumeNextFrame(buffer); + } + + // Now, try to report a playout event for the first frame, which is now too + // old. + const SimulatedFrame kTooOldFrame(start_time, 0); + const Error error = receiver()->ReportPlayoutEvent( + kTooOldFrame.frame_id, kTooOldFrame.rtp_timestamp, now()); + EXPECT_EQ(error.code(), Error::Code::kParameterOutOfRange); +} + +// This test verifies that when the calculated playout delay is negative, it +// is correctly reported as zero (adjusting for the clock offset and network +// delay). +TEST_F(ReceiverTest, ReportPlayoutEventWithZeroDelay) { + const Clock::time_point start_time = FakeClock::now(); + ExchangeInitialReportPackets(start_time); + + const SimulatedFrame kPlayedOutFrame(start_time, 0); + + // Send a frame, and consume it. + ReceiveFrame(0, start_time, kTargetPlayoutDelay, kRoundTripNetworkDelay); + ConsumeAndVerifyFrame(kPlayedOutFrame); + + // Intercept the playout event logs sent to the sender. + EXPECT_CALL(*sender(), OnCastReceiverFrameLogMessages(_)) + .WillOnce([&](std::vector logs) { + ASSERT_EQ(logs.size(), 1u); + const auto& frame_log = logs[0]; + EXPECT_EQ(frame_log.rtp_timestamp, kPlayedOutFrame.rtp_timestamp); + ASSERT_EQ(frame_log.messages.size(), 1u); + + const auto& event_log1 = frame_log.messages[0]; + EXPECT_EQ(event_log1.type, StatisticsEvent::Type::kFramePlayedOut); + + EXPECT_THAT(event_log1.delay, EqualsDuration(milliseconds(9))); + EXPECT_THAT(event_log1.timestamp, + EqualsDuration(start_time + 2 * kRoundTripNetworkDelay)); + }); + + receiver()->ReportPlayoutEvent(kPlayedOutFrame.frame_id, + kPlayedOutFrame.rtp_timestamp, now()); + AdvanceClockAndRunTasks(kRtcpReportInterval + kOneWayNetworkDelay); +} + +} // namespace +} // namespace openscreen::cast diff --git a/cast/streaming/impl/receiver_message_unittest.cc b/cast/streaming/impl/receiver_message_unittest.cc index 1bf0f6411..1efe3796b 100644 --- a/cast/streaming/impl/receiver_message_unittest.cc +++ b/cast/streaming/impl/receiver_message_unittest.cc @@ -46,4 +46,16 @@ TEST(ReceiverMessageTest, ReceiverErrorToError) { ReceiverError(1234, "message two").ToError()); } +TEST(ReceiverMessageTest, ErrorOnNonObjectMessage) { + Json::Value array_val(Json::arrayValue); + EXPECT_TRUE(ReceiverMessage::Parse(array_val).is_error()); + EXPECT_TRUE(ReceiverError::Parse(array_val).is_error()); + EXPECT_TRUE(ReceiverCapability::Parse(array_val).is_error()); + + Json::Value string_val("string"); + EXPECT_TRUE(ReceiverMessage::Parse(string_val).is_error()); + EXPECT_TRUE(ReceiverError::Parse(string_val).is_error()); + EXPECT_TRUE(ReceiverCapability::Parse(string_val).is_error()); +} + } // namespace openscreen::cast diff --git a/cast/streaming/impl/receiver_packet_router.cc b/cast/streaming/impl/receiver_packet_router.cc index 23789ca88..44a1d28f5 100644 --- a/cast/streaming/impl/receiver_packet_router.cc +++ b/cast/streaming/impl/receiver_packet_router.cc @@ -7,7 +7,6 @@ #include #include "cast/streaming/impl/packet_util.h" -#include "cast/streaming/public/receiver.h" #include "platform/base/span.h" #include "util/osp_logging.h" #include "util/stringprintf.h" @@ -21,23 +20,23 @@ ReceiverPacketRouter::~ReceiverPacketRouter() { OSP_CHECK(receivers_.empty()); } -void ReceiverPacketRouter::OnReceiverCreated(Ssrc sender_ssrc, - Receiver* receiver) { +void ReceiverPacketRouter::RegisterPacketConsumer(Ssrc sender_ssrc, + PacketConsumer* receiver) { OSP_CHECK(receivers_.find(sender_ssrc) == receivers_.end()); receivers_.emplace_back(sender_ssrc, receiver); - // If there were no Receiver instances before, resume receiving packets for - // dispatch. Reset/Clear the remote endpoint, in preparation for later setting - // it to the source of the first packet received. + // If there were no PacketConsumer instances before, resume receiving packets + // for dispatch. Reset/Clear the remote endpoint, in preparation for later + // setting it to the source of the first packet received. if (receivers_.size() == 1) { environment_.set_remote_endpoint(IPEndpoint{}); environment_.ConsumeIncomingPackets(this); } } -void ReceiverPacketRouter::OnReceiverDestroyed(Ssrc sender_ssrc) { +void ReceiverPacketRouter::DeregisterPacketConsumer(Ssrc sender_ssrc) { receivers_.erase_key(sender_ssrc); - // If there are no longer any Receivers, suspend receiving packets. + // If there are no longer any PacketConsumers, suspend receiving packets. if (receivers_.empty()) { environment_.DropIncomingPackets(); } @@ -93,7 +92,7 @@ void ReceiverPacketRouter::OnReceivedPacket(const IPEndpoint& source, if (seems_like.first == ApparentPacketType::RTP) { it->second->OnReceivedRtpPacket(arrival_time, std::move(packet)); } else if (seems_like.first == ApparentPacketType::RTCP) { - it->second->OnReceivedRtcpPacket(arrival_time, std::move(packet)); + it->second->OnReceivedRtcpPacket(arrival_time, packet); } } diff --git a/cast/streaming/impl/receiver_packet_router.h b/cast/streaming/impl/receiver_packet_router.h index 206c37897..9e93eaf8a 100644 --- a/cast/streaming/impl/receiver_packet_router.h +++ b/cast/streaming/impl/receiver_packet_router.h @@ -26,21 +26,29 @@ class Receiver; // filtered-out. class ReceiverPacketRouter final : public Environment::PacketConsumer { public: + class PacketConsumer { + public: + virtual void OnReceivedRtpPacket(Clock::time_point arrival_time, + std::vector packet) = 0; + virtual void OnReceivedRtcpPacket(Clock::time_point arrival_time, + std::span packet) = 0; + + protected: + virtual ~PacketConsumer() = default; + }; + explicit ReceiverPacketRouter(Environment& environment); ~ReceiverPacketRouter() final; - protected: - friend class Receiver; - - // Called from a Receiver constructor/destructor to register/deregister a - // Receiver instance that processes RTP/RTCP packets from a Sender having the - // given SSRC. - void OnReceiverCreated(Ssrc sender_ssrc, Receiver* receiver); - void OnReceiverDestroyed(Ssrc sender_ssrc); + // Called from a PacketConsumer constructor/destructor to register/deregister + // a PacketConsumer instance that processes RTP/RTCP packets from a Sender + // having the given SSRC. + void RegisterPacketConsumer(Ssrc sender_ssrc, PacketConsumer* consumer); + void DeregisterPacketConsumer(Ssrc sender_ssrc); - // Called by a Receiver to send a RTCP packet back to the source from which - // earlier packets were received, or does nothing if OnReceivedPacket() has - // not been called yet. + // Called by a PacketConsumer to send a RTCP packet back to the source from + // which earlier packets were received, or does nothing if OnReceivedPacket() + // has not been called yet. void SendRtcpPacket(ByteView packet); private: @@ -51,7 +59,7 @@ class ReceiverPacketRouter final : public Environment::PacketConsumer { Environment& environment_; - FlatMap receivers_; + FlatMap receivers_; }; } // namespace openscreen::cast diff --git a/cast/streaming/impl/receiver_session_unittest.cc b/cast/streaming/impl/receiver_session_unittest.cc index 5230866c8..83aa5cd53 100644 --- a/cast/streaming/impl/receiver_session_unittest.cc +++ b/cast/streaming/impl/receiver_session_unittest.cc @@ -4,8 +4,12 @@ #include "cast/streaming/public/receiver_session.h" +#include +#include #include +#include "cast/streaming/impl/message_constants.h" +#include "cast/streaming/input.pb.h" #include "cast/streaming/public/receiver.h" #include "cast/streaming/testing/mock_environment.h" #include "cast/streaming/testing/simple_message_port.h" @@ -16,10 +20,10 @@ #include "platform/test/fake_task_runner.h" #include "util/chrono_helpers.h" #include "util/json/json_serialization.h" +#include "util/std_util.h" using ::testing::_; using ::testing::InSequence; -using ::testing::Invoke; using ::testing::NiceMock; using ::testing::Return; using ::testing::StrictMock; @@ -48,6 +52,7 @@ constexpr char kValidOfferMessage[] = R"({ "level": "4", "aesKey": "bbf109bf84513b456b13a184453b66ce", "aesIvMask": "edaf9e4536e2b66191f560d9c04b2a69", + "receiverRtcpDscp": 46, "resolutions": [ { "width": 1280, @@ -69,6 +74,7 @@ constexpr char kValidOfferMessage[] = R"({ "level": "4", "aesKey": "040d756791711fd3adb939066e6d8690", "aesIvMask": "9ff0f022a959150e70a2d05a6c184aed", + "receiverRtcpDscp": 46, "resolutions": [ { "width": 1280, @@ -89,6 +95,7 @@ constexpr char kValidOfferMessage[] = R"({ "maxBitRate": 5000000, "aesKey": "040d756791711fd3adb939066e6d8690", "aesIvMask": "9ff0f022a959150e70a2d05a6c184aed", + "receiverRtcpDscp": 46, "resolutions": [ { "width": 1920, @@ -107,7 +114,8 @@ constexpr char kValidOfferMessage[] = R"({ "timeBase": "1/48000", "channels": 2, "aesKey": "51027e4e2347cbcb49d57ef10177aebc", - "aesIvMask": "7f12a19be62a36c04ae4116caaeff6d1" + "aesIvMask": "7f12a19be62a36c04ae4116caaeff6d1", + "receiverRtcpDscp": 46 } ] } @@ -311,6 +319,44 @@ constexpr char kRpcMessage[] = R"({ "type" : "RPC" })"; +constexpr char kValidOfferMessageWithInput[] = R"({ + "type": "OFFER", + "seqNum": 1337, + "offer": { + "castMode": "mirroring", + "supportedStreams": [ + { + "index": 31338, + "type": "video_source", + "codecName": "vp8", + "rtpProfile": "cast", + "rtpPayloadType": 127, + "ssrc": 19088745, + "maxFrameRate": "60000/1000", + "timeBase": "1/90000", + "maxBitRate": 5000000, + "profile": "main", + "level": "4", + "aesKey": "040d756791711fd3adb939066e6d8690", + "aesIvMask": "9ff0f022a959150e70a2d05a6c184aed", + "rtpExtensions": ["input_events"], + "resolutions": [ + { + "width": 1280, + "height": 720 + } + ] + } + ] + } +})"; + +constexpr char kInputMessage[] = R"({ + "input" : "CGQQnBiCGQgSAggMGgIIBg==", + "seqNum" : 3, + "type" : "INPUT" +})"; + class FakeClient : public ReceiverSession::Client { public: MOCK_METHOD(void, @@ -1028,4 +1074,178 @@ TEST_F(ReceiverSessionTest, HandlesRpcMessage) { ASSERT_TRUE(received_initialize_message); } +TEST_F(ReceiverSessionTest, EnablesDscpInAnswer) { + ReceiverConstraints constraints; + constraints.enable_dscp = true; + SetUpWithConstraints(std::move(constraints)); + EXPECT_CALL(client_, OnNegotiated(session_.get(), _)); + EXPECT_CALL(client_, + OnReceiversDestroying(session_.get(), + ReceiverSession::Client::kEndOfSession)); + message_port_->ReceiveMessage(kValidOfferMessage); + const std::vector& messages = message_port_->posted_messages(); + ASSERT_EQ(1u, messages.size()); + Json::Value message = ExpectIsValidAnswer(messages[0]); + const Json::Value& answer = message["answer"]; + ASSERT_TRUE(answer.isObject()); + const Json::Value& dscp = answer["receiverRtcpDscp"]; + ASSERT_FALSE(dscp.empty()); + EXPECT_EQ(dscp[0], 1337); + EXPECT_EQ(dscp[1], 31338); + + message_port_->clear(); + ReceiverConstraints constraints2; + constraints2.enable_dscp = false; + SetUpWithConstraints(std::move(constraints2)); + EXPECT_CALL(client_, OnNegotiated(session_.get(), _)); + EXPECT_CALL(client_, + OnReceiversDestroying(session_.get(), + ReceiverSession::Client::kEndOfSession)); + message_port_->ReceiveMessage(kValidOfferMessage); // Re-send offer message + const std::vector& messages2 = message_port_->posted_messages(); + ASSERT_EQ(1u, messages2.size()); + message = ExpectIsValidAnswer(messages2[0]); + const Json::Value& answer2 = message["answer"]; + ASSERT_TRUE(answer2.isObject()); + const Json::Value& dscp2 = answer2["receiverRtcpDscp"]; + ASSERT_TRUE(dscp2.empty()); +} + +TEST_F(ReceiverSessionTest, InputEventsOptIn) { + ReceiverConstraints constraints; + constraints.supports_input_events = true; + SetUpWithConstraints(std::move(constraints)); + + EXPECT_CALL(client_, OnNegotiated(session_.get(), _)) + .WillOnce([](const ReceiverSession* session, + ReceiverSession::ConfiguredReceivers cr) { + EXPECT_TRUE(cr.input_enabled); + }); + EXPECT_CALL(client_, + OnReceiversDestroying(session_.get(), + ReceiverSession::Client::kEndOfSession)); + + message_port_->ReceiveMessage(kValidOfferMessageWithInput); + + const std::vector& messages = message_port_->posted_messages(); + ASSERT_EQ(1u, messages.size()); + Json::Value message = ExpectIsValidAnswer(messages[0]); + const Json::Value& answer = message["answer"]; + const Json::Value& extensions = answer["rtpExtensions"]; + EXPECT_TRUE( + std::ranges::any_of(extensions, [](const Json::Value& stream_extensions) { + return Contains(stream_extensions, kInputEventsRtpExtension); + })); +} + +TEST_F(ReceiverSessionTest, HandlesInputMessage) { + ReceiverConstraints constraints; + constraints.supports_input_events = true; + SetUpWithConstraints(std::move(constraints)); + + message_port_->ReceiveMessage(kInputMessage); + // Nothing should happen yet, the session doesn't have a messenger. + ASSERT_EQ(0u, message_port_->posted_messages().size()); + + InSequence s; + bool received_negotiation = false; + EXPECT_CALL(client_, OnNegotiated(session_.get(), _)) + .WillOnce([&received_negotiation]( + const ReceiverSession* session, + ReceiverSession::ConfiguredReceivers receivers) mutable { + ASSERT_TRUE(receivers.input_enabled); + received_negotiation = true; + }); + EXPECT_CALL(client_, + OnReceiversDestroying(session_.get(), + ReceiverSession::Client::kEndOfSession)); + + message_port_->ReceiveMessage(kValidOfferMessageWithInput); + ASSERT_TRUE(received_negotiation); +} + +TEST_F(ReceiverSessionTest, HandlesInputMessengerNotNegotiated) { + ReceiverConstraints constraints; + constraints.supports_input_events = false; + SetUpWithConstraints(std::move(constraints)); + + EXPECT_CALL(client_, OnNegotiated(session_.get(), _)) + .WillOnce([](const ReceiverSession* session, + ReceiverSession::ConfiguredReceivers cr) { + EXPECT_FALSE(cr.input_enabled); + }); + EXPECT_CALL(client_, + OnReceiversDestroying(session_.get(), + ReceiverSession::Client::kEndOfSession)); + + message_port_->ReceiveMessage(kValidOfferMessageWithInput); +} + +TEST_F(ReceiverSessionTest, HandlesInputMessengerSendsMessage) { + ReceiverConstraints constraints; + constraints.supports_input_events = true; + SetUpWithConstraints(std::move(constraints)); + + EXPECT_CALL(client_, OnNegotiated(session_.get(), _)) + .WillOnce([this](const ReceiverSession* session, + ReceiverSession::ConfiguredReceivers cr) { + ASSERT_TRUE(cr.input_enabled); + + InputMessage message; + auto* event = message.add_events(); + event->set_type(InputMessage::INPUT_TYPE_KEY_DOWN); + this->session_->SendInputMessage(message); + }); + EXPECT_CALL(client_, + OnReceiversDestroying(session_.get(), + ReceiverSession::Client::kEndOfSession)); + + message_port_->ReceiveMessage(kValidOfferMessageWithInput); + + // Verify message was sent through message port. + const auto& messages = message_port_->posted_messages(); + + // We expect at least 2 messages: ANSWER and INPUT. + ASSERT_GE(messages.size(), 2u); + + // Index 0 is ANSWER. + ExpectIsValidAnswer(messages[0]); + + // Index 1 should be INPUT. + ErrorOr input_json = json::Parse(messages[1]); + ASSERT_TRUE(input_json.is_value()); + EXPECT_EQ("INPUT", input_json.value()["type"].asString()); + EXPECT_FALSE(input_json.value()["input"].asString().empty()); +} + +TEST_F(ReceiverSessionTest, HandlesInputMessengerReceivesMessage) { + ReceiverConstraints constraints; + constraints.supports_input_events = true; + SetUpWithConstraints(std::move(constraints)); + + bool received_input = false; + session_->SetInputCallback( + [&received_input](InputMessage message) { received_input = true; }); + + EXPECT_CALL(client_, OnNegotiated(session_.get(), _)) + .WillOnce([](const ReceiverSession* session, + ReceiverSession::ConfiguredReceivers cr) { + ASSERT_TRUE(cr.input_enabled); + }); + + message_port_->ReceiveMessage(kValidOfferMessageWithInput); + + message_port_->ReceiveMessage(kInputMessage); + ASSERT_TRUE(received_input); + + EXPECT_CALL(client_, + OnReceiversDestroying(session_.get(), + ReceiverSession::Client::kEndOfSession)); +} + +TEST_F(ReceiverSessionTest, UnsubscribesFromEnvironmentOnDestruction) { + session_.reset(); + environment_->SetSocketStateForTesting(Environment::SocketState::kReady); +} + } // namespace openscreen::cast diff --git a/cast/streaming/impl/receiver_unittest.cc b/cast/streaming/impl/receiver_unittest.cc deleted file mode 100644 index 231490f3b..000000000 --- a/cast/streaming/impl/receiver_unittest.cc +++ /dev/null @@ -1,852 +0,0 @@ -// Copyright 2019 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "cast/streaming/public/receiver.h" - -#include - -#include -#include -#include -#include - -#include "cast/streaming/impl/compound_rtcp_parser.h" -#include "cast/streaming/impl/frame_crypto.h" -#include "cast/streaming/impl/receiver_packet_router.h" -#include "cast/streaming/impl/rtcp_common.h" -#include "cast/streaming/impl/rtcp_session.h" -#include "cast/streaming/impl/rtp_defines.h" -#include "cast/streaming/impl/rtp_packetizer.h" -#include "cast/streaming/impl/sender_report_builder.h" -#include "cast/streaming/impl/session_config.h" -#include "cast/streaming/public/constants.h" -#include "cast/streaming/public/encoded_frame.h" -#include "cast/streaming/public/environment.h" -#include "cast/streaming/rtp_time.h" -#include "cast/streaming/ssrc.h" -#include "cast/streaming/testing/mock_environment.h" -#include "cast/streaming/testing/simple_socket_subscriber.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "platform/api/time.h" -#include "platform/api/udp_socket.h" -#include "platform/base/error.h" -#include "platform/base/ip_address.h" -#include "platform/base/span.h" -#include "platform/base/udp_packet.h" -#include "platform/test/byte_view_test_util.h" -#include "platform/test/fake_clock.h" -#include "platform/test/fake_task_runner.h" -#include "util/chrono_helpers.h" -#include "util/osp_logging.h" - -using testing::_; -using testing::AtLeast; -using testing::Gt; -using testing::Invoke; -using testing::SaveArg; - -namespace openscreen::cast { -namespace { - -// Receiver configuration. - -constexpr Ssrc kSenderSsrc = 1; -constexpr Ssrc kReceiverSsrc = 2; -constexpr int kRtpTimebase = 48000; -constexpr milliseconds kTargetPlayoutDelay(100); -constexpr auto kAesKey = - std::array{{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}}; -constexpr auto kCastIvMask = - std::array{{0xf0, 0xe0, 0xd0, 0xc0, 0xb0, 0xa0, 0x90, 0x80, - 0x70, 0x60, 0x50, 0x40, 0x30, 0x20, 0x10, 0x00}}; - -constexpr milliseconds kTargetPlayoutDelayChange(800); -// Additional configuration for the Sender. -constexpr RtpPayloadType kRtpPayloadType = RtpPayloadType::kVideoVp8; -constexpr int kMaxRtpPacketSize = 64; - -// A simulated one-way network delay, and round-trip network delay. -constexpr auto kOneWayNetworkDelay = milliseconds(3); -constexpr auto kRoundTripNetworkDelay = 2 * kOneWayNetworkDelay; -static_assert(kRoundTripNetworkDelay < kTargetPlayoutDelay && - kRoundTripNetworkDelay < kTargetPlayoutDelayChange, - "Network delay must be smaller than target playout delay."); - -// An EncodedFrame for unit testing, one of a sequence of simulated frames, each -// of 10 ms duration. The first frame will be a key frame; and any later frames -// will be non-key, dependent on the prior frame. Frame 5 (the 6th frame in the -// zero-based sequence) will include a target playout delay change, an increase -// to 800 ms. Frames with different IDs will contain vary in their payload data -// size, but are always 3 or more packets' worth of data. -struct SimulatedFrame : public EncodedFrame { - static constexpr milliseconds kFrameDuration = milliseconds(10); - static constexpr milliseconds kTargetPlayoutDelayChange = milliseconds(800); - - static constexpr int kPlayoutChangeAtFrame = 5; - - SimulatedFrame(Clock::time_point first_frame_reference_time, int which) { - frame_id = FrameId::first() + which; - if (which == 0) { - dependency = EncodedFrame::Dependency::kKeyFrame; - referenced_frame_id = frame_id; - } else { - dependency = EncodedFrame::Dependency::kDependent; - referenced_frame_id = frame_id - 1; - } - rtp_timestamp = - GetRtpStartTime() + - RtpTimeDelta::FromDuration(kFrameDuration * which, kRtpTimebase); - reference_time = first_frame_reference_time + kFrameDuration * which; - if (which == kPlayoutChangeAtFrame) { - new_playout_delay = kTargetPlayoutDelayChange; - } - constexpr int kAdditionalBytesEachSuccessiveFrame = 3; - buffer_.resize(3 * kMaxRtpPacketSize + - which * kAdditionalBytesEachSuccessiveFrame); - for (size_t i = 0; i < buffer_.size(); ++i) { - buffer_[i] = static_cast(which + static_cast(i)); - } - data = buffer_; - } - - static RtpTimeTicks GetRtpStartTime() { - return RtpTimeTicks::FromTimeSinceOrigin(seconds(0), kRtpTimebase); - } - - static milliseconds GetExpectedPlayoutDelay(int which) { - return (which < kPlayoutChangeAtFrame) ? kTargetPlayoutDelay - : kTargetPlayoutDelayChange; - } - - private: - std::vector buffer_; -}; - -// static -constexpr milliseconds SimulatedFrame::kFrameDuration; -constexpr milliseconds SimulatedFrame::kTargetPlayoutDelayChange; -constexpr int SimulatedFrame::kPlayoutChangeAtFrame; - -// Processes packets from the Receiver under test, as a real Sender might, and -// allows the unit tests to set expectations on events of interest to confirm -// proper behavior of the Receiver. -class MockSender : public CompoundRtcpParser::Client { - public: - MockSender(TaskRunner& task_runner, UdpSocket::Client* receiver) - : task_runner_(task_runner), - receiver_(receiver), - sender_endpoint_{ - // Use a random IPv6 address in the range reserved for - // "documentation purposes." Thus, the following is a fake address - // that should be blocked by the OS (and all network packet - // routers). But, these tests don't use real sockets, so... - IPAddress::Parse("2001:db8:0d93:69c2:fd1a:49a6:a7c0:e8a6").value(), - 2344}, - rtcp_session_(kSenderSsrc, kReceiverSsrc, FakeClock::now()), - sender_report_builder_(rtcp_session_), - rtcp_parser_(rtcp_session_, *this), - crypto_(kAesKey, kCastIvMask), - rtp_packetizer_(kRtpPayloadType, kSenderSsrc, kMaxRtpPacketSize) {} - - ~MockSender() override = default; - - void set_max_feedback_frame_id(FrameId f) { max_feedback_frame_id_ = f; } - - // Called by the test procedures to generate a Sender Report containing the - // given lip-sync timestamps, and send it to the Receiver. The caller must - // spin the TaskRunner for the RTCP packet to be delivered to the Receiver. - StatusReportId SendSenderReport(Clock::time_point reference_time, - RtpTimeTicks rtp_timestamp) { - // Generate the Sender Report RTCP packet. - uint8_t buffer[kMaxRtpPacketSizeForIpv4UdpOnEthernet]; - RtcpSenderReport sender_report; - sender_report.reference_time = reference_time; - sender_report.rtp_timestamp = rtp_timestamp; - const auto packet_and_report_id = - sender_report_builder_.BuildPacket(sender_report, buffer); - - // Send the RTCP packet as a UdpPacket directly to the Receiver instance. - UdpPacket packet_to_send(packet_and_report_id.first.begin(), - packet_and_report_id.first.end()); - packet_to_send.set_source(sender_endpoint_); - task_runner_.PostTaskWithDelay( - [receiver = receiver_, packet = std::move(packet_to_send)]() mutable { - receiver->OnRead(nullptr, ErrorOr(std::move(packet))); - }, - kOneWayNetworkDelay); - - return packet_and_report_id.second; - } - - // Sets which frame is currently being sent by this MockSender. Test code must - // call SendRtpPackets() to send the packets. - void SetFrameBeingSent(const EncodedFrame& frame) { - frame_being_sent_ = crypto_.Encrypt(frame); - } - - // Returns a vector containing each packet ID once (of the current frame being - // sent). `permutation` controls the sort order of the vector: zero will - // provide all the packet IDs in order, and greater values will provide them - // in a different, predictable order. - std::vector GetAllPacketIds(int permutation) { - const int num_packets = - rtp_packetizer_.ComputeNumberOfPackets(frame_being_sent_); - OSP_CHECK_GT(num_packets, 0); - std::vector ids; - ids.reserve(num_packets); - const FramePacketId last_packet_id = - static_cast(num_packets - 1); - for (FramePacketId packet_id = 0; packet_id <= last_packet_id; - ++packet_id) { - ids.push_back(packet_id); - } - for (int i = 0; i < permutation; ++i) { - std::next_permutation(ids.begin(), ids.end()); - } - return ids; - } - - // Send the specified packets of the current frame being sent. - void SendRtpPackets(const std::vector& packets_to_send) { - uint8_t buffer[kMaxRtpPacketSize]; - for (FramePacketId packet_id : packets_to_send) { - const auto span = rtp_packetizer_.GeneratePacket( - frame_being_sent_, packet_id, ByteBuffer(buffer, kMaxRtpPacketSize)); - UdpPacket packet_to_send(span.begin(), span.end()); - packet_to_send.set_source(sender_endpoint_); - task_runner_.PostTaskWithDelay( - [receiver = receiver_, packet = std::move(packet_to_send)]() mutable { - receiver->OnRead(nullptr, ErrorOr(std::move(packet))); - }, - kOneWayNetworkDelay); - } - } - - // Called to process a packet from the Receiver. - void OnPacketFromReceiver(ByteView packet) { - EXPECT_TRUE(rtcp_parser_.Parse(packet, max_feedback_frame_id_)); - } - - // CompoundRtcpParser::Client implementation: Tests set expectations on these - // mocks to confirm that the receiver is providing the right data to the - // sender in its RTCP packets. - MOCK_METHOD1(OnReceiverReferenceTimeAdvanced, - void(Clock::time_point reference_time)); - MOCK_METHOD1(OnReceiverReport, void(const RtcpReportBlock& receiver_report)); - MOCK_METHOD0(OnReceiverIndicatesPictureLoss, void()); - MOCK_METHOD2(OnReceiverCheckpoint, - void(FrameId frame_id, milliseconds playout_delay)); - MOCK_METHOD1(OnReceiverHasFrames, void(std::vector acks)); - MOCK_METHOD1(OnReceiverIsMissingPackets, void(std::vector nacks)); - - private: - TaskRunner& task_runner_; - UdpSocket::Client* const receiver_; - const IPEndpoint sender_endpoint_; - RtcpSession rtcp_session_; - SenderReportBuilder sender_report_builder_; - CompoundRtcpParser rtcp_parser_; - FrameCrypto crypto_; - RtpPacketizer rtp_packetizer_; - FrameId max_feedback_frame_id_ = FrameId::first() + kMaxUnackedFrames; - - EncryptedFrame frame_being_sent_; -}; - -class MockConsumer : public Receiver::Consumer { - public: - MOCK_METHOD1(OnFramesReady, void(int next_frame_buffer_size)); -}; - -class ReceiverTest : public testing::Test { - public: - ReceiverTest() - : clock_(Clock::now()), - task_runner_(clock_), - env_(&FakeClock::now, task_runner_), - packet_router_(env_), - receiver_(env_, - packet_router_, - {/* .sender_ssrc = */ kSenderSsrc, - /* .receiver_ssrc = */ kReceiverSsrc, - /* .rtp_timebase = */ kRtpTimebase, - /* .channels = */ 2, - /* .target_playout_delay = */ kTargetPlayoutDelay, - /* .aes_secret_key = */ kAesKey, - /* .aes_iv_mask = */ kCastIvMask, - /* .is_pli_enabled = */ true}), - sender_(task_runner_, &env_) { - env_.SetSocketSubscriber(&socket_subscriber_); - ON_CALL(env_, SendPacket(_, _)) - .WillByDefault(Invoke([this](ByteView packet, PacketMetadata metadata) { - task_runner_.PostTaskWithDelay( - [sender = &sender_, copy_of_packet = std::vector( - packet.begin(), packet.end())]() mutable { - sender->OnPacketFromReceiver(std::move(copy_of_packet)); - }, - kOneWayNetworkDelay); - })); - receiver_.SetConsumer(&consumer_); - } - - ~ReceiverTest() override = default; - - Receiver* receiver() { return &receiver_; } - MockSender* sender() { return &sender_; } - MockConsumer* consumer() { return &consumer_; } - - void AdvanceClockAndRunTasks(Clock::duration delta) { clock_.Advance(delta); } - void RunTasksUntilIdle() { task_runner_.RunTasksUntilIdle(); } - - // Sends the initial Sender Report with lip-sync timing information to - // "unblock" the Receiver, and confirms the Receiver immediately replies with - // a corresponding Receiver Report. - void ExchangeInitialReportPackets() { - const Clock::time_point start_time = FakeClock::now(); - sender_.SendSenderReport(start_time, SimulatedFrame::GetRtpStartTime()); - AdvanceClockAndRunTasks( - kOneWayNetworkDelay); // Transmit report to Receiver. - // The Receiver will immediately reply with a Receiver Report. - EXPECT_CALL(sender_, - OnReceiverCheckpoint(FrameId::leader(), kTargetPlayoutDelay)) - .Times(1); - AdvanceClockAndRunTasks(kOneWayNetworkDelay); // Transmit reply to Sender. - testing::Mock::VerifyAndClearExpectations(&sender_); - } - - // Consume one frame from the Receiver, and verify that it is the same as the - // `sent_frame`. Exception: The `reference_time` is the playout time on the - // Receiver's end, while it refers to the capture time on the Sender's end. - void ConsumeAndVerifyFrame(const SimulatedFrame& sent_frame) { - SCOPED_TRACE(testing::Message() << "for frame " << sent_frame.frame_id); - - const int payload_size = receiver()->AdvanceToNextFrame(); - ASSERT_NE(Receiver::kNoFramesReady, payload_size); - std::vector buffer(payload_size); - EncodedFrame received_frame = receiver()->ConsumeNextFrame(buffer); - - EXPECT_EQ(sent_frame.dependency, received_frame.dependency); - EXPECT_EQ(sent_frame.frame_id, received_frame.frame_id); - EXPECT_EQ(sent_frame.referenced_frame_id, - received_frame.referenced_frame_id); - EXPECT_EQ(sent_frame.rtp_timestamp, received_frame.rtp_timestamp); - EXPECT_EQ(sent_frame.reference_time + kOneWayNetworkDelay + - SimulatedFrame::GetExpectedPlayoutDelay(sent_frame.frame_id - - FrameId::first()), - received_frame.reference_time); - EXPECT_EQ(sent_frame.new_playout_delay, received_frame.new_playout_delay); - ExpectByteViewsHaveSameBytes(sent_frame.data, received_frame.data); - } - - // Consume zero or more frames from the Receiver, verifying that they are the - // same as the SimulatedFrame that was sent. - void ConsumeAndVerifyFrames(int first, - int last, - Clock::time_point start_time) { - for (int i = first; i <= last; ++i) { - ConsumeAndVerifyFrame(SimulatedFrame(start_time, i)); - } - } - - private: - FakeClock clock_; - FakeTaskRunner task_runner_; - testing::NiceMock env_; - ReceiverPacketRouter packet_router_; - Receiver receiver_; - testing::NiceMock sender_; - testing::NiceMock consumer_; - SimpleSubscriber socket_subscriber_; -}; - -// Tests that the Receiver processes RTCP packets correctly and sends RTCP -// reports at regular intervals. -TEST_F(ReceiverTest, ReceivesAndSendsRtcpPackets) { - // Sender-side expectations, after the Receiver has processed the first Sender - // Report. - Clock::time_point receiver_reference_time{}; - EXPECT_CALL(*sender(), OnReceiverReferenceTimeAdvanced(_)) - .WillOnce(SaveArg<0>(&receiver_reference_time)); - RtcpReportBlock receiver_report; - EXPECT_CALL(*sender(), OnReceiverReport(_)) - .WillOnce(SaveArg<0>(&receiver_report)); - EXPECT_CALL(*sender(), - OnReceiverCheckpoint(FrameId::leader(), kTargetPlayoutDelay)) - .Times(1); - - // Have the MockSender send a Sender Report with lip-sync timing information. - const Clock::time_point sender_reference_time = FakeClock::now(); - const RtpTimeTicks sender_rtp_timestamp = - RtpTimeTicks::FromTimeSinceOrigin(seconds(1), kRtpTimebase); - const StatusReportId sender_report_id = - sender()->SendSenderReport(sender_reference_time, sender_rtp_timestamp); - - AdvanceClockAndRunTasks(kRoundTripNetworkDelay); - - // Expect the MockSender got back a Receiver Report that includes its SSRC and - // the last Sender Report ID. - testing::Mock::VerifyAndClearExpectations(sender()); - EXPECT_EQ(kSenderSsrc, receiver_report.ssrc); - EXPECT_EQ(sender_report_id, receiver_report.last_status_report_id); - - // Confirm the clock offset math: Since the Receiver and MockSender share the - // same underlying FakeClock, the Receiver should be ahead of the Sender, - // which reflects the simulated one-way network packet travel time (of the - // Sender Report). - // - // Note: The offset can be affected by the lossy conversion when going to and - // from the wire-format NtpTimestamps. See the unit tests in - // ntp_time_unittest.cc for further discussion. - constexpr auto kAllowedNtpRoundingError = microseconds(2); - EXPECT_NEAR(to_microseconds(kOneWayNetworkDelay).count(), - static_cast(to_microseconds(receiver_reference_time - - sender_reference_time) - .count()), - kAllowedNtpRoundingError.count()); - - // Without the Sender doing anything, the Receiver should continue providing - // RTCP reports at regular intervals. Simulate three intervals of time, - // verifying that the Receiver did send reports. - Clock::time_point last_receiver_reference_time = receiver_reference_time; - for (int i = 0; i < 3; ++i) { - receiver_reference_time = Clock::time_point(); - EXPECT_CALL(*sender(), OnReceiverReferenceTimeAdvanced(_)) - .WillRepeatedly(SaveArg<0>(&receiver_reference_time)); - AdvanceClockAndRunTasks(kRtcpReportInterval); - testing::Mock::VerifyAndClearExpectations(sender()); - EXPECT_LT(last_receiver_reference_time, receiver_reference_time); - last_receiver_reference_time = receiver_reference_time; - } -} - -// Tests that the Receiver processes RTP packets, which might arrive in-order or -// out of order, but such that each frame is completely received in-order. Also, -// confirms that target playout delay changes are processed/applied correctly. -TEST_F(ReceiverTest, ReceivesFramesInOrder) { - const Clock::time_point start_time = FakeClock::now(); - ExchangeInitialReportPackets(); - - EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(10); - for (int i = 0; i <= 9; ++i) { - EXPECT_CALL(*sender(), OnReceiverCheckpoint( - FrameId::first() + i, - SimulatedFrame::GetExpectedPlayoutDelay(i))) - .Times(1); - EXPECT_CALL(*sender(), OnReceiverIsMissingPackets(_)).Times(0); - - sender()->SetFrameBeingSent(SimulatedFrame(start_time, i)); - // Send the frame's packets in-order half the time, out-of-order the other - // half. - const int permutation = (i % 2) ? i : 0; - sender()->SendRtpPackets(sender()->GetAllPacketIds(permutation)); - - AdvanceClockAndRunTasks(kRoundTripNetworkDelay); - - // The Receiver should immediately ACK once it has received all the RTP - // packets to complete the frame. - testing::Mock::VerifyAndClearExpectations(sender()); - - // Advance to next frame transmission time. - AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration - - kRoundTripNetworkDelay); - } - - // When the Receiver has all of the frames and they are complete, it should - // send out a low-frequency periodic RTCP "ping." Verify that there is one and - // only one "ping" sent when the clock moves forward by one default report - // interval during a period of inactivity. - EXPECT_CALL(*sender(), OnReceiverCheckpoint(FrameId::first() + 9, - kTargetPlayoutDelayChange)) - .Times(1); - AdvanceClockAndRunTasks(kRtcpReportInterval); - testing::Mock::VerifyAndClearExpectations(sender()); - - ConsumeAndVerifyFrames(0, 9, start_time); - EXPECT_EQ(Receiver::kNoFramesReady, receiver()->AdvanceToNextFrame()); -} - -// Tests that the Receiver processes RTP packets, can receive frames out of -// order, and issues the appropriate ACK/NACK feedback to the Sender as it -// realizes what it has and what it's missing. -TEST_F(ReceiverTest, ReceivesFramesOutOfOrder) { - const Clock::time_point start_time = FakeClock::now(); - ExchangeInitialReportPackets(); - - constexpr static int kOutOfOrderFrames[] = {3, 4, 2, 0, 1}; - for (int i : kOutOfOrderFrames) { - // Expectations are different as each frame is sent and received. - switch (i) { - case 3: { - // Note that frame 4 will not yet be known to the Receiver, and so it - // should not be mentioned in any of the feedback for this case. - EXPECT_CALL(*sender(), OnReceiverCheckpoint(FrameId::leader(), - kTargetPlayoutDelay)) - .Times(AtLeast(1)); - EXPECT_CALL( - *sender(), - OnReceiverHasFrames(std::vector({FrameId::first() + 3}))) - .Times(AtLeast(1)); - EXPECT_CALL(*sender(), - OnReceiverIsMissingPackets(std::vector({ - PacketNack{FrameId::first(), kAllPacketsLost}, - PacketNack{FrameId::first() + 1, kAllPacketsLost}, - PacketNack{FrameId::first() + 2, kAllPacketsLost}, - }))) - .Times(AtLeast(1)); - EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); - break; - } - - case 4: { - EXPECT_CALL(*sender(), OnReceiverCheckpoint(FrameId::leader(), - kTargetPlayoutDelay)) - .Times(AtLeast(1)); - EXPECT_CALL(*sender(), - OnReceiverHasFrames(std::vector( - {FrameId::first() + 3, FrameId::first() + 4}))) - .Times(AtLeast(1)); - EXPECT_CALL(*sender(), - OnReceiverIsMissingPackets(std::vector({ - PacketNack{FrameId::first(), kAllPacketsLost}, - PacketNack{FrameId::first() + 1, kAllPacketsLost}, - PacketNack{FrameId::first() + 2, kAllPacketsLost}, - }))) - .Times(AtLeast(1)); - EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); - break; - } - - case 2: { - EXPECT_CALL(*sender(), OnReceiverCheckpoint(FrameId::leader(), - kTargetPlayoutDelay)) - .Times(AtLeast(1)); - EXPECT_CALL(*sender(), OnReceiverHasFrames(std::vector( - {FrameId::first() + 2, FrameId::first() + 3, - FrameId::first() + 4}))) - .Times(AtLeast(1)); - EXPECT_CALL(*sender(), - OnReceiverIsMissingPackets(std::vector({ - PacketNack{FrameId::first(), kAllPacketsLost}, - PacketNack{FrameId::first() + 1, kAllPacketsLost}, - }))) - .Times(AtLeast(1)); - EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); - break; - } - - case 0: { - EXPECT_CALL(*sender(), - OnReceiverCheckpoint(FrameId::first(), kTargetPlayoutDelay)) - .Times(AtLeast(1)); - EXPECT_CALL(*sender(), OnReceiverHasFrames(std::vector( - {FrameId::first() + 2, FrameId::first() + 3, - FrameId::first() + 4}))) - .Times(AtLeast(1)); - EXPECT_CALL(*sender(), - OnReceiverIsMissingPackets(std::vector( - {PacketNack{FrameId::first() + 1, kAllPacketsLost}}))) - .Times(AtLeast(1)); - EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); - break; - } - - case 1: { - EXPECT_CALL(*sender(), OnReceiverCheckpoint(FrameId::first() + 4, - kTargetPlayoutDelay)) - .Times(AtLeast(1)); - EXPECT_CALL(*sender(), OnReceiverHasFrames(_)).Times(0); - EXPECT_CALL(*sender(), OnReceiverIsMissingPackets(_)).Times(0); - EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); - break; - } - - default: - OSP_NOTREACHED(); - } - - sender()->SetFrameBeingSent(SimulatedFrame(start_time, i)); - sender()->SendRtpPackets(sender()->GetAllPacketIds(i)); - - AdvanceClockAndRunTasks(kRoundTripNetworkDelay); - - // While there are known incomplete frames, the Receiver should send RTCP - // packets more frequently than the default "ping" interval. Thus, advancing - // the clock by this much should result in several feedback reports - // transmitted to the Sender. - AdvanceClockAndRunTasks(kRtcpReportInterval - kRoundTripNetworkDelay); - - testing::Mock::VerifyAndClearExpectations(sender()); - testing::Mock::VerifyAndClearExpectations(consumer()); - } - - ConsumeAndVerifyFrames(0, 4, start_time); - EXPECT_EQ(Receiver::kNoFramesReady, receiver()->AdvanceToNextFrame()); -} - -// Tests that the Receiver will respond to a key frame request from its client -// by sending a Picture Loss Indicator (PLI) to the Sender, and then will -// automatically stop sending the PLI once a key frame has been received. -TEST_F(ReceiverTest, RequestsKeyFrameToRectifyPictureLoss) { - const Clock::time_point start_time = FakeClock::now(); - ExchangeInitialReportPackets(); - - // Send and Receive three frames in-order, normally. - for (int i = 0; i <= 2; ++i) { - EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); - EXPECT_CALL(*sender(), - OnReceiverCheckpoint(FrameId::first() + i, kTargetPlayoutDelay)) - .Times(1); - sender()->SetFrameBeingSent(SimulatedFrame(start_time, i)); - sender()->SendRtpPackets(sender()->GetAllPacketIds(0)); - AdvanceClockAndRunTasks(kRoundTripNetworkDelay); - testing::Mock::VerifyAndClearExpectations(sender()); - testing::Mock::VerifyAndClearExpectations(consumer()); - // Advance to next frame transmission time. - AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration - - kRoundTripNetworkDelay); - } - ConsumeAndVerifyFrames(0, 2, start_time); - - // Simulate the Consumer requesting a key frame after picture loss (e.g., a - // decoder failure). Ensure the Sender is immediately notified. - EXPECT_CALL(*sender(), OnReceiverIndicatesPictureLoss()).Times(1); - receiver()->RequestKeyFrame(); - AdvanceClockAndRunTasks(kOneWayNetworkDelay); // Propagate request to Sender. - testing::Mock::VerifyAndClearExpectations(sender()); - - // The Sender sends another frame that is not a key frame and, upon receipt, - // the Receiver should repeat its "cry" for a key frame. - EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); - EXPECT_CALL(*sender(), - OnReceiverCheckpoint(FrameId::first() + 3, kTargetPlayoutDelay)) - .Times(1); - EXPECT_CALL(*sender(), OnReceiverIndicatesPictureLoss()).Times(AtLeast(1)); - sender()->SetFrameBeingSent(SimulatedFrame(start_time, 3)); - sender()->SendRtpPackets(sender()->GetAllPacketIds(0)); - AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration - kOneWayNetworkDelay); - testing::Mock::VerifyAndClearExpectations(sender()); - testing::Mock::VerifyAndClearExpectations(consumer()); - ConsumeAndVerifyFrames(3, 3, start_time); - - // Finally, the Sender responds to the PLI condition by sending a key frame. - // Confirm the Receiver has stopped indicating picture loss after having - // received the key frame. - EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); - EXPECT_CALL(*sender(), - OnReceiverCheckpoint(FrameId::first() + 4, kTargetPlayoutDelay)) - .Times(1); - EXPECT_CALL(*sender(), OnReceiverIndicatesPictureLoss()).Times(0); - SimulatedFrame key_frame(start_time, 4); - key_frame.dependency = EncodedFrame::Dependency::kKeyFrame; - key_frame.referenced_frame_id = key_frame.frame_id; - sender()->SetFrameBeingSent(key_frame); - sender()->SendRtpPackets(sender()->GetAllPacketIds(0)); - AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); - testing::Mock::VerifyAndClearExpectations(sender()); - testing::Mock::VerifyAndClearExpectations(consumer()); - - // The client has not yet consumed the key frame, so any calls to - // RequestKeyFrame() should not set the PLI condition again. - EXPECT_CALL(*sender(), OnReceiverIndicatesPictureLoss()).Times(0); - receiver()->RequestKeyFrame(); - AdvanceClockAndRunTasks(kOneWayNetworkDelay); - testing::Mock::VerifyAndClearExpectations(sender()); - - // After consuming the requested key frame, the client should be able to set - // the PLI condition again with another RequestKeyFrame() call. - ConsumeAndVerifyFrame(key_frame); - EXPECT_CALL(*sender(), OnReceiverIndicatesPictureLoss()).Times(1); - receiver()->RequestKeyFrame(); - AdvanceClockAndRunTasks(kOneWayNetworkDelay); - testing::Mock::VerifyAndClearExpectations(sender()); -} - -TEST_F(ReceiverTest, PLICanBeDisabled) { - receiver()->SetPliEnabledForTesting(false); - - EXPECT_CALL(*sender(), OnReceiverIndicatesPictureLoss()).Times(0); - receiver()->RequestKeyFrame(); - AdvanceClockAndRunTasks(kOneWayNetworkDelay); - testing::Mock::VerifyAndClearExpectations(sender()); -} - -// Tests that the Receiver will start dropping packets once its frame queue is -// full (i.e., when the consumer is not pulling them out of the queue). Since -// the Receiver will stop ACK'ing frames, the Sender will become stalled. -TEST_F(ReceiverTest, EatsItsFill) { - const Clock::time_point start_time = FakeClock::now(); - ExchangeInitialReportPackets(); - - // Send and Receive the maximum possible number of frames in-order, normally. - for (int i = 0; i < kMaxUnackedFrames; ++i) { - EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); - EXPECT_CALL(*sender(), OnReceiverCheckpoint( - FrameId::first() + i, - SimulatedFrame::GetExpectedPlayoutDelay(i))) - .Times(1); - sender()->SetFrameBeingSent(SimulatedFrame(start_time, i)); - sender()->SendRtpPackets(sender()->GetAllPacketIds(0)); - AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); - testing::Mock::VerifyAndClearExpectations(sender()); - testing::Mock::VerifyAndClearExpectations(consumer()); - } - - // Sending one more frame should be ignored. Over and over. None of the - // feedback reports from the Receiver should indicate it is collecting packets - // for future frames. - int ignored_frame = kMaxUnackedFrames; - for (int i = 0; i < 5; ++i) { - EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); - EXPECT_CALL(*sender(), - OnReceiverCheckpoint(FrameId::first() + (ignored_frame - 1), - kTargetPlayoutDelayChange)) - .Times(AtLeast(0)); - EXPECT_CALL(*sender(), OnReceiverIsMissingPackets(_)).Times(0); - sender()->SetFrameBeingSent(SimulatedFrame(start_time, ignored_frame)); - sender()->SendRtpPackets(sender()->GetAllPacketIds(0)); - AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); - testing::Mock::VerifyAndClearExpectations(sender()); - testing::Mock::VerifyAndClearExpectations(consumer()); - } - - // Consume only one frame, and confirm the Receiver allows only one frame more - // to be received. - ConsumeAndVerifyFrames(0, 0, start_time); - int no_longer_ignored_frame = ignored_frame; - ++ignored_frame; - EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(AtLeast(1)); - EXPECT_CALL(*sender(), - OnReceiverCheckpoint(FrameId::first() + no_longer_ignored_frame, - kTargetPlayoutDelayChange)) - .Times(AtLeast(1)); - EXPECT_CALL(*sender(), OnReceiverIsMissingPackets(_)).Times(0); - // This frame should be received successfully. - sender()->SetFrameBeingSent( - SimulatedFrame(start_time, no_longer_ignored_frame)); - sender()->SendRtpPackets(sender()->GetAllPacketIds(0)); - AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); - // This second frame should be ignored, however. - sender()->SetFrameBeingSent(SimulatedFrame(start_time, ignored_frame)); - sender()->SendRtpPackets(sender()->GetAllPacketIds(0)); - AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); - testing::Mock::VerifyAndClearExpectations(sender()); - testing::Mock::VerifyAndClearExpectations(consumer()); -} - -// Tests that incomplete frames that would be played-out too late are dropped, -// but only as inter-frame data dependency requirements permit, and only if no -// target playout delay change information would have been missed. -TEST_F(ReceiverTest, DropsLateFrames) { - const Clock::time_point start_time = FakeClock::now(); - ExchangeInitialReportPackets(); - - // Before any packets have been sent/received, the Receiver should indicate no - // frames are ready. - EXPECT_EQ(Receiver::kNoFramesReady, receiver()->AdvanceToNextFrame()); - - // Set a ridiculously-large estimated player processing time so that the logic - // thinks every frame going to play out too late. - receiver()->SetPlayerProcessingTime(seconds(3)); - - // In this test there are eight frames total: - // - Frame 0: Key frame. - // - Frames 1-4: Non-key frames. - // - Frame 5: Non-key frame that contains a target playout delay change. - // - Frame 6: Key frame. - // - Frame 7: Non-key frame. - ASSERT_EQ(SimulatedFrame::kPlayoutChangeAtFrame, 5); - SimulatedFrame frames[8] = {{start_time, 0}, {start_time, 1}, {start_time, 2}, - {start_time, 3}, {start_time, 4}, {start_time, 5}, - {start_time, 6}, {start_time, 7}}; - frames[6].dependency = EncodedFrame::Dependency::kKeyFrame; - frames[6].referenced_frame_id = frames[6].frame_id; - - // Send just packet 1 (NOT packet 0) of all the frames. The Receiver should - // never notify the consumer via the callback, nor report that any frames are - // ready, because none of the frames have been completely received. - EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); - EXPECT_CALL(*sender(), OnReceiverCheckpoint(_, _)).Times(0); - for (int i = 0; i <= 7; ++i) { - sender()->SetFrameBeingSent(frames[i]); - // Assumption: There are at least three packets in each frame, else the test - // is not exercising the logic meaningfully. - ASSERT_LE(size_t{3}, sender()->GetAllPacketIds(0).size()); - sender()->SendRtpPackets({FramePacketId{1}}); - AdvanceClockAndRunTasks(SimulatedFrame::kFrameDuration); - } - testing::Mock::VerifyAndClearExpectations(consumer()); - testing::Mock::VerifyAndClearExpectations(sender()); - EXPECT_EQ(Receiver::kNoFramesReady, receiver()->AdvanceToNextFrame()); - - // Send all the packets of Frame 6 (the second key frame) and Frame 7. The - // Receiver still cannot drop any frames because it has not seen packet 0 of - // every prior frame. In other words, it cannot ignore any possibility of a - // target playout delay change from the Sender. - EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); - EXPECT_CALL(*sender(), OnReceiverCheckpoint(_, _)).Times(0); - for (int i = 6; i <= 7; ++i) { - sender()->SetFrameBeingSent(frames[i]); - sender()->SendRtpPackets(sender()->GetAllPacketIds(0)); - } - AdvanceClockAndRunTasks(kRoundTripNetworkDelay); - testing::Mock::VerifyAndClearExpectations(consumer()); - testing::Mock::VerifyAndClearExpectations(sender()); - EXPECT_EQ(Receiver::kNoFramesReady, receiver()->AdvanceToNextFrame()); - - // Send packet 0 for all but Frame 5, which contains a target playout delay - // change. All but the last two frames will still be incomplete. The Receiver - // still cannot drop any frames because it doesn't know whether Frame 5 had a - // target playout delay change. - EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); - EXPECT_CALL(*sender(), OnReceiverCheckpoint(_, _)).Times(0); - for (int i = 0; i <= 7; ++i) { - if (i == 5) { - continue; - } - sender()->SetFrameBeingSent(frames[i]); - sender()->SendRtpPackets({FramePacketId{0}}); - } - AdvanceClockAndRunTasks(kRoundTripNetworkDelay); - testing::Mock::VerifyAndClearExpectations(consumer()); - testing::Mock::VerifyAndClearExpectations(sender()); - EXPECT_EQ(Receiver::kNoFramesReady, receiver()->AdvanceToNextFrame()); - - // Finally, send packet 0 for Frame 5. Now, the Receiver will drop every frame - // before the completely-received second key frame, as they are all still - // incomplete and will play-out too late. When it drops the frames, it will - // notify the sender of the new checkpoint so that it stops trying to - // re-transmit the dropped frames. - EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); - EXPECT_CALL(*sender(), OnReceiverCheckpoint(FrameId::first() + 7, - kTargetPlayoutDelayChange)) - .Times(1); - sender()->SetFrameBeingSent(frames[5]); - sender()->SendRtpPackets({FramePacketId{0}}); - AdvanceClockAndRunTasks(kRoundTripNetworkDelay); - // Note: Consuming Frame 6 will trigger the checkpoint advancement, since the - // call to AdvanceToNextFrame() contains the frame skipping/dropping logic. - ConsumeAndVerifyFrame(frames[6]); - testing::Mock::VerifyAndClearExpectations(consumer()); - testing::Mock::VerifyAndClearExpectations(sender()); - - // After consuming Frame 6, the Receiver knows Frame 7 is also available and - // should have scheduled an immediate task to notify the Consumer of this. - EXPECT_CALL(*consumer(), OnFramesReady(Gt(0))).Times(1); - AdvanceClockAndRunTasks(kOneWayNetworkDelay); - testing::Mock::VerifyAndClearExpectations(consumer()); - - // Now consume Frame 7. This shouldn't trigger any further checkpoint - // advancement. - EXPECT_CALL(*consumer(), OnFramesReady(_)).Times(0); - EXPECT_CALL(*sender(), OnReceiverCheckpoint(_, _)).Times(0); - ConsumeAndVerifyFrame(frames[7]); - AdvanceClockAndRunTasks(kOneWayNetworkDelay); - testing::Mock::VerifyAndClearExpectations(consumer()); - testing::Mock::VerifyAndClearExpectations(sender()); -} - -} // namespace -} // namespace openscreen::cast diff --git a/cast/streaming/impl/rpc_messenger_unittest.cc b/cast/streaming/impl/rpc_messenger_unittest.cc index f4b346fe4..323d850d3 100644 --- a/cast/streaming/impl/rpc_messenger_unittest.cc +++ b/cast/streaming/impl/rpc_messenger_unittest.cc @@ -15,7 +15,6 @@ #include "platform/base/span.h" using testing::_; -using testing::Invoke; using testing::Return; namespace openscreen::cast { @@ -77,7 +76,7 @@ class RpcMessengerTest : public testing::Test { void ProcessMessage(const RpcMessage& rpc) { std::vector message(rpc.ByteSizeLong()); rpc.SerializeToArray(message.data(), message.size()); - rpc_messenger_->ProcessMessageFromRemote(message.data(), message.size()); + rpc_messenger_->ProcessMessageFromRemote(message); } std::unique_ptr fake_messenger_; diff --git a/cast/streaming/impl/rtcp_common.cc b/cast/streaming/impl/rtcp_common.cc index 40d2cc123..78f0287e8 100644 --- a/cast/streaming/impl/rtcp_common.cc +++ b/cast/streaming/impl/rtcp_common.cc @@ -35,15 +35,13 @@ void RtcpCommonHeader::AppendFields(ByteBuffer& buffer) const { switch (with.subtype) { case RtcpSubtype::kPictureLossIndicator: case RtcpSubtype::kFeedback: + case RtcpSubtype::kReceiverLog: byte0 |= static_cast(with.subtype); break; - // TODO(issuetracker.google.com/298085631): implement support for - // sending receiver logs over RTCP. - case RtcpSubtype::kReceiverLog: - OSP_UNIMPLEMENTED(); - break; - default: + // We should not be creating application or payload specific packets + // with an unknown or null subtype -- they will just be ignored. + case RtcpSubtype::kNull: OSP_NOTREACHED(); } break; @@ -215,7 +213,7 @@ std::optional RtcpReportBlock::ParseOne(ByteView buffer, for (int block = 0; block < report_count; ++block) { if (ConsumeField(buffer) != ssrc) { // Skip-over report block meant for some other recipient. - buffer.remove_prefix(kRtcpReportBlockSize - sizeof(uint32_t)); + buffer = buffer.subspan(kRtcpReportBlockSize - sizeof(uint32_t)); continue; } diff --git a/cast/streaming/impl/rtcp_common.h b/cast/streaming/impl/rtcp_common.h index c094f2ac4..7f13d1ec4 100644 --- a/cast/streaming/impl/rtcp_common.h +++ b/cast/streaming/impl/rtcp_common.h @@ -13,7 +13,7 @@ #include "cast/streaming/impl/ntp_time.h" #include "cast/streaming/impl/rtp_defines.h" -#include "cast/streaming/impl/statistics_defines.h" +#include "cast/streaming/impl/statistics_common.h" #include "cast/streaming/public/frame_id.h" #include "cast/streaming/rtp_time.h" #include "cast/streaming/ssrc.h" @@ -177,7 +177,7 @@ struct PacketNack { struct RtcpReceiverEventLogMessage { // The statistics event type, may be either a receiver side frame event or // packet event. - StatisticsEventType type; + StatisticsEvent::Type type; // The time at which this event occurred. Clock::time_point timestamp; diff --git a/cast/streaming/impl/rtp_packet_parser.cc b/cast/streaming/impl/rtp_packet_parser.cc index 22f51a317..76f573371 100644 --- a/cast/streaming/impl/rtp_packet_parser.cc +++ b/cast/streaming/impl/rtp_packet_parser.cc @@ -92,7 +92,7 @@ std::optional RtpPacketParser::Parse( result.new_playout_delay = std::chrono::milliseconds(ReadBigEndian(buffer.data())); } - buffer.remove_prefix(size); + buffer = buffer.subspan(size); } // All remaining data in the packet is the payload. diff --git a/cast/streaming/impl/rtp_packetizer.cc b/cast/streaming/impl/rtp_packetizer.cc index e2c88802b..0e9f21ea6 100644 --- a/cast/streaming/impl/rtp_packetizer.cc +++ b/cast/streaming/impl/rtp_packetizer.cc @@ -109,10 +109,6 @@ ByteBuffer RtpPacketizer::GeneratePacket(const EncryptedFrame& frame, AppendField(frame.new_playout_delay.count(), buffer); } - // Sanity-check the pointer math, to ensure the packet is being entirely - // populated, with no underrun or overrun. - OSP_CHECK_EQ(buffer.data() + data_chunk_size, packet.end()); - // Copy the encrypted payload data into the packet. auto data_chunk = frame.data.subspan(data_chunk_start, data_chunk_size); std::copy(data_chunk.begin(), data_chunk.end(), buffer.data()); diff --git a/cast/streaming/impl/sender_impl.cc b/cast/streaming/impl/sender_impl.cc new file mode 100644 index 000000000..dedd34c46 --- /dev/null +++ b/cast/streaming/impl/sender_impl.cc @@ -0,0 +1,659 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cast/streaming/impl/sender_impl.h" + +#include +#include +#include +#include + +#include "cast/streaming/impl/rtp_defines.h" +#include "cast/streaming/impl/statistics_common.h" +#include "cast/streaming/public/session_config.h" +#include "platform/base/trivial_clock_traits.h" +#include "util/chrono_helpers.h" +#include "util/osp_logging.h" +#include "util/std_util.h" +#include "util/string_util.h" +#include "util/trace_logging.h" + +namespace openscreen::cast { + +using clock_operators::operator<<; + +SenderImpl::SenderImpl(Environment& environment, + SenderPacketRouter& packet_router, + SessionConfig config, + RtpPayloadType rtp_payload_type) + : config_(config), + packet_router_(packet_router), + rtcp_session_(config.sender_ssrc, + config.receiver_ssrc, + environment.now()), + rtcp_parser_(rtcp_session_, *this), + sender_report_builder_(rtcp_session_), + rtp_packetizer_(rtp_payload_type, + config.sender_ssrc, + packet_router_.max_packet_size()), + rtp_timebase_(config.rtp_timebase), + crypto_(config.aes_secret_key, config.aes_iv_mask), + statistics_dispatcher_(environment), + target_playout_delay_(config.target_playout_delay) { + OSP_CHECK_NE(rtcp_session_.sender_ssrc(), rtcp_session_.receiver_ssrc()); + OSP_CHECK_GT(rtp_timebase_, 0); + OSP_CHECK_GT(target_playout_delay_, milliseconds::zero()); + + pending_sender_report_.reference_time = SenderPacketRouter::kNever; + + packet_router_.OnSenderCreated(rtcp_session_.receiver_ssrc(), this); +} + +SenderImpl::~SenderImpl() { + packet_router_.OnSenderDestroyed(rtcp_session_.receiver_ssrc()); +} + +void SenderImpl::SetObserver(openscreen::cast::Sender::Observer* observer) { + OSP_CHECK_NE(observer_, observer); + observer_ = observer; +} + +size_t SenderImpl::GetInFlightFrameCount() const { + return num_frames_in_flight_; +} + +Clock::duration SenderImpl::GetInFlightMediaDuration( + RtpTimeTicks next_frame_rtp_timestamp) const { + if (num_frames_in_flight_ == 0) { + return Clock::duration::zero(); // No frames are currently in-flight. + } + + const PendingFrameSlot& oldest_slot = get_slot_for(checkpoint_frame_id_ + 1); + // Note: The oldest slot's frame cannot have been canceled because the + // protocol does not allow ACK'ing this particular frame without also moving + // the checkpoint forward. See "CST2 feedback" discussion in rtp_defines.h. + OSP_CHECK(oldest_slot.is_active_for_frame(checkpoint_frame_id_ + 1)); + + return (next_frame_rtp_timestamp - oldest_slot.frame->rtp_timestamp) + .ToDuration(rtp_timebase_); +} + +Clock::duration SenderImpl::GetMaxInFlightMediaDuration() const { + // Assumption: The total amount of allowed in-flight media should equal the + // half of the playout delay window, plus the amount of time it takes to + // receive an ACK from the Receiver. + // + // Why half of the playout delay window? It's assumed here that capture and + // media encoding, which occur before EnqueueFrame() is called, are executing + // within the first half of the playout delay window. This leaves the second + // half for executing all network transmits/re-transmits, plus decoding and + // play-out at the Receiver. + // + // TODO(crbug.com/498035450): this needs to be modernized and is based on + // outdated assumptions. + return (target_playout_delay_ / 2) + (round_trip_time_ / 2); +} + +bool SenderImpl::NeedsKeyFrame() const { + return last_enqueued_key_frame_id_ <= picture_lost_at_frame_id_; +} + +FrameId SenderImpl::GetNextFrameId() const { + return last_enqueued_frame_id_ + 1; +} + +Clock::duration SenderImpl::GetCurrentRoundTripTime() const { + return round_trip_time_; +} + +openscreen::cast::Sender::EnqueueFrameResult SenderImpl::EnqueueFrame( + const EncodedFrame& frame) { + // Assume the fields of the `frame` have all been set correctly, with + // monotonically increasing timestamps and a valid pointer to the data. + OSP_CHECK_EQ(frame.frame_id, GetNextFrameId()); + OSP_CHECK_GE(frame.referenced_frame_id, FrameId::first()); + if (frame.frame_id != FrameId::first()) { + OSP_CHECK_GT(frame.rtp_timestamp, pending_sender_report_.rtp_timestamp); + if (frame.reference_time <= pending_sender_report_.reference_time) { + OSP_DLOG_WARN << "Frame " << frame.frame_id + << " has non-monotonic reference_time: " + << frame.reference_time + << " <= " << pending_sender_report_.reference_time; + } + } + OSP_CHECK(frame.data.data()); + + const auto capture_begin_time = + (frame.capture_begin_time > Clock::time_point::min()) + ? frame.capture_begin_time + : Clock::now(); + + TRACE_FLOW_BEGIN_WITH_TIME(TraceCategory::kSender, "Frame.Capture", + frame.frame_id, capture_begin_time); + + if (frame.capture_end_time > Clock::time_point::min()) { + TRACE_FLOW_STEP_WITH_TIME(TraceCategory::kSender, "Frame.Capture.End", + frame.frame_id, frame.capture_end_time); + } + + TRACE_FLOW_STEP(TraceCategory::kSender, "Frame.Encode.End", frame.frame_id); + + // Check whether enqueuing the frame would exceed the design limit for the + // span of FrameIds. Even if `num_frames_in_flight_` is less than + // kMaxUnackedFrames, it's the span of FrameIds that is restricted. + if ((frame.frame_id - checkpoint_frame_id_) > kMaxUnackedFrames) { + return REACHED_ID_SPAN_LIMIT; + } + + // Check whether enqueuing the frame would exceed the current maximum media + // duration limit. + if (GetInFlightMediaDuration(frame.rtp_timestamp) > + GetMaxInFlightMediaDuration()) { + return MAX_DURATION_IN_FLIGHT; + } + + // Encrypt the frame and initialize the slot tracking its sending. + PendingFrameSlot& slot = get_slot_for(frame.frame_id); + OSP_CHECK(!slot.frame); + slot.frame = crypto_.Encrypt(frame); + const int packet_count = rtp_packetizer_.ComputeNumberOfPackets(*slot.frame); + if (packet_count <= 0) { + slot.frame.reset(); + return PAYLOAD_TOO_LARGE; + } + slot.send_flags.Resize(packet_count, BitVector::SET); + slot.packet_sent_times.assign(packet_count, SenderPacketRouter::kNever); + + // Officially record the "enqueue." + ++num_frames_in_flight_; + last_enqueued_frame_id_ = slot.frame->frame_id; + OSP_CHECK_LE( + num_frames_in_flight_, + static_cast(last_enqueued_frame_id_ - checkpoint_frame_id_)); + if (slot.frame->dependency == EncodedFrame::Dependency::kKeyFrame) { + last_enqueued_key_frame_id_ = slot.frame->frame_id; + } + TRACE_FLOW_STEP(TraceCategory::kSender, "Frame.Enqueued", frame.frame_id); + + // Update the target playout delay, if necessary. + if (slot.frame->new_playout_delay > milliseconds::zero()) { + target_playout_delay_ = slot.frame->new_playout_delay; + playout_delay_change_at_frame_id_ = slot.frame->frame_id; + } + + // Update the lip-sync information for the next Sender Report, ensuring that + // the reference time is monotonically increasing. + pending_sender_report_.reference_time = + frame.frame_id == FrameId::first() + ? slot.frame->reference_time + : std::max(slot.frame->reference_time, + pending_sender_report_.reference_time); + pending_sender_report_.rtp_timestamp = slot.frame->rtp_timestamp; + + // If the round trip time hasn't been computed yet, immediately send a RTCP + // packet (i.e., before the RTP packets are sent). The RTCP packet will + // provide a Sender Report which contains the required lip-sync information + // the Receiver needs for timing the media playout. + // + // Detail: Working backwards, if the round trip time is not known, then this + // Sender has never processed a Receiver Report. Thus, the Receiver has never + // provided a Receiver Report, which it can only do after having processed a + // Sender Report from this Sender. Thus, this Sender really needs to send + // that, right now! + if (round_trip_time_ == Clock::duration::zero()) { + packet_router_.RequestRtcpSend(rtcp_session_.receiver_ssrc()); + } + + // Re-activate RTP sending if it was suspended. + packet_router_.RequestRtpSend(rtcp_session_.receiver_ssrc()); + statistics_dispatcher_.DispatchEnqueueEvents(config_.stream_type, frame); + + return OK; +} + +void SenderImpl::CancelInFlightData() { + TRACE_DEFAULT_SCOPED1( + TraceCategory::kSender, "frames_in_flight", + std::to_string(last_enqueued_frame_id_ - checkpoint_frame_id_)); + + while (checkpoint_frame_id_ < last_enqueued_frame_id_) { + ++checkpoint_frame_id_; + CancelPendingFrame(checkpoint_frame_id_, /*was_acked*/ false); + } + DispatchCancellations(); +} + +void SenderImpl::ReportFrameDropEvent(FrameId frame_id, + RtpTimeTicks rtp_timestamp, + Clock::time_point drop_time) { + statistics_dispatcher_.DispatchFrameDropEvent(config_.stream_type, frame_id, + rtp_timestamp, drop_time); +} + +void SenderImpl::OnReceivedRtcpPacket(Clock::time_point arrival_time, + ByteView packet) { + rtcp_packet_arrival_time_ = arrival_time; + // This call to Parse() invoke zero or more of the OnReceiverXYZ() methods in + // the current call stack: + if (rtcp_parser_.Parse(packet, last_enqueued_frame_id_)) { + packet_router_.OnRtcpReceived(arrival_time, round_trip_time_); + } +} + +ByteBuffer SenderImpl::GetRtcpPacketForImmediateSend( + Clock::time_point send_time, + ByteBuffer buffer) { + if (pending_sender_report_.reference_time == SenderPacketRouter::kNever) { + // Cannot send a report if one is not available (i.e., a frame has never + // been enqueued). + return buffer.subspan(0, 0); + } + + // The Sender Report to be sent is a snapshot of the "pending Sender Report," + // but with its timestamp fields modified. First, the reference time is set to + // the RTCP packet's send time. Then, the corresponding RTP timestamp is + // translated to match (for lip-sync). + RtcpSenderReport sender_report = pending_sender_report_; + sender_report.reference_time = send_time; + sender_report.rtp_timestamp += RtpTimeDelta::FromDuration( + sender_report.reference_time - pending_sender_report_.reference_time, + rtp_timebase_); + + return sender_report_builder_.BuildPacket(sender_report, buffer).first; +} + +ByteBuffer SenderImpl::GetRtpPacketForImmediateSend(Clock::time_point send_time, + ByteBuffer buffer) { + ChosenPacket chosen = ChooseNextRtpPacketNeedingSend(); + + // If no packets need sending (i.e., all packets have been sent at least once + // and do not need to be re-sent yet), check whether a Kickstart packet should + // be sent. It's possible that there has been complete packet loss of some + // frames, and the Receiver may not be aware of the existence of the latest + // frame(s). Kickstarting is the only way the Receiver can discover the newer + // frames it doesn't know about. + if (!chosen) { + const ChosenPacketAndWhen kickstart = ChooseKickstartPacket(); + if (kickstart.when > send_time) { + // Nothing to send, so return "empty" signal to the packet router. The + // packet router will suspend RTP sending until this Sender explicitly + // resumes it. + return buffer.subspan(0, 0); + } + chosen = kickstart; + OSP_CHECK(chosen); + } + + const ByteBuffer result = rtp_packetizer_.GeneratePacket( + *chosen.slot->frame, chosen.packet_id, buffer); + chosen.slot->send_flags.Clear(chosen.packet_id); + chosen.slot->packet_sent_times[chosen.packet_id] = send_time; + + ++pending_sender_report_.send_packet_count; + // According to RFC3550, the octet count does not include the RTP header. The + // following is just a good approximation, however, because the header size + // will very infrequently be 4 bytes greater (see + // RtpPacketizer::kAdaptiveLatencyHeaderSize). No known Cast Streaming + // Receiver implementations use this for anything, and so this should be fine. + const int approximate_octet_count = + static_cast(result.size()) - RtpPacketizer::kBaseRtpHeaderSize; + OSP_CHECK_GE(approximate_octet_count, 0); + pending_sender_report_.send_octet_count += approximate_octet_count; + + return result; +} + +Clock::time_point SenderImpl::GetRtpResumeTime() { + if (ChooseNextRtpPacketNeedingSend()) { + return Alarm::kImmediately; + } + return ChooseKickstartPacket().when; +} + +RtpTimeTicks SenderImpl::GetLastRtpTimestamp() const { + return {}; +} + +StreamType SenderImpl::GetStreamType() const { + return config_.stream_type; +} + +void SenderImpl::OnReceiverReferenceTimeAdvanced( + Clock::time_point reference_time) { + // Not used. +} + +void SenderImpl::OnReceiverReport(const RtcpReportBlock& receiver_report) { + OSP_CHECK_NE(rtcp_packet_arrival_time_, SenderPacketRouter::kNever); + + const Clock::duration total_delay = + rtcp_packet_arrival_time_ - + sender_report_builder_.GetRecentReportTime( + receiver_report.last_status_report_id, rtcp_packet_arrival_time_); + const auto non_network_delay = + Clock::to_duration(receiver_report.delay_since_last_report); + + // Round trip time measurement: This is the time elapsed since the Sender + // Report was sent, minus the time the Receiver did other stuff before sending + // the Receiver Report back. + // + // If the round trip time seems to be less than or equal to zero, assume clock + // imprecision by one or both peers caused a bad value to be calculated. The + // true value is likely very close to zero (i.e., this is ideal network + // behavior); and so just represent this as 75 µs, an optimistic + // wired-Ethernet LAN ping time. + constexpr auto kNearZeroRoundTripTime = Clock::to_duration(microseconds(75)); + static_assert(kNearZeroRoundTripTime > Clock::duration::zero(), + "More precision in Clock::duration needed!"); + const Clock::duration measurement = + std::max(total_delay - non_network_delay, kNearZeroRoundTripTime); + + // Validate the measurement by using the current target playout delay as a + // "reasonable upper-bound." It's certainly possible that the actual network + // round-trip time could exceed the target playout delay, but that would mean + // the current network performance is totally inadequate for streaming anyway. + // We cap the measurement here instead of ignoring it so the Sender still + // backs off its estimates during severe network congestion. + Clock::duration clamped_measurement = measurement; + if (clamped_measurement > target_playout_delay_) { + OSP_LOG_WARN << "Capping round-trip time measurement (" << measurement + << ") to the current target playout delay (" + << target_playout_delay_ << ")."; + clamped_measurement = target_playout_delay_; + } + + // Measurements will typically have high variance. Use a simple smoothing + // filter to track a short-term average that changes less drastically. + if (round_trip_time_ == Clock::duration::zero()) { + round_trip_time_ = clamped_measurement; + } else { + // Arbitrary constant, to provide 1/8 weight to the new measurement, and 7/8 + // weight to the old estimate, which seems to work well for de-noising the + // estimate. + constexpr int kInertia = 7; + round_trip_time_ = + (kInertia * round_trip_time_ + clamped_measurement) / (kInertia + 1); + } + TRACE_SCOPED1(TraceCategory::kSender, "UpdatedRoundTripTime", + "round_trip_time", ToString(round_trip_time_)); +} + +void SenderImpl::OnCastReceiverFrameLogMessages( + std::vector messages) { + statistics_dispatcher_.DispatchFrameLogMessages(config_.stream_type, + messages); +} + +void SenderImpl::OnReceiverIndicatesPictureLoss() { + TRACE_DEFAULT_SCOPED1(TraceCategory::kSender, "last_received_frame_id", + picture_lost_at_frame_id_.ToString()); + // The Receiver will continue the PLI notifications until it has received a + // key frame. Thus, if a key frame is already in-flight, don't make a state + // change that would cause this Sender to force another expensive key frame. + if (checkpoint_frame_id_ < last_enqueued_key_frame_id_) { + return; + } + + picture_lost_at_frame_id_ = checkpoint_frame_id_; + + if (observer_) { + observer_->OnPictureLost(); + } + + // Note: It may seem that all pending frames should be canceled until + // EnqueueFrame() is called with a key frame. However: + // + // 1. The Receiver should still be the main authority on what frames/packets + // are being ACK'ed and NACK'ed. + // + // 2. It may be desirable for the Receiver to be "limping along" in the + // meantime. For example, video may be corrupted but mostly watchable, + // and so it's best for the Sender to continue sending the non-key frames + // until the Receiver indicates otherwise. +} + +void SenderImpl::OnReceiverCheckpoint(FrameId frame_id, + milliseconds playout_delay) { + TRACE_DEFAULT_SCOPED2(TraceCategory::kSender, "frame_id", frame_id.ToString(), + "playout_delay", ToString(playout_delay)); + if (frame_id > last_enqueued_frame_id_) { + TRACE_SET_RESULT(Error::Code::kParameterOutOfRange); + OSP_LOG_ERROR + << "Ignoring checkpoint for " << latest_expected_frame_id_ + << " because this Sender could not have sent any frames after " + << last_enqueued_frame_id_ << '.'; + return; + } + // CompoundRtcpParser should guarantee this: + OSP_CHECK_GE(playout_delay, milliseconds::zero()); + while (checkpoint_frame_id_ < frame_id) { + ++checkpoint_frame_id_; + PendingFrameSlot& slot = get_slot_for(checkpoint_frame_id_); + if (slot.is_active_for_frame(checkpoint_frame_id_)) { + const RtpTimeTicks rtp_timestamp = slot.frame->rtp_timestamp; + statistics_dispatcher_.DispatchAckEvent( + config_.stream_type, rtp_timestamp, checkpoint_frame_id_); + CancelPendingFrame(checkpoint_frame_id_, /*was_acked*/ true); + + TRACE_FLOW_STEP(TraceCategory::kSender, "Frame.Acked", + checkpoint_frame_id_); + } + } + latest_expected_frame_id_ = std::max(latest_expected_frame_id_, frame_id); + DispatchCancellations(); + + if (playout_delay != target_playout_delay_ && + frame_id >= playout_delay_change_at_frame_id_) { + OSP_LOG_WARN << "Sender's target playout delay (" << target_playout_delay_ + << ") disagrees with the Receiver's (" << playout_delay << ")"; + } +} + +void SenderImpl::OnReceiverHasFrames(std::vector acks) { + OSP_DCHECK(!acks.empty() && AreElementsSortedAndUnique(acks)); + TRACE_DEFAULT_SCOPED1(TraceCategory::kSender, "frame_ids", + string_util::Join(acks)); + + if (acks.back() > last_enqueued_frame_id_) { + TRACE_SET_RESULT(Error::Code::kParameterOutOfRange); + OSP_LOG_ERROR << "Ignoring individual frame ACKs: ACKing frame " + << latest_expected_frame_id_ + << " is invalid because this Sender could not have sent any " + "frames after " + << last_enqueued_frame_id_ << '.'; + return; + } + + for (FrameId id : acks) { + TRACE_FLOW_STEP(TraceCategory::kSender, "Frame.Acked", id); + PendingFrameSlot& slot = get_slot_for(id); + if (slot.is_active_for_frame(id)) { + const RtpTimeTicks rtp_timestamp = slot.frame->rtp_timestamp; + statistics_dispatcher_.DispatchAckEvent(config_.stream_type, + rtp_timestamp, id); + } + CancelPendingFrame(id, /*was_acked*/ true); + } + latest_expected_frame_id_ = std::max(latest_expected_frame_id_, acks.back()); + DispatchCancellations(); +} + +void SenderImpl::OnReceiverIsMissingPackets(std::vector nacks) { + TRACE_DEFAULT_SCOPED1(TraceCategory::kSender, "number_of_packets", + std::to_string(nacks.size())); + OSP_DCHECK(!nacks.empty() && AreElementsSortedAndUnique(nacks)); + OSP_CHECK_NE(rtcp_packet_arrival_time_, SenderPacketRouter::kNever); + + // This is a point-in-time threshold that indicates whether each NACK will + // trigger a packet retransmit. The threshold is based on the network round + // trip time because a Receiver's NACK may have been issued while the needed + // packet was in-flight from the Sender. In such cases, the Receiver's NACK is + // likely stale and this Sender should not redundantly re-transmit the packet + // again. + const Clock::time_point too_recent_a_send_time = + rtcp_packet_arrival_time_ - round_trip_time_; + + // Iterate over all the NACKs... + bool need_to_send = false; + for (auto nack_it = nacks.begin(); nack_it != nacks.end();) { + // Find the slot associated with the NACK's frame ID. + const FrameId frame_id = nack_it->frame_id; + PendingFrameSlot* slot = nullptr; + if (frame_id <= last_enqueued_frame_id_) { + PendingFrameSlot& candidate_slot = get_slot_for(frame_id); + if (candidate_slot.is_active_for_frame(frame_id)) { + slot = &candidate_slot; + } + } + + // If no slot was found (i.e., the NACK is invalid) for the frame, skip-over + // all other NACKs for the same frame. While it seems to be a bug that the + // Receiver would attempt to NACK a frame that does not yet exist, this can + // happen in rare cases where RTCP packets arrive out-of-order (i.e., the + // network shuffled them). + if (!slot) { + TRACE_SCOPED1(TraceCategory::kSender, "MissingNackSlot", "frame_id", + frame_id.ToString()); + for (++nack_it; nack_it != nacks.end() && nack_it->frame_id == frame_id; + ++nack_it) { + } + continue; + } + + latest_expected_frame_id_ = std::max(latest_expected_frame_id_, frame_id); + + const auto HandleIndividualNack = [&](FramePacketId packet_id) { + if (slot->packet_sent_times[packet_id] <= too_recent_a_send_time) { + slot->send_flags.Set(packet_id); + need_to_send = true; + } + }; + const FramePacketId range_end = slot->packet_sent_times.size(); + if (nack_it->packet_id == kAllPacketsLost) { + for (FramePacketId packet_id = 0; packet_id < range_end; ++packet_id) { + HandleIndividualNack(packet_id); + } + ++nack_it; + } else { + do { + if (nack_it->packet_id < range_end) { + HandleIndividualNack(nack_it->packet_id); + } else { + OSP_LOG_WARN + << "Ignoring NACK for packet that doesn't exist in frame " + << frame_id << ": " << static_cast(nack_it->packet_id); + } + ++nack_it; + } while (nack_it != nacks.end() && nack_it->frame_id == frame_id); + } + } + + if (need_to_send) { + packet_router_.RequestRtpSend(rtcp_session_.receiver_ssrc()); + } +} + +SenderImpl::ChosenPacket SenderImpl::ChooseNextRtpPacketNeedingSend() { + // Find the oldest packet needing to be sent (or re-sent). + for (FrameId frame_id = checkpoint_frame_id_ + 1; + frame_id <= last_enqueued_frame_id_; ++frame_id) { + PendingFrameSlot& slot = get_slot_for(frame_id); + if (!slot.is_active_for_frame(frame_id)) { + continue; // Frame was canceled. None of its packets need to be sent. + } + const FramePacketId packet_id = slot.send_flags.FindFirstSet(); + if (packet_id < slot.send_flags.size()) { + return {&slot, packet_id}; + } + } + + return {}; // Nothing needs to be sent. +} + +SenderImpl::ChosenPacketAndWhen SenderImpl::ChooseKickstartPacket() { + if (latest_expected_frame_id_ >= last_enqueued_frame_id_) { + // Since the Receiver must know about all of the frames currently queued, no + // Kickstart packet is necessary. + return {}; + } + + // The Kickstart packet is always in the last-enqueued frame, so that the + // Receiver will know about every frame the Sender has. However, which packet + // should be chosen? Any would do, since all packets contain the frame's total + // packet count. For historical reasons, all sender implementations have + // always just sent the last packet; and so that tradition is continued here. + ChosenPacketAndWhen chosen; + chosen.slot = &get_slot_for(last_enqueued_frame_id_); + // Note: This frame cannot have been canceled since + // `latest_expected_frame_id_` hasn't yet reached this point. + OSP_CHECK(chosen.slot->is_active_for_frame(last_enqueued_frame_id_)); + chosen.packet_id = chosen.slot->send_flags.size() - 1; + + const Clock::time_point time_last_sent = + chosen.slot->packet_sent_times[chosen.packet_id]; + // Sanity-check: This method should not be called to choose a packet while + // there are still unsent packets. + OSP_CHECK_NE(time_last_sent, SenderPacketRouter::kNever); + + // The desired Kickstart interval is a fraction of the total + // `target_playout_delay_`. The reason for the specific ratio here is based on + // lost knowledge (from legacy implementations); but it makes sense (i.e., to + // be a good "network citizen") to be less aggressive for larger playout delay + // windows, and more aggressive for shorter ones to avoid too-late packet + // arrivals. + using kWaitFraction = std::ratio<1, 20>; + const Clock::duration desired_kickstart_interval = + Clock::to_duration(target_playout_delay_) * kWaitFraction::num / + kWaitFraction::den; + // The actual interval used is increased, if current network performance + // warrants waiting longer. Don't send a Kickstart packet until no NACKs + // have been received for two network round-trip periods. + constexpr int kLowerBoundRoundTrips = 2; + const Clock::duration kickstart_interval = std::max( + desired_kickstart_interval, round_trip_time_ * kLowerBoundRoundTrips); + chosen.when = time_last_sent + kickstart_interval; + + return chosen; +} + +void SenderImpl::CancelPendingFrame(FrameId frame_id, bool was_acked) { + TRACE_FLOW_END(TraceCategory::kSender, "Frame.Cancelled", frame_id); + + PendingFrameSlot& slot = get_slot_for(frame_id); + if (!slot.is_active_for_frame(frame_id)) { + return; // Frame was already canceled. + } + + if (was_acked) { + packet_router_.OnPayloadReceived( + slot.frame->data.size(), rtcp_packet_arrival_time_, round_trip_time_); + } + + slot.frame.reset(); + OSP_CHECK_GT(num_frames_in_flight_, 0); + --num_frames_in_flight_; + if (observer_) { + pending_cancellations_.emplace_back(frame_id); + } +} + +void SenderImpl::DispatchCancellations() { + if (observer_) { + for (const FrameId id : pending_cancellations_) { + observer_->OnFrameCanceled(id); + } + } + pending_cancellations_.clear(); + + // At this point, there should either be no frames in flight, or the frame + // immediately after `checkpoint_frame_id_` must be valid. + OSP_DCHECK((num_frames_in_flight_ == 0) || + get_slot_for(checkpoint_frame_id_ + 1) + .is_active_for_frame(checkpoint_frame_id_ + 1)); +} + +SenderImpl::PendingFrameSlot::PendingFrameSlot() = default; +SenderImpl::PendingFrameSlot::~PendingFrameSlot() = default; + +} // namespace openscreen::cast diff --git a/cast/streaming/impl/sender_impl.h b/cast/streaming/impl/sender_impl.h new file mode 100644 index 000000000..8c43c5922 --- /dev/null +++ b/cast/streaming/impl/sender_impl.h @@ -0,0 +1,242 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CAST_STREAMING_IMPL_SENDER_IMPL_H_ +#define CAST_STREAMING_IMPL_SENDER_IMPL_H_ + +#include + +#include +#include +#include +#include + +#include "cast/streaming/impl/compound_rtcp_parser.h" +#include "cast/streaming/impl/frame_crypto.h" +#include "cast/streaming/impl/rtcp_common.h" +#include "cast/streaming/impl/rtp_defines.h" +#include "cast/streaming/impl/rtp_packetizer.h" +#include "cast/streaming/impl/sender_report_builder.h" +#include "cast/streaming/impl/statistics_dispatcher.h" +#include "cast/streaming/public/constants.h" +#include "cast/streaming/public/frame_id.h" +#include "cast/streaming/public/sender.h" +#include "cast/streaming/public/session_config.h" +#include "cast/streaming/rtp_time.h" +#include "cast/streaming/sender_packet_router.h" +#include "platform/api/time.h" +#include "platform/base/span.h" +#include "util/bit_vector.h" + +namespace openscreen::cast { + +class Environment; + +// The Cast Streaming Sender, a peer corresponding to some Cast Streaming +// Receiver at the other end of a network link. See class level comments for +// Receiver for a high-level overview. +class SenderImpl final : public Sender, + public SenderPacketRouter::Sender, + public CompoundRtcpParser::Client { + public: + // Constructs a Sender that attaches to the given `environment`-provided + // resources and `packet_router`. The `config` contains the settings that were + // agreed-upon by both sides from the OFFER/ANSWER exchange (i.e., the part of + // the overall end-to-end connection process that occurs before Cast Streaming + // is started). The `rtp_payload_type` does not affect the behavior of this + // Sender. It is simply passed along to a Receiver in the RTP packet stream. + SenderImpl(Environment& environment, + SenderPacketRouter& packet_router, + SessionConfig config, + RtpPayloadType rtp_payload_type); + + ~SenderImpl() final; + + // Sender overrides. + const SessionConfig& config() const override { return config_; } + void SetObserver(Observer* observer) override; + size_t GetInFlightFrameCount() const override; + Clock::duration GetInFlightMediaDuration( + RtpTimeTicks next_frame_rtp_timestamp) const override; + Clock::duration GetMaxInFlightMediaDuration() const override; + bool NeedsKeyFrame() const override; + FrameId GetNextFrameId() const override; + Clock::duration GetCurrentRoundTripTime() const override; + [[nodiscard]] EnqueueFrameResult EnqueueFrame( + const EncodedFrame& frame) override; + void CancelInFlightData() override; + void ReportFrameDropEvent(FrameId frame_id, + RtpTimeTicks rtp_timestamp, + Clock::time_point drop_time) override; + + private: + // Tracking/Storage for frames that are ready-to-send, and until they are + // fully received at the other end. + struct PendingFrameSlot { + // The frame to send, or nullopt if this slot is not in use. + std::optional frame; + + // Represents which packets need to be sent. Elements are indexed by + // FramePacketId. A set bit means a packet needs to be sent (or re-sent). + BitVector send_flags; + + // The time when each of the packets was last sent, or + // `SenderPacketRouter::kNever` if the packet has not been sent yet. + // Elements are indexed by FramePacketId. This is used to avoid + // re-transmitting any given packet too frequently. + std::vector packet_sent_times; + + PendingFrameSlot(); + ~PendingFrameSlot(); + + bool is_active_for_frame(FrameId frame_id) const { + return frame && frame->frame_id == frame_id; + } + }; + + // Return value from the ChooseXYZ() helper methods. + struct ChosenPacket { + PendingFrameSlot* slot = nullptr; + FramePacketId packet_id{}; + + explicit operator bool() const { return !!slot; } + }; + + // An extension of ChosenPacket that also includes the point-in-time when the + // packet should be sent. + struct ChosenPacketAndWhen : public ChosenPacket { + Clock::time_point when = SenderPacketRouter::kNever; + }; + + // SenderPacketRouter::Sender implementation. + void OnReceivedRtcpPacket(Clock::time_point arrival_time, + ByteView packet) final; + ByteBuffer GetRtcpPacketForImmediateSend(Clock::time_point send_time, + ByteBuffer buffer) final; + ByteBuffer GetRtpPacketForImmediateSend(Clock::time_point send_time, + ByteBuffer buffer) final; + Clock::time_point GetRtpResumeTime() final; + RtpTimeTicks GetLastRtpTimestamp() const final; + StreamType GetStreamType() const final; + + // CompoundRtcpParser::Client implementation. + void OnReceiverReferenceTimeAdvanced(Clock::time_point reference_time) final; + void OnReceiverReport(const RtcpReportBlock& receiver_report) final; + void OnCastReceiverFrameLogMessages( + std::vector messages) final; + void OnReceiverIndicatesPictureLoss() final; + void OnReceiverCheckpoint(FrameId frame_id, + std::chrono::milliseconds playout_delay) final; + void OnReceiverHasFrames(std::vector acks) final; + void OnReceiverIsMissingPackets(std::vector nacks) final; + + // Helper to choose which packet to send, from those that have been flagged as + // "need to send." Returns a "false" result if nothing needs to be sent. + ChosenPacket ChooseNextRtpPacketNeedingSend(); + + // Helper that returns the packet that should be used to kick-start the + // Receiver, and the time at which the packet should be sent. Returns a kNever + // result if kick-starting is not needed. + ChosenPacketAndWhen ChooseKickstartPacket(); + + // Cancels sending (or resending) the given frame once it is known to have + // been either: + // 1. Cancelled by the sender (was_acked must be false); + // 2. Fully received based on the ACK feedback in a receiver RTCP report + // (was_acked must be true); + // 3. The receiver sent a checkpoint frame ID (was_acked must be true). + // + // This clears the corresponding entry in `pending_frames_` and + // adds `frame_id` to the list of pending cancellations to be dispatched as + // part of DispatchCancellations(). + // + // NOTE: Every frame_id ends up being "cancelled" at least once. + void CancelPendingFrame(FrameId frame_id, bool was_acked); + + // Must be called after one or a series of CancelPendingFrame() calls in order + // to notify the observer, if any, about cancellations. + void DispatchCancellations(); + + // Inline helper to return the slot that would contain the tracking info for + // the given `frame_id`. + const PendingFrameSlot& get_slot_for(FrameId frame_id) const { + return pending_frames_[(frame_id - FrameId::first()) % + pending_frames_.size()]; + } + PendingFrameSlot& get_slot_for(FrameId frame_id) { + return pending_frames_[(frame_id - FrameId::first()) % + pending_frames_.size()]; + } + + const SessionConfig config_; + SenderPacketRouter& packet_router_; + RtcpSession rtcp_session_; + CompoundRtcpParser rtcp_parser_; + SenderReportBuilder sender_report_builder_; + RtpPacketizer rtp_packetizer_; + const int rtp_timebase_; + FrameCrypto crypto_; + StatisticsDispatcher statistics_dispatcher_; + + // Ring buffer of PendingFrameSlots. The frame having FrameId x will always + // be slotted at position x % pending_frames_.size(). Use get_slot_for() to + // access the correct slot for a given FrameId. + std::array pending_frames_ = {}; + + // A count of the number of frames in-flight (i.e., the number of active + // entries in `pending_frames_`). + size_t num_frames_in_flight_ = 0; + + // The ID of the last frame enqueued. + FrameId last_enqueued_frame_id_ = FrameId::leader(); + + // Indicates that all of the packets for all frames up to and including this + // FrameId have been successfully received (or otherwise do not need to be + // re-transmitted). + FrameId checkpoint_frame_id_ = FrameId::leader(); + + // The ID of the latest frame the Receiver seems to be aware of. + FrameId latest_expected_frame_id_ = FrameId::leader(); + + // The target playout delay for the last-enqueued frame. This is auto-updated + // when a frame is enqueued that changes the delay. + std::chrono::milliseconds target_playout_delay_; + FrameId playout_delay_change_at_frame_id_ = FrameId::first(); + + // The exact arrival time of the last RTCP packet. + Clock::time_point rtcp_packet_arrival_time_ = SenderPacketRouter::kNever; + + // The near-term average round trip time. This is updated with each Sender + // Report → Receiver Report round trip. This is initially zero, indicating the + // round trip time has not been measured yet. + Clock::duration round_trip_time_ = {}; + + // Maintain current stats in a Sender Report that is ready for sending at any + // time. This includes up-to-date lip-sync information, and packet and byte + // count stats. + RtcpSenderReport pending_sender_report_; + + // These are used to determine whether a key frame needs to be sent to the + // Receiver. When the Receiver provides a picture loss notification, the + // current checkpoint frame ID is stored in `picture_lost_at_frame_id_`. Then, + // while `last_enqueued_key_frame_id_` is less than or equal to + // `picture_lost_at_frame_id_`, the Sender knows it still needs to send a key + // frame to resolve the picture loss condition. In all other cases, the + // Receiver is either in a good state or is in the process of receiving the + // key frame that will make that happen. + FrameId picture_lost_at_frame_id_ = FrameId::leader(); + FrameId last_enqueued_key_frame_id_ = FrameId::leader(); + + // The current observer (optional). + Observer* observer_ = nullptr; + + // Because the observer may take action when frames are cancelled, such as + // calling APIs like EnqueueFrame(), `this` must be in a good state before + // the observer is notified of any pending frame cancellations. + std::vector pending_cancellations_; +}; + +} // namespace openscreen::cast + +#endif // CAST_STREAMING_IMPL_SENDER_IMPL_H_ diff --git a/cast/streaming/impl/sender_impl_unittest.cc b/cast/streaming/impl/sender_impl_unittest.cc new file mode 100644 index 000000000..14f212f13 --- /dev/null +++ b/cast/streaming/impl/sender_impl_unittest.cc @@ -0,0 +1,1244 @@ +// Copyright 2020 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cast/streaming/impl/sender_impl.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cast/streaming/impl/compound_rtcp_builder.h" +#include "cast/streaming/impl/frame_collector.h" +#include "cast/streaming/impl/frame_crypto.h" +#include "cast/streaming/impl/packet_util.h" +#include "cast/streaming/impl/rtcp_session.h" +#include "cast/streaming/impl/rtp_defines.h" +#include "cast/streaming/impl/rtp_packet_parser.h" +#include "cast/streaming/impl/sender_report_parser.h" +#include "cast/streaming/public/constants.h" +#include "cast/streaming/public/encoded_frame.h" +#include "cast/streaming/public/frame_id.h" +#include "cast/streaming/public/session_config.h" +#include "cast/streaming/sender_packet_router.h" +#include "cast/streaming/ssrc.h" +#include "cast/streaming/testing/mock_environment.h" +#include "cast/streaming/testing/simple_socket_subscriber.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "platform/base/span.h" +#include "platform/test/fake_clock.h" +#include "platform/test/fake_task_runner.h" +#include "util/alarm.h" +#include "util/bit_vector.h" +#include "util/chrono_helpers.h" +#include "util/std_util.h" + +using testing::_; +using testing::AtLeast; +using testing::InvokeWithoutArgs; +using testing::Mock; +using testing::NiceMock; +using testing::Return; +using testing::Sequence; +using testing::StrictMock; + +namespace openscreen::cast { +namespace { + +// Sender configuration. +constexpr Ssrc kSenderSsrc = 1; +constexpr Ssrc kReceiverSsrc = 2; +constexpr int kRtpTimebase = 48000; +constexpr milliseconds kTargetPlayoutDelay(400); +constexpr auto kAesKey = + std::array{{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}}; +constexpr auto kCastIvMask = + std::array{{0xf0, 0xe0, 0xd0, 0xc0, 0xb0, 0xa0, 0x90, 0x80, + 0x70, 0x60, 0x50, 0x40, 0x30, 0x20, 0x10, 0x00}}; +constexpr RtpPayloadType kRtpPayloadType = RtpPayloadType::kVideoVp8; + +// The number of RTP ticks advanced per frame, for 100 FPS media. +constexpr int kRtpTicksPerFrame = kRtpTimebase / 100; + +// The number of milliseconds advanced per frame, for 100 FPS media. +constexpr milliseconds kFrameDuration{1000 / 100}; +static_assert(kFrameDuration < (kTargetPlayoutDelay / 10), + "Kickstart test assumes frame duration is far less than the " + "playout delay."); + +// An Encoded frame that also holds onto its own copy of data. +struct EncodedFrameWithBuffer : public EncodedFrame { + // `EncodedFrame::data` always points inside buffer.begin()...buffer.end(). + std::vector buffer; +}; + +// SenderPacketRouter configuration for these tests. +constexpr int kNumPacketsPerBurst = 20; +constexpr milliseconds kBurstInterval(10); + +// An arbitrary value, subtracted from "now," to specify the reference_time on +// frames that are about to be enqueued. This simulates that capture+encode +// happened in the past, before Sender::EnqueueFrame() is called. +constexpr milliseconds kCaptureDelay(11); + +// In some tests, the computed time values could be off a little bit due to +// imprecision in certain wire-format timestamp types. The following macro +// behaves just like Gtest's EXPECT_NEAR(), but works with all the time types +// too. +#define EXPECT_NEARLY_EQUAL(duration_a, duration_b, epsilon) \ + if ((duration_a) >= (duration_b)) { \ + EXPECT_LE((duration_a), (duration_b) + (epsilon)); \ + } else { \ + EXPECT_GE((duration_a), (duration_b) - (epsilon)); \ + } + +void OverrideRtpTimestamp(int frame_count, EncodedFrame* frame, int fps) { + const int ticks = frame_count * kRtpTimebase / fps; + frame->rtp_timestamp = RtpTimeTicks() + RtpTimeDelta::FromTicks(ticks); +} + +// Simulates UDP/IPv6 traffic in one direction (from Sender→Receiver, or +// Receiver→Sender), with a settable amount of delay. +class SimulatedNetworkPipe { + public: + SimulatedNetworkPipe(TaskRunner& task_runner, + Environment::PacketConsumer& remote) + : task_runner_(task_runner), remote_(remote) { + // Create a fake IPv6 address using the "documentative purposes" prefix + // concatenated with the `this` pointer. + std::array hextets{}; + hextets[0] = 0x2001; + hextets[1] = 0x0db8; + auto* const this_pointer = this; + static_assert(sizeof(this_pointer) <= (6 * sizeof(uint16_t)), ""); + memcpy(&hextets[2], &this_pointer, sizeof(this_pointer)); + local_endpoint_ = IPEndpoint{IPAddress(hextets), 2344}; + } + + const IPEndpoint& local_endpoint() const { return local_endpoint_; } + + Clock::duration network_delay() const { return network_delay_; } + void set_network_delay(Clock::duration delay) { network_delay_ = delay; } + + // The caller needs to spin the task runner before `packet` will reach the + // other side. + void StartPacketTransmission(std::vector packet) { + task_runner_.PostTaskWithDelay( + [this, pkt = std::move(packet)]() mutable { + remote_.OnReceivedPacket(local_endpoint_, FakeClock::now(), + std::move(pkt)); + }, + network_delay_); + } + + private: + TaskRunner& task_runner_; + Environment::PacketConsumer& remote_; + + IPEndpoint local_endpoint_; + + // The amount of time for the packet to transmit over this simulated network + // pipe. Defaults to zero to simplify the tests that don't care about delays. + Clock::duration network_delay_{}; +}; + +// Processes packets from the Sender under test, allowing unit tests to set +// expectations for parsed RTP or RTCP packets, to confirm proper behavior of +// the Sender. +class MockReceiver : public Environment::PacketConsumer { + public: + explicit MockReceiver(SimulatedNetworkPipe& pipe_to_sender) + : pipe_to_sender_(pipe_to_sender), + rtcp_session_(kSenderSsrc, kReceiverSsrc, FakeClock::now()), + sender_report_parser_(rtcp_session_), + rtcp_builder_(rtcp_session_), + rtp_parser_(kSenderSsrc), + crypto_(kAesKey, kCastIvMask) { + rtcp_builder_.SetPlayoutDelay(kTargetPlayoutDelay); + } + + ~MockReceiver() override = default; + + // Simulate the Receiver ACK'ing all frames up to and including the + // `new_checkpoint`. + void SetCheckpointFrame(FrameId new_checkpoint) { + OSP_CHECK_GE(new_checkpoint, rtcp_builder_.checkpoint_frame()); + rtcp_builder_.SetCheckpointFrame(new_checkpoint); + } + + // Automatically advances the checkpoint based on what is found in + // `complete_frames_`, returning true if the checkpoint moved forward. + bool AutoAdvanceCheckpoint() { + const FrameId old_checkpoint = rtcp_builder_.checkpoint_frame(); + FrameId new_checkpoint = old_checkpoint; + for (auto it = complete_frames_.upper_bound(old_checkpoint); + it != complete_frames_.end(); ++it) { + if (it->first != new_checkpoint + 1) { + break; + } + ++new_checkpoint; + } + if (new_checkpoint > old_checkpoint) { + rtcp_builder_.SetCheckpointFrame(new_checkpoint); + return true; + } + return false; + } + + void SetPictureLossIndicator(bool picture_is_lost) { + rtcp_builder_.SetPictureLossIndicator(picture_is_lost); + } + + void SetReceiverReport(StatusReportId reply_for, + RtcpReportBlock::Delay processing_delay) { + RtcpReportBlock receiver_report; + receiver_report.ssrc = kSenderSsrc; + receiver_report.last_status_report_id = reply_for; + receiver_report.delay_since_last_report = processing_delay; + rtcp_builder_.IncludeReceiverReportInNextPacket(receiver_report); + } + + void SetNacksAndAcks(std::vector packet_nacks, + std::vector frame_acks) { + rtcp_builder_.IncludeFeedbackInNextPacket(std::move(packet_nacks), + std::move(frame_acks)); + } + + // Builds and sends a RTCP packet containing one or more of: checkpoint, PLI, + // Receiver Report, NACKs, ACKs. + void TransmitRtcpFeedbackPacket() { + uint8_t buffer[kMaxRtpPacketSizeForIpv6UdpOnEthernet]; + const ByteBuffer packet = + rtcp_builder_.BuildPacket(FakeClock::now(), buffer); + pipe_to_sender_.StartPacketTransmission( + std::vector(packet.begin(), packet.end())); + } + + // Used by tests to simulate the Receiver not seeing specific packets come in + // from the network (e.g., because the network dropped the packets). + void SetIgnoreList(std::vector ignore_list) { + ignore_list_ = ignore_list; + } + + // Environment::PacketConsumer implementation. + // + // Called to process a packet from the Sender, simulating basic RTP frame + // collection and Sender Report parsing/handling. + void OnReceivedPacket(const IPEndpoint& source, + Clock::time_point arrival_time, + std::vector packet) override { + const auto type_and_ssrc = InspectPacketForRouting(packet); + EXPECT_NE(ApparentPacketType::UNKNOWN, type_and_ssrc.first); + EXPECT_EQ(kSenderSsrc, type_and_ssrc.second); + if (type_and_ssrc.first == ApparentPacketType::RTP) { + const std::optional part_of_frame = + rtp_parser_.Parse(packet); + ASSERT_TRUE(part_of_frame); + + // Return early if simulating packet drops over the network. + if (ContainsIf(ignore_list_, [&](const PacketNack& baddie) { + return (baddie.frame_id == part_of_frame->frame_id && + (baddie.packet_id == kAllPacketsLost || + baddie.packet_id == part_of_frame->packet_id)); + })) { + return; + } + + OnRtpPacket(*part_of_frame); + CollectRtpPacket(*part_of_frame, std::move(packet)); + } else if (type_and_ssrc.first == ApparentPacketType::RTCP) { + std::optional report = + sender_report_parser_.Parse(packet); + ASSERT_TRUE(report); + OnSenderReport(*report); + } + } + + std::map TakeCompleteFrames() { + std::map result; + result.swap(complete_frames_); + return result; + } + + // Tests set expectations on these mocks to monitor events of interest, and/or + // invoke additional behaviors. + MOCK_METHOD1(OnRtpPacket, + void(const RtpPacketParser::ParseResult& parsed_packet)); + MOCK_METHOD1(OnFrameComplete, void(FrameId frame_id)); + MOCK_METHOD1(OnSenderReport, + void(const SenderReportParser::SenderReportWithId& report)); + + private: + // Collects the individual RTP packets until a whole frame can be formed, then + // calls OnFrameComplete(). Ignores extra RTP packets that are no longer + // needed. + void CollectRtpPacket(const RtpPacketParser::ParseResult& part_of_frame, + std::vector packet) { + const FrameId frame_id = part_of_frame.frame_id; + if (complete_frames_.find(frame_id) != complete_frames_.end()) { + return; + } + FrameCollector& collector = incomplete_frames_[frame_id]; + collector.set_frame_id(frame_id); + EXPECT_TRUE(collector.CollectRtpPacket(part_of_frame, &packet)); + if (!collector.is_complete()) { + return; + } + const EncodedFrame& metadata = collector.PeekFrameMetadata(); + const size_t payload_size = collector.GetFramePayloadSize(); + EncodedFrameWithBuffer& decrypted = complete_frames_[frame_id]; + // Note: Not setting decrypted->reference_time here since the logic around + // calculating the playout time is rather complex, and is definitely outside + // the scope of the testing being done in this module. Instead, end-to-end + // testing should exist elsewhere to confirm frame play-out times with real + // Receivers. + decrypted.buffer.resize(payload_size); + crypto_.Decrypt(metadata.frame_id, collector.GetPayloadChunks(), + decrypted.buffer); + metadata.CopyMetadataTo(&decrypted); + decrypted.data = decrypted.buffer; + incomplete_frames_.erase(frame_id); + OnFrameComplete(frame_id); + } + + SimulatedNetworkPipe& pipe_to_sender_; + RtcpSession rtcp_session_; + SenderReportParser sender_report_parser_; + CompoundRtcpBuilder rtcp_builder_; + RtpPacketParser rtp_parser_; + FrameCrypto crypto_; + + std::vector ignore_list_; + std::map incomplete_frames_; + std::map complete_frames_; +}; + +class MockObserver : public Sender::Observer { + public: + MOCK_METHOD1(OnFrameCanceled, void(FrameId frame_id)); + MOCK_METHOD0(OnPictureLost, void()); +}; + +class SenderTest : public testing::Test { + public: + SenderTest() + : fake_clock_(Clock::now()), + task_runner_(fake_clock_), + sender_environment_(&FakeClock::now, task_runner_), + sender_packet_router_(sender_environment_, + kNumPacketsPerBurst, + kBurstInterval), + sender_(sender_environment_, + sender_packet_router_, + {/* .sender_ssrc = */ kSenderSsrc, + /* .receiver_ssrc = */ kReceiverSsrc, + /* .rtp_timebase = */ kRtpTimebase, + /* .channels = */ 2, + /* .target_playout_delay = */ kTargetPlayoutDelay, + /* .aes_secret_key = */ kAesKey, + /* .aes_iv_mask = */ kCastIvMask, + /* .is_pli_enabled = */ true}, + kRtpPayloadType), + receiver_to_sender_pipe_(task_runner_, sender_packet_router_), + receiver_(receiver_to_sender_pipe_), + sender_to_receiver_pipe_(task_runner_, receiver_) { + sender_environment_.SetSocketSubscriber(&socket_subscriber_); + sender_environment_.set_remote_endpoint( + receiver_to_sender_pipe_.local_endpoint()); + ON_CALL(sender_environment_, SendPacket(_, _)) + .WillByDefault([this](ByteView packet, PacketMetadata metadata) { + sender_to_receiver_pipe_.StartPacketTransmission( + std::vector(packet.begin(), packet.end())); + }); + } + + ~SenderTest() override = default; + + SenderImpl* sender() { return &sender_; } + MockReceiver* receiver() { return &receiver_; } + + void SetReceiverToSenderNetworkDelay(Clock::duration delay) { + receiver_to_sender_pipe_.set_network_delay(delay); + } + + void SetSenderToReceiverNetworkDelay(Clock::duration delay) { + sender_to_receiver_pipe_.set_network_delay(delay); + } + + void SimulateExecution(Clock::duration how_long = Clock::duration::zero()) { + fake_clock_.Advance(how_long); + } + + static void PopulateFramePayloadBuffer(int seed, + int num_bytes, + std::vector* payload) { + payload->clear(); + payload->reserve(num_bytes); + for (int i = 0; i < num_bytes; ++i) { + payload->push_back(static_cast(seed + i)); + } + } + + static void PopulateFrameWithDefaults(FrameId frame_id, + Clock::time_point reference_time, + int seed, + int num_payload_bytes, + EncodedFrameWithBuffer* frame) { + frame->dependency = (frame_id == FrameId::first()) + ? EncodedFrame::Dependency::kKeyFrame + : EncodedFrame::Dependency::kDependent; + frame->frame_id = frame_id; + frame->referenced_frame_id = frame->frame_id; + if (frame_id != FrameId::first()) { + --frame->referenced_frame_id; + } + frame->rtp_timestamp = + RtpTimeTicks() + (RtpTimeDelta::FromTicks(kRtpTicksPerFrame) * + (frame_id - FrameId::first())); + frame->reference_time = reference_time; + PopulateFramePayloadBuffer(seed, num_payload_bytes, &frame->buffer); + frame->data = frame->buffer; + } + + // Confirms that all `sent_frames` exist in `received_frames`, with identical + // data and metadata. + static void ExpectFramesReceivedCorrectly( + Span sent_frames, + const std::map received_frames) { + ASSERT_EQ(sent_frames.size(), received_frames.size()); + + for (const EncodedFrameWithBuffer& sent_frame : sent_frames) { + SCOPED_TRACE(testing::Message() + << "Checking sent frame " << sent_frame.frame_id); + const auto received_it = received_frames.find(sent_frame.frame_id); + if (received_it == received_frames.end()) { + ADD_FAILURE() << "Did not receive frame."; + continue; + } + const EncodedFrame& received_frame = received_it->second; + EXPECT_EQ(sent_frame.dependency, received_frame.dependency); + EXPECT_EQ(sent_frame.referenced_frame_id, + received_frame.referenced_frame_id); + EXPECT_EQ(sent_frame.rtp_timestamp, received_frame.rtp_timestamp); + EXPECT_THAT(sent_frame.data, + testing::ElementsAreArray(received_frame.data)); + } + } + + private: + FakeClock fake_clock_; + FakeTaskRunner task_runner_; + NiceMock sender_environment_; + SenderPacketRouter sender_packet_router_; + SenderImpl sender_; + SimulatedNetworkPipe receiver_to_sender_pipe_; + NiceMock receiver_; + SimulatedNetworkPipe sender_to_receiver_pipe_; + SimpleSubscriber socket_subscriber_; +}; + +// Tests that the Sender can send EncodedFrames over an ideal network (i.e., low +// latency, no loss), and does so without having to transmit the same packet +// twice. +TEST_F(SenderTest, SendsFramesEfficiently) { + constexpr milliseconds kOneWayNetworkDelay(1); + SetSenderToReceiverNetworkDelay(kOneWayNetworkDelay); + SetReceiverToSenderNetworkDelay(kOneWayNetworkDelay); + + // Expect that each packet is only sent once. + std::set> received_packets; + EXPECT_CALL(*receiver(), OnRtpPacket(_)) + .WillRepeatedly([&](const RtpPacketParser::ParseResult& parsed_packet) { + std::pair id(parsed_packet.frame_id, + parsed_packet.packet_id); + const auto insert_result = received_packets.insert(id); + EXPECT_TRUE(insert_result.second) + << "Received duplicate packet: " << id.first << ':' + << static_cast(id.second); + }); + + // Simulate normal frame ACK'ing behavior. + ON_CALL(*receiver(), OnFrameComplete(_)).WillByDefault(InvokeWithoutArgs([&] { + if (receiver()->AutoAdvanceCheckpoint()) { + receiver()->TransmitRtcpFeedbackPacket(); + } + })); + + StrictMock observer; + EXPECT_CALL(observer, OnFrameCanceled(FrameId::first())); + EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 1)); + EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 2)); + sender()->SetObserver(&observer); + + EncodedFrameWithBuffer frames[3]; + constexpr int kFrameDataSizes[] = {8196, 12, 1900}; + for (int i = 0; i < 3; ++i) { + if (i == 0) { + EXPECT_TRUE(sender()->NeedsKeyFrame()); + } else { + EXPECT_FALSE(sender()->NeedsKeyFrame()); + } + PopulateFrameWithDefaults(FrameId::first() + i, + FakeClock::now() - kCaptureDelay, 0xbf - i, + kFrameDataSizes[i], &frames[i]); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); + SimulateExecution(kFrameDuration); + } + SimulateExecution(kTargetPlayoutDelay); + + ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); +} + +// Tests that the Sender properly updates the checkpoint frame ID while +// it is cancelling frames. See https://crbug.com/1433584 for an example crash +// where the checkpoint frame ID is invalid. +TEST_F(SenderTest, WaitsUntilEndOfReportToUpdateObservers) { + constexpr milliseconds kOneWayNetworkDelay(1); + SetSenderToReceiverNetworkDelay(kOneWayNetworkDelay); + SetReceiverToSenderNetworkDelay(kOneWayNetworkDelay); + + // Expect that each packet is only sent once. + std::set> received_packets; + EXPECT_CALL(*receiver(), OnRtpPacket(_)) + .WillRepeatedly([&](const RtpPacketParser::ParseResult& parsed_packet) { + std::pair id(parsed_packet.frame_id, + parsed_packet.packet_id); + const auto insert_result = received_packets.insert(id); + EXPECT_TRUE(insert_result.second) + << "Received duplicate packet: " << id.first << ':' + << static_cast(id.second); + }); + + StrictMock observer; + + // The sender should be in a valid state during frame cancellations. Since + // these all came from the same report, the sender shouldn't have any frames + // in flight. + EXPECT_CALL(observer, OnFrameCanceled(_)) + .Times(3) + .WillRepeatedly([sender = sender()](FrameId id) { + EXPECT_EQ(0u, sender->GetInFlightFrameCount()); + + // Since no frames are in flight, the next frame timestamp should not + // matter. + EXPECT_EQ(Clock::duration::zero(), + sender->GetInFlightMediaDuration(RtpTimeTicks(123456789))); + }); + + // Don't ACK frames and return a report until the third frame. + EXPECT_CALL(*receiver(), OnFrameComplete(_)).Times(2); + EXPECT_CALL(*receiver(), OnFrameComplete(FrameId::first() + 2)) + .WillOnce(InvokeWithoutArgs([&] { + if (receiver()->AutoAdvanceCheckpoint()) { + receiver()->TransmitRtcpFeedbackPacket(); + } + })); + + sender()->SetObserver(&observer); + + EncodedFrameWithBuffer frames[3]; + constexpr int kFrameDataSizes[] = {8196, 12, 1900}; + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(i == 0, sender()->NeedsKeyFrame()); + PopulateFrameWithDefaults(FrameId::first() + i, + FakeClock::now() - kCaptureDelay, 0xbf - i, + kFrameDataSizes[i], &frames[i]); + + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); + SimulateExecution(kFrameDuration); + } + SimulateExecution(kTargetPlayoutDelay); + + ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); +} + +// Tests that the Sender correctly computes the current in-flight media +// duration, a backlog signal for clients. +TEST_F(SenderTest, ComputesInFlightMediaDuration) { + // With no frames enqueued, the in-flight media duration should be zero. + EXPECT_EQ(Clock::duration::zero(), + sender()->GetInFlightMediaDuration(RtpTimeTicks())); + EXPECT_EQ(Clock::duration::zero(), + sender()->GetInFlightMediaDuration( + RtpTimeTicks() + RtpTimeDelta::FromTicks(kRtpTicksPerFrame))); + + // Enqueue a frame. + EncodedFrameWithBuffer frame; + PopulateFrameWithDefaults(FrameId::first(), FakeClock::now(), 0, + 13 /* bytes */, &frame); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); + + // Now, the in-flight media duration should depend on the RTP timestamp of the + // next frame. + EXPECT_EQ(kFrameDuration, sender()->GetInFlightMediaDuration( + frame.rtp_timestamp + + RtpTimeDelta::FromTicks(kRtpTicksPerFrame))); + EXPECT_EQ(10 * kFrameDuration, + sender()->GetInFlightMediaDuration( + frame.rtp_timestamp + + RtpTimeDelta::FromTicks(10 * kRtpTicksPerFrame))); +} + +// Tests that the Sender computes the maximum in-flight media duration based on +// its analysis of current network conditions. By implication, this demonstrates +// that the Sender is also measuring the network round-trip time. +TEST_F(SenderTest, RespondsToNetworkLatencyChanges) { + // The expected maximum error in time calculations is one tick of the RTCP + // report block's delay type. + constexpr auto kEpsilon = to_nanoseconds(RtcpReportBlock::Delay(1)); + + // Before the Sender has the necessary information to compute the network + // round-trip time, GetMaxInFlightMediaDuration() will return half the target + // playout delay. + EXPECT_NEARLY_EQUAL(kTargetPlayoutDelay / 2, + sender()->GetMaxInFlightMediaDuration(), kEpsilon); + + // No network is perfect. Simulate different one-way network delays. + constexpr milliseconds kOutboundDelay(2); + constexpr milliseconds kInboundDelay(4); + constexpr milliseconds kRoundTripDelay(kOutboundDelay + kInboundDelay); + SetSenderToReceiverNetworkDelay(kOutboundDelay); + SetReceiverToSenderNetworkDelay(kInboundDelay); + + // Enqueue a frame in the Sender to start emitting periodic RTCP reports. + { + EncodedFrameWithBuffer frame; + PopulateFrameWithDefaults(FrameId::first(), FakeClock::now(), 0, + 1 /* byte */, &frame); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); + } + + // Run one network round-trip from Sender→Receiver→Sender. + StatusReportId sender_report_id{}; + EXPECT_CALL(*receiver(), OnSenderReport(_)) + .WillOnce( + [&](const SenderReportParser::SenderReportWithId& sender_report) { + sender_report_id = sender_report.report_id; + }); + // Simulate the passage of time for the Sender Report to reach the Receiver. + SimulateExecution(kOutboundDelay); + // The Receiver should have received the Sender Report at this point. + Mock::VerifyAndClearExpectations(receiver()); + ASSERT_NE(StatusReportId{}, sender_report_id); + // Simulate the passage of time in the Receiver doing "other tasks" before + // replying back to the Sender. This delay is included in the Receiver Report + // so that the Sender can isolate the delays caused by the network. + constexpr milliseconds kReceiverProcessingDelay(2); + SimulateExecution(kReceiverProcessingDelay); + // Create the Receiver Report "reply," and simulate it being sent across the + // network, back to the Sender. + receiver()->SetReceiverReport( + sender_report_id, std::chrono::duration_cast( + kReceiverProcessingDelay)); + receiver()->TransmitRtcpFeedbackPacket(); + SimulateExecution(kInboundDelay); + + // At this point, the Sender should have computed the network round-trip time, + // and so GetMaxInFlightMediaDuration() will return half the target playout + // delay PLUS half the network round-trip time. + EXPECT_NEARLY_EQUAL(kTargetPlayoutDelay / 2 + kRoundTripDelay / 2, + sender()->GetMaxInFlightMediaDuration(), kEpsilon); + + // Increase the outbound delay, which will increase the total round-trip time. + constexpr milliseconds kIncreasedOutboundDelay(6); + constexpr milliseconds kIncreasedRoundTripDelay(kIncreasedOutboundDelay + + kInboundDelay); + SetSenderToReceiverNetworkDelay(kIncreasedOutboundDelay); + + // With increased network delay, run several more network round-trips. Expect + // the Sender to gradually converge towards the new network round-trip time. + constexpr int kNumReportIntervals = 50; + EXPECT_CALL(*receiver(), OnSenderReport(_)) + .Times(kNumReportIntervals) + .WillRepeatedly( + [&](const SenderReportParser::SenderReportWithId& sender_report) { + receiver()->SetReceiverReport(sender_report.report_id, + RtcpReportBlock::Delay::zero()); + receiver()->TransmitRtcpFeedbackPacket(); + }); + Clock::duration last_max = sender()->GetMaxInFlightMediaDuration(); + for (int i = 0; i < kNumReportIntervals; ++i) { + SimulateExecution(kRtcpReportInterval); + const Clock::duration updated_value = + sender()->GetMaxInFlightMediaDuration(); + EXPECT_LE(last_max, updated_value); + last_max = updated_value; + } + EXPECT_NEARLY_EQUAL(kTargetPlayoutDelay / 2 + kIncreasedRoundTripDelay / 2, + sender()->GetMaxInFlightMediaDuration(), kEpsilon); +} + +// Tests that the Sender rejects frames if too large a span of FrameIds would be +// in-flight at once. +TEST_F(SenderTest, RejectsEnqueuingBeforeProtocolDesignLimit) { + // For this test, use 1000 FPS. This makes the frames all one millisecond + // apart to avoid triggering the media-duration rejection logic. + constexpr int kFramesPerSecond = 1000; + constexpr milliseconds kSmallFrameDuration(1); + + // Send the absolute design-limit maximum number of frames. + int frame_count = 0; + for (; frame_count < kMaxUnackedFrames; ++frame_count) { + EncodedFrameWithBuffer frame; + PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, + 13 /* bytes */, &frame); + OverrideRtpTimestamp(frame_count, &frame, kFramesPerSecond); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); + SimulateExecution(kSmallFrameDuration); + } + + // Now, attempting to enqueue just one more frame should fail. + EncodedFrameWithBuffer one_frame_too_much; + PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, + 13 /* bytes */, &one_frame_too_much); + OverrideRtpTimestamp(frame_count++, &one_frame_too_much, kFramesPerSecond); + EXPECT_EQ(Sender::REACHED_ID_SPAN_LIMIT, + sender()->EnqueueFrame(one_frame_too_much)); + SimulateExecution(kSmallFrameDuration); + + // Now, simulate the Receiver ACKing the first frame, and enqueuing should + // then succeed again. + receiver()->SetCheckpointFrame(FrameId::first()); + receiver()->TransmitRtcpFeedbackPacket(); + SimulateExecution(); // RTCP transmitted to Sender. + EXPECT_EQ(Sender::OK, sender()->EnqueueFrame(one_frame_too_much)); + SimulateExecution(kSmallFrameDuration); + + // Finally, attempting to enqueue another frame should fail again. + EncodedFrameWithBuffer another_frame_too_much; + PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, + 13 /* bytes */, &another_frame_too_much); + OverrideRtpTimestamp(frame_count++, &another_frame_too_much, + kFramesPerSecond); + EXPECT_EQ(Sender::REACHED_ID_SPAN_LIMIT, + sender()->EnqueueFrame(another_frame_too_much)); + SimulateExecution(kSmallFrameDuration); +} + +TEST_F(SenderTest, CanCancelAllInFlightFrames) { + StrictMock observer; + sender()->SetObserver(&observer); + + // Send the absolute design-limit maximum number of frames. + for (int i = 0; i < kMaxUnackedFrames; ++i) { + EncodedFrameWithBuffer frame; + PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, + 13 /* bytes */, &frame); + OverrideRtpTimestamp(i, &frame, 1000 /* fps */); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); + SimulateExecution(kFrameDuration); + } + + EXPECT_CALL(observer, OnFrameCanceled(_)).Times(kMaxUnackedFrames); + sender()->CancelInFlightData(); +} + +// Tests that the Sender rejects frames if too-long a media duration is +// in-flight. This is the Sender's primary flow control mechanism. +TEST_F(SenderTest, RejectsEnqueuingIfTooLongMediaDurationIsInFlight) { + // For this test, use 20 FPS. This makes all frames 50 ms apart, which should + // make it easy to trigger the media-duration rejection logic. + constexpr int kFramesPerSecond = 20; + constexpr milliseconds kLargeFrameDuration(50); + + // Enqueue frames until one is rejected because the in-flight duration would + // be too high. + EncodedFrameWithBuffer frame; + int frame_count = 0; + for (; frame_count < kMaxUnackedFrames; ++frame_count) { + PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, + 13 /* bytes */, &frame); + OverrideRtpTimestamp(frame_count, &frame, kFramesPerSecond); + const auto result = sender()->EnqueueFrame(frame); + SimulateExecution(kLargeFrameDuration); + if (result == Sender::MAX_DURATION_IN_FLIGHT) { + break; + } + ASSERT_EQ(Sender::OK, result); + } + + // Now, simulate the Receiver ACKing the first frame, and enqueuing should + // then succeed again. + receiver()->SetCheckpointFrame(FrameId::first()); + receiver()->TransmitRtcpFeedbackPacket(); + SimulateExecution(); // RTCP transmitted to Sender. + EXPECT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); + SimulateExecution(kLargeFrameDuration); + + // However, attempting to enqueue another frame should fail again. + EncodedFrameWithBuffer one_frame_too_much; + PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, + 13 /* bytes */, &one_frame_too_much); + OverrideRtpTimestamp(++frame_count, &one_frame_too_much, kFramesPerSecond); + EXPECT_EQ(Sender::MAX_DURATION_IN_FLIGHT, + sender()->EnqueueFrame(one_frame_too_much)); + SimulateExecution(kLargeFrameDuration); +} + +// Tests that the Sender correctly dispatches frame drop events to the +// statistics collector. +TEST_F(SenderTest, ReportFrameDropEvent) { + StrictMock observer; + sender()->SetObserver(&observer); + + const FrameId frame_id = FrameId::first(); + const RtpTimeTicks rtp_timestamp(12345); + const Clock::time_point drop_time = FakeClock::now() + milliseconds(10); + + sender()->ReportFrameDropEvent(frame_id, rtp_timestamp, drop_time); + + // Verification relies on the underlying StatisticsCollector and Dispatcher + // being properly hooked up, which is tested in their respective unit tests. + // In `SenderTest`, we primarily just make sure calling this does not crash. + // We can also verify it through the `TakeRecentFrameEvents()` on a mock/fake + // if `StatisticsCollector` is accessible, but `SenderTest` environment uses + // the real one. +} + +// Tests that the Sender propagates the Receiver's picture loss indicator to the +// Observer::OnPictureLost(), and via calls to NeedsKeyFrame(); but only when +// producing a key frame is absolutely necessary. +TEST_F(SenderTest, ManagesReceiverPictureLossWorkflow) { + StrictMock observer; + sender()->SetObserver(&observer); + + // Send three frames... + EncodedFrameWithBuffer frames[6]; + for (int i = 0; i < 3; ++i) { + if (i == 0) { + EXPECT_TRUE(sender()->NeedsKeyFrame()); + } else { + EXPECT_FALSE(sender()->NeedsKeyFrame()); + } + PopulateFrameWithDefaults(FrameId::first() + i, + FakeClock::now() - kCaptureDelay, 0, + 24 /* bytes */, &frames[i]); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); + SimulateExecution(kFrameDuration); + } + SimulateExecution(kTargetPlayoutDelay); + + // Simulate the Receiver ACK'ing the first three frames. + EXPECT_CALL(observer, OnFrameCanceled(FrameId::first())); + EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 1)); + EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 2)); + EXPECT_CALL(observer, OnPictureLost()).Times(0); + receiver()->SetCheckpointFrame(frames[2].frame_id); + receiver()->TransmitRtcpFeedbackPacket(); + SimulateExecution(); // RTCP transmitted to Sender. + Mock::VerifyAndClearExpectations(&observer); + + // Simulate something going wrong in the Receiver, and have it report picture + // loss to the Sender. The Sender should then propagate this to its Observer + // and return true when NeedsKeyFrame() is called. + EXPECT_CALL(observer, OnFrameCanceled(_)).Times(0); + EXPECT_CALL(observer, OnPictureLost()); + EXPECT_FALSE(sender()->NeedsKeyFrame()); + receiver()->SetPictureLossIndicator(true); + receiver()->TransmitRtcpFeedbackPacket(); + SimulateExecution(); // RTCP transmitted to Sender. + Mock::VerifyAndClearExpectations(&observer); + EXPECT_TRUE(sender()->NeedsKeyFrame()); + + // Send a non-key frame, and expect NeedsKeyFrame() still returns true. The + // Observer is not re-notified. This accounts for the case where a client's + // media encoder had frames in its processing pipeline before NeedsKeyFrame() + // began returning true. + EXPECT_CALL(observer, OnFrameCanceled(_)).Times(0); + EXPECT_CALL(observer, OnPictureLost()).Times(0); + EncodedFrameWithBuffer& nonkey_frame = frames[3]; + PopulateFrameWithDefaults(FrameId::first() + 3, + FakeClock::now() - kCaptureDelay, 0, 24 /* bytes */, + &nonkey_frame); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(nonkey_frame)); + SimulateExecution(kFrameDuration); + Mock::VerifyAndClearExpectations(&observer); + EXPECT_TRUE(sender()->NeedsKeyFrame()); + + // Now send a key frame, and expect NeedsKeyFrame() returns false. Note that + // the Receiver hasn't cleared the PLI condition, but the Sender knows more + // key frames won't be needed. + EXPECT_CALL(observer, OnFrameCanceled(_)).Times(0); + EXPECT_CALL(observer, OnPictureLost()).Times(0); + EncodedFrameWithBuffer& recovery_frame = frames[4]; + PopulateFrameWithDefaults(FrameId::first() + 4, + FakeClock::now() - kCaptureDelay, 0, 24 /* bytes */, + &recovery_frame); + recovery_frame.dependency = EncodedFrame::Dependency::kKeyFrame; + recovery_frame.referenced_frame_id = recovery_frame.frame_id; + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(recovery_frame)); + SimulateExecution(kFrameDuration); + Mock::VerifyAndClearExpectations(&observer); + EXPECT_FALSE(sender()->NeedsKeyFrame()); + + // Let's say the Receiver hasn't received the key frame yet, and it reports + // its picture loss again to the Sender. Observer::OnPictureLost() should not + // be called, and NeedsKeyFrame() should NOT return true, because the Sender + // knows the Receiver hasn't acknowledged the key frame (just sent) yet. + EXPECT_CALL(observer, OnFrameCanceled(nonkey_frame.frame_id)); + EXPECT_CALL(observer, OnPictureLost()).Times(0); + receiver()->SetCheckpointFrame(nonkey_frame.frame_id); + receiver()->SetPictureLossIndicator(true); + receiver()->TransmitRtcpFeedbackPacket(); + SimulateExecution(); // RTCP transmitted to Sender. + Mock::VerifyAndClearExpectations(&observer); + EXPECT_FALSE(sender()->NeedsKeyFrame()); + + // Now, simulate the Receiver getting the key frame, but NOT recovering. This + // should cause Observer::OnPictureLost() to be called, and cause + // NeedsKeyFrame() to return true again. + EXPECT_CALL(observer, OnFrameCanceled(recovery_frame.frame_id)); + EXPECT_CALL(observer, OnPictureLost()); + receiver()->SetCheckpointFrame(recovery_frame.frame_id); + receiver()->SetPictureLossIndicator(true); + receiver()->TransmitRtcpFeedbackPacket(); + SimulateExecution(); // RTCP transmitted to Sender. + Mock::VerifyAndClearExpectations(&observer); + EXPECT_TRUE(sender()->NeedsKeyFrame()); + + // Send another key frame, and expect NeedsKeyFrame() returns false. + EXPECT_CALL(observer, OnFrameCanceled(_)).Times(0); + EXPECT_CALL(observer, OnPictureLost()).Times(0); + EncodedFrameWithBuffer& another_recovery_frame = frames[5]; + PopulateFrameWithDefaults(FrameId::first() + 5, + FakeClock::now() - kCaptureDelay, 0, 24 /* bytes */, + &another_recovery_frame); + another_recovery_frame.dependency = EncodedFrame::Dependency::kKeyFrame; + another_recovery_frame.referenced_frame_id = another_recovery_frame.frame_id; + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(another_recovery_frame)); + SimulateExecution(kFrameDuration); + Mock::VerifyAndClearExpectations(&observer); + EXPECT_FALSE(sender()->NeedsKeyFrame()); + + // Now, simulate the Receiver recovering. It will report this to the Sender, + // and NeedsKeyFrame() will still return false. + EXPECT_CALL(observer, OnFrameCanceled(another_recovery_frame.frame_id)); + EXPECT_CALL(observer, OnPictureLost()).Times(0); + receiver()->SetCheckpointFrame(another_recovery_frame.frame_id); + receiver()->SetPictureLossIndicator(false); + receiver()->TransmitRtcpFeedbackPacket(); + SimulateExecution(); // RTCP transmitted to Sender. + Mock::VerifyAndClearExpectations(&observer); + EXPECT_FALSE(sender()->NeedsKeyFrame()); + + ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); +} + +// Tests that the Receiver should get a Sender Report just before the first RTP +// packet, and at regular intervals thereafter. The Sender Report contains the +// lip-sync information necessary for play-out timing. +TEST_F(SenderTest, ProvidesSenderReports) { + std::vector sender_reports; + Sequence packet_sequence; + EXPECT_CALL(*receiver(), OnSenderReport(_)) + .InSequence(packet_sequence) + .WillOnce([&](const SenderReportParser::SenderReportWithId& report) { + sender_reports.push_back(report); + }) + .RetiresOnSaturation(); + EXPECT_CALL(*receiver(), OnRtpPacket(_)).InSequence(packet_sequence); + EXPECT_CALL(*receiver(), OnSenderReport(_)) + .Times(3) + .InSequence(packet_sequence) + .WillRepeatedly( + [&](const SenderReportParser::SenderReportWithId& report) { + sender_reports.push_back(report); + }); + + EncodedFrameWithBuffer frame; + constexpr int kFrameDataSize = 250; + PopulateFrameWithDefaults(FrameId::first(), FakeClock::now(), 0, + kFrameDataSize, &frame); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); + SimulateExecution(); // Should send one Sender Report + one RTP packet. + EXPECT_EQ(size_t{1}, sender_reports.size()); + + // Have the Receiver ACK the frame to prevent retransmitting the RTP packet. + receiver()->SetCheckpointFrame(FrameId::first()); + receiver()->TransmitRtcpFeedbackPacket(); + SimulateExecution(); // RTCP transmitted to Sender. + + // Advance through three more reporting intervals. One Sender Report should be + // sent each interval, making a total of 4 reports sent. + constexpr auto kThreeReportIntervals = 3 * kRtcpReportInterval; + SimulateExecution(kThreeReportIntervals); // Three more Sender Reports. + ASSERT_EQ(size_t{4}, sender_reports.size()); + + // The first report should contain the same timestamps as the frame because + // the Clock did not advance. Also, its packet count and octet count fields + // should be zero since the report was sent before the RTP packet. + EXPECT_EQ(frame.reference_time, sender_reports.front().reference_time); + EXPECT_EQ(frame.rtp_timestamp, sender_reports.front().rtp_timestamp); + EXPECT_EQ(uint32_t{0}, sender_reports.front().send_packet_count); + EXPECT_EQ(uint32_t{0}, sender_reports.front().send_octet_count); + + // The last report should contain the timestamps extrapolated into the future + // because the Clock did move forward. Also, the packet count and octet fields + // should now be non-zero because the report was sent after the RTP packet. + EXPECT_EQ(frame.reference_time + kThreeReportIntervals, + sender_reports.back().reference_time); + EXPECT_EQ(frame.rtp_timestamp + + RtpTimeDelta::FromDuration(kThreeReportIntervals, kRtpTimebase), + sender_reports.back().rtp_timestamp); + EXPECT_EQ(uint32_t{1}, sender_reports.back().send_packet_count); + EXPECT_EQ(uint32_t{kFrameDataSize}, sender_reports.back().send_octet_count); +} + +TEST_F(SenderTest, ReferenceTimesCanBeNonMonotonic) { + // This tests that the sender is robust to encoded frames with non-monotonic + // reference times. This situation does not prevent frames from being + // transmitted in correct order; however, lip sync will suffer if reference + // times do not correspond to RTP timestamps. + EXPECT_CALL(*receiver(), OnRtpPacket(_)).Times(AtLeast(1)); + EXPECT_CALL(*receiver(), OnSenderReport(_)).Times(AtLeast(1)); + + // Send the 10 frames with non-monotonic reference times. + Clock::time_point reference_time = FakeClock::now(); + for (int i = 0; i < 10; ++i) { + EncodedFrameWithBuffer frame; + PopulateFrameWithDefaults(sender()->GetNextFrameId(), reference_time, 0, + 13 /* bytes */, &frame); + // OverrideRtpTimestamp(i, &frame, 1000 /* fps */); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); + SimulateExecution(kFrameDuration); + reference_time -= microseconds(10); + } +} + +// Tests that the Sender provides Kickstart packets whenever the Receiver may +// not know about new frames. +TEST_F(SenderTest, ProvidesKickstartPacketsIfReceiverDoesNotACK) { + // Have the Receiver move the checkpoint forward only for the first frame, and + // none of the later frames. This will force the Sender to eventually send a + // Kickstart packet. + ON_CALL(*receiver(), OnFrameComplete(_)).WillByDefault([&](FrameId frame_id) { + if (frame_id == FrameId::first()) { + receiver()->SetCheckpointFrame(FrameId::first()); + receiver()->TransmitRtcpFeedbackPacket(); + } + }); + + // Send three frames, paced to the media. + EncodedFrameWithBuffer frames[3]; + for (int i = 0; i < 3; ++i) { + PopulateFrameWithDefaults(FrameId::first() + i, + FakeClock::now() - kCaptureDelay, i, + 48 /* bytes */, &frames[i]); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); + SimulateExecution(kFrameDuration); + } + + // Now, do nothing for a while. Because the Receiver isn't moving the + // checkpoint forward, the Sender will have sent all the RTP packets at least + // once, and then will start sending just Kickstart packets. + SimulateExecution(kTargetPlayoutDelay); + + // Keep doing nothing for a while, and confirm the Sender is just sending the + // same Kickstart packet over and over. The Kickstart packet is supposed to be + // the last packet of the latest frame. + std::set> unique_received_packet_ids; + EXPECT_CALL(*receiver(), OnRtpPacket(_)) + .WillRepeatedly([&](const RtpPacketParser::ParseResult& parsed_packet) { + unique_received_packet_ids.emplace(parsed_packet.frame_id, + parsed_packet.packet_id); + }); + SimulateExecution(kTargetPlayoutDelay); + Mock::VerifyAndClearExpectations(receiver()); + EXPECT_EQ(size_t{1}, unique_received_packet_ids.size()); + EXPECT_EQ(frames[2].frame_id, unique_received_packet_ids.begin()->first); + + // Now, simulate the Receiver ACKing all the frames. + receiver()->SetCheckpointFrame(frames[2].frame_id); + receiver()->TransmitRtcpFeedbackPacket(); + SimulateExecution(); // RTCP transmitted to Sender. + + // With all the frames sent, the Sender should not be transmitting anything. + EXPECT_CALL(*receiver(), OnRtpPacket(_)).Times(0); + SimulateExecution(10 * kTargetPlayoutDelay); + + ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); +} + +// Tests that the Sender only retransmits packets specifically NACK'ed by the +// Receiver. +TEST_F(SenderTest, ResendsIndividuallyNackedPackets) { + // Populate the frame data in each frame with enough bytes to force at least + // three RTP packets per frame. + constexpr int kFrameDataSize = 3 * kMaxRtpPacketSizeForIpv6UdpOnEthernet; + + // Use a 1ms network delay in each direction to make the sequence of events + // clearer in this test. + constexpr milliseconds kOneWayNetworkDelay(1); + SetSenderToReceiverNetworkDelay(kOneWayNetworkDelay); + SetReceiverToSenderNetworkDelay(kOneWayNetworkDelay); + + // Simulate that three specific packets will be dropped by the network, one + // from each frame (about to be sent). + const std::vector dropped_packets{ + {FrameId::first(), FramePacketId{2}}, + {FrameId::first() + 1, FramePacketId{1}}, + {FrameId::first() + 2, FramePacketId{0}}, + }; + receiver()->SetIgnoreList(dropped_packets); + + // Send three frames, paced to the media. The Receiver won't completely + // receive any of these frames due to dropped packets. + EXPECT_CALL(*receiver(), OnFrameComplete(_)).Times(0); + EncodedFrameWithBuffer frames[3]; + for (int i = 0; i < 3; ++i) { + PopulateFrameWithDefaults(FrameId::first() + i, + FakeClock::now() - kCaptureDelay, i, + kFrameDataSize, &frames[i]); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); + SimulateExecution(kFrameDuration); + } + SimulateExecution(kTargetPlayoutDelay); + Mock::VerifyAndClearExpectations(receiver()); + EXPECT_EQ(3u, sender()->GetInFlightFrameCount()); + + // The Receiver NACKs the three dropped packets... + receiver()->SetNacksAndAcks(dropped_packets, {}); + receiver()->TransmitRtcpFeedbackPacket(); + + // In the meantime, the network recovers (i.e., no more dropped packets)... + receiver()->SetIgnoreList({}); + + // The NACKs reach the Sender, and it acts on them by retransmitting. + SimulateExecution(kOneWayNetworkDelay); + + // As each retransmitted packet arrives at the Receiver, advance the + // checkpoint forward to notify the Sender of frames that are now completely + // received. Also, confirm that only the three specifically-NACK'ed packets + // were retransmitted. + EXPECT_CALL(*receiver(), OnFrameComplete(_)) + .Times(3) + .WillRepeatedly(InvokeWithoutArgs([&] { + if (receiver()->AutoAdvanceCheckpoint()) { + receiver()->TransmitRtcpFeedbackPacket(); + } + })); + EXPECT_CALL(*receiver(), OnRtpPacket(_)) + .Times(3) + .WillRepeatedly([&](const RtpPacketParser::ParseResult& packet) { + EXPECT_TRUE(Contains(dropped_packets, + PacketNack{packet.frame_id, packet.packet_id})); + }); + SimulateExecution(kOneWayNetworkDelay); + Mock::VerifyAndClearExpectations(receiver()); + + // The Receiver checkpoint feedback(s) travel back to the Sender, and there + // should no longer be any frames in-flight. + SimulateExecution(kOneWayNetworkDelay); + EXPECT_EQ(0u, sender()->GetInFlightFrameCount()); + + // The Sender should not be transmitting anything from now on since all frames + // are known to have been completely received. + EXPECT_CALL(*receiver(), OnRtpPacket(_)).Times(0); + SimulateExecution(10 * kTargetPlayoutDelay); + + ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); +} + +// Tests that the Sender retransmits an entire frame if the Receiver requests it +// (i.e., a full frame NACK), but does not retransmit any packets for frames +// (before or after) that have been acknowledged. +TEST_F(SenderTest, ResendsMissingFrames) { + // Populate the frame data in each frame with enough bytes to force at least + // three RTP packets per frame. + constexpr int kFrameDataSize = 3 * kMaxRtpPacketSizeForIpv6UdpOnEthernet; + + // Use a 1ms network delay in each direction to make the sequence of events + // clearer in this test. + constexpr milliseconds kOneWayNetworkDelay(1); + SetSenderToReceiverNetworkDelay(kOneWayNetworkDelay); + SetReceiverToSenderNetworkDelay(kOneWayNetworkDelay); + + // Simulate that all of the packets for the second frame will be dropped by + // the network, but only the packets for that frame. + const std::vector dropped_packets{ + {FrameId::first() + 1, kAllPacketsLost}, + }; + receiver()->SetIgnoreList(dropped_packets); + + StrictMock observer; + sender()->SetObserver(&observer); + + // The expectations below track the story and execute simulated Receiver + // responses. The Sender will have three frames enqueued by its client, and + // then... + // + // The first frame is received and the Receiver ACKs it by moving the + // checkpoint forward. + Sequence completion_sequence; + EXPECT_CALL(*receiver(), OnFrameComplete(FrameId::first())) + .InSequence(completion_sequence) + .WillOnce(InvokeWithoutArgs([&] { + receiver()->SetCheckpointFrame(FrameId::first()); + receiver()->TransmitRtcpFeedbackPacket(); + })); + // Since all of the packets for the second frame are being dropped, the third + // frame will finish next. The Receiver responds by NACKing the second frame + // and ACKing the third frame. The checkpoint does not move forward because + // the second frame has not been received yet. + // + // NETWORK CHANGE: After the third frame is received, stop dropping packets. + EXPECT_CALL(*receiver(), OnFrameComplete(FrameId::first() + 2)) + .InSequence(completion_sequence) + .WillOnce(InvokeWithoutArgs([&] { + receiver()->SetNacksAndAcks(dropped_packets, + std::vector{FrameId::first() + 2}); + receiver()->TransmitRtcpFeedbackPacket(); + receiver()->SetIgnoreList({}); + })); + // Finally, the Sender should respond to the whole-frame NACK by re-sending + // all of the packets for the second frame, and so the Receiver should + // completely receive the frame. + EXPECT_CALL(*receiver(), OnFrameComplete(FrameId::first() + 1)) + .InSequence(completion_sequence) + .WillOnce(InvokeWithoutArgs([&] { + receiver()->SetCheckpointFrame(FrameId::first() + 2); + receiver()->TransmitRtcpFeedbackPacket(); + })); + + // From the Sender's perspective, the Receiver will ACK the first frame, then + // the third frame, then the second frame. + Sequence cancel_sequence; + EXPECT_CALL(observer, OnFrameCanceled(FrameId::first())) + .InSequence(cancel_sequence); + EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 2)) + .InSequence(cancel_sequence); + EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 1)) + .InSequence(cancel_sequence); + + // With all the expectations/sequences in-place, let 'er rip! + EncodedFrameWithBuffer frames[3]; + for (int i = 0; i < 3; ++i) { + PopulateFrameWithDefaults(FrameId::first() + i, + FakeClock::now() - kCaptureDelay, i, + kFrameDataSize, &frames[i]); + ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); + SimulateExecution(kFrameDuration); + } + SimulateExecution(kTargetPlayoutDelay); + Mock::VerifyAndClearExpectations(receiver()); + EXPECT_EQ(0u, sender()->GetInFlightFrameCount()); + + // The Sender should not be transmitting anything from now on since all frames + // are known to have been completely received. + EXPECT_CALL(*receiver(), OnRtpPacket(_)).Times(0); + SimulateExecution(10 * kTargetPlayoutDelay); + + ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); +} + +} // namespace +} // namespace openscreen::cast diff --git a/cast/streaming/impl/sender_message_unittest.cc b/cast/streaming/impl/sender_message_unittest.cc new file mode 100644 index 000000000..504079930 --- /dev/null +++ b/cast/streaming/impl/sender_message_unittest.cc @@ -0,0 +1,196 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cast/streaming/sender_message.h" + +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "util/json/json_serialization.h" + +namespace openscreen::cast { + +TEST(SenderMessageTest, ErrorOnNonObjectMessage) { + Json::Value array_val(Json::arrayValue); + EXPECT_TRUE(SenderMessage::Parse(array_val).is_error()); + + Json::Value string_val("string"); + EXPECT_TRUE(SenderMessage::Parse(string_val).is_error()); + + Json::Value int_val(42); + EXPECT_TRUE(SenderMessage::Parse(int_val).is_error()); +} + +TEST(SenderMessageTest, ParsesGetCapabilities) { + ErrorOr root = json::Parse(R"({ + "type": "GET_CAPABILITIES", + "seqNum": 123 + })"); + ASSERT_TRUE(root.is_value()); + + ErrorOr message = SenderMessage::Parse(root.value()); + ASSERT_TRUE(message.is_value()); + EXPECT_EQ(SenderMessage::Type::kGetCapabilities, message.value().type); + EXPECT_EQ(123, message.value().sequence_number); + EXPECT_TRUE(message.value().valid); + + ErrorOr serialized = message.value().ToJson(); + ASSERT_TRUE(serialized.is_value()); + EXPECT_EQ(json::Stringify(root.value()).value(), + json::Stringify(serialized.value()).value()); +} + +TEST(SenderMessageTest, ParsesRpc) { + ErrorOr root = json::Parse(R"({ + "type": "RPC", + "seqNum": 456, + "rpc": "SGVsbG8=" + })"); + ASSERT_TRUE(root.is_value()); + + ErrorOr message = SenderMessage::Parse(root.value()); + ASSERT_TRUE(message.is_value()); + EXPECT_EQ(SenderMessage::Type::kRpc, message.value().type); + EXPECT_EQ(456, message.value().sequence_number); + EXPECT_TRUE(message.value().valid); + ASSERT_TRUE( + std::holds_alternative>(message.value().body)); + std::vector expected_body = {'H', 'e', 'l', 'l', 'o'}; + EXPECT_EQ(expected_body, + std::get>(message.value().body)); + + ErrorOr serialized = message.value().ToJson(); + ASSERT_TRUE(serialized.is_value()); + EXPECT_EQ(json::Stringify(root.value()).value(), + json::Stringify(serialized.value()).value()); +} + +TEST(SenderMessageTest, ParsesInput) { + ErrorOr root = json::Parse(R"({ + "type": "INPUT", + "seqNum": 789, + "input": "V29ybGQ=" + })"); + ASSERT_TRUE(root.is_value()); + + ErrorOr message = SenderMessage::Parse(root.value()); + ASSERT_TRUE(message.is_value()); + EXPECT_EQ(SenderMessage::Type::kInput, message.value().type); + EXPECT_EQ(789, message.value().sequence_number); + EXPECT_TRUE(message.value().valid); + ASSERT_TRUE( + std::holds_alternative>(message.value().body)); + std::vector expected_body = {'W', 'o', 'r', 'l', 'd'}; + EXPECT_EQ(expected_body, + std::get>(message.value().body)); + + ErrorOr serialized = message.value().ToJson(); + ASSERT_TRUE(serialized.is_value()); + EXPECT_EQ(json::Stringify(root.value()).value(), + json::Stringify(serialized.value()).value()); +} + +TEST(SenderMessageTest, ParsesOffer) { + ErrorOr root = json::Parse(R"({ + "type": "OFFER", + "seqNum": 101, + "offer": { + "castMode": "mirroring", + "supportedStreams": null + } + })"); + ASSERT_TRUE(root.is_value()); + + // ... (The json parser fails on supportedStreams being null or empty array + // because Offer requires it to be an array and have items. Let's just + // use a valid Offer that actually deserializes to the exact same JSON. + + const char kOfferJson[] = R"({ + "type": "OFFER", + "seqNum": 101, + "offer": { + "castMode": "mirroring", + "supportedStreams": [ + { + "index": 2, + "type": "audio_source", + "codecName": "opus", + "rtpProfile": "cast", + "rtpPayloadType": 96, + "ssrc": 19088743, + "bitRate": 124000, + "timeBase": "1/48000", + "channels": 2, + "aesKey": "51027e4e2347cbcb49d57ef10177aebc", + "aesIvMask": "7f12a19be62a36c04ae4116caaeff6d1", + "codecParameter": "", + "receiverRtcpEventLog": true, + "targetDelay": 400 + } + ] + } + })"; + + root = json::Parse(kOfferJson); + ASSERT_TRUE(root.is_value()); + + ErrorOr message = SenderMessage::Parse(root.value()); + ASSERT_TRUE(message.is_value()); + EXPECT_EQ(SenderMessage::Type::kOffer, message.value().type); + EXPECT_EQ(101, message.value().sequence_number); + EXPECT_TRUE(message.value().valid); + ASSERT_TRUE(std::holds_alternative(message.value().body)); + + ErrorOr serialized = message.value().ToJson(); + ASSERT_TRUE(serialized.is_value()); + EXPECT_EQ(json::Stringify(root.value()).value(), + json::Stringify(serialized.value()).value()); +} + +TEST(SenderMessageTest, HandlesMissingSequenceNumber) { + ErrorOr root = json::Parse(R"({ + "type": "GET_CAPABILITIES" + })"); + ASSERT_TRUE(root.is_value()); + + ErrorOr message = SenderMessage::Parse(root.value()); + ASSERT_TRUE(message.is_value()); + EXPECT_EQ(SenderMessage::Type::kGetCapabilities, message.value().type); + EXPECT_EQ(-1, message.value().sequence_number); + EXPECT_TRUE(message.value().valid); + + ErrorOr serialized = message.value().ToJson(); + ASSERT_TRUE(serialized.is_value()); + EXPECT_EQ(json::Stringify(root.value()).value(), + json::Stringify(serialized.value()).value()); +} + +TEST(SenderMessageTest, InvalidMissingBody) { + ErrorOr root = json::Parse(R"({ + "type": "RPC", + "seqNum": 456 + })"); + ASSERT_TRUE(root.is_value()); + + ErrorOr message = SenderMessage::Parse(root.value()); + ASSERT_TRUE(message.is_value()); + EXPECT_EQ(SenderMessage::Type::kRpc, message.value().type); + EXPECT_FALSE(message.value().valid); +} + +TEST(SenderMessageTest, HandlesUnknownType) { + ErrorOr root = json::Parse(R"({ + "type": "UNKNOWN_GARBAGE", + "seqNum": 123 + })"); + ASSERT_TRUE(root.is_value()); + + ErrorOr message = SenderMessage::Parse(root.value()); + ASSERT_TRUE(message.is_value()); + EXPECT_EQ(SenderMessage::Type::kUnknown, message.value().type); + EXPECT_FALSE(message.value().valid); +} + +} // namespace openscreen::cast diff --git a/cast/streaming/impl/sender_report_parser.cc b/cast/streaming/impl/sender_report_parser.cc index 4575e0df3..041c817cb 100644 --- a/cast/streaming/impl/sender_report_parser.cc +++ b/cast/streaming/impl/sender_report_parser.cc @@ -29,12 +29,12 @@ std::optional SenderReportParser::Parse( if (!header) { return std::nullopt; } - buffer.remove_prefix(kRtcpCommonHeaderSize); + buffer = buffer.subspan(kRtcpCommonHeaderSize); if (static_cast(buffer.size()) < header->payload_size) { return std::nullopt; } auto chunk = buffer.subspan(0, header->payload_size); - buffer.remove_prefix(header->payload_size); + buffer = buffer.subspan(header->payload_size); // Only process Sender Reports with a matching SSRC. if (header->packet_type != RtcpPacketType::kSenderReport) { diff --git a/cast/streaming/impl/sender_report_parser_fuzzer.cc b/cast/streaming/impl/sender_report_parser_fuzzer.cc index 54597b6be..d415e225a 100644 --- a/cast/streaming/impl/sender_report_parser_fuzzer.cc +++ b/cast/streaming/impl/sender_report_parser_fuzzer.cc @@ -22,14 +22,12 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { // also contains a NtpTimeConverter, which samples the system clock at // construction time. There is no reason to re-construct these objects for // each fuzzer test input. -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wexit-time-destructors" - static RtcpSession session(kSenderSsrcInSeedCorpus, kReceiverSsrcInSeedCorpus, - openscreen::Clock::now()); - static SenderReportParser parser(session); -#pragma clang diagnostic pop - - parser.Parse(openscreen::ByteView(data, size)); + static auto* session = + new RtcpSession(kSenderSsrcInSeedCorpus, kReceiverSsrcInSeedCorpus, + openscreen::Clock::now()); + static auto* parser = new SenderReportParser(*session); + + parser->Parse(openscreen::ByteView(data, size)); return 0; } diff --git a/cast/streaming/impl/sender_session_unittest.cc b/cast/streaming/impl/sender_session_unittest.cc index b3367c903..886a8d150 100644 --- a/cast/streaming/impl/sender_session_unittest.cc +++ b/cast/streaming/impl/sender_session_unittest.cc @@ -5,10 +5,15 @@ #include "cast/streaming/public/sender_session.h" #include +#include +#include #include #include "cast/streaming/capture_configs.h" +#include "cast/streaming/impl/message_constants.h" +#include "cast/streaming/input.pb.h" #include "cast/streaming/public/capture_recommendations.h" +#include "cast/streaming/public/protobuf_messenger.h" #include "cast/streaming/testing/mock_environment.h" #include "cast/streaming/testing/simple_message_port.h" #include "gmock/gmock.h" @@ -17,11 +22,11 @@ #include "platform/test/fake_clock.h" #include "platform/test/fake_task_runner.h" #include "util/chrono_helpers.h" +#include "util/no_destructor.h" #include "util/stringprintf.h" using ::testing::_; using ::testing::InSequence; -using ::testing::Invoke; using ::testing::NiceMock; using ::testing::Return; using ::testing::StrictMock; @@ -114,46 +119,59 @@ constexpr char kCapabilitiesErrorResponse[] = R"({ } })"; -const AudioCaptureConfig kAudioCaptureConfigInvalidChannels{ - AudioCodec::kAac, -1 /* channels */, 44000 /* bit_rate */, - 96000 /* sample_rate */ -}; +constexpr char kInputMessage[] = R"({ + "seqNum": 1, + "type": "INPUT", + "input": "CGQQnBiCGQgSAggMGgIIBg==" +})"; + +const AudioCaptureConfig& GetAudioCaptureConfigInvalidChannels() { + static const NoDestructor + kAudioCaptureConfigInvalidChannels(AudioCodec::kAac, -1 /* channels */, + 44000 /* bit_rate */, + 96000 /* sample_rate */); + return *kAudioCaptureConfigInvalidChannels; +} + +const AudioCaptureConfig& GetAudioCaptureConfigValid() { + static const NoDestructor kAudioCaptureConfigValid( + AudioCodec::kAac, 5 /* channels */, 32000 /* bit_rate */, + 44000 /* sample_rate */, std::chrono::milliseconds(300), "mp4a.40.5"); + return *kAudioCaptureConfigValid; +} + +const VideoCaptureConfig& GetVideoCaptureConfigMissingResolutions() { + static const NoDestructor + kVideoCaptureConfigMissingResolutions( + VideoCodec::kHevc, SimpleFraction{60, 1}, 300000 /* max_bit_rate */, + std::vector{}, std::chrono::milliseconds(500), + "hev1.1.6.L150.B0"); + return *kVideoCaptureConfigMissingResolutions; +} -const AudioCaptureConfig kAudioCaptureConfigValid{ - AudioCodec::kAac, - 5 /* channels */, - 32000 /* bit_rate */, - 44000 /* sample_rate */, - std::chrono::milliseconds(300), - "mp4a.40.5"}; - -const VideoCaptureConfig kVideoCaptureConfigMissingResolutions{ - VideoCodec::kHevc, - {60, 1}, - 300000 /* max_bit_rate */, - std::vector{}, - std::chrono::milliseconds(500), - "hev1.1.6.L150.B0"}; - -const VideoCaptureConfig kVideoCaptureConfigInvalid{ - VideoCodec::kHevc, - {60, 1}, - -300000 /* max_bit_rate */, - std::vector{Resolution{1920, 1080}, Resolution{1280, 720}}}; - -const VideoCaptureConfig kVideoCaptureConfigValid{ - VideoCodec::kHevc, - {60, 1}, - 300000 /* max_bit_rate */, - std::vector{Resolution{1280, 720}, Resolution{1920, 1080}}, - std::chrono::milliseconds(250), - "hev1.1.6.L150.B0"}; - -const VideoCaptureConfig kVideoCaptureConfigValidSimplest{ - VideoCodec::kHevc, - {60, 1}, - 300000 /* max_bit_rate */, - std::vector{Resolution{1920, 1080}}}; +const VideoCaptureConfig& GetVideoCaptureConfigInvalid() { + static const NoDestructor kVideoCaptureConfigInvalid( + VideoCodec::kHevc, SimpleFraction{60, 1}, -300000 /* max_bit_rate */, + std::vector{Resolution{1920, 1080}, Resolution{1280, 720}}); + return *kVideoCaptureConfigInvalid; +} + +const VideoCaptureConfig& GetVideoCaptureConfigValid() { + static const NoDestructor kVideoCaptureConfigValid( + VideoCodec::kHevc, SimpleFraction{60, 1}, 300000 /* max_bit_rate */, + std::vector{Resolution{1280, 720}, Resolution{1920, 1080}}, + std::chrono::milliseconds(250), "hev1.1.6.L150.B0"); + return *kVideoCaptureConfigValid; +} + +[[maybe_unused]] const VideoCaptureConfig& +GetVideoCaptureConfigValidSimplest() { + static const NoDestructor + kVideoCaptureConfigValidSimplest( + VideoCodec::kHevc, SimpleFraction{60, 1}, 300000 /* max_bit_rate */, + std::vector{Resolution{1920, 1080}}); + return *kVideoCaptureConfigValidSimplest; +} class FakeClient : public SenderSession::Client { public: @@ -173,6 +191,8 @@ class FakeClient : public SenderSession::Client { (override)); }; +// TODO(jophba): this matcher is likely more generally useful and should +// be refactored. MATCHER_P(CodeEquals, code, "Checks error codes but not messages.") { return arg.code() == code; } @@ -202,20 +222,21 @@ class SenderSessionTest : public ::testing::Test { message_port_.get(), "sender-12345", "receiver-12345", - /* use_android_rtp_hack */ true}; + /* use_android_rtp_hack */ true, + /* use_dscp */ true}; session_ = std::make_unique(std::move(config)); } void NegotiateMirroringWithValidConfigs() { const Error error = session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); ASSERT_TRUE(error.ok()); } void NegotiateRemotingWithValidConfigs() { - const Error error = session_->NegotiateRemoting(kAudioCaptureConfigValid, - kVideoCaptureConfigValid); + const Error error = session_->NegotiateRemoting( + GetAudioCaptureConfigValid(), GetVideoCaptureConfigValid()); ASSERT_TRUE(error.ok()); } @@ -250,18 +271,18 @@ class SenderSessionTest : public ::testing::Test { const int video_index = video_stream["index"].asInt(); const int video_ssrc = video_stream["ssrc"].asUInt(); - constexpr char kAnswerTemplate[] = R"({ + static constexpr char kAnswerTemplate[] = R"({{ "type": "ANSWER", - "seqNum": %d, + "seqNum": {}, "result": "ok", - "answer": { - "castMode": "%s", + "answer": {{ + "castMode": "{}", "udpPort": 1234, - "sendIndexes": [%d, %d], - "ssrcs": [%d, %d] - } - })"; - return StringPrintf(kAnswerTemplate, offer["seqNum"].asInt(), + "sendIndexes": [{}, {}], + "ssrcs": [{}, {}] + }} + }})"; + return StringFormat(kAnswerTemplate, offer["seqNum"].asInt(), mode == CastMode::kMirroring ? "mirroring" : "remoting", audio_index, video_index, audio_ssrc + 1, video_ssrc + 1); @@ -287,7 +308,7 @@ TEST_F(SenderSessionTest, ComplainsIfNoConfigsToOffer) { TEST_F(SenderSessionTest, ComplainsIfInvalidAudioCaptureConfig) { const Error error = session_->Negotiate( - std::vector{kAudioCaptureConfigInvalidChannels}, + std::vector{GetAudioCaptureConfigInvalidChannels()}, std::vector{}); EXPECT_EQ(error, @@ -297,23 +318,24 @@ TEST_F(SenderSessionTest, ComplainsIfInvalidAudioCaptureConfig) { TEST_F(SenderSessionTest, ComplainsIfInvalidVideoCaptureConfig) { const Error error = session_->Negotiate( std::vector{}, - std::vector{kVideoCaptureConfigInvalid}); + std::vector{GetVideoCaptureConfigInvalid()}); EXPECT_EQ(error, Error(Error::Code::kParameterInvalid, "Invalid configs provided.")); } TEST_F(SenderSessionTest, ComplainsIfMissingResolutions) { - const Error error = session_->Negotiate( - std::vector{}, - std::vector{kVideoCaptureConfigMissingResolutions}); + const Error error = + session_->Negotiate(std::vector{}, + std::vector{ + GetVideoCaptureConfigMissingResolutions()}); EXPECT_EQ(error, Error(Error::Code::kParameterInvalid, "Invalid configs provided.")); } TEST_F(SenderSessionTest, SendsOfferWithZeroBitrateOptions) { - VideoCaptureConfig video_config = kVideoCaptureConfigValid; + VideoCaptureConfig video_config = GetVideoCaptureConfigValid(); video_config.max_bit_rate = 0; - AudioCaptureConfig audio_config = kAudioCaptureConfigValid; + AudioCaptureConfig audio_config = GetAudioCaptureConfigValid(); audio_config.bit_rate = 0; const Error error = @@ -332,7 +354,7 @@ TEST_F(SenderSessionTest, SendsOfferWithZeroBitrateOptions) { TEST_F(SenderSessionTest, SendsOfferWithSimpleVideoOnly) { const Error error = session_->Negotiate( std::vector{}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetVideoCaptureConfigValid()}); EXPECT_TRUE(error.ok()); const auto& messages = message_port_->posted_messages(); @@ -345,7 +367,7 @@ TEST_F(SenderSessionTest, SendsOfferWithSimpleVideoOnly) { TEST_F(SenderSessionTest, SendsOfferAudioOnly) { const Error error = session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, + std::vector{GetAudioCaptureConfigValid()}, std::vector{}); EXPECT_TRUE(error.ok()); @@ -359,8 +381,8 @@ TEST_F(SenderSessionTest, SendsOfferAudioOnly) { TEST_F(SenderSessionTest, SendsOfferMessage) { session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); const auto& messages = message_port_->posted_messages(); ASSERT_EQ(1u, messages.size()); @@ -417,9 +439,9 @@ TEST_F(SenderSessionTest, HandlesStreamTypes) { // configured with the proper StreamType. ON_CALL(client_, OnNegotiated(session_.get(), _, _)) .WillByDefault( - Invoke([&](const SenderSession* sender_session, - SenderSession::ConfiguredSenders senders, - capture_recommendations::Recommendations recommendations) { + [&](const SenderSession* sender_session, + SenderSession::ConfiguredSenders senders, + capture_recommendations::Recommendations recommendations) { StreamType audio_stream_type = senders.audio_sender->config().stream_type; StreamType video_stream_type = @@ -427,7 +449,7 @@ TEST_F(SenderSessionTest, HandlesStreamTypes) { EXPECT_EQ(audio_stream_type, StreamType::kAudio); EXPECT_EQ(video_stream_type, StreamType::kVideo); - })); + }); EXPECT_CALL(client_, OnNegotiated(session_.get(), _, _)); message_port_->ReceiveMessage(answer); } @@ -440,8 +462,8 @@ TEST_F(SenderSessionTest, HandlesInvalidNamespace) { TEST_F(SenderSessionTest, HandlesMalformedAnswer) { session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); // Note that unlike when we simply don't select any streams, when the answer // is actually malformed we have no way of knowing it was an answer at all, @@ -452,8 +474,8 @@ TEST_F(SenderSessionTest, HandlesMalformedAnswer) { TEST_F(SenderSessionTest, HandlesImproperlyFormattedAnswer) { session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); EXPECT_CALL(client_, OnError(session_.get(), CodeEquals(Error::Code::kInvalidAnswer))); @@ -462,8 +484,8 @@ TEST_F(SenderSessionTest, HandlesImproperlyFormattedAnswer) { TEST_F(SenderSessionTest, HandlesInvalidAnswer) { const Error error = session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); EXPECT_CALL(client_, OnError(session_.get(), CodeEquals(Error::Code::kInvalidAnswer))); @@ -472,8 +494,8 @@ TEST_F(SenderSessionTest, HandlesInvalidAnswer) { TEST_F(SenderSessionTest, HandlesNullAnswer) { const Error error = session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); EXPECT_TRUE(error.ok()); EXPECT_CALL(client_, OnError(session_.get(), _)); @@ -482,8 +504,8 @@ TEST_F(SenderSessionTest, HandlesNullAnswer) { TEST_F(SenderSessionTest, HandlesAnswerTimeout) { const Error error = session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); EXPECT_TRUE(error.ok()); // No ANSWER received in time, should report an error. @@ -504,8 +526,8 @@ TEST_F(SenderSessionTest, HandlesCapabilitiesTimeout) { TEST_F(SenderSessionTest, HandlesInvalidSequenceNumber) { const Error error = session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); // We should just discard messages with an invalid sequence number. message_port_->ReceiveMessage(kInvalidSequenceNumberMessage); @@ -513,8 +535,8 @@ TEST_F(SenderSessionTest, HandlesInvalidSequenceNumber) { TEST_F(SenderSessionTest, HandlesUnknownTypeMessageWithValidSeqNum) { session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); // If a message is of unknown type but has an expected seqnum, it's // probably a malformed response. @@ -524,8 +546,8 @@ TEST_F(SenderSessionTest, HandlesUnknownTypeMessageWithValidSeqNum) { TEST_F(SenderSessionTest, HandlesInvalidTypeMessageWithValidSeqNum) { session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); // If a message is of unknown type but has an expected seqnum, it's // probably a malformed response. The sender session will end up @@ -538,8 +560,8 @@ TEST_F(SenderSessionTest, HandlesInvalidTypeMessageWithValidSeqNum) { TEST_F(SenderSessionTest, HandlesInvalidTypeMessage) { session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); // We should just discard messages with an invalid message type and // no sequence number. @@ -548,8 +570,8 @@ TEST_F(SenderSessionTest, HandlesInvalidTypeMessage) { TEST_F(SenderSessionTest, HandlesErrorMessage) { session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); // We should report error responses. NOTE: according to the spec, // RPC messages should never be an error result. @@ -565,8 +587,8 @@ TEST_F(SenderSessionTest, HandlesErrorMessage) { TEST_F(SenderSessionTest, HandlesMessagePortError) { session_->Negotiate( - std::vector{kAudioCaptureConfigValid}, - std::vector{kVideoCaptureConfigValid}); + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); // We should report message port errors. EXPECT_CALL(client_, OnError(session_.get(), _)); @@ -582,20 +604,20 @@ TEST_F(SenderSessionTest, ReportsZeroBandwidthWhenNoPacketsSent) { TEST_F(SenderSessionTest, ComplainsIfInvalidAudioCaptureConfigRemoting) { const Error error = session_->NegotiateRemoting( - kAudioCaptureConfigInvalidChannels, kVideoCaptureConfigValid); + GetAudioCaptureConfigInvalidChannels(), GetVideoCaptureConfigValid()); EXPECT_EQ(error.code(), Error::Code::kParameterInvalid); } TEST_F(SenderSessionTest, ComplainsIfInvalidVideoCaptureConfigRemoting) { - const Error error = session_->NegotiateRemoting(kAudioCaptureConfigValid, - kVideoCaptureConfigInvalid); + const Error error = session_->NegotiateRemoting( + GetAudioCaptureConfigValid(), GetVideoCaptureConfigInvalid()); EXPECT_EQ(error.code(), Error::Code::kParameterInvalid); } TEST_F(SenderSessionTest, ComplainsIfMissingResolutionsRemoting) { const Error error = session_->NegotiateRemoting( - kAudioCaptureConfigValid, kVideoCaptureConfigMissingResolutions); + GetAudioCaptureConfigValid(), GetVideoCaptureConfigMissingResolutions()); EXPECT_EQ(error.code(), Error::Code::kParameterInvalid); } @@ -646,4 +668,157 @@ TEST_F(SenderSessionTest, SuccessfulGetCapabilitiesRequest) { EXPECT_THAT(capabilities.video, testing::ElementsAre(VideoCapability::kVp8)); } +TEST_F(SenderSessionTest, SendsOfferWithoutDscp) { + // Tear down the default session before attempting to do a custom one here. + session_.reset(); + message_port_ = std::make_unique("receiver-12345"); + environment_ = MakeEnvironment(); + + SenderSession::Configuration config{ + IPAddress::kV4LoopbackAddress(), + client_, + environment_.get(), + message_port_.get(), + "sender-12345", + "receiver-12345", + true, // use_android_rtp_hack + false // enable_dscp + }; + session_ = std::make_unique(std::move(config)); + + session_->Negotiate( + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); + + const auto& messages = message_port_->posted_messages(); + ASSERT_EQ(1u, messages.size()); + + auto message_body = json::Parse(messages[0]); + ASSERT_TRUE(message_body.is_value()); + const Json::Value offer = std::move(message_body.value()); + const Json::Value& offer_body = offer["offer"]; + const Json::Value& streams = offer_body["supportedStreams"]; + EXPECT_TRUE(streams.isArray()); + EXPECT_EQ(2u, streams.size()); + + const Json::Value& audio_stream = streams[0]; + EXPECT_TRUE(audio_stream["receiverRtcpDscp"].isNull()); + + const Json::Value& video_stream = streams[1]; + EXPECT_TRUE(video_stream["receiverRtcpDscp"].isNull()); +} + +TEST_F(SenderSessionTest, SendsOfferWithDscp) { + // Tear down the default session before attempting to do a custom one here. + session_.reset(); + message_port_ = std::make_unique("receiver-12345"); + environment_ = MakeEnvironment(); + + SenderSession::Configuration config{ + IPAddress::kV4LoopbackAddress(), + client_, + environment_.get(), + message_port_.get(), + "sender-12345", + "receiver-12345", + true, // use_android_rtp_hack + true // enable_dscp + }; + session_ = std::make_unique(std::move(config)); + + session_->Negotiate( + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); + + const auto& messages = message_port_->posted_messages(); + ASSERT_EQ(1u, messages.size()); + + auto message_body = json::Parse(messages[0]); + ASSERT_TRUE(message_body.is_value()); + const Json::Value offer = std::move(message_body.value()); + const Json::Value& offer_body = offer["offer"]; + const Json::Value& streams = offer_body["supportedStreams"]; + EXPECT_TRUE(streams.isArray()); + EXPECT_EQ(2u, streams.size()); + + const Json::Value& audio_stream = streams[0]; + EXPECT_EQ(static_cast(UdpSocket::DscpMode::kAF41), + audio_stream["receiverRtcpDscp"].asInt()); + + const Json::Value& video_stream = streams[1]; + EXPECT_EQ(static_cast(UdpSocket::DscpMode::kAF41), + video_stream["receiverRtcpDscp"].asInt()); +} + +TEST_F(SenderSessionTest, InputEventsOptIn) { + // Opt-in using SetInputCallback. + session_->SetInputCallback([](InputMessage message) {}); + + session_->Negotiate( + std::vector{GetAudioCaptureConfigValid()}, + std::vector{GetVideoCaptureConfigValid()}); + + const auto& messages = message_port_->posted_messages(); + ASSERT_EQ(1u, messages.size()); + auto message_body = json::Parse(messages[0]); + ASSERT_TRUE(message_body.is_value()); + const Json::Value& offer = message_body.value()["offer"]; + + // Verify input_events extension is present in the offer. + bool found_extension = false; + for (const auto& stream : offer["supportedStreams"]) { + for (const auto& ext : stream["rtpExtensions"]) { + if (ext.asString() == kInputEventsRtpExtension) { + found_extension = true; + break; + } + } + } + EXPECT_TRUE(found_extension); +} + +TEST_F(SenderSessionTest, HandlesSetInputCallbackUpdate) { + session_->SetInputCallback([](InputMessage message) {}); +} + +TEST_F(SenderSessionTest, HandlesSetInputCallbackNull) { + session_->SetInputCallback([](InputMessage message) {}); + + session_->SetInputCallback(nullptr); +} + +TEST_F(SenderSessionTest, HandlesInputMessage) { + NegotiateMirroringWithValidConfigs(); + EXPECT_CALL(client_, OnNegotiated(session_.get(), _, _)); + message_port_->ReceiveMessage(ConstructAnswerFromOffer(CastMode::kMirroring)); + + bool called = false; + session_->SetInputCallback( + [&called](InputMessage message) { called = true; }); + message_port_->ReceiveMessage(kInputMessage); + EXPECT_TRUE(called); +} + +TEST_F(SenderSessionTest, SendsInputMessage) { + NegotiateMirroringWithValidConfigs(); + EXPECT_CALL(client_, OnNegotiated(session_.get(), _, _)); + message_port_->ReceiveMessage(ConstructAnswerFromOffer(CastMode::kMirroring)); + + session_->SetInputCallback([](InputMessage message) {}); + + InputMessage message; + auto* event = message.add_events(); + event->set_type(InputMessage::INPUT_TYPE_KEY_DOWN); + + message_port_->clear(); + session_->SendInputMessage(message); + + const auto& messages = message_port_->posted_messages(); + ASSERT_EQ(1u, messages.size()); + auto message_body = json::Parse(messages[0]); + ASSERT_TRUE(message_body.is_value()); + EXPECT_EQ("INPUT", message_body.value()["type"].asString()); + EXPECT_FALSE(message_body.value()["input"].asString().empty()); +} + } // namespace openscreen::cast diff --git a/cast/streaming/impl/sender_unittest.cc b/cast/streaming/impl/sender_unittest.cc deleted file mode 100644 index b7b2d2c2f..000000000 --- a/cast/streaming/impl/sender_unittest.cc +++ /dev/null @@ -1,1228 +0,0 @@ -// Copyright 2020 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "cast/streaming/public/sender.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "cast/streaming/impl/compound_rtcp_builder.h" -#include "cast/streaming/impl/frame_collector.h" -#include "cast/streaming/impl/frame_crypto.h" -#include "cast/streaming/impl/packet_util.h" -#include "cast/streaming/impl/rtcp_session.h" -#include "cast/streaming/impl/rtp_defines.h" -#include "cast/streaming/impl/rtp_packet_parser.h" -#include "cast/streaming/impl/sender_report_parser.h" -#include "cast/streaming/impl/session_config.h" -#include "cast/streaming/public/constants.h" -#include "cast/streaming/public/encoded_frame.h" -#include "cast/streaming/public/frame_id.h" -#include "cast/streaming/sender_packet_router.h" -#include "cast/streaming/ssrc.h" -#include "cast/streaming/testing/mock_environment.h" -#include "cast/streaming/testing/simple_socket_subscriber.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "platform/base/span.h" -#include "platform/test/byte_view_test_util.h" -#include "platform/test/fake_clock.h" -#include "platform/test/fake_task_runner.h" -#include "util/alarm.h" -#include "util/chrono_helpers.h" -#include "util/std_util.h" -#include "util/yet_another_bit_vector.h" - -using testing::_; -using testing::AtLeast; -using testing::Invoke; -using testing::InvokeWithoutArgs; -using testing::Mock; -using testing::NiceMock; -using testing::Return; -using testing::Sequence; -using testing::StrictMock; - -namespace openscreen::cast { -namespace { - -// Sender configuration. -constexpr Ssrc kSenderSsrc = 1; -constexpr Ssrc kReceiverSsrc = 2; -constexpr int kRtpTimebase = 48000; -constexpr milliseconds kTargetPlayoutDelay(400); -constexpr auto kAesKey = - std::array{{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}}; -constexpr auto kCastIvMask = - std::array{{0xf0, 0xe0, 0xd0, 0xc0, 0xb0, 0xa0, 0x90, 0x80, - 0x70, 0x60, 0x50, 0x40, 0x30, 0x20, 0x10, 0x00}}; -constexpr RtpPayloadType kRtpPayloadType = RtpPayloadType::kVideoVp8; - -// The number of RTP ticks advanced per frame, for 100 FPS media. -constexpr int kRtpTicksPerFrame = kRtpTimebase / 100; - -// The number of milliseconds advanced per frame, for 100 FPS media. -constexpr milliseconds kFrameDuration{1000 / 100}; -static_assert(kFrameDuration < (kTargetPlayoutDelay / 10), - "Kickstart test assumes frame duration is far less than the " - "playout delay."); - -// An Encoded frame that also holds onto its own copy of data. -struct EncodedFrameWithBuffer : public EncodedFrame { - // `EncodedFrame::data` always points inside buffer.begin()...buffer.end(). - std::vector buffer; -}; - -// SenderPacketRouter configuration for these tests. -constexpr int kNumPacketsPerBurst = 20; -constexpr milliseconds kBurstInterval(10); - -// An arbitrary value, subtracted from "now," to specify the reference_time on -// frames that are about to be enqueued. This simulates that capture+encode -// happened in the past, before Sender::EnqueueFrame() is called. -constexpr milliseconds kCaptureDelay(11); - -// In some tests, the computed time values could be off a little bit due to -// imprecision in certain wire-format timestamp types. The following macro -// behaves just like Gtest's EXPECT_NEAR(), but works with all the time types -// too. -#define EXPECT_NEARLY_EQUAL(duration_a, duration_b, epsilon) \ - if ((duration_a) >= (duration_b)) { \ - EXPECT_LE((duration_a), (duration_b) + (epsilon)); \ - } else { \ - EXPECT_GE((duration_a), (duration_b) - (epsilon)); \ - } - -void OverrideRtpTimestamp(int frame_count, EncodedFrame* frame, int fps) { - const int ticks = frame_count * kRtpTimebase / fps; - frame->rtp_timestamp = RtpTimeTicks() + RtpTimeDelta::FromTicks(ticks); -} - -// Simulates UDP/IPv6 traffic in one direction (from Sender→Receiver, or -// Receiver→Sender), with a settable amount of delay. -class SimulatedNetworkPipe { - public: - SimulatedNetworkPipe(TaskRunner& task_runner, - Environment::PacketConsumer& remote) - : task_runner_(task_runner), remote_(remote) { - // Create a fake IPv6 address using the "documentative purposes" prefix - // concatenated with the `this` pointer. - std::array hextets{}; - hextets[0] = 0x2001; - hextets[1] = 0x0db8; - auto* const this_pointer = this; - static_assert(sizeof(this_pointer) <= (6 * sizeof(uint16_t)), ""); - memcpy(&hextets[2], &this_pointer, sizeof(this_pointer)); - local_endpoint_ = IPEndpoint{IPAddress(hextets), 2344}; - } - - const IPEndpoint& local_endpoint() const { return local_endpoint_; } - - Clock::duration network_delay() const { return network_delay_; } - void set_network_delay(Clock::duration delay) { network_delay_ = delay; } - - // The caller needs to spin the task runner before `packet` will reach the - // other side. - void StartPacketTransmission(std::vector packet) { - task_runner_.PostTaskWithDelay( - [this, pkt = std::move(packet)]() mutable { - remote_.OnReceivedPacket(local_endpoint_, FakeClock::now(), - std::move(pkt)); - }, - network_delay_); - } - - private: - TaskRunner& task_runner_; - Environment::PacketConsumer& remote_; - - IPEndpoint local_endpoint_; - - // The amount of time for the packet to transmit over this simulated network - // pipe. Defaults to zero to simplify the tests that don't care about delays. - Clock::duration network_delay_{}; -}; - -// Processes packets from the Sender under test, allowing unit tests to set -// expectations for parsed RTP or RTCP packets, to confirm proper behavior of -// the Sender. -class MockReceiver : public Environment::PacketConsumer { - public: - explicit MockReceiver(SimulatedNetworkPipe& pipe_to_sender) - : pipe_to_sender_(pipe_to_sender), - rtcp_session_(kSenderSsrc, kReceiverSsrc, FakeClock::now()), - sender_report_parser_(rtcp_session_), - rtcp_builder_(rtcp_session_), - rtp_parser_(kSenderSsrc), - crypto_(kAesKey, kCastIvMask) { - rtcp_builder_.SetPlayoutDelay(kTargetPlayoutDelay); - } - - ~MockReceiver() override = default; - - // Simulate the Receiver ACK'ing all frames up to and including the - // `new_checkpoint`. - void SetCheckpointFrame(FrameId new_checkpoint) { - OSP_CHECK_GE(new_checkpoint, rtcp_builder_.checkpoint_frame()); - rtcp_builder_.SetCheckpointFrame(new_checkpoint); - } - - // Automatically advances the checkpoint based on what is found in - // `complete_frames_`, returning true if the checkpoint moved forward. - bool AutoAdvanceCheckpoint() { - const FrameId old_checkpoint = rtcp_builder_.checkpoint_frame(); - FrameId new_checkpoint = old_checkpoint; - for (auto it = complete_frames_.upper_bound(old_checkpoint); - it != complete_frames_.end(); ++it) { - if (it->first != new_checkpoint + 1) { - break; - } - ++new_checkpoint; - } - if (new_checkpoint > old_checkpoint) { - rtcp_builder_.SetCheckpointFrame(new_checkpoint); - return true; - } - return false; - } - - void SetPictureLossIndicator(bool picture_is_lost) { - rtcp_builder_.SetPictureLossIndicator(picture_is_lost); - } - - void SetReceiverReport(StatusReportId reply_for, - RtcpReportBlock::Delay processing_delay) { - RtcpReportBlock receiver_report; - receiver_report.ssrc = kSenderSsrc; - receiver_report.last_status_report_id = reply_for; - receiver_report.delay_since_last_report = processing_delay; - rtcp_builder_.IncludeReceiverReportInNextPacket(receiver_report); - } - - void SetNacksAndAcks(std::vector packet_nacks, - std::vector frame_acks) { - rtcp_builder_.IncludeFeedbackInNextPacket(std::move(packet_nacks), - std::move(frame_acks)); - } - - // Builds and sends a RTCP packet containing one or more of: checkpoint, PLI, - // Receiver Report, NACKs, ACKs. - void TransmitRtcpFeedbackPacket() { - uint8_t buffer[kMaxRtpPacketSizeForIpv6UdpOnEthernet]; - const ByteBuffer packet = - rtcp_builder_.BuildPacket(FakeClock::now(), buffer); - pipe_to_sender_.StartPacketTransmission( - std::vector(packet.begin(), packet.end())); - } - - // Used by tests to simulate the Receiver not seeing specific packets come in - // from the network (e.g., because the network dropped the packets). - void SetIgnoreList(std::vector ignore_list) { - ignore_list_ = ignore_list; - } - - // Environment::PacketConsumer implementation. - // - // Called to process a packet from the Sender, simulating basic RTP frame - // collection and Sender Report parsing/handling. - void OnReceivedPacket(const IPEndpoint& source, - Clock::time_point arrival_time, - std::vector packet) override { - const auto type_and_ssrc = InspectPacketForRouting(packet); - EXPECT_NE(ApparentPacketType::UNKNOWN, type_and_ssrc.first); - EXPECT_EQ(kSenderSsrc, type_and_ssrc.second); - if (type_and_ssrc.first == ApparentPacketType::RTP) { - const std::optional part_of_frame = - rtp_parser_.Parse(packet); - ASSERT_TRUE(part_of_frame); - - // Return early if simulating packet drops over the network. - if (ContainsIf(ignore_list_, [&](const PacketNack& baddie) { - return (baddie.frame_id == part_of_frame->frame_id && - (baddie.packet_id == kAllPacketsLost || - baddie.packet_id == part_of_frame->packet_id)); - })) { - return; - } - - OnRtpPacket(*part_of_frame); - CollectRtpPacket(*part_of_frame, std::move(packet)); - } else if (type_and_ssrc.first == ApparentPacketType::RTCP) { - std::optional report = - sender_report_parser_.Parse(packet); - ASSERT_TRUE(report); - OnSenderReport(*report); - } - } - - std::map TakeCompleteFrames() { - std::map result; - result.swap(complete_frames_); - return result; - } - - // Tests set expectations on these mocks to monitor events of interest, and/or - // invoke additional behaviors. - MOCK_METHOD1(OnRtpPacket, - void(const RtpPacketParser::ParseResult& parsed_packet)); - MOCK_METHOD1(OnFrameComplete, void(FrameId frame_id)); - MOCK_METHOD1(OnSenderReport, - void(const SenderReportParser::SenderReportWithId& report)); - - private: - // Collects the individual RTP packets until a whole frame can be formed, then - // calls OnFrameComplete(). Ignores extra RTP packets that are no longer - // needed. - void CollectRtpPacket(const RtpPacketParser::ParseResult& part_of_frame, - std::vector packet) { - const FrameId frame_id = part_of_frame.frame_id; - if (complete_frames_.find(frame_id) != complete_frames_.end()) { - return; - } - FrameCollector& collector = incomplete_frames_[frame_id]; - collector.set_frame_id(frame_id); - EXPECT_TRUE(collector.CollectRtpPacket(part_of_frame, &packet)); - if (!collector.is_complete()) { - return; - } - const EncryptedFrame& encrypted = collector.PeekAtAssembledFrame(); - EncodedFrameWithBuffer& decrypted = complete_frames_[frame_id]; - // Note: Not setting decrypted->reference_time here since the logic around - // calculating the playout time is rather complex, and is definitely outside - // the scope of the testing being done in this module. Instead, end-to-end - // testing should exist elsewhere to confirm frame play-out times with real - // Receivers. - decrypted.buffer.resize(FrameCrypto::GetPlaintextSize(encrypted)); - crypto_.Decrypt(encrypted, decrypted.buffer); - encrypted.CopyMetadataTo(&decrypted); - decrypted.data = decrypted.buffer; - incomplete_frames_.erase(frame_id); - OnFrameComplete(frame_id); - } - - SimulatedNetworkPipe& pipe_to_sender_; - RtcpSession rtcp_session_; - SenderReportParser sender_report_parser_; - CompoundRtcpBuilder rtcp_builder_; - RtpPacketParser rtp_parser_; - FrameCrypto crypto_; - - std::vector ignore_list_; - std::map incomplete_frames_; - std::map complete_frames_; -}; - -class MockObserver : public Sender::Observer { - public: - MOCK_METHOD1(OnFrameCanceled, void(FrameId frame_id)); - MOCK_METHOD0(OnPictureLost, void()); -}; - -class SenderTest : public testing::Test { - public: - SenderTest() - : fake_clock_(Clock::now()), - task_runner_(fake_clock_), - sender_environment_(&FakeClock::now, task_runner_), - sender_packet_router_(sender_environment_, - kNumPacketsPerBurst, - kBurstInterval), - sender_(sender_environment_, - sender_packet_router_, - {/* .sender_ssrc = */ kSenderSsrc, - /* .receiver_ssrc = */ kReceiverSsrc, - /* .rtp_timebase = */ kRtpTimebase, - /* .channels = */ 2, - /* .target_playout_delay = */ kTargetPlayoutDelay, - /* .aes_secret_key = */ kAesKey, - /* .aes_iv_mask = */ kCastIvMask, - /* .is_pli_enabled = */ true}, - kRtpPayloadType), - receiver_to_sender_pipe_(task_runner_, sender_packet_router_), - receiver_(receiver_to_sender_pipe_), - sender_to_receiver_pipe_(task_runner_, receiver_) { - sender_environment_.SetSocketSubscriber(&socket_subscriber_); - sender_environment_.set_remote_endpoint( - receiver_to_sender_pipe_.local_endpoint()); - ON_CALL(sender_environment_, SendPacket(_, _)) - .WillByDefault(Invoke([this](ByteView packet, PacketMetadata metadata) { - sender_to_receiver_pipe_.StartPacketTransmission( - std::vector(packet.begin(), packet.end())); - })); - } - - ~SenderTest() override = default; - - Sender* sender() { return &sender_; } - MockReceiver* receiver() { return &receiver_; } - - void SetReceiverToSenderNetworkDelay(Clock::duration delay) { - receiver_to_sender_pipe_.set_network_delay(delay); - } - - void SetSenderToReceiverNetworkDelay(Clock::duration delay) { - sender_to_receiver_pipe_.set_network_delay(delay); - } - - void SimulateExecution(Clock::duration how_long = Clock::duration::zero()) { - fake_clock_.Advance(how_long); - } - - static void PopulateFramePayloadBuffer(int seed, - int num_bytes, - std::vector* payload) { - payload->clear(); - payload->reserve(num_bytes); - for (int i = 0; i < num_bytes; ++i) { - payload->push_back(static_cast(seed + i)); - } - } - - static void PopulateFrameWithDefaults(FrameId frame_id, - Clock::time_point reference_time, - int seed, - int num_payload_bytes, - EncodedFrameWithBuffer* frame) { - frame->dependency = (frame_id == FrameId::first()) - ? EncodedFrame::Dependency::kKeyFrame - : EncodedFrame::Dependency::kDependent; - frame->frame_id = frame_id; - frame->referenced_frame_id = frame->frame_id; - if (frame_id != FrameId::first()) { - --frame->referenced_frame_id; - } - frame->rtp_timestamp = - RtpTimeTicks() + (RtpTimeDelta::FromTicks(kRtpTicksPerFrame) * - (frame_id - FrameId::first())); - frame->reference_time = reference_time; - PopulateFramePayloadBuffer(seed, num_payload_bytes, &frame->buffer); - frame->data = frame->buffer; - } - - // Confirms that all `sent_frames` exist in `received_frames`, with identical - // data and metadata. - static void ExpectFramesReceivedCorrectly( - Span sent_frames, - const std::map received_frames) { - ASSERT_EQ(sent_frames.size(), received_frames.size()); - - for (const EncodedFrameWithBuffer& sent_frame : sent_frames) { - SCOPED_TRACE(testing::Message() - << "Checking sent frame " << sent_frame.frame_id); - const auto received_it = received_frames.find(sent_frame.frame_id); - if (received_it == received_frames.end()) { - ADD_FAILURE() << "Did not receive frame."; - continue; - } - const EncodedFrame& received_frame = received_it->second; - EXPECT_EQ(sent_frame.dependency, received_frame.dependency); - EXPECT_EQ(sent_frame.referenced_frame_id, - received_frame.referenced_frame_id); - EXPECT_EQ(sent_frame.rtp_timestamp, received_frame.rtp_timestamp); - ExpectByteViewsHaveSameBytes(sent_frame.data, received_frame.data); - } - } - - private: - FakeClock fake_clock_; - FakeTaskRunner task_runner_; - NiceMock sender_environment_; - SenderPacketRouter sender_packet_router_; - Sender sender_; - SimulatedNetworkPipe receiver_to_sender_pipe_; - NiceMock receiver_; - SimulatedNetworkPipe sender_to_receiver_pipe_; - SimpleSubscriber socket_subscriber_; -}; - -// Tests that the Sender can send EncodedFrames over an ideal network (i.e., low -// latency, no loss), and does so without having to transmit the same packet -// twice. -TEST_F(SenderTest, SendsFramesEfficiently) { - constexpr milliseconds kOneWayNetworkDelay(1); - SetSenderToReceiverNetworkDelay(kOneWayNetworkDelay); - SetReceiverToSenderNetworkDelay(kOneWayNetworkDelay); - - // Expect that each packet is only sent once. - std::set> received_packets; - EXPECT_CALL(*receiver(), OnRtpPacket(_)) - .WillRepeatedly( - Invoke([&](const RtpPacketParser::ParseResult& parsed_packet) { - std::pair id(parsed_packet.frame_id, - parsed_packet.packet_id); - const auto insert_result = received_packets.insert(id); - EXPECT_TRUE(insert_result.second) - << "Received duplicate packet: " << id.first << ':' - << static_cast(id.second); - })); - - // Simulate normal frame ACK'ing behavior. - ON_CALL(*receiver(), OnFrameComplete(_)).WillByDefault(InvokeWithoutArgs([&] { - if (receiver()->AutoAdvanceCheckpoint()) { - receiver()->TransmitRtcpFeedbackPacket(); - } - })); - - StrictMock observer; - EXPECT_CALL(observer, OnFrameCanceled(FrameId::first())); - EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 1)); - EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 2)); - sender()->SetObserver(&observer); - - EncodedFrameWithBuffer frames[3]; - constexpr int kFrameDataSizes[] = {8196, 12, 1900}; - for (int i = 0; i < 3; ++i) { - if (i == 0) { - EXPECT_TRUE(sender()->NeedsKeyFrame()); - } else { - EXPECT_FALSE(sender()->NeedsKeyFrame()); - } - PopulateFrameWithDefaults(FrameId::first() + i, - FakeClock::now() - kCaptureDelay, 0xbf - i, - kFrameDataSizes[i], &frames[i]); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); - SimulateExecution(kFrameDuration); - } - SimulateExecution(kTargetPlayoutDelay); - - ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); -} - -// Tests that the Sender properly updates the checkpoint frame ID while -// it is cancelling frames. See https://crbug.com/1433584 for an example crash -// where the checkpoint frame ID is invalid. -TEST_F(SenderTest, WaitsUntilEndOfReportToUpdateObservers) { - constexpr milliseconds kOneWayNetworkDelay(1); - SetSenderToReceiverNetworkDelay(kOneWayNetworkDelay); - SetReceiverToSenderNetworkDelay(kOneWayNetworkDelay); - - // Expect that each packet is only sent once. - std::set> received_packets; - EXPECT_CALL(*receiver(), OnRtpPacket(_)) - .WillRepeatedly( - Invoke([&](const RtpPacketParser::ParseResult& parsed_packet) { - std::pair id(parsed_packet.frame_id, - parsed_packet.packet_id); - const auto insert_result = received_packets.insert(id); - EXPECT_TRUE(insert_result.second) - << "Received duplicate packet: " << id.first << ':' - << static_cast(id.second); - })); - - StrictMock observer; - - // The sender should be in a valid state during frame cancellations. Since - // these all came from the same report, the sender shouldn't have any frames - // in flight. - EXPECT_CALL(observer, OnFrameCanceled(_)) - .Times(3) - .WillRepeatedly([sender = sender()](FrameId id) { - EXPECT_EQ(0, sender->GetInFlightFrameCount()); - - // Since no frames are in flight, the next frame timestamp should not - // matter. - EXPECT_EQ(Clock::duration::zero(), - sender->GetInFlightMediaDuration(RtpTimeTicks(123456789))); - }); - - // Don't ACK frames and return a report until the third frame. - EXPECT_CALL(*receiver(), OnFrameComplete(_)).Times(2); - EXPECT_CALL(*receiver(), OnFrameComplete(FrameId::first() + 2)) - .WillOnce(InvokeWithoutArgs([&] { - if (receiver()->AutoAdvanceCheckpoint()) { - receiver()->TransmitRtcpFeedbackPacket(); - } - })); - - sender()->SetObserver(&observer); - - EncodedFrameWithBuffer frames[3]; - constexpr int kFrameDataSizes[] = {8196, 12, 1900}; - for (int i = 0; i < 3; ++i) { - EXPECT_EQ(i == 0, sender()->NeedsKeyFrame()); - PopulateFrameWithDefaults(FrameId::first() + i, - FakeClock::now() - kCaptureDelay, 0xbf - i, - kFrameDataSizes[i], &frames[i]); - - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); - SimulateExecution(kFrameDuration); - } - SimulateExecution(kTargetPlayoutDelay); - - ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); -} - -// Tests that the Sender correctly computes the current in-flight media -// duration, a backlog signal for clients. -TEST_F(SenderTest, ComputesInFlightMediaDuration) { - // With no frames enqueued, the in-flight media duration should be zero. - EXPECT_EQ(Clock::duration::zero(), - sender()->GetInFlightMediaDuration(RtpTimeTicks())); - EXPECT_EQ(Clock::duration::zero(), - sender()->GetInFlightMediaDuration( - RtpTimeTicks() + RtpTimeDelta::FromTicks(kRtpTicksPerFrame))); - - // Enqueue a frame. - EncodedFrameWithBuffer frame; - PopulateFrameWithDefaults(FrameId::first(), FakeClock::now(), 0, - 13 /* bytes */, &frame); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); - - // Now, the in-flight media duration should depend on the RTP timestamp of the - // next frame. - EXPECT_EQ(kFrameDuration, sender()->GetInFlightMediaDuration( - frame.rtp_timestamp + - RtpTimeDelta::FromTicks(kRtpTicksPerFrame))); - EXPECT_EQ(10 * kFrameDuration, - sender()->GetInFlightMediaDuration( - frame.rtp_timestamp + - RtpTimeDelta::FromTicks(10 * kRtpTicksPerFrame))); -} - -// Tests that the Sender computes the maximum in-flight media duration based on -// its analysis of current network conditions. By implication, this demonstrates -// that the Sender is also measuring the network round-trip time. -TEST_F(SenderTest, RespondsToNetworkLatencyChanges) { - // The expected maximum error in time calculations is one tick of the RTCP - // report block's delay type. - constexpr auto kEpsilon = to_nanoseconds(RtcpReportBlock::Delay(1)); - - // Before the Sender has the necessary information to compute the network - // round-trip time, GetMaxInFlightMediaDuration() will return half the target - // playout delay. - EXPECT_NEARLY_EQUAL(kTargetPlayoutDelay / 2, - sender()->GetMaxInFlightMediaDuration(), kEpsilon); - - // No network is perfect. Simulate different one-way network delays. - constexpr milliseconds kOutboundDelay(2); - constexpr milliseconds kInboundDelay(4); - constexpr milliseconds kRoundTripDelay(kOutboundDelay + kInboundDelay); - SetSenderToReceiverNetworkDelay(kOutboundDelay); - SetReceiverToSenderNetworkDelay(kInboundDelay); - - // Enqueue a frame in the Sender to start emitting periodic RTCP reports. - { - EncodedFrameWithBuffer frame; - PopulateFrameWithDefaults(FrameId::first(), FakeClock::now(), 0, - 1 /* byte */, &frame); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); - } - - // Run one network round-trip from Sender→Receiver→Sender. - StatusReportId sender_report_id{}; - EXPECT_CALL(*receiver(), OnSenderReport(_)) - .WillOnce(Invoke( - [&](const SenderReportParser::SenderReportWithId& sender_report) { - sender_report_id = sender_report.report_id; - })); - // Simulate the passage of time for the Sender Report to reach the Receiver. - SimulateExecution(kOutboundDelay); - // The Receiver should have received the Sender Report at this point. - Mock::VerifyAndClearExpectations(receiver()); - ASSERT_NE(StatusReportId{}, sender_report_id); - // Simulate the passage of time in the Receiver doing "other tasks" before - // replying back to the Sender. This delay is included in the Receiver Report - // so that the Sender can isolate the delays caused by the network. - constexpr milliseconds kReceiverProcessingDelay(2); - SimulateExecution(kReceiverProcessingDelay); - // Create the Receiver Report "reply," and simulate it being sent across the - // network, back to the Sender. - receiver()->SetReceiverReport( - sender_report_id, std::chrono::duration_cast( - kReceiverProcessingDelay)); - receiver()->TransmitRtcpFeedbackPacket(); - SimulateExecution(kInboundDelay); - - // At this point, the Sender should have computed the network round-trip time, - // and so GetMaxInFlightMediaDuration() will return half the target playout - // delay PLUS half the network round-trip time. - EXPECT_NEARLY_EQUAL(kTargetPlayoutDelay / 2 + kRoundTripDelay / 2, - sender()->GetMaxInFlightMediaDuration(), kEpsilon); - - // Increase the outbound delay, which will increase the total round-trip time. - constexpr milliseconds kIncreasedOutboundDelay(6); - constexpr milliseconds kIncreasedRoundTripDelay(kIncreasedOutboundDelay + - kInboundDelay); - SetSenderToReceiverNetworkDelay(kIncreasedOutboundDelay); - - // With increased network delay, run several more network round-trips. Expect - // the Sender to gradually converge towards the new network round-trip time. - constexpr int kNumReportIntervals = 50; - EXPECT_CALL(*receiver(), OnSenderReport(_)) - .Times(kNumReportIntervals) - .WillRepeatedly(Invoke( - [&](const SenderReportParser::SenderReportWithId& sender_report) { - receiver()->SetReceiverReport(sender_report.report_id, - RtcpReportBlock::Delay::zero()); - receiver()->TransmitRtcpFeedbackPacket(); - })); - Clock::duration last_max = sender()->GetMaxInFlightMediaDuration(); - for (int i = 0; i < kNumReportIntervals; ++i) { - SimulateExecution(kRtcpReportInterval); - const Clock::duration updated_value = - sender()->GetMaxInFlightMediaDuration(); - EXPECT_LE(last_max, updated_value); - last_max = updated_value; - } - EXPECT_NEARLY_EQUAL(kTargetPlayoutDelay / 2 + kIncreasedRoundTripDelay / 2, - sender()->GetMaxInFlightMediaDuration(), kEpsilon); -} - -// Tests that the Sender rejects frames if too large a span of FrameIds would be -// in-flight at once. -TEST_F(SenderTest, RejectsEnqueuingBeforeProtocolDesignLimit) { - // For this test, use 1000 FPS. This makes the frames all one millisecond - // apart to avoid triggering the media-duration rejection logic. - constexpr int kFramesPerSecond = 1000; - constexpr milliseconds kSmallFrameDuration(1); - - // Send the absolute design-limit maximum number of frames. - int frame_count = 0; - for (; frame_count < kMaxUnackedFrames; ++frame_count) { - EncodedFrameWithBuffer frame; - PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, - 13 /* bytes */, &frame); - OverrideRtpTimestamp(frame_count, &frame, kFramesPerSecond); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); - SimulateExecution(kSmallFrameDuration); - } - - // Now, attempting to enqueue just one more frame should fail. - EncodedFrameWithBuffer one_frame_too_much; - PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, - 13 /* bytes */, &one_frame_too_much); - OverrideRtpTimestamp(frame_count++, &one_frame_too_much, kFramesPerSecond); - EXPECT_EQ(Sender::REACHED_ID_SPAN_LIMIT, - sender()->EnqueueFrame(one_frame_too_much)); - SimulateExecution(kSmallFrameDuration); - - // Now, simulate the Receiver ACKing the first frame, and enqueuing should - // then succeed again. - receiver()->SetCheckpointFrame(FrameId::first()); - receiver()->TransmitRtcpFeedbackPacket(); - SimulateExecution(); // RTCP transmitted to Sender. - EXPECT_EQ(Sender::OK, sender()->EnqueueFrame(one_frame_too_much)); - SimulateExecution(kSmallFrameDuration); - - // Finally, attempting to enqueue another frame should fail again. - EncodedFrameWithBuffer another_frame_too_much; - PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, - 13 /* bytes */, &another_frame_too_much); - OverrideRtpTimestamp(frame_count++, &another_frame_too_much, - kFramesPerSecond); - EXPECT_EQ(Sender::REACHED_ID_SPAN_LIMIT, - sender()->EnqueueFrame(another_frame_too_much)); - SimulateExecution(kSmallFrameDuration); -} - -TEST_F(SenderTest, CanCancelAllInFlightFrames) { - StrictMock observer; - sender()->SetObserver(&observer); - - // Send the absolute design-limit maximum number of frames. - for (int i = 0; i < kMaxUnackedFrames; ++i) { - EncodedFrameWithBuffer frame; - PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, - 13 /* bytes */, &frame); - OverrideRtpTimestamp(i, &frame, 1000 /* fps */); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); - SimulateExecution(kFrameDuration); - } - - EXPECT_CALL(observer, OnFrameCanceled(_)).Times(kMaxUnackedFrames); - sender()->CancelInFlightData(); -} - -// Tests that the Sender rejects frames if too-long a media duration is -// in-flight. This is the Sender's primary flow control mechanism. -TEST_F(SenderTest, RejectsEnqueuingIfTooLongMediaDurationIsInFlight) { - // For this test, use 20 FPS. This makes all frames 50 ms apart, which should - // make it easy to trigger the media-duration rejection logic. - constexpr int kFramesPerSecond = 20; - constexpr milliseconds kLargeFrameDuration(50); - - // Enqueue frames until one is rejected because the in-flight duration would - // be too high. - EncodedFrameWithBuffer frame; - int frame_count = 0; - for (; frame_count < kMaxUnackedFrames; ++frame_count) { - PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, - 13 /* bytes */, &frame); - OverrideRtpTimestamp(frame_count, &frame, kFramesPerSecond); - const auto result = sender()->EnqueueFrame(frame); - SimulateExecution(kLargeFrameDuration); - if (result == Sender::MAX_DURATION_IN_FLIGHT) { - break; - } - ASSERT_EQ(Sender::OK, result); - } - - // Now, simulate the Receiver ACKing the first frame, and enqueuing should - // then succeed again. - receiver()->SetCheckpointFrame(FrameId::first()); - receiver()->TransmitRtcpFeedbackPacket(); - SimulateExecution(); // RTCP transmitted to Sender. - EXPECT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); - SimulateExecution(kLargeFrameDuration); - - // However, attempting to enqueue another frame should fail again. - EncodedFrameWithBuffer one_frame_too_much; - PopulateFrameWithDefaults(sender()->GetNextFrameId(), FakeClock::now(), 0, - 13 /* bytes */, &one_frame_too_much); - OverrideRtpTimestamp(++frame_count, &one_frame_too_much, kFramesPerSecond); - EXPECT_EQ(Sender::MAX_DURATION_IN_FLIGHT, - sender()->EnqueueFrame(one_frame_too_much)); - SimulateExecution(kLargeFrameDuration); -} - -// Tests that the Sender propagates the Receiver's picture loss indicator to the -// Observer::OnPictureLost(), and via calls to NeedsKeyFrame(); but only when -// producing a key frame is absolutely necessary. -TEST_F(SenderTest, ManagesReceiverPictureLossWorkflow) { - StrictMock observer; - sender()->SetObserver(&observer); - - // Send three frames... - EncodedFrameWithBuffer frames[6]; - for (int i = 0; i < 3; ++i) { - if (i == 0) { - EXPECT_TRUE(sender()->NeedsKeyFrame()); - } else { - EXPECT_FALSE(sender()->NeedsKeyFrame()); - } - PopulateFrameWithDefaults(FrameId::first() + i, - FakeClock::now() - kCaptureDelay, 0, - 24 /* bytes */, &frames[i]); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); - SimulateExecution(kFrameDuration); - } - SimulateExecution(kTargetPlayoutDelay); - - // Simulate the Receiver ACK'ing the first three frames. - EXPECT_CALL(observer, OnFrameCanceled(FrameId::first())); - EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 1)); - EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 2)); - EXPECT_CALL(observer, OnPictureLost()).Times(0); - receiver()->SetCheckpointFrame(frames[2].frame_id); - receiver()->TransmitRtcpFeedbackPacket(); - SimulateExecution(); // RTCP transmitted to Sender. - Mock::VerifyAndClearExpectations(&observer); - - // Simulate something going wrong in the Receiver, and have it report picture - // loss to the Sender. The Sender should then propagate this to its Observer - // and return true when NeedsKeyFrame() is called. - EXPECT_CALL(observer, OnFrameCanceled(_)).Times(0); - EXPECT_CALL(observer, OnPictureLost()); - EXPECT_FALSE(sender()->NeedsKeyFrame()); - receiver()->SetPictureLossIndicator(true); - receiver()->TransmitRtcpFeedbackPacket(); - SimulateExecution(); // RTCP transmitted to Sender. - Mock::VerifyAndClearExpectations(&observer); - EXPECT_TRUE(sender()->NeedsKeyFrame()); - - // Send a non-key frame, and expect NeedsKeyFrame() still returns true. The - // Observer is not re-notified. This accounts for the case where a client's - // media encoder had frames in its processing pipeline before NeedsKeyFrame() - // began returning true. - EXPECT_CALL(observer, OnFrameCanceled(_)).Times(0); - EXPECT_CALL(observer, OnPictureLost()).Times(0); - EncodedFrameWithBuffer& nonkey_frame = frames[3]; - PopulateFrameWithDefaults(FrameId::first() + 3, - FakeClock::now() - kCaptureDelay, 0, 24 /* bytes */, - &nonkey_frame); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(nonkey_frame)); - SimulateExecution(kFrameDuration); - Mock::VerifyAndClearExpectations(&observer); - EXPECT_TRUE(sender()->NeedsKeyFrame()); - - // Now send a key frame, and expect NeedsKeyFrame() returns false. Note that - // the Receiver hasn't cleared the PLI condition, but the Sender knows more - // key frames won't be needed. - EXPECT_CALL(observer, OnFrameCanceled(_)).Times(0); - EXPECT_CALL(observer, OnPictureLost()).Times(0); - EncodedFrameWithBuffer& recovery_frame = frames[4]; - PopulateFrameWithDefaults(FrameId::first() + 4, - FakeClock::now() - kCaptureDelay, 0, 24 /* bytes */, - &recovery_frame); - recovery_frame.dependency = EncodedFrame::Dependency::kKeyFrame; - recovery_frame.referenced_frame_id = recovery_frame.frame_id; - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(recovery_frame)); - SimulateExecution(kFrameDuration); - Mock::VerifyAndClearExpectations(&observer); - EXPECT_FALSE(sender()->NeedsKeyFrame()); - - // Let's say the Receiver hasn't received the key frame yet, and it reports - // its picture loss again to the Sender. Observer::OnPictureLost() should not - // be called, and NeedsKeyFrame() should NOT return true, because the Sender - // knows the Receiver hasn't acknowledged the key frame (just sent) yet. - EXPECT_CALL(observer, OnFrameCanceled(nonkey_frame.frame_id)); - EXPECT_CALL(observer, OnPictureLost()).Times(0); - receiver()->SetCheckpointFrame(nonkey_frame.frame_id); - receiver()->SetPictureLossIndicator(true); - receiver()->TransmitRtcpFeedbackPacket(); - SimulateExecution(); // RTCP transmitted to Sender. - Mock::VerifyAndClearExpectations(&observer); - EXPECT_FALSE(sender()->NeedsKeyFrame()); - - // Now, simulate the Receiver getting the key frame, but NOT recovering. This - // should cause Observer::OnPictureLost() to be called, and cause - // NeedsKeyFrame() to return true again. - EXPECT_CALL(observer, OnFrameCanceled(recovery_frame.frame_id)); - EXPECT_CALL(observer, OnPictureLost()); - receiver()->SetCheckpointFrame(recovery_frame.frame_id); - receiver()->SetPictureLossIndicator(true); - receiver()->TransmitRtcpFeedbackPacket(); - SimulateExecution(); // RTCP transmitted to Sender. - Mock::VerifyAndClearExpectations(&observer); - EXPECT_TRUE(sender()->NeedsKeyFrame()); - - // Send another key frame, and expect NeedsKeyFrame() returns false. - EXPECT_CALL(observer, OnFrameCanceled(_)).Times(0); - EXPECT_CALL(observer, OnPictureLost()).Times(0); - EncodedFrameWithBuffer& another_recovery_frame = frames[5]; - PopulateFrameWithDefaults(FrameId::first() + 5, - FakeClock::now() - kCaptureDelay, 0, 24 /* bytes */, - &another_recovery_frame); - another_recovery_frame.dependency = EncodedFrame::Dependency::kKeyFrame; - another_recovery_frame.referenced_frame_id = another_recovery_frame.frame_id; - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(another_recovery_frame)); - SimulateExecution(kFrameDuration); - Mock::VerifyAndClearExpectations(&observer); - EXPECT_FALSE(sender()->NeedsKeyFrame()); - - // Now, simulate the Receiver recovering. It will report this to the Sender, - // and NeedsKeyFrame() will still return false. - EXPECT_CALL(observer, OnFrameCanceled(another_recovery_frame.frame_id)); - EXPECT_CALL(observer, OnPictureLost()).Times(0); - receiver()->SetCheckpointFrame(another_recovery_frame.frame_id); - receiver()->SetPictureLossIndicator(false); - receiver()->TransmitRtcpFeedbackPacket(); - SimulateExecution(); // RTCP transmitted to Sender. - Mock::VerifyAndClearExpectations(&observer); - EXPECT_FALSE(sender()->NeedsKeyFrame()); - - ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); -} - -// Tests that the Receiver should get a Sender Report just before the first RTP -// packet, and at regular intervals thereafter. The Sender Report contains the -// lip-sync information necessary for play-out timing. -TEST_F(SenderTest, ProvidesSenderReports) { - std::vector sender_reports; - Sequence packet_sequence; - EXPECT_CALL(*receiver(), OnSenderReport(_)) - .InSequence(packet_sequence) - .WillOnce( - Invoke([&](const SenderReportParser::SenderReportWithId& report) { - sender_reports.push_back(report); - })) - .RetiresOnSaturation(); - EXPECT_CALL(*receiver(), OnRtpPacket(_)).InSequence(packet_sequence); - EXPECT_CALL(*receiver(), OnSenderReport(_)) - .Times(3) - .InSequence(packet_sequence) - .WillRepeatedly( - Invoke([&](const SenderReportParser::SenderReportWithId& report) { - sender_reports.push_back(report); - })); - - EncodedFrameWithBuffer frame; - constexpr int kFrameDataSize = 250; - PopulateFrameWithDefaults(FrameId::first(), FakeClock::now(), 0, - kFrameDataSize, &frame); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); - SimulateExecution(); // Should send one Sender Report + one RTP packet. - EXPECT_EQ(size_t{1}, sender_reports.size()); - - // Have the Receiver ACK the frame to prevent retransmitting the RTP packet. - receiver()->SetCheckpointFrame(FrameId::first()); - receiver()->TransmitRtcpFeedbackPacket(); - SimulateExecution(); // RTCP transmitted to Sender. - - // Advance through three more reporting intervals. One Sender Report should be - // sent each interval, making a total of 4 reports sent. - constexpr auto kThreeReportIntervals = 3 * kRtcpReportInterval; - SimulateExecution(kThreeReportIntervals); // Three more Sender Reports. - ASSERT_EQ(size_t{4}, sender_reports.size()); - - // The first report should contain the same timestamps as the frame because - // the Clock did not advance. Also, its packet count and octet count fields - // should be zero since the report was sent before the RTP packet. - EXPECT_EQ(frame.reference_time, sender_reports.front().reference_time); - EXPECT_EQ(frame.rtp_timestamp, sender_reports.front().rtp_timestamp); - EXPECT_EQ(uint32_t{0}, sender_reports.front().send_packet_count); - EXPECT_EQ(uint32_t{0}, sender_reports.front().send_octet_count); - - // The last report should contain the timestamps extrapolated into the future - // because the Clock did move forward. Also, the packet count and octet fields - // should now be non-zero because the report was sent after the RTP packet. - EXPECT_EQ(frame.reference_time + kThreeReportIntervals, - sender_reports.back().reference_time); - EXPECT_EQ(frame.rtp_timestamp + - RtpTimeDelta::FromDuration(kThreeReportIntervals, kRtpTimebase), - sender_reports.back().rtp_timestamp); - EXPECT_EQ(uint32_t{1}, sender_reports.back().send_packet_count); - EXPECT_EQ(uint32_t{kFrameDataSize}, sender_reports.back().send_octet_count); -} - -TEST_F(SenderTest, ReferenceTimesCanBeNonMonotonic) { - // This tests that the sender is robust to encoded frames with non-monotonic - // reference times. This situation does not prevent frames from being - // transmitted in correct order; however, lip sync will suffer if reference - // times do not correspond to RTP timestamps. - EXPECT_CALL(*receiver(), OnRtpPacket(_)).Times(AtLeast(1)); - EXPECT_CALL(*receiver(), OnSenderReport(_)).Times(AtLeast(1)); - - // Send the 10 frames with non-monotonic reference times. - Clock::time_point reference_time = FakeClock::now(); - for (int i = 0; i < 10; ++i) { - EncodedFrameWithBuffer frame; - PopulateFrameWithDefaults(sender()->GetNextFrameId(), reference_time, 0, - 13 /* bytes */, &frame); - // OverrideRtpTimestamp(i, &frame, 1000 /* fps */); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frame)); - SimulateExecution(kFrameDuration); - reference_time -= microseconds(10); - } -} - -// Tests that the Sender provides Kickstart packets whenever the Receiver may -// not know about new frames. -TEST_F(SenderTest, ProvidesKickstartPacketsIfReceiverDoesNotACK) { - // Have the Receiver move the checkpoint forward only for the first frame, and - // none of the later frames. This will force the Sender to eventually send a - // Kickstart packet. - ON_CALL(*receiver(), OnFrameComplete(_)) - .WillByDefault(Invoke([&](FrameId frame_id) { - if (frame_id == FrameId::first()) { - receiver()->SetCheckpointFrame(FrameId::first()); - receiver()->TransmitRtcpFeedbackPacket(); - } - })); - - // Send three frames, paced to the media. - EncodedFrameWithBuffer frames[3]; - for (int i = 0; i < 3; ++i) { - PopulateFrameWithDefaults(FrameId::first() + i, - FakeClock::now() - kCaptureDelay, i, - 48 /* bytes */, &frames[i]); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); - SimulateExecution(kFrameDuration); - } - - // Now, do nothing for a while. Because the Receiver isn't moving the - // checkpoint forward, the Sender will have sent all the RTP packets at least - // once, and then will start sending just Kickstart packets. - SimulateExecution(kTargetPlayoutDelay); - - // Keep doing nothing for a while, and confirm the Sender is just sending the - // same Kickstart packet over and over. The Kickstart packet is supposed to be - // the last packet of the latest frame. - std::set> unique_received_packet_ids; - EXPECT_CALL(*receiver(), OnRtpPacket(_)) - .WillRepeatedly( - Invoke([&](const RtpPacketParser::ParseResult& parsed_packet) { - unique_received_packet_ids.emplace(parsed_packet.frame_id, - parsed_packet.packet_id); - })); - SimulateExecution(kTargetPlayoutDelay); - Mock::VerifyAndClearExpectations(receiver()); - EXPECT_EQ(size_t{1}, unique_received_packet_ids.size()); - EXPECT_EQ(frames[2].frame_id, unique_received_packet_ids.begin()->first); - - // Now, simulate the Receiver ACKing all the frames. - receiver()->SetCheckpointFrame(frames[2].frame_id); - receiver()->TransmitRtcpFeedbackPacket(); - SimulateExecution(); // RTCP transmitted to Sender. - - // With all the frames sent, the Sender should not be transmitting anything. - EXPECT_CALL(*receiver(), OnRtpPacket(_)).Times(0); - SimulateExecution(10 * kTargetPlayoutDelay); - - ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); -} - -// Tests that the Sender only retransmits packets specifically NACK'ed by the -// Receiver. -TEST_F(SenderTest, ResendsIndividuallyNackedPackets) { - // Populate the frame data in each frame with enough bytes to force at least - // three RTP packets per frame. - constexpr int kFrameDataSize = 3 * kMaxRtpPacketSizeForIpv6UdpOnEthernet; - - // Use a 1ms network delay in each direction to make the sequence of events - // clearer in this test. - constexpr milliseconds kOneWayNetworkDelay(1); - SetSenderToReceiverNetworkDelay(kOneWayNetworkDelay); - SetReceiverToSenderNetworkDelay(kOneWayNetworkDelay); - - // Simulate that three specific packets will be dropped by the network, one - // from each frame (about to be sent). - const std::vector dropped_packets{ - {FrameId::first(), FramePacketId{2}}, - {FrameId::first() + 1, FramePacketId{1}}, - {FrameId::first() + 2, FramePacketId{0}}, - }; - receiver()->SetIgnoreList(dropped_packets); - - // Send three frames, paced to the media. The Receiver won't completely - // receive any of these frames due to dropped packets. - EXPECT_CALL(*receiver(), OnFrameComplete(_)).Times(0); - EncodedFrameWithBuffer frames[3]; - for (int i = 0; i < 3; ++i) { - PopulateFrameWithDefaults(FrameId::first() + i, - FakeClock::now() - kCaptureDelay, i, - kFrameDataSize, &frames[i]); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); - SimulateExecution(kFrameDuration); - } - SimulateExecution(kTargetPlayoutDelay); - Mock::VerifyAndClearExpectations(receiver()); - EXPECT_EQ(3, sender()->GetInFlightFrameCount()); - - // The Receiver NACKs the three dropped packets... - receiver()->SetNacksAndAcks(dropped_packets, {}); - receiver()->TransmitRtcpFeedbackPacket(); - - // In the meantime, the network recovers (i.e., no more dropped packets)... - receiver()->SetIgnoreList({}); - - // The NACKs reach the Sender, and it acts on them by retransmitting. - SimulateExecution(kOneWayNetworkDelay); - - // As each retransmitted packet arrives at the Receiver, advance the - // checkpoint forward to notify the Sender of frames that are now completely - // received. Also, confirm that only the three specifically-NACK'ed packets - // were retransmitted. - EXPECT_CALL(*receiver(), OnFrameComplete(_)) - .Times(3) - .WillRepeatedly(InvokeWithoutArgs([&] { - if (receiver()->AutoAdvanceCheckpoint()) { - receiver()->TransmitRtcpFeedbackPacket(); - } - })); - EXPECT_CALL(*receiver(), OnRtpPacket(_)) - .Times(3) - .WillRepeatedly(Invoke([&](const RtpPacketParser::ParseResult& packet) { - EXPECT_TRUE(Contains(dropped_packets, - PacketNack{packet.frame_id, packet.packet_id})); - })); - SimulateExecution(kOneWayNetworkDelay); - Mock::VerifyAndClearExpectations(receiver()); - - // The Receiver checkpoint feedback(s) travel back to the Sender, and there - // should no longer be any frames in-flight. - SimulateExecution(kOneWayNetworkDelay); - EXPECT_EQ(0, sender()->GetInFlightFrameCount()); - - // The Sender should not be transmitting anything from now on since all frames - // are known to have been completely received. - EXPECT_CALL(*receiver(), OnRtpPacket(_)).Times(0); - SimulateExecution(10 * kTargetPlayoutDelay); - - ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); -} - -// Tests that the Sender retransmits an entire frame if the Receiver requests it -// (i.e., a full frame NACK), but does not retransmit any packets for frames -// (before or after) that have been acknowledged. -TEST_F(SenderTest, ResendsMissingFrames) { - // Populate the frame data in each frame with enough bytes to force at least - // three RTP packets per frame. - constexpr int kFrameDataSize = 3 * kMaxRtpPacketSizeForIpv6UdpOnEthernet; - - // Use a 1ms network delay in each direction to make the sequence of events - // clearer in this test. - constexpr milliseconds kOneWayNetworkDelay(1); - SetSenderToReceiverNetworkDelay(kOneWayNetworkDelay); - SetReceiverToSenderNetworkDelay(kOneWayNetworkDelay); - - // Simulate that all of the packets for the second frame will be dropped by - // the network, but only the packets for that frame. - const std::vector dropped_packets{ - {FrameId::first() + 1, kAllPacketsLost}, - }; - receiver()->SetIgnoreList(dropped_packets); - - StrictMock observer; - sender()->SetObserver(&observer); - - // The expectations below track the story and execute simulated Receiver - // responses. The Sender will have three frames enqueued by its client, and - // then... - // - // The first frame is received and the Receiver ACKs it by moving the - // checkpoint forward. - Sequence completion_sequence; - EXPECT_CALL(*receiver(), OnFrameComplete(FrameId::first())) - .InSequence(completion_sequence) - .WillOnce(InvokeWithoutArgs([&] { - receiver()->SetCheckpointFrame(FrameId::first()); - receiver()->TransmitRtcpFeedbackPacket(); - })); - // Since all of the packets for the second frame are being dropped, the third - // frame will finish next. The Receiver responds by NACKing the second frame - // and ACKing the third frame. The checkpoint does not move forward because - // the second frame has not been received yet. - // - // NETWORK CHANGE: After the third frame is received, stop dropping packets. - EXPECT_CALL(*receiver(), OnFrameComplete(FrameId::first() + 2)) - .InSequence(completion_sequence) - .WillOnce(InvokeWithoutArgs([&] { - receiver()->SetNacksAndAcks(dropped_packets, - std::vector{FrameId::first() + 2}); - receiver()->TransmitRtcpFeedbackPacket(); - receiver()->SetIgnoreList({}); - })); - // Finally, the Sender should respond to the whole-frame NACK by re-sending - // all of the packets for the second frame, and so the Receiver should - // completely receive the frame. - EXPECT_CALL(*receiver(), OnFrameComplete(FrameId::first() + 1)) - .InSequence(completion_sequence) - .WillOnce(InvokeWithoutArgs([&] { - receiver()->SetCheckpointFrame(FrameId::first() + 2); - receiver()->TransmitRtcpFeedbackPacket(); - })); - - // From the Sender's perspective, the Receiver will ACK the first frame, then - // the third frame, then the second frame. - Sequence cancel_sequence; - EXPECT_CALL(observer, OnFrameCanceled(FrameId::first())) - .InSequence(cancel_sequence); - EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 2)) - .InSequence(cancel_sequence); - EXPECT_CALL(observer, OnFrameCanceled(FrameId::first() + 1)) - .InSequence(cancel_sequence); - - // With all the expectations/sequences in-place, let 'er rip! - EncodedFrameWithBuffer frames[3]; - for (int i = 0; i < 3; ++i) { - PopulateFrameWithDefaults(FrameId::first() + i, - FakeClock::now() - kCaptureDelay, i, - kFrameDataSize, &frames[i]); - ASSERT_EQ(Sender::OK, sender()->EnqueueFrame(frames[i])); - SimulateExecution(kFrameDuration); - } - SimulateExecution(kTargetPlayoutDelay); - Mock::VerifyAndClearExpectations(receiver()); - EXPECT_EQ(0, sender()->GetInFlightFrameCount()); - - // The Sender should not be transmitting anything from now on since all frames - // are known to have been completely received. - EXPECT_CALL(*receiver(), OnRtpPacket(_)).Times(0); - SimulateExecution(10 * kTargetPlayoutDelay); - - ExpectFramesReceivedCorrectly(frames, receiver()->TakeCompleteFrames()); -} - -} // namespace -} // namespace openscreen::cast diff --git a/cast/streaming/impl/session_messenger_unittest.cc b/cast/streaming/impl/session_messenger_unittest.cc index 25565f0f8..8ce74a5b5 100644 --- a/cast/streaming/impl/session_messenger_unittest.cc +++ b/cast/streaming/impl/session_messenger_unittest.cc @@ -5,12 +5,15 @@ #include "cast/streaming/public/session_messenger.h" #include +#include +#include "cast/streaming/impl/message_constants.h" #include "cast/streaming/testing/message_pipe.h" #include "cast/streaming/testing/simple_message_port.h" #include "gtest/gtest.h" #include "platform/test/fake_clock.h" #include "platform/test/fake_task_runner.h" +#include "util/no_destructor.h" namespace openscreen::cast { @@ -23,39 +26,48 @@ constexpr char kReceiverId[] = "receiver-12345"; // Generally the messages are inlined below, with the exception of the Offer, // simply because it is massive. -Offer kExampleOffer{ - CastMode::kMirroring, - {AudioStream{Stream{0, - Stream::Type::kAudioSource, - 2, - RtpPayloadType::kAudioOpus, - 12344442, - std::chrono::milliseconds{2000}, - {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - false, - "", - 48000}, - AudioCodec::kOpus, 1400}}, - {VideoStream{Stream{1, - Stream::Type::kVideoSource, - 1, - RtpPayloadType::kVideoVp8, - 12344444, - std::chrono::milliseconds{2000}, - {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - false, - "", - 90000}, - VideoCodec::kVp8, - SimpleFraction{30, 1}, - 3000000, +const Offer& GetExampleOffer() { + static const NoDestructor kExampleOffer( + CastMode::kMirroring, + std::vector{AudioStream{ + Stream{0, + Stream::Type::kAudioSource, + 2, + RtpPayloadType::kAudioOpus, + 12344442, + std::chrono::milliseconds{2000}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + false, + std::nullopt, + 48000, "", + {kInputEventsRtpExtension}}, + AudioCodec::kOpus, 1400}}, + std::vector{VideoStream{ + Stream{1, + Stream::Type::kVideoSource, + 1, + RtpPayloadType::kVideoVp8, + 12344444, + std::chrono::milliseconds{2000}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + false, + std::nullopt, + 90000, "", - "", - {Resolution{640, 480}}, - ""}}}; + {kInputEventsRtpExtension}}, + VideoCodec::kVp8, + SimpleFraction{30, 1}, + 3000000, + "", + "", + "", + {Resolution{640, 480}}, + ""}}); + return *kExampleOffer; +} struct SessionMessageStore { public: @@ -129,9 +141,9 @@ class SessionMessengerTest : public ::testing::Test { }; TEST_F(SessionMessengerTest, RpcMessaging) { - static const std::vector kSenderMessage = {1, 2, 3, 4, 5}; - static const std::vector kSenderMessageTwo = {11, 12, 13}; - static const std::vector kReceiverResponse = {6, 7, 8, 9}; + const std::vector kSenderMessage = {1, 2, 3, 4, 5}; + const std::vector kSenderMessageTwo = {11, 12, 13}; + const std::vector kReceiverResponse = {6, 7, 8, 9}; ASSERT_TRUE( sender_messenger_ ->SendOutboundMessage(SenderMessage{SenderMessage::Type::kRpc, 123, @@ -145,13 +157,13 @@ TEST_F(SessionMessengerTest, RpcMessaging) { EXPECT_EQ(SenderMessage::Type::kRpc, message_store_.sender_messages[0].second.type); ASSERT_TRUE(message_store_.sender_messages[0].second.valid); - EXPECT_EQ(kSenderMessage, absl::get>( + EXPECT_EQ(kSenderMessage, std::get>( message_store_.sender_messages[0].second.body)); EXPECT_EQ(SenderMessage::Type::kRpc, message_store_.sender_messages[1].second.type); ASSERT_TRUE(message_store_.sender_messages[1].second.valid); EXPECT_EQ(kSenderMessageTwo, - absl::get>( + std::get>( message_store_.sender_messages[1].second.body)); message_store_.sender_messages.clear(); @@ -168,7 +180,48 @@ TEST_F(SessionMessengerTest, RpcMessaging) { message_store_.receiver_messages[0].value().type); EXPECT_TRUE(message_store_.receiver_messages[0].value().valid); EXPECT_EQ(kReceiverResponse, - absl::get>( + std::get>( + message_store_.receiver_messages[0].value().body)); +} + +TEST_F(SessionMessengerTest, InputMessaging) { + const std::vector kSenderMessage = {1, 2, 3, 4, 5}; + const std::vector kSenderMessageTwo = {11, 12, 13}; + const std::vector kReceiverResponse = {6, 7, 8, 9}; + + sender_messenger_->SetHandler(ReceiverMessage::Type::kInput, + message_store_.GetReplyCallback()); + receiver_messenger_->SetHandler(SenderMessage::Type::kInput, + message_store_.GetRequestCallback()); + + ASSERT_TRUE(sender_messenger_->SendInputMessage(kSenderMessage).ok()); + ASSERT_TRUE(sender_messenger_->SendInputMessage(kSenderMessageTwo).ok()); + + ASSERT_EQ(2u, message_store_.sender_messages.size()); + ASSERT_TRUE(message_store_.receiver_messages.empty()); + EXPECT_EQ(SenderMessage::Type::kInput, + message_store_.sender_messages[0].second.type); + ASSERT_TRUE(message_store_.sender_messages[0].second.valid); + EXPECT_EQ(kSenderMessage, std::get>( + message_store_.sender_messages[0].second.body)); + EXPECT_EQ(SenderMessage::Type::kInput, + message_store_.sender_messages[1].second.type); + ASSERT_TRUE(message_store_.sender_messages[1].second.valid); + EXPECT_EQ(kSenderMessageTwo, + std::get>( + message_store_.sender_messages[1].second.body)); + + message_store_.sender_messages.clear(); + ASSERT_TRUE( + receiver_messenger_->SendInputMessage(kSenderId, kReceiverResponse).ok()); + + ASSERT_TRUE(message_store_.sender_messages.empty()); + ASSERT_EQ(1u, message_store_.receiver_messages.size()); + EXPECT_EQ(ReceiverMessage::Type::kInput, + message_store_.receiver_messages[0].value().type); + EXPECT_TRUE(message_store_.receiver_messages[0].value().valid); + EXPECT_EQ(kReceiverResponse, + std::get>( message_store_.receiver_messages[0].value().body)); } @@ -205,7 +258,7 @@ TEST_F(SessionMessengerTest, CapabilitiesMessaging) { message_store_.receiver_messages[0].value().type); EXPECT_TRUE(message_store_.receiver_messages[0].value().valid); - const auto& capability = absl::get( + const auto& capability = std::get( message_store_.receiver_messages[0].value().body); EXPECT_EQ(47, capability.remoting_version); EXPECT_THAT(capability.media_capabilities, @@ -213,12 +266,13 @@ TEST_F(SessionMessengerTest, CapabilitiesMessaging) { } TEST_F(SessionMessengerTest, OfferAnswerMessaging) { - ASSERT_TRUE(sender_messenger_ - ->SendRequest(SenderMessage{SenderMessage::Type::kOffer, 42, - true /* valid */, kExampleOffer}, - ReceiverMessage::Type::kAnswer, - message_store_.GetReplyCallback()) - .ok()); + ASSERT_TRUE( + sender_messenger_ + ->SendRequest(SenderMessage{SenderMessage::Type::kOffer, 42, + true /* valid */, GetExampleOffer()}, + ReceiverMessage::Type::kAnswer, + message_store_.GetReplyCallback()) + .ok()); ASSERT_EQ(1u, message_store_.sender_messages.size()); ASSERT_TRUE(message_store_.receiver_messages.empty()); @@ -252,7 +306,7 @@ TEST_F(SessionMessengerTest, OfferAnswerMessaging) { EXPECT_TRUE(message_store_.receiver_messages[0].value().valid); const auto& answer = - absl::get(message_store_.receiver_messages[0].value().body); + std::get(message_store_.receiver_messages[0].value().body); EXPECT_EQ(1234, answer.udp_port); EXPECT_THAT(answer.send_indexes, ElementsAre(0, 1)); @@ -260,12 +314,13 @@ TEST_F(SessionMessengerTest, OfferAnswerMessaging) { } TEST_F(SessionMessengerTest, OfferAndReceiverError) { - ASSERT_TRUE(sender_messenger_ - ->SendRequest(SenderMessage{SenderMessage::Type::kOffer, 42, - true /* valid */, kExampleOffer}, - ReceiverMessage::Type::kAnswer, - message_store_.GetReplyCallback()) - .ok()); + ASSERT_TRUE( + sender_messenger_ + ->SendRequest(SenderMessage{SenderMessage::Type::kOffer, 42, + true /* valid */, GetExampleOffer()}, + ReceiverMessage::Type::kAnswer, + message_store_.GetReplyCallback()) + .ok()); ASSERT_EQ(1u, message_store_.sender_messages.size()); ASSERT_TRUE(message_store_.receiver_messages.empty()); @@ -289,8 +344,8 @@ TEST_F(SessionMessengerTest, OfferAndReceiverError) { message_store_.receiver_messages[0].value().type); EXPECT_FALSE(message_store_.receiver_messages[0].value().valid); - const auto& error = absl::get( - message_store_.receiver_messages[0].value().body); + const auto& error = + std::get(message_store_.receiver_messages[0].value().body); EXPECT_EQ(123, error.code); EXPECT_EQ("Something real bad happened", error.description); } @@ -305,12 +360,13 @@ TEST_F(SessionMessengerTest, UnknownSenderMessageTypesDontGetSent) { } TEST_F(SessionMessengerTest, UnknownReceiverMessageTypesDontGetSent) { - ASSERT_TRUE(sender_messenger_ - ->SendRequest(SenderMessage{SenderMessage::Type::kOffer, 42, - true /* valid */, kExampleOffer}, - ReceiverMessage::Type::kAnswer, - message_store_.GetReplyCallback()) - .ok()); + ASSERT_TRUE( + sender_messenger_ + ->SendRequest(SenderMessage{SenderMessage::Type::kOffer, 42, + true /* valid */, GetExampleOffer()}, + ReceiverMessage::Type::kAnswer, + message_store_.GetReplyCallback()) + .ok()); EXPECT_DEATH_IF_SUPPORTED( receiver_messenger_ @@ -334,12 +390,13 @@ TEST_F(SessionMessengerTest, SenderHandlesUnknownMessageType) { // test elsewhere that messages with the wrong sequence number are ignored, // here if the type is unknown but the message contains a valid sequence // number we just treat it as a bad response/same as a timeout. - ASSERT_TRUE(sender_messenger_ - ->SendRequest(SenderMessage{SenderMessage::Type::kOffer, 42, - true /* valid */, kExampleOffer}, - ReceiverMessage::Type::kAnswer, - message_store_.GetReplyCallback()) - .ok()); + ASSERT_TRUE( + sender_messenger_ + ->SendRequest(SenderMessage{SenderMessage::Type::kOffer, 42, + true /* valid */, GetExampleOffer()}, + ReceiverMessage::Type::kAnswer, + message_store_.GetReplyCallback()) + .ok()); sender_pipe_end().ReceiveMessage(kCastWebrtcNamespace, R"({ "type": "ANSWER_VERSION_2", "seqNum": 42 @@ -602,4 +659,98 @@ TEST_F(SessionMessengerTest, UnknownNamespaceMessagesGetDropped) { ASSERT_TRUE(message_store_.receiver_messages.empty()); } +TEST_F(SessionMessengerTest, CustomNamespaceMessaging) { + constexpr char kCustomNamespace[] = "urn:x-cast:com.google.custom"; + constexpr char kMessage[] = "Hello from the Custom Sender!"; + std::string received_source_id; + std::string received_namespace; + std::string received_message; + + receiver_messenger_->SetCustomMessageHandler( + kCustomNamespace, + [&](const std::string& source_id, const std::string& message_namespace, + const std::string& message) { + received_source_id = source_id; + received_namespace = message_namespace; + received_message = message; + }); + + receiver_pipe_end().ReceiveMessage(kSenderId, kCustomNamespace, kMessage); + + EXPECT_EQ(kSenderId, received_source_id); + EXPECT_EQ(kCustomNamespace, received_namespace); + EXPECT_EQ(kMessage, received_message); + + constexpr char kReply[] = "Hello from the Custom Receiver!"; + ASSERT_TRUE( + receiver_messenger_->SendMessage(kSenderId, kCustomNamespace, kReply) + .ok()); +} + +TEST_F(SessionMessengerTest, MultipleCustomNamespaceMessaging) { + constexpr char kNamespace1[] = "urn:x-cast:com.google.custom.1"; + constexpr char kNamespace2[] = "urn:x-cast:com.google.custom.2"; + constexpr char kMessage1[] = "Message for namespace 1"; + constexpr char kMessage2[] = "Message for namespace 2"; + + std::string received_ns; + std::string received_msg; + + auto handler1 = [&](const std::string& source_id, + const std::string& message_namespace, + const std::string& message) { + received_ns = message_namespace; + received_msg = message; + }; + + auto handler2 = [&](const std::string& source_id, + const std::string& message_namespace, + const std::string& message) { + received_ns = message_namespace; + received_msg = message; + }; + + receiver_messenger_->SetCustomMessageHandler(kNamespace1, handler1); + receiver_messenger_->SetCustomMessageHandler(kNamespace2, handler2); + + // Send to namespace 1 + receiver_pipe_end().ReceiveMessage(kSenderId, kNamespace1, kMessage1); + EXPECT_EQ(kNamespace1, received_ns); + EXPECT_EQ(kMessage1, received_msg); + + // Send to namespace 2 + receiver_pipe_end().ReceiveMessage(kSenderId, kNamespace2, kMessage2); + EXPECT_EQ(kNamespace2, received_ns); + EXPECT_EQ(kMessage2, received_msg); + + // Attempt to update handler for namespace 1. This should be ignored. + constexpr char kMessage1Updated[] = "Updated message for namespace 1"; + receiver_messenger_->SetCustomMessageHandler( + kNamespace1, + [&](const std::string& source_id, const std::string& message_namespace, + const std::string& message) { + received_ns = "updated-" + message_namespace; + received_msg = message; + }); + + receiver_pipe_end().ReceiveMessage(kSenderId, kNamespace1, kMessage1Updated); + // Verify the original handler still handles the message. + EXPECT_EQ(kNamespace1, received_ns); + EXPECT_EQ(kMessage1Updated, received_msg); + + // Remove handler for namespace 2 + received_ns = ""; + received_msg = ""; + receiver_messenger_->SetCustomMessageHandler(kNamespace2, nullptr); + receiver_pipe_end().ReceiveMessage(kSenderId, kNamespace2, kMessage2); + EXPECT_EQ("", received_ns); + EXPECT_EQ("", received_msg); + + // Verify SendMessage for multiple namespaces + EXPECT_TRUE( + receiver_messenger_->SendMessage(kSenderId, kNamespace1, kMessage1).ok()); + EXPECT_TRUE( + receiver_messenger_->SendMessage(kSenderId, kNamespace2, kMessage2).ok()); +} + } // namespace openscreen::cast diff --git a/cast/streaming/impl/statistics_analyzer.cc b/cast/streaming/impl/statistics_analyzer.cc index 58f32978a..bd2fc9fe9 100644 --- a/cast/streaming/impl/statistics_analyzer.cc +++ b/cast/streaming/impl/statistics_analyzer.cc @@ -6,11 +6,14 @@ #include +#include "cast/streaming/impl/statistics_common.h" #include "platform/base/trivial_clock_traits.h" #include "util/chrono_helpers.h" namespace openscreen::cast { +using openscreen::clock_operators::operator<<; + namespace { constexpr Clock::duration kAnalysisInterval = std::chrono::milliseconds(500); @@ -25,11 +28,11 @@ double InMilliseconds(Clock::duration duration) { return static_cast(to_milliseconds(duration).count()); } -bool IsReceiverEvent(StatisticsEventType event) { - return event == StatisticsEventType::kFrameAckSent || - event == StatisticsEventType::kFrameDecoded || - event == StatisticsEventType::kFramePlayedOut || - event == StatisticsEventType::kPacketReceived; +bool IsReceiverEvent(StatisticsEvent::Type event) { + return event == StatisticsEvent::Type::kFrameAckSent || + event == StatisticsEvent::Type::kFrameDecoded || + event == StatisticsEvent::Type::kFramePlayedOut || + event == StatisticsEvent::Type::kPacketReceived; } } // namespace @@ -81,10 +84,10 @@ void StatisticsAnalyzer::SendStatistics() { const Clock::time_point end_time = now_(); stats_client_->OnStatisticsUpdated(SenderStats{ .audio_statistics = - ConstructStatisticsList(end_time, StatisticsEventMediaType::kAudio), + ConstructStatisticsList(end_time, StatisticsEvent::MediaType::kAudio), .audio_histograms = histograms_.audio, .video_statistics = - ConstructStatisticsList(end_time, StatisticsEventMediaType::kVideo), + ConstructStatisticsList(end_time, StatisticsEvent::MediaType::kVideo), .video_histograms = histograms_.video}); } @@ -131,10 +134,11 @@ void StatisticsAnalyzer::ProcessPacketEvents( } RecordEventTimes(packet_event); - if (packet_event.type == StatisticsEventType::kPacketSentToNetwork || - packet_event.type == StatisticsEventType::kPacketReceived) { + if (packet_event.type == StatisticsEvent::Type::kPacketSentToNetwork || + packet_event.type == StatisticsEvent::Type::kPacketReceived) { RecordPacketLatencies(packet_event); - } else if (packet_event.type == StatisticsEventType::kPacketRetransmitted) { + } else if (packet_event.type == + StatisticsEvent::Type::kPacketRetransmitted) { // We only measure network latency for packets that are not retransmitted. ErasePacketInfo(packet_event); } @@ -163,11 +167,11 @@ void StatisticsAnalyzer::RecordFrameLatencies(const FrameEvent& frame_event) { } switch (frame_event.type) { - case StatisticsEventType::kFrameCaptureBegin: + case StatisticsEvent::Type::kFrameCaptureBegin: it->second.capture_begin_time = frame_event.timestamp; break; - case StatisticsEventType::kFrameCaptureEnd: { + case StatisticsEvent::Type::kFrameCaptureEnd: { it->second.capture_end_time = frame_event.timestamp; if (it->second.capture_begin_time != Clock::time_point::min()) { const Clock::duration capture_latency = @@ -179,7 +183,7 @@ void StatisticsAnalyzer::RecordFrameLatencies(const FrameEvent& frame_event) { } } break; - case StatisticsEventType::kFrameEncoded: { + case StatisticsEvent::Type::kFrameEncoded: { it->second.encode_end_time = frame_event.timestamp; if (it->second.capture_end_time != Clock::time_point::min()) { const Clock::duration encode_latency = @@ -193,7 +197,7 @@ void StatisticsAnalyzer::RecordFrameLatencies(const FrameEvent& frame_event) { // Frame latency is the time from when the frame is encoded until the // receiver ack for the frame is sent. - case StatisticsEventType::kFrameAckSent: { + case StatisticsEvent::Type::kFrameAckSent: { const auto adjusted_timestamp = ToSenderTimestamp(frame_event.timestamp, frame_event.media_type); if (!adjusted_timestamp) { @@ -208,7 +212,7 @@ void StatisticsAnalyzer::RecordFrameLatencies(const FrameEvent& frame_event) { } } break; - case StatisticsEventType::kFramePlayedOut: { + case StatisticsEvent::Type::kFramePlayedOut: { const auto adjusted_timestamp = ToSenderTimestamp(frame_event.timestamp, frame_event.media_type); if (!adjusted_timestamp) { @@ -243,7 +247,7 @@ void StatisticsAnalyzer::RecordPacketLatencies( // Queueing latency is the time from when a frame is encoded to when the // packet is first sent. - if (packet_event.type == StatisticsEventType::kPacketSentToNetwork) { + if (packet_event.type == StatisticsEvent::Type::kPacketSentToNetwork) { const auto it = frame_infos.find(packet_event.rtp_timestamp); // We have an encode end time for a frame associated with this packet. @@ -272,15 +276,16 @@ void StatisticsAnalyzer::RecordPacketLatencies( } } else { // We know when this packet was sent, and when it arrived. PacketInfo value = it->second; - StatisticsEventType recorded_type = value.type; + StatisticsEvent::Type recorded_type = value.type; Clock::time_point packet_sent_time; Clock::time_point packet_received_time; - if (recorded_type == StatisticsEventType::kPacketSentToNetwork && - packet_event.type == StatisticsEventType::kPacketReceived) { + if (recorded_type == StatisticsEvent::Type::kPacketSentToNetwork && + packet_event.type == StatisticsEvent::Type::kPacketReceived) { packet_sent_time = value.timestamp; packet_received_time = packet_event.timestamp; - } else if (recorded_type == StatisticsEventType::kPacketReceived && - packet_event.type == StatisticsEventType::kPacketSentToNetwork) { + } else if (recorded_type == StatisticsEvent::Type::kPacketReceived && + packet_event.type == + StatisticsEvent::Type::kPacketSentToNetwork) { packet_sent_time = packet_event.timestamp; packet_received_time = value.timestamp; } else { @@ -298,15 +303,11 @@ void StatisticsAnalyzer::RecordPacketLatencies( } packet_received_time -= *receiver_offset; - // Network latency is the time between when a packet is sent and when it - // is received. - const Clock::duration network_latency = - packet_received_time - packet_sent_time; - RecordEstimatedNetworkLatency(network_latency); - AddToLatencyAggregrate(StatisticType::kAvgNetworkLatencyMs, network_latency, + const auto latency = packet_received_time - packet_sent_time; + AddToLatencyAggregrate(StatisticType::kAvgNetworkLatencyMs, latency, packet_event.media_type); AddToHistogram(HistogramType::kNetworkLatencyMs, packet_event.media_type, - InMilliseconds(network_latency)); + InMilliseconds(latency)); // Packet latency is the time from when a frame is encoded until when the // packet is received. @@ -327,10 +328,13 @@ void StatisticsAnalyzer::RecordEventTimes(const StatisticsEvent& event) { Clock::time_point sender_timestamp = event.timestamp; if (IsReceiverEvent(event.type)) { - const Clock::time_point estimated_sent_time = - event.received_timestamp - estimated_network_latency_; - session_stats.last_response_received_time = std::max( - session_stats.last_response_received_time, estimated_sent_time); + const auto latency = offset_estimator_->GetEstimatedLatency(); + if (latency) { + const Clock::time_point estimated_sent_time = + event.received_timestamp - *latency; + session_stats.last_response_received_time = std::max( + session_stats.last_response_received_time, estimated_sent_time); + } const auto result = ToSenderTimestamp(event.timestamp, event.media_type); if (!result) { @@ -356,7 +360,7 @@ void StatisticsAnalyzer::ErasePacketInfo(const PacketEvent& packet_event) { void StatisticsAnalyzer::AddToLatencyAggregrate( StatisticType latency_stat, Clock::duration latency_delta, - StatisticsEventMediaType media_type) { + StatisticsEvent::MediaType media_type) { LatencyStatsMap& latency_stats = latency_stats_.Get(media_type); auto it = latency_stats.find(latency_stat); @@ -371,23 +375,26 @@ void StatisticsAnalyzer::AddToLatencyAggregrate( } void StatisticsAnalyzer::AddToHistogram(HistogramType histogram, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, int64_t sample) { histograms_.Get(media_type)[static_cast(histogram)].Add(sample); } SenderStats::StatisticsList StatisticsAnalyzer::ConstructStatisticsList( Clock::time_point end_time, - StatisticsEventMediaType media_type) { + StatisticsEvent::MediaType media_type) { SenderStats::StatisticsList stats_list; - // TODO(b/298205111): Support kNumFramesDroppedByEncoder stat. - PopulateFrameCountStat(StatisticsEventType::kFrameCaptureEnd, + PopulateFrameCountStat(StatisticsEvent::Type::kFrameDroppedByEncoder, + StatisticType::kNumFramesDroppedByEncoder, media_type, + stats_list); + + PopulateFrameCountStat(StatisticsEvent::Type::kFrameCaptureEnd, StatisticType::kNumFramesCaptured, media_type, stats_list); // kEnqueueFps - PopulateFpsStat(StatisticsEventType::kFrameEncoded, + PopulateFpsStat(StatisticsEvent::Type::kFrameEncoded, StatisticType::kEnqueueFps, media_type, end_time, stats_list); constexpr StatisticType kSupportedLatencyStats[] = { @@ -401,22 +408,22 @@ SenderStats::StatisticsList StatisticsAnalyzer::ConstructStatisticsList( } // kEncodeRateKbps - PopulateFrameBitrateStat(StatisticsEventType::kFrameEncoded, + PopulateFrameBitrateStat(StatisticsEvent::Type::kFrameEncoded, StatisticType::kEncodeRateKbps, media_type, end_time, stats_list); // kPacketTransmissionRateKbps - PopulatePacketBitrateStat(StatisticsEventType::kPacketSentToNetwork, + PopulatePacketBitrateStat(StatisticsEvent::Type::kPacketSentToNetwork, StatisticType::kPacketTransmissionRateKbps, media_type, end_time, stats_list); // kNumPacketsSent - PopulatePacketCountStat(StatisticsEventType::kPacketSentToNetwork, + PopulatePacketCountStat(StatisticsEvent::Type::kPacketSentToNetwork, StatisticType::kNumPacketsSent, media_type, stats_list); // kNumPacketsReceived - PopulatePacketCountStat(StatisticsEventType::kPacketReceived, + PopulatePacketCountStat(StatisticsEvent::Type::kPacketReceived, StatisticType::kNumPacketsReceived, media_type, stats_list); @@ -430,9 +437,9 @@ SenderStats::StatisticsList StatisticsAnalyzer::ConstructStatisticsList( } void StatisticsAnalyzer::PopulatePacketCountStat( - StatisticsEventType event, + StatisticsEvent::Type event, StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, SenderStats::StatisticsList& stats_list) { PacketStatsMap& stats_map = packet_stats_.Get(media_type); @@ -443,9 +450,9 @@ void StatisticsAnalyzer::PopulatePacketCountStat( } void StatisticsAnalyzer::PopulateFrameCountStat( - StatisticsEventType event, + StatisticsEvent::Type event, StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, SenderStats::StatisticsList& stats_list) { FrameStatsMap& stats_map = frame_stats_.Get(media_type); @@ -456,9 +463,9 @@ void StatisticsAnalyzer::PopulateFrameCountStat( } void StatisticsAnalyzer::PopulateFpsStat( - StatisticsEventType event, + StatisticsEvent::Type event, StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, Clock::time_point end_time, SenderStats::StatisticsList& stats_list) { FrameStatsMap& stats_map = frame_stats_.Get(media_type); @@ -476,7 +483,7 @@ void StatisticsAnalyzer::PopulateFpsStat( void StatisticsAnalyzer::PopulateAvgLatencyStat( StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, SenderStats::StatisticsList& stats_list ) { @@ -491,9 +498,9 @@ void StatisticsAnalyzer::PopulateAvgLatencyStat( } void StatisticsAnalyzer::PopulateFrameBitrateStat( - StatisticsEventType event, + StatisticsEvent::Type event, StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, Clock::time_point end_time, SenderStats::StatisticsList& stats_list) { FrameStatsMap& stats_map = frame_stats_.Get(media_type); @@ -509,9 +516,9 @@ void StatisticsAnalyzer::PopulateFrameBitrateStat( } void StatisticsAnalyzer::PopulatePacketBitrateStat( - StatisticsEventType event, + StatisticsEvent::Type event, StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, Clock::time_point end_time, SenderStats::StatisticsList& stats_list) { PacketStatsMap& stats_map = packet_stats_.Get(media_type); @@ -527,7 +534,7 @@ void StatisticsAnalyzer::PopulatePacketBitrateStat( } void StatisticsAnalyzer::PopulateSessionStats( - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, Clock::time_point end_time, SenderStats::StatisticsList& stats_list) { SessionStats& session_stats = session_stats_.Get(media_type); @@ -554,28 +561,13 @@ void StatisticsAnalyzer::PopulateSessionStats( std::optional StatisticsAnalyzer::ToSenderTimestamp( Clock::time_point receiver_timestamp, - StatisticsEventMediaType media_type) const { + StatisticsEvent::MediaType media_type) const { const std::optional receiver_offset = offset_estimator_->GetEstimatedOffset(); if (!receiver_offset) { return {}; } - return receiver_timestamp + estimated_network_latency_ - *receiver_offset; -} - -void StatisticsAnalyzer::RecordEstimatedNetworkLatency( - Clock::duration latency) { - if (estimated_network_latency_ == Clock::duration{}) { - estimated_network_latency_ = latency; - return; - } - - // We use an exponential moving average for recording the network latency. - // NOTE: value chosen experimentally to perform some smoothing and represent - // the past few seconds of data. - constexpr double kWeight = 2.0 / 301.0; - estimated_network_latency_ = to_microseconds( - latency * kWeight + estimated_network_latency_ * (1.0 - kWeight)); + return receiver_timestamp - *receiver_offset; } } // namespace openscreen::cast diff --git a/cast/streaming/impl/statistics_analyzer.h b/cast/streaming/impl/statistics_analyzer.h index 851403fe7..036cd20d2 100644 --- a/cast/streaming/impl/statistics_analyzer.h +++ b/cast/streaming/impl/statistics_analyzer.h @@ -59,7 +59,7 @@ class StatisticsAnalyzer { struct PacketInfo { Clock::time_point timestamp; - StatisticsEventType type; + StatisticsEvent::Type type; }; struct SessionStats { @@ -75,20 +75,20 @@ class StatisticsAnalyzer { T audio; T video; - const T& Get(StatisticsEventMediaType media_type) const { - if (media_type == StatisticsEventMediaType::kAudio) { + const T& Get(StatisticsEvent::MediaType media_type) const { + if (media_type == StatisticsEvent::MediaType::kAudio) { return audio; } - OSP_CHECK(media_type == StatisticsEventMediaType::kVideo); + OSP_CHECK(media_type == StatisticsEvent::MediaType::kVideo); return video; } - T& Get(StatisticsEventMediaType media_type) { + T& Get(StatisticsEvent::MediaType media_type) { return const_cast(const_cast(this)->Get(media_type)); } }; - using FrameStatsMap = std::map; - using PacketStatsMap = std::map; + using FrameStatsMap = std::map; + using PacketStatsMap = std::map; using LatencyStatsMap = std::map; using FrameInfoMap = std::map; @@ -116,50 +116,50 @@ class StatisticsAnalyzer { void ErasePacketInfo(const PacketEvent& packet_event); void AddToLatencyAggregrate(StatisticType latency_stat, Clock::duration latency_delta, - StatisticsEventMediaType media_type); + StatisticsEvent::MediaType media_type); void AddToHistogram(HistogramType histogram, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, int64_t sample); // Creates a stats list, and populates the entries based on stored stats info // / aggregates for each stat field. SenderStats::StatisticsList ConstructStatisticsList( Clock::time_point end_time, - StatisticsEventMediaType media_type); + StatisticsEvent::MediaType media_type); - void PopulatePacketCountStat(StatisticsEventType event, + void PopulatePacketCountStat(StatisticsEvent::Type event, StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, SenderStats::StatisticsList& stats_list); - void PopulateFrameCountStat(StatisticsEventType event, + void PopulateFrameCountStat(StatisticsEvent::Type event, StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, SenderStats::StatisticsList& stats_list); - void PopulateFpsStat(StatisticsEventType event, + void PopulateFpsStat(StatisticsEvent::Type event, StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, Clock::time_point end_time, SenderStats::StatisticsList& stats_list); void PopulateAvgLatencyStat(StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, SenderStats::StatisticsList& stats_list); - void PopulateFrameBitrateStat(StatisticsEventType event, + void PopulateFrameBitrateStat(StatisticsEvent::Type event, StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, Clock::time_point end_time, SenderStats::StatisticsList& stats_list); - void PopulatePacketBitrateStat(StatisticsEventType event, + void PopulatePacketBitrateStat(StatisticsEvent::Type event, StatisticType stat, - StatisticsEventMediaType media_type, + StatisticsEvent::MediaType media_type, Clock::time_point end_time, SenderStats::StatisticsList& stats_list); - void PopulateSessionStats(StatisticsEventMediaType media_type, + void PopulateSessionStats(StatisticsEvent::MediaType media_type, Clock::time_point end_time, SenderStats::StatisticsList& stats_list); @@ -167,11 +167,7 @@ class StatisticsAnalyzer { // the sender-side version of this receiver timestamp, if possible. std::optional ToSenderTimestamp( Clock::time_point receiver_timestamp, - StatisticsEventMediaType media_type) const; - - // Records the network latency estimate, which is then weighted and used as - // part of the moving network latency estimate. - void RecordEstimatedNetworkLatency(Clock::duration latency); + StatisticsEvent::MediaType media_type) const; // The statistics client to which we report analyzed statistics. SenderStatsClient* const stats_client_; @@ -187,12 +183,6 @@ class StatisticsAnalyzer { Alarm alarm_; Clock::time_point start_time_; - // Keep track of the currently estimated network latency. - // - // NOTE: though we currently record the average network latency separately for - // audio and video, they use the same network so the value should be the same. - Clock::duration estimated_network_latency_{}; - // Maps of frame / packet infos used for stats that rely on seeing multiple // events. For example, network latency is the calculated time difference // between went a packet is sent, and when it is received. diff --git a/cast/streaming/impl/statistics_analyzer_unittest.cc b/cast/streaming/impl/statistics_analyzer_unittest.cc index cb5ee9ac7..b6c7e905b 100644 --- a/cast/streaming/impl/statistics_analyzer_unittest.cc +++ b/cast/streaming/impl/statistics_analyzer_unittest.cc @@ -18,7 +18,6 @@ using testing::_; using testing::AtLeast; -using testing::Invoke; using testing::InvokeWithoutArgs; using testing::Mock; using testing::NiceMock; @@ -44,8 +43,8 @@ constexpr int kDefaultSizeBytes = 10; constexpr int kDefaultStatIntervalMs = 5; constexpr FrameEvent kDefaultFrameEvent(FrameId::first(), - StatisticsEventType::kFrameEncoded, - StatisticsEventMediaType::kVideo, + StatisticsEvent::Type::kFrameEncoded, + StatisticsEvent::MediaType::kVideo, RtpTimeTicks(), kDefaultSizeBytes, Clock::time_point::min(), @@ -58,8 +57,8 @@ constexpr FrameEvent kDefaultFrameEvent(FrameId::first(), constexpr PacketEvent kDefaultPacketEvent( FrameId::first(), - StatisticsEventType::kPacketSentToNetwork, - StatisticsEventMediaType::kVideo, + StatisticsEvent::Type::kPacketSentToNetwork, + StatisticsEvent::MediaType::kVideo, RtpTimeTicks(), kDefaultSizeBytes, Clock::time_point::min(), @@ -109,6 +108,10 @@ class FakeClockOffsetEstimator : public ClockOffsetEstimator { GetEstimatedOffset, (), (const, override)); + MOCK_METHOD(std::optional, + GetEstimatedLatency, + (), + (const, override)); }; } // namespace @@ -122,13 +125,14 @@ class StatisticsAnalyzerTest : public ::testing::Test { // In general, use an estimator that doesn't have an offset. // TODO(issuetracker.google.com/298085631): add test coverage for the // estimator usage in this class. - auto fake_estimator = + auto fake_estimator_unique_ptr = std::make_unique>(); - ON_CALL(*fake_estimator, GetEstimatedOffset()) + fake_estimator_ = fake_estimator_unique_ptr.get(); + ON_CALL(*fake_estimator_, GetEstimatedOffset()) .WillByDefault(Return(Clock::duration{})); analyzer_ = std::make_unique( &stats_client_, fake_clock_.now, fake_task_runner_, - std::move(fake_estimator)); + std::move(fake_estimator_unique_ptr)); collector_ = analyzer_->statistics_collector(); } @@ -166,7 +170,7 @@ TEST_F(StatisticsAnalyzerTest, FrameEncoded) { analyzer_->ScheduleAnalysis(); Clock::time_point first_event_time = fake_clock_.now(); - Clock::time_point last_event_time; + Clock::time_point last_event_time = Clock::time_point::min(); RtpTimeTicks rtp_timestamp; for (int i = 0; i < kDefaultNumEvents; i++) { @@ -178,7 +182,7 @@ TEST_F(StatisticsAnalyzerTest, FrameEncoded) { } EXPECT_CALL(stats_client_, OnStatisticsUpdated(_)) - .WillOnce(Invoke([&](const SenderStats& stats) { + .WillOnce([&](const SenderStats& stats) { double expected_fps = kDefaultNumEvents / (kDefaultStatsAnalysisIntervalMs / 1000.0); ExpectStatEq(stats.video_statistics, StatisticType::kEnqueueFps, @@ -198,7 +202,7 @@ TEST_F(StatisticsAnalyzerTest, FrameEncoded) { stats.video_statistics, StatisticType::kLastEventTimeMs, static_cast( to_milliseconds(last_event_time.time_since_epoch()).count())); - })); + }); fake_clock_.Advance( milliseconds(kDefaultStatsAnalysisIntervalMs - @@ -219,7 +223,7 @@ TEST_F(StatisticsAnalyzerTest, FrameEncodedAndAckSent) { total_frame_latency += random_latency; FrameEvent event2 = MakeFrameEvent(i, rtp_timestamp); - event2.type = StatisticsEventType::kFrameAckSent; + event2.type = StatisticsEvent::Type::kFrameAckSent; event2.timestamp += random_latency; event2.received_timestamp += random_latency * 2; @@ -230,13 +234,13 @@ TEST_F(StatisticsAnalyzerTest, FrameEncodedAndAckSent) { } EXPECT_CALL(stats_client_, OnStatisticsUpdated(_)) - .WillOnce(Invoke([&](const SenderStats& stats) { + .WillOnce([&](const SenderStats& stats) { double expected_avg_frame_latency = static_cast(to_milliseconds(total_frame_latency).count()) / kDefaultNumEvents; ExpectStatEq(stats.video_statistics, StatisticType::kAvgFrameLatencyMs, expected_avg_frame_latency); - })); + }); fake_clock_.Advance( milliseconds(kDefaultStatsAnalysisIntervalMs - @@ -259,7 +263,7 @@ TEST_F(StatisticsAnalyzerTest, FramePlayedOut) { auto delay_delta = milliseconds(60 - (20 * (i % 5))); FrameEvent event2 = MakeFrameEvent(i, rtp_timestamp); - event2.type = StatisticsEventType::kFramePlayedOut; + event2.type = StatisticsEvent::Type::kFramePlayedOut; event2.timestamp += random_latency; event2.received_timestamp += random_latency * 2; event2.delay_delta = delay_delta; @@ -275,7 +279,7 @@ TEST_F(StatisticsAnalyzerTest, FramePlayedOut) { } EXPECT_CALL(stats_client_, OnStatisticsUpdated(_)) - .WillOnce(Invoke([&](const SenderStats& stats) { + .WillOnce([&](const SenderStats& stats) { ExpectStatEq(stats.video_statistics, StatisticType::kNumLateFrames, total_late_frames); @@ -287,7 +291,33 @@ TEST_F(StatisticsAnalyzerTest, FramePlayedOut) { /* 80-99 */ 0}; ExpectHistoBuckets(stats.video_histograms, HistogramType::kFrameLatenessMs, kExpectedBuckets); - })); + }); + + fake_clock_.Advance( + milliseconds(kDefaultStatsAnalysisIntervalMs - + (kDefaultStatIntervalMs * kDefaultNumEvents))); +} + +TEST_F(StatisticsAnalyzerTest, FrameDroppedByEncoder) { + analyzer_->ScheduleAnalysis(); + + RtpTimeTicks rtp_timestamp; + + for (int i = 0; i < kDefaultNumEvents; i++) { + FrameEvent event = MakeFrameEvent(i, rtp_timestamp); + event.type = StatisticsEvent::Type::kFrameDroppedByEncoder; + collector_->CollectFrameEvent(event); + + fake_clock_.Advance(milliseconds(kDefaultStatIntervalMs)); + rtp_timestamp += RtpTimeDelta::FromTicks(90); + } + + EXPECT_CALL(stats_client_, OnStatisticsUpdated(_)) + .WillOnce([&](const SenderStats& stats) { + ExpectStatEq(stats.video_statistics, + StatisticType::kNumFramesDroppedByEncoder, + kDefaultNumEvents); + }); fake_clock_.Advance( milliseconds(kDefaultStatsAnalysisIntervalMs - @@ -295,10 +325,12 @@ TEST_F(StatisticsAnalyzerTest, FramePlayedOut) { } TEST_F(StatisticsAnalyzerTest, AllFrameEvents) { - constexpr std::array kEventsToReport{ - StatisticsEventType::kFrameCaptureBegin, - StatisticsEventType::kFrameCaptureEnd, StatisticsEventType::kFrameEncoded, - StatisticsEventType::kFrameAckSent, StatisticsEventType::kFramePlayedOut}; + constexpr std::array kEventsToReport{ + StatisticsEvent::Type::kFrameCaptureBegin, + StatisticsEvent::Type::kFrameCaptureEnd, + StatisticsEvent::Type::kFrameEncoded, + StatisticsEvent::Type::kFrameAckSent, + StatisticsEvent::Type::kFramePlayedOut}; constexpr int kNumFrames = 5; constexpr int kNumEvents = kNumFrames * kEventsToReport.size(); @@ -317,7 +349,7 @@ TEST_F(StatisticsAnalyzerTest, AllFrameEvents) { RtpTimeTicks rtp_timestamp; int current_event = 0; for (int frame_id = 0; frame_id < kNumFrames; frame_id++) { - for (StatisticsEventType event_type : kEventsToReport) { + for (StatisticsEvent::Type event_type : kEventsToReport) { FrameEvent event = MakeFrameEvent(frame_id, rtp_timestamp); event.type = event_type; event.timestamp += milliseconds(kTimestampOffsetsMs[current_event]); @@ -352,7 +384,7 @@ TEST_F(StatisticsAnalyzerTest, AllFrameEvents) { {HistogramType::kFrameLatenessMs, {0, 4, 0, 1}}}}; EXPECT_CALL(stats_client_, OnStatisticsUpdated(_)) - .WillOnce(Invoke([&](const SenderStats& stats) { + .WillOnce([&](const SenderStats& stats) { for (const auto& stat_pair : kExpectedStats) { ExpectStatEq(stats.video_statistics, stat_pair.first, stat_pair.second); @@ -361,7 +393,7 @@ TEST_F(StatisticsAnalyzerTest, AllFrameEvents) { ExpectHistoBuckets(stats.video_histograms, histogram_pair.first, histogram_pair.second); } - })); + }); fake_clock_.Advance(milliseconds(kDefaultStatsAnalysisIntervalMs - (kDefaultStatIntervalMs * kNumEvents))); @@ -390,7 +422,7 @@ TEST_F(StatisticsAnalyzerTest, FrameEncodedAndPacketSent) { } EXPECT_CALL(stats_client_, OnStatisticsUpdated(_)) - .WillOnce(Invoke([&](const SenderStats& stats) { + .WillOnce([&](const SenderStats& stats) { constexpr double kExpectedKbps = kDefaultSizeBytes * 8 * kDefaultNumEvents / static_cast(kDefaultStatsAnalysisIntervalMs); @@ -414,7 +446,7 @@ TEST_F(StatisticsAnalyzerTest, FrameEncodedAndPacketSent) { /* 100-119 */ 0}; ExpectHistoBuckets(stats.video_histograms, HistogramType::kQueueingLatencyMs, kExpectedBuckets); - })); + }); fake_clock_.Advance( milliseconds(kDefaultStatsAnalysisIntervalMs - @@ -438,7 +470,7 @@ TEST_F(StatisticsAnalyzerTest, PacketSentAndReceived) { event2.frame_id = FrameId(i); event2.timestamp += network_latency; event2.received_timestamp += network_latency * 2; - event2.type = StatisticsEventType::kPacketReceived; + event2.type = StatisticsEvent::Type::kPacketReceived; collector_->CollectPacketEvent(event1); collector_->CollectPacketEvent(event2); @@ -447,7 +479,7 @@ TEST_F(StatisticsAnalyzerTest, PacketSentAndReceived) { } EXPECT_CALL(stats_client_, OnStatisticsUpdated(_)) - .WillOnce(Invoke([&](const SenderStats& stats) { + .WillOnce([&](const SenderStats& stats) { const double expected_avg_network_latency = static_cast( to_milliseconds(total_network_latency).count()) / @@ -465,7 +497,7 @@ TEST_F(StatisticsAnalyzerTest, PacketSentAndReceived) { /* 100-119 */ 0}; ExpectHistoBuckets(stats.video_histograms, HistogramType::kNetworkLatencyMs, kExpectedBuckets); - })); + }); fake_clock_.Advance( milliseconds(kDefaultStatsAnalysisIntervalMs - @@ -473,11 +505,14 @@ TEST_F(StatisticsAnalyzerTest, PacketSentAndReceived) { } TEST_F(StatisticsAnalyzerTest, FrameEncodedPacketSentAndReceived) { + EXPECT_CALL(*fake_estimator_, GetEstimatedLatency()) + .WillRepeatedly(Return(std::optional(milliseconds(40)))); + analyzer_->ScheduleAnalysis(); Clock::duration total_packet_latency = milliseconds(0); RtpTimeTicks rtp_timestamp; - Clock::time_point last_event_time; + Clock::time_point last_event_time = Clock::time_point::min(); for (int i = 0; i < kDefaultNumEvents; i++) { FrameEvent event1 = MakeFrameEvent(i, rtp_timestamp); @@ -487,14 +522,16 @@ TEST_F(StatisticsAnalyzerTest, FrameEncodedPacketSentAndReceived) { // Let packet latency be either 20, 40, 60, 80, or 100 ms. Clock::duration packet_latency = milliseconds(100 - (20 * (i % 5))); total_packet_latency += packet_latency; - if (fake_clock_.now() + packet_latency > last_event_time) { - last_event_time = fake_clock_.now() + packet_latency; - } PacketEvent event3 = MakePacketEvent(i, rtp_timestamp); event3.timestamp += packet_latency; event3.received_timestamp += packet_latency * 2; - event3.type = StatisticsEventType::kPacketReceived; + event3.type = StatisticsEvent::Type::kPacketReceived; + + if (event3.type == StatisticsEvent::Type::kPacketReceived && + event3.received_timestamp > last_event_time) { + last_event_time = event3.received_timestamp; + } collector_->CollectFrameEvent(event1); collector_->CollectPacketEvent(event2); @@ -504,7 +541,7 @@ TEST_F(StatisticsAnalyzerTest, FrameEncodedPacketSentAndReceived) { } EXPECT_CALL(stats_client_, OnStatisticsUpdated(_)) - .WillOnce(Invoke([&](const SenderStats& stats) { + .WillOnce([&](const SenderStats& stats) { ExpectStatEq(stats.video_statistics, StatisticType::kNumPacketsSent, kDefaultNumEvents); ExpectStatEq(stats.video_statistics, StatisticType::kNumPacketsReceived, @@ -512,8 +549,8 @@ TEST_F(StatisticsAnalyzerTest, FrameEncodedPacketSentAndReceived) { const double expected_time_since_last_receiver_response = static_cast( - (to_milliseconds(fake_clock_.now() - last_event_time) - - milliseconds(25)) + to_milliseconds(fake_clock_.now() - + (last_event_time - milliseconds(40))) .count()); ExpectStatEq(stats.video_statistics, StatisticType::kTimeSinceLastReceiverResponseMs, @@ -535,7 +572,7 @@ TEST_F(StatisticsAnalyzerTest, FrameEncodedPacketSentAndReceived) { /* 120-139 */ 0}; ExpectHistoBuckets(stats.video_histograms, HistogramType::kPacketLatencyMs, kExpectedBuckets); - })); + }); fake_clock_.Advance( milliseconds(kDefaultStatsAnalysisIntervalMs - @@ -555,9 +592,9 @@ TEST_F(StatisticsAnalyzerTest, AudioAndVideoFrameEncodedPacketSentAndReceived) { int total_video_events = 0; for (int i = 0; i < num_events; i++) { - StatisticsEventMediaType media_type = StatisticsEventMediaType::kVideo; + StatisticsEvent::MediaType media_type = StatisticsEvent::MediaType::kVideo; if (i % 2 == 0) { - media_type = StatisticsEventMediaType::kAudio; + media_type = StatisticsEvent::MediaType::kAudio; } FrameEvent event1 = MakeFrameEvent(i, rtp_timestamp); @@ -569,17 +606,17 @@ TEST_F(StatisticsAnalyzerTest, AudioAndVideoFrameEncodedPacketSentAndReceived) { // Let packet latency be either 20, 40, 60, 80, or 100 ms. Clock::duration packet_latency = milliseconds(100 - (20 * (i % 5))); - if (media_type == StatisticsEventMediaType::kAudio) { + if (media_type == StatisticsEvent::MediaType::kAudio) { total_audio_events++; total_audio_packet_latency += packet_latency; - } else if (media_type == StatisticsEventMediaType::kVideo) { + } else if (media_type == StatisticsEvent::MediaType::kVideo) { total_video_events++; total_video_packet_latency += packet_latency; } PacketEvent event3 = MakePacketEvent(i, rtp_timestamp); event3.timestamp += packet_latency; - event3.type = StatisticsEventType::kPacketReceived; + event3.type = StatisticsEvent::Type::kPacketReceived; event3.media_type = media_type; collector_->CollectFrameEvent(event1); @@ -590,7 +627,7 @@ TEST_F(StatisticsAnalyzerTest, AudioAndVideoFrameEncodedPacketSentAndReceived) { } EXPECT_CALL(stats_client_, OnStatisticsUpdated(_)) - .WillOnce(Invoke([&](const SenderStats& stats) { + .WillOnce([&](const SenderStats& stats) { ExpectStatEq(stats.audio_statistics, StatisticType::kNumPacketsSent, total_audio_events); ExpectStatEq(stats.audio_statistics, StatisticType::kNumPacketsReceived, @@ -612,17 +649,19 @@ TEST_F(StatisticsAnalyzerTest, AudioAndVideoFrameEncodedPacketSentAndReceived) { total_video_events; ExpectStatEq(stats.video_statistics, StatisticType::kAvgPacketLatencyMs, expected_video_avg_packet_latency); - })); + }); fake_clock_.Advance(milliseconds(kDefaultStatsAnalysisIntervalMs - (frame_interval_ms * num_events))); } TEST_F(StatisticsAnalyzerTest, LotsOfEventsStillWorksProperly) { - constexpr std::array kEventsToReport{ - StatisticsEventType::kFrameCaptureBegin, - StatisticsEventType::kFrameCaptureEnd, StatisticsEventType::kFrameEncoded, - StatisticsEventType::kFrameAckSent, StatisticsEventType::kFramePlayedOut}; + constexpr std::array kEventsToReport{ + StatisticsEvent::Type::kFrameCaptureBegin, + StatisticsEvent::Type::kFrameCaptureEnd, + StatisticsEvent::Type::kFrameEncoded, + StatisticsEvent::Type::kFrameAckSent, + StatisticsEvent::Type::kFramePlayedOut}; constexpr int kNumFrames = 1000; constexpr int kNumEvents = kNumFrames * kEventsToReport.size(); @@ -663,7 +702,7 @@ TEST_F(StatisticsAnalyzerTest, LotsOfEventsStillWorksProperly) { testing::InSequence s; EXPECT_CALL(stats_client_, OnStatisticsUpdated(_)).Times(49); EXPECT_CALL(stats_client_, OnStatisticsUpdated(_)) - .WillOnce(Invoke([&](const SenderStats& stats) { + .WillOnce([&](const SenderStats& stats) { for (const auto& stat_pair : kExpectedStats) { ExpectStatEq(stats.video_statistics, stat_pair.first, stat_pair.second); @@ -672,14 +711,14 @@ TEST_F(StatisticsAnalyzerTest, LotsOfEventsStillWorksProperly) { ExpectHistoBuckets(stats.video_histograms, histogram_pair.first, histogram_pair.second); } - })); + }); } analyzer_->ScheduleAnalysis(); RtpTimeTicks rtp_timestamp; int current_event = 0; for (int frame_id = 0; frame_id < kNumFrames; frame_id++) { - for (StatisticsEventType event_type : kEventsToReport) { + for (StatisticsEvent::Type event_type : kEventsToReport) { FrameEvent event = MakeFrameEvent(frame_id, rtp_timestamp); event.type = event_type; event.timestamp += milliseconds( diff --git a/cast/streaming/impl/statistics_collector.cc b/cast/streaming/impl/statistics_collector.cc index 0f6c27332..daed12cbd 100644 --- a/cast/streaming/impl/statistics_collector.cc +++ b/cast/streaming/impl/statistics_collector.cc @@ -23,7 +23,7 @@ void StatisticsCollector::CollectPacketSentEvent(ByteView packet, // Populate the new PacketEvent by parsing the wire-format `packet`. event.timestamp = now_(); - event.type = StatisticsEventType::kPacketSentToNetwork; + event.type = StatisticsEvent::Type::kPacketSentToNetwork; BigEndianReader reader(packet.data(), packet.size()); bool success = reader.Skip(4); @@ -32,7 +32,7 @@ void StatisticsCollector::CollectPacketSentEvent(ByteView packet, success &= reader.Skip(4); event.rtp_timestamp = metadata.rtp_timestamp.Expand(truncated_rtp_timestamp); - event.media_type = ToMediaType(metadata.stream_type); + event.media_type = StatisticsEvent::ToMediaType(metadata.stream_type); success &= reader.Skip(2); success &= reader.Read(&event.packet_id); diff --git a/cast/streaming/impl/statistics_collector.h b/cast/streaming/impl/statistics_collector.h index cc9aedd78..6067e5d11 100644 --- a/cast/streaming/impl/statistics_collector.h +++ b/cast/streaming/impl/statistics_collector.h @@ -7,7 +7,7 @@ #include -#include "cast/streaming/impl/statistics_defines.h" +#include "cast/streaming/impl/statistics_common.h" #include "platform/api/time.h" #include "platform/base/span.h" diff --git a/cast/streaming/impl/statistics_collector_unittest.cc b/cast/streaming/impl/statistics_collector_unittest.cc index cd129dd50..746a2e716 100644 --- a/cast/streaming/impl/statistics_collector_unittest.cc +++ b/cast/streaming/impl/statistics_collector_unittest.cc @@ -6,7 +6,7 @@ #include -#include "cast/streaming/impl/statistics_defines.h" +#include "cast/streaming/impl/statistics_common.h" #include "gtest/gtest.h" #include "platform/api/time.h" #include "platform/base/span.h" @@ -35,8 +35,8 @@ TEST_F(StatisticsCollectorTest, CanCollectPacketEvents) { // clang-format off constexpr PacketEvent kEventOne( FrameId(5000), - StatisticsEventType::kPacketSentToNetwork, - StatisticsEventMediaType::kAudio, + StatisticsEvent::Type::kPacketSentToNetwork, + StatisticsEvent::MediaType::kAudio, RtpTimeTicks(47474838), 1234u, Clock::time_point(milliseconds(12455680)), @@ -46,8 +46,8 @@ TEST_F(StatisticsCollectorTest, CanCollectPacketEvents) { constexpr PacketEvent kEventTwo( FrameId(20000), - StatisticsEventType::kPacketSentToNetwork, - StatisticsEventMediaType::kVideo, + StatisticsEvent::Type::kPacketSentToNetwork, + StatisticsEvent::MediaType::kVideo, RtpTimeTicks(4747900), 553u, Clock::time_point(milliseconds(12455880)), @@ -87,8 +87,8 @@ TEST_F(StatisticsCollectorTest, CanCollectPacketSentEvents) { EXPECT_EQ(FrameId(), events[0].frame_id); EXPECT_EQ(20u, events[0].size); EXPECT_GT(Clock::now(), events[0].timestamp); - EXPECT_EQ(StatisticsEventType::kPacketSentToNetwork, events[0].type); - EXPECT_EQ(StatisticsEventMediaType::kAudio, events[0].media_type); + EXPECT_EQ(StatisticsEvent::Type::kPacketSentToNetwork, events[0].type); + EXPECT_EQ(StatisticsEvent::MediaType::kAudio, events[0].media_type); EXPECT_EQ(3599u, events[1].packet_id); EXPECT_EQ(4113u, events[1].max_packet_id); @@ -96,16 +96,16 @@ TEST_F(StatisticsCollectorTest, CanCollectPacketSentEvents) { EXPECT_EQ(FrameId(), events[1].frame_id); EXPECT_EQ(21u, events[1].size); EXPECT_GT(Clock::now(), events[1].timestamp); - EXPECT_EQ(StatisticsEventType::kPacketSentToNetwork, events[1].type); - EXPECT_EQ(StatisticsEventMediaType::kVideo, events[1].media_type); + EXPECT_EQ(StatisticsEvent::Type::kPacketSentToNetwork, events[1].type); + EXPECT_EQ(StatisticsEvent::MediaType::kVideo, events[1].media_type); } TEST_F(StatisticsCollectorTest, CanCollectFrameEvents) { // clang-format off constexpr FrameEvent kEventOne( FrameId(1), - StatisticsEventType::kFrameAckReceived, - StatisticsEventMediaType::kVideo, + StatisticsEvent::Type::kFrameAckReceived, + StatisticsEvent::MediaType::kVideo, RtpTimeTicks(1233), /* size= */ 0, Clock::time_point(milliseconds(12345678)), @@ -118,8 +118,8 @@ TEST_F(StatisticsCollectorTest, CanCollectFrameEvents) { constexpr FrameEvent kEventTwo( FrameId(2), - StatisticsEventType::kFramePlayedOut, - StatisticsEventMediaType::kAudio, + StatisticsEvent::Type::kFramePlayedOut, + StatisticsEvent::MediaType::kAudio, RtpTimeTicks(1733), /* size= */ 6000, Clock::time_point(milliseconds(12455680)), diff --git a/cast/streaming/impl/statistics_common.cc b/cast/streaming/impl/statistics_common.cc new file mode 100644 index 000000000..a25f56205 --- /dev/null +++ b/cast/streaming/impl/statistics_common.cc @@ -0,0 +1,115 @@ +// Copyright 2023 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cast/streaming/impl/statistics_common.h" + +#include "util/osp_logging.h" + +namespace openscreen::cast { + +// static +StatisticsEvent::Type StatisticsEvent::FromWireType(WireType wire_type) { + switch (wire_type) { + case WireType::kAudioAckSent: + case WireType::kVideoAckSent: + case WireType::kUnifiedAckSent: + return Type::kFrameAckSent; + + case WireType::kAudioPlayoutDelay: + case WireType::kVideoRenderDelay: + case WireType::kUnifiedRenderDelay: + return Type::kFramePlayedOut; + + case WireType::kAudioFrameDecoded: + case WireType::kVideoFrameDecoded: + case WireType::kUnifiedFrameDecoded: + return Type::kFrameDecoded; + + case WireType::kAudioPacketReceived: + case WireType::kVideoPacketReceived: + case WireType::kUnifiedPacketReceived: + return Type::kPacketReceived; + + default: + OSP_VLOG << "Unexpected RTCP log message received: " + << static_cast(wire_type); + return Type::kUnknown; + } +} + +// static +StatisticsEvent::WireType StatisticsEvent::ToWireType(Type type) { + switch (type) { + case Type::kUnknown: + return WireType::kUnknown; + + case Type::kFrameAckSent: + return WireType::kUnifiedAckSent; + + case Type::kFramePlayedOut: + return WireType::kUnifiedRenderDelay; + + case Type::kFrameDecoded: + return WireType::kUnifiedFrameDecoded; + + case Type::kPacketReceived: + return WireType::kUnifiedPacketReceived; + + default: + OSP_VLOG << "Unknown RTCP log message event type: " + << static_cast(type); + return WireType::kUnknown; + } +} + +// static +StatisticsEvent::MediaType StatisticsEvent::ToMediaType(StreamType type) { + switch (type) { + case StreamType::kUnknown: + return MediaType::kUnknown; + case StreamType::kAudio: + return MediaType::kAudio; + case StreamType::kVideo: + return MediaType::kVideo; + } + + OSP_NOTREACHED(); +} + +StatisticsEvent::StatisticsEvent(const StatisticsEvent& other) = default; +StatisticsEvent::StatisticsEvent(StatisticsEvent&& other) noexcept = default; +StatisticsEvent& StatisticsEvent::operator=(const StatisticsEvent& other) = + default; +StatisticsEvent& StatisticsEvent::operator=(StatisticsEvent&& other) = default; + +bool StatisticsEvent::operator==(const StatisticsEvent& other) const { + return frame_id == other.frame_id && type == other.type && + media_type == other.media_type && + rtp_timestamp == other.rtp_timestamp && size == other.size && + timestamp == other.timestamp && + received_timestamp == other.received_timestamp; +} + +FrameEvent::FrameEvent(const FrameEvent& other) = default; +FrameEvent::FrameEvent(FrameEvent&& other) noexcept = default; +FrameEvent& FrameEvent::operator=(const FrameEvent& other) = default; +FrameEvent& FrameEvent::operator=(FrameEvent&& other) = default; + +bool FrameEvent::operator==(const FrameEvent& other) const { + return StatisticsEvent::operator==(other) && width == other.width && + height == other.height && delay_delta == other.delay_delta && + key_frame == other.key_frame && target_bitrate == other.target_bitrate; +} + +PacketEvent::PacketEvent(const PacketEvent& other) = default; +PacketEvent::PacketEvent(PacketEvent&& other) noexcept = default; +PacketEvent& PacketEvent::operator=(const PacketEvent& other) = default; +PacketEvent& PacketEvent::operator=(PacketEvent&& other) = default; + +bool PacketEvent::operator==(const PacketEvent& other) const { + return StatisticsEvent::operator==(other) && packet_id == other.packet_id && + max_packet_id == other.max_packet_id; +} + +} // namespace openscreen::cast diff --git a/cast/streaming/impl/statistics_common.h b/cast/streaming/impl/statistics_common.h new file mode 100644 index 000000000..69f2dba72 --- /dev/null +++ b/cast/streaming/impl/statistics_common.h @@ -0,0 +1,223 @@ +// Copyright 2023 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CAST_STREAMING_IMPL_STATISTICS_COMMON_H_ +#define CAST_STREAMING_IMPL_STATISTICS_COMMON_H_ + +#include +#include + +#include "cast/streaming/public/constants.h" +#include "cast/streaming/public/frame_id.h" +#include "cast/streaming/rtp_time.h" +#include "platform/api/time.h" + +namespace openscreen::cast { + +struct StatisticsEvent { + enum class Type : int { + kUnknown = 0, + + // Sender side frame events. + kFrameCaptureBegin = 1, + kFrameCaptureEnd = 2, + kFrameEncoded = 3, + kFrameAckReceived = 4, + + // Receiver side frame events. + kFrameAckSent = 5, + kFrameDecoded = 6, + kFramePlayedOut = 7, + + // Sender side packet events. + kPacketSentToNetwork = 8, + kPacketRetransmitted = 9, + kPacketRtxRejected = 10, + + // Receiver side packet events. + kPacketReceived = 11, + kFrameDroppedByEncoder = 15, + + kNumOfEvents = kFrameDroppedByEncoder + 1 + }; + + // Serialized values for the statistics events for use by the RTCP builder + // and parser logic. *Do not modify existing values* since they are shared by + // both libcast-based devices as well as a variety of legacy implementations. + // + // NOTE: Events 1 to 8 have been replaced with events 11 to 14 (e.g. + // kAudioAckSent and kVideoAckSent merged into a single event kAckSent). + // Events 9 and 10 (to log duplicated packets) have been fully removed. Future + // events may reuse those values. + enum class WireType : uint8_t { + kUnknown = 0, + + // Legacy audio event types. + kAudioAckSent = 1, + kAudioPlayoutDelay = 2, + kAudioFrameDecoded = 3, + kAudioPacketReceived = 4, + + // Legacy video event types. + kVideoAckSent = 5, + kVideoRenderDelay = 6, + kVideoFrameDecoded = 7, + kVideoPacketReceived = 8, + + // New unified event types. + kUnifiedAckSent = 11, + kUnifiedRenderDelay = 12, + kUnifiedFrameDecoded = 13, + kUnifiedPacketReceived = 14, + + kNumOfEvents = kUnifiedPacketReceived + 1 + }; + + enum class MediaType : int { kUnknown = 0, kAudio = 1, kVideo = 2 }; + + static Type FromWireType(WireType wire_type); + static WireType ToWireType(Type type); + static MediaType ToMediaType(StreamType type); + + constexpr StatisticsEvent(FrameId frame_id, + Type type, + MediaType media_type, + RtpTimeTicks rtp_timestamp, + uint32_t size, + Clock::time_point timestamp, + Clock::time_point received_timestamp) + : frame_id(frame_id), + type(type), + media_type(media_type), + rtp_timestamp(rtp_timestamp), + size(size), + timestamp(timestamp), + received_timestamp(received_timestamp) {} + + constexpr StatisticsEvent() = default; + StatisticsEvent(const StatisticsEvent& other); + StatisticsEvent(StatisticsEvent&& other) noexcept; + StatisticsEvent& operator=(const StatisticsEvent& other); + StatisticsEvent& operator=(StatisticsEvent&& other); + ~StatisticsEvent() = default; + + bool operator==(const StatisticsEvent& other) const; + + // The frame this event is associated with. + FrameId frame_id; + + // The type of this frame event. + Type type = Type::kUnknown; + + // Whether this was audio or video (or unknown). + MediaType media_type = MediaType::kUnknown; + + // The RTP timestamp of the frame this event is associated with. + RtpTimeTicks rtp_timestamp; + + // Size of this packet, or the frame it is associated with. + // Note: we use uint32_t instead of size_t for byte count because this struct + // is sent over IPC which could span 32 & 64 bit processes. + uint32_t size = 0; + + // Time of event logged. + Clock::time_point timestamp; + + // Time that the event was received by the sender. Only set for receiver-side + // events. + Clock::time_point received_timestamp; +}; + +struct FrameEvent : public StatisticsEvent { + constexpr FrameEvent(FrameId frame_id_in, + Type type_in, + MediaType media_type_in, + RtpTimeTicks rtp_timestamp_in, + uint32_t size_in, + Clock::time_point timestamp_in, + Clock::time_point received_timestamp_in, + int width, + int height, + Clock::duration delay_delta, + bool key_frame, + int target_bitrate) + : StatisticsEvent(frame_id_in, + type_in, + media_type_in, + rtp_timestamp_in, + size_in, + timestamp_in, + received_timestamp_in), + width(width), + height(height), + delay_delta(delay_delta), + key_frame(key_frame), + target_bitrate(target_bitrate) {} + + constexpr FrameEvent() = default; + FrameEvent(const FrameEvent& other); + FrameEvent(FrameEvent&& other) noexcept; + FrameEvent& operator=(const FrameEvent& other); + FrameEvent& operator=(FrameEvent&& other); + ~FrameEvent() = default; + + bool operator==(const FrameEvent& other) const; + + // Resolution of the frame. Only set for video FRAME_CAPTURE_END events. + int width = 0; + int height = 0; + + // Only set for FRAME_PLAYOUT events. + // If this value is zero the frame is rendered on time. + // If this value is positive it means the frame is rendered late. + // If this value is negative it means the frame is rendered early. + Clock::duration delay_delta{}; + + // Whether the frame is a key frame. Only set for video FRAME_ENCODED event. + bool key_frame = false; + + // The requested target bitrate of the encoder at the time the frame is + // encoded. Only set for video FRAME_ENCODED event. + int target_bitrate = 0; +}; + +struct PacketEvent : public StatisticsEvent { + constexpr PacketEvent(FrameId frame_id_in, + Type type_in, + MediaType media_type_in, + RtpTimeTicks rtp_timestamp_in, + uint32_t size_in, + Clock::time_point timestamp_in, + Clock::time_point received_timestamp_in, + uint16_t packet_id, + uint16_t max_packet_id) + : StatisticsEvent(frame_id_in, + type_in, + media_type_in, + rtp_timestamp_in, + size_in, + timestamp_in, + received_timestamp_in), + packet_id(packet_id), + max_packet_id(max_packet_id) {} + + constexpr PacketEvent() = default; + PacketEvent(const PacketEvent& other); + PacketEvent(PacketEvent&& other) noexcept; + PacketEvent& operator=(const PacketEvent& other); + PacketEvent& operator=(PacketEvent&& other); + ~PacketEvent() = default; + + bool operator==(const PacketEvent& other) const; + + // The packet this event is associated with. + uint16_t packet_id = 0; + + // The highest packet ID seen so far at time of event. + uint16_t max_packet_id = 0; +}; + +} // namespace openscreen::cast + +#endif // CAST_STREAMING_IMPL_STATISTICS_COMMON_H_ diff --git a/cast/streaming/impl/statistics_defines.cc b/cast/streaming/impl/statistics_defines.cc deleted file mode 100644 index 8ec1c7396..000000000 --- a/cast/streaming/impl/statistics_defines.cc +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2023 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "cast/streaming/impl/statistics_defines.h" - -namespace openscreen::cast { - -StatisticsEventMediaType ToMediaType(StreamType type) { - switch (type) { - case StreamType::kUnknown: - return StatisticsEventMediaType::kUnknown; - case StreamType::kAudio: - return StatisticsEventMediaType::kAudio; - case StreamType::kVideo: - return StatisticsEventMediaType::kVideo; - } - - OSP_NOTREACHED(); -} - -StatisticsEvent::StatisticsEvent(const StatisticsEvent& other) = default; -StatisticsEvent::StatisticsEvent(StatisticsEvent&& other) noexcept = default; -StatisticsEvent& StatisticsEvent::operator=(const StatisticsEvent& other) = - default; -StatisticsEvent& StatisticsEvent::operator=(StatisticsEvent&& other) = default; - -bool StatisticsEvent::operator==(const StatisticsEvent& other) const { - return frame_id == other.frame_id && type == other.type && - media_type == other.media_type && - rtp_timestamp == other.rtp_timestamp && size == other.size && - timestamp == other.timestamp && - received_timestamp == other.received_timestamp; -} - -FrameEvent::FrameEvent(const FrameEvent& other) = default; -FrameEvent::FrameEvent(FrameEvent&& other) noexcept = default; -FrameEvent& FrameEvent::operator=(const FrameEvent& other) = default; -FrameEvent& FrameEvent::operator=(FrameEvent&& other) = default; - -bool FrameEvent::operator==(const FrameEvent& other) const { - return StatisticsEvent::operator==(other) && width == other.width && - height == other.height && delay_delta == other.delay_delta && - key_frame == other.key_frame && target_bitrate == other.target_bitrate; -} - -PacketEvent::PacketEvent(const PacketEvent& other) = default; -PacketEvent::PacketEvent(PacketEvent&& other) noexcept = default; -PacketEvent& PacketEvent::operator=(const PacketEvent& other) = default; -PacketEvent& PacketEvent::operator=(PacketEvent&& other) = default; - -bool PacketEvent::operator==(const PacketEvent& other) const { - return StatisticsEvent::operator==(other) && packet_id == other.packet_id && - max_packet_id == other.max_packet_id; -} - -} // namespace openscreen::cast diff --git a/cast/streaming/impl/statistics_defines.h b/cast/streaming/impl/statistics_defines.h deleted file mode 100644 index 736cd675b..000000000 --- a/cast/streaming/impl/statistics_defines.h +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2023 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CAST_STREAMING_IMPL_STATISTICS_DEFINES_H_ -#define CAST_STREAMING_IMPL_STATISTICS_DEFINES_H_ - -#include -#include - -#include "cast/streaming/public/constants.h" -#include "cast/streaming/public/frame_id.h" -#include "cast/streaming/rtp_time.h" -#include "platform/api/time.h" - -namespace openscreen::cast { - -enum class StatisticsEventType : int { - kUnknown = 0, - - // Sender side frame events. - kFrameCaptureBegin = 1, - kFrameCaptureEnd = 2, - kFrameEncoded = 3, - kFrameAckReceived = 4, - - // Receiver side frame events. - kFrameAckSent = 5, - kFrameDecoded = 6, - kFramePlayedOut = 7, - - // Sender side packet events. - kPacketSentToNetwork = 8, - kPacketRetransmitted = 9, - kPacketRtxRejected = 10, - - // Receiver side packet events. - kPacketReceived = 11, - - kNumOfEvents = kPacketReceived + 1 -}; - -enum class StatisticsEventMediaType : int { - kUnknown = 0, - kAudio = 1, - kVideo = 2 -}; - -StatisticsEventMediaType ToMediaType(StreamType type); - -struct StatisticsEvent { - constexpr StatisticsEvent(FrameId frame_id, - StatisticsEventType type, - StatisticsEventMediaType media_type, - RtpTimeTicks rtp_timestamp, - uint32_t size, - Clock::time_point timestamp, - Clock::time_point received_timestamp) - : frame_id(frame_id), - type(type), - media_type(media_type), - rtp_timestamp(rtp_timestamp), - size(size), - timestamp(timestamp), - received_timestamp(received_timestamp) {} - - constexpr StatisticsEvent() = default; - StatisticsEvent(const StatisticsEvent& other); - StatisticsEvent(StatisticsEvent&& other) noexcept; - StatisticsEvent& operator=(const StatisticsEvent& other); - StatisticsEvent& operator=(StatisticsEvent&& other); - ~StatisticsEvent() = default; - - bool operator==(const StatisticsEvent& other) const; - - // The frame this event is associated with. - FrameId frame_id; - - // The type of this frame event. - StatisticsEventType type = StatisticsEventType::kUnknown; - - // Whether this was audio or video (or unknown). - StatisticsEventMediaType media_type = StatisticsEventMediaType::kUnknown; - - // The RTP timestamp of the frame this event is associated with. - RtpTimeTicks rtp_timestamp; - - // Size of this packet, or the frame it is associated with. - // Note: we use uint32_t instead of size_t for byte count because this struct - // is sent over IPC which could span 32 & 64 bit processes. - uint32_t size = 0; - - // Time of event logged. - Clock::time_point timestamp; - - // Time that the event was received by the sender. Only set for receiver-side - // events. - Clock::time_point received_timestamp; -}; - -struct FrameEvent : public StatisticsEvent { - constexpr FrameEvent(FrameId frame_id_in, - StatisticsEventType type_in, - StatisticsEventMediaType media_type_in, - RtpTimeTicks rtp_timestamp_in, - uint32_t size_in, - Clock::time_point timestamp_in, - Clock::time_point received_timestamp_in, - int width, - int height, - Clock::duration delay_delta, - bool key_frame, - int target_bitrate) - : StatisticsEvent(frame_id_in, - type_in, - media_type_in, - rtp_timestamp_in, - size_in, - timestamp_in, - received_timestamp_in), - width(width), - height(height), - delay_delta(delay_delta), - key_frame(key_frame), - target_bitrate(target_bitrate) {} - - constexpr FrameEvent() = default; - FrameEvent(const FrameEvent& other); - FrameEvent(FrameEvent&& other) noexcept; - FrameEvent& operator=(const FrameEvent& other); - FrameEvent& operator=(FrameEvent&& other); - ~FrameEvent() = default; - - bool operator==(const FrameEvent& other) const; - - // Resolution of the frame. Only set for video FRAME_CAPTURE_END events. - int width = 0; - int height = 0; - - // Only set for FRAME_PLAYOUT events. - // If this value is zero the frame is rendered on time. - // If this value is positive it means the frame is rendered late. - // If this value is negative it means the frame is rendered early. - Clock::duration delay_delta{}; - - // Whether the frame is a key frame. Only set for video FRAME_ENCODED event. - bool key_frame = false; - - // The requested target bitrate of the encoder at the time the frame is - // encoded. Only set for video FRAME_ENCODED event. - int target_bitrate = 0; -}; - -struct PacketEvent : public StatisticsEvent { - constexpr PacketEvent(FrameId frame_id_in, - StatisticsEventType type_in, - StatisticsEventMediaType media_type_in, - RtpTimeTicks rtp_timestamp_in, - uint32_t size_in, - Clock::time_point timestamp_in, - Clock::time_point received_timestamp_in, - uint16_t packet_id, - uint16_t max_packet_id) - : StatisticsEvent(frame_id_in, - type_in, - media_type_in, - rtp_timestamp_in, - size_in, - timestamp_in, - received_timestamp_in), - packet_id(packet_id), - max_packet_id(max_packet_id) {} - - constexpr PacketEvent() = default; - PacketEvent(const PacketEvent& other); - PacketEvent(PacketEvent&& other) noexcept; - PacketEvent& operator=(const PacketEvent& other); - PacketEvent& operator=(PacketEvent&& other); - ~PacketEvent() = default; - - bool operator==(const PacketEvent& other) const; - - // The packet this event is associated with. - uint16_t packet_id = 0; - - // The highest packet ID seen so far at time of event. - uint16_t max_packet_id = 0; -}; - -} // namespace openscreen::cast - -#endif // CAST_STREAMING_IMPL_STATISTICS_DEFINES_H_ diff --git a/cast/streaming/impl/statistics_dispatcher.cc b/cast/streaming/impl/statistics_dispatcher.cc new file mode 100644 index 000000000..21a8c7b2e --- /dev/null +++ b/cast/streaming/impl/statistics_dispatcher.cc @@ -0,0 +1,162 @@ +// Copyright 2025 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cast/streaming/impl/statistics_dispatcher.h" + +#include + +#include "cast/streaming/impl/rtcp_common.h" +#include "cast/streaming/impl/rtp_defines.h" +#include "cast/streaming/impl/statistics_collector.h" +#include "cast/streaming/impl/statistics_common.h" +#include "cast/streaming/public/encoded_frame.h" +#include "cast/streaming/public/environment.h" +#include "cast/streaming/public/session_config.h" +#include "platform/base/trivial_clock_traits.h" +#include "util/chrono_helpers.h" +#include "util/osp_logging.h" +#include "util/std_util.h" +#include "util/trace_logging.h" + +namespace openscreen::cast { + +using clock_operators::operator<<; + +StatisticsDispatcher::StatisticsDispatcher(Environment& environment) + : environment_(environment) {} +StatisticsDispatcher::~StatisticsDispatcher() = default; + +void StatisticsDispatcher::DispatchEnqueueEvents(StreamType stream_type, + const EncodedFrame& frame) { + if (!environment_.statistics_collector()) { + return; + } + const auto media_type = StatisticsEvent::ToMediaType(stream_type); + + // Submit a capture begin event. + FrameEvent capture_begin_event; + capture_begin_event.type = StatisticsEvent::Type::kFrameCaptureBegin; + capture_begin_event.media_type = media_type; + capture_begin_event.rtp_timestamp = frame.rtp_timestamp; + capture_begin_event.timestamp = + (frame.capture_begin_time > Clock::time_point::min()) + ? frame.capture_begin_time + : environment_.now(); + environment_.statistics_collector()->CollectFrameEvent( + std::move(capture_begin_event)); + + // Submit a capture end event. + FrameEvent capture_end_event; + capture_end_event.type = StatisticsEvent::Type::kFrameCaptureEnd; + capture_end_event.media_type = media_type; + capture_end_event.rtp_timestamp = frame.rtp_timestamp; + capture_end_event.timestamp = + (frame.capture_end_time > Clock::time_point::min()) + ? frame.capture_end_time + : environment_.now(); + environment_.statistics_collector()->CollectFrameEvent( + std::move(capture_end_event)); + + // Submit an encoded event. + FrameEvent encode_event; + encode_event.timestamp = environment_.now(); + encode_event.type = StatisticsEvent::Type::kFrameEncoded; + encode_event.media_type = media_type; + encode_event.rtp_timestamp = frame.rtp_timestamp; + encode_event.frame_id = frame.frame_id; + encode_event.size = static_cast(frame.data.size()); + encode_event.key_frame = + frame.dependency == openscreen::cast::EncodedFrame::Dependency::kKeyFrame; + + environment_.statistics_collector()->CollectFrameEvent( + std::move(encode_event)); +} + +void StatisticsDispatcher::DispatchAckEvent(StreamType stream_type, + RtpTimeTicks rtp_timestamp, + FrameId frame_id) { + if (!environment_.statistics_collector()) { + return; + } + + FrameEvent ack_event; + ack_event.timestamp = environment_.now(); + ack_event.type = StatisticsEvent::Type::kFrameAckReceived; + ack_event.media_type = StatisticsEvent::ToMediaType(stream_type); + ack_event.rtp_timestamp = rtp_timestamp; + ack_event.frame_id = frame_id; + + environment_.statistics_collector()->CollectFrameEvent(std::move(ack_event)); +} + +void StatisticsDispatcher::DispatchFrameDropEvent(StreamType stream_type, + FrameId frame_id, + RtpTimeTicks rtp_timestamp, + Clock::time_point drop_time) { + if (!environment_.statistics_collector()) { + return; + } + + FrameEvent drop_event; + drop_event.timestamp = drop_time; + drop_event.type = StatisticsEvent::Type::kFrameDroppedByEncoder; + drop_event.media_type = StatisticsEvent::ToMediaType(stream_type); + drop_event.rtp_timestamp = rtp_timestamp; + drop_event.frame_id = frame_id; + + environment_.statistics_collector()->CollectFrameEvent(std::move(drop_event)); +} + +void StatisticsDispatcher::DispatchFrameLogMessages( + StreamType stream_type, + const std::vector& messages) { + if (!environment_.statistics_collector()) { + return; + } + + const Clock::time_point now = environment_.now(); + const auto media_type = StatisticsEvent::ToMediaType(stream_type); + for (const RtcpReceiverFrameLogMessage& log_message : messages) { + for (const RtcpReceiverEventLogMessage& event_message : + log_message.messages) { + switch (event_message.type) { + case StatisticsEvent::Type::kPacketReceived: { + PacketEvent event; + event.timestamp = event_message.timestamp; + event.received_timestamp = now; + event.type = event_message.type; + event.media_type = media_type; + event.rtp_timestamp = log_message.rtp_timestamp; + event.packet_id = event_message.packet_id; + environment_.statistics_collector()->CollectPacketEvent( + std::move(event)); + } break; + + case StatisticsEvent::Type::kFrameAckSent: + case StatisticsEvent::Type::kFrameDecoded: + case StatisticsEvent::Type::kFramePlayedOut: { + FrameEvent event; + event.timestamp = event_message.timestamp; + event.received_timestamp = now; + event.type = event_message.type; + event.media_type = media_type; + event.rtp_timestamp = log_message.rtp_timestamp; + if (event.type == StatisticsEvent::Type::kFramePlayedOut) { + event.delay_delta = event_message.delay; + } + environment_.statistics_collector()->CollectFrameEvent( + std::move(event)); + } break; + + default: + OSP_VLOG << "Received log message via RTCP that we did not expect, " + "StatisticsEvent::Type=" + << static_cast(event_message.type); + break; + } + } + } +} + +} // namespace openscreen::cast diff --git a/cast/streaming/impl/statistics_dispatcher.h b/cast/streaming/impl/statistics_dispatcher.h new file mode 100644 index 000000000..38dbc8636 --- /dev/null +++ b/cast/streaming/impl/statistics_dispatcher.h @@ -0,0 +1,57 @@ +// Copyright 2025 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CAST_STREAMING_IMPL_STATISTICS_DISPATCHER_H_ +#define CAST_STREAMING_IMPL_STATISTICS_DISPATCHER_H_ + +#include + +#include "cast/streaming/impl/statistics_common.h" +#include "platform/api/time.h" +#include "platform/base/span.h" + +namespace openscreen::cast { + +class StatisticsCollector; +class Environment; +struct EncodedFrame; +struct RtcpReceiverFrameLogMessage; + +// This class is responsible for dispatching statistics events. +class StatisticsDispatcher { + public: + explicit StatisticsDispatcher(Environment& environment); + + StatisticsDispatcher(const StatisticsDispatcher&) = delete; + StatisticsDispatcher& operator=(const StatisticsDispatcher&) = delete; + StatisticsDispatcher(StatisticsDispatcher&&) noexcept = delete; + StatisticsDispatcher& operator=(StatisticsDispatcher&&) = delete; + ~StatisticsDispatcher(); + + // Dispatches enqueue events for a given frame. + void DispatchEnqueueEvents(StreamType stream_type, const EncodedFrame& frame); + + // Dispatches frame log messages. + void DispatchFrameLogMessages( + StreamType stream_type, + const std::vector& messages); + + // Dispatches an ack event. + void DispatchAckEvent(StreamType stream_type, + RtpTimeTicks rtp_timestamp, + FrameId frame_id); + + // Dispatches a frame dropped by encoder event. + void DispatchFrameDropEvent(StreamType stream_type, + FrameId frame_id, + RtpTimeTicks rtp_timestamp, + Clock::time_point drop_time); + + private: + Environment& environment_; +}; + +} // namespace openscreen::cast + +#endif // CAST_STREAMING_IMPL_STATISTICS_DISPATCHER_H_ diff --git a/cast/streaming/impl/statistics_dispatcher_unittest.cc b/cast/streaming/impl/statistics_dispatcher_unittest.cc new file mode 100644 index 000000000..9e50dcba1 --- /dev/null +++ b/cast/streaming/impl/statistics_dispatcher_unittest.cc @@ -0,0 +1,230 @@ +// Copyright 2025 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "cast/streaming/impl/statistics_dispatcher.h" + +#include "cast/streaming/impl/rtcp_common.h" +#include "cast/streaming/impl/rtp_defines.h" +#include "cast/streaming/impl/statistics_collector.h" +#include "cast/streaming/impl/statistics_common.h" +#include "cast/streaming/public/encoded_frame.h" +#include "cast/streaming/public/frame_id.h" +#include "cast/streaming/testing/mock_environment.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "platform/api/time.h" +#include "platform/base/error.h" +#include "platform/test/fake_clock.h" +#include "platform/test/fake_task_runner.h" +#include "util/chrono_helpers.h" + +namespace openscreen::cast { + +namespace { + +using ::testing::_; +using ::testing::ElementsAre; +using ::testing::IsEmpty; +using ::testing::Mock; +using ::testing::SaveArg; +using ::testing::StrictMock; + +class StatisticsDispatcherTest : public ::testing::Test { + public: + StatisticsDispatcherTest() + : environment_(&FakeClock::now, task_runner_), + collector_(&clock_.now), + dispatcher_(environment_) { + environment_.SetStatisticsCollector(&collector_); + } + + ~StatisticsDispatcherTest() override { + environment_.SetStatisticsCollector(nullptr); + } + + protected: + FakeClock clock_{Clock::now()}; + FakeTaskRunner task_runner_{clock_}; + testing::NiceMock environment_; + StatisticsCollector collector_; + StatisticsDispatcher dispatcher_; +}; + +TEST_F(StatisticsDispatcherTest, DispatchEnqueueEvents) { + EncodedFrame frame; + frame.rtp_timestamp = RtpTimeTicks(12345); + frame.frame_id = FrameId::first(); + frame.dependency = EncodedFrame::Dependency::kKeyFrame; + frame.data = ByteView(reinterpret_cast("test"), 4); + frame.capture_begin_time = clock_.now() + milliseconds(10); + frame.capture_end_time = clock_.now() + milliseconds(20); + + dispatcher_.DispatchEnqueueEvents(StreamType::kVideo, frame); + const std::vector events = collector_.TakeRecentFrameEvents(); + ASSERT_EQ(3u, events.size()); + EXPECT_EQ(events[0].type, StatisticsEvent::Type::kFrameCaptureBegin); + EXPECT_EQ(events[0].media_type, StatisticsEvent::MediaType::kVideo); + EXPECT_EQ(events[0].rtp_timestamp, frame.rtp_timestamp); + EXPECT_EQ(events[0].timestamp, frame.capture_begin_time); + + EXPECT_EQ(events[1].type, StatisticsEvent::Type::kFrameCaptureEnd); + EXPECT_EQ(events[1].media_type, StatisticsEvent::MediaType::kVideo); + EXPECT_EQ(events[1].rtp_timestamp, frame.rtp_timestamp); + EXPECT_EQ(events[1].timestamp, frame.capture_end_time); + + EXPECT_EQ(events[2].type, StatisticsEvent::Type::kFrameEncoded); + EXPECT_EQ(events[2].media_type, StatisticsEvent::MediaType::kVideo); + EXPECT_EQ(events[2].rtp_timestamp, frame.rtp_timestamp); + EXPECT_EQ(events[2].frame_id, frame.frame_id); + EXPECT_EQ(events[2].size, 4u); + EXPECT_EQ(events[2].key_frame, true); +} + +TEST_F(StatisticsDispatcherTest, DispatchFrameDropEvent) { + const FrameId frame_id = FrameId::first(); + const RtpTimeTicks rtp_timestamp(12345); + const Clock::time_point drop_time = clock_.now() + milliseconds(10); + + dispatcher_.DispatchFrameDropEvent(StreamType::kVideo, frame_id, + rtp_timestamp, drop_time); + + const std::vector events = collector_.TakeRecentFrameEvents(); + ASSERT_EQ(1u, events.size()); + + EXPECT_EQ(events[0].type, StatisticsEvent::Type::kFrameDroppedByEncoder); + EXPECT_EQ(events[0].media_type, StatisticsEvent::MediaType::kVideo); + EXPECT_EQ(events[0].frame_id, frame_id); + EXPECT_EQ(events[0].rtp_timestamp, rtp_timestamp); + EXPECT_EQ(events[0].timestamp, drop_time); +} + +TEST_F(StatisticsDispatcherTest, DispatchEnqueueEventsWithDefaultTimes) { + EncodedFrame frame; + frame.rtp_timestamp = RtpTimeTicks(12345); + frame.frame_id = FrameId::first(); + frame.dependency = EncodedFrame::Dependency::kKeyFrame; + frame.data = ByteView(reinterpret_cast("test"), 4); + + dispatcher_.DispatchEnqueueEvents(StreamType::kVideo, frame); + const std::vector events = collector_.TakeRecentFrameEvents(); + ASSERT_EQ(3u, events.size()); + + EXPECT_EQ(events[0].type, StatisticsEvent::Type::kFrameCaptureBegin); + EXPECT_EQ(events[0].media_type, StatisticsEvent::MediaType::kVideo); + EXPECT_EQ(events[0].rtp_timestamp, frame.rtp_timestamp); + EXPECT_EQ(events[0].timestamp, clock_.now()); + + EXPECT_EQ(events[1].type, StatisticsEvent::Type::kFrameCaptureEnd); + EXPECT_EQ(events[1].media_type, StatisticsEvent::MediaType::kVideo); + EXPECT_EQ(events[1].rtp_timestamp, frame.rtp_timestamp); + EXPECT_EQ(events[1].timestamp, clock_.now()); + + EXPECT_EQ(events[2].type, StatisticsEvent::Type::kFrameEncoded); + EXPECT_EQ(events[2].media_type, StatisticsEvent::MediaType::kVideo); + EXPECT_EQ(events[2].rtp_timestamp, frame.rtp_timestamp); + EXPECT_EQ(events[2].frame_id, frame.frame_id); + EXPECT_EQ(events[2].size, 4u); + EXPECT_EQ(events[2].key_frame, true); +} + +TEST_F(StatisticsDispatcherTest, DispatchAckEvent) { + const RtpTimeTicks kRtpTimestamp(54321); + const FrameId kFrameId = FrameId::first() + 1; + + dispatcher_.DispatchAckEvent(StreamType::kAudio, kRtpTimestamp, kFrameId); + const std::vector events = collector_.TakeRecentFrameEvents(); + + EXPECT_EQ(events[0].type, StatisticsEvent::Type::kFrameAckReceived); + EXPECT_EQ(events[0].media_type, StatisticsEvent::MediaType::kAudio); + EXPECT_EQ(events[0].rtp_timestamp, kRtpTimestamp); + EXPECT_EQ(events[0].frame_id, kFrameId); +} + +TEST_F(StatisticsDispatcherTest, DispatchFrameLogMessages) { + std::vector messages; + RtcpReceiverFrameLogMessage log_message; + log_message.rtp_timestamp = RtpTimeTicks(98765); + + RtcpReceiverEventLogMessage packet_received_message; + packet_received_message.type = StatisticsEvent::Type::kPacketReceived; + packet_received_message.timestamp = clock_.now() + milliseconds(5); + packet_received_message.packet_id = 10; + log_message.messages.push_back(packet_received_message); + + RtcpReceiverEventLogMessage frame_ack_sent_message; + frame_ack_sent_message.type = StatisticsEvent::Type::kFrameAckSent; + frame_ack_sent_message.timestamp = clock_.now() + milliseconds(10); + log_message.messages.push_back(frame_ack_sent_message); + + RtcpReceiverEventLogMessage frame_decoded_message; + frame_decoded_message.type = StatisticsEvent::Type::kFrameDecoded; + frame_decoded_message.timestamp = clock_.now() + milliseconds(15); + log_message.messages.push_back(frame_decoded_message); + + RtcpReceiverEventLogMessage frame_played_out_message; + frame_played_out_message.type = StatisticsEvent::Type::kFramePlayedOut; + frame_played_out_message.timestamp = clock_.now() + milliseconds(20); + frame_played_out_message.delay = milliseconds(10); + log_message.messages.push_back(frame_played_out_message); + messages.push_back(log_message); + + dispatcher_.DispatchFrameLogMessages(StreamType::kAudio, messages); + const std::vector frame_events = + collector_.TakeRecentFrameEvents(); + const std::vector packet_events = + collector_.TakeRecentPacketEvents(); + ASSERT_EQ(3u, frame_events.size()); + ASSERT_EQ(1u, packet_events.size()); + + EXPECT_EQ(packet_events[0].type, StatisticsEvent::Type::kPacketReceived); + EXPECT_EQ(packet_events[0].media_type, StatisticsEvent::MediaType::kAudio); + EXPECT_EQ(packet_events[0].rtp_timestamp, log_message.rtp_timestamp); + EXPECT_EQ(packet_events[0].packet_id, packet_received_message.packet_id); + EXPECT_EQ(packet_events[0].timestamp, packet_received_message.timestamp); + EXPECT_EQ(packet_events[0].received_timestamp, clock_.now()); + + EXPECT_EQ(frame_events[0].type, StatisticsEvent::Type::kFrameAckSent); + EXPECT_EQ(frame_events[0].media_type, StatisticsEvent::MediaType::kAudio); + EXPECT_EQ(frame_events[0].rtp_timestamp, log_message.rtp_timestamp); + EXPECT_EQ(frame_events[0].timestamp, frame_ack_sent_message.timestamp); + EXPECT_EQ(frame_events[0].received_timestamp, clock_.now()); + + EXPECT_EQ(frame_events[1].type, StatisticsEvent::Type::kFrameDecoded); + EXPECT_EQ(frame_events[1].media_type, StatisticsEvent::MediaType::kAudio); + EXPECT_EQ(frame_events[1].rtp_timestamp, log_message.rtp_timestamp); + EXPECT_EQ(frame_events[1].timestamp, frame_decoded_message.timestamp); + EXPECT_EQ(frame_events[1].received_timestamp, clock_.now()); + + EXPECT_EQ(frame_events[2].type, StatisticsEvent::Type::kFramePlayedOut); + EXPECT_EQ(frame_events[2].media_type, StatisticsEvent::MediaType::kAudio); + EXPECT_EQ(frame_events[2].rtp_timestamp, log_message.rtp_timestamp); + EXPECT_EQ(frame_events[2].timestamp, frame_played_out_message.timestamp); + EXPECT_EQ(frame_events[2].received_timestamp, clock_.now()); + EXPECT_EQ(frame_events[2].delay_delta, frame_played_out_message.delay); +} + +TEST_F(StatisticsDispatcherTest, DispatchFrameLogMessagesWithUnknownEventType) { + std::vector messages; + RtcpReceiverFrameLogMessage log_message; + log_message.rtp_timestamp = RtpTimeTicks(98765); + + RtcpReceiverEventLogMessage unknown_event_message; + unknown_event_message.type = StatisticsEvent::Type::kUnknown; + unknown_event_message.timestamp = clock_.now() + milliseconds(5); + log_message.messages.push_back(unknown_event_message); + + messages.push_back(log_message); + + dispatcher_.DispatchFrameLogMessages(StreamType::kAudio, messages); + + const std::vector frame_events = + collector_.TakeRecentFrameEvents(); + const std::vector packet_events = + collector_.TakeRecentPacketEvents(); + EXPECT_EQ(0u, frame_events.size()); + EXPECT_EQ(0u, packet_events.size()); +} + +} // namespace +} // namespace openscreen::cast diff --git a/cast/streaming/input.proto b/cast/streaming/input.proto new file mode 100644 index 000000000..f5d98edc3 --- /dev/null +++ b/cast/streaming/input.proto @@ -0,0 +1,184 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto3"; + +package openscreen.cast; + +option optimize_for = LITE_RUNTIME; + +// Messages used for sending input events (like touch, mouse, key) between a +// Cast Sender and a Cast Receiver. This allows for interactive applications +// where the user interacts with one device, and those interactions are +// forwarded to the other device. +message InputMessage { + enum InputType { + INPUT_TYPE_KEY_DOWN = 0; + INPUT_TYPE_KEY_UP = 1; + INPUT_TYPE_MOUSE_DOWN = 2; + INPUT_TYPE_MOUSE_UP = 3; + INPUT_TYPE_MOUSE_MOVE = 4; + INPUT_TYPE_MOUSE_ENTER = 5; + INPUT_TYPE_MOUSE_LEAVE = 6; + INPUT_TYPE_MOUSE_WHEEL = 7; + INPUT_TYPE_TOUCH = 8; + } + + enum ModifierKeys { + MODIFIER_KEY_UNKNOWN = 0; + MODIFIER_KEY_ALT = 1; + MODIFIER_KEY_CTRL = 2; + MODIFIER_KEY_SHIFT = 3; + MODIFIER_KEY_META = 4; + MODIFIER_KEY_CAPS_LOCK = 5; + } + + enum MouseButton { + MOUSE_BUTTON_UNKNOWN = 0; + MOUSE_BUTTON_PRIMARY = 1; + MOUSE_BUTTON_SECONDARY = 2; + MOUSE_BUTTON_AUXILIARY = 3; + MOUSE_BUTTON_BROWSER_BACK = 4; + MOUSE_BUTTON_BROWSER_FORWARD = 5; + } + + enum TouchState { + TOUCH_STATE_UNKNOWN = 0; + TOUCH_STATE_START = 1; + TOUCH_STATE_MOVE = 2; + TOUCH_STATE_END = 3; + TOUCH_STATE_CANCEL = 4; + } + + enum WheelMode { + WHEEL_MODE_UNKNOWN = 0; + WHEEL_MODE_PIXEL = 1; + WHEEL_MODE_LINE = 2; + WHEEL_MODE_PAGE = 3; + } + + // Input events use normalized [0.0, 1.0] x and y coordinates, where origin + // (0, 0) is defined as the upper left of the viewport. + // + // The "viewport" is defined as the stream of pixels shared between a Cast + // sender and receiver. + message NormalizedPoint { + // The x-coordinate, normalized to [0.0, 1.0]. + float x = 1; + // The y-coordinate, normalized to [0.0, 1.0]. + float y = 2; + } + + // A Timestamp represents a point in time independent of any time zone or + // local calendar, encoded as a count of seconds and fractions of seconds + // at nanosecond resolution. The count is relative to an epoch at UTC + // midnight on January 1, 1970, in the proleptic Gregorian calendar which + // extends the Gregorian calendar backwards to year one. + // + // Based on the google.protobuf.Timestamp type. + message Timestamp { + int64 seconds = 1; + int32 nanos = 2; + } + + message Touch { + // Unique identifier to keep track of ongoing touch events. + int32 id = 1; + + // Location of the touch event. + NormalizedPoint location = 2; + + // The state of this specific touch point. + TouchState state = 3; + } + + message TouchEvent { + // Touch objects related to this event. + repeated Touch touches = 1; + + // Keys that were being pressed when the touch event fired. + repeated ModifierKeys modifiers = 2; + } + + message MouseEvent { + // Location of the mouse event. + NormalizedPoint location = 1; + + // Normalized change in x and y coordinates. + NormalizedPoint delta = 2; + + // The buttons being pressed when the mouse event was fired. + repeated MouseButton buttons = 3; + + // The modifier keys being pressed when the mouse event fired. + repeated ModifierKeys modifiers = 4; + } + + message WheelEvent { + // Location of the wheel event. + NormalizedPoint location = 1; + + // The buttons being pressed when the wheel event was fired. + repeated MouseButton buttons = 2; + + // The modifier keys being pressed when the wheel event fired. + repeated ModifierKeys modifiers = 3; + + // Scroll deltas. + float delta_x = 4; + float delta_y = 5; + float delta_z = 6; + + // The units of the scroll delta values. + WheelMode delta_mode = 7; + } + + message KeyEvent { + // The physical key that was pressed. + // Maps to the UI Events KeyboardEvent code + // https://www.w3.org/TR/uievents-code/#code-value-tables + string key_code = 1; + + // The represented value of the key when pressed (depends on layout and caps + // lock status). + string key_value = 2; + + // Modifier keys (CTRL, ALT, SHIFT, etc.). + repeated ModifierKeys modifiers = 3; + + // True if the key is being held down and automatically repeating. + bool repeat = 4; + } + + message InputEvent { + InputType type = 1; + + // Timestamp of the system clock of the generating device. + // The receiver of this input event may wish to translate this value into + // its own system clock using a clock drift smoother. + Timestamp timestamp = 2; + + oneof event { + TouchEvent touch_event = 3; + MouseEvent mouse_event = 4; + KeyEvent key_event = 5; + WheelEvent wheel_event = 6; + } + + // ID of the device that generated this event, with `0` being a valid ID + // corresponding to the default or built-in device. Input event sources that + // do not differentiate between input devices may provide 0 for all events. + int32 device_id = 7; + } + + // Batch of input events. + repeated InputEvent events = 1; + + // All of the input events for a single input message should share the same + // viewport dimensions so that all NormalizedPoints are in the same domain. + // If a resize occurs while generating input events, it is encouraged to send + // two separate input event messages. + int32 viewport_width = 2; + int32 viewport_height = 3; +} diff --git a/cast/streaming/message_fields.h b/cast/streaming/message_fields.h index 3dc169e7a..feba973b2 100644 --- a/cast/streaming/message_fields.h +++ b/cast/streaming/message_fields.h @@ -43,6 +43,7 @@ inline constexpr char kErrorDescription[] = "description"; // Other message fields. inline constexpr char kRpcMessageBody[] = "rpc"; +inline constexpr char kInputMessageBody[] = "input"; inline constexpr char kCapabilitiesMessageBody[] = "capabilities"; inline constexpr char kStatusMessageBody[] = "status"; diff --git a/cast/streaming/offer_messages.h b/cast/streaming/offer_messages.h deleted file mode 100644 index dae04875f..000000000 --- a/cast/streaming/offer_messages.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_OFFER_MESSAGES_H_ -#define CAST_STREAMING_OFFER_MESSAGES_H_ - -#include "cast/streaming/public/offer_messages.h" - -#endif // CAST_STREAMING_OFFER_MESSAGES_H_ diff --git a/cast/streaming/public/answer_messages.cc b/cast/streaming/public/answer_messages.cc index 0bdba0afd..25235bea8 100644 --- a/cast/streaming/public/answer_messages.cc +++ b/cast/streaming/public/answer_messages.cc @@ -102,7 +102,7 @@ constexpr char kDisplay[] = "display"; // std::optional array of numbers specifying the indexes of streams that will // send event logs through RTCP. constexpr char kReceiverRtcpEventLog[] = "receiverRtcpEventLog"; -// OPtional array of numbers specifying the indexes of streams that will use +// Optional array of numbers specifying the indexes of streams that will use // DSCP values specified in the OFFER message for RTCP packets. constexpr char kReceiverRtcpDscp[] = "receiverRtcpDscp"; // If this optional field is present the receiver supports the specific @@ -118,56 +118,57 @@ Json::Value AspectRatioConstraintToJson(AspectRatioConstraint aspect_ratio) { .value(kScalingSender)); } -bool TryParseAspectRatioConstraint(const Json::Value& value, - AspectRatioConstraint* out) { +std::optional TryParseAspectRatioConstraint( + const Json::Value& value) { std::string aspect_ratio; if (!json::TryParseString(value, &aspect_ratio)) { - return false; + return std::nullopt; } ErrorOr constraint = GetEnum(kAspectRatioConstraintNames, aspect_ratio); if (constraint.is_error()) { - return false; + return std::nullopt; } - *out = constraint.value(); - return true; + return constraint.value(); } template -bool ParseOptional(const Json::Value& value, std::optional* out) { - // It's fine if the value is empty. +ErrorOr> ParseOptional(const Json::Value& value) { if (!value) { - return true; + return std::optional{}; } - T tentative_out; - if (!T::TryParse(value, &tentative_out)) { - return false; + auto out = T::TryParse(value); + if (out.is_error()) { + return out.error(); } - *out = tentative_out; - return true; + return std::optional{std::move(out.value())}; } } // namespace // static -bool AspectRatio::TryParse(const Json::Value& value, AspectRatio* out) { +ErrorOr AspectRatio::TryParse(const Json::Value& value) { std::string parsed_value; if (!json::TryParseString(value, &parsed_value)) { - return false; + return Error(Error::Code::kJsonParseError, "Invalid aspect ratio string"); } std::vector fields = string_util::Split(parsed_value, kAspectRatioDelimiter); if (fields.size() != 2) { - return false; + return Error(Error::Code::kJsonParseError, "Invalid aspect ratio format"); } - if (!string_parse::ParseAsciiNumber(fields[0], out->width) || - !string_parse::ParseAsciiNumber(fields[1], out->height)) { - return false; + AspectRatio out; + if (!string_parse::ParseAsciiNumber(fields[0], out.width) || + !string_parse::ParseAsciiNumber(fields[1], out.height)) { + return Error(Error::Code::kJsonParseError, "Invalid aspect ratio values"); } - return out->IsValid(); + if (!out.IsValid()) { + return Error(Error::Code::kJsonParseError, "Invalid aspect ratio"); + } + return out; } bool AspectRatio::IsValid() const { @@ -175,23 +176,31 @@ bool AspectRatio::IsValid() const { } // static -bool AudioConstraints::TryParse(const Json::Value& root, - AudioConstraints* out) { - if (!json::TryParseInt(root[kMaxSampleRate], &(out->max_sample_rate)) || - !json::TryParseInt(root[kMaxChannels], &(out->max_channels)) || - !json::TryParseInt(root[kMaxBitRate], &(out->max_bit_rate))) { - return false; +ErrorOr AudioConstraints::TryParse(const Json::Value& root) { + if (!root.isObject()) { + return Error(Error::Code::kJsonParseError, + "Audio constraints is not a JSON object"); + } + + AudioConstraints out; + if (!json::TryParseInt(root[kMaxSampleRate], &out.max_sample_rate) || + !json::TryParseInt(root[kMaxChannels], &out.max_channels) || + !json::TryParseInt(root[kMaxBitRate], &out.max_bit_rate)) { + return Error(Error::Code::kJsonParseError, "Invalid audio constraints"); } std::chrono::milliseconds max_delay; if (json::TryParseMilliseconds(root[kMaxDelay], &max_delay)) { - out->max_delay = max_delay; + out.max_delay = max_delay; } - if (!json::TryParseInt(root[kMinBitRate], &(out->min_bit_rate))) { - out->min_bit_rate = kDefaultAudioMinBitRate; + if (!json::TryParseInt(root[kMinBitRate], &out.min_bit_rate)) { + out.min_bit_rate = kDefaultAudioMinBitRate; + } + if (!out.IsValid()) { + return Error(Error::Code::kJsonParseError, "Invalid audio constraints"); } - return out->IsValid(); + return out; } Json::Value AudioConstraints::ToJson() const { @@ -213,29 +222,48 @@ bool AudioConstraints::IsValid() const { } // static -bool VideoConstraints::TryParse(const Json::Value& root, - VideoConstraints* out) { - if (!Dimensions::TryParse(root[kMaxDimensions], &(out->max_dimensions)) || - !json::TryParseInt(root[kMaxBitRate], &(out->max_bit_rate)) || - !ParseOptional(root[kMinResolution], - &(out->min_resolution))) { - return false; +ErrorOr VideoConstraints::TryParse(const Json::Value& root) { + if (!root.isObject()) { + return Error(Error::Code::kJsonParseError, + "Video constraints is not a JSON object"); + } + + VideoConstraints out; + + auto max_dimensions = Dimensions::TryParse(root[kMaxDimensions]); + if (max_dimensions.is_error()) { + return max_dimensions.error(); + } + out.max_dimensions = std::move(max_dimensions.value()); + + if (!json::TryParseInt(root[kMaxBitRate], &out.max_bit_rate)) { + return Error(Error::Code::kJsonParseError, + "Invalid video constraints: missing maxBitRate"); } + auto min_resolution = ParseOptional(root[kMinResolution]); + if (min_resolution.is_error()) { + return min_resolution.error(); + } + out.min_resolution = std::move(min_resolution.value()); + std::chrono::milliseconds max_delay; if (json::TryParseMilliseconds(root[kMaxDelay], &max_delay)) { - out->max_delay = max_delay; + out.max_delay = max_delay; } double max_pixels_per_second; if (json::TryParseDouble(root[kMaxPixelsPerSecond], &max_pixels_per_second)) { - out->max_pixels_per_second = max_pixels_per_second; + out.max_pixels_per_second = max_pixels_per_second; } - if (!json::TryParseInt(root[kMinBitRate], &(out->min_bit_rate))) { - out->min_bit_rate = kDefaultVideoMinBitRate; + if (!json::TryParseInt(root[kMinBitRate], &out.min_bit_rate)) { + out.min_bit_rate = kDefaultVideoMinBitRate; + } + if (!out.IsValid()) { + return Error(Error::Code::kJsonParseError, "Invalid video constraints"); } - return out->IsValid(); + return out; } bool VideoConstraints::IsValid() const { @@ -268,12 +296,30 @@ Json::Value VideoConstraints::ToJson() const { } // static -bool Constraints::TryParse(const Json::Value& root, Constraints* out) { - if (!AudioConstraints::TryParse(root[kAudio], &(out->audio)) || - !VideoConstraints::TryParse(root[kVideo], &(out->video))) { - return false; +ErrorOr Constraints::TryParse(const Json::Value& root) { + if (!root.isObject()) { + return Error(Error::Code::kJsonParseError, + "Constraints is not a JSON object"); } - return out->IsValid(); + + Constraints out; + + auto audio = AudioConstraints::TryParse(root[kAudio]); + if (audio.is_error()) { + return audio.error(); + } + out.audio = std::move(audio.value()); + + auto video = VideoConstraints::TryParse(root[kVideo]); + if (video.is_error()) { + return video.error(); + } + out.video = std::move(video.value()); + + if (!out.IsValid()) { + return Error(Error::Code::kJsonParseError, "Invalid constraints"); + } + return out; } bool Constraints::IsValid() const { @@ -289,22 +335,38 @@ Json::Value Constraints::ToJson() const { } // static -bool DisplayDescription::TryParse(const Json::Value& root, - DisplayDescription* out) { - if (!ParseOptional(root[kDimensions], &(out->dimensions)) || - !ParseOptional(root[kAspectRatio], &(out->aspect_ratio))) { - return false; +ErrorOr DisplayDescription::TryParse( + const Json::Value& root) { + if (!root.isObject()) { + return Error(Error::Code::kJsonParseError, + "Display description is not a JSON object"); } - AspectRatioConstraint constraint; - if (TryParseAspectRatioConstraint(root[kScaling], &constraint)) { - out->aspect_ratio_constraint = - std::optional(std::move(constraint)); + DisplayDescription out; + + auto dimensions = ParseOptional(root[kDimensions]); + if (dimensions.is_error()) { + return dimensions.error(); + } + out.dimensions = std::move(dimensions.value()); + + auto aspect_ratio = ParseOptional(root[kAspectRatio]); + if (aspect_ratio.is_error()) { + return aspect_ratio.error(); + } + out.aspect_ratio = std::move(aspect_ratio.value()); + + auto constraint = TryParseAspectRatioConstraint(root[kScaling]); + if (constraint.has_value()) { + out.aspect_ratio_constraint = constraint.value(); } else { - out->aspect_ratio_constraint = std::nullopt; + out.aspect_ratio_constraint = std::nullopt; } - return out->IsValid(); + if (!out.IsValid()) { + return Error(Error::Code::kJsonParseError, "Invalid display description"); + } + return out; } bool DisplayDescription::IsValid() const { @@ -334,7 +396,7 @@ Json::Value DisplayDescription::ToJson() const { Json::Value root; if (aspect_ratio.has_value()) { root[kAspectRatio] = - StringPrintf("%d%c%d", aspect_ratio->width, kAspectRatioDelimiter, + StringFormat("{}{}{}", aspect_ratio->width, kAspectRatioDelimiter, aspect_ratio->height); } if (dimensions.has_value()) { @@ -347,23 +409,42 @@ Json::Value DisplayDescription::ToJson() const { return root; } -bool Answer::TryParse(const Json::Value& root, Answer* out) { - if (!json::TryParseInt(root[kUdpPort], &(out->udp_port)) || - !json::TryParseIntArray(root[kSendIndexes], &(out->send_indexes)) || - !json::TryParseUintArray(root[kSsrcs], &(out->ssrcs)) || - !ParseOptional(root[kConstraints], &(out->constraints)) || - !ParseOptional(root[kDisplay], &(out->display))) { - return false; +ErrorOr Answer::TryParse(const Json::Value& root) { + if (!root.isObject()) { + return Error(Error::Code::kJsonParseError, "Answer is not a JSON object"); + } + + Answer out; + if (!json::TryParseInt(root[kUdpPort], &out.udp_port) || + !json::TryParseIntArray(root[kSendIndexes], &out.send_indexes) || + !json::TryParseUintArray(root[kSsrcs], &out.ssrcs)) { + return Error(Error::Code::kJsonParseError, + "Invalid answer: missing or invalid mandatory fields"); } - // These function set to empty array if not present, so we can ignore + auto constraints = ParseOptional(root[kConstraints]); + if (constraints.is_error()) { + return constraints.error(); + } + out.constraints = std::move(constraints.value()); + + auto display = ParseOptional(root[kDisplay]); + if (display.is_error()) { + return display.error(); + } + out.display = std::move(display.value()); + + // These functions set to empty array if not present, so we can ignore // the return value for optional values. json::TryParseIntArray(root[kReceiverRtcpEventLog], - &(out->receiver_rtcp_event_log)); - json::TryParseIntArray(root[kReceiverRtcpDscp], &(out->receiver_rtcp_dscp)); - json::TryParseStringArray(root[kRtpExtensions], &(out->rtp_extensions)); + &out.receiver_rtcp_event_log); + json::TryParseIntArray(root[kReceiverRtcpDscp], &out.receiver_rtcp_dscp); + json::TryParseNestedStringArray(root[kRtpExtensions], &out.rtp_extensions); - return out->IsValid(); + if (!out.IsValid()) { + return Error(Error::Code::kJsonParseError, "Invalid answer"); + } + return out; } bool Answer::IsValid() const { @@ -409,7 +490,7 @@ Json::Value Answer::ToJson() const { root[kReceiverRtcpDscp] = json::PrimitiveVectorToJson(receiver_rtcp_dscp); } if (!rtp_extensions.empty()) { - root[kRtpExtensions] = json::PrimitiveVectorToJson(rtp_extensions); + root[kRtpExtensions] = json::NestedStringArrayToJson(rtp_extensions); } return root; } diff --git a/cast/streaming/public/answer_messages.h b/cast/streaming/public/answer_messages.h index 50199c1e0..00949f6d3 100644 --- a/cast/streaming/public/answer_messages.h +++ b/cast/streaming/public/answer_messages.h @@ -35,7 +35,7 @@ namespace openscreen::cast { // (3) IsValid. Used by both TryParse and ToJson to ensure that the // object is in a good state. struct AudioConstraints { - static bool TryParse(const Json::Value& value, AudioConstraints* out); + static ErrorOr TryParse(const Json::Value& value); Json::Value ToJson() const; bool IsValid() const; @@ -47,7 +47,7 @@ struct AudioConstraints { }; struct VideoConstraints { - static bool TryParse(const Json::Value& value, VideoConstraints* out); + static ErrorOr TryParse(const Json::Value& value); Json::Value ToJson() const; bool IsValid() const; @@ -60,7 +60,7 @@ struct VideoConstraints { }; struct Constraints { - static bool TryParse(const Json::Value& value, Constraints* out); + static ErrorOr TryParse(const Json::Value& value); Json::Value ToJson() const; bool IsValid() const; @@ -74,7 +74,7 @@ struct Constraints { enum class AspectRatioConstraint : uint8_t { kVariable = 0, kFixed }; struct AspectRatio { - static bool TryParse(const Json::Value& value, AspectRatio* out); + static ErrorOr TryParse(const Json::Value& value); bool IsValid() const; bool operator==(const AspectRatio& other) const { @@ -86,7 +86,7 @@ struct AspectRatio { }; struct DisplayDescription { - static bool TryParse(const Json::Value& value, DisplayDescription* out); + static ErrorOr TryParse(const Json::Value& value); Json::Value ToJson() const; bool IsValid() const; @@ -98,7 +98,7 @@ struct DisplayDescription { }; struct Answer { - static bool TryParse(const Json::Value& value, Answer* out); + static ErrorOr TryParse(const Json::Value& value); Json::Value ToJson() const; bool IsValid() const; @@ -114,7 +114,7 @@ struct Answer { std::vector receiver_rtcp_dscp; // RTP extensions should be empty, but not null. - std::vector rtp_extensions = {}; + std::vector> rtp_extensions = {}; }; } // namespace openscreen::cast diff --git a/cast/streaming/public/constants.h b/cast/streaming/public/constants.h index 5eb7d8a90..f4e37368e 100644 --- a/cast/streaming/public/constants.h +++ b/cast/streaming/public/constants.h @@ -96,6 +96,9 @@ inline constexpr std::chrono::milliseconds kDefaultMaxDelayMs(1500); // to 3. inline constexpr int kSupportedRemotingVersion = 2; +// Used for RTCP message support. +constexpr uint32_t kCastName = ('C' << 24) + ('A' << 16) + ('S' << 8) + 'T'; + // Codecs known and understood by cast senders and receivers. Note: receivers // are required to implement the following codecs to be Cast V2 compliant: H264, // VP8, AAC, Opus. Senders have to implement at least one codec from this diff --git a/cast/streaming/public/encoded_frame.h b/cast/streaming/public/encoded_frame.h index c02395a96..c38e47151 100644 --- a/cast/streaming/public/encoded_frame.h +++ b/cast/streaming/public/encoded_frame.h @@ -13,7 +13,6 @@ #include "cast/streaming/public/frame_id.h" #include "cast/streaming/rtp_time.h" #include "platform/api/time.h" -#include "platform/base/macros.h" #include "platform/base/span.h" namespace openscreen::cast { @@ -58,10 +57,11 @@ struct EncodedFrame { std::chrono::milliseconds new_playout_delay, ByteView data); EncodedFrame(); - ~EncodedFrame(); - + EncodedFrame(const EncodedFrame&) = delete; + EncodedFrame& operator=(const EncodedFrame&) = delete; EncodedFrame(EncodedFrame&&) noexcept; EncodedFrame& operator=(EncodedFrame&&); + ~EncodedFrame(); // Copies all members except `data` to `dest`. Does not modify |dest->data|. void CopyMetadataTo(EncodedFrame* dest) const; @@ -112,8 +112,6 @@ struct EncodedFrame { // context, this points to the data to be sent. In the receiver context, this // is set to the region of a client-provided buffer that was populated. ByteView data; - - OSP_DISALLOW_COPY_AND_ASSIGN(EncodedFrame); }; } // namespace openscreen::cast diff --git a/cast/streaming/public/environment.cc b/cast/streaming/public/environment.cc index 03cab93e5..e068ea26f 100644 --- a/cast/streaming/public/environment.cc +++ b/cast/streaming/public/environment.cc @@ -93,6 +93,12 @@ int Environment::GetMaxPacketSize() const { } } +void Environment::SetDscp(UdpSocket::DscpMode mode) { + if (socket_) { + socket_->SetDscp(mode); + } +} + void Environment::SendPacket(ByteView packet, PacketMetadata metadata) { OSP_CHECK(remote_endpoint_.address); OSP_CHECK_NE(remote_endpoint_.port, 0); diff --git a/cast/streaming/public/environment.h b/cast/streaming/public/environment.h index 124a78ae3..d30cf99cd 100644 --- a/cast/streaming/public/environment.h +++ b/cast/streaming/public/environment.h @@ -121,6 +121,9 @@ class Environment : public UdpSocket::Client { // value of at least kRequiredNetworkPacketSize. int GetMaxPacketSize() const; + // Sets the DSCP value for the underlying UDP socket. + void SetDscp(UdpSocket::DscpMode mode); + // Sends the given `packet` to the remote endpoint, best-effort. // set_remote_endpoint() must be called beforehand with a valid IPEndpoint. // diff --git a/cast/streaming/public/frame_id.h b/cast/streaming/public/frame_id.h index db3384a01..f17a0ad58 100644 --- a/cast/streaming/public/frame_id.h +++ b/cast/streaming/public/frame_id.h @@ -59,37 +59,37 @@ class FrameId : public ExpandedValueBase { } // Operators to compute advancement by incremental amounts. - FrameId operator+(int64_t rhs) const { + constexpr FrameId operator+(int64_t rhs) const { OSP_CHECK(!is_null()); return FrameId(value_ + rhs); } - FrameId operator-(int64_t rhs) const { + constexpr FrameId operator-(int64_t rhs) const { OSP_CHECK(!is_null()); return FrameId(value_ - rhs); } - FrameId& operator+=(int64_t rhs) { + constexpr FrameId& operator+=(int64_t rhs) { OSP_CHECK(!is_null()); return (*this = (*this + rhs)); } - FrameId& operator-=(int64_t rhs) { + constexpr FrameId& operator-=(int64_t rhs) { OSP_CHECK(!is_null()); return (*this = (*this - rhs)); } - FrameId& operator++() { + constexpr FrameId& operator++() { OSP_CHECK(!is_null()); ++value_; return *this; } - FrameId& operator--() { + constexpr FrameId& operator--() { OSP_CHECK(!is_null()); --value_; return *this; } - FrameId operator++(int) { + constexpr FrameId operator++(int) { OSP_CHECK(!is_null()); return FrameId(value_++); } - FrameId operator--(int) { + constexpr FrameId operator--(int) { OSP_CHECK(!is_null()); return FrameId(value_--); } diff --git a/cast/streaming/public/offer_messages.cc b/cast/streaming/public/offer_messages.cc index e3149a4be..0efe0255b 100644 --- a/cast/streaming/public/offer_messages.cc +++ b/cast/streaming/public/offer_messages.cc @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -31,23 +32,24 @@ constexpr char kAudioSourceType[] = "audio_source"; constexpr char kVideoSourceType[] = "video_source"; constexpr char kStreamType[] = "type"; -bool CodecParameterIsValid(VideoCodec codec, - const std::string& codec_parameter) { - if (codec_parameter.empty()) { +[[nodiscard]] constexpr bool CodecParameterIsValid(VideoCodec codec, + std::string_view parameter) { + if (parameter.empty()) { return true; } switch (codec) { - case VideoCodec::kVp8: - return string_util::starts_with(codec_parameter, "vp08"); - case VideoCodec::kVp9: - return string_util::starts_with(codec_parameter, "vp09"); - case VideoCodec::kAv1: - return string_util::starts_with(codec_parameter, "av01"); - case VideoCodec::kHevc: - return string_util::starts_with(codec_parameter, "hev1"); - case VideoCodec::kH264: - return string_util::starts_with(codec_parameter, "avc1"); - case VideoCodec::kNotSpecified: + using enum VideoCodec; + case kVp8: + return parameter.starts_with("vp08"); + case kVp9: + return parameter.starts_with("vp09"); + case kAv1: + return parameter.starts_with("av01"); + case kHevc: + return parameter.starts_with("hev1"); + case kH264: + return parameter.starts_with("avc1"); + case kNotSpecified: return false; } OSP_NOTREACHED(); @@ -60,7 +62,7 @@ bool CodecParameterIsValid(AudioCodec codec, } switch (codec) { case AudioCodec::kAac: - return string_util::starts_with(codec_parameter, "mp4a."); + return codec_parameter.starts_with("mp4a."); // Opus doesn't use codec parameters. case AudioCodec::kOpus: // fallthrough @@ -157,12 +159,12 @@ bool TryParseResolutions(const Json::Value& value, } for (Json::ArrayIndex i = 0; i < value.size(); ++i) { - Resolution resolution; - if (!Resolution::TryParse(value[i], &resolution)) { + auto resolution = Resolution::TryParse(value[i]); + if (resolution.is_error()) { out->clear(); return false; } - out->push_back(std::move(resolution)); + out->push_back(std::move(resolution.value())); } return true; @@ -170,54 +172,66 @@ bool TryParseResolutions(const Json::Value& value, } // namespace -Error Stream::TryParse(const Json::Value& value, - Stream::Type type, - Stream* out) { - out->type = type; +ErrorOr Stream::TryParse(const Json::Value& value, Stream::Type type) { + if (!value.isObject()) { + return Error(Error::Code::kJsonParseError, "Stream is not a JSON object"); + } + + Stream out; + out.type = type; - if (!json::TryParseInt(value["index"], &out->index) || - !json::TryParseUint(value["ssrc"], &out->ssrc) || - !TryParseRtpPayloadType(value["rtpPayloadType"], - &out->rtp_payload_type) || - !TryParseRtpTimebase(value["timeBase"], &out->rtp_timebase)) { + if (!json::TryParseInt(value["index"], &out.index) || + !json::TryParseUint(value["ssrc"], &out.ssrc) || + !TryParseRtpPayloadType(value["rtpPayloadType"], &out.rtp_payload_type) || + !TryParseRtpTimebase(value["timeBase"], &out.rtp_timebase)) { return Error(Error::Code::kJsonParseError, "Offer stream has missing or invalid mandatory field"); } - if (!json::TryParseInt(value["channels"], &out->channels)) { - out->channels = out->type == Stream::Type::kAudioSource - ? kDefaultNumAudioChannels - : kDefaultNumVideoChannels; - } else if (out->channels <= 0) { + if (!json::TryParseInt(value["channels"], &out.channels)) { + out.channels = out.type == Stream::Type::kAudioSource + ? kDefaultNumAudioChannels + : kDefaultNumVideoChannels; + } else if (out.channels <= 0) { return Error(Error::Code::kJsonParseError, "Invalid channel count"); } - if (!TryParseAesHexBytes(value["aesKey"], &out->aes_key) || - !TryParseAesHexBytes(value["aesIvMask"], &out->aes_iv_mask)) { + if (!TryParseAesHexBytes(value["aesKey"], &out.aes_key) || + !TryParseAesHexBytes(value["aesIvMask"], &out.aes_iv_mask)) { return Error(Error::Code::kUnencryptedOffer, "Offer stream must have both a valid aesKey and aesIvMask"); } - if (out->rtp_timebase < + if (out.rtp_timebase < std::min(kDefaultAudioMinSampleRate, kRtpVideoTimebase) || - out->rtp_timebase > kRtpVideoTimebase) { + out.rtp_timebase > kRtpVideoTimebase) { return Error(Error::Code::kJsonParseError, "rtp_timebase (sample rate)"); } - out->target_delay = kDefaultTargetPlayoutDelay; + out.target_delay = kDefaultTargetPlayoutDelay; int target_delay; if (json::TryParseInt(value["targetDelay"], &target_delay)) { auto d = std::chrono::milliseconds(target_delay); if (kMinTargetPlayoutDelay <= d && d <= kMaxTargetPlayoutDelay) { - out->target_delay = d; + out.target_delay = d; } } json::TryParseBool(value["receiverRtcpEventLog"], - &out->receiver_rtcp_event_log); - json::TryParseString(value["receiverRtcpDscp"], &out->receiver_rtcp_dscp); - json::TryParseString(value["codecParameter"], &out->codec_parameter); + &out.receiver_rtcp_event_log); + int dscp_value; + if (json::TryParseInt(value["receiverRtcpDscp"], &dscp_value)) { + // DSCP values are clamped to [0, 63]. + if (dscp_value < 0 || dscp_value > 63) { + return Error(Error::Code::kJsonParseError, + "receiverRtcpDscp (invalid DSCP value)"); + } + out.receiver_rtcp_dscp = dscp_value; + } - return Error::None(); + json::TryParseStringArray(value["rtpExtensions"], &out.rtp_extensions); + json::TryParseString(value["codecParameter"], &out.codec_parameter); + + return out; } Json::Value Stream::ToJson() const { @@ -238,9 +252,14 @@ Json::Value Stream::ToJson() const { root["aesKey"] = HexEncode(aes_key.data(), aes_key.size()); root["aesIvMask"] = HexEncode(aes_iv_mask.data(), aes_iv_mask.size()); root["receiverRtcpEventLog"] = receiver_rtcp_event_log; - root["receiverRtcpDscp"] = receiver_rtcp_dscp; + if (receiver_rtcp_dscp.has_value()) { + root["receiverRtcpDscp"] = receiver_rtcp_dscp.value(); + } root["timeBase"] = "1/" + std::to_string(rtp_timebase); root["codecParameter"] = codec_parameter; + if (!rtp_extensions.empty()) { + root["rtpExtensions"] = json::PrimitiveVectorToJson(rtp_extensions); + } return root; } @@ -250,16 +269,22 @@ bool Stream::IsValid() const { rtp_timebase >= 1; } -Error AudioStream::TryParse(const Json::Value& value, AudioStream* out) { - Error error = - Stream::TryParse(value, Stream::Type::kAudioSource, &out->stream); - if (!error.ok()) { - return error; +ErrorOr AudioStream::TryParse(const Json::Value& value) { + if (!value.isObject()) { + return Error(Error::Code::kJsonParseError, + "Audio stream is not a JSON object"); } + auto stream_or_error = Stream::TryParse(value, Stream::Type::kAudioSource); + if (stream_or_error.is_error()) { + return stream_or_error.error(); + } + + AudioStream out; + out.stream = std::move(stream_or_error.value()); + std::string codec_name; - if (!json::TryParseInt(value["bitRate"], &out->bit_rate) || - out->bit_rate < 0 || + if (!json::TryParseInt(value["bitRate"], &out.bit_rate) || out.bit_rate < 0 || !json::TryParseString(value[kCodecName], &codec_name)) { return Error(Error::Code::kJsonParseError, "Invalid audio stream field"); } @@ -268,14 +293,14 @@ Error AudioStream::TryParse(const Json::Value& value, AudioStream* out) { return Error(Error::Code::kUnknownCodec, "Codec is not known, can't use stream"); } - out->codec = codec.value(); - if (!CodecParameterIsValid(codec.value(), out->stream.codec_parameter)) { + out.codec = codec.value(); + if (!CodecParameterIsValid(codec.value(), out.stream.codec_parameter)) { return Error(Error::Code::kInvalidCodecParameter, - StringPrintf("Invalid audio codec parameter (%s for codec %s)", - out->stream.codec_parameter.c_str(), + StringFormat("Invalid audio codec parameter ({} for codec {})", + out.stream.codec_parameter.c_str(), CodecToString(codec.value()))); } - return Error::None(); + return out; } Json::Value AudioStream::ToJson() const { @@ -291,13 +316,20 @@ bool AudioStream::IsValid() const { return bit_rate >= 0 && stream.IsValid(); } -Error VideoStream::TryParse(const Json::Value& value, VideoStream* out) { - Error error = - Stream::TryParse(value, Stream::Type::kVideoSource, &out->stream); - if (!error.ok()) { - return error; +ErrorOr VideoStream::TryParse(const Json::Value& value) { + if (!value.isObject()) { + return Error(Error::Code::kJsonParseError, + "Video stream is not a JSON object"); + } + + auto stream_or_error = Stream::TryParse(value, Stream::Type::kVideoSource); + if (stream_or_error.is_error()) { + return stream_or_error.error(); } + VideoStream out; + out.stream = std::move(stream_or_error.value()); + std::string codec_name; if (!json::TryParseString(value[kCodecName], &codec_name)) { return Error(Error::Code::kJsonParseError, "Video stream missing codec"); @@ -307,33 +339,33 @@ Error VideoStream::TryParse(const Json::Value& value, VideoStream* out) { return Error(Error::Code::kUnknownCodec, "Codec is not known, can't use stream"); } - out->codec = codec.value(); - if (!CodecParameterIsValid(codec.value(), out->stream.codec_parameter)) { + out.codec = codec.value(); + if (!CodecParameterIsValid(codec.value(), out.stream.codec_parameter)) { return Error(Error::Code::kInvalidCodecParameter, - StringPrintf("Invalid video codec parameter (%s for codec %s)", - out->stream.codec_parameter.c_str(), + StringFormat("Invalid video codec parameter ({} for codec {})", + out.stream.codec_parameter.c_str(), CodecToString(codec.value()))); } - out->max_frame_rate = SimpleFraction{kDefaultMaxFrameRate, 1}; + out.max_frame_rate = SimpleFraction{kDefaultMaxFrameRate, 1}; std::string raw_max_frame_rate; if (json::TryParseString(value["maxFrameRate"], &raw_max_frame_rate)) { auto parsed = SimpleFraction::FromString(raw_max_frame_rate); if (parsed.is_value() && parsed.value().is_positive()) { - out->max_frame_rate = parsed.value(); + out.max_frame_rate = parsed.value(); } } - TryParseResolutions(value["resolutions"], &out->resolutions); - json::TryParseString(value["profile"], &out->profile); - json::TryParseString(value["protection"], &out->protection); - json::TryParseString(value["level"], &out->level); - json::TryParseString(value["errorRecoveryMode"], &out->error_recovery_mode); - if (!json::TryParseInt(value["maxBitRate"], &out->max_bit_rate)) { - out->max_bit_rate = 4 << 20; + TryParseResolutions(value["resolutions"], &out.resolutions); + json::TryParseString(value["profile"], &out.profile); + json::TryParseString(value["protection"], &out.protection); + json::TryParseString(value["level"], &out.level); + json::TryParseString(value["errorRecoveryMode"], &out.error_recovery_mode); + if (!json::TryParseInt(value["maxBitRate"], &out.max_bit_rate)) { + out.max_bit_rate = 4 << 20; } - return Error::None(); + return out; } Json::Value VideoStream::ToJson() const { @@ -361,7 +393,7 @@ bool VideoStream::IsValid() const { } // static -Error Offer::TryParse(const Json::Value& root, Offer* out) { +ErrorOr Offer::TryParse(const Json::Value& root) { if (!root.isObject()) { return Error(Error::Code::kJsonParseError, "null offer"); } @@ -374,6 +406,9 @@ Error Offer::TryParse(const Json::Value& root, Offer* out) { std::vector audio_streams; std::vector video_streams; + + using Dscp = std::optional; + std::optional receiver_rtcp_dscp; for (Json::ArrayIndex i = 0; i < supported_streams.size(); ++i) { const Json::Value& fields = supported_streams[i]; std::string type; @@ -381,18 +416,34 @@ Error Offer::TryParse(const Json::Value& root, Offer* out) { return Error(Error::Code::kJsonParseError, "Missing stream type"); } - Error error; + Error error = Error::None(); if (type == kAudioSourceType) { - AudioStream stream; - error = AudioStream::TryParse(fields, &stream); - if (error.ok()) { + auto stream_or_error = AudioStream::TryParse(fields); + if (stream_or_error.is_value()) { + auto stream = std::move(stream_or_error.value()); + if (!receiver_rtcp_dscp) { + receiver_rtcp_dscp.emplace(stream.stream.receiver_rtcp_dscp); + } else if (stream.stream.receiver_rtcp_dscp != *receiver_rtcp_dscp) { + return Error(Error::Code::kJsonParseError, + "Mixed DSCP values in offer"); + } audio_streams.push_back(std::move(stream)); + } else { + error = stream_or_error.error(); } } else if (type == kVideoSourceType) { - VideoStream stream; - error = VideoStream::TryParse(fields, &stream); - if (error.ok()) { + auto stream_or_error = VideoStream::TryParse(fields); + if (stream_or_error.is_value()) { + auto stream = std::move(stream_or_error.value()); + if (!receiver_rtcp_dscp) { + receiver_rtcp_dscp.emplace(stream.stream.receiver_rtcp_dscp); + } else if (stream.stream.receiver_rtcp_dscp != *receiver_rtcp_dscp) { + return Error(Error::Code::kJsonParseError, + "Mixed DSCP values in offer"); + } video_streams.push_back(std::move(stream)); + } else { + error = stream_or_error.error(); } } @@ -406,9 +457,8 @@ Error Offer::TryParse(const Json::Value& root, Offer* out) { } } - *out = Offer{cast_mode.value(CastMode::kMirroring), std::move(audio_streams), + return Offer{cast_mode.value(CastMode::kMirroring), std::move(audio_streams), std::move(video_streams)}; - return Error::None(); } Json::Value Offer::ToJson() const { @@ -429,9 +479,9 @@ Json::Value Offer::ToJson() const { } bool Offer::IsValid() const { - return std::all_of(audio_streams.begin(), audio_streams.end(), - [](const AudioStream& a) { return a.IsValid(); }) && - std::all_of(video_streams.begin(), video_streams.end(), - [](const VideoStream& v) { return v.IsValid(); }); + return std::ranges::all_of( + audio_streams, [](const AudioStream& a) { return a.IsValid(); }) && + std::ranges::all_of(video_streams, + [](const VideoStream& v) { return v.IsValid(); }); } } // namespace openscreen::cast diff --git a/cast/streaming/public/offer_messages.h b/cast/streaming/public/offer_messages.h index 98dd9031d..042f7ff10 100644 --- a/cast/streaming/public/offer_messages.h +++ b/cast/streaming/public/offer_messages.h @@ -10,8 +10,8 @@ #include #include "cast/streaming/impl/rtp_defines.h" -#include "cast/streaming/impl/session_config.h" #include "cast/streaming/message_fields.h" +#include "cast/streaming/public/session_config.h" #include "cast/streaming/resolution.h" #include "json/value.h" #include "platform/base/error.h" @@ -43,9 +43,7 @@ inline constexpr int kDefaultNumAudioChannels = 2; struct Stream { enum class Type : uint8_t { kAudioSource, kVideoSource }; - static Error TryParse(const Json::Value& root, - Stream::Type type, - Stream* out); + static ErrorOr TryParse(const Json::Value& root, Stream::Type type); Json::Value ToJson() const; bool IsValid() const; @@ -62,17 +60,22 @@ struct Stream { // must be converted to a 16 digit byte array. std::array aes_key = {}; std::array aes_iv_mask = {}; - bool receiver_rtcp_event_log = false; - std::string receiver_rtcp_dscp; + + // The event logs are generally recommended for use in gathering statistics + // for the sender session. + bool receiver_rtcp_event_log = true; + std::optional receiver_rtcp_dscp; int rtp_timebase = 0; // The codec parameter field honors the format laid out in RFC 6381: // https://datatracker.ietf.org/doc/html/rfc6381. std::string codec_parameter; + + std::vector rtp_extensions; }; struct AudioStream { - static Error TryParse(const Json::Value& root, AudioStream* out); + static ErrorOr TryParse(const Json::Value& root); Json::Value ToJson() const; bool IsValid() const; @@ -82,7 +85,7 @@ struct AudioStream { }; struct VideoStream { - static Error TryParse(const Json::Value& root, VideoStream* out); + static ErrorOr TryParse(const Json::Value& root); Json::Value ToJson() const; bool IsValid() const; @@ -98,7 +101,7 @@ struct VideoStream { }; struct Offer { - static Error TryParse(const Json::Value& root, Offer* out); + static ErrorOr TryParse(const Json::Value& root); Json::Value ToJson() const; bool IsValid() const; diff --git a/cast/streaming/public/protobuf_messenger.h b/cast/streaming/public/protobuf_messenger.h new file mode 100644 index 000000000..c8e81465f --- /dev/null +++ b/cast/streaming/public/protobuf_messenger.h @@ -0,0 +1,89 @@ +// Copyright 2026 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CAST_STREAMING_PUBLIC_PROTOBUF_MESSENGER_H_ +#define CAST_STREAMING_PUBLIC_PROTOBUF_MESSENGER_H_ + +#include +#include +#include +#include +#include + +#include "google/protobuf/message_lite.h" +#include "platform/base/span.h" +#include "util/osp_logging.h" + +namespace openscreen::cast { + +// This class handles the common tasks of parsing and serializing protobuf +// messages for Cast Streaming messengers. +template + requires std::is_base_of_v<::google::protobuf::MessageLite, T> +class ProtobufMessenger { + public: + using SendMessageCallback = std::function)>; + using ReceiveMessageCallback = std::function)>; + + explicit ProtobufMessenger(SendMessageCallback send_message_cb) + : ProtobufMessenger(std::move(send_message_cb), {}) {} + + ProtobufMessenger(SendMessageCallback send_message_cb, + ReceiveMessageCallback receive_message_cb) + : send_message_cb_(std::move(send_message_cb)), + receive_message_cb_(std::move(receive_message_cb)) { + OSP_DCHECK(send_message_cb_); + } + + ProtobufMessenger(const ProtobufMessenger&) = delete; + ProtobufMessenger& operator=(const ProtobufMessenger&) = delete; + + ProtobufMessenger(ProtobufMessenger&& other) noexcept = default; + ProtobufMessenger& operator=(ProtobufMessenger&& other) noexcept = default; + + virtual ~ProtobufMessenger() = default; + + // Distributes an incoming message to the subclass. + // The `message` should be already base64-decoded. + void ProcessMessageFromRemote(ByteView message) { + auto proto = std::make_unique(); + if (proto->ParseFromArray(message.data(), message.size())) { + OnMessage(std::move(proto)); + } else { + OSP_DLOG_WARN << "Failed to parse protobuf message from remote"; + } + } + + // Executes the `send_message_cb_` using `message`. + void SendMessageToRemote(const T& message) { + std::vector serialized(message.ByteSizeLong()); + if (message.SerializeToArray(serialized.data(), serialized.size())) { + send_message_cb_(std::move(serialized)); + } else { + OSP_DLOG_WARN << "Failed to serialize protobuf message for remote"; + } + } + + void SetReceiveMessageCallback(ReceiveMessageCallback receive_message_cb) { + receive_message_cb_ = std::move(receive_message_cb); + } + + const ReceiveMessageCallback& receive_message_cb() const { + return receive_message_cb_; + } + + protected: + virtual void OnMessage(std::unique_ptr message) { + if (receive_message_cb_) { + receive_message_cb_(std::move(message)); + } + } + + SendMessageCallback send_message_cb_; + ReceiveMessageCallback receive_message_cb_; +}; + +} // namespace openscreen::cast + +#endif // CAST_STREAMING_PUBLIC_PROTOBUF_MESSENGER_H_ diff --git a/cast/streaming/public/receiver.cc b/cast/streaming/public/receiver.cc index a164d0bd2..3487badf5 100644 --- a/cast/streaming/public/receiver.cc +++ b/cast/streaming/public/receiver.cc @@ -1,485 +1,15 @@ -// Copyright 2019 The Chromium Authors +// Copyright 2021 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "cast/streaming/public/receiver.h" -#include -#include - -#include "cast/streaming/impl/receiver_packet_router.h" -#include "cast/streaming/impl/session_config.h" -#include "cast/streaming/public/constants.h" -#include "platform/base/span.h" -#include "platform/base/trivial_clock_traits.h" -#include "util/chrono_helpers.h" -#include "util/osp_logging.h" -#include "util/std_util.h" -#include "util/trace_logging.h" - namespace openscreen::cast { -using clock_operators::operator<<; - -// Conveniences for ensuring logging output includes the SSRC of the Receiver, -// to help distinguish one out of multiple instances in a Cast Streaming -// session. -// -#define RECEIVER_LOG(level) OSP_LOG_##level << "[SSRC:" << ssrc() << "] " -#define RECEIVER_VLOG OSP_VLOG << "[SSRC:" << ssrc() << "] " - -Receiver::Receiver(Environment& environment, - ReceiverPacketRouter& packet_router, - SessionConfig config) - : now_(environment.now_function()), - packet_router_(packet_router), - config_(config), - rtcp_session_(config.sender_ssrc, config.receiver_ssrc, now_()), - rtcp_parser_(rtcp_session_), - rtcp_builder_(rtcp_session_), - stats_tracker_(config.rtp_timebase), - rtp_parser_(config.sender_ssrc), - rtp_timebase_(config.rtp_timebase), - crypto_(config.aes_secret_key, config.aes_iv_mask), - is_pli_enabled_(config.is_pli_enabled), - rtcp_alarm_(environment.now_function(), environment.task_runner()), - smoothed_clock_offset_(ClockDriftSmoother::kDefaultTimeConstant), - consumption_alarm_(environment.now_function(), - environment.task_runner()) { - OSP_CHECK_EQ(checkpoint_frame(), FrameId::leader()); - - rtcp_buffer_.assign(environment.GetMaxPacketSize(), 0); - OSP_CHECK_GT(rtcp_buffer_.size(), 0); - - rtcp_builder_.SetPlayoutDelay(config.target_playout_delay); - playout_delay_changes_.emplace_back(FrameId::leader(), - config.target_playout_delay); - - packet_router_.OnReceiverCreated(rtcp_session_.sender_ssrc(), this); -} - -Receiver::~Receiver() { - packet_router_.OnReceiverDestroyed(rtcp_session_.sender_ssrc()); -} - -const SessionConfig& Receiver::config() const { - return config_; -} -int Receiver::rtp_timebase() const { - return rtp_timebase_; -} -Ssrc Receiver::ssrc() const { - return rtcp_session_.receiver_ssrc(); -} - -void Receiver::SetConsumer(Consumer* consumer) { - consumer_ = consumer; - ScheduleFrameReadyCheck(); -} - -void Receiver::SetPlayerProcessingTime(Clock::duration needed_time) { - player_processing_time_ = std::max(Clock::duration::zero(), needed_time); -} - -void Receiver::RequestKeyFrame() { - // If we don't have picture loss indication enabled, we should not request - // any key frames. - if (!is_pli_enabled_) { - OSP_LOG_WARN << "Should not request any key frames when picture loss " - "indication is not enabled"; - return; - } - - if (!last_key_frame_received_.is_null() && - last_frame_consumed_ >= last_key_frame_received_ && - !rtcp_builder_.is_picture_loss_indicator_set()) { - rtcp_builder_.SetPictureLossIndicator(true); - SendRtcp(); - } -} - -int Receiver::AdvanceToNextFrame() { - TRACE_DEFAULT_SCOPED(TraceCategory::kReceiver); - const FrameId immediate_next_frame = last_frame_consumed_ + 1; - - // Scan the queue for the next frame that should be consumed. Typically, this - // is the very next frame; but if it is incomplete and already late for - // playout, consider skipping-ahead. - for (FrameId f = immediate_next_frame; f <= latest_frame_expected_; ++f) { - PendingFrame& entry = GetQueueEntry(f); - if (entry.collector.is_complete()) { - const EncryptedFrame& encrypted_frame = - entry.collector.PeekAtAssembledFrame(); - if (f == immediate_next_frame) { // Typical case. - return FrameCrypto::GetPlaintextSize(encrypted_frame); - } - if (encrypted_frame.dependency != EncodedFrame::Dependency::kDependent) { - // Found a frame after skipping past some frames. Drop the ones being - // skipped, advancing `last_frame_consumed_` before returning. - DropAllFramesBefore(f); - return FrameCrypto::GetPlaintextSize(encrypted_frame); - } - // Conclusion: The frame in the current queue entry is complete, but - // depends on a prior incomplete frame. Continue scanning... - } - - // Do not consider skipping past this frame if its estimated capture time is - // unknown. The implication here is that, if `estimated_capture_time` is - // set, the Receiver also knows whether any target playout delay changes - // were communicated from the Sender in the frame's first RTP packet. - if (!entry.estimated_capture_time) { - break; - } - - // If this incomplete frame is not yet late for playout, simply wait for the - // rest of its packets to come in. However, do schedule a check to - // re-examine things at the time it should be processed. - const auto process_time = *entry.estimated_capture_time + - ResolveTargetPlayoutDelay(f) - - player_processing_time_; - if (process_time > now_()) { - ScheduleFrameReadyCheck(process_time); - break; - } - } - - return kNoFramesReady; -} - -EncodedFrame Receiver::ConsumeNextFrame(ByteBuffer buffer) { - TRACE_DEFAULT_SCOPED(TraceCategory::kReceiver); - // Assumption: The required call to AdvanceToNextFrame() ensures that - // `last_frame_consumed_` is set to one before the frame to be consumed here. - const FrameId frame_id = last_frame_consumed_ + 1; - OSP_CHECK_LE(frame_id, checkpoint_frame()); - - // Decrypt the frame, populating the given output `frame`. - PendingFrame& entry = GetQueueEntry(frame_id); - OSP_CHECK(entry.collector.is_complete()); - OSP_CHECK(entry.estimated_capture_time); - - const EncryptedFrame& encrypted_frame = - entry.collector.PeekAtAssembledFrame(); - - // `buffer` will contain the decrypted frame contents. - crypto_.Decrypt(encrypted_frame, buffer); - EncodedFrame frame; - encrypted_frame.CopyMetadataTo(&frame); - frame.data = buffer; - frame.reference_time = - *entry.estimated_capture_time + ResolveTargetPlayoutDelay(frame_id); - - RECEIVER_VLOG << "ConsumeNextFrame → " << frame.frame_id << ": " - << frame.data.size() << " payload bytes, RTP Timestamp " - << frame.rtp_timestamp.ToTimeSinceOrigin( - rtp_timebase_) - << ", to play-out " << (frame.reference_time - now_()) - << " from now."; - - entry.Reset(); - last_frame_consumed_ = frame_id; - - // Ensure the Consumer is notified if there are already more frames ready for - // consumption, and it hasn't explicitly called AdvanceToNextFrame() to check - // for itself. - ScheduleFrameReadyCheck(); - - return frame; -} - -void Receiver::OnReceivedRtpPacket(Clock::time_point arrival_time, - std::vector packet) { - const std::optional part = - rtp_parser_.Parse(packet); - if (!part) { - RECEIVER_LOG(WARN) << "Parsing of " << packet.size() - << " bytes as an RTP packet failed."; - return; - } - stats_tracker_.OnReceivedValidRtpPacket(part->sequence_number, - part->rtp_timestamp, arrival_time); - - // Ignore packets for frames the Receiver is no longer interested in. - if (part->frame_id <= checkpoint_frame()) { - return; - } - - // Extend the range of frames known to this Receiver, within the capacity of - // this Receiver's queue. Prepare the FrameCollectors to receive any - // newly-discovered frames. - if (part->frame_id > latest_frame_expected_) { - const FrameId max_allowed_frame_id = - last_frame_consumed_ + kMaxUnackedFrames; - if (part->frame_id > max_allowed_frame_id) { - return; - } - do { - ++latest_frame_expected_; - GetQueueEntry(latest_frame_expected_) - .collector.set_frame_id(latest_frame_expected_); - } while (latest_frame_expected_ < part->frame_id); - } - - // Start-up edge case: Blatantly drop the first packet of all frames until the - // Receiver has processed at least one Sender Report containing the necessary - // clock-drift and lip-sync information (see OnReceivedRtcpPacket()). This is - // an inescapable data dependency. Note that this special case should almost - // never trigger, since a well-behaving Sender will send the first Sender - // Report RTCP packet before any of the RTP packets. - if (!last_sender_report_ && part->packet_id == FramePacketId{0}) { - RECEIVER_LOG(WARN) << "Dropping packet 0 of frame " << part->frame_id - << " because it arrived before the first Sender Report."; - // Note: The Sender will have to re-transmit this dropped packet after the - // Sender Report to allow the Receiver to move forward. - return; - } - - PendingFrame& pending_frame = GetQueueEntry(part->frame_id); - FrameCollector& collector = pending_frame.collector; - if (collector.is_complete()) { - // An extra, redundant `packet` was received. Do nothing since the frame was - // already complete. - return; - } - - if (!collector.CollectRtpPacket(*part, &packet)) { - return; // Bad data in the parsed packet. Ignore it. - } - - // The first packet in a frame contains timing information critical for - // computing this frame's (and all future frames') playout time. Process that, - // but only once. - if (part->packet_id == FramePacketId{0} && - !pending_frame.estimated_capture_time) { - // Estimate the original capture time of this frame (at the Sender), in - // terms of the Receiver's clock: First, start with a reference time point - // from the Sender's clock (the one from the last Sender Report). Then, - // translate it into the equivalent reference time point in terms of the - // Receiver's clock by applying the measured offset between the two clocks. - // Finally, apply the RTP timestamp difference between the Sender Report and - // this frame to determine what the original capture time of this frame was. - pending_frame.estimated_capture_time = - last_sender_report_->reference_time + smoothed_clock_offset_.Current() + - (part->rtp_timestamp - last_sender_report_->rtp_timestamp) - .ToDuration(rtp_timebase_); - - // If a target playout delay change was included in this packet, record it. - if (part->new_playout_delay > milliseconds::zero()) { - RecordNewTargetPlayoutDelay(part->frame_id, part->new_playout_delay); - } - - // Now that the estimated capture time is known, other frames may have just - // become ready, per the frame-skipping logic in AdvanceToNextFrame(). - ScheduleFrameReadyCheck(); - } - - if (!collector.is_complete()) { - return; // Wait for the rest of the packets to come in. - } - const EncryptedFrame& encrypted_frame = collector.PeekAtAssembledFrame(); - - // Whenever a key frame has been received, the decoder has what it needs to - // recover. In this case, clear the PLI condition. - if (encrypted_frame.dependency == EncryptedFrame::Dependency::kKeyFrame) { - rtcp_builder_.SetPictureLossIndicator(false); - last_key_frame_received_ = part->frame_id; - } - - // If this just-completed frame is the one right after the checkpoint frame, - // advance the checkpoint forward. - if (part->frame_id == (checkpoint_frame() + 1)) { - AdvanceCheckpoint(part->frame_id); - } - - // Since a frame has become complete, schedule a check to see whether this or - // any other frames have become ready for consumption. - ScheduleFrameReadyCheck(); -} - -void Receiver::OnReceivedRtcpPacket(Clock::time_point arrival_time, - std::vector packet) { - TRACE_DEFAULT_SCOPED(TraceCategory::kReceiver); - std::optional parsed_report = - rtcp_parser_.Parse(packet); - if (!parsed_report) { - RECEIVER_LOG(WARN) << "Parsing of " << packet.size() - << " bytes as an RTCP packet failed."; - return; - } - last_sender_report_ = std::move(parsed_report); - last_sender_report_arrival_time_ = arrival_time; - - // Measure the offset between the Sender's clock and the Receiver's Clock. - // This will be used to translate reference timestamps from the Sender into - // timestamps that represent the exact same moment in time at the Receiver. - // - // Note: Due to design limitations in the Cast Streaming spec, the Receiver - // has no way to compute how long it took the Sender Report to travel over the - // network. The calculation here just ignores that, and so the - // `measured_offset` below will be larger than the true value by that amount. - // This will have the effect of a later-than-configured playout delay. - const Clock::duration measured_offset = - arrival_time - last_sender_report_->reference_time; - smoothed_clock_offset_.Update(arrival_time, measured_offset); - - RtcpReportBlock report; - report.ssrc = rtcp_session_.sender_ssrc(); - stats_tracker_.PopulateNextReport(&report); - report.last_status_report_id = last_sender_report_->report_id; - report.SetDelaySinceLastReport(now_() - last_sender_report_arrival_time_); - rtcp_builder_.IncludeReceiverReportInNextPacket(report); - - SendRtcp(); -} - -void Receiver::SendRtcp() { - // Collect ACK/NACK feedback for all active frames in the queue. - std::vector packet_nacks; - std::vector frame_acks; - for (FrameId f = checkpoint_frame() + 1; f <= latest_frame_expected_; ++f) { - const FrameCollector& collector = GetQueueEntry(f).collector; - if (collector.is_complete()) { - frame_acks.push_back(f); - } else { - collector.GetMissingPackets(&packet_nacks); - } - } - - // Build and send a compound RTCP packet. - const bool no_nacks = packet_nacks.empty(); - rtcp_builder_.IncludeFeedbackInNextPacket(std::move(packet_nacks), - std::move(frame_acks)); - last_rtcp_send_time_ = now_(); - packet_router_.SendRtcpPacket( - rtcp_builder_.BuildPacket(last_rtcp_send_time_, rtcp_buffer_)); - - // Schedule the automatic sending of another RTCP packet, if this method is - // not called within some bounded amount of time. While incomplete frames - // exist in the queue, send RTCP packets (with ACK/NACK feedback) frequently. - // When there are no incomplete frames, use a longer "keepalive" interval. - const Clock::duration interval = - (no_nacks ? kRtcpReportInterval : kNackFeedbackInterval); - rtcp_alarm_.Schedule([this] { SendRtcp(); }, last_rtcp_send_time_ + interval); -} - -const Receiver::PendingFrame& Receiver::GetQueueEntry(FrameId frame_id) const { - return const_cast(this)->GetQueueEntry(frame_id); -} - -Receiver::PendingFrame& Receiver::GetQueueEntry(FrameId frame_id) { - return pending_frames_[(frame_id - FrameId::first()) % - pending_frames_.size()]; -} - -void Receiver::RecordNewTargetPlayoutDelay(FrameId as_of_frame, - milliseconds delay) { - OSP_CHECK_GT(as_of_frame, checkpoint_frame()); - - // Prune-out entries from `playout_delay_changes_` that are no longer needed. - // At least one entry must always be kept (i.e., there must always be a - // "current" setting). - const FrameId next_frame = last_frame_consumed_ + 1; - const auto keep_one_before_it = std::find_if( - std::next(playout_delay_changes_.begin()), playout_delay_changes_.end(), - [&](const auto& entry) { return entry.first > next_frame; }); - playout_delay_changes_.erase(playout_delay_changes_.begin(), - std::prev(keep_one_before_it)); - - // Insert the delay change entry, maintaining the ascending ordering of the - // vector. - const auto insert_it = std::find_if( - playout_delay_changes_.begin(), playout_delay_changes_.end(), - [&](const auto& entry) { return entry.first > as_of_frame; }); - playout_delay_changes_.emplace(insert_it, as_of_frame, delay); - - OSP_DCHECK(AreElementsSortedAndUnique(playout_delay_changes_)); -} - -milliseconds Receiver::ResolveTargetPlayoutDelay(FrameId frame_id) const { - OSP_CHECK_GT(frame_id, last_frame_consumed_); - -#if OSP_DCHECK_IS_ON() - // Extra precaution: Ensure all possible playout delay changes are known. In - // other words, every unconsumed frame in the queue, up to (and including) - // `frame_id`, must have an assigned estimated_capture_time. - for (FrameId f = last_frame_consumed_ + 1; f <= frame_id; ++f) { - OSP_CHECK(GetQueueEntry(f).estimated_capture_time) - << " don't know whether there was a playout delay change for frame " - << f; - } -#endif - - const auto it = std::find_if( - playout_delay_changes_.crbegin(), playout_delay_changes_.crend(), - [&](const auto& entry) { return entry.first <= frame_id; }); - OSP_CHECK(it != playout_delay_changes_.crend()); - return it->second; -} - -void Receiver::AdvanceCheckpoint(FrameId new_checkpoint) { - TRACE_DEFAULT_SCOPED(TraceCategory::kReceiver); - OSP_CHECK_GT(new_checkpoint, checkpoint_frame()); - OSP_CHECK_LE(new_checkpoint, latest_frame_expected_); - - while (new_checkpoint < latest_frame_expected_) { - const FrameId next = new_checkpoint + 1; - if (!GetQueueEntry(next).collector.is_complete()) { - break; - } - new_checkpoint = next; - } - - set_checkpoint_frame(new_checkpoint); - rtcp_builder_.SetPlayoutDelay(ResolveTargetPlayoutDelay(new_checkpoint)); - SendRtcp(); -} - -void Receiver::DropAllFramesBefore(FrameId first_kept_frame) { - // The following CHECKs are verifying that this method is only being called - // because one or more incomplete frames are being skipped-over. - const FrameId first_to_drop = last_frame_consumed_ + 1; - OSP_CHECK_GT(first_kept_frame, first_to_drop); - OSP_CHECK_GT(first_kept_frame, checkpoint_frame()); - OSP_CHECK_LE(first_kept_frame, latest_frame_expected_); - - // Reset each of the frames being dropped, pretending that they were consumed. - for (FrameId f = first_to_drop; f < first_kept_frame; ++f) { - PendingFrame& entry = GetQueueEntry(f); - // Pedantic sanity-check: Ensure the "target playout delay change" data - // dependency was satisfied. See comments in AdvanceToNextFrame(). - OSP_CHECK(entry.estimated_capture_time); - entry.Reset(); - } - last_frame_consumed_ = first_kept_frame - 1; - - RECEIVER_LOG(INFO) << "Artificially advancing checkpoint after skipping."; - AdvanceCheckpoint(first_kept_frame); -} - -void Receiver::ScheduleFrameReadyCheck(Clock::time_point when) { - consumption_alarm_.Schedule( - [this] { - if (consumer_) { - const int next_frame_buffer_size = AdvanceToNextFrame(); - if (next_frame_buffer_size != kNoFramesReady) { - consumer_->OnFramesReady(next_frame_buffer_size); - } - } - }, - when); -} - -Receiver::PendingFrame::PendingFrame() = default; -Receiver::PendingFrame::~PendingFrame() = default; +Receiver::Consumer::~Consumer() = default; -void Receiver::PendingFrame::Reset() { - collector.Reset(); - estimated_capture_time = std::nullopt; -} +Receiver::Receiver() = default; -// static -constexpr milliseconds Receiver::kDefaultPlayerProcessingTime; -constexpr int Receiver::kNoFramesReady; -constexpr milliseconds Receiver::kNackFeedbackInterval; +Receiver::~Receiver() = default; } // namespace openscreen::cast diff --git a/cast/streaming/public/receiver.h b/cast/streaming/public/receiver.h index 40ca85755..e57e4a5ae 100644 --- a/cast/streaming/public/receiver.h +++ b/cast/streaming/public/receiver.h @@ -1,41 +1,21 @@ -// Copyright 2019 The Chromium Authors +// Copyright 2021 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #ifndef CAST_STREAMING_PUBLIC_RECEIVER_H_ #define CAST_STREAMING_PUBLIC_RECEIVER_H_ -#include - -#include #include -#include -#include -#include -#include -#include "cast/streaming/impl/clock_drift_smoother.h" -#include "cast/streaming/impl/compound_rtcp_builder.h" -#include "cast/streaming/impl/frame_collector.h" -#include "cast/streaming/impl/packet_receive_stats_tracker.h" -#include "cast/streaming/impl/receiver_base.h" -#include "cast/streaming/impl/rtcp_common.h" -#include "cast/streaming/impl/rtcp_session.h" -#include "cast/streaming/impl/rtp_packet_parser.h" -#include "cast/streaming/impl/sender_report_parser.h" -#include "cast/streaming/impl/session_config.h" -#include "cast/streaming/public/environment.h" -#include "cast/streaming/public/frame_id.h" +#include "cast/streaming/public/encoded_frame.h" +#include "cast/streaming/public/session_config.h" #include "cast/streaming/ssrc.h" #include "platform/api/time.h" +#include "platform/base/error.h" #include "platform/base/span.h" -#include "util/alarm.h" namespace openscreen::cast { -struct EncodedFrame; -class ReceiverPacketRouter; - // The Cast Streaming Receiver, a peer corresponding to some Cast Streaming // Sender at the other end of a network link. // @@ -54,39 +34,9 @@ class ReceiverPacketRouter; // // See the Receiver Demo app for a reference implementation that both shows and // explains how Receivers are properly configured and started, integrated with a -// decoder, and the resulting decoded media is played out. Also, here is a -// general usage example: -// -// class MyPlayer : public openscreen::cast::Receiver::Consumer { -// public: -// explicit MyPlayer(Receiver* receiver) : receiver_(receiver) { -// recevier_->SetPlayerProcessingTime(std::chrono::milliseconds(10)); -// receiver_->SetConsumer(this); -// } -// -// ~MyPlayer() override { -// receiver_->SetConsumer(nullptr); -// } -// -// private: -// // Receiver::Consumer implementation. -// void OnFramesReady(int next_frame_buffer_size) override { -// std::vector buffer; -// buffer.resize(next_frame_buffer_size); -// openscreen::cast::EncodedFrame encoded_frame = -// receiver_->ConsumeNextFrame(buffer); -// -// display_.RenderFrame(decoder_.DecodeFrame(encoded_frame.data)); -// -// // Note: An implementation could call receiver_->AdvanceToNextFrame() -// // and receiver_->ConsumeNextFrame() in a loop here, to consume all the -// // remaining frames that are ready. -// } -// -// Receiver* const receiver_; -// MyDecoder decoder_; -// MyDisplay display_; -// }; +// decoder, and the resulting decoded media is played out. The +// Receiver::Consumer is implemented by the SDLPlayerBase class in +// //cast/standalone_receiver/sdl_player_base.h. // // Internally, a queue of complete and partially-received frames is maintained. // The queue is a circular queue of FrameCollectors that each maintain the @@ -102,207 +52,93 @@ class ReceiverPacketRouter; // received and processed. // 3. Last Frame Consumed: The FrameId of last frame consumed (see // ConsumeNextFrame()). Once a frame is consumed, all internal resources -// related to the frame can be freed and/or re-used for later frames. -class Receiver : public ReceiverBase { +// related to the frame can be freed and/or reused for later frames. +class Receiver { public: - using ReceiverBase::Consumer; - - // Constructs a Receiver that attaches to the given `environment` and - // `packet_router`. The config contains the settings that were - // agreed-upon by both sides from the OFFER/ANSWER exchange (i.e., the part of - // the overall end-to-end connection process that occurs before Cast Streaming - // is started). - Receiver(Environment& environment, - ReceiverPacketRouter& packet_router, - SessionConfig config); - ~Receiver() override; - - // ReceiverBase overrides. - const SessionConfig& config() const override; - int rtp_timebase() const override; - Ssrc ssrc() const override; - void SetConsumer(Consumer* consumer) override; - void SetPlayerProcessingTime(Clock::duration needed_time) override; - void RequestKeyFrame() override; - int AdvanceToNextFrame() override; - EncodedFrame ConsumeNextFrame(ByteBuffer buffer) override; - - // Allows setting picture loss indication for testing. In production, this - // should be done using the config. - void SetPliEnabledForTesting(bool is_pli_enabled) { - is_pli_enabled_ = is_pli_enabled; - } - - // The default "player processing time" amount. See SetPlayerProcessingTime(). - static constexpr std::chrono::milliseconds kDefaultPlayerProcessingTime = - ReceiverBase::kDefaultPlayerProcessingTime; - - // Returned by AdvanceToNextFrame() when there are no frames currently ready - // for consumption. - static constexpr int kNoFramesReady = ReceiverBase::kNoFramesReady; - - protected: - friend class ReceiverPacketRouter; - - // Called by ReceiverPacketRouter to provide this Receiver with what looks - // like a RTP/RTCP packet meant for it specifically (among other Receivers). - void OnReceivedRtpPacket(Clock::time_point arrival_time, - std::vector packet); - void OnReceivedRtcpPacket(Clock::time_point arrival_time, - std::vector packet); - - private: - // An entry in the circular queue (see `pending_frames_`). - struct PendingFrame { - FrameCollector collector; - - // The Receiver's [local] Clock time when this frame was originally captured - // at the Sender. This is computed and assigned when the RTP packet with ID - // 0 is processed. Add the target playout delay to this to get the target - // playout time. - std::optional estimated_capture_time; - - PendingFrame(); - ~PendingFrame(); - - // Reset this entry to its initial state, freeing resources. - void Reset(); + class Consumer { + public: + virtual ~Consumer(); + + // Called whenever one or more frames have become ready for consumption. The + // `next_frame_buffer_size` argument is identical to the result of calling + // AdvanceToNextFrame(), and so the Consumer only needs to prepare a buffer + // and call ConsumeNextFrame(). It may then call AdvanceToNextFrame() to + // check whether there are any more frames ready, but this is not mandatory. + // See usage example in SDLPlayerBase::OnFramesReady. + virtual void OnFramesReady(size_t next_frame_buffer_size) = 0; }; - // Get/Set the checkpoint FrameId. This indicates that all of the packets for - // all frames up to and including this FrameId have been successfully received - // (or otherwise do not need to be re-transmitted). - FrameId checkpoint_frame() const { return rtcp_builder_.checkpoint_frame(); } - void set_checkpoint_frame(FrameId frame_id) { - rtcp_builder_.SetCheckpointFrame(frame_id); - } - - // Send an RTCP packet to the Sender immediately, to acknowledge the complete - // reception of one or more additional frames, to reply to a Sender Report, or - // to request re-transmits. Calling this also schedules additional RTCP - // packets to be sent periodically for the life of this Receiver. - void SendRtcp(); - - // Helpers to map the given `frame_id` to the element in the `pending_frames_` - // circular queue. There are both const and non-const versions, but neither - // mutate any state (i.e., they are just look-ups). - const PendingFrame& GetQueueEntry(FrameId frame_id) const; - PendingFrame& GetQueueEntry(FrameId frame_id); - - // Record that the target playout delay has changed starting with the given - // FrameId. - void RecordNewTargetPlayoutDelay(FrameId as_of_frame, - std::chrono::milliseconds delay); - - // Examine the known target playout delay changes to determine what setting is - // in-effect for the given frame. - std::chrono::milliseconds ResolveTargetPlayoutDelay(FrameId frame_id) const; - - // Called to move the checkpoint forward. This scans the queue, starting from - // `new_checkpoint`, to find the latest in a contiguous sequence of completed - // frames. Then, it records that frame as the new checkpoint, and immediately - // sends a feedback RTCP packet to the Sender. - void AdvanceCheckpoint(FrameId new_checkpoint); - - // Helper to force-drop all frames before `first_kept_frame`, even if they - // were never consumed. This will also auto-cancel frames that were never - // completely received, artificially moving the checkpoint forward, and - // notifying the Sender of that. The caller of this method is responsible for - // making sure that frame data dependencies will not be broken by dropping the - // frames. - void DropAllFramesBefore(FrameId first_kept_frame); - - // Sets the `consumption_alarm_` to check whether any frames are ready, - // including possibly skipping over late frames in order to make not-yet-late - // frames become ready. The default argument value means "without delay." - void ScheduleFrameReadyCheck(Clock::time_point when = Alarm::kImmediately); - - const ClockNowFunctionPtr now_; - ReceiverPacketRouter& packet_router_; - const SessionConfig config_; - RtcpSession rtcp_session_; - SenderReportParser rtcp_parser_; - CompoundRtcpBuilder rtcp_builder_; - PacketReceiveStatsTracker stats_tracker_; // Tracks transmission stats. - RtpPacketParser rtp_parser_; - const int rtp_timebase_; // RTP timestamp ticks per second. - const FrameCrypto crypto_; // Decrypts assembled frames. - bool is_pli_enabled_; // Whether picture loss indication is enabled. - - // Buffer for serializing/sending RTCP packets. - std::vector rtcp_buffer_; - - // Schedules tasks to ensure RTCP reports are sent within a bounded interval. - // Not scheduled until after this Receiver has processed the first packet from - // the Sender. - Alarm rtcp_alarm_; - Clock::time_point last_rtcp_send_time_ = Clock::time_point::min(); - - // The last Sender Report received and when the packet containing it had - // arrived. This contains lip-sync timestamps used as part of the calculation - // of playout times for the received frames, as well as ping-pong data bounced - // back to the Sender in the Receiver Reports. It is nullopt until the first - // parseable Sender Report is received. - std::optional last_sender_report_; - Clock::time_point last_sender_report_arrival_time_; - - // Tracks the offset between the Receiver's [local] clock and the Sender's - // clock. This is invalid until the first Sender Report has been successfully - // processed (i.e., `last_sender_report_` is not nullopt). - ClockDriftSmoother smoothed_clock_offset_; - - // The ID of the latest frame whose existence is known to this Receiver. This - // value must always be greater than or equal to `checkpoint_frame()`. - FrameId latest_frame_expected_ = FrameId::leader(); - - // The ID of the last frame consumed. This value must always be less than or - // equal to `checkpoint_frame()`, since it's impossible to consume incomplete - // frames! - FrameId last_frame_consumed_ = FrameId::leader(); - - // The ID of the latest key frame known to be in-flight. This is used by - // RequestKeyFrame() to ensure the PLI condition doesn't get set again until - // after the consumer has seen a key frame that would clear the condition. - FrameId last_key_frame_received_; - - // The frame queue (circular), which tracks which frames are in-flight, stores - // data for partially-received frames, and holds onto completed frames until - // the consumer consumes them. + Receiver(); + virtual ~Receiver(); + + // The session configuration for this sender. The configuration is generated + // from the offer/answer exchange, and includes critical information like the + // RTP timebase, SSRCs for sending and receiving, and the AES configuration. + virtual const SessionConfig& config() const = 0; + + // Set the Consumer receiving notifications when new frames are ready for + // consumption. Frames received before this method is called will remain in + // the queue indefinitely. The `consumer` pointer is expected to remain valid + // for the lifetime of the Receiver (or until SetConsumer() is called again + // with a new value, either a new consumer or nullptr). + virtual void SetConsumer(Consumer* consumer) = 0; + + // Sets how much time the consumer will need to decode/buffer/render/etc., and + // otherwise fully process a frame for on-time playback. This information is + // used by the Receiver to decide whether to skip past frames that have + // arrived too late, as well as adjust the reference time of the frame to + // factor in the player processing time -- resulting in the frame being + // scheduled for playback earlier and decreasing total playout delay. This + // method can be called repeatedly to make adjustments based on changing + // environmental conditions. It is HIGHLY recommended that consumers of this + // API provide a proper processing time, otherwise there may be significantly + // larger playout delays. // - // Use GetQueueEntry() to access a slot. The currently-active slots are those - // for the frames after `last_frame_consumed_` and up-to/including - // `latest_frame_expected_`. - std::array pending_frames_{}; + // Default setting: kDefaultPlayerProcessingTime + virtual void SetPlayerProcessingTime(Clock::duration needed_time) = 0; - // Tracks the recent changes to the target playout delay, which is controlled - // by the Sender. The FrameId indicates the first frame where a new delay - // setting takes effect. This vector is never empty, is kept sorted, and is - // pruned to remain as small as possible. + // Called by the consumer to report that a frame has been played out. This is + // used to report playback statistics to the sender. // - // The target playout delay is the amount of time between a frame's - // capture/recording on the Sender and when it should be played-out at the - // Receiver. - std::vector> - playout_delay_changes_; - - // The consumer to notify when there are one or more frames completed and - // ready to be consumed. - Consumer* consumer_ = nullptr; - - // The additional time needed to decode/play-out each frame after being - // consumed from this Receiver. - Clock::duration player_processing_time_ = kDefaultPlayerProcessingTime; - - // Scheduled to check whether there are frames ready and, if there are, to - // notify the Consumer via OnFramesReady(). - Alarm consumption_alarm_; - - // The interval between sending ACK/NACK feedback RTCP messages while - // incomplete frames exist in the queue. + // NOTE: the consumer has until `kMaxUnackedFrames` additional frames have + // been consumed *after* `frame_id` to report the playout event, otherwise a + // kParameterOutOfRange error will be returned. + virtual Error ReportPlayoutEvent(FrameId frame_id, + RtpTimeTicks rtp_timestamp, + Clock::time_point playout_time) = 0; + + // Propagates a "picture loss indicator" notification to the Sender, + // requesting a key frame so that decode/playout can recover. It is safe to + // call this redundantly. The Receiver will clear the picture loss condition + // automatically, once a key frame is received (i.e., before + // ConsumeNextFrame() is called to access it). + virtual void RequestKeyFrame() = 0; + + // Advances to the next frame ready for consumption. This may skip-over + // incomplete frames that will not play out on-time; but only if there are + // completed frames further down the queue that have no dependency + // relationship with them (e.g., key frames). // - // TODO(jophba): This should be a function of the current target playout - // delay, similar to the Sender's kickstart interval logic. - static constexpr std::chrono::milliseconds kNackFeedbackInterval{30}; + // This method returns std::nullopt if there is not currently a frame ready + // for consumption. The caller should wait for a Consumer::OnFramesReady() + // notification before trying again. Otherwise, the number of bytes of encoded + // data is returned, and the caller should use this to ensure the buffer it + // passes to ConsumeNextFrame() is large enough. + virtual std::optional AdvanceToNextFrame() = 0; + + // Returns the next frame, both metadata and payload data. The Consumer calls + // this method after being notified via OnFramesReady(), and it can also call + // this whenever AdvanceToNextFrame() indicates another frame is ready. + // `buffer` must point to a sufficiently-sized buffer (enforced by CHECK) that + // will be populated with the frame's payload data. The returned frame's + // `data` will be set to the portion of the buffer that was populated. + virtual EncodedFrame ConsumeNextFrame(ByteBuffer buffer) = 0; + + // The default "player processing time" amount. See SetPlayerProcessingTime(). + // This value is based on real world experimentation, however may vary + // widely depending on the platform of the receiver and what type of + // decoder is available. + static constexpr std::chrono::milliseconds kDefaultPlayerProcessingTime{50}; }; } // namespace openscreen::cast diff --git a/cast/streaming/public/receiver_constraints.h b/cast/streaming/public/receiver_constraints.h index dc60e9832..5a4cd220a 100644 --- a/cast/streaming/public/receiver_constraints.h +++ b/cast/streaming/public/receiver_constraints.h @@ -114,7 +114,7 @@ struct RemotingConstraints { // ALAC (Apple Lossless) // AC-3 (Dolby Digital) // These properties are tied directly to what Chrome supports. See: - // https://source.chromium.org/chromium/chromium/src/+/master:media/base/audio_codecs.h + // https://source.chromium.org/chromium/chromium/src/+/main:media/base/audio_codecs.h bool supports_chrome_audio_codecs = false; // Current remoting senders assume that the receiver supports 4K for all @@ -143,7 +143,7 @@ class ReceiverConstraints { // Returns true if all configurations supported by `other` are also // supported by this instance. // - // TODO(crbug.com/1356762): Implement receiver-side session renegotation + // TODO(crbug.com/1356762): Implement receiver-side session renegotiation // so we can eliminate complicated logic for constraints compatiblity. bool IsSupersetOf(const ReceiverConstraints& other) const; @@ -167,6 +167,15 @@ class ReceiverConstraints { // offers may provide a set of remoting constraints, or leave nullptr for // all remoting OFFERs to be rejected in favor of continuing streaming. std::unique_ptr remoting; + + // If true, allows the receiver to negotiate DSCP marking for RTP and RTCP + // packets, which can help improve Quality of Service in some environments. + // This feature is off by default to allow embedders to opt in and experiment + // as desired. + bool enable_dscp = false; + + // If true, the receiver supports sending input events to the sender. + bool supports_input_events = false; }; } // namespace openscreen::cast diff --git a/cast/streaming/public/receiver_message.cc b/cast/streaming/public/receiver_message.cc index 30d523094..9654f6699 100644 --- a/cast/streaming/public/receiver_message.cc +++ b/cast/streaming/public/receiver_message.cc @@ -5,6 +5,7 @@ #include "cast/streaming/public/receiver_message.h" #include +#include #include "cast/streaming/message_fields.h" #include "json/reader.h" @@ -22,10 +23,11 @@ namespace openscreen::cast { namespace { -EnumNameTable kMessageTypeNames{ +EnumNameTable kMessageTypeNames{ {{kMessageTypeAnswer, ReceiverMessage::Type::kAnswer}, {"CAPABILITIES_RESPONSE", ReceiverMessage::Type::kCapabilitiesResponse}, - {"RPC", ReceiverMessage::Type::kRpc}}}; + {"RPC", ReceiverMessage::Type::kRpc}, + {"INPUT", ReceiverMessage::Type::kInput}}}; EnumNameTable kMediaCapabilityNames{ {{"audio", MediaCapability::kAudio}, @@ -92,7 +94,7 @@ ReceiverError::~ReceiverError() = default; // static ErrorOr ReceiverError::Parse(const Json::Value& value) { - if (!value) { + if (!value.isObject()) { return Error(Error::Code::kParameterInvalid, "Empty JSON in receiver error parsing"); } @@ -121,7 +123,7 @@ Error ReceiverError::ToError() const { return Error(*openscreen_code, description); } - std::string full_description = StringPrintf("Error code: %d, description: %s", + std::string full_description = StringFormat("Error code: {}, description: {}", code, description.c_str()); return Error(Error::Code::kUnknownError, std::move(full_description)); } @@ -129,7 +131,7 @@ Error ReceiverError::ToError() const { // static ErrorOr ReceiverCapability::Parse( const Json::Value& value) { - if (!value) { + if (!value.isObject()) { return Error(Error::Code::kParameterInvalid, "Empty JSON in capabilities parsing"); } @@ -163,7 +165,7 @@ Json::Value ReceiverCapability::ToJson() const { // static ErrorOr ReceiverMessage::Parse(const Json::Value& value) { ReceiverMessage message; - if (!value) { + if (!value.isObject()) { return Error(Error::Code::kJsonParseError, "Invalid message body"); } @@ -174,9 +176,11 @@ ErrorOr ReceiverMessage::Parse(const Json::Value& value) { message.type = GetMessageType(value); message.valid = - (result == kResultOk || message.type == ReceiverMessage::Type::kRpc); + (result == kResultOk || message.type == ReceiverMessage::Type::kRpc || + message.type == ReceiverMessage::Type::kInput); - if (message.type != ReceiverMessage::Type::kRpc) { + if (message.type != ReceiverMessage::Type::kRpc && + message.type != ReceiverMessage::Type::kInput) { if (!json::TryParseInt(value[kSequenceNumber], &(message.sequence_number))) { message.sequence_number = -1; @@ -199,10 +203,10 @@ ErrorOr ReceiverMessage::Parse(const Json::Value& value) { switch (message.type) { case Type::kAnswer: { - Answer answer; - if (openscreen::cast::Answer::TryParse(value[kAnswerMessageBody], - &answer)) { - message.body = std::move(answer); + auto answer_or_error = + openscreen::cast::Answer::TryParse(value[kAnswerMessageBody]); + if (answer_or_error.is_value()) { + message.body = std::move(answer_or_error.value()); message.valid = true; } } break; @@ -226,6 +230,16 @@ ErrorOr ReceiverMessage::Parse(const Json::Value& value) { } } break; + case Type::kInput: { + std::string encoded_input; + std::vector input; + if (json::TryParseString(value[kInputMessageBody], &encoded_input) && + base64::Decode(encoded_input, &input)) { + message.body = std::move(input); + message.valid = true; + } + } break; + default: break; } @@ -247,10 +261,10 @@ ErrorOr ReceiverMessage::ToJson() const { case ReceiverMessage::Type::kAnswer: if (valid) { root[kResult] = kResultOk; - root[kAnswerMessageBody] = absl::get(body).ToJson(); + root[kAnswerMessageBody] = std::get(body).ToJson(); } else { root[kResult] = kResultError; - root[kErrorMessageBody] = absl::get(body).ToJson(); + root[kErrorMessageBody] = std::get(body).ToJson(); } break; @@ -258,17 +272,22 @@ ErrorOr ReceiverMessage::ToJson() const { if (valid) { root[kResult] = kResultOk; root[kCapabilitiesMessageBody] = - absl::get(body).ToJson(); + std::get(body).ToJson(); } else { root[kResult] = kResultError; - root[kErrorMessageBody] = absl::get(body).ToJson(); + root[kErrorMessageBody] = std::get(body).ToJson(); } break; // NOTE: RPC messages do NOT have a result field. case ReceiverMessage::Type::kRpc: root[kRpcMessageBody] = - base64::Encode(absl::get>(body)); + base64::Encode(std::get>(body)); + break; + + case ReceiverMessage::Type::kInput: + root[kInputMessageBody] = + base64::Encode(std::get>(body)); break; default: diff --git a/cast/streaming/public/receiver_message.h b/cast/streaming/public/receiver_message.h index d01e77e28..6bf545784 100644 --- a/cast/streaming/public/receiver_message.h +++ b/cast/streaming/public/receiver_message.h @@ -9,9 +9,9 @@ #include #include #include +#include #include -#include "absl/types/variant.h" #include "cast/streaming/public/answer_messages.h" #include "json/value.h" #include "util/osp_logging.h" @@ -90,6 +90,9 @@ struct ReceiverMessage { // Rpc binary messages. The payload is base64-encoded. kRpc, + + // Input-related binary messages. The payload is base64-encoded. + kInput, }; static ErrorOr Parse(const Json::Value& value); @@ -101,11 +104,11 @@ struct ReceiverMessage { bool valid = false; - absl::variant, // Binary-encoded RPC message. - ReceiverCapability, - ReceiverError> + std::variant, // Binary-encoded protobuf message. + ReceiverCapability, + ReceiverError> body; }; diff --git a/cast/streaming/public/receiver_session.cc b/cast/streaming/public/receiver_session.cc index c1fcec9d1..51fd8986b 100644 --- a/cast/streaming/public/receiver_session.cc +++ b/cast/streaming/public/receiver_session.cc @@ -6,13 +6,18 @@ #include #include +#include #include #include +#include #include "cast/common/channel/message_util.h" #include "cast/common/public/message_port.h" +#include "cast/streaming/impl/message_constants.h" +#include "cast/streaming/impl/receiver_impl.h" #include "cast/streaming/message_fields.h" #include "cast/streaming/public/answer_messages.h" +#include "cast/streaming/public/constants.h" #include "cast/streaming/public/environment.h" #include "cast/streaming/public/offer_messages.h" #include "cast/streaming/public/receiver.h" @@ -74,6 +79,18 @@ MediaCapability ToCapability(VideoCodec codec) { } } +void MaybeSetDscp(AudioStream* audio, VideoStream* video, Environment& env) { + std::optional dscp_value; + if (audio) { + dscp_value = audio->stream.receiver_rtcp_dscp; + } else if (video) { + dscp_value = video->stream.receiver_rtcp_dscp; + } + if (dscp_value) { + env.SetDscp(static_cast(dscp_value.value())); + } +} + } // namespace ReceiverSession::Client::~Client() = default; @@ -86,13 +103,23 @@ ReceiverSession::ReceiverSession(Client& client, environment_(environment), constraints_(std::move(constraints)), session_id_(MakeUniqueSessionId("streaming_receiver")), - messenger_(message_port, - session_id_, - [this](Error error) { - OSP_DLOG_WARN << "Got a session messenger error: " << error; - client_.OnError(this, error); - }), - packet_router_(environment_) { + messenger_(std::make_unique( + message_port, + session_id_, + [this](Error error) { + OSP_DLOG_WARN << "Got a session messenger error: " << error; + client_.OnError(this, error); + })), + packet_router_(environment_), + input_messenger_([this](std::vector message) { + if (!negotiated_sender_id_.empty()) { + const Error error = this->messenger_->SendInputMessage( + negotiated_sender_id_, message); + if (!error.ok()) { + OSP_LOG_WARN << "Failed to send input message: " << error; + } + } + }) { OSP_CHECK(std::none_of( constraints_.video_codecs.begin(), constraints_.video_codecs.end(), [](VideoCodec c) { return c == VideoCodec::kNotSpecified; })); @@ -100,28 +127,66 @@ ReceiverSession::ReceiverSession(Client& client, constraints_.audio_codecs.begin(), constraints_.audio_codecs.end(), [](AudioCodec c) { return c == AudioCodec::kNotSpecified; })); - messenger_.SetHandler( + messenger_->SetHandler( SenderMessage::Type::kOffer, [this](const std::string& sender_id, SenderMessage message) { OnOffer(sender_id, std::move(message)); }); - messenger_.SetHandler( + messenger_->SetHandler( SenderMessage::Type::kGetCapabilities, [this](const std::string& sender_id, SenderMessage message) { OnCapabilitiesRequest(sender_id, std::move(message)); }); - messenger_.SetHandler( + messenger_->SetHandler( SenderMessage::Type::kRpc, [this](const std::string& sender_id, SenderMessage message) { this->OnRpcMessage(sender_id, std::move(message)); }); + messenger_->SetHandler( + SenderMessage::Type::kInput, + [this](const std::string& sender_id, SenderMessage message) { + this->OnInputMessage(sender_id, std::move(message)); + }); environment_.SetSocketSubscriber(this); } ReceiverSession::~ReceiverSession() { + environment_.SetSocketSubscriber(nullptr); + // messenger_ must be destroyed before OnReceiversDestroying runs, since it + // calls message_port_.ResetClient(). That MessagePort is destroyed before + // OnReceiversDestroying returns. See crbug.com/374199735 for more details. + messenger_.reset(); ResetReceivers(Client::kEndOfSession); } +void ReceiverSession::SetInputCallback( + std::function callback) { + if (callback) { + input_messenger_.SetReceiveMessageCallback( + [cb = std::move(callback)](std::unique_ptr message) { + cb(std::move(*message)); + }); + } else { + input_messenger_.SetReceiveMessageCallback(nullptr); + } +} + +void ReceiverSession::SetCustomMessageHandler( + std::string_view message_namespace, + ReceiverSessionMessenger::CustomMessageCallback cb) { + messenger_->SetCustomMessageHandler(message_namespace, std::move(cb)); +} + +void ReceiverSession::SendInputMessage(const InputMessage& message) { + if (negotiated_sender_id_.empty()) { + OSP_DLOG_WARN << "Can't send an INPUT message without a currently " + "negotiated session."; + return; + } + + input_messenger_.SendMessageToRemote(message); +} + void ReceiverSession::OnSocketReady() { if (pending_offer_) { InitializeSession(*pending_offer_); @@ -169,7 +234,7 @@ void ReceiverSession::OnOffer(const std::string& sender_id, properties->sender_id = sender_id; properties->sequence_number = message.sequence_number; - const Offer& offer = absl::get(message.body); + const Offer& offer = std::get(message.body); if (offer.cast_mode == CastMode::kRemoting) { if (!constraints_.remoting) { @@ -245,7 +310,7 @@ void ReceiverSession::OnCapabilitiesRequest(const std::string& sender_id, // NOTE: we respond to any arbitrary sender here, to allow sender to get // capabilities before making an OFFER. - const Error result = messenger_.SendMessage(sender_id, std::move(response)); + const Error result = messenger_->SendMessage(sender_id, std::move(response)); if (!result.ok()) { client_.OnError(this, result); } @@ -265,29 +330,30 @@ void ReceiverSession::OnRpcMessage(const std::string& sender_id, return; } - const auto& body = absl::get>(message.body); + const auto& body = std::get>(message.body); if (!rpc_messenger_) { OSP_DLOG_INFO << "Received an RPC message without having a messenger."; return; } - rpc_messenger_->ProcessMessageFromRemote(body.data(), body.size()); + rpc_messenger_->ProcessMessageFromRemote(body); } -void ReceiverSession::SendRpcMessage(std::vector message) { - if (negotiated_sender_id_.empty()) { - OSP_DLOG_WARN - << "Can't send an RPC message without a currently negotiated session."; +void ReceiverSession::OnInputMessage(const std::string& sender_id, + SenderMessage message) { + if (!message.valid) { + OSP_DLOG_WARN << "Bad INPUT message. This may or may not represent a " + "serious problem."; return; } - const Error error = messenger_.SendMessage( - negotiated_sender_id_, - ReceiverMessage{ReceiverMessage::Type::kRpc, -1, true /* valid */, - std::move(message)}); - - if (!error.ok()) { - OSP_LOG_WARN << "Failed to send RPC message: " << error; + if (sender_id != negotiated_sender_id_) { + OSP_DLOG_INFO << "Received an INPUT message from sender " << sender_id + << "--which we haven't negotiated with, dropping."; + return; } + + const auto& body = std::get>(message.body); + input_messenger_.ProcessMessageFromRemote(body); } void ReceiverSession::SelectStreams(const Offer& offer, @@ -303,7 +369,6 @@ void ReceiverSession::SelectStreams(const Offer& offer, } } else { OSP_CHECK(offer.cast_mode == CastMode::kRemoting); - if (offer.audio_streams.size() == 1) { properties->selected_audio = std::make_unique(offer.audio_streams[0]); @@ -316,6 +381,12 @@ void ReceiverSession::SelectStreams(const Offer& offer, } void ReceiverSession::InitializeSession(const PendingOffer& properties) { + // Enable DSCP on the UDP socket, if enabled and offered by the sender. + if (constraints_.enable_dscp) { + MaybeSetDscp(properties.selected_audio.get(), + properties.selected_video.get(), environment_); + } + Answer answer = ConstructAnswer(properties); if (!answer.IsValid()) { // If the answer message is invalid, there is no point in setting up a @@ -326,42 +397,49 @@ void ReceiverSession::InitializeSession(const PendingOffer& properties) { return; } - // Only spawn receivers if we know we have a valid answer message. + // Send the ANSWER before informing the client, in case we have an error, or + // the client happens to decide to send a message that depends on the ANSWER + // being sent. + const Error result = messenger_->SendMessage( + properties.sender_id, + ReceiverMessage{ReceiverMessage::Type::kAnswer, + properties.sequence_number, true /* valid */, + std::move(answer)}); + if (!result.ok()) { + client_.OnError(this, result); + } + ConfiguredReceivers receivers = SpawnReceivers(properties); negotiated_sender_id_ = properties.sender_id; + if (properties.mode == CastMode::kMirroring) { client_.OnNegotiated(this, std::move(receivers)); } else { rpc_messenger_ = std::make_unique([this](std::vector message) { - this->SendRpcMessage(std::move(message)); + const Error error = this->messenger_->SendRpcMessage( + this->negotiated_sender_id_, message); + if (!error.ok()) { + OSP_LOG_WARN << "Failed to send RPC message: " << error; + } }); client_.OnRemotingNegotiated( this, RemotingNegotiation{std::move(receivers), rpc_messenger_.get()}); } - - const Error result = messenger_.SendMessage( - negotiated_sender_id_, - ReceiverMessage{ReceiverMessage::Type::kAnswer, - properties.sequence_number, true /* valid */, - std::move(answer)}); - if (!result.ok()) { - client_.OnError(this, result); - } } std::unique_ptr ReceiverSession::ConstructReceiver( const Stream& stream) { // Session config is currently only for mirroring. - SessionConfig config = {stream.ssrc, stream.ssrc + 1, - stream.rtp_timebase, stream.channels, - stream.target_delay, stream.aes_key, - stream.aes_iv_mask, /* is_pli_enabled */ true}; + SessionConfig config(stream.ssrc, stream.ssrc + 1, stream.rtp_timebase, + stream.channels, stream.target_delay, stream.aes_key, + stream.aes_iv_mask, /* is_pli_enabled */ true, + StreamType::kUnknown, stream.receiver_rtcp_event_log); if (!config.IsValid()) { return nullptr; } - return std::make_unique(environment_, packet_router_, - std::move(config)); + return std::make_unique(environment_, packet_router_, + std::move(config)); } ReceiverSession::ConfiguredReceivers ReceiverSession::SpawnReceivers( @@ -395,10 +473,21 @@ ReceiverSession::ConfiguredReceivers ReceiverSession::SpawnReceivers( properties.selected_video->stream.codec_parameter}; } + bool input_enabled = false; + if (constraints_.supports_input_events) { + const auto* video = properties.selected_video.get(); + if (video && + Contains(video->stream.rtp_extensions, kInputEventsRtpExtension)) { + input_enabled = true; + } + } + return ConfiguredReceivers{current_audio_receiver_.get(), std::move(audio_config), current_video_receiver_.get(), - std::move(video_config), properties.sender_id}; + std::move(video_config), + input_enabled, + properties.sender_id}; } void ReceiverSession::ResetReceivers(Client::ReceiversDestroyingReason reason) { @@ -413,13 +502,22 @@ void ReceiverSession::ResetReceivers(Client::ReceiversDestroyingReason reason) { Answer ReceiverSession::ConstructAnswer(const PendingOffer& properties) { OSP_CHECK(properties.IsValid()); + // NOTE: The stream_indexes are intended to be always audio first, video + // second. Related arrays, such as stream_ssrcs and rtp_extensions, are + // expected to follow the same ordering. std::vector stream_indexes; std::vector stream_ssrcs; + std::vector stream_indexes_with_events; Constraints constraints; if (properties.selected_audio) { stream_indexes.push_back(properties.selected_audio->stream.index); stream_ssrcs.push_back(properties.selected_audio->stream.ssrc + 1); + if (properties.selected_audio->stream.receiver_rtcp_event_log) { + stream_indexes_with_events.push_back( + properties.selected_audio->stream.index); + } + for (const auto& limit : constraints_.audio_limits) { if (limit.codec == properties.selected_audio->codec || limit.applies_to_all_codecs) { @@ -433,9 +531,6 @@ Answer ReceiverSession::ConstructAnswer(const PendingOffer& properties) { } if (properties.selected_video) { - stream_indexes.push_back(properties.selected_video->stream.index); - stream_ssrcs.push_back(properties.selected_video->stream.ssrc + 1); - for (const auto& limit : constraints_.video_limits) { if (limit.codec == properties.selected_video->codec || limit.applies_to_all_codecs) { @@ -464,9 +559,48 @@ Answer ReceiverSession::ConstructAnswer(const PendingOffer& properties) { if (constraints.IsValid()) { answer_constraints = std::move(constraints); } - return Answer{environment_.GetBoundLocalEndpoint().port, - std::move(stream_indexes), std::move(stream_ssrcs), - answer_constraints, std::move(display)}; + + std::vector receiver_rtcp_dscp; + if (constraints_.enable_dscp) { + if (properties.selected_audio && + properties.selected_audio->stream.receiver_rtcp_dscp) { + receiver_rtcp_dscp.push_back(properties.selected_audio->stream.index); + } + if (properties.selected_video && + properties.selected_video->stream.receiver_rtcp_dscp) { + receiver_rtcp_dscp.push_back(properties.selected_video->stream.index); + } + } + + if (properties.selected_video) { + stream_indexes.push_back(properties.selected_video->stream.index); + stream_ssrcs.push_back(properties.selected_video->stream.ssrc + 1); + } + + std::vector> rtp_extensions; + if (constraints_.supports_input_events) { + const bool sender_requested_input_events = + properties.selected_video && + Contains(properties.selected_video->stream.rtp_extensions, + kInputEventsRtpExtension); + + if (sender_requested_input_events) { + rtp_extensions.emplace_back(); + rtp_extensions.push_back({kInputEventsRtpExtension}); + } + } + + return Answer{ + .udp_port = environment_.GetBoundLocalEndpoint().port, + .send_indexes = std::move(stream_indexes), + .ssrcs = std::move(stream_ssrcs), + .constraints = answer_constraints, + .display = std::move(display), + .receiver_rtcp_event_log = std::move(stream_indexes_with_events), + .receiver_rtcp_dscp = receiver_rtcp_dscp, + + // TODO(crbug.com/40238532): re-add support for adaptive playout delay?? + .rtp_extensions = std::move(rtp_extensions)}; } ReceiverCapability ReceiverSession::CreateRemotingCapabilityV2() { @@ -496,7 +630,7 @@ void ReceiverSession::SendErrorAnswerReply(const std::string& sender_id, int sequence_number, const Error& error) { OSP_DLOG_WARN << error; - const Error result = messenger_.SendMessage( + const Error result = messenger_->SendMessage( sender_id, ReceiverMessage{ReceiverMessage::Type::kAnswer, sequence_number, false /* valid */, ReceiverError(error)}); diff --git a/cast/streaming/public/receiver_session.h b/cast/streaming/public/receiver_session.h index 5df8a210f..ee2e753ac 100644 --- a/cast/streaming/public/receiver_session.h +++ b/cast/streaming/public/receiver_session.h @@ -5,6 +5,7 @@ #ifndef CAST_STREAMING_PUBLIC_RECEIVER_SESSION_H_ #define CAST_STREAMING_PUBLIC_RECEIVER_SESSION_H_ +#include #include #include #include @@ -13,18 +14,21 @@ #include "cast/common/public/message_port.h" #include "cast/streaming/capture_configs.h" #include "cast/streaming/impl/receiver_packet_router.h" -#include "cast/streaming/impl/session_config.h" +#include "cast/streaming/input.pb.h" #include "cast/streaming/public/constants.h" +#include "cast/streaming/public/environment.h" #include "cast/streaming/public/offer_messages.h" +#include "cast/streaming/public/protobuf_messenger.h" #include "cast/streaming/public/receiver_constraints.h" #include "cast/streaming/public/rpc_messenger.h" +#include "cast/streaming/public/session_config.h" #include "cast/streaming/public/session_messenger.h" #include "cast/streaming/resolution.h" #include "cast/streaming/sender_message.h" +#include "platform/base/ip_address.h" namespace openscreen::cast { -class Environment; class Receiver; // This class is responsible for listening for streaming requests from Cast @@ -62,6 +66,10 @@ class ReceiverSession final : public Environment::SocketSubscriber { Receiver* video_receiver; VideoCaptureConfig video_config; + // Set to true if input events were successfully negotiated for this + // session. + bool input_enabled; + // The ID of the sender that this set of receivers was configured to // communicate with. std::string sender_id; @@ -148,8 +156,28 @@ class ReceiverSession final : public Environment::SocketSubscriber { ReceiverSession& operator=(ReceiverSession&&) = delete; ~ReceiverSession() override; + // The RPC messenger for this session. NOTE: RPC messages may come at + // any time from the receiver, so subscriptions to RPC remoting messages + // should be done before calling `NegotiateRemoting`. + RpcMessenger* rpc_messenger() { return rpc_messenger_.get(); } + const std::string& session_id() const { return session_id_; } + // Set the callback for handling input events. If set, future negotiations + // will include support for input events. If not set, negotiations will not + // include input event support and further input event messages will be + // ignored. + void SetInputCallback(std::function callback); + + // Sends an input message to the currently negotiated sender. + void SendInputMessage(const InputMessage& message); + + void SetCustomMessageHandler( + std::string_view message_namespace, + ReceiverSessionMessenger::CustomMessageCallback cb); + + ReceiverSessionMessenger* messenger() { return messenger_.get(); } + // Environment::SocketSubscriber event callbacks. void OnSocketReady() override; void OnSocketInvalid(const Error& error) override; @@ -182,6 +210,7 @@ class ReceiverSession final : public Environment::SocketSubscriber { void OnCapabilitiesRequest(const std::string& sender_id, SenderMessage message); void OnRpcMessage(const std::string& sender_id, SenderMessage message); + void OnInputMessage(const std::string& sender_id, SenderMessage message); // Sends an RPC message to the currently negotiated sender. void SendRpcMessage(std::vector message); @@ -228,7 +257,7 @@ class ReceiverSession final : public Environment::SocketSubscriber { std::string negotiated_sender_id_; // The session messenger used for the lifetime of this session. - ReceiverSessionMessenger messenger_; + std::unique_ptr messenger_; // The packet router to be used for all Receivers spawned by this session. ReceiverPacketRouter packet_router_; @@ -244,6 +273,10 @@ class ReceiverSession final : public Environment::SocketSubscriber { // If remoting, we store the RpcMessenger used by the embedder to send RPC // messages from the remoting protobuf specification. std::unique_ptr rpc_messenger_; + + // The INPUT messenger, which uses the session messenger for sending INPUT + // messages. + ProtobufMessenger input_messenger_; }; } // namespace openscreen::cast diff --git a/cast/streaming/public/rpc_messenger.cc b/cast/streaming/public/rpc_messenger.cc index c63b0fc38..ddebcac65 100644 --- a/cast/streaming/public/rpc_messenger.cc +++ b/cast/streaming/public/rpc_messenger.cc @@ -41,10 +41,6 @@ constexpr RpcMessenger::Handle RpcMessenger::kAcquireRendererHandle; constexpr RpcMessenger::Handle RpcMessenger::kAcquireDemuxerHandle; constexpr RpcMessenger::Handle RpcMessenger::kFirstHandle; -RpcMessenger::RpcMessenger(SendMessageCallback send_message_cb) - : next_handle_(kFirstHandle), - send_message_cb_(std::move(send_message_cb)) {} - RpcMessenger::~RpcMessenger() { receive_callbacks_.clear(); } @@ -66,33 +62,14 @@ void RpcMessenger::UnregisterMessageReceiverCallback( receive_callbacks_.erase_key(handle); } -void RpcMessenger::ProcessMessageFromRemote(const uint8_t* message, - std::size_t message_len) { - auto rpc = std::make_unique(); - if (!rpc->ParseFromArray(message, message_len)) { - OSP_DLOG_WARN << "Failed to parse RPC message from remote: \"" << message - << "\""; - return; - } - ProcessMessageFromRemote(std::move(rpc)); -} - void RpcMessenger::ProcessMessageFromRemote( std::unique_ptr message) { - const auto entry = receive_callbacks_.find(message->handle()); - if (entry == receive_callbacks_.end()) { - OSP_VLOG << "Dropping message due to unregistered handle: " - << message->handle(); - return; - } - entry->second(std::move(message)); + OnMessage(std::move(message)); } void RpcMessenger::SendMessageToRemote(const RpcMessage& rpc) { OSP_VLOG << "Sending RPC message: " << rpc; - std::vector message(rpc.ByteSizeLong()); - rpc.SerializeToArray(message.data(), message.size()); - send_message_cb_(std::move(message)); + ProtobufMessenger::SendMessageToRemote(rpc); } bool RpcMessenger::IsRegisteredForTesting(RpcMessenger::Handle handle) { @@ -103,4 +80,14 @@ WeakPtr RpcMessenger::GetWeakPtr() { return weak_factory_.GetWeakPtr(); } +void RpcMessenger::OnMessage(std::unique_ptr message) { + const auto entry = receive_callbacks_.find(message->handle()); + if (entry == receive_callbacks_.end()) { + OSP_VLOG << "Dropping message due to unregistered handle: " + << message->handle(); + return; + } + entry->second(std::move(message)); +} + } // namespace openscreen::cast diff --git a/cast/streaming/public/rpc_messenger.h b/cast/streaming/public/rpc_messenger.h index 3032edd7f..50a3401a3 100644 --- a/cast/streaming/public/rpc_messenger.h +++ b/cast/streaming/public/rpc_messenger.h @@ -11,6 +11,7 @@ #include #include +#include "cast/streaming/public/protobuf_messenger.h" #include "cast/streaming/remoting.pb.h" #include "util/flat_map.h" #include "util/weak_ptr.h" @@ -24,24 +25,14 @@ namespace openscreen::cast { // Before RPC communication starts, both sides need to negotiate the handle // value in the existing RPC communication channel using the special handles // |kAcquire*Handle|. -// -// NOTE: RpcMessenger doesn't actually send RPC messages to the remote. The -// session messenger needs to set SendMessageCallback, and call -// ProcessMessageFromRemote as appropriate. The RpcMessenger then distributes -// each RPC message to the subscribed component. -class RpcMessenger { +class RpcMessenger final : public ProtobufMessenger { public: using Handle = int; using ReceiveMessageCallback = std::function)>; - using SendMessageCallback = std::function)>; - explicit RpcMessenger(SendMessageCallback send_message_cb); - RpcMessenger(const RpcMessenger&) = delete; - RpcMessenger(RpcMessenger&&) noexcept; - RpcMessenger& operator=(const RpcMessenger&) = delete; - RpcMessenger& operator=(RpcMessenger&&); - ~RpcMessenger(); + using ProtobufMessenger::ProtobufMessenger; + ~RpcMessenger() override; // Get unique handle value for RPC message handles. Handle GetUniqueHandle(); @@ -57,11 +48,8 @@ class RpcMessenger { // Allows components to unregister in order to stop receiving message. void UnregisterMessageReceiverCallback(Handle handle); - // Distributes an incoming RPC message to the registered (if any) component. - // The `serialized_message` should be already base64-decoded and ready for - // deserialization by protobuf. - void ProcessMessageFromRemote(const uint8_t* message, - std::size_t message_len); + using ProtobufMessenger::ProcessMessageFromRemote; + // This overload distributes an already-deserialized message to the // registered component. void ProcessMessageFromRemote(std::unique_ptr message); @@ -75,12 +63,6 @@ class RpcMessenger { // Weak pointer creator. WeakPtr GetWeakPtr(); - // Consumers of RPCMessenger may set the send message callback post-hoc - // in order to simulate different scenarios. - void set_send_message_cb_for_testing(SendMessageCallback cb) { - send_message_cb_ = std::move(cb); - } - // Predefined invalid handle value for RPC message. static constexpr Handle kInvalidHandle = -1; @@ -92,16 +74,16 @@ class RpcMessenger { // The first handle to return from GetUniqueHandle(). static constexpr Handle kFirstHandle = 100; + protected: + void OnMessage(std::unique_ptr message) override; + private: // Next unique handle to return from GetUniqueHandle(). - Handle next_handle_; + Handle next_handle_ = kFirstHandle; // Maps of handle values to associated MessageReceivers. FlatMap receive_callbacks_; - // Callback that is ran to send a serialized message. - SendMessageCallback send_message_cb_; - WeakPtrFactory weak_factory_{this}; }; diff --git a/cast/streaming/public/sender.cc b/cast/streaming/public/sender.cc index b514f3bf4..a910b1e9a 100644 --- a/cast/streaming/public/sender.cc +++ b/cast/streaming/public/sender.cc @@ -4,744 +4,9 @@ #include "cast/streaming/public/sender.h" -#include -#include -#include -#include - -#include "cast/streaming/impl/rtp_defines.h" -#include "cast/streaming/impl/session_config.h" -#include "cast/streaming/impl/statistics_defines.h" -#include "platform/base/trivial_clock_traits.h" -#include "util/chrono_helpers.h" -#include "util/osp_logging.h" -#include "util/std_util.h" -#include "util/trace_logging.h" - namespace openscreen::cast { -using clock_operators::operator<<; - -namespace { - -void DispatchEnqueueEvents(StreamType stream_type, - const EncodedFrame& frame, - Environment& environment) { - if (!environment.statistics_collector()) { - return; - } - - const StatisticsEventMediaType media_type = ToMediaType(stream_type); - - // Submit a capture begin event. - FrameEvent capture_begin_event; - capture_begin_event.type = StatisticsEventType::kFrameCaptureBegin; - capture_begin_event.media_type = media_type; - capture_begin_event.rtp_timestamp = frame.rtp_timestamp; - capture_begin_event.timestamp = - (frame.capture_begin_time > Clock::time_point::min()) - ? frame.capture_begin_time - : environment.now(); - environment.statistics_collector()->CollectFrameEvent( - std::move(capture_begin_event)); - - // Submit a capture end event. - FrameEvent capture_end_event; - capture_end_event.type = StatisticsEventType::kFrameCaptureEnd; - capture_end_event.media_type = media_type; - capture_end_event.rtp_timestamp = frame.rtp_timestamp; - capture_end_event.timestamp = - (frame.capture_end_time > Clock::time_point::min()) - ? frame.capture_end_time - : environment.now(); - environment.statistics_collector()->CollectFrameEvent( - std::move(capture_end_event)); - - // Submit an encoded event. - FrameEvent encode_event; - encode_event.timestamp = environment.now(); - encode_event.type = StatisticsEventType::kFrameEncoded; - encode_event.media_type = media_type; - encode_event.rtp_timestamp = frame.rtp_timestamp; - encode_event.frame_id = frame.frame_id; - encode_event.size = static_cast(frame.data.size()); - encode_event.key_frame = - frame.dependency == openscreen::cast::EncodedFrame::Dependency::kKeyFrame; - - environment.statistics_collector()->CollectFrameEvent( - std::move(encode_event)); -} - -void DispatchAckEvent(StreamType stream_type, - RtpTimeTicks rtp_timestamp, - FrameId frame_id, - Environment& environment) { - if (!environment.statistics_collector()) { - return; - } - - FrameEvent ack_event; - ack_event.timestamp = environment.now(); - ack_event.type = StatisticsEventType::kFrameAckReceived; - ack_event.media_type = ToMediaType(stream_type); - ack_event.rtp_timestamp = rtp_timestamp; - ack_event.frame_id = frame_id; - - environment.statistics_collector()->CollectFrameEvent(std::move(ack_event)); -} - -// TODO(issuetracker.google.com/298277160): move into a helper file, add tests. -void DispatchFrameLogMessages( - StreamType stream_type, - const std::vector& messages, - Environment& environment) { - if (!environment.statistics_collector()) { - return; - } - - const Clock::time_point now = environment.now(); - const StatisticsEventMediaType media_type = ToMediaType(stream_type); - for (const RtcpReceiverFrameLogMessage& log_message : messages) { - for (const RtcpReceiverEventLogMessage& event_message : - log_message.messages) { - switch (event_message.type) { - case StatisticsEventType::kPacketReceived: { - PacketEvent event; - event.timestamp = event_message.timestamp; - event.received_timestamp = now; - event.type = event_message.type; - event.media_type = media_type; - event.rtp_timestamp = log_message.rtp_timestamp; - event.packet_id = event_message.packet_id; - environment.statistics_collector()->CollectPacketEvent( - std::move(event)); - } break; - - case StatisticsEventType::kFrameAckSent: - case StatisticsEventType::kFrameDecoded: - case StatisticsEventType::kFramePlayedOut: { - FrameEvent event; - event.timestamp = event_message.timestamp; - event.received_timestamp = now; - event.type = event_message.type; - event.media_type = media_type; - event.rtp_timestamp = log_message.rtp_timestamp; - if (event.type == StatisticsEventType::kFramePlayedOut) { - event.delay_delta = event_message.delay; - } - environment.statistics_collector()->CollectFrameEvent( - std::move(event)); - } break; - - default: - OSP_VLOG << "Received log message via RTCP that we did not expect, " - "StatisticsEventType=" - << static_cast(event_message.type); - break; - } - } - } -} - -} // namespace - -Sender::Sender(Environment& environment, - SenderPacketRouter& packet_router, - SessionConfig config, - RtpPayloadType rtp_payload_type) - : environment_(environment), - config_(config), - packet_router_(packet_router), - rtcp_session_(config.sender_ssrc, - config.receiver_ssrc, - environment.now()), - rtcp_parser_(rtcp_session_, *this), - sender_report_builder_(rtcp_session_), - rtp_packetizer_(rtp_payload_type, - config.sender_ssrc, - packet_router_.max_packet_size()), - rtp_timebase_(config.rtp_timebase), - crypto_(config.aes_secret_key, config.aes_iv_mask), - target_playout_delay_(config.target_playout_delay) { - OSP_CHECK_NE(rtcp_session_.sender_ssrc(), rtcp_session_.receiver_ssrc()); - OSP_CHECK_GT(rtp_timebase_, 0); - OSP_CHECK_GT(target_playout_delay_, milliseconds::zero()); - - pending_sender_report_.reference_time = SenderPacketRouter::kNever; - - packet_router_.OnSenderCreated(rtcp_session_.receiver_ssrc(), this); -} - -Sender::~Sender() { - packet_router_.OnSenderDestroyed(rtcp_session_.receiver_ssrc()); -} - -void Sender::SetObserver(Sender::Observer* observer) { - observer_ = observer; -} - -int Sender::GetInFlightFrameCount() const { - return num_frames_in_flight_; -} - -Clock::duration Sender::GetInFlightMediaDuration( - RtpTimeTicks next_frame_rtp_timestamp) const { - if (num_frames_in_flight_ == 0) { - return Clock::duration::zero(); // No frames are currently in-flight. - } - - const PendingFrameSlot& oldest_slot = get_slot_for(checkpoint_frame_id_ + 1); - // Note: The oldest slot's frame cannot have been canceled because the - // protocol does not allow ACK'ing this particular frame without also moving - // the checkpoint forward. See "CST2 feedback" discussion in rtp_defines.h. - OSP_CHECK(oldest_slot.is_active_for_frame(checkpoint_frame_id_ + 1)); - - return (next_frame_rtp_timestamp - oldest_slot.frame->rtp_timestamp) - .ToDuration(rtp_timebase_); -} - -Clock::duration Sender::GetMaxInFlightMediaDuration() const { - // Assumption: The total amount of allowed in-flight media should equal the - // half of the playout delay window, plus the amount of time it takes to - // receive an ACK from the Receiver. - // - // Why half of the playout delay window? It's assumed here that capture and - // media encoding, which occur before EnqueueFrame() is called, are executing - // within the first half of the playout delay window. This leaves the second - // half for executing all network transmits/re-transmits, plus decoding and - // play-out at the Receiver. - return (target_playout_delay_ / 2) + (round_trip_time_ / 2); -} - -bool Sender::NeedsKeyFrame() const { - return last_enqueued_key_frame_id_ <= picture_lost_at_frame_id_; -} - -FrameId Sender::GetNextFrameId() const { - return last_enqueued_frame_id_ + 1; -} - -Clock::duration Sender::GetCurrentRoundTripTime() const { - return round_trip_time_; -} - -Sender::EnqueueFrameResult Sender::EnqueueFrame(const EncodedFrame& frame) { - // Assume the fields of the `frame` have all been set correctly, with - // monotonically increasing timestamps and a valid pointer to the data. - OSP_CHECK_EQ(frame.frame_id, GetNextFrameId()); - OSP_CHECK_GE(frame.referenced_frame_id, FrameId::first()); - if (frame.frame_id != FrameId::first()) { - OSP_CHECK_GT(frame.rtp_timestamp, pending_sender_report_.rtp_timestamp); - if (frame.reference_time <= pending_sender_report_.reference_time) { - OSP_DLOG_WARN << "Frame " << frame.frame_id - << " has non-monotonic reference_time: " - << frame.reference_time - << " <= " << pending_sender_report_.reference_time; - } - } - OSP_CHECK(frame.data.data()); - - // Check whether enqueuing the frame would exceed the design limit for the - // span of FrameIds. Even if `num_frames_in_flight_` is less than - // kMaxUnackedFrames, it's the span of FrameIds that is restricted. - if ((frame.frame_id - checkpoint_frame_id_) > kMaxUnackedFrames) { - return REACHED_ID_SPAN_LIMIT; - } - - // Check whether enqueuing the frame would exceed the current maximum media - // duration limit. - if (GetInFlightMediaDuration(frame.rtp_timestamp) > - GetMaxInFlightMediaDuration()) { - return MAX_DURATION_IN_FLIGHT; - } - - // Encrypt the frame and initialize the slot tracking its sending. - PendingFrameSlot& slot = get_slot_for(frame.frame_id); - OSP_CHECK(!slot.frame); - slot.frame = crypto_.Encrypt(frame); - const int packet_count = rtp_packetizer_.ComputeNumberOfPackets(*slot.frame); - if (packet_count <= 0) { - slot.frame.reset(); - return PAYLOAD_TOO_LARGE; - } - slot.send_flags.Resize(packet_count, YetAnotherBitVector::SET); - slot.packet_sent_times.assign(packet_count, SenderPacketRouter::kNever); - - // Officially record the "enqueue." - ++num_frames_in_flight_; - last_enqueued_frame_id_ = slot.frame->frame_id; - OSP_CHECK_LE(num_frames_in_flight_, - last_enqueued_frame_id_ - checkpoint_frame_id_); - if (slot.frame->dependency == EncodedFrame::Dependency::kKeyFrame) { - last_enqueued_key_frame_id_ = slot.frame->frame_id; - } - - // Update the target playout delay, if necessary. - if (slot.frame->new_playout_delay > milliseconds::zero()) { - target_playout_delay_ = slot.frame->new_playout_delay; - playout_delay_change_at_frame_id_ = slot.frame->frame_id; - } - - // Update the lip-sync information for the next Sender Report, ensuring that - // the reference time is monotonically increasing. - pending_sender_report_.reference_time = - frame.frame_id == FrameId::first() - ? slot.frame->reference_time - : std::max(slot.frame->reference_time, - pending_sender_report_.reference_time); - pending_sender_report_.rtp_timestamp = slot.frame->rtp_timestamp; - - // If the round trip time hasn't been computed yet, immediately send a RTCP - // packet (i.e., before the RTP packets are sent). The RTCP packet will - // provide a Sender Report which contains the required lip-sync information - // the Receiver needs for timing the media playout. - // - // Detail: Working backwards, if the round trip time is not known, then this - // Sender has never processed a Receiver Report. Thus, the Receiver has never - // provided a Receiver Report, which it can only do after having processed a - // Sender Report from this Sender. Thus, this Sender really needs to send - // that, right now! - if (round_trip_time_ == Clock::duration::zero()) { - packet_router_.RequestRtcpSend(rtcp_session_.receiver_ssrc()); - } - - // Re-activate RTP sending if it was suspended. - packet_router_.RequestRtpSend(rtcp_session_.receiver_ssrc()); - DispatchEnqueueEvents(config_.stream_type, frame, environment_); - - return OK; -} - -void Sender::CancelInFlightData() { - TRACE_SCOPED1(TraceCategory::kSender, "CancelInFlightData", - "frames_in_flight", - std::to_string(last_enqueued_frame_id_ - checkpoint_frame_id_)); - - while (checkpoint_frame_id_ < last_enqueued_frame_id_) { - ++checkpoint_frame_id_; - CancelPendingFrame(checkpoint_frame_id_, /*was_acked*/ false); - } - DispatchCancellations(); -} - -void Sender::OnReceivedRtcpPacket(Clock::time_point arrival_time, - ByteView packet) { - rtcp_packet_arrival_time_ = arrival_time; - // This call to Parse() invoke zero or more of the OnReceiverXYZ() methods in - // the current call stack: - if (rtcp_parser_.Parse(packet, last_enqueued_frame_id_)) { - packet_router_.OnRtcpReceived(arrival_time, round_trip_time_); - } -} - -ByteBuffer Sender::GetRtcpPacketForImmediateSend(Clock::time_point send_time, - ByteBuffer buffer) { - if (pending_sender_report_.reference_time == SenderPacketRouter::kNever) { - // Cannot send a report if one is not available (i.e., a frame has never - // been enqueued). - return buffer.subspan(0, 0); - } - - // The Sender Report to be sent is a snapshot of the "pending Sender Report," - // but with its timestamp fields modified. First, the reference time is set to - // the RTCP packet's send time. Then, the corresponding RTP timestamp is - // translated to match (for lip-sync). - RtcpSenderReport sender_report = pending_sender_report_; - sender_report.reference_time = send_time; - sender_report.rtp_timestamp += RtpTimeDelta::FromDuration( - sender_report.reference_time - pending_sender_report_.reference_time, - rtp_timebase_); - - return sender_report_builder_.BuildPacket(sender_report, buffer).first; -} - -ByteBuffer Sender::GetRtpPacketForImmediateSend(Clock::time_point send_time, - ByteBuffer buffer) { - ChosenPacket chosen = ChooseNextRtpPacketNeedingSend(); - - // If no packets need sending (i.e., all packets have been sent at least once - // and do not need to be re-sent yet), check whether a Kickstart packet should - // be sent. It's possible that there has been complete packet loss of some - // frames, and the Receiver may not be aware of the existence of the latest - // frame(s). Kickstarting is the only way the Receiver can discover the newer - // frames it doesn't know about. - if (!chosen) { - const ChosenPacketAndWhen kickstart = ChooseKickstartPacket(); - if (kickstart.when > send_time) { - // Nothing to send, so return "empty" signal to the packet router. The - // packet router will suspend RTP sending until this Sender explicitly - // resumes it. - return buffer.subspan(0, 0); - } - chosen = kickstart; - OSP_CHECK(chosen); - } - - const ByteBuffer result = rtp_packetizer_.GeneratePacket( - *chosen.slot->frame, chosen.packet_id, buffer); - chosen.slot->send_flags.Clear(chosen.packet_id); - chosen.slot->packet_sent_times[chosen.packet_id] = send_time; - - ++pending_sender_report_.send_packet_count; - // According to RFC3550, the octet count does not include the RTP header. The - // following is just a good approximation, however, because the header size - // will very infrequently be 4 bytes greater (see - // RtpPacketizer::kAdaptiveLatencyHeaderSize). No known Cast Streaming - // Receiver implementations use this for anything, and so this should be fine. - const int approximate_octet_count = - static_cast(result.size()) - RtpPacketizer::kBaseRtpHeaderSize; - OSP_CHECK_GE(approximate_octet_count, 0); - pending_sender_report_.send_octet_count += approximate_octet_count; - - return result; -} - -Clock::time_point Sender::GetRtpResumeTime() { - if (ChooseNextRtpPacketNeedingSend()) { - return Alarm::kImmediately; - } - return ChooseKickstartPacket().when; -} - -RtpTimeTicks Sender::GetLastRtpTimestamp() const { - return {}; -} - -StreamType Sender::GetStreamType() const { - return config_.stream_type; -} - -void Sender::OnReceiverReferenceTimeAdvanced(Clock::time_point reference_time) { - // Not used. -} - -void Sender::OnReceiverReport(const RtcpReportBlock& receiver_report) { - OSP_CHECK_NE(rtcp_packet_arrival_time_, SenderPacketRouter::kNever); - - const Clock::duration total_delay = - rtcp_packet_arrival_time_ - - sender_report_builder_.GetRecentReportTime( - receiver_report.last_status_report_id, rtcp_packet_arrival_time_); - const auto non_network_delay = - Clock::to_duration(receiver_report.delay_since_last_report); - - // Round trip time measurement: This is the time elapsed since the Sender - // Report was sent, minus the time the Receiver did other stuff before sending - // the Receiver Report back. - // - // If the round trip time seems to be less than or equal to zero, assume clock - // imprecision by one or both peers caused a bad value to be calculated. The - // true value is likely very close to zero (i.e., this is ideal network - // behavior); and so just represent this as 75 µs, an optimistic - // wired-Ethernet LAN ping time. - constexpr auto kNearZeroRoundTripTime = Clock::to_duration(microseconds(75)); - static_assert(kNearZeroRoundTripTime > Clock::duration::zero(), - "More precision in Clock::duration needed!"); - const Clock::duration measurement = - std::max(total_delay - non_network_delay, kNearZeroRoundTripTime); - - // Validate the measurement by using the current target playout delay as a - // "reasonable upper-bound." It's certainly possible that the actual network - // round-trip time could exceed the target playout delay, but that would mean - // the current network performance is totally inadequate for streaming anyway. - if (measurement > target_playout_delay_) { - OSP_LOG_WARN << "Invalidating a round-trip time measurement (" - << measurement - << ") since it exceeds the current target playout delay (" - << target_playout_delay_ << ")."; - return; - } - - // Measurements will typically have high variance. Use a simple smoothing - // filter to track a short-term average that changes less drastically. - if (round_trip_time_ == Clock::duration::zero()) { - round_trip_time_ = measurement; - } else { - // Arbitrary constant, to provide 1/8 weight to the new measurement, and 7/8 - // weight to the old estimate, which seems to work well for de-noising the - // estimate. - constexpr int kInertia = 7; - round_trip_time_ = - (kInertia * round_trip_time_ + measurement) / (kInertia + 1); - } - TRACE_SCOPED1(TraceCategory::kSender, "UpdatedRoundTripTime", - "round_trip_time", ToString(round_trip_time_)); -} - -void Sender::OnCastReceiverFrameLogMessages( - std::vector messages) { - DispatchFrameLogMessages(config_.stream_type, messages, environment_); -} - -void Sender::OnReceiverIndicatesPictureLoss() { - TRACE_SCOPED1(TraceCategory::kSender, "OnReceiverIndicatesPictureLoss", - "last_received_frame_id", picture_lost_at_frame_id_.ToString()); - // The Receiver will continue the PLI notifications until it has received a - // key frame. Thus, if a key frame is already in-flight, don't make a state - // change that would cause this Sender to force another expensive key frame. - if (checkpoint_frame_id_ < last_enqueued_key_frame_id_) { - return; - } - - picture_lost_at_frame_id_ = checkpoint_frame_id_; - - if (observer_) { - observer_->OnPictureLost(); - } - - // Note: It may seem that all pending frames should be canceled until - // EnqueueFrame() is called with a key frame. However: - // - // 1. The Receiver should still be the main authority on what frames/packets - // are being ACK'ed and NACK'ed. - // - // 2. It may be desirable for the Receiver to be "limping along" in the - // meantime. For example, video may be corrupted but mostly watchable, - // and so it's best for the Sender to continue sending the non-key frames - // until the Receiver indicates otherwise. -} - -void Sender::OnReceiverCheckpoint(FrameId frame_id, - milliseconds playout_delay) { - TRACE_SCOPED2(TraceCategory::kSender, "OnReceiverCheckpoint", "frame_id", - frame_id.ToString(), "playout_delay", ToString(playout_delay)); - if (frame_id > last_enqueued_frame_id_) { - TRACE_SET_RESULT(Error::Code::kParameterOutOfRange); - OSP_LOG_ERROR - << "Ignoring checkpoint for " << latest_expected_frame_id_ - << " because this Sender could not have sent any frames after " - << last_enqueued_frame_id_ << '.'; - return; - } - // CompoundRtcpParser should guarantee this: - OSP_CHECK_GE(playout_delay, milliseconds::zero()); - while (checkpoint_frame_id_ < frame_id) { - ++checkpoint_frame_id_; - PendingFrameSlot& slot = get_slot_for(checkpoint_frame_id_); - if (slot.is_active_for_frame(checkpoint_frame_id_)) { - const RtpTimeTicks rtp_timestamp = slot.frame->rtp_timestamp; - DispatchAckEvent(config_.stream_type, rtp_timestamp, checkpoint_frame_id_, - environment_); - CancelPendingFrame(checkpoint_frame_id_, /*was_acked*/ true); - } - } - latest_expected_frame_id_ = std::max(latest_expected_frame_id_, frame_id); - DispatchCancellations(); - - if (playout_delay != target_playout_delay_ && - frame_id >= playout_delay_change_at_frame_id_) { - OSP_LOG_WARN << "Sender's target playout delay (" << target_playout_delay_ - << ") disagrees with the Receiver's (" << playout_delay << ")"; - } -} - -void Sender::OnReceiverHasFrames(std::vector acks) { - OSP_DCHECK(!acks.empty() && AreElementsSortedAndUnique(acks)); - TRACE_SCOPED1(TraceCategory::kSender, "OnReceiverHasFrames", "frame_ids", - Join(acks)); - - if (acks.back() > last_enqueued_frame_id_) { - TRACE_SET_RESULT(Error::Code::kParameterOutOfRange); - OSP_LOG_ERROR << "Ignoring individual frame ACKs: ACKing frame " - << latest_expected_frame_id_ - << " is invalid because this Sender could not have sent any " - "frames after " - << last_enqueued_frame_id_ << '.'; - return; - } - - for (FrameId id : acks) { - PendingFrameSlot& slot = get_slot_for(id); - if (slot.is_active_for_frame(id)) { - const RtpTimeTicks rtp_timestamp = slot.frame->rtp_timestamp; - DispatchAckEvent(config_.stream_type, rtp_timestamp, id, environment_); - } - CancelPendingFrame(id, /*was_acked*/ true); - } - latest_expected_frame_id_ = std::max(latest_expected_frame_id_, acks.back()); - DispatchCancellations(); -} - -void Sender::OnReceiverIsMissingPackets(std::vector nacks) { - TRACE_SCOPED1(TraceCategory::kSender, "OnReceiverIsMissingPackets", - "number_of_packets", std::to_string(nacks.size())); - OSP_DCHECK(!nacks.empty() && AreElementsSortedAndUnique(nacks)); - OSP_CHECK_NE(rtcp_packet_arrival_time_, SenderPacketRouter::kNever); - - // This is a point-in-time threshold that indicates whether each NACK will - // trigger a packet retransmit. The threshold is based on the network round - // trip time because a Receiver's NACK may have been issued while the needed - // packet was in-flight from the Sender. In such cases, the Receiver's NACK is - // likely stale and this Sender should not redundantly re-transmit the packet - // again. - const Clock::time_point too_recent_a_send_time = - rtcp_packet_arrival_time_ - round_trip_time_; - - // Iterate over all the NACKs... - bool need_to_send = false; - for (auto nack_it = nacks.begin(); nack_it != nacks.end();) { - // Find the slot associated with the NACK's frame ID. - const FrameId frame_id = nack_it->frame_id; - PendingFrameSlot* slot = nullptr; - if (frame_id <= last_enqueued_frame_id_) { - PendingFrameSlot& candidate_slot = get_slot_for(frame_id); - if (candidate_slot.is_active_for_frame(frame_id)) { - slot = &candidate_slot; - } - } - - // If no slot was found (i.e., the NACK is invalid) for the frame, skip-over - // all other NACKs for the same frame. While it seems to be a bug that the - // Receiver would attempt to NACK a frame that does not yet exist, this can - // happen in rare cases where RTCP packets arrive out-of-order (i.e., the - // network shuffled them). - if (!slot) { - TRACE_SCOPED1(TraceCategory::kSender, "MissingNackSlot", "frame_id", - frame_id.ToString()); - for (++nack_it; nack_it != nacks.end() && nack_it->frame_id == frame_id; - ++nack_it) { - } - continue; - } - - latest_expected_frame_id_ = std::max(latest_expected_frame_id_, frame_id); - - const auto HandleIndividualNack = [&](FramePacketId packet_id) { - if (slot->packet_sent_times[packet_id] <= too_recent_a_send_time) { - slot->send_flags.Set(packet_id); - need_to_send = true; - } - }; - const FramePacketId range_end = slot->packet_sent_times.size(); - if (nack_it->packet_id == kAllPacketsLost) { - for (FramePacketId packet_id = 0; packet_id < range_end; ++packet_id) { - HandleIndividualNack(packet_id); - } - ++nack_it; - } else { - do { - if (nack_it->packet_id < range_end) { - HandleIndividualNack(nack_it->packet_id); - } else { - OSP_LOG_WARN - << "Ignoring NACK for packet that doesn't exist in frame " - << frame_id << ": " << static_cast(nack_it->packet_id); - } - ++nack_it; - } while (nack_it != nacks.end() && nack_it->frame_id == frame_id); - } - } - - if (need_to_send) { - packet_router_.RequestRtpSend(rtcp_session_.receiver_ssrc()); - } -} - -Sender::ChosenPacket Sender::ChooseNextRtpPacketNeedingSend() { - // Find the oldest packet needing to be sent (or re-sent). - for (FrameId frame_id = checkpoint_frame_id_ + 1; - frame_id <= last_enqueued_frame_id_; ++frame_id) { - PendingFrameSlot& slot = get_slot_for(frame_id); - if (!slot.is_active_for_frame(frame_id)) { - continue; // Frame was canceled. None of its packets need to be sent. - } - const FramePacketId packet_id = slot.send_flags.FindFirstSet(); - if (packet_id < slot.send_flags.size()) { - return {&slot, packet_id}; - } - } - - return {}; // Nothing needs to be sent. -} - -Sender::ChosenPacketAndWhen Sender::ChooseKickstartPacket() { - if (latest_expected_frame_id_ >= last_enqueued_frame_id_) { - // Since the Receiver must know about all of the frames currently queued, no - // Kickstart packet is necessary. - return {}; - } - - // The Kickstart packet is always in the last-enqueued frame, so that the - // Receiver will know about every frame the Sender has. However, which packet - // should be chosen? Any would do, since all packets contain the frame's total - // packet count. For historical reasons, all sender implementations have - // always just sent the last packet; and so that tradition is continued here. - ChosenPacketAndWhen chosen; - chosen.slot = &get_slot_for(last_enqueued_frame_id_); - // Note: This frame cannot have been canceled since - // `latest_expected_frame_id_` hasn't yet reached this point. - OSP_CHECK(chosen.slot->is_active_for_frame(last_enqueued_frame_id_)); - chosen.packet_id = chosen.slot->send_flags.size() - 1; - - const Clock::time_point time_last_sent = - chosen.slot->packet_sent_times[chosen.packet_id]; - // Sanity-check: This method should not be called to choose a packet while - // there are still unsent packets. - OSP_CHECK_NE(time_last_sent, SenderPacketRouter::kNever); - - // The desired Kickstart interval is a fraction of the total - // `target_playout_delay_`. The reason for the specific ratio here is based on - // lost knowledge (from legacy implementations); but it makes sense (i.e., to - // be a good "network citizen") to be less aggressive for larger playout delay - // windows, and more aggressive for shorter ones to avoid too-late packet - // arrivals. - using kWaitFraction = std::ratio<1, 20>; - const Clock::duration desired_kickstart_interval = - Clock::to_duration(target_playout_delay_) * kWaitFraction::num / - kWaitFraction::den; - // The actual interval used is increased, if current network performance - // warrants waiting longer. Don't send a Kickstart packet until no NACKs - // have been received for two network round-trip periods. - constexpr int kLowerBoundRoundTrips = 2; - const Clock::duration kickstart_interval = std::max( - desired_kickstart_interval, round_trip_time_ * kLowerBoundRoundTrips); - chosen.when = time_last_sent + kickstart_interval; - - return chosen; -} - -void Sender::CancelPendingFrame(FrameId frame_id, bool was_acked) { - TRACE_SCOPED1(TraceCategory::kSender, "CancelPendingFrame", "frame_id", - frame_id.ToString()); - - PendingFrameSlot& slot = get_slot_for(frame_id); - if (!slot.is_active_for_frame(frame_id)) { - return; // Frame was already canceled. - } - - if (was_acked) { - packet_router_.OnPayloadReceived( - slot.frame->data.size(), rtcp_packet_arrival_time_, round_trip_time_); - } - - slot.frame.reset(); - OSP_CHECK_GT(num_frames_in_flight_, 0); - --num_frames_in_flight_; - if (observer_) { - pending_cancellations_.emplace_back(frame_id); - } -} - -void Sender::DispatchCancellations() { - if (observer_) { - for (const FrameId id : pending_cancellations_) { - observer_->OnFrameCanceled(id); - } - } - pending_cancellations_.clear(); - - // At this point, there should either be no frames in flight, or the frame - // immediately after `checkpoint_frame_id_` must be valid. - OSP_DCHECK((num_frames_in_flight_ == 0) || - get_slot_for(checkpoint_frame_id_ + 1) - .is_active_for_frame(checkpoint_frame_id_ + 1)); -} - -void Sender::Observer::OnFrameCanceled(FrameId frame_id) {} -void Sender::Observer::OnPictureLost() {} Sender::Observer::~Observer() = default; - -Sender::PendingFrameSlot::PendingFrameSlot() = default; -Sender::PendingFrameSlot::~PendingFrameSlot() = default; +Sender::~Sender() = default; } // namespace openscreen::cast diff --git a/cast/streaming/public/sender.h b/cast/streaming/public/sender.h index 1b14f8b2d..f6f34e4c1 100644 --- a/cast/streaming/public/sender.h +++ b/cast/streaming/public/sender.h @@ -7,33 +7,19 @@ #include -#include #include -#include -#include -#include "cast/streaming/impl/compound_rtcp_parser.h" -#include "cast/streaming/impl/frame_crypto.h" -#include "cast/streaming/impl/rtcp_common.h" -#include "cast/streaming/impl/rtp_defines.h" -#include "cast/streaming/impl/rtp_packetizer.h" -#include "cast/streaming/impl/sender_report_builder.h" -#include "cast/streaming/impl/session_config.h" -#include "cast/streaming/public/constants.h" +#include "cast/streaming/public/encoded_frame.h" #include "cast/streaming/public/frame_id.h" +#include "cast/streaming/public/session_config.h" #include "cast/streaming/rtp_time.h" -#include "cast/streaming/sender_packet_router.h" +#include "cast/streaming/ssrc.h" #include "platform/api/time.h" -#include "platform/base/span.h" -#include "util/yet_another_bit_vector.h" namespace openscreen::cast { -class Environment; - // The Cast Streaming Sender, a peer corresponding to some Cast Streaming -// Receiver at the other end of a network link. See class level comments for -// Receiver for a high-level overview. +// Receiver at the other end of a network link. // // The Sender is the peer responsible for enqueuing EncodedFrames for streaming, // guaranteeing their delivery to a Receiver, and handling feedback events from @@ -64,26 +50,28 @@ class Environment; // EncodedFrame struct should be prepared with its frame_id field set to // whatever GetNextFrameId() returns. Please see method comments for // more-detailed usage info. -class Sender final : public SenderPacketRouter::Sender, - public CompoundRtcpParser::Client { +class Sender { public: // Interface for receiving notifications about events of possible interest. - // Handling each of these is optional, but some may be mandatory for certain - // applications (see method comments below). class Observer { public: - // Called when a frame was canceled. "Canceled" means that the Receiver has - // either acknowledged successful receipt of the frame or has decided to - // skip over it. Note: Frame cancellations may occur out-of-order. - virtual void OnFrameCanceled(FrameId frame_id); + // Called when a frame was canceled, which may occur in the following cases: + // - The Receiver acknowledged successful receipt of the frame. + // - The Receiver decided to skip over the frame (e.g. it was too late). + // - The Sender decided to skip the frame (e.g. OnFrameCanceled() called). + // + // Note: Frame cancellations may occur out-of-order. + virtual void OnFrameCanceled(FrameId frame_id) = 0; // Called when a Receiver begins reporting picture loss, and there is no key // frame currently enqueued in the Sender. The application should enqueue a - // key frame as soon as possible. Note: An application that pauses frame - // sending (e.g., screen mirroring when the screen is not changing) should - // use this notification to send an out-of-band "refresh frame," encoded as - // a key frame. - virtual void OnPictureLost(); + // key frame as soon as possible. + // + // This acts as a "push" notification, which is useful for immediately + // waking up an application that may be waiting for the next capture tick. + // For "pull" state checking inside a continuous encoding loop, see + // NeedsKeyFrame(). + virtual void OnPictureLost() = 0; protected: virtual ~Observer(); @@ -94,46 +82,31 @@ class Sender final : public SenderPacketRouter::Sender, // The frame has been queued for sending. OK, - // The frame's payload was too large. This is typically triggered when - // submitting a payload of several dozen megabytes or more. This result code - // likely indicates some kind of upstream bug. + // The frame's payload was too large. PAYLOAD_TOO_LARGE, - // The span of FrameIds is too large. Cast Streaming's protocol design - // imposes a limit in the maximum difference between the highest-valued - // in-flight FrameId and the least-valued one. + // The span of FrameIds is too large. REACHED_ID_SPAN_LIMIT, - // Too-large a media duration is in-flight. Enqueuing another frame would - // automatically cause late play-out at the Receiver. + // Too-large a media duration is in-flight. MAX_DURATION_IN_FLIGHT, }; - // Constructs a Sender that attaches to the given `environment`-provided - // resources and `packet_router`. The `config` contains the settings that were - // agreed-upon by both sides from the OFFER/ANSWER exchange (i.e., the part of - // the overall end-to-end connection process that occurs before Cast Streaming - // is started). The `rtp_payload_type` does not affect the behavior of this - // Sender. It is simply passed along to a Receiver in the RTP packet stream. - Sender(Environment& environment, - SenderPacketRouter& packet_router, - SessionConfig config, - RtpPayloadType rtp_payload_type); - - ~Sender() final; + virtual ~Sender(); - const SessionConfig& config() const { return config_; } - Ssrc ssrc() const { return rtcp_session_.sender_ssrc(); } - int rtp_timebase() const { return rtp_timebase_; } + // The session configuration for this sender. The configuration is generated + // from the offer/answer exchange, and includes critical information like the + // RTP timebase, SSRCs for sending and receiving, and the AES configuration. + virtual const SessionConfig& config() const = 0; // Sets an observer for receiving notifications. Call with nullptr to stop // observing. - void SetObserver(Observer* observer); + virtual void SetObserver(Observer* observer) = 0; // Returns the number of frames currently in-flight. This is only meant to be // informative. Clients should use GetInFlightMediaDuration() to make // throttling decisions. - int GetInFlightFrameCount() const; + virtual size_t GetInFlightFrameCount() const = 0; // Returns the total media duration of the frames currently in-flight, // assuming the next not-yet-enqueued frame will have the given RTP timestamp. @@ -141,30 +114,36 @@ class Sender final : public SenderPacketRouter::Sender, // GetMaxInFlightMediaDuration(), and media encoding should be throttled down // before additional EnqueueFrame() calls would cause this to reach the // current maximum limit. - Clock::duration GetInFlightMediaDuration( - RtpTimeTicks next_frame_rtp_timestamp) const; + virtual Clock::duration GetInFlightMediaDuration( + RtpTimeTicks next_frame_rtp_timestamp) const = 0; // Return the maximum acceptable in-flight media duration, given the current // target playout delay setting and end-to-end network/system conditions. - Clock::duration GetMaxInFlightMediaDuration() const; + virtual Clock::duration GetMaxInFlightMediaDuration() const = 0; // Returns true if the Receiver requires a key frame. Note that this will // return true until a key frame is accepted by EnqueueFrame(). Thus, when // encoding is pipelined, care should be taken to instruct the encoder to // produce just ONE forced key frame. - bool NeedsKeyFrame() const; + // + // This acts as a stateful "pull" check, which is useful for an encoder loop + // to poll right before processing the next image. For "push" notifications + // to wake up an idle application, see Observer::OnPictureLost(). + virtual bool NeedsKeyFrame() const = 0; // Returns the next FrameId, the one after the frame enqueued by the last call // to EnqueueFrame(). Note that the next call to EnqueueFrame() assumes this // frame ID be used. - FrameId GetNextFrameId() const; + virtual FrameId GetNextFrameId() const = 0; // Get the current round trip time, defined as the total time between when the // sender report is sent and the receiver report is received. This value is // updated with each receiver report using a weighted moving average of 1/8 // for the new value and 7/8 for the previous value. Will be set to // Clock::duration::zero() if no reports have been received yet. - Clock::duration GetCurrentRoundTripTime() const; + // TODO(crbug.com/498036656): move to a more modern approach for estimating + // bandwidth. + virtual Clock::duration GetCurrentRoundTripTime() const = 0; // Enqueues the given `frame` for sending as soon as possible. Returns OK if // the frame is accepted, and some time later Observer::OnFrameCanceled() will @@ -174,177 +153,19 @@ class Sender final : public SenderPacketRouter::Sender, // be the same as GetNextFrameId(); both the `rtp_timestamp` and // `reference_time` fields must be monotonically increasing relative to the // prior frame; and the frame's `data` pointer must be set. - [[nodiscard]] EnqueueFrameResult EnqueueFrame(const EncodedFrame& frame); + [[nodiscard]] virtual EnqueueFrameResult EnqueueFrame( + const EncodedFrame& frame) = 0; // Causes all pending operations to discard data when they are processed - // later. - void CancelInFlightData(); - - private: - // Tracking/Storage for frames that are ready-to-send, and until they are - // fully received at the other end. - struct PendingFrameSlot { - // The frame to send, or nullopt if this slot is not in use. - std::optional frame; - - // Represents which packets need to be sent. Elements are indexed by - // FramePacketId. A set bit means a packet needs to be sent (or re-sent). - YetAnotherBitVector send_flags; - - // The time when each of the packets was last sent, or - // `SenderPacketRouter::kNever` if the packet has not been sent yet. - // Elements are indexed by FramePacketId. This is used to avoid - // re-transmitting any given packet too frequently. - std::vector packet_sent_times; - - PendingFrameSlot(); - ~PendingFrameSlot(); - - bool is_active_for_frame(FrameId frame_id) const { - return frame && frame->frame_id == frame_id; - } - }; - - // Return value from the ChooseXYZ() helper methods. - struct ChosenPacket { - PendingFrameSlot* slot = nullptr; - FramePacketId packet_id{}; - - explicit operator bool() const { return !!slot; } - }; - - // An extension of ChosenPacket that also includes the point-in-time when the - // packet should be sent. - struct ChosenPacketAndWhen : public ChosenPacket { - Clock::time_point when = SenderPacketRouter::kNever; - }; - - // SenderPacketRouter::Sender implementation. - void OnReceivedRtcpPacket(Clock::time_point arrival_time, - ByteView packet) final; - ByteBuffer GetRtcpPacketForImmediateSend(Clock::time_point send_time, - ByteBuffer buffer) final; - ByteBuffer GetRtpPacketForImmediateSend(Clock::time_point send_time, - ByteBuffer buffer) final; - Clock::time_point GetRtpResumeTime() final; - RtpTimeTicks GetLastRtpTimestamp() const final; - StreamType GetStreamType() const final; - - // CompoundRtcpParser::Client implementation. - void OnReceiverReferenceTimeAdvanced(Clock::time_point reference_time) final; - void OnReceiverReport(const RtcpReportBlock& receiver_report) final; - void OnCastReceiverFrameLogMessages( - std::vector messages) final; - void OnReceiverIndicatesPictureLoss() final; - void OnReceiverCheckpoint(FrameId frame_id, - std::chrono::milliseconds playout_delay) final; - void OnReceiverHasFrames(std::vector acks) final; - void OnReceiverIsMissingPackets(std::vector nacks) final; - - // Helper to choose which packet to send, from those that have been flagged as - // "need to send." Returns a "false" result if nothing needs to be sent. - ChosenPacket ChooseNextRtpPacketNeedingSend(); - - // Helper that returns the packet that should be used to kick-start the - // Receiver, and the time at which the packet should be sent. Returns a kNever - // result if kick-starting is not needed. - ChosenPacketAndWhen ChooseKickstartPacket(); - - // Cancels sending (or resending) the given frame once it is known to have - // been either: - // 1. Cancelled by the sender (was_acked must be false); - // 2. Fully received based on the ACK feedback in a receiver RTCP report - // (was_acked must be true); - // 3. The receiver sent a checkpoint frame ID (was_acked must be true). - // - // This clears the corresponding entry in `pending_frames_` and - // adds `frame_id` to the list of pending cancellations to be dispatched as - // part of DispatchCancellations(). - // - // NOTE: Every frame_id ends up being "cancelled" at least once. - void CancelPendingFrame(FrameId frame_id, bool was_acked); - - // Must be called after one or a series of CancelPendingFrame() calls in order - // to notify the observer, if any, about cancellations. - void DispatchCancellations(); - - // Inline helper to return the slot that would contain the tracking info for - // the given `frame_id`. - const PendingFrameSlot& get_slot_for(FrameId frame_id) const { - return pending_frames_[(frame_id - FrameId::first()) % - pending_frames_.size()]; - } - PendingFrameSlot& get_slot_for(FrameId frame_id) { - return pending_frames_[(frame_id - FrameId::first()) % - pending_frames_.size()]; - } - - Environment& environment_; - const SessionConfig config_; - SenderPacketRouter& packet_router_; - RtcpSession rtcp_session_; - CompoundRtcpParser rtcp_parser_; - SenderReportBuilder sender_report_builder_; - RtpPacketizer rtp_packetizer_; - const int rtp_timebase_; - FrameCrypto crypto_; - - // Ring buffer of PendingFrameSlots. The frame having FrameId x will always - // be slotted at position x % pending_frames_.size(). Use get_slot_for() to - // access the correct slot for a given FrameId. - std::array pending_frames_{}; - - // A count of the number of frames in-flight (i.e., the number of active - // entries in `pending_frames_`). - int num_frames_in_flight_ = 0; - - // The ID of the last frame enqueued. - FrameId last_enqueued_frame_id_ = FrameId::leader(); - - // Indicates that all of the packets for all frames up to and including this - // FrameId have been successfully received (or otherwise do not need to be - // re-transmitted). - FrameId checkpoint_frame_id_ = FrameId::leader(); - - // The ID of the latest frame the Receiver seems to be aware of. - FrameId latest_expected_frame_id_ = FrameId::leader(); - - // The target playout delay for the last-enqueued frame. This is auto-updated - // when a frame is enqueued that changes the delay. - std::chrono::milliseconds target_playout_delay_; - FrameId playout_delay_change_at_frame_id_ = FrameId::first(); - - // The exact arrival time of the last RTCP packet. - Clock::time_point rtcp_packet_arrival_time_ = SenderPacketRouter::kNever; - - // The near-term average round trip time. This is updated with each Sender - // Report → Receiver Report round trip. This is initially zero, indicating the - // round trip time has not been measured yet. - Clock::duration round_trip_time_{0}; - - // Maintain current stats in a Sender Report that is ready for sending at any - // time. This includes up-to-date lip-sync information, and packet and byte - // count stats. - RtcpSenderReport pending_sender_report_; - - // These are used to determine whether a key frame needs to be sent to the - // Receiver. When the Receiver provides a picture loss notification, the - // current checkpoint frame ID is stored in `picture_lost_at_frame_id_`. Then, - // while `last_enqueued_key_frame_id_` is less than or equal to - // `picture_lost_at_frame_id_`, the Sender knows it still needs to send a key - // frame to resolve the picture loss condition. In all other cases, the - // Receiver is either in a good state or is in the process of receiving the - // key frame that will make that happen. - FrameId picture_lost_at_frame_id_ = FrameId::leader(); - FrameId last_enqueued_key_frame_id_ = FrameId::leader(); - - // The current observer (optional). - Observer* observer_ = nullptr; - - // Because the observer may take action when frames are cancelled, such as - // calling APIs like EnqueueFrame(), `this` must be in a good state before - // the observer is notified of any pending frame cancellations. - std::vector pending_cancellations_; + // later. This will notify observers by invoking OnFrameCanceled() for each + // canceled frame. + virtual void CancelInFlightData() = 0; + + // May be called by the consumer to report that a frame has been dropped. This + // is used to report drop statistics to the sender's statistics collector. + virtual void ReportFrameDropEvent(FrameId frame_id, + RtpTimeTicks rtp_timestamp, + Clock::time_point drop_time) = 0; }; } // namespace openscreen::cast diff --git a/cast/streaming/public/sender_session.cc b/cast/streaming/public/sender_session.cc index afc66b53c..0b8c33b96 100644 --- a/cast/streaming/public/sender_session.cc +++ b/cast/streaming/public/sender_session.cc @@ -11,8 +11,11 @@ #include #include #include +#include #include "cast/streaming/impl/clock_offset_estimator.h" +#include "cast/streaming/impl/message_constants.h" +#include "cast/streaming/impl/sender_impl.h" #include "cast/streaming/message_fields.h" #include "cast/streaming/public/capture_recommendations.h" #include "cast/streaming/public/environment.h" @@ -23,6 +26,7 @@ #include "util/json/json_helpers.h" #include "util/json/json_serialization.h" #include "util/osp_logging.h" +#include "util/no_destructor.h" #include "util/stringprintf.h" namespace openscreen::cast { @@ -30,70 +34,101 @@ namespace openscreen::cast { namespace { // Default error message for a bad CAPABILITIES_RESPONSE message. const Error& InvalidCapabilitiesResponseError() { - static const Error kError( + static const openscreen::NoDestructor kError( Error::Code::kRemotingNotSupported, "Invalid CAPABILITIES_RESPONSE message, assuming remoting is not " "supported"); - return kError; + return *kError; } // Default error message for a bad ANSWER message. const Error& InvalidAnswerError() { - static const Error kError(Error::Code::kInvalidAnswer, - "Invalid ANSWER message."); - return kError; + static const openscreen::NoDestructor kError( + Error::Code::kInvalidAnswer, "Invalid ANSWER message."); + return *kError; } // Error message for an ANSWER timeout. const Error& AnswerTimeoutError() { - static const Error kError(Error::Code::kAnswerTimeout, - "Didn't receive an ANSWER message before timeout."); - return kError; + static const openscreen::NoDestructor kError( + Error::Code::kAnswerTimeout, + "Didn't receive an ANSWER message before timeout."); + return *kError; } // Default error message for a bad RPC message. const Error& InvalidRpcError() { - static const Error kError(Error::Code::kJsonParseError, - "Invalid RPC message."); - return kError; + static const openscreen::NoDestructor kError( + Error::Code::kJsonParseError, "Invalid RPC message."); + return *kError; +} + +// Default error message for a bad INPUT message. +const Error& InvalidInputError() { + static const openscreen::NoDestructor kError( + Error::Code::kJsonParseError, "Invalid INPUT message."); + return *kError; +} + +// Returns DSCP suggestions based on Table 1 in RFC 8837. +// https://datatracker.ietf.org/doc/html/rfc8837#name-dscp-mappings +UdpSocket::DscpMode GetDscpSuggestion(bool has_audio, bool has_video) { + if (has_video) { + // Whether or not we have audio, use Assured Forwarding with + // lowest level drop precedence (AF1). + return UdpSocket::DscpMode::kAF41; + } else if (has_audio) { + // If this is an audio only session, the spec indicates that Expedited + // Forwarding (EF) should be used. + return UdpSocket::DscpMode::kEF; + } + + // No audio or video means no session, which means this function should not be + // called. + OSP_NOTREACHED(); +} + +std::optional ToWire(std::optional mode) { + if (mode) { + return static_cast(mode.value()); + } + return std::nullopt; } AudioStream CreateStream(int index, const AudioCaptureConfig& config, - bool use_android_rtp_hack) { - return AudioStream{Stream{index, - Stream::Type::kAudioSource, - config.channels, - GetPayloadType(config.codec, use_android_rtp_hack), - GenerateSsrc(true /*high_priority*/), - config.target_playout_delay, - GenerateRandomBytes16(), - GenerateRandomBytes16(), - true /* receiver_rtcp_event_log */, - {} /* receiver_rtcp_dscp */, - config.sample_rate, - config.codec_parameter}, - config.codec, - std::max(config.bit_rate, kDefaultAudioMinBitRate)}; + bool use_android_rtp_hack, + std::optional dscp_mode, + bool /* supports_input_events */) { + std::vector rtp_extensions; + return AudioStream{ + Stream{index, Stream::Type::kAudioSource, config.channels, + GetPayloadType(config.codec, use_android_rtp_hack), + GenerateSsrc(true /*high_priority*/), config.target_playout_delay, + GenerateRandomBytes16(), GenerateRandomBytes16(), + true /* receiver_rtcp_event_log */, ToWire(dscp_mode), + config.sample_rate, config.codec_parameter}, + config.codec, std::max(config.bit_rate, kDefaultAudioMinBitRate)}; } VideoStream CreateStream(int index, const VideoCaptureConfig& config, - bool use_android_rtp_hack) { + bool use_android_rtp_hack, + std::optional dscp_mode, + bool supports_input_events) { constexpr int kVideoStreamChannelCount = 1; + std::vector rtp_extensions; + if (supports_input_events) { + rtp_extensions.push_back(kInputEventsRtpExtension); + } return VideoStream{ - Stream{index, - Stream::Type::kVideoSource, - kVideoStreamChannelCount, + Stream{index, Stream::Type::kVideoSource, kVideoStreamChannelCount, GetPayloadType(config.codec, use_android_rtp_hack), - GenerateSsrc(false /*high_priority*/), - config.target_playout_delay, - GenerateRandomBytes16(), - GenerateRandomBytes16(), - true /* receiver_rtcp_event_log */, - {} /* receiver_rtcp_dscp */, - kRtpVideoTimebase, - config.codec_parameter}, + GenerateSsrc(false /*high_priority*/), config.target_playout_delay, + GenerateRandomBytes16(), GenerateRandomBytes16(), + true /* receiver_rtcp_event_log */, ToWire(dscp_mode), + kRtpVideoTimebase, config.codec_parameter, + std::move(rtp_extensions)}, config.codec, config.max_frame_rate, (config.max_bit_rate >= kDefaultVideoMinBitRate) @@ -111,45 +146,64 @@ template void CreateStreamList(int offset_index, const std::vector& configs, bool use_android_rtp_hack, + std::optional dscp_mode, + bool supports_input_events, std::vector* out) { out->reserve(configs.size()); for (size_t i = 0; i < configs.size(); ++i) { - out->emplace_back( - CreateStream(i + offset_index, configs[i], use_android_rtp_hack)); + out->emplace_back(CreateStream(i + offset_index, configs[i], + use_android_rtp_hack, dscp_mode, + supports_input_events)); } } Offer CreateMirroringOffer(const std::vector& audio_configs, const std::vector& video_configs, - bool use_android_rtp_hack) { + bool use_android_rtp_hack, + bool enable_dscp, + bool supports_input_events) { Offer offer; offer.cast_mode = CastMode::kMirroring; + const bool has_audio = !audio_configs.empty(); + const bool has_video = !video_configs.empty(); + std::optional dscp_mode; + if (enable_dscp) { + dscp_mode = GetDscpSuggestion(has_audio, has_video); + } + // NOTE here: IDs will always follow the pattern: // [0.. audio streams... N - 1][N.. video streams.. K] - CreateStreamList(0, audio_configs, use_android_rtp_hack, - &offer.audio_streams); + CreateStreamList(0, audio_configs, use_android_rtp_hack, dscp_mode, + supports_input_events, &offer.audio_streams); CreateStreamList(audio_configs.size(), video_configs, use_android_rtp_hack, - &offer.video_streams); + dscp_mode, supports_input_events, &offer.video_streams); return offer; } Offer CreateRemotingOffer(const AudioCaptureConfig& audio_config, const VideoCaptureConfig& video_config, - bool use_android_rtp_hack) { + bool use_android_rtp_hack, + bool enable_dscp, + bool supports_input_events) { Offer offer; offer.cast_mode = CastMode::kRemoting; - AudioStream audio_stream = - CreateStream(0, audio_config, use_android_rtp_hack); + std::optional dscp_mode; + if (enable_dscp) { + dscp_mode = GetDscpSuggestion(true, true); + } + + AudioStream audio_stream = CreateStream(0, audio_config, use_android_rtp_hack, + dscp_mode, supports_input_events); audio_stream.codec = AudioCodec::kNotSpecified; audio_stream.stream.rtp_payload_type = GetPayloadType(AudioCodec::kNotSpecified, use_android_rtp_hack); offer.audio_streams.push_back(std::move(audio_stream)); - VideoStream video_stream = - CreateStream(1, video_config, use_android_rtp_hack); + VideoStream video_stream = CreateStream(1, video_config, use_android_rtp_hack, + dscp_mode, supports_input_events); video_stream.codec = VideoCodec::kNotSpecified; video_stream.stream.rtp_payload_type = GetPayloadType(VideoCodec::kNotSpecified, use_android_rtp_hack); @@ -227,6 +281,38 @@ RemotingCapabilities ToCapabilities(const ReceiverCapability& capability) { return out; } +void MaybeSetDscp(const Offer& offer, const Answer& answer, Environment& env) { + if (answer.send_indexes.empty() || answer.receiver_rtcp_dscp.empty()) { + return; + } + + // DSCP support is all or nothing, which is both recommended by the RFC + // spec and also a practical result of sharing one UDP socket for audio + // and video streams. Thus, only enable it if the receiver wanted it on + // all selected streams. + std::vector sorted_indexes = answer.send_indexes; + std::vector sorted_dscp_indexes = answer.receiver_rtcp_dscp; + std::ranges::sort(sorted_indexes); + std::ranges::sort(sorted_dscp_indexes); + if (sorted_indexes != sorted_dscp_indexes) { + return; + } + + // To be spec compliant, the DSCP value is nested on each stream. However, + // we either use one DSCP value for every stream, or none at all. If dynamic + // DSCP values end up being supported, we may need to look up the original + // stream here. + std::optional suggested_dscp; + if (!offer.audio_streams.empty()) { + suggested_dscp = offer.audio_streams.front().stream.receiver_rtcp_dscp; + } else if (!offer.video_streams.empty()) { + suggested_dscp = offer.video_streams.front().stream.receiver_rtcp_dscp; + } + if (suggested_dscp) { + env.SetDscp(static_cast(suggested_dscp.value())); + } +} + } // namespace SenderSession::Client::~Client() = default; @@ -243,7 +329,16 @@ SenderSession::SenderSession(Configuration config) }, config_.environment->task_runner()), rpc_messenger_([this](std::vector message) { - SendRpcMessage(std::move(message)); + const Error error = this->messenger_.SendRpcMessage(message); + if (!error.ok()) { + OSP_LOG_WARN << "Failed to send RPC message: " << error; + } + }), + input_messenger_([this](std::vector message) { + const Error error = this->messenger_.SendInputMessage(message); + if (!error.ok()) { + OSP_LOG_WARN << "Failed to send INPUT message: " << error; + } }), packet_router_(*config_.environment) { // We may or may not do remoting this session, however our RPC handler @@ -253,9 +348,15 @@ SenderSession::SenderSession(Configuration config) [this](ErrorOr message) { this->OnRpcMessage(std::move(message)); }); + messenger_.SetHandler(ReceiverMessage::Type::kInput, + [this](ErrorOr message) { + this->OnInputMessage(std::move(message)); + }); } -SenderSession::~SenderSession() = default; +SenderSession::~SenderSession() { + config_.environment->SetStatisticsCollector(nullptr); +} Error SenderSession::Negotiate(std::vector audio_configs, std::vector video_configs) { @@ -268,8 +369,9 @@ Error SenderSession::Negotiate(std::vector audio_configs, return Error(Error::Code::kParameterInvalid, "Invalid configs provided."); } - Offer offer = CreateMirroringOffer(audio_configs, video_configs, - config_.use_android_rtp_hack); + Offer offer = CreateMirroringOffer( + audio_configs, video_configs, config_.use_android_rtp_hack, + config_.enable_dscp, input_messenger_.receive_message_cb() != nullptr); return StartNegotiation(std::move(audio_configs), std::move(video_configs), std::move(offer)); } @@ -283,8 +385,9 @@ Error SenderSession::NegotiateRemoting(AudioCaptureConfig audio_config, "Passed invalid audio or video config."); } - Offer offer = CreateRemotingOffer(audio_config, video_config, - config_.use_android_rtp_hack); + Offer offer = CreateRemotingOffer( + audio_config, video_config, config_.use_android_rtp_hack, + config_.enable_dscp, input_messenger_.receive_message_cb() != nullptr); return StartNegotiation({audio_config}, {video_config}, std::move(offer)); } @@ -321,6 +424,28 @@ void SenderSession::SetStatsClient(SenderStatsClient* client) { stats_analyzer_->ScheduleAnalysis(); } +void SenderSession::SetInputCallback( + std::function callback) { + if (callback) { + input_messenger_.SetReceiveMessageCallback( + [cb = std::move(callback)](std::unique_ptr message) { + cb(std::move(*message)); + }); + } else { + input_messenger_.SetReceiveMessageCallback(nullptr); + } +} + +void SenderSession::SendInputMessage(const InputMessage& message) { + if (state_ == State::kIdle) { + OSP_DLOG_WARN << "Can't send an INPUT message without a currently " + "negotiated session."; + return; + } + + input_messenger_.SendMessageToRemote(message); +} + void SenderSession::ResetState() { state_ = State::kIdle; current_negotiation_.reset(); @@ -360,7 +485,7 @@ void SenderSession::OnAnswer(ErrorOr message) { return; } - const Answer& answer = absl::get(message.value().body); + const Answer& answer = std::get(message.value().body); ConfiguredSenders senders = SelectSenders(answer); // If we didn't select any senders, the negotiation was unsuccessful. if (!senders.audio_sender && !senders.video_sender) { @@ -385,7 +510,7 @@ void SenderSession::OnCapabilitiesResponse(ErrorOr message) { // error response to indicate remoting is not supported. if (!message) { config_.client.OnError(this, Error(Error::Code::kRemotingNotSupported, - message.error().ToString())); + message.error().message())); return; } if (!message.value().valid || @@ -395,7 +520,7 @@ void SenderSession::OnCapabilitiesResponse(ErrorOr message) { } const ReceiverCapability& caps = - absl::get(message.value().body); + std::get(message.value().body); int remoting_version = caps.remoting_version; // If not set, we assume it is version 1. if (remoting_version == ReceiverCapability::kRemotingVersionUnknown) { @@ -403,8 +528,8 @@ void SenderSession::OnCapabilitiesResponse(ErrorOr message) { } if (remoting_version > kSupportedRemotingVersion) { - std::string error_message = StringPrintf( - "Receiver is using too new of a version for remoting (%d > %d)", + std::string error_message = StringFormat( + "Receiver is using too new of a version for remoting ({} > {})", remoting_version, kSupportedRemotingVersion); config_.client.OnError(this, Error(Error::Code::kRemotingNotSupported, std::move(error_message))); @@ -426,15 +551,31 @@ void SenderSession::OnRpcMessage(ErrorOr message) { return; } - const auto& body = absl::get>(message.value().body); - rpc_messenger_.ProcessMessageFromRemote(body.data(), body.size()); + const auto& body = std::get>(message.value().body); + rpc_messenger_.ProcessMessageFromRemote(body); +} + +void SenderSession::OnInputMessage(ErrorOr message) { + if (!message) { + config_.client.OnError(this, message.error()); + return; + } + + if (!message.value().valid || + message.value().type != ReceiverMessage::Type::kInput) { + HandleErrorMessage(message.value(), InvalidInputError()); + return; + } + + const auto& body = std::get>(message.value().body); + input_messenger_.ProcessMessageFromRemote(body); } void SenderSession::HandleErrorMessage(ReceiverMessage message, const Error& default_error) { OSP_CHECK(!message.valid); - if (absl::holds_alternative(message.body)) { - const ReceiverError& error = absl::get(message.body); + if (std::holds_alternative(message.body)) { + const ReceiverError& error = std::get(message.body); Error converted_error = error.ToError(); // If the receiver error code was an invalid value, fallback to @@ -462,8 +603,8 @@ std::unique_ptr SenderSession::CreateSender(Ssrc receiver_ssrc, /* is_pli_enabled*/ true, ToStreamType(type, config_.use_android_rtp_hack)}; OSP_DCHECK(config.IsValid()); - return std::make_unique(*config_.environment, packet_router_, - std::move(config), type); + return std::make_unique(*config_.environment, packet_router_, + std::move(config), type); } void SenderSession::SpawnAudioSender(ConfiguredSenders* senders, @@ -514,6 +655,11 @@ SenderSession::ConfiguredSenders SenderSession::SelectSenders( OSP_LOG_INFO << "Streaming to " << config_.environment->remote_endpoint() << "..."; + // If DSCP was successfully negotiated, enable it. + if (config_.enable_dscp) { + MaybeSetDscp(current_negotiation_->offer, answer, *config_.environment); + } + ConfiguredSenders senders; for (size_t i = 0; i < answer.send_indexes.size(); ++i) { const Ssrc receiver_ssrc = answer.ssrcs[i]; @@ -531,14 +677,4 @@ SenderSession::ConfiguredSenders SenderSession::SelectSenders( return senders; } -void SenderSession::SendRpcMessage(std::vector message_body) { - Error error = this->messenger_.SendOutboundMessage(SenderMessage{ - SenderMessage::Type::kRpc, ++(this->current_sequence_number_), true, - std::move(message_body)}); - - if (!error.ok()) { - OSP_LOG_WARN << "Failed to send RPC message: " << error; - } -} - } // namespace openscreen::cast diff --git a/cast/streaming/public/sender_session.h b/cast/streaming/public/sender_session.h index d67260019..bef38f15d 100644 --- a/cast/streaming/public/sender_session.h +++ b/cast/streaming/public/sender_session.h @@ -5,6 +5,7 @@ #ifndef CAST_STREAMING_PUBLIC_SENDER_SESSION_H_ #define CAST_STREAMING_PUBLIC_SENDER_SESSION_H_ +#include #include #include #include @@ -12,18 +13,20 @@ #include "cast/common/public/message_port.h" #include "cast/streaming/capture_configs.h" -#include "cast/streaming/impl/session_config.h" #include "cast/streaming/impl/statistics_analyzer.h" +#include "cast/streaming/input.pb.h" #include "cast/streaming/public/answer_messages.h" #include "cast/streaming/public/capture_recommendations.h" -#include "cast/streaming/public/offer_messages.h" +#include "cast/streaming/public/protobuf_messenger.h" #include "cast/streaming/public/rpc_messenger.h" #include "cast/streaming/public/sender.h" +#include "cast/streaming/public/session_config.h" #include "cast/streaming/public/session_messenger.h" #include "cast/streaming/public/statistics.h" #include "cast/streaming/remoting_capabilities.h" #include "cast/streaming/sender_packet_router.h" #include "json/value.h" +#include "platform/base/ip_address.h" #include "util/json/json_serialization.h" namespace openscreen::cast { @@ -123,6 +126,12 @@ class SenderSession final { // Whether or not the android RTP value hack should be used (for legacy // android devices). For more information, see https://crbug.com/631828. bool use_android_rtp_hack = true; + + // If true, allows the sender to negotiate DSCP marking for RTP and RTCP + // packets, which can help provide Quality-of-Service in some + // environments. This feature is off by default to allow embedders to opt + // in and experiment as desired. + bool enable_dscp = false; }; // The SenderSession assumes that the passed in client, environment, and @@ -167,6 +176,16 @@ class SenderSession final { // recorded unless this field is set. void SetStatsClient(SenderStatsClient* client); + // Set the callback for handling input events. If set, future negotiations + // will include support for input events. If not set, the sender will not + // request input messaging with the receiver. + // NOTE: resetting the callback to null will not force a renegotiation -- the + // embedder is responsible for deciding what happens next with the session. + void SetInputCallback(std::function callback); + + // Sends an input message to the remote. + void SendInputMessage(const InputMessage& message); + // The RPC messenger for this session. NOTE: RPC messages may come at // any time from the receiver, so subscriptions to RPC remoting messages // should be done before calling `NegotiateRemoting`. @@ -223,6 +242,7 @@ class SenderSession final { void OnAnswer(ErrorOr message); void OnCapabilitiesResponse(ErrorOr message); void OnRpcMessage(ErrorOr message); + void OnInputMessage(ErrorOr message); // Handles an error `message` response from a receiver. If the receiver does // not contain any error information, `default_error` will be reported @@ -247,9 +267,6 @@ class SenderSession final { // Spawn a set of configured senders from the currently stored negotiation. ConfiguredSenders SelectSenders(const Answer& answer); - // Used by the RPC messenger to send outbound messages. - void SendRpcMessage(std::vector message_body); - // This session's configuration. Configuration config_; @@ -258,10 +275,14 @@ class SenderSession final { // cast/protocol/castv2/streaming_schema.json. SenderSessionMessenger messenger_; - // The RPC messenger, which uses the session messager for sending RPC messages - // and handles subscriptions to RPC messages. + // The RPC messenger, which uses the session messenger for sending RPC + // messages and handles subscriptions to RPC messages. RpcMessenger rpc_messenger_; + // The INPUT messenger, which uses the session messenger for sending INPUT + // messages. + ProtobufMessenger input_messenger_; + // The packet router used for RTP/RTCP messaging across all senders. SenderPacketRouter packet_router_; diff --git a/cast/streaming/impl/session_config.cc b/cast/streaming/public/session_config.cc similarity index 86% rename from cast/streaming/impl/session_config.cc rename to cast/streaming/public/session_config.cc index 0b98ca207..e819b4f6a 100644 --- a/cast/streaming/impl/session_config.cc +++ b/cast/streaming/public/session_config.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "cast/streaming/impl/session_config.h" +#include "cast/streaming/public/session_config.h" #include #include @@ -25,7 +25,8 @@ SessionConfig::SessionConfig(Ssrc sender_ssrc, std::array aes_secret_key, std::array aes_iv_mask, bool is_pli_enabled, - StreamType stream_type) + StreamType stream_type, + bool are_receiver_event_logs_enabled) : sender_ssrc(sender_ssrc), receiver_ssrc(receiver_ssrc), rtp_timebase(rtp_timebase), @@ -34,7 +35,8 @@ SessionConfig::SessionConfig(Ssrc sender_ssrc, aes_secret_key(std::move(aes_secret_key)), aes_iv_mask(std::move(aes_iv_mask)), is_pli_enabled(is_pli_enabled), - stream_type(stream_type) {} + stream_type(stream_type), + are_receiver_event_logs_enabled(are_receiver_event_logs_enabled) {} SessionConfig::SessionConfig(const SessionConfig& other) = default; SessionConfig::SessionConfig(SessionConfig&& other) noexcept = default; diff --git a/cast/streaming/impl/session_config.h b/cast/streaming/public/session_config.h similarity index 81% rename from cast/streaming/impl/session_config.h rename to cast/streaming/public/session_config.h index 4fab4ed3d..e77f1fb3b 100644 --- a/cast/streaming/impl/session_config.h +++ b/cast/streaming/public/session_config.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef CAST_STREAMING_IMPL_SESSION_CONFIG_H_ -#define CAST_STREAMING_IMPL_SESSION_CONFIG_H_ +#ifndef CAST_STREAMING_PUBLIC_SESSION_CONFIG_H_ +#define CAST_STREAMING_PUBLIC_SESSION_CONFIG_H_ #include #include @@ -25,7 +25,8 @@ struct SessionConfig final { std::array aes_secret_key, std::array aes_iv_mask, bool is_pli_enabled = false, - StreamType stream_type = StreamType::kUnknown); + StreamType stream_type = StreamType::kUnknown, + bool are_receiver_event_logs_enabled = true); SessionConfig(const SessionConfig& other); SessionConfig(SessionConfig&& other) noexcept; SessionConfig& operator=(const SessionConfig& other); @@ -59,8 +60,12 @@ struct SessionConfig final { // The type (e.g. audio or video) of the stream. StreamType stream_type = StreamType::kUnknown; + + // Whether RTCP event logs from the Receiver are enabled. These are used for + // generating statistics. It is recommended that this generally be true. + bool are_receiver_event_logs_enabled = true; }; } // namespace openscreen::cast -#endif // CAST_STREAMING_IMPL_SESSION_CONFIG_H_ +#endif // CAST_STREAMING_PUBLIC_SESSION_CONFIG_H_ diff --git a/cast/streaming/public/session_messenger.cc b/cast/streaming/public/session_messenger.cc index cc56d0e3d..ec6586b22 100644 --- a/cast/streaming/public/session_messenger.cc +++ b/cast/streaming/public/session_messenger.cc @@ -4,6 +4,7 @@ #include "cast/streaming/public/session_messenger.h" +#include #include #include @@ -94,18 +95,29 @@ SenderSessionMessenger::SenderSessionMessenger(MessagePort& message_port, void SenderSessionMessenger::SetHandler(ReceiverMessage::Type type, ReplyCallback cb) { - // Currently the only handler allowed is for RPC messages. - OSP_CHECK(type == ReceiverMessage::Type::kRpc); - rpc_callback_ = std::move(cb); + // Currently the only handlers allowed are for RPC and INPUT messages. + if (type == ReceiverMessage::Type::kRpc) { + rpc_callback_ = std::move(cb); + } else if (type == ReceiverMessage::Type::kInput) { + input_callback_ = std::move(cb); + } else { + OSP_NOTREACHED(); + } } void SenderSessionMessenger::ResetHandler(ReceiverMessage::Type type) { - OSP_CHECK(type == ReceiverMessage::Type::kRpc); - rpc_callback_ = {}; + if (type == ReceiverMessage::Type::kRpc) { + rpc_callback_ = {}; + } else if (type == ReceiverMessage::Type::kInput) { + input_callback_ = {}; + } else { + OSP_NOTREACHED(); + } } Error SenderSessionMessenger::SendOutboundMessage(SenderMessage message) { - const auto namespace_ = (message.type == SenderMessage::Type::kRpc) + const auto namespace_ = (message.type == SenderMessage::Type::kRpc || + message.type == SenderMessage::Type::kInput) ? kCastRemotingNamespace : kCastWebrtcNamespace; @@ -122,11 +134,19 @@ Error SenderSessionMessenger::SendRpcMessage(ByteView message) { std::vector(message.begin(), message.end())}); } +Error SenderSessionMessenger::SendInputMessage(ByteView message) { + return SendOutboundMessage(SenderMessage{ + openscreen::cast::SenderMessage::Type::kInput, + -1 /* sequence_number, unused by INPUT messages */, true /* valid */, + std::vector(message.begin(), message.end())}); +} + Error SenderSessionMessenger::SendRequest(SenderMessage message, ReceiverMessage::Type reply_type, ReplyCallback cb) { - // RPC messages are not meant to be request/reply. + // RPC and INPUT messages are not meant to be request/reply. OSP_CHECK(reply_type != ReceiverMessage::Type::kRpc); + OSP_CHECK(reply_type != ReceiverMessage::Type::kInput); if (!cb) { return Error(Error::Code::kParameterInvalid, @@ -168,7 +188,7 @@ void SenderSessionMessenger::OnMessage(const std::string& source_id, } ErrorOr message_body = json::Parse(message); - if (!message_body) { + if (!message_body || !message_body.value().isObject()) { ReportError(message_body.error()); OSP_DLOG_WARN << "Received an invalid message: " << message; return; @@ -194,6 +214,12 @@ void SenderSessionMessenger::OnMessage(const std::string& source_id, } else { OSP_DLOG_INFO << "Received RPC message but no callback, dropping"; } + } else if (receiver_message.value().type == ReceiverMessage::Type::kInput) { + if (input_callback_) { + input_callback_(receiver_message.value()); + } else { + OSP_DLOG_INFO << "Received INPUT message but no callback, dropping"; + } } else { const int sequence_number = receiver_message.value().sequence_number; auto it = awaiting_replies_.find(sequence_number); @@ -229,6 +255,24 @@ void ReceiverSessionMessenger::ResetHandler(SenderMessage::Type type) { callbacks_.erase_key(type); } +Error ReceiverSessionMessenger::SendRpcMessage(const std::string& source_id, + ByteView message) { + return SendMessage( + source_id, + ReceiverMessage{ReceiverMessage::Type::kRpc, -1 /* sequence_number */, + true /* valid */, + std::vector(message.begin(), message.end())}); +} + +Error ReceiverSessionMessenger::SendInputMessage(const std::string& source_id, + ByteView message) { + return SendMessage( + source_id, + ReceiverMessage{ReceiverMessage::Type::kInput, -1 /* sequence_number */, + true /* valid */, + std::vector(message.begin(), message.end())}); +} + Error ReceiverSessionMessenger::SendMessage(const std::string& source_id, ReceiverMessage message) { if (source_id.empty()) { @@ -236,7 +280,8 @@ Error ReceiverSessionMessenger::SendMessage(const std::string& source_id, "Cannot send a message without a current source ID."); } - const auto namespace_ = (message.type == ReceiverMessage::Type::kRpc) + const auto namespace_ = (message.type == ReceiverMessage::Type::kRpc || + message.type == ReceiverMessage::Type::kInput) ? kCastRemotingNamespace : kCastWebrtcNamespace; @@ -246,11 +291,55 @@ Error ReceiverSessionMessenger::SendMessage(const std::string& source_id, message_json.value()); } +void ReceiverSessionMessenger::SetCustomMessageHandler( + std::string_view message_namespace, + CustomMessageCallback cb) { + auto it = std::find_if(custom_message_handlers_.begin(), + custom_message_handlers_.end(), + [&message_namespace](const auto& pair) { + return pair.first == message_namespace; + }); + + if (!cb) { + if (it != custom_message_handlers_.end()) { + custom_message_handlers_.erase(it); + } + return; + } + + if (it != custom_message_handlers_.end()) { + OSP_LOG_ERROR << "Handler already exists for namespace: " + << message_namespace; + return; + } else { + custom_message_handlers_.emplace_back(std::string(message_namespace), + std::move(cb)); + } +} + +Error ReceiverSessionMessenger::SendMessage(std::string_view destination_id, + std::string_view message_namespace, + std::string_view message) { + message_port().PostMessage(std::string(destination_id), + std::string(message_namespace), + std::string(message)); + return Error::None(); +} + void ReceiverSessionMessenger::OnMessage(const std::string& source_id, const std::string& message_namespace, const std::string& message) { if (message_namespace != kCastWebrtcNamespace && message_namespace != kCastRemotingNamespace) { + auto it = std::find_if(custom_message_handlers_.begin(), + custom_message_handlers_.end(), + [&message_namespace](const auto& pair) { + return pair.first == message_namespace; + }); + if (it != custom_message_handlers_.end()) { + it->second(source_id, message_namespace, message); + return; + } OSP_DLOG_WARN << "Received message from unknown namespace: " << message_namespace; return; @@ -259,7 +348,7 @@ void ReceiverSessionMessenger::OnMessage(const std::string& source_id, // If the message is bad JSON, the sender is in a funky state so we // report an error. ErrorOr message_body = json::Parse(message); - if (message_body.is_error()) { + if (message_body.is_error() || !message_body.value().isObject()) { ReportError(message_body.error()); return; } diff --git a/cast/streaming/public/session_messenger.h b/cast/streaming/public/session_messenger.h index aa0d5d709..3702b4a4d 100644 --- a/cast/streaming/public/session_messenger.h +++ b/cast/streaming/public/session_messenger.h @@ -10,7 +10,6 @@ #include #include -#include "absl/types/variant.h" #include "cast/common/public/message_port.h" #include "cast/streaming/public/answer_messages.h" #include "cast/streaming/public/offer_messages.h" @@ -35,6 +34,8 @@ class SessionMessenger : public MessagePort::Client { ErrorCallback cb); ~SessionMessenger() override; + MessagePort& message_port() { return message_port_; } + protected: // Barebones message sending method shared by both children. [[nodiscard]] Error SendMessage(const std::string& destination_id, @@ -78,6 +79,9 @@ class SenderSessionMessenger final : public SessionMessenger { // Convenience method for sending a valid RPC message. [[nodiscard]] Error SendRpcMessage(ByteView message); + // Convenience method for sending a valid INPUT message. + [[nodiscard]] Error SendInputMessage(ByteView message); + // Send a request (with optional reply callback). [[nodiscard]] Error SendRequest(SenderMessage message, ReceiverMessage::Type reply_type, @@ -104,6 +108,7 @@ class SenderSessionMessenger final : public SessionMessenger { // Currently we can only set a handler for RPC messages, so no need for // a flatmap here. ReplyCallback rpc_callback_; + ReplyCallback input_callback_; WeakPtrFactory weak_factory_{this}; }; @@ -121,10 +126,30 @@ class ReceiverSessionMessenger final : public SessionMessenger { void SetHandler(SenderMessage::Type type, RequestCallback cb); void ResetHandler(SenderMessage::Type type); + // Convenience method for sending a valid RPC message. + [[nodiscard]] Error SendRpcMessage(const std::string& source_id, + ByteView message); + + // Convenience method for sending a valid INPUT message. + [[nodiscard]] Error SendInputMessage(const std::string& source_id, + ByteView message); + // Send a JSON message. [[nodiscard]] Error SendMessage(const std::string& source_id, ReceiverMessage message); + // Send a raw string message to a custom namespace. + [[nodiscard]] Error SendMessage(std::string_view destination_id, + std::string_view message_namespace, + std::string_view message); + + using CustomMessageCallback = + std::function; + void SetCustomMessageHandler(std::string_view message_namespace, + CustomMessageCallback cb); + // MessagePort::Client overrides void OnMessage(const std::string& source_id, const std::string& message_namespace, @@ -133,6 +158,8 @@ class ReceiverSessionMessenger final : public SessionMessenger { private: FlatMap callbacks_; + std::vector> + custom_message_handlers_; }; } // namespace openscreen::cast diff --git a/cast/streaming/public/statistics.cc b/cast/streaming/public/statistics.cc index 77ae7e8ca..81667f698 100644 --- a/cast/streaming/public/statistics.cc +++ b/cast/streaming/public/statistics.cc @@ -154,7 +154,7 @@ std::string SimpleHistogram::GetBucketName(size_t index) const { // are calculated. const int bucket_min = min + width * (index - 1); const int bucket_max = min + index * width - 1; - return StringPrintf("%d-%d", bucket_min, bucket_max); + return StringFormat("{}-{}", bucket_min, bucket_max); } Json::Value SenderStats::ToJson() const { @@ -170,6 +170,14 @@ std::string SenderStats::ToString() const { return json::Stringify(ToJson()).value(); } +std::ostream& operator<<(std::ostream& out, const SenderStats& stats) { + return out << stats.ToString(); +} + +std::ostream& operator<<(std::ostream& out, const SimpleHistogram& histogram) { + return out << histogram.ToString(); +} + SenderStatsClient::~SenderStatsClient() {} } // namespace openscreen::cast diff --git a/cast/streaming/public/statistics.h b/cast/streaming/public/statistics.h index 86e6a88cb..660dfb119 100644 --- a/cast/streaming/public/statistics.h +++ b/cast/streaming/public/statistics.h @@ -151,6 +151,8 @@ struct SimpleHistogram { std::string GetBucketName(size_t index) const; }; +std::ostream& operator<<(std::ostream& out, const SimpleHistogram& histogram); + struct SenderStats { using StatisticsList = std::array(StatisticType::kNumTypes)>; @@ -159,21 +161,23 @@ struct SenderStats { static_cast(HistogramType::kNumTypes)>; // The current audio statistics. - StatisticsList audio_statistics; + StatisticsList audio_statistics = {}; // The current audio histograms. - HistogramsList audio_histograms; + HistogramsList audio_histograms = {}; // The current video statistics. - StatisticsList video_statistics; + StatisticsList video_statistics = {}; // The current video histograms. - HistogramsList video_histograms; + HistogramsList video_histograms = {}; Json::Value ToJson() const; std::string ToString() const; }; +std::ostream& operator<<(std::ostream& out, const SenderStats& stats); + // The consumer may provide a statistics client if they are interested in // getting statistics about the ongoing session. class SenderStatsClient { diff --git a/cast/streaming/receiver.h b/cast/streaming/receiver.h deleted file mode 100644 index a1b6255d3..000000000 --- a/cast/streaming/receiver.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_RECEIVER_H_ -#define CAST_STREAMING_RECEIVER_H_ - -#include "cast/streaming/public/receiver.h" - -#endif // CAST_STREAMING_RECEIVER_H_ diff --git a/cast/streaming/receiver_constraints.h b/cast/streaming/receiver_constraints.h deleted file mode 100644 index f19e90cd3..000000000 --- a/cast/streaming/receiver_constraints.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_RECEIVER_CONSTRAINTS_H_ -#define CAST_STREAMING_RECEIVER_CONSTRAINTS_H_ - -#include "cast/streaming/public/receiver_constraints.h" - -#endif // CAST_STREAMING_RECEIVER_CONSTRAINTS_H_ diff --git a/cast/streaming/receiver_message.h b/cast/streaming/receiver_message.h deleted file mode 100644 index f14129332..000000000 --- a/cast/streaming/receiver_message.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_RECEIVER_MESSAGE_H_ -#define CAST_STREAMING_RECEIVER_MESSAGE_H_ - -#include "cast/streaming/public/receiver_message.h" - -#endif // CAST_STREAMING_RECEIVER_MESSAGE_H_ diff --git a/cast/streaming/receiver_session.h b/cast/streaming/receiver_session.h deleted file mode 100644 index 72a858c63..000000000 --- a/cast/streaming/receiver_session.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_RECEIVER_SESSION_H_ -#define CAST_STREAMING_RECEIVER_SESSION_H_ - -#include "cast/streaming/public/receiver_session.h" - -#endif // CAST_STREAMING_RECEIVER_SESSION_H_ diff --git a/cast/streaming/resolution.cc b/cast/streaming/resolution.cc index 942a9fb5f..8727f848d 100644 --- a/cast/streaming/resolution.cc +++ b/cast/streaming/resolution.cc @@ -38,12 +38,21 @@ bool FrameRateEquals(double a, double b) { } // namespace -bool Resolution::TryParse(const Json::Value& root, Resolution* out) { - if (!json::TryParseInt(root[kWidth], &(out->width)) || - !json::TryParseInt(root[kHeight], &(out->height))) { - return false; +ErrorOr Resolution::TryParse(const Json::Value& root) { + if (!root.isObject()) { + return Error(Error::Code::kJsonParseError, + "Resolution is not a JSON object"); } - return out->IsValid(); + + Resolution out; + if (!json::TryParseInt(root[kWidth], &out.width) || + !json::TryParseInt(root[kHeight], &out.height)) { + return Error(Error::Code::kJsonParseError, "Invalid resolution"); + } + if (!out.IsValid()) { + return Error(Error::Code::kJsonParseError, "Invalid resolution values"); + } + return out; } bool Resolution::IsValid() const { @@ -71,14 +80,28 @@ bool Resolution::IsSupersetOf(const Resolution& other) const { return width >= other.width && height >= other.height; } -bool Dimensions::TryParse(const Json::Value& root, Dimensions* out) { - if (!json::TryParseInt(root[kWidth], &(out->width)) || - !json::TryParseInt(root[kHeight], &(out->height)) || - !(root[kFrameRate].isNull() || - json::TryParseSimpleFraction(root[kFrameRate], &(out->frame_rate)))) { - return false; +ErrorOr Dimensions::TryParse(const Json::Value& root) { + if (!root.isObject()) { + return Error(Error::Code::kJsonParseError, + "Dimensions is not a JSON object"); + } + + Dimensions out; + if (!json::TryParseInt(root[kWidth], &out.width) || + !json::TryParseInt(root[kHeight], &out.height)) { + return Error(Error::Code::kJsonParseError, "Invalid dimensions"); + } + + if (!root[kFrameRate].isNull()) { + if (!json::TryParseSimpleFraction(root[kFrameRate], &out.frame_rate)) { + return Error(Error::Code::kJsonParseError, "Invalid frame rate"); + } + } + + if (!out.IsValid()) { + return Error(Error::Code::kJsonParseError, "Invalid dimensions values"); } - return out->IsValid(); + return out; } bool Dimensions::IsValid() const { diff --git a/cast/streaming/resolution.h b/cast/streaming/resolution.h index 1e52eabcd..038d82fd6 100644 --- a/cast/streaming/resolution.h +++ b/cast/streaming/resolution.h @@ -17,7 +17,7 @@ namespace openscreen::cast { // A resolution in pixels. struct Resolution { - static bool TryParse(const Json::Value& value, Resolution* out); + static ErrorOr TryParse(const Json::Value& value); bool IsValid() const; Json::Value ToJson() const; @@ -35,7 +35,7 @@ struct Resolution { // A resolution in pixels and a frame rate. struct Dimensions { - static bool TryParse(const Json::Value& value, Dimensions* out); + static ErrorOr TryParse(const Json::Value& value); bool IsValid() const; Json::Value ToJson() const; diff --git a/cast/streaming/rpc_messenger.h b/cast/streaming/rpc_messenger.h deleted file mode 100644 index dcd8755c5..000000000 --- a/cast/streaming/rpc_messenger.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_RPC_MESSENGER_H_ -#define CAST_STREAMING_RPC_MESSENGER_H_ - -#include "cast/streaming/public/rpc_messenger.h" - -#endif // CAST_STREAMING_RPC_MESSENGER_H_ diff --git a/cast/streaming/sender.h b/cast/streaming/sender.h deleted file mode 100644 index ebae3bd0a..000000000 --- a/cast/streaming/sender.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_SENDER_H_ -#define CAST_STREAMING_SENDER_H_ - -#include "cast/streaming/public/sender.h" - -#endif // CAST_STREAMING_SENDER_H_ diff --git a/cast/streaming/sender_message.cc b/cast/streaming/sender_message.cc index 287c77159..1494af6b9 100644 --- a/cast/streaming/sender_message.cc +++ b/cast/streaming/sender_message.cc @@ -5,6 +5,7 @@ #include "cast/streaming/sender_message.h" #include +#include #include "cast/streaming/message_fields.h" #include "util/base64.h" @@ -17,10 +18,11 @@ namespace openscreen::cast { namespace { -EnumNameTable kMessageTypeNames{ +EnumNameTable kMessageTypeNames{ {{kMessageTypeOffer, SenderMessage::Type::kOffer}, {"GET_CAPABILITIES", SenderMessage::Type::kGetCapabilities}, - {"RPC", SenderMessage::Type::kRpc}}}; + {"RPC", SenderMessage::Type::kRpc}, + {"INPUT", SenderMessage::Type::kInput}}}; SenderMessage::Type GetMessageType(const Json::Value& root) { std::string type; @@ -37,8 +39,9 @@ SenderMessage::Type GetMessageType(const Json::Value& root) { // static ErrorOr SenderMessage::Parse(const Json::Value& value) { - if (!value) { - return Error(Error::Code::kParameterInvalid, "Empty JSON"); + if (!value.isObject()) { + return Error(Error::Code::kParameterInvalid, + "SenderMessage body is not a JSON object"); } SenderMessage message; @@ -49,9 +52,9 @@ ErrorOr SenderMessage::Parse(const Json::Value& value) { message.type = GetMessageType(value); switch (message.type) { case Type::kOffer: { - Offer offer; - if (Offer::TryParse(value[kOfferMessageBody], &offer).ok()) { - message.body = std::move(offer); + auto offer_or_error = Offer::TryParse(value[kOfferMessageBody]); + if (offer_or_error.is_value()) { + message.body = std::move(offer_or_error.value()); message.valid = true; } } break; @@ -66,6 +69,16 @@ ErrorOr SenderMessage::Parse(const Json::Value& value) { } } break; + case Type::kInput: { + std::string input_body; + std::vector input; + if (json::TryParseString(value[kInputMessageBody], &input_body) && + base64::Decode(input_body, &input)) { + message.body = input; + message.valid = true; + } + } break; + case Type::kGetCapabilities: message.valid = true; break; @@ -90,12 +103,17 @@ ErrorOr SenderMessage::ToJson() const { switch (type) { case SenderMessage::Type::kOffer: - root[kOfferMessageBody] = absl::get(body).ToJson(); + root[kOfferMessageBody] = std::get(body).ToJson(); break; case SenderMessage::Type::kRpc: root[kRpcMessageBody] = - base64::Encode(absl::get>(body)); + base64::Encode(std::get>(body)); + break; + + case SenderMessage::Type::kInput: + root[kInputMessageBody] = + base64::Encode(std::get>(body)); break; case SenderMessage::Type::kGetCapabilities: diff --git a/cast/streaming/sender_message.h b/cast/streaming/sender_message.h index a7c8b07fe..24a779bf5 100644 --- a/cast/streaming/sender_message.h +++ b/cast/streaming/sender_message.h @@ -7,9 +7,9 @@ #include #include +#include #include -#include "absl/types/variant.h" #include "cast/streaming/public/offer_messages.h" #include "json/value.h" #include "platform/base/error.h" @@ -32,6 +32,9 @@ struct SenderMessage { // Rpc binary messages. The payload is base64-encoded. kRpc, + + // Input-related binary messages. The payload is base64-encoded. + kInput, }; static ErrorOr Parse(const Json::Value& value); @@ -40,10 +43,10 @@ struct SenderMessage { Type type = Type::kUnknown; int32_t sequence_number = -1; bool valid = false; - absl::variant, // Binary-encoded RPC message. - Offer, - std::string> + std::variant, // Binary-encoded protobuf message. + Offer, + std::string> body; }; diff --git a/cast/streaming/sender_packet_router_unittest.cc b/cast/streaming/sender_packet_router_unittest.cc index 4fd51fd7d..b5bc8fb0d 100644 --- a/cast/streaming/sender_packet_router_unittest.cc +++ b/cast/streaming/sender_packet_router_unittest.cc @@ -20,7 +20,6 @@ using testing::_; using testing::ElementsAreArray; -using testing::Invoke; using testing::Mock; using testing::Return; @@ -144,8 +143,8 @@ class MockSender : public SenderPacketRouter::Sender { (Clock::time_point send_time, ByteBuffer buffer), (override)); MOCK_METHOD(Clock::time_point, GetRtpResumeTime, (), (override)); - MOCK_METHOD(RtpTimeTicks, GetLastRtpTimestamp, (), (const override)); - MOCK_METHOD(StreamType, GetStreamType, (), (const override)); + MOCK_METHOD(RtpTimeTicks, GetLastRtpTimestamp, (), (const, override)); + MOCK_METHOD(StreamType, GetStreamType, (), (const, override)); }; class SenderPacketRouterTest : public testing::Test { @@ -261,10 +260,10 @@ TEST_F(SenderPacketRouterTest, RoutesRTCPPacketsFromReceivers) { Clock::time_point arrival_time{}; std::vector received_packet; EXPECT_CALL(*audio_sender(), OnReceivedRtcpPacket(_, _)) - .WillOnce(Invoke([&](Clock::time_point when, ByteView packet) { + .WillOnce([&](Clock::time_point when, ByteView packet) { arrival_time = when; received_packet.assign(packet.begin(), packet.end()); - })); + }); EXPECT_CALL(*video_sender(), OnReceivedRtcpPacket(_, _)).Times(0); const Clock::time_point expected_arrival_time = env()->now(); @@ -287,15 +286,15 @@ TEST_F(SenderPacketRouterTest, RoutesRTCPPacketsFromReceivers) { Clock::time_point audio_arrival_time{}, video_arrival_time{}; std::vector received_audio_packet, received_video_packet; EXPECT_CALL(*audio_sender(), OnReceivedRtcpPacket(_, _)) - .WillOnce(Invoke([&](Clock::time_point when, ByteView packet) { + .WillOnce([&](Clock::time_point when, ByteView packet) { audio_arrival_time = when; received_audio_packet.assign(packet.begin(), packet.end()); - })); + }); EXPECT_CALL(*video_sender(), OnReceivedRtcpPacket(_, _)) - .WillOnce(Invoke([&](Clock::time_point when, ByteView packet) { + .WillOnce([&](Clock::time_point when, ByteView packet) { video_arrival_time = when; received_video_packet.assign(packet.begin(), packet.end()); - })); + }); const Clock::time_point expected_audio_arrival_time = env()->now(); SimulatePacketArrivedNow(kRemoteEndpoint, kValidAudioRtcpPacket); @@ -329,7 +328,7 @@ TEST_F(SenderPacketRouterTest, SchedulesPeriodicTransmissionOfRTCPPackets) { EXPECT_CALL(*audio_sender(), OnReceivedRtcpPacket(_, _)).Times(0); EXPECT_CALL(*audio_sender(), GetRtcpPacketForImmediateSend(_, _)) .Times(kNumIterations) - .WillRepeatedly(Invoke(&MakeFakePacket)); + .WillRepeatedly(&MakeFakePacket); EXPECT_CALL(*audio_sender(), GetRtpPacketForImmediateSend(_, _)).Times(0); ON_CALL(*audio_sender(), GetRtpResumeTime()) .WillByDefault(Return(SenderPacketRouter::kNever)); @@ -337,9 +336,9 @@ TEST_F(SenderPacketRouterTest, SchedulesPeriodicTransmissionOfRTCPPackets) { // Capture every packet sent for analysis at the end of this test. std::vector> packets_sent; EXPECT_CALL(*env(), SendPacket(_, _)) - .WillRepeatedly(Invoke([&](ByteView packet, PacketMetadata metadata) { + .WillRepeatedly([&](ByteView packet, PacketMetadata metadata) { packets_sent.emplace_back(packet.begin(), packet.end()); - })); + }); const Clock::time_point first_send_time = env()->now(); router()->RequestRtcpSend(kAudioReceiverSsrc); @@ -369,9 +368,9 @@ TEST_F(SenderPacketRouterTest, SchedulesAndTransmitsRTPBursts) { // Capture every packet sent for analysis at the end of this test. std::vector> packets_sent; EXPECT_CALL(*env(), SendPacket(_, _)) - .WillRepeatedly(Invoke([&](ByteView packet, PacketMetadata metadata) { + .WillRepeatedly([&](ByteView packet, PacketMetadata metadata) { packets_sent.emplace_back(packet.begin(), packet.end()); - })); + }); // Simulate a typical video Sender RTP at-startup sending sequence: First, at // t=0ms, the Sender wants to send its large 10-packet key frame. This will @@ -393,26 +392,25 @@ TEST_F(SenderPacketRouterTest, SchedulesAndTransmitsRTPBursts) { int num_get_rtp_calls = 0; EXPECT_CALL(*video_sender(), GetRtpPacketForImmediateSend(_, _)) .Times(14 + 2) - .WillRepeatedly( - Invoke([&](Clock::time_point send_time, ByteBuffer buffer) { - ++num_get_rtp_calls; - - // 14 packets are sent: The first through fourth bursts send three - // packets each, and the fifth burst sends two. - if (num_get_rtp_calls <= 14) { - return MakeFakePacket(send_time, buffer); - } - - // 2 "done signals" are then sent: One is at the end of the fifth - // burst, one is for a "nothing to send" sixth burst. - return ToEmptyPacketBuffer(send_time, buffer); - })); + .WillRepeatedly([&](Clock::time_point send_time, ByteBuffer buffer) { + ++num_get_rtp_calls; + + // 14 packets are sent: The first through fourth bursts send three + // packets each, and the fifth burst sends two. + if (num_get_rtp_calls <= 14) { + return MakeFakePacket(send_time, buffer); + } + + // 2 "done signals" are then sent: One is at the end of the fifth + // burst, one is for a "nothing to send" sixth burst. + return ToEmptyPacketBuffer(send_time, buffer); + }); const Clock::time_point kickstart_time = start_time + 4 * kBurstInterval + milliseconds(25); int num_get_resume_calls = 0; EXPECT_CALL(*video_sender(), GetRtpResumeTime()) .Times(4 + 1 + 1) - .WillRepeatedly(Invoke([&] { + .WillRepeatedly([&] { ++num_get_resume_calls; // After each of the first through fourth bursts, the Sender wants to @@ -429,7 +427,7 @@ TEST_F(SenderPacketRouterTest, SchedulesAndTransmitsRTPBursts) { // After the sixth burst, the Sender pauses RTP sending indefinitely. return SenderPacketRouter::kNever; - })); + }); router()->RequestRtpSend(kVideoReceiverSsrc); // Execute first burst. RunTasksUntilIdle(); @@ -444,8 +442,8 @@ TEST_F(SenderPacketRouterTest, SchedulesAndTransmitsRTPBursts) { // Now, resume RTP sending for one more 1-packet frame, and then pause RTP // sending again. EXPECT_CALL(*video_sender(), GetRtpPacketForImmediateSend(_, _)) - .WillOnce(Invoke(&MakeFakePacket)) // Frame 2, only packet. - .WillOnce(Invoke(&ToEmptyPacketBuffer)); // Done for now. + .WillOnce(&MakeFakePacket) // Frame 2, only packet. + .WillOnce(&ToEmptyPacketBuffer); // Done for now. // After the seventh burst, the Sender pauses RTP sending again. EXPECT_CALL(*video_sender(), GetRtpResumeTime()) .WillOnce(Return(SenderPacketRouter::kNever)); @@ -495,9 +493,9 @@ TEST_F(SenderPacketRouterTest, SchedulesAndTransmitsAccountingForPriority) { // Capture every packet sent for analysis at the end of this test. std::vector> packets_sent; EXPECT_CALL(*env(), SendPacket(_, _)) - .WillRepeatedly(Invoke([&](ByteView packet, PacketMetadata metadata) { + .WillRepeatedly([&](ByteView packet, PacketMetadata metadata) { packets_sent.emplace_back(packet.begin(), packet.end()); - })); + }); // These indicate how often one packet will be sent from each Sender. constexpr Clock::duration kAudioRtpInterval = milliseconds(10); @@ -506,45 +504,41 @@ TEST_F(SenderPacketRouterTest, SchedulesAndTransmitsAccountingForPriority) { // Note: The priority flags used in this test ('0'..'3') indicate // lowest-to-highest priority. EXPECT_CALL(*audio_sender(), GetRtcpPacketForImmediateSend(_, _)) - .WillRepeatedly( - Invoke([](Clock::time_point send_time, ByteBuffer buffer) { - return MakeFakePacketWithFlag('3', send_time, buffer); - })); + .WillRepeatedly([](Clock::time_point send_time, ByteBuffer buffer) { + return MakeFakePacketWithFlag('3', send_time, buffer); + }); int num_audio_get_rtp_calls = 0; EXPECT_CALL(*audio_sender(), GetRtpPacketForImmediateSend(_, _)) - .WillRepeatedly( - Invoke([&](Clock::time_point send_time, ByteBuffer buffer) { - // Alternate between returning a single packet and a "done for now" - // signal. - ++num_audio_get_rtp_calls; - if (num_audio_get_rtp_calls % 2) { - return MakeFakePacketWithFlag('1', send_time, buffer); - } - return buffer.subspan(0, 0); - })); + .WillRepeatedly([&](Clock::time_point send_time, ByteBuffer buffer) { + // Alternate between returning a single packet and a "done for now" + // signal. + ++num_audio_get_rtp_calls; + if (num_audio_get_rtp_calls % 2) { + return MakeFakePacketWithFlag('1', send_time, buffer); + } + return buffer.subspan(0, 0); + }); EXPECT_CALL(*video_sender(), GetRtcpPacketForImmediateSend(_, _)) - .WillRepeatedly( - Invoke([](Clock::time_point send_time, ByteBuffer buffer) { - return MakeFakePacketWithFlag('2', send_time, buffer); - })); + .WillRepeatedly([](Clock::time_point send_time, ByteBuffer buffer) { + return MakeFakePacketWithFlag('2', send_time, buffer); + }); int num_video_get_rtp_calls = 0; EXPECT_CALL(*video_sender(), GetRtpPacketForImmediateSend(_, _)) - .WillRepeatedly( - Invoke([&](Clock::time_point send_time, ByteBuffer buffer) { - // Alternate between returning a single packet and a "done for now" - // signal. - ++num_video_get_rtp_calls; - if (num_video_get_rtp_calls % 2) { - return MakeFakePacketWithFlag('0', send_time, buffer); - } - return buffer.subspan(0, 0); - })); - EXPECT_CALL(*audio_sender(), GetRtpResumeTime()).WillRepeatedly(Invoke([&] { + .WillRepeatedly([&](Clock::time_point send_time, ByteBuffer buffer) { + // Alternate between returning a single packet and a "done for now" + // signal. + ++num_video_get_rtp_calls; + if (num_video_get_rtp_calls % 2) { + return MakeFakePacketWithFlag('0', send_time, buffer); + } + return buffer.subspan(0, 0); + }); + EXPECT_CALL(*audio_sender(), GetRtpResumeTime()).WillRepeatedly([&] { return env()->now() + kAudioRtpInterval; - })); - EXPECT_CALL(*video_sender(), GetRtpResumeTime()).WillRepeatedly(Invoke([&] { + }); + EXPECT_CALL(*video_sender(), GetRtpResumeTime()).WillRepeatedly([&] { return env()->now() + kVideoRtpInterval; - })); + }); // Request starting both RTCP and RTP sends for both Senders, in a random // order. diff --git a/cast/streaming/sender_session.h b/cast/streaming/sender_session.h deleted file mode 100644 index 23ae7d514..000000000 --- a/cast/streaming/sender_session.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_SENDER_SESSION_H_ -#define CAST_STREAMING_SENDER_SESSION_H_ - -#include "cast/streaming/public/sender_session.h" - -#endif // CAST_STREAMING_SENDER_SESSION_H_ diff --git a/cast/streaming/session_messenger.h b/cast/streaming/session_messenger.h deleted file mode 100644 index cf9e93b9e..000000000 --- a/cast/streaming/session_messenger.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_SESSION_MESSENGER_H_ -#define CAST_STREAMING_SESSION_MESSENGER_H_ - -#include "cast/streaming/public/session_messenger.h" - -#endif // CAST_STREAMING_SESSION_MESSENGER_H_ diff --git a/cast/streaming/statistics.h b/cast/streaming/statistics.h deleted file mode 100644 index 8e10a43ac..000000000 --- a/cast/streaming/statistics.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2024 The Chromium Authors -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(issuetracker.google.com/154090565): Remove this once clients are updated - -#ifndef CAST_STREAMING_STATISTICS_H_ -#define CAST_STREAMING_STATISTICS_H_ - -#include "cast/streaming/public/statistics.h" - -#endif // CAST_STREAMING_STATISTICS_H_ diff --git a/cast/streaming/testing/simple_message_port.h b/cast/streaming/testing/simple_message_port.h index 43f7a94c1..a9230001b 100644 --- a/cast/streaming/testing/simple_message_port.h +++ b/cast/streaming/testing/simple_message_port.h @@ -57,6 +57,8 @@ class SimpleMessagePort : public MessagePort { return posted_messages_; } + void clear() { posted_messages_.clear(); } + private: MessagePort::Client* client_ = nullptr; std::string destination_id_; diff --git a/cast/test/BUILD.gn b/cast/test/BUILD.gn index 704f8c647..f34e5d6e2 100644 --- a/cast/test/BUILD.gn +++ b/cast/test/BUILD.gn @@ -36,7 +36,6 @@ if (is_posix && !build_with_chromium) { "../../platform", "../../platform:standalone_impl", "../../testing/util", - "../../third_party/abseil", "../../third_party/boringssl", "../../third_party/getopt", "../../third_party/googletest:gmock", diff --git a/cast/test/cast_socket_e2e_test.cc b/cast/test/cast_socket_e2e_test.cc index 637d99f84..187ce1c02 100644 --- a/cast/test/cast_socket_e2e_test.cc +++ b/cast/test/cast_socket_e2e_test.cc @@ -55,7 +55,7 @@ class SenderSocketsClient : public SenderSocketFactory::Client, // SenderSocketFactory::Client overrides. void OnConnected(SenderSocketFactory* factory, const IPEndpoint& endpoint, - std::unique_ptr socket) { + std::unique_ptr socket) override { OSP_CHECK(!socket_); OSP_LOG_INFO << kLogDecorator << "Sender connected to endpoint: " << endpoint; @@ -234,6 +234,11 @@ class CastSocketE2ETest : public ::testing::Test { // TODO(issuetracker.google.com/169967989): Would like to have a symmetric // OnClose check. EXPECT_CALL(*client, OnCloseMock(client->socket())); + // Verification of SSL_shutdown (issuetracker.google.com/169966671): + // If the peer does not call SSL_shutdown during socket closure, BoringSSL + // will report a protocol error (missing close notify) resulting in + // Error::Code::kFatalSSLError. The expectation of kSocketClosedFailure + // here explicitly verifies that SSL_shutdown was called and succeeded. EXPECT_CALL(*peer_client, OnErrorMock(peer_client->socket(), _)) .WillOnce([](CastSocket* socket, const Error& error) { EXPECT_EQ(error.code(), Error::Code::kSocketClosedFailure); diff --git a/cast/test/device_auth_test.cc b/cast/test/device_auth_test.cc index 080a932a8..cb3758af8 100644 --- a/cast/test/device_auth_test.cc +++ b/cast/test/device_auth_test.cc @@ -19,6 +19,7 @@ #include "cast/sender/channel/message_util.h" #include "gtest/gtest.h" #include "platform/test/paths.h" +#include "util/no_destructor.h" #include "util/read_file.h" namespace openscreen::cast { @@ -28,11 +29,11 @@ using proto::CastMessage; using proto::DeviceAuthMessage; using ::testing::_; -using ::testing::Invoke; const std::string& GetSpecificTestDataPath() { - static std::string data_path = GetTestDataPath() + "cast/receiver/channel/"; - return data_path; + static const NoDestructor data_path(GetTestDataPath() + + "cast/receiver/channel/"); + return *data_path; } class DeviceAuthTest : public ::testing::Test { @@ -80,10 +81,9 @@ class DeviceAuthTest : public ::testing::Test { } CastMessage challenge_reply; EXPECT_CALL(fake_cast_socket_pair_.mock_peer_client, OnMessage(_, _)) - .WillOnce( - Invoke([&challenge_reply](CastSocket* socket, CastMessage message) { - challenge_reply = std::move(message); - })); + .WillOnce([&challenge_reply](CastSocket* socket, CastMessage message) { + challenge_reply = std::move(message); + }); ASSERT_TRUE( fake_cast_socket_pair_.peer_socket->Send(std::move(auth_challenge)) .ok()); diff --git a/discovery/BUILD.gn b/discovery/BUILD.gn index 256e81af8..eb5ed41b3 100644 --- a/discovery/BUILD.gn +++ b/discovery/BUILD.gn @@ -70,7 +70,6 @@ openscreen_source_set("mdns") { "mdns/impl/mdns_trackers.h", ] - public_deps = [ "../third_party/abseil" ] deps = [ ":public", "../platform", @@ -115,7 +114,6 @@ openscreen_source_set("dnssd") { deps = [ ":mdns", ":public", - "../third_party/abseil", "../util", ] friend = [ ":unittests" ] @@ -135,7 +133,6 @@ openscreen_source_set("testing") { deps = [ ":mdns", ":public", - "../third_party/abseil", "../third_party/googletest:gmock", "../third_party/googletest:gtest", ] @@ -176,7 +173,6 @@ openscreen_source_set("unittests") { ":public", ":testing", "../platform:test", - "../third_party/abseil", "../third_party/googletest:gmock", "../third_party/googletest:gtest", "../util", diff --git a/discovery/README.md b/discovery/README.md index f04ba90e6..314b0bc3c 100644 --- a/discovery/README.md +++ b/discovery/README.md @@ -5,4 +5,4 @@ defined in [RFC 6762](https://tools.ietf.org/html/rfc6762) and [RFC 6763](https://tools.ietf.org/html/rfc6763). For more details, see the -[full documentation](https://chromium.googlesource.com/openscreen/+/refs/heads/master/docs/discovery.md). +[full documentation](https://chromium.googlesource.com/openscreen/+/refs/heads/main/docs/discovery.md). diff --git a/discovery/dnssd/impl/conversion_layer.cc b/discovery/dnssd/impl/conversion_layer.cc index 7d3df5596..370f37b33 100644 --- a/discovery/dnssd/impl/conversion_layer.cc +++ b/discovery/dnssd/impl/conversion_layer.cc @@ -6,6 +6,7 @@ #include #include +#include #include #include "discovery/dnssd/impl/constants.h" @@ -14,7 +15,6 @@ #include "discovery/dnssd/public/dns_sd_instance.h" #include "discovery/mdns/public/mdns_constants.h" #include "discovery/mdns/public/mdns_records.h" -#include "util/span_util.h" #include "util/string_util.h" namespace openscreen::discovery { @@ -146,7 +146,7 @@ DomainName GetDomainName(const InstanceKey& key) { DomainName GetDomainName(const MdnsRecord& record) { return IsPtrRecord(record) - ? absl::get(record.rdata()).ptr_domain() + ? std::get(record.rdata()).ptr_domain() : record.name(); } diff --git a/discovery/dnssd/impl/conversion_layer_unittest.cc b/discovery/dnssd/impl/conversion_layer_unittest.cc index 3a80691dc..ea3be7024 100644 --- a/discovery/dnssd/impl/conversion_layer_unittest.cc +++ b/discovery/dnssd/impl/conversion_layer_unittest.cc @@ -6,6 +6,7 @@ #include #include +#include #include #include "discovery/dnssd/impl/instance_key.h" @@ -14,7 +15,6 @@ #include "discovery/mdns/testing/mdns_test_util.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "util/span_util.h" #include "util/std_util.h" namespace openscreen::discovery { @@ -115,7 +115,7 @@ TEST(DnsSdConversionLayerTest, GetDnsRecordsPtr) { FakeDnsRecordFactory::kServiceNameProtocolPart); EXPECT_EQ(it->name().labels()[2], FakeDnsRecordFactory::kDomainName); - const auto& rdata = absl::get(it->rdata()); + const auto& rdata = std::get(it->rdata()); EXPECT_EQ(rdata.ptr_domain().labels().size(), size_t{4}); EXPECT_EQ(rdata.ptr_domain().labels()[0], FakeDnsRecordFactory::kInstanceName); @@ -152,7 +152,7 @@ TEST(DnsSdConversionLayerTest, GetDnsRecordsSrv) { FakeDnsRecordFactory::kServiceNameProtocolPart); EXPECT_EQ(it->name().labels()[3], FakeDnsRecordFactory::kDomainName); - const auto& rdata = absl::get(it->rdata()); + const auto& rdata = std::get(it->rdata()); EXPECT_EQ(rdata.priority(), 0); EXPECT_EQ(rdata.weight(), 0); EXPECT_EQ(rdata.port(), FakeDnsRecordFactory::kPortNum); @@ -184,7 +184,7 @@ TEST(DnsSdConversionLayerTest, GetDnsRecordsAPresent) { FakeDnsRecordFactory::kServiceNameProtocolPart); EXPECT_EQ(it->name().labels()[3], FakeDnsRecordFactory::kDomainName); - const auto& rdata = absl::get(it->rdata()); + const auto& rdata = std::get(it->rdata()); EXPECT_EQ(rdata.ipv4_address(), IPAddress(FakeDnsRecordFactory::kV4AddressOctets)); } @@ -228,7 +228,7 @@ TEST(DnsSdConversionLayerTest, GetDnsRecordsAAAAPresent) { FakeDnsRecordFactory::kServiceNameProtocolPart); EXPECT_EQ(it->name().labels()[3], FakeDnsRecordFactory::kDomainName); - const auto& rdata = absl::get(it->rdata()); + const auto& rdata = std::get(it->rdata()); EXPECT_EQ(rdata.ipv6_address(), IPAddress(FakeDnsRecordFactory::kV6AddressHextets)); } @@ -275,7 +275,7 @@ TEST(DnsSdConversionLayerTest, GetDnsRecordsTxt) { FakeDnsRecordFactory::kServiceNameProtocolPart); EXPECT_EQ(it->name().labels()[3], FakeDnsRecordFactory::kDomainName); - const auto& rdata = absl::get(it->rdata()); + const auto& rdata = std::get(it->rdata()); EXPECT_EQ(rdata.texts().size(), size_t{2}); EXPECT_TRUE(Contains( rdata.texts(), diff --git a/discovery/dnssd/impl/dns_data_graph.cc b/discovery/dnssd/impl/dns_data_graph.cc index 482ce2452..475eb8f3e 100644 --- a/discovery/dnssd/impl/dns_data_graph.cc +++ b/discovery/dnssd/impl/dns_data_graph.cc @@ -6,6 +6,7 @@ #include #include +#include #include "discovery/dnssd/impl/conversion_layer.h" #include "discovery/dnssd/impl/instance_key.h" @@ -109,7 +110,7 @@ class DnsDataGraphImpl : public DnsDataGraph { if (it == records_.end()) { return std::nullopt; } else { - return std::cref(absl::get(it->rdata())); + return std::cref(std::get(it->rdata())); } } @@ -251,14 +252,14 @@ Error DnsDataGraphImpl::Node::ApplyDataRecordChange(MdnsRecord record, std::vector::iterator it; if (record.dns_type() == DnsType::kPTR) { - child_name = absl::get(record.rdata()).ptr_domain(); + child_name = std::get(record.rdata()).ptr_domain(); it = std::find_if(records_.begin(), records_.end(), [record](const MdnsRecord& rhs) { return record.IsReannouncementOf(rhs); }); } else { if (record.dns_type() == DnsType::kSRV) { - child_name = absl::get(record.rdata()).target(); + child_name = std::get(record.rdata()).target(); } it = FindRecord(record.dns_type()); } @@ -545,7 +546,7 @@ DnsDataGraphImpl::CalculatePtrRecordEndpoints(Node* node) const { } const DomainName domain = - absl::get(record.rdata()).ptr_domain(); + std::get(record.rdata()).ptr_domain(); const Node* child = nodes_.find(domain)->second.get(); std::vector> child_endpoints = CreateEndpoints(DomainGroup::kSrvAndTxt, child->name()); diff --git a/discovery/dnssd/impl/dns_data_graph_unittest.cc b/discovery/dnssd/impl/dns_data_graph_unittest.cc index 465d32311..d63b4e17d 100644 --- a/discovery/dnssd/impl/dns_data_graph_unittest.cc +++ b/discovery/dnssd/impl/dns_data_graph_unittest.cc @@ -35,7 +35,6 @@ IPAddress GetAddressV6(const DnsSdInstanceEndpoint endpoint) { } // namespace using testing::_; -using testing::Invoke; using testing::Return; using testing::StrictMock; diff --git a/discovery/dnssd/impl/publisher_impl.cc b/discovery/dnssd/impl/publisher_impl.cc index 733eaf0e0..6cbc16e17 100644 --- a/discovery/dnssd/impl/publisher_impl.cc +++ b/discovery/dnssd/impl/publisher_impl.cc @@ -280,7 +280,7 @@ void PublisherImpl::OnDomainFound(const DomainName& requested_name, Error result = mdns_publisher_.RegisterRecord(mdns_record); if (!result.ok()) { reporting_client_.OnRecoverableError( - Error(Error::Code::kRecordPublicationError, result.ToString())); + Error(Error::Code::kRecordPublicationError, result.message())); } } diff --git a/discovery/dnssd/impl/publisher_impl_unittest.cc b/discovery/dnssd/impl/publisher_impl_unittest.cc index 293728d4f..80d75eb8c 100644 --- a/discovery/dnssd/impl/publisher_impl_unittest.cc +++ b/discovery/dnssd/impl/publisher_impl_unittest.cc @@ -5,6 +5,7 @@ #include "discovery/dnssd/impl/publisher_impl.h" #include +#include #include #include "discovery/common/testing/mock_reporting_client.h" @@ -23,8 +24,10 @@ using testing::StrictMock; class MockClient : public DnsSdPublisher::Client { public: - MOCK_METHOD2(OnEndpointClaimed, - void(const DnsSdInstance&, const DnsSdInstanceEndpoint&)); + MOCK_METHOD(void, + OnEndpointClaimed, + (const DnsSdInstance&, const DnsSdInstanceEndpoint&), + (override)); }; class MockMdnsService : public MdnsService { @@ -45,12 +48,16 @@ class MockMdnsService : public MdnsService { void ReinitializeQueries(const DomainName& name) override { FAIL(); } - MOCK_METHOD3(StartProbe, - Error(MdnsDomainConfirmedProvider*, DomainName, IPAddress)); - MOCK_METHOD2(UpdateRegisteredRecord, - Error(const MdnsRecord&, const MdnsRecord&)); - MOCK_METHOD1(RegisterRecord, Error(const MdnsRecord& record)); - MOCK_METHOD1(UnregisterRecord, Error(const MdnsRecord& record)); + MOCK_METHOD(Error, + StartProbe, + (MdnsDomainConfirmedProvider*, DomainName, IPAddress), + (override)); + MOCK_METHOD(Error, + UpdateRegisteredRecord, + (const MdnsRecord&, const MdnsRecord&), + (override)); + MOCK_METHOD(Error, RegisterRecord, (const MdnsRecord& record), (override)); + MOCK_METHOD(Error, UnregisterRecord, (const MdnsRecord& record), (override)); }; class PublisherImplTest : public testing::Test { @@ -98,13 +105,12 @@ TEST_F(PublisherImplTest, TestRegistrationAndDegrestration) { .WillRepeatedly([&seen, &address, &domain2](const MdnsRecord& record) mutable -> Error { if (record.dns_type() == DnsType::kA) { - const ARecordRdata& data = absl::get(record.rdata()); + const ARecordRdata& data = std::get(record.rdata()); if (data.ipv4_address() == address) { seen++; } } else if (record.dns_type() == DnsType::kSRV) { - const SrvRecordRdata& data = - absl::get(record.rdata()); + const SrvRecordRdata& data = std::get(record.rdata()); if (data.port() == 80) { seen++; } @@ -131,13 +137,12 @@ TEST_F(PublisherImplTest, TestRegistrationAndDegrestration) { .WillRepeatedly([&seen, &address](const MdnsRecord& record) mutable -> Error { if (record.dns_type() == DnsType::kA) { - const ARecordRdata& data = absl::get(record.rdata()); + const ARecordRdata& data = std::get(record.rdata()); if (data.ipv4_address() == address) { seen++; } } else if (record.dns_type() == DnsType::kSRV) { - const SrvRecordRdata& data = - absl::get(record.rdata()); + const SrvRecordRdata& data = std::get(record.rdata()); if (data.port() == 80) { seen++; } diff --git a/discovery/dnssd/impl/querier_impl.cc b/discovery/dnssd/impl/querier_impl.cc index 19b2393f4..9e461abc4 100644 --- a/discovery/dnssd/impl/querier_impl.cc +++ b/discovery/dnssd/impl/querier_impl.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "discovery/common/reporting_client.h" @@ -328,7 +329,7 @@ std::vector QuerierImpl::OnRecordChanged( const DomainName& create_endpoints_domain = record.dns_type() != DnsType::kPTR ? record.name() - : absl::get(record.rdata()).ptr_domain(); + : std::get(record.rdata()).ptr_domain(); const DnsDataGraph::DomainGroup create_endpoints_group = record.dns_type() != DnsType::kPTR ? DnsDataGraph::GetDomainGroup(record) diff --git a/discovery/dnssd/impl/service_key.h b/discovery/dnssd/impl/service_key.h index cc206509a..43b6ea9aa 100644 --- a/discovery/dnssd/impl/service_key.h +++ b/discovery/dnssd/impl/service_key.h @@ -5,11 +5,13 @@ #ifndef DISCOVERY_DNSSD_IMPL_SERVICE_KEY_H_ #define DISCOVERY_DNSSD_IMPL_SERVICE_KEY_H_ +#include #include #include #include #include "platform/base/error.h" +#include "util/hashing.h" namespace openscreen::discovery { @@ -45,9 +47,6 @@ class ServiceKey { std::string service_id_; std::string domain_id_; - template - friend H AbslHashValue(H h, const ServiceKey& key); - friend bool operator<(const ServiceKey& lhs, const ServiceKey& rhs); // Validation method which needs the same code as CreateFromRecord(). Use a @@ -57,12 +56,6 @@ class ServiceKey { friend bool HasValidDnsRecordAddress(const DomainName& domain); }; -// Hashing functions to allow for using with absl::Hash<...>. -template -H AbslHashValue(H h, const ServiceKey& key) { - return H::combine(std::move(h), key.service_id_, key.domain_id_); -} - // Comparison operators for using above keys with an std::map inline bool operator<(const ServiceKey& lhs, const ServiceKey& rhs) { int comp = lhs.domain_id_.compare(rhs.domain_id_); @@ -94,4 +87,15 @@ inline bool operator!=(const ServiceKey& lhs, const ServiceKey& rhs) { } // namespace openscreen::discovery +namespace std { + +template <> +struct hash { + size_t operator()(const openscreen::discovery::ServiceKey& key) const { + return openscreen::ComputeAggregateHash(key.service_id(), key.domain_id()); + } +}; + +} // namespace std + #endif // DISCOVERY_DNSSD_IMPL_SERVICE_KEY_H_ diff --git a/discovery/dnssd/public/dns_sd_txt_record.cc b/discovery/dnssd/public/dns_sd_txt_record.cc index 684fd1acf..acb9656c2 100644 --- a/discovery/dnssd/public/dns_sd_txt_record.cc +++ b/discovery/dnssd/public/dns_sd_txt_record.cc @@ -7,8 +7,6 @@ #include #include -#include "util/span_util.h" - namespace openscreen::discovery { // static diff --git a/discovery/mdns/impl/mdns_probe_manager_unittest.cc b/discovery/mdns/impl/mdns_probe_manager_unittest.cc index c6c0c9f2b..8d4f1771f 100644 --- a/discovery/mdns/impl/mdns_probe_manager_unittest.cc +++ b/discovery/mdns/impl/mdns_probe_manager_unittest.cc @@ -19,7 +19,6 @@ #include "platform/test/fake_udp_socket.h" using testing::_; -using testing::Invoke; using testing::Return; using testing::StrictMock; diff --git a/discovery/mdns/impl/mdns_probe_unittest.cc b/discovery/mdns/impl/mdns_probe_unittest.cc index ab79327b1..a809dd680 100644 --- a/discovery/mdns/impl/mdns_probe_unittest.cc +++ b/discovery/mdns/impl/mdns_probe_unittest.cc @@ -19,7 +19,6 @@ #include "platform/test/fake_udp_socket.h" using testing::_; -using testing::Invoke; using testing::Return; using testing::StrictMock; @@ -90,25 +89,13 @@ class MdnsProbeTests : public testing::Test { const IPEndpoint endpoint_v4_{address_v4_, 80}; }; -// TODO(issuetracker.google.com/243611087): Occasionally flaky in bots. -TEST_F(MdnsProbeTests, DISABLED_TestNoCancelationFlow) { - EXPECT_CALL(sender_, SendMulticast(_)); - clock_.Advance(kDelayBetweenProbeQueries); - EXPECT_EQ(task_runner_.delayed_task_count(), 1); - testing::Mock::VerifyAndClearExpectations(&sender_); - - EXPECT_CALL(sender_, SendMulticast(_)); - clock_.Advance(kDelayBetweenProbeQueries); - EXPECT_EQ(task_runner_.delayed_task_count(), 1); - testing::Mock::VerifyAndClearExpectations(&sender_); - - EXPECT_CALL(sender_, SendMulticast(_)); - clock_.Advance(kDelayBetweenProbeQueries); - EXPECT_EQ(task_runner_.delayed_task_count(), 1); - testing::Mock::VerifyAndClearExpectations(&sender_); - +TEST_F(MdnsProbeTests, TestNoCancelationFlow) { + // 3 probes should be sent. + EXPECT_CALL(sender_, SendMulticast(_)).Times(3); EXPECT_CALL(observer_, OnProbeSuccess(probe_.get())).Times(1); - clock_.Advance(kDelayBetweenProbeQueries); + + // Advance enough time for all probes to finish (initial delay + 3 intervals). + clock_.Advance(std::chrono::seconds(1)); EXPECT_EQ(task_runner_.delayed_task_count(), 0); } @@ -117,14 +104,16 @@ TEST_F(MdnsProbeTests, CancelationWhenMatchingMessageReceived) { OnMessageReceived(CreateMessage(name_)); } -// TODO(issuetracker.google.com/243611087): Occasionally flaky in bots. -TEST_F(MdnsProbeTests, DISABLED_TestNoCancelationOnUnrelatedMessages) { +TEST_F(MdnsProbeTests, TestNoCancelationOnUnrelatedMessages) { OnMessageReceived(CreateMessage(name2_)); - EXPECT_CALL(sender_, SendMulticast(_)); - clock_.Advance(kDelayBetweenProbeQueries); - EXPECT_EQ(task_runner_.delayed_task_count(), 1); - testing::Mock::VerifyAndClearExpectations(&sender_); + // 3 probes should be sent. + EXPECT_CALL(sender_, SendMulticast(_)).Times(3); + EXPECT_CALL(observer_, OnProbeSuccess(probe_.get())).Times(1); + + // Advance enough time for all probes to finish (initial delay + 3 intervals). + clock_.Advance(std::chrono::seconds(1)); + EXPECT_EQ(task_runner_.delayed_task_count(), 0); } } // namespace openscreen::discovery diff --git a/discovery/mdns/impl/mdns_publisher.cc b/discovery/mdns/impl/mdns_publisher.cc index 4bc63cdb4..e7063ac4d 100644 --- a/discovery/mdns/impl/mdns_publisher.cc +++ b/discovery/mdns/impl/mdns_publisher.cc @@ -6,6 +6,7 @@ #include #include +#include #include "discovery/common/config.h" #include "discovery/mdns/impl/mdns_probe_manager.h" @@ -43,6 +44,13 @@ inline MdnsRecord CreateGoodbyeRecord(const MdnsRecord& record) { } // namespace +MdnsPublisher::RecordAnnouncerPtr MdnsPublisher::CreateAnnouncer( + MdnsRecord record) { + return std::make_unique(std::move(record), *this, + task_runner_, now_function_, + max_announcement_attempts_); +} + MdnsPublisher::MdnsPublisher(MdnsSender& sender, MdnsProbeManager& ownership_manager, TaskRunner& task_runner, @@ -236,7 +244,7 @@ Error MdnsPublisher::RemoveRecord(const MdnsRecord& record, bool MdnsPublisher::IsRecordNameClaimed(const MdnsRecord& record) const { const DomainName& name = record.dns_type() == DnsType::kPTR - ? absl::get(record.rdata()).ptr_domain() + ? std::get(record.rdata()).ptr_domain() : record.name(); return ownership_manager_.IsDomainClaimed(name); } diff --git a/discovery/mdns/impl/mdns_publisher.h b/discovery/mdns/impl/mdns_publisher.h index d8c30c146..134019172 100644 --- a/discovery/mdns/impl/mdns_publisher.h +++ b/discovery/mdns/impl/mdns_publisher.h @@ -51,6 +51,10 @@ class MdnsPublisher : public MdnsResponder::RecordHandler { TaskRunner& task_runner, ClockNowFunctionPtr now_function, const Config& config); + MdnsPublisher(const MdnsPublisher&) = delete; + MdnsPublisher(MdnsPublisher&&) noexcept = delete; + MdnsPublisher& operator=(const MdnsPublisher&) = delete; + MdnsPublisher& operator=(MdnsPublisher&&) = delete; ~MdnsPublisher() override; // Registers a new mDNS record for advertisement by this service. For A, AAAA, @@ -78,8 +82,6 @@ class MdnsPublisher : public MdnsResponder::RecordHandler { // Returns the total number of records currently registered; size_t GetRecordCount() const; - OSP_DISALLOW_COPY_AND_ASSIGN(MdnsPublisher); - private: // Class responsible for sending announcement and goodbye messages for // MdnsRecord instances when they are published, updated, or unpublished. The @@ -146,11 +148,7 @@ class MdnsPublisher : public MdnsResponder::RecordHandler { friend class MdnsPublisherTesting; // Creates a new published from the provided record. - RecordAnnouncerPtr CreateAnnouncer(MdnsRecord record) { - return std::make_unique(std::move(record), *this, - task_runner_, now_function_, - max_announcement_attempts_); - } + RecordAnnouncerPtr CreateAnnouncer(MdnsRecord record); // Removes the given record from the `records_` map. A goodbye record is only // sent for this removal if `should_announce_deletion` is true. diff --git a/discovery/mdns/impl/mdns_publisher_unittest.cc b/discovery/mdns/impl/mdns_publisher_unittest.cc index 8055cffef..aae7b4e2f 100644 --- a/discovery/mdns/impl/mdns_publisher_unittest.cc +++ b/discovery/mdns/impl/mdns_publisher_unittest.cc @@ -16,7 +16,6 @@ #include "util/std_util.h" using testing::_; -using testing::Invoke; using testing::Return; using testing::StrictMock; diff --git a/discovery/mdns/impl/mdns_querier.cc b/discovery/mdns/impl/mdns_querier.cc index 639c16b0a..b167f7ca7 100644 --- a/discovery/mdns/impl/mdns_querier.cc +++ b/discovery/mdns/impl/mdns_querier.cc @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "discovery/common/config.h" @@ -31,7 +32,7 @@ bool IsNegativeResponseFor(const MdnsRecord& record, DnsType type) { return false; } - const NsecRecordRdata& nsec = absl::get(record.rdata()); + const NsecRecordRdata& nsec = std::get(record.rdata()); // RFC 6762 section 6.1, the NSEC bit must NOT be set in the received NSEC // record to indicate this is an mDNS NSEC record rather than a traditional @@ -165,7 +166,7 @@ void RemoveInvalidNsecFlags(std::vector* records) { bool has_changed = false; // The types for the new record to create, if `has_changed`. - const NsecRecordRdata& nsec_rdata = absl::get(it->rdata()); + const NsecRecordRdata& nsec_rdata = std::get(it->rdata()); DnsTypeBitSet types; for (DnsType type : nsec_rdata.types()) { types.Insert(type); @@ -181,7 +182,7 @@ void RemoveInvalidNsecFlags(std::vector* records) { while (it != records->end() && it->name() == nsec->name() && it->dns_type() == DnsType::kNSEC) { has_changed |= - types.Insert(absl::get(it->rdata()).types()); + types.Insert(std::get(it->rdata()).types()); new_ttl = std::min(new_ttl, it->ttl()); it = records->erase(it); } @@ -304,7 +305,7 @@ int MdnsQuerier::RecordTrackerLruCache::Update( if (result.is_error()) { reporting_client_.OnRecoverableError( Error(Error::Code::kUpdateReceivedRecordFailure, - result.error().ToString())); + result.error().message())); continue; } @@ -581,7 +582,7 @@ bool MdnsQuerier::ShouldAnswerRecordBeProcessed(const MdnsRecord& answer) { // which is no longer active. std::vector types = {answer.dns_type()}; if (answer.dns_type() == DnsType::kNSEC) { - const auto& nsec_rdata = absl::get(answer.rdata()); + const auto& nsec_rdata = std::get(answer.rdata()); types = nsec_rdata.types(); } @@ -631,7 +632,7 @@ void MdnsQuerier::ProcessRecord(const MdnsRecord& record) { size_t types_count = 0; const DnsType* types_ptr = nullptr; if (record.dns_type() == DnsType::kNSEC) { - const auto& nsec_rdata = absl::get(record.rdata()); + const auto& nsec_rdata = std::get(record.rdata()); if (Contains(nsec_rdata.types(), DnsType::kANY)) { types_ptr = kTranslatedNsecAnyQueryTypes.data(); types_count = kTranslatedNsecAnyQueryTypes.size(); diff --git a/discovery/mdns/impl/mdns_querier_unittest.cc b/discovery/mdns/impl/mdns_querier_unittest.cc index ae1c19f7f..a5b57eca6 100644 --- a/discovery/mdns/impl/mdns_querier_unittest.cc +++ b/discovery/mdns/impl/mdns_querier_unittest.cc @@ -27,7 +27,6 @@ namespace openscreen::discovery { using testing::_; using testing::Args; -using testing::Invoke; using testing::Return; using testing::StrictMock; using testing::WithArgs; diff --git a/discovery/mdns/impl/mdns_responder.cc b/discovery/mdns/impl/mdns_responder.cc index 6721e9b10..8f18f7008 100644 --- a/discovery/mdns/impl/mdns_responder.cc +++ b/discovery/mdns/impl/mdns_responder.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include "discovery/mdns/impl/mdns_probe_manager.h" #include "discovery/mdns/impl/mdns_publisher.h" @@ -160,7 +161,7 @@ void ApplyQueryResults(MdnsMessage* message, OSP_CHECK(record.dns_type() == DnsType::kPTR); const DomainName& target = - absl::get(record.rdata()).ptr_domain(); + std::get(record.rdata()).ptr_domain(); AddAdditionalRecords(message, record_handler, target, known_answers, DnsType::kSRV, clazz, true); AddAdditionalRecords(message, record_handler, target, known_answers, @@ -178,7 +179,7 @@ void ApplyQueryResults(MdnsMessage* message, { const MdnsRecord& srv_record = message->additional_records()[i]; const DomainName& target = - absl::get(srv_record.rdata()).target(); + std::get(srv_record.rdata()).target(); AddAdditionalRecords(message, record_handler, target, known_answers, DnsType::kA, clazz, target == domain); } @@ -188,7 +189,7 @@ void ApplyQueryResults(MdnsMessage* message, { const MdnsRecord& srv_record = message->additional_records()[i]; const DomainName& target = - absl::get(srv_record.rdata()).target(); + std::get(srv_record.rdata()).target(); AddAdditionalRecords(message, record_handler, target, known_answers, DnsType::kAAAA, clazz, target == domain); } @@ -203,7 +204,7 @@ void ApplyQueryResults(MdnsMessage* message, OSP_CHECK(srv_record.dns_type() == DnsType::kSRV); const DomainName& target = - absl::get(srv_record.rdata()).target(); + std::get(srv_record.rdata()).target(); AddAdditionalRecords(message, record_handler, target, known_answers, DnsType::kA, clazz, target == domain); AddAdditionalRecords(message, record_handler, target, known_answers, diff --git a/discovery/mdns/impl/mdns_responder.h b/discovery/mdns/impl/mdns_responder.h index 0c60b60de..a67853634 100644 --- a/discovery/mdns/impl/mdns_responder.h +++ b/discovery/mdns/impl/mdns_responder.h @@ -12,7 +12,6 @@ #include "discovery/common/config.h" #include "discovery/mdns/public/mdns_records.h" #include "platform/api/time.h" -#include "platform/base/macros.h" #include "util/alarm.h" namespace openscreen { @@ -73,7 +72,10 @@ class MdnsResponder { const Config& config); ~MdnsResponder(); - OSP_DISALLOW_COPY_AND_ASSIGN(MdnsResponder); + MdnsResponder(const MdnsResponder&) = delete; + MdnsResponder(MdnsResponder&&) noexcept = delete; + MdnsResponder& operator=(const MdnsResponder&) = delete; + MdnsResponder& operator=(MdnsResponder&&) = delete; private: // Class which handles processing and responding to queries segmented into @@ -90,9 +92,8 @@ class MdnsResponder { const Config& config); TruncatedQuery(const TruncatedQuery& other) = delete; TruncatedQuery(TruncatedQuery&& other) noexcept = delete; - TruncatedQuery& operator=(const TruncatedQuery& other) = delete; - TruncatedQuery& operator=(TruncatedQuery&& other) noexcept = delete; + TruncatedQuery& operator=(TruncatedQuery&& other) = delete; // Sets the query associated with this instance. Must only be called if no // query has already been set, here or through the ctor. diff --git a/discovery/mdns/impl/mdns_responder_unittest.cc b/discovery/mdns/impl/mdns_responder_unittest.cc index d59011085..5ef0738ef 100644 --- a/discovery/mdns/impl/mdns_responder_unittest.cc +++ b/discovery/mdns/impl/mdns_responder_unittest.cc @@ -5,6 +5,7 @@ #include "discovery/mdns/impl/mdns_responder.h" #include +#include #include "discovery/common/config.h" #include "discovery/mdns/impl/mdns_probe_manager.h" @@ -33,7 +34,7 @@ void CheckSingleNsecRecordType(const MdnsMessage& message, DnsType type) { const MdnsRecord record = message.answers()[0]; ASSERT_EQ(record.dns_type(), DnsType::kNSEC); - const NsecRecordRdata& rdata = absl::get(record.rdata()); + const NsecRecordRdata& rdata = std::get(record.rdata()); ASSERT_EQ(rdata.types().size(), size_t{1}); EXPECT_EQ(rdata.types()[0], type); @@ -41,7 +42,7 @@ void CheckSingleNsecRecordType(const MdnsMessage& message, DnsType type) { void CheckPtrDomain(const MdnsRecord& record, const DomainName& domain) { ASSERT_EQ(record.dns_type(), DnsType::kPTR); - const PtrRecordRdata& rdata = absl::get(record.rdata()); + const PtrRecordRdata& rdata = std::get(record.rdata()); EXPECT_EQ(rdata.ptr_domain(), domain); } @@ -53,7 +54,7 @@ void ExpectContainsNsecRecordType(const std::vector& records, return false; } - const NsecRecordRdata& rdata = absl::get(record.rdata()); + const NsecRecordRdata& rdata = std::get(record.rdata()); return rdata.types().size() == 1 && rdata.types()[0] == type; })); } @@ -62,7 +63,6 @@ void ExpectContainsNsecRecordType(const std::vector& records, using testing::_; using testing::Args; -using testing::Invoke; using testing::Return; using testing::StrictMock; @@ -70,7 +70,10 @@ class MockRecordHandler : public MdnsResponder::RecordHandler { public: void AddRecord(MdnsRecord record) { records_.push_back(record); } - MOCK_METHOD3(HasRecords, bool(const DomainName&, DnsType, DnsClass)); + MOCK_METHOD(bool, + HasRecords, + (const DomainName&, DnsType, DnsClass), + (override)); std::vector GetRecords(const DomainName& name, DnsType type, @@ -104,16 +107,20 @@ class MockMdnsSender : public MdnsSender { public: explicit MockMdnsSender(UdpSocket& socket) : MdnsSender(socket) {} - MOCK_METHOD1(SendMulticast, Error(const MdnsMessage& message)); - MOCK_METHOD2(SendMessage, - Error(const MdnsMessage& message, const IPEndpoint& endpoint)); + MOCK_METHOD(Error, SendMulticast, (const MdnsMessage& message), (override)); + MOCK_METHOD(Error, + SendMessage, + (const MdnsMessage& message, const IPEndpoint& endpoint), + (override)); }; class MockProbeManager : public MdnsProbeManager { public: - MOCK_CONST_METHOD1(IsDomainClaimed, bool(const DomainName&)); - MOCK_METHOD2(RespondToProbeQuery, - void(const MdnsMessage&, const IPEndpoint&)); + MOCK_METHOD(bool, IsDomainClaimed, (const DomainName&), (const, override)); + MOCK_METHOD(void, + RespondToProbeQuery, + (const MdnsMessage&, const IPEndpoint&), + (override)); }; class MdnsResponderTest : public testing::Test { diff --git a/discovery/mdns/impl/mdns_sender_unittest.cc b/discovery/mdns/impl/mdns_sender_unittest.cc index 99867b3a5..e157fc1ae 100644 --- a/discovery/mdns/impl/mdns_sender_unittest.cc +++ b/discovery/mdns/impl/mdns_sender_unittest.cc @@ -11,7 +11,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "platform/base/span.h" -#include "platform/test/byte_view_test_util.h" #include "platform/test/fake_udp_socket.h" #include "platform/test/mock_udp_socket.h" @@ -24,8 +23,7 @@ using testing::StrictMock; using testing::WithArgs; ACTION_P(MatchesBytes, expected_data) { - const ByteView& actual_data = arg0; - ExpectByteViewsHaveSameBytes(actual_data, expected_data); + EXPECT_THAT(arg0, testing::ElementsAreArray(expected_data)); } class MdnsSenderTest : public testing::Test { diff --git a/discovery/mdns/impl/mdns_trackers.cc b/discovery/mdns/impl/mdns_trackers.cc index 14d20ea2b..4457b1e6f 100644 --- a/discovery/mdns/impl/mdns_trackers.cc +++ b/discovery/mdns/impl/mdns_trackers.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include "discovery/common/config.h" #include "discovery/mdns/impl/mdns_random.h" @@ -46,7 +47,7 @@ bool IsNegativeResponseForType(const MdnsRecord& record, DnsType dns_type) { return false; } - const auto& nsec_types = absl::get(record.rdata()).types(); + const auto& nsec_types = std::get(record.rdata()).types(); return ContainsIf(nsec_types, [dns_type](DnsType type) { return type == dns_type || type == DnsType::kANY; }); diff --git a/discovery/mdns/impl/mdns_trackers_unittest.cc b/discovery/mdns/impl/mdns_trackers_unittest.cc index 8b49b2d42..394439234 100644 --- a/discovery/mdns/impl/mdns_trackers_unittest.cc +++ b/discovery/mdns/impl/mdns_trackers_unittest.cc @@ -27,7 +27,6 @@ constexpr Clock::duration kOneSecond = using testing::_; using testing::Args; using testing::DoAll; -using testing::Invoke; using testing::Return; using testing::StrictMock; using testing::WithArgs; diff --git a/discovery/mdns/public/mdns_reader.cc b/discovery/mdns/public/mdns_reader.cc index fad1c39ca..f194b7354 100644 --- a/discovery/mdns/public/mdns_reader.cc +++ b/discovery/mdns/public/mdns_reader.cc @@ -365,7 +365,8 @@ bool MdnsReader::Read(IPAddress::Version version, IPAddress* out) { : IPAddress::kV4Size; const uint8_t* const address_bytes = current(); if (Skip(ipaddress_size)) { - *out = IPAddress(version, address_bytes); + *out = IPAddress(version, + std::span(address_bytes, ipaddress_size)); return true; } return false; diff --git a/discovery/mdns/public/mdns_reader_unittest.cc b/discovery/mdns/public/mdns_reader_unittest.cc index 41a1c8c83..7532c83cf 100644 --- a/discovery/mdns/public/mdns_reader_unittest.cc +++ b/discovery/mdns/public/mdns_reader_unittest.cc @@ -279,9 +279,11 @@ TEST(MdnsReaderTest, ReadAAAARecordRdata) { 0x02, 0x02, 0xb3, 0xff, 0xfe, 0x1e, 0x83, 0x29, }; // clang-format on - TestReadEntrySucceeds(kAAAARecordRdata, sizeof(kAAAARecordRdata), - AAAARecordRdata(IPAddress(IPAddress::Version::kV6, - kAAAARecordRdata + 2))); + TestReadEntrySucceeds( + kAAAARecordRdata, sizeof(kAAAARecordRdata), + AAAARecordRdata(IPAddress( + IPAddress::Version::kV6, + std::span(kAAAARecordRdata).subspan(2, IPAddress::kV6Size)))); } TEST(MdnsReaderTest, ReadAAAARecordRdata_TooShort) { diff --git a/discovery/mdns/public/mdns_records.cc b/discovery/mdns/public/mdns_records.cc index 44b5c8088..bf67b9438 100644 --- a/discovery/mdns/public/mdns_records.cc +++ b/discovery/mdns/public/mdns_records.cc @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "discovery/mdns/public/mdns_writer.h" @@ -42,8 +43,8 @@ inline int CompareIgnoreCase(const std::string& x, const std::string& y) { template bool IsGreaterThan(const Rdata& lhs, const Rdata& rhs) { - const RDataType& lhs_cast = absl::get(lhs); - const RDataType& rhs_cast = absl::get(rhs); + const RDataType& lhs_cast = std::get(lhs); + const RDataType& rhs_cast = std::get(rhs); // The Extra 2 in length is from the record size that Write() prepends to the // result. @@ -583,20 +584,20 @@ bool MdnsRecord::IsValidConfig(const DomainName& name, // has been removed. return ttl.count() <= std::numeric_limits::max() && ((dns_type == DnsType::kSRV && - absl::holds_alternative(rdata)) || + std::holds_alternative(rdata)) || (dns_type == DnsType::kA && - absl::holds_alternative(rdata)) || + std::holds_alternative(rdata)) || (dns_type == DnsType::kAAAA && - absl::holds_alternative(rdata)) || + std::holds_alternative(rdata)) || (dns_type == DnsType::kPTR && - absl::holds_alternative(rdata)) || + std::holds_alternative(rdata)) || (dns_type == DnsType::kTXT && - absl::holds_alternative(rdata)) || + std::holds_alternative(rdata)) || (dns_type == DnsType::kNSEC && - absl::holds_alternative(rdata)) || + std::holds_alternative(rdata)) || (dns_type == DnsType::kOPT && - absl::holds_alternative(rdata)) || - absl::holds_alternative(rdata)); + std::holds_alternative(rdata)) || + std::holds_alternative(rdata)); } bool MdnsRecord::operator==(const MdnsRecord& rhs) const { @@ -655,7 +656,7 @@ bool MdnsRecord::IsReannouncementOf(const MdnsRecord& rhs) const { size_t MdnsRecord::MaxWireSize() const { auto wire_size_visitor = [](auto&& arg) { return arg.MaxWireSize(); }; // NAME size, 2-byte TYPE, 2-byte CLASS, 4-byte TTL, RDATA size - return name_.MaxWireSize() + absl::visit(wire_size_visitor, rdata_) + 8; + return name_.MaxWireSize() + std::visit(wire_size_visitor, rdata_) + 8; } #ifdef _DEBUG @@ -665,14 +666,14 @@ std::ostream& operator<<(std::ostream& os, const MdnsRecord& mdns_record) { if (mdns_record.dns_type_ == DnsType::kPTR) { const DomainName& target = - absl::get(mdns_record.rdata_).ptr_domain(); + std::get(mdns_record.rdata_).ptr_domain(); os << ", target: '" << target << "'"; } else if (mdns_record.dns_type_ == DnsType::kSRV) { const DomainName& target = - absl::get(mdns_record.rdata_).target(); + std::get(mdns_record.rdata_).target(); os << ", target: '" << target << "'"; } else if (mdns_record.dns_type_ == DnsType::kNSEC) { - const auto& nsec_rdata = absl::get(mdns_record.rdata_); + const auto& nsec_rdata = std::get(mdns_record.rdata_); std::vector types = nsec_rdata.types(); os << ", representing ["; if (!types.empty()) { diff --git a/discovery/mdns/public/mdns_records.h b/discovery/mdns/public/mdns_records.h index db5289880..f9c70c25d 100644 --- a/discovery/mdns/public/mdns_records.h +++ b/discovery/mdns/public/mdns_records.h @@ -13,13 +13,14 @@ #include #include #include +#include #include -#include "absl/types/variant.h" #include "discovery/mdns/public/mdns_constants.h" #include "platform/base/error.h" #include "platform/base/interface_info.h" #include "platform/base/ip_address.h" +#include "util/hashing.h" #include "util/osp_logging.h" #include "util/string_util.h" @@ -83,15 +84,6 @@ class DomainName { bool IsRoot() const { return labels_.empty(); } const std::vector& labels() const { return labels_; } - template - friend H AbslHashValue(H h, const DomainName& domain_name) { - std::vector labels_clone = domain_name.labels_; - for (auto& label : labels_clone) { - ::openscreen::string_util::AsciiStrToLower(label); - } - return H::combine(std::move(h), std::move(labels_clone)); - } - private: DomainName(std::vector labels, size_t max_wire_size); @@ -125,11 +117,6 @@ class RawRecordRdata { uint16_t size() const { return rdata_.size(); } const uint8_t* data() const { return rdata_.data(); } - template - friend H AbslHashValue(H h, const RawRecordRdata& rdata) { - return H::combine(std::move(h), rdata.rdata_); - } - private: std::vector rdata_; }; @@ -160,12 +147,6 @@ class SrvRecordRdata { uint16_t port() const { return port_; } const DomainName& target() const { return target_; } - template - friend H AbslHashValue(H h, const SrvRecordRdata& rdata) { - return H::combine(std::move(h), rdata.priority_, rdata.weight_, rdata.port_, - rdata.target_); - } - private: uint16_t priority_ = 0; uint16_t weight_ = 0; @@ -192,12 +173,6 @@ class ARecordRdata { const IPAddress& ipv4_address() const { return ipv4_address_; } NetworkInterfaceIndex interface_index() const { return interface_index_; } - template - friend H AbslHashValue(H h, const ARecordRdata& rdata) { - const auto& bytes = rdata.ipv4_address_.bytes(); - return H::combine_contiguous(std::move(h), bytes, 4); - } - private: IPAddress ipv4_address_{0, 0, 0, 0}; NetworkInterfaceIndex interface_index_; @@ -222,12 +197,6 @@ class AAAARecordRdata { const IPAddress& ipv6_address() const { return ipv6_address_; } NetworkInterfaceIndex interface_index() const { return interface_index_; } - template - friend H AbslHashValue(H h, const AAAARecordRdata& rdata) { - const auto& bytes = rdata.ipv6_address_.bytes(); - return H::combine_contiguous(std::move(h), bytes, 16); - } - private: IPAddress ipv6_address_{0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000}; @@ -251,11 +220,6 @@ class PtrRecordRdata { size_t MaxWireSize() const; const DomainName& ptr_domain() const { return ptr_domain_; } - template - friend H AbslHashValue(H h, const PtrRecordRdata& rdata) { - return H::combine(std::move(h), rdata.ptr_domain_); - } - private: DomainName ptr_domain_; }; @@ -285,11 +249,6 @@ class TxtRecordRdata { size_t MaxWireSize() const; const std::vector& texts() const { return texts_; } - template - friend H AbslHashValue(H h, const TxtRecordRdata& rdata) { - return H::combine(std::move(h), rdata.texts_); - } - private: TxtRecordRdata(std::vector texts, size_t max_wire_size); @@ -346,11 +305,6 @@ class NsecRecordRdata { const std::vector& types() const { return types_; } const std::vector& encoded_types() const { return encoded_types_; } - template - friend H AbslHashValue(H h, const NsecRecordRdata& rdata) { - return H::combine(std::move(h), rdata.types_, rdata.next_domain_name_); - } - private: std::vector encoded_types_; std::vector types_; @@ -371,11 +325,6 @@ class OptRecordRdata { bool operator==(const Option& rhs) const; bool operator!=(const Option& rhs) const; - template - friend H AbslHashValue(H h, const Option& option) { - return H::combine(std::move(h), option.code, option.length, option.data); - } - // Code assigned by the Expert Review process as defined by the DNSEXT // working group and the IESG, as specified in RFC6891 section 9.1. For // specific assignments, see: @@ -414,11 +363,6 @@ class OptRecordRdata { // Set of options stored in this OPT record. const std::vector