diff --git a/CMakeLists.txt b/CMakeLists.txt index 852da24..313a90b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,12 @@ set(modules VTKStreaming::WEBM ) +# VideoToolbox is an Apple-only system framework; only build the hardware +# encoder backend that uses it on macOS. Non-Apple builds are unaffected. +if(APPLE) + list(APPEND modules VTKStreaming::VTEncode) +endif() + include(VTKSDKPythonWheelHelper) vtksdk_build_modules(${SKBUILD_PROJECT_NAME} MODULES ${modules}) vtksdk_generate_package_init(${SKBUILD_PROJECT_NAME} MODULES ${modules}) diff --git a/README.md b/README.md index 368f7d8..b87206d 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,9 @@ This module provides classes to encode and stream frames from a VTK OpenGL render window using video codecs. It supports video encoding with VP9 (through [libvpx](https://chromium.googlesource.com/webm/libvpx/)) -and H.264/H.265 (through [NVENC](https://developer.nvidia.com/nvidia-video-codec-sdk/download)). +and H.264/H.265 through hardware encoders: [NVENC](https://developer.nvidia.com/nvidia-video-codec-sdk/download) +on NVIDIA GPUs, and [VideoToolbox](https://developer.apple.com/documentation/videotoolbox) +on macOS (Apple Silicon and Intel). ## Installation @@ -127,7 +129,7 @@ pip install wheelhouse/vtk_streaming-*.whl 1. [examples/simple_encoder_decoder.py](./examples/simple_encoder_decoder.py) - Live VP9 encode/decode round-trip with two render windows side by side. 2. [examples/resize_encoder_decoder.py](./examples/resize_encoder_decoder.py) - VP9 encode/decode round-trip that survives window resizes. -3. [examples/simple_nvenc_record.py](./examples/simple_nvenc_record.py) - Record a render window for later playback using NVENC. This needs `ffplay` to playback the .h264 file. +3. [examples/simple_hardware_encoder_record.py](./examples/simple_hardware_encoder_record.py) - Record a render window for later playback using whichever hardware H.264 encoder `vtkEncoderFactory` selects (Apple VideoToolbox on macOS, NVENC on NVIDIA GPUs). This needs `ffplay` to playback the .h264 file. ## Getting help diff --git a/Streaming/Encode/CMakeLists.txt b/Streaming/Encode/CMakeLists.txt index 5191d8c..2801654 100644 --- a/Streaming/Encode/CMakeLists.txt +++ b/Streaming/Encode/CMakeLists.txt @@ -1,5 +1,6 @@ -set(classes +set(classes vtkVideoEncoder + vtkEncoderFactory # TEMP: interim encoder selector; delete for VTK 9.7 (override attributes) ) vtk_module_add_module(VTKStreaming::Encode diff --git a/Streaming/Encode/vtkEncoderFactory.cxx b/Streaming/Encode/vtkEncoderFactory.cxx new file mode 100644 index 0000000..64006f4 --- /dev/null +++ b/Streaming/Encode/vtkEncoderFactory.cxx @@ -0,0 +1,411 @@ +/*========================================================================= + + Program: Visualization Toolkit + Module: vtkEncoderFactory.cxx + + Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen + All rights reserved. + See Copyright.txt or http://www.kitware.com/Copyright.htm for details. + + This software is distributed WITHOUT ANY WARRANTY; without even + the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + PURPOSE. See the above copyright notice for more information. + +=========================================================================*/ + +#include "vtkEncoderFactory.h" + +#include "vtkObjectFactory.h" +#include "vtkVideoEncoder.h" + +#include +#include +#include + +//------------------------------------------------------------------------------ +vtkStandardNewMacro(vtkEncoderFactory); + +namespace +{ +// An ordered list of preferences: each entry is a key with its ranked values, both kept +// in the order the user specified them (strongest first). Keys are lowercased; values keep +// their original case. +using PreferenceList = std::vector>>; + +//------------------------------------------------------------------------------ +// Process-wide registry and preferences, held as function-local statics so they are +// constructed on first use. A file-scope static would risk being constructed after a +// backend's static initializer runs (initialization order across shared libraries is +// unspecified), which is exactly when RegisterBackend is first called. +std::vector& Registry() +{ + static std::vector registry; + return registry; +} + +PreferenceList& Preferences() +{ + static PreferenceList preferences; + return preferences; +} + +//------------------------------------------------------------------------------ +std::string Trim(const std::string& s) +{ + const auto notSpace = [](unsigned char c) { return std::isspace(c) == 0; }; + auto begin = std::find_if(s.begin(), s.end(), notSpace); + auto end = std::find_if(s.rbegin(), s.rend(), notSpace).base(); + return (begin < end) ? std::string(begin, end) : std::string(); +} + +std::string ToLower(std::string s) +{ + std::transform(s.begin(), s.end(), s.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return s; +} + +bool CaseInsensitiveEqual(const std::string& a, const std::string& b) +{ + return ToLower(a) == ToLower(b); +} + +//------------------------------------------------------------------------------ +// Normalize a codec spelling for comparison: lowercase, drop a leading "vtkvc_", and +// remove '.' so that "H.265", "H265", "VTKVC_H265" all collapse to "h265". +std::string NormalizeCodec(const std::string& raw) +{ + std::string s = ToLower(Trim(raw)); + const std::string prefix = "vtkvc_"; + if (s.rfind(prefix, 0) == 0) + { + s = s.substr(prefix.size()); + } + s.erase(std::remove(s.begin(), s.end(), '.'), s.end()); + // A couple of friendly aliases. + if (s == "hevc") + { + return "h265"; + } + if (s == "avc") + { + return "h264"; + } + return s; +} + +// Map a codec name to the enum. Returns true and sets @a codec on success. Compares the +// normalized input against the normalized vtkVideoCodecTypeUtilities::ToString of every +// enum value, so adding a codec to Core needs no change here. +bool CodecFromString(const std::string& raw, VTKVideoCodecType& codec) +{ + const std::string want = NormalizeCodec(raw); + if (want.empty()) + { + return false; + } + for (int i = 0; i < VTKVideoCodecType::VTKVC_MaxNumberOfSupportedCodecs; ++i) + { + const auto candidate = static_cast(i); + if (NormalizeCodec(vtkVideoCodecTypeUtilities::ToString(candidate)) == want) + { + codec = candidate; + return true; + } + } + return false; +} + +//------------------------------------------------------------------------------ +bool SupportsCodec(const vtkEncoderFactory::BackendDescriptor& d, VTKVideoCodecType codec) +{ + return std::find(d.Codecs.begin(), d.Codecs.end(), codec) != d.Codecs.end(); +} + +bool BoolOf(const std::string& s) +{ + const std::string v = ToLower(Trim(s)); + return v == "true" || v == "1" || v == "yes" || v == "on"; +} + +//------------------------------------------------------------------------------ +// Rank of a backend against one preference key's ordered value list (lower is better): +// explicit match at value index j -> j +// attribute not declared by the backend -> WILDCARD (== values.size()): compatible, but +// ranked after any explicit match +// no listed value matches -> NO_MATCH (== values.size() + 1): worst +int RankForKey(const vtkEncoderFactory::BackendDescriptor& d, const std::string& key, + const std::vector& values) +{ + const int wildcard = static_cast(values.size()); + const int noMatch = wildcard + 1; + + if (key == "codec") + { + // Every backend declares its codecs. Unresolvable values (e.g. a stray "hardware=true" + // parsed as a codec) simply never match. + for (int j = 0; j < static_cast(values.size()); ++j) + { + VTKVideoCodecType codec; + if (CodecFromString(values[j], codec) && SupportsCodec(d, codec)) + { + return j; + } + } + return noMatch; + } + if (key == "hardware") + { + // Every backend declares Hardware, so there is no wildcard case here. + for (int j = 0; j < static_cast(values.size()); ++j) + { + if (BoolOf(values[j]) == d.Hardware) + { + return j; + } + } + return noMatch; + } + // Generic attribute (e.g. "platform"). Attribute keys are stored in their original case, + // so match them case-insensitively. + const std::string* declared = nullptr; + for (const auto& attribute : d.Attributes) + { + if (CaseInsensitiveEqual(attribute.first, key)) + { + declared = &attribute.second; + break; + } + } + if (declared == nullptr) + { + return wildcard; // backend does not constrain this attribute -> compatible with any value + } + for (int j = 0; j < static_cast(values.size()); ++j) + { + if (CaseInsensitiveEqual(*declared, values[j])) + { + return j; + } + } + return noMatch; +} + +// Per-key ranks for a backend, in preference order. Compared lexicographically (std::vector +// provides that via operator<): the first (strongest) key dominates, later keys break ties. +std::vector ScoreBackend( + const vtkEncoderFactory::BackendDescriptor& d, const PreferenceList& prefs) +{ + std::vector score; + score.reserve(prefs.size()); + for (const auto& pref : prefs) + { + score.push_back(RankForKey(d, pref.first, pref.second)); + } + return score; +} +} // anonymous namespace + +//------------------------------------------------------------------------------ +void vtkEncoderFactory::PrintSelf(ostream& os, vtkIndent indent) +{ + this->Superclass::PrintSelf(os, indent); + os << indent << "Registered backends: " << GetNumberOfRegisteredBackends() << '\n'; + for (const auto& d : Registry()) + { + os << indent << " " << d.SubclassName << " (hardware=" << d.Hardware << ")\n"; + } + os << indent << "Preferences:\n"; + for (const auto& pref : Preferences()) + { + os << indent << " " << pref.first << " ="; + for (const auto& value : pref.second) + { + os << ' ' << value; + } + os << '\n'; + } +} + +//------------------------------------------------------------------------------ +void vtkEncoderFactory::RegisterBackend(const BackendDescriptor& descriptor) +{ + auto& registry = Registry(); + for (const auto& existing : registry) + { + if (existing.SubclassName == descriptor.SubclassName) + { + return; // idempotent: guard against a module being loaded more than once + } + } + registry.push_back(descriptor); +} + +//------------------------------------------------------------------------------ +int vtkEncoderFactory::GetNumberOfRegisteredBackends() +{ + return static_cast(Registry().size()); +} + +//------------------------------------------------------------------------------ +bool vtkEncoderFactory::CheckAvailability(VTKVideoCodecType codec) +{ + for (const auto& d : Registry()) + { + if (!SupportsCodec(d, codec)) + { + continue; + } + if (d.Available == nullptr || d.Available()) + { + return true; + } + } + return false; +} + +//------------------------------------------------------------------------------ +void vtkEncoderFactory::SetPreferences(const char* preferences) +{ + auto& prefs = Preferences(); + prefs.clear(); + if (preferences == nullptr) + { + return; + } + // Format: "key1=v1a,v1b,...;key2=v2a,...;...". ';' separates keys (ordered strongest to + // weakest); ',' separates a key's values (also ordered strongest to weakest). + const std::string s(preferences); + std::string::size_type start = 0; + while (start <= s.size()) + { + const std::string::size_type semicolon = s.find(';', start); + const std::string group = + s.substr(start, semicolon == std::string::npos ? std::string::npos : semicolon - start); + const std::string::size_type eq = group.find('='); + if (eq != std::string::npos) + { + const std::string key = ToLower(Trim(group.substr(0, eq))); + if (!key.empty()) + { + std::vector values; + const std::string valueList = group.substr(eq + 1); + std::string::size_type vstart = 0; + while (vstart <= valueList.size()) + { + const std::string::size_type comma = valueList.find(',', vstart); + std::string value = Trim(valueList.substr( + vstart, comma == std::string::npos ? std::string::npos : comma - vstart)); + if (!value.empty()) // keep value case (e.g. Platform names) + { + values.push_back(value); + } + if (comma == std::string::npos) + { + break; + } + vstart = comma + 1; + } + // A repeated key replaces in place, keeping its first position. + bool replaced = false; + for (auto& existing : prefs) + { + if (existing.first == key) + { + existing.second = values; + replaced = true; + break; + } + } + if (!replaced) + { + prefs.emplace_back(key, values); + } + } + } + if (semicolon == std::string::npos) + { + break; + } + start = semicolon + 1; + } +} + +//------------------------------------------------------------------------------ +void vtkEncoderFactory::ClearPreferences() +{ + Preferences().clear(); +} + +//------------------------------------------------------------------------------ +vtkVideoEncoder* vtkEncoderFactory::CreateEncoder() +{ + const auto& prefs = Preferences(); + auto& registry = Registry(); + + // Rank every backend by preference first (no availability probe: ranking is pure attribute + // comparison), best (lowest score) first. + std::vector ranked; + ranked.reserve(registry.size()); + for (const auto& d : registry) + { + ranked.push_back(&d); + } + std::stable_sort(ranked.begin(), ranked.end(), + [&prefs](const BackendDescriptor* a, const BackendDescriptor* b) + { + const std::vector sa = ScoreBackend(*a, prefs); + const std::vector sb = ScoreBackend(*b, prefs); + if (sa != sb) + { + return sa < sb; // lexicographic: strongest key dominates, lower rank is better + } + if (a->Hardware != b->Hardware) + { + return a->Hardware; // deterministic tiebreak: prefer hardware + } + return a->SubclassName < b->SubclassName; // then alphabetical + }); + + // Availability is the only hard requirement: probe in rank order and take the first + // available backend. This runs each backend's probe at most once, and only until a winner + // is found (so e.g. NVENC's GL-window probe runs only if it out-ranks everything). + const BackendDescriptor* best = nullptr; + for (const auto* d : ranked) + { + if (d->Available == nullptr || d->Available()) + { + best = d; + break; + } + } + if (best == nullptr) + { + return nullptr; + } + + vtkVideoEncoder* encoder = best->Create(); + // Configure the encoder with the strongest preferred codec it supports. Without this the + // encoder keeps its default codec (VP9), which a backend picked for H.264/H.265 rejects. + if (encoder != nullptr) + { + for (const auto& pref : prefs) + { + if (pref.first != "codec") + { + continue; + } + for (const auto& value : pref.second) + { + VTKVideoCodecType codec; + if (CodecFromString(value, codec) && SupportsCodec(*best, codec)) + { + encoder->SetCodec(codec); + break; + } + } + break; + } + } + return encoder; +} diff --git a/Streaming/Encode/vtkEncoderFactory.h b/Streaming/Encode/vtkEncoderFactory.h new file mode 100644 index 0000000..abba7bb --- /dev/null +++ b/Streaming/Encode/vtkEncoderFactory.h @@ -0,0 +1,145 @@ +/*========================================================================= + + Program: Visualization Toolkit + Module: vtkEncoderFactory.h + + Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen + All rights reserved. + See Copyright.txt or http://www.kitware.com/Copyright.htm for details. + + This software is distributed WITHOUT ANY WARRANTY; without even + the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + PURPOSE. See the above copyright notice for more information. + +=========================================================================*/ +/** + * @class vtkEncoderFactory + * @brief TEMPORARY preference-driven selector for concrete vtkVideoEncoder backends + * + * vtkEncoderFactory picks the best available concrete encoder for a set of + * key=value preferences (Codec, Hardware, Platform). It exists only because this + * project targets VTK 9.6, which lacks the object-factory *override attribute* API + * (vtkOverrideAttribute + vtkObjectFactory::SetPreferences) that ships in VTK 9.7. + * + * @warning This class is temporary scaffolding. When the project upgrades to VTK + * 9.7, delete this class and its per-backend registrars, and instead have each + * concrete encoder return its attributes from CreateOverrideAttributes() so that + * vtkObjectFactory::SetPreferences() + vtkVideoEncoder::New() do the selection. The + * BackendDescriptor::Attributes maps deliberately use the 9.7 attribute vocabulary + * so that migration is mechanical. + * + * Backends register themselves (see RegisterBackend) from a static initializer in + * their own translation unit, so this class never has to #include a backend header + * (which would create a dependency cycle: backends DEPEND on this Encode module). + * A backend only becomes visible after its shared library is loaded; in Python this + * happens automatically when `vtk_streaming` is imported. + * + * @sa vtkVideoEncoder, vtkVideoCodecTypes + */ + +#ifndef vtkEncoderFactory_h +#define vtkEncoderFactory_h + +#include "vtkObject.h" + +#include "vtkStreamingEncodeModule.h" // for export macro +#include "vtkVideoCodecTypes.h" // for VTKVideoCodecType + +#include // for the descriptor attribute map +#include // for descriptor fields +#include // for the codec list + +class vtkVideoEncoder; + +class VTKSTREAMINGENCODE_EXPORT vtkEncoderFactory : public vtkObject +{ +public: + vtkTypeMacro(vtkEncoderFactory, vtkObject); + static vtkEncoderFactory* New(); + void PrintSelf(ostream& os, vtkIndent indent) override; + + ///@{ + /** + * Raw C function pointers so a descriptor stays trivially copyable and free of + * heap allocation. Create() returns a new encoder (VTK New() ownership); Available() + * probes whether a usable encoder is present on this machine. + */ + using CreateFn = vtkVideoEncoder* (*)(); + using AvailFn = bool (*)(); + ///@} + + /** + * Everything vtkEncoderFactory needs to know about one concrete backend. Registered + * by the backend itself so this module stays decoupled from concrete encoders. + */ + struct BackendDescriptor + { + std::string SubclassName; // e.g. "vtkVideoToolboxEncoder" + CreateFn Create = nullptr; // upcasts the concrete New() to the base + AvailFn Available = nullptr; // nullptr => always available (software) + bool Hardware = false; // hardware-accelerated backend? + std::vector Codecs; // codecs this backend supports + std::map Attributes; // 9.7 vocabulary: Platform, Hardware, ... + }; + + /** + * Register a concrete backend. Called by each backend's static initializer at library + * load time. Idempotent: a second registration with the same SubclassName is ignored, + * so re-importing a module does not double-register. + */ + static void RegisterBackend(const BackendDescriptor& descriptor); + + /** + * Returns true when any registered backend supports @a codec and is available on this + * machine. Ignores the current preferences. + */ + static bool CheckAvailability(VTKVideoCodecType codec); + + /** + * Set the selection preferences from a ranked, multi-value string in the VTK 9.7 override + * format: "key1=v1a,v1b,...;key2=v2a,...;...". ';' separates keys; ',' separates a key's + * values. Ordering is strength: keys are ranked strongest-to-weakest, and so are the values + * within each key. e.g. "Codec=H265,H264,VP9;Hardware=true;Platform=macOS". + * + * Selection (see CreateEncoder) is a lexicographic *soft* ranking: availability is the only + * hard requirement, every key is a preference, and a mismatch merely lowers rank. Put a key + * first to make it decisive (e.g. "Hardware=true;Codec=..." makes hardware dominate codec). + * + * Keys are case-insensitive; whitespace is trimmed; a repeated key keeps its first position + * with the last-specified values. Passing nullptr clears preferences. Recognized keys: Codec + * (vp9/av1/h264/h265, tolerant of "H.264"/"HEVC" spellings), Hardware (true/false), Platform + * (macOS/Linux/Windows). Note ';' is required between keys: a "key=value" token sitting after + * a comma is treated as a value of the preceding key (and dropped if it does not resolve). + */ + static void SetPreferences(const char* preferences); + + /** + * Clear all preferences (equivalent to SetPreferences(nullptr)). + */ + static void ClearPreferences(); + + /** + * Create the best available encoder for the current preferences, or nullptr when none + * matches. The caller owns the returned object (standard VTK New() semantics). + * + * Among available backends that satisfy every set preference (codec support, Platform, + * Hardware), hardware backends are preferred, with a deterministic alphabetical + * SubclassName tiebreak. + */ + static vtkVideoEncoder* CreateEncoder(); + + /** + * Number of registered backends. Mostly useful for tests/diagnostics. + */ + static int GetNumberOfRegisteredBackends(); + +protected: + vtkEncoderFactory() = default; + ~vtkEncoderFactory() override = default; + +private: + vtkEncoderFactory(const vtkEncoderFactory&) = delete; + void operator=(const vtkEncoderFactory&) = delete; +}; + +#endif // vtkEncoderFactory_h diff --git a/Streaming/NvEncode/vtkNvEncoderGL.cxx b/Streaming/NvEncode/vtkNvEncoderGL.cxx index 9b05864..473c9f0 100644 --- a/Streaming/NvEncode/vtkNvEncoderGL.cxx +++ b/Streaming/NvEncode/vtkNvEncoderGL.cxx @@ -77,6 +77,10 @@ vtkNvEncoderGL::vtkNvEncoderGL() { this->ResourceCallback = new vtkOpenGLResourceFreeCallback(this, &vtkNvEncoderGL::ReleaseGLResources); + // Default to a codec this backend supports (H.264, its internal default), so an instance + // created without an explicit SetCodec is valid. The base class default is VP9, which + // NVENC does not support. + this->Codec = VTKVideoCodecType::VTKVC_H264; } //------------------------------------------------------------------------------ @@ -467,3 +471,43 @@ void vtkNvEncoderGL::ReleaseGLResources(vtkWindow* window) internals.NvEncInputFrames.clear(); internals.NvEncInputResources.clear(); } + +//------------------------------------------------------------------------------ +// TEMP (VTK 9.6): self-register this backend with vtkEncoderFactory so it can be +// selected by preferences. Delete this block for VTK 9.7 and instead return these +// attributes from vtkNvEncoderGL::CreateOverrideAttributes(). +#include "vtkEncoderFactory.h" +namespace +{ +vtkVideoEncoder* CreateNvEncoderGL() +{ + return vtkNvEncoderGL::New(); +} + +// The NvEncode module is built for Linux and Windows, so the Platform attribute is +// resolved at compile time rather than hard-coded. +#if defined(_WIN32) +constexpr const char* kNvEncodePlatform = "Windows"; +#elif defined(__APPLE__) +constexpr const char* kNvEncodePlatform = "macOS"; +#else +constexpr const char* kNvEncodePlatform = "Linux"; +#endif + +struct vtkNvEncoderGLRegistrar +{ + vtkNvEncoderGLRegistrar() + { + vtkEncoderFactory::BackendDescriptor d; + d.SubclassName = "vtkNvEncoderGL"; + d.Create = &CreateNvEncoderGL; + d.Available = &vtkNvEncoderGL::CheckAvailability; + d.Hardware = true; + d.Codecs = { VTKVideoCodecType::VTKVC_H264, VTKVideoCodecType::VTKVC_H265 }; + d.Attributes = { { "Platform", kNvEncodePlatform }, { "Hardware", "true" } }; + vtkEncoderFactory::RegisterBackend(d); + } +}; +// Runs when the vtkStreamingNvEncode library is loaded (e.g. on `import vtk_streaming`). +const vtkNvEncoderGLRegistrar sNvEncoderGLRegistrar; +} // anonymous namespace diff --git a/Streaming/VTEncode/CMakeLists.txt b/Streaming/VTEncode/CMakeLists.txt new file mode 100644 index 0000000..fb5f2a6 --- /dev/null +++ b/Streaming/VTEncode/CMakeLists.txt @@ -0,0 +1,16 @@ +set(classes + vtkVideoToolboxEncoder +) + +vtk_module_add_module(VTKStreaming::VTEncode + CLASSES ${classes} +) + +if(APPLE) + vtk_module_link(VTKStreaming::VTEncode + PRIVATE + "-framework VideoToolbox" + "-framework CoreMedia" + "-framework CoreVideo" + "-framework CoreFoundation") +endif() diff --git a/Streaming/VTEncode/vtk.module b/Streaming/VTEncode/vtk.module new file mode 100644 index 0000000..08362dc --- /dev/null +++ b/Streaming/VTEncode/vtk.module @@ -0,0 +1,7 @@ +NAME + VTKStreaming::VTEncode +LIBRARY_NAME + vtkStreamingVTEncode +DEPENDS + VTKStreaming::Core + VTKStreaming::Encode diff --git a/Streaming/VTEncode/vtkVideoToolboxEncoder.cxx b/Streaming/VTEncode/vtkVideoToolboxEncoder.cxx new file mode 100644 index 0000000..dc1e173 --- /dev/null +++ b/Streaming/VTEncode/vtkVideoToolboxEncoder.cxx @@ -0,0 +1,818 @@ +/*========================================================================= + + Program: Visualization Toolkit + Module: vtkVideoToolboxEncoder.cxx + + Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen + All rights reserved. + See Copyright.txt or http://www.kitware.com/Copyright.htm for details. + + This software is distributed WITHOUT ANY WARRANTY; without even + the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + PURPOSE. See the above copyright notice for more information. + +=========================================================================*/ + +// This module is only added to the build on Apple platforms (see the top-level +// CMakeLists.txt `if(APPLE)` guard), so VideoToolbox is always available here. + +#include "vtkVideoToolboxEncoder.h" + +#include "vtkLogger.h" +#include "vtkMath.h" +#include "vtkObjectFactory.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace +{ +//------------------------------------------------------------------------------ +// Small RAII helper so CoreFoundation objects are released on scope exit. +template +struct CFHandle +{ + T Ref = nullptr; + CFHandle() = default; + explicit CFHandle(T ref) + : Ref(ref) + { + } + ~CFHandle() + { + if (this->Ref != nullptr) + { + CFRelease(this->Ref); + } + } + CFHandle(const CFHandle&) = delete; + CFHandle& operator=(const CFHandle&) = delete; + operator T() const { return this->Ref; } +}; + +//------------------------------------------------------------------------------ +// Set a numeric session property from an int/double/bool. +void SetSessionProperty(VTCompressionSessionRef session, CFStringRef key, int32_t value) +{ + CFHandle number(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value)); + VTSessionSetProperty(session, key, number); +} +void SetSessionProperty(VTCompressionSessionRef session, CFStringRef key, double value) +{ + CFHandle number(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &value)); + VTSessionSetProperty(session, key, number); +} +void SetSessionProperty(VTCompressionSessionRef session, CFStringRef key, bool value) +{ + VTSessionSetProperty(session, key, value ? kCFBooleanTrue : kCFBooleanFalse); +} + +// Same as above but reports the status, for properties an encoder may reject +// (e.g. ConstantBitRate, which not every hardware encoder supports). +OSStatus SetSessionPropertyChecked(VTCompressionSessionRef session, CFStringRef key, int32_t value) +{ + CFHandle number(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value)); + return VTSessionSetProperty(session, key, number); +} + +// Cap the instantaneous data rate to `bitsPerSecond` averaged over one second. +// Combined with an average bitrate this bounds VBR; used alone it approximates +// CBR on encoders that lack a true constant-bitrate mode. +void SetDataRateLimit(VTCompressionSessionRef session, unsigned int bitsPerSecond) +{ + if (bitsPerSecond == 0) + { + return; + } + const int64_t bytesPerSecond = bitsPerSecond / 8; + const double seconds = 1.0; + CFHandle bytes( + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &bytesPerSecond)); + CFHandle secs(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &seconds)); + const void* limitValues[] = { bytes.Ref, secs.Ref }; + CFHandle limits( + CFArrayCreate(kCFAllocatorDefault, limitValues, 2, &kCFTypeArrayCallBacks)); + VTSessionSetProperty(session, kVTCompressionPropertyKey_DataRateLimits, limits); +} + +//------------------------------------------------------------------------------ +// The VideoToolbox pixel format matching a VTKStreaming pixel format, or 0 when +// the format is unsupported. +OSType ToCVPixelFormat(VTKPixelFormatType format) +{ + switch (format) + { + case VTKPixelFormatType::VTKPF_NV12: + return kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; + case VTKPixelFormatType::VTKPF_IYUV: + return kCVPixelFormatType_420YpCbCr8Planar; + case VTKPixelFormatType::VTKPF_RGBA32: + return kCVPixelFormatType_32BGRA; + case VTKPixelFormatType::VTKPF_RGB24: + default: + return 0; + } +} +} // anonymous namespace + +//------------------------------------------------------------------------------ +struct vtkVideoToolboxEncoderInternals +{ + VTCompressionSessionRef Session = nullptr; + CMVideoCodecType CodecType = kCMVideoCodecType_H264; + // Filled synchronously by the output callback, drained by EncodeInternal. + std::vector> PendingPackets; + VTKVideoProcessingStatusType LastStatus = VTKVideoProcessingStatusType::VTKVPStatus_Success; + int64_t FrameCounter = 0; + // RFC 6381 codec string, parsed from the SPS on key frames and reused for the + // delta frames that follow (which carry no parameter sets). + std::string CodecName; + bool Initialized = false; + // timing for encode and copy ops. + std::chrono::high_resolution_clock::duration dtEncode{}, dtCopy{}; +}; + +vtkStandardNewMacro(vtkVideoToolboxEncoder); + +//------------------------------------------------------------------------------ +vtkVideoToolboxEncoder::vtkVideoToolboxEncoder() + : Internals(new vtkVideoToolboxEncoderInternals()) +{ + // Default to a codec this backend supports (matches Internals->CodecType), so an + // instance created without an explicit SetCodec is valid. The base class default is VP9, + // which VideoToolbox does not support. + this->Codec = VTKVideoCodecType::VTKVC_H264; +} + +//------------------------------------------------------------------------------ +vtkVideoToolboxEncoder::~vtkVideoToolboxEncoder() +{ + this->Shutdown(); +} + +//------------------------------------------------------------------------------ +void vtkVideoToolboxEncoder::PrintSelf(ostream& os, vtkIndent indent) +{ + this->Superclass::PrintSelf(os, indent); +} + +//------------------------------------------------------------------------------ +vtkIdType vtkVideoToolboxEncoder::GetLastEncodeTimeNS() const noexcept +{ + return std::chrono::duration_cast(this->Internals->dtEncode).count(); +} + +//------------------------------------------------------------------------------ +vtkIdType vtkVideoToolboxEncoder::GetLastScaleTimeNS() const noexcept +{ + return std::chrono::duration_cast(this->Internals->dtCopy).count(); +} + +//------------------------------------------------------------------------------ +bool vtkVideoToolboxEncoder::SupportsCodec(VTKVideoCodecType codec) const noexcept +{ + return codec == VTKVideoCodecType::VTKVC_H264 || codec == VTKVideoCodecType::VTKVC_H265; +} + +namespace +{ +// The 4-byte Annex B NAL unit start code. +const unsigned char kAnnexBStartCode[4] = { 0x00, 0x00, 0x00, 0x01 }; + +//------------------------------------------------------------------------------ +// The byte length of the AVCC/HVCC NAL length prefix VideoToolbox emits (usually +// 4). Reported by the parameter-set accessors on the format description. +int GetNalHeaderLength(CMFormatDescriptionRef format) +{ + if (format == nullptr) + { + return 4; + } + const CMVideoCodecType codec = CMFormatDescriptionGetMediaSubType(format); + int nalHeaderLength = 4; + size_t count = 0; + if (codec == kCMVideoCodecType_HEVC) + { + CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( + format, 0, nullptr, nullptr, &count, &nalHeaderLength); + } + else + { + CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + format, 0, nullptr, nullptr, &count, &nalHeaderLength); + } + return nalHeaderLength > 0 ? nalHeaderLength : 4; +} + +//------------------------------------------------------------------------------ +// Prepend each parameter set NAL (SPS/PPS, plus VPS for HEVC) to `out` as an +// Annex B unit (start code + NAL) so a key frame is self-describing. +void AppendParameterSetsAnnexB(CMSampleBufferRef sampleBuffer, std::vector& out) +{ + CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer); + if (format == nullptr) + { + return; + } + const CMVideoCodecType codec = CMFormatDescriptionGetMediaSubType(format); + size_t count = 0; + // Query the parameter set count (H.264 and HEVC have dedicated accessors). + if (codec == kCMVideoCodecType_HEVC) + { + if (CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( + format, 0, nullptr, nullptr, &count, nullptr) != noErr) + { + return; + } + } + else + { + if (CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + format, 0, nullptr, nullptr, &count, nullptr) != noErr) + { + return; + } + } + for (size_t i = 0; i < count; ++i) + { + const uint8_t* ps = nullptr; + size_t psSize = 0; + OSStatus status = (codec == kCMVideoCodecType_HEVC) + ? CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( + format, i, &ps, &psSize, nullptr, nullptr) + : CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + format, i, &ps, &psSize, nullptr, nullptr); + if (status != noErr || ps == nullptr) + { + continue; + } + out.insert(out.end(), kAnnexBStartCode, kAnnexBStartCode + 4); + out.insert(out.end(), ps, ps + psSize); + } +} + +//------------------------------------------------------------------------------ +// Convert a run of AVCC/HVCC length-prefixed NAL units into Annex B (each NAL +// prefixed by a start code) and append to `out`. This matches the NVENC backend +// and makes the raw elementary stream directly playable. +void AppendAnnexB( + const unsigned char* data, size_t length, int nalHeaderLength, std::vector& out) +{ + size_t offset = 0; + while (offset + static_cast(nalHeaderLength) <= length) + { + uint32_t nalLength = 0; + for (int i = 0; i < nalHeaderLength; ++i) + { + nalLength = (nalLength << 8) | data[offset + i]; + } + offset += nalHeaderLength; + if (nalLength == 0 || offset + nalLength > length) + { + break; + } + out.insert(out.end(), kAnnexBStartCode, kAnnexBStartCode + 4); + out.insert(out.end(), data + offset, data + offset + nalLength); + offset += nalLength; + } +} + +//------------------------------------------------------------------------------ +// Build a WebCodecs/RFC-6381 codec string from the sequence parameter set (SPS) +// carried by an Annex-B key frame bitstream. Returns an empty string when no SPS +// is present (e.g. a delta frame) or the codec is unsupported. Mirrors the +// NVENC backend's BuildCodecString so both encoders report identical strings. +std::string BuildCodecString(CMVideoCodecType codec, const unsigned char* data, std::size_t size) +{ + const bool isH264 = (codec == kCMVideoCodecType_H264); + const bool isHEVC = (codec == kCMVideoCodecType_HEVC); + if (data == nullptr || (!isH264 && !isHEVC)) + { + return {}; + } + // Walk the Annex-B NAL units looking for the SPS. A leading zero of a 4-byte + // start code is skipped naturally by advancing one byte at a time. + for (std::size_t i = 0; i + 3 < size; ++i) + { + if (data[i] != 0 || data[i + 1] != 0 || data[i + 2] != 1) + { + continue; + } + const std::size_t nal = i + 3; + if (isH264) + { + const int nalType = data[nal] & 0x1F; + if (nalType == 7 && nal + 3 < size) // Sequence parameter set. + { + // avc1.PPCCLL: profile_idc, constraint_set flags byte, level_idc. + char buf[16]; + std::snprintf( + buf, sizeof(buf), "avc1.%02X%02X%02X", data[nal + 1], data[nal + 2], data[nal + 3]); + return buf; + } + } + else // HEVC + { + const int nalType = (data[nal] >> 1) & 0x3F; + if (nalType == 33 && nal + 14 < size) // Sequence parameter set. + { + // Skip the 2-byte NAL header and the byte holding + // sps_video_parameter_set_id/sps_max_sub_layers_minus1/ + // sps_temporal_id_nesting_flag to reach profile_tier_level(). + const unsigned char* ptl = &data[nal + 3]; + const int profileSpace = (ptl[0] >> 6) & 0x03; + const int tierFlag = (ptl[0] >> 5) & 0x01; + const int profileIdc = ptl[0] & 0x1F; + const std::uint32_t compat = (std::uint32_t(ptl[1]) << 24) | (std::uint32_t(ptl[2]) << 16) | + (std::uint32_t(ptl[3]) << 8) | std::uint32_t(ptl[4]); + const int levelIdc = ptl[11]; + std::ostringstream oss; + oss << "hvc1."; + if (profileSpace > 0) + { + oss << static_cast('A' + profileSpace - 1); + } + oss << profileIdc << '.' << std::uppercase << std::hex << compat << std::dec << '.' + << (tierFlag ? 'H' : 'L') << levelIdc; + // Six constraint-indicator-flag bytes, trailing zero bytes trimmed. + int last = 5; + while (last >= 0 && ptl[5 + last] == 0) + { + --last; + } + for (int c = 0; c <= last; ++c) + { + char cbuf[8]; + std::snprintf(cbuf, sizeof(cbuf), ".%02X", ptl[5 + c]); + oss << cbuf; + } + return oss.str(); + } + } + } + return {}; +} + +//------------------------------------------------------------------------------ +// Whether a sample buffer holds a sync sample (key frame). Absence of the +// attachment array, or NotSync being false/absent, means it is a key frame. +bool IsKeyFrame(CMSampleBufferRef sampleBuffer) +{ + CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, false); + if (attachments == nullptr || CFArrayGetCount(attachments) == 0) + { + return true; + } + CFDictionaryRef dict = static_cast(CFArrayGetValueAtIndex(attachments, 0)); + CFBooleanRef notSync = nullptr; + if (CFDictionaryGetValueIfPresent( + dict, kCMSampleAttachmentKey_NotSync, reinterpret_cast(¬Sync))) + { + return !CFBooleanGetValue(notSync); + } + return true; +} + +//------------------------------------------------------------------------------ +// VideoToolbox output callback. Runs synchronously with respect to +// EncodeInternal thanks to VTCompressionSessionCompleteFrames. +void CompressionOutputCallback(void* outputCallbackRefCon, void* /*sourceFrameRefCon*/, + OSStatus status, VTEncodeInfoFlags infoFlags, CMSampleBufferRef sampleBuffer) +{ + auto* internals = static_cast(outputCallbackRefCon); + if (status != noErr) + { + vtkLogF(ERROR, "VideoToolbox encode failed with OSStatus %d", static_cast(status)); + internals->LastStatus = VTKVideoProcessingStatusType::VTKVPStatus_UnknownError; + return; + } + if ((infoFlags & kVTEncodeInfo_FrameDropped) != 0 || sampleBuffer == nullptr || + !CMSampleBufferDataIsReady(sampleBuffer)) + { + return; + } + + CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sampleBuffer); + if (block == nullptr) + { + return; + } + size_t totalLength = 0; + char* dataPointer = nullptr; + if (CMBlockBufferGetDataPointer(block, 0, nullptr, &totalLength, &dataPointer) != noErr || + dataPointer == nullptr) + { + return; + } + + const bool keyFrame = IsKeyFrame(sampleBuffer); + CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer); + const int nalHeaderLength = GetNalHeaderLength(format); + std::vector payload; + payload.reserve(totalLength + 256); + if (keyFrame) + { + // Inline the parameter sets ahead of the key frame (Annex B). + AppendParameterSetsAnnexB(sampleBuffer, payload); + } + // Rewrite the AVCC/HVCC length prefixes as Annex B start codes so the stream + // matches the NVENC backend and plays directly. + AppendAnnexB( + reinterpret_cast(dataPointer), totalLength, nalHeaderLength, payload); + + CMTime pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); + CMVideoDimensions dims = { 0, 0 }; + if (format != nullptr) + { + dims = CMVideoFormatDescriptionGetDimensions(format); + } + + // Key frames carry the SPS; parse the RFC 6381 codec string from it and reuse + // it for the delta frames that follow. + if (keyFrame) + { + std::string codecName = BuildCodecString(internals->CodecType, payload.data(), payload.size()); + if (!codecName.empty()) + { + internals->CodecName = codecName; + } + } + + auto chunk = vtk::TakeSmartPointer(vtkCompressedVideoPacket::New()); + chunk->SetIsKeyFrame(keyFrame); + chunk->SetCodecLongName(internals->CodecName.c_str()); + chunk->SetDisplayWidth(dims.width); + chunk->SetDisplayHeight(dims.height); + chunk->SetCodedWidth(dims.width); + chunk->SetCodedHeight(dims.height); + chunk->SetPresentationTS(CMTIME_IS_VALID(pts) ? pts.value : internals->FrameCounter); + chunk->CopyData(payload.data(), static_cast(payload.size())); + vtkLogF(TRACE, "%lld|%s|%d bytes", chunk->GetPresentationTS(), + chunk->GetIsKeyFrame() ? "key" : "delta", chunk->GetSize()); + internals->PendingPackets.emplace_back(chunk); +} +} // anonymous namespace + +//------------------------------------------------------------------------------ +bool vtkVideoToolboxEncoder::InitializeInternal() +{ + auto& internals = (*this->Internals); + internals.FrameCounter = 0; + internals.CodecName.clear(); + internals.LastStatus = VTKVideoProcessingStatusType::VTKVPStatus_Success; + switch (this->Codec) + { + case VTKVideoCodecType::VTKVC_H264: + internals.CodecType = kCMVideoCodecType_H264; + break; + case VTKVideoCodecType::VTKVC_H265: + internals.CodecType = kCMVideoCodecType_HEVC; + break; + default: + vtkLogF(ERROR, "VideoToolbox encoder only supports H.264 and H.265. Got codec %s.", + vtkVideoCodecTypeUtilities::ToString(this->Codec)); + return false; + } + return true; +} + +//------------------------------------------------------------------------------ +void vtkVideoToolboxEncoder::ShutdownInternal() +{ + auto& internals = (*this->Internals); + internals.PendingPackets.clear(); + internals.Initialized = false; +} + +//------------------------------------------------------------------------------ +bool vtkVideoToolboxEncoder::SetupEncoderFrame(int width, int height) +{ + vtkLogF(TRACE, "%s, %dx%d", __func__, width, height); + auto& internals = (*this->Internals); + + const OSType cvFormat = ToCVPixelFormat(this->InputPixelFormat); + if (cvFormat == 0) + { + vtkLogF(ERROR, "Unsupported input pixel format %s for VideoToolbox.", + vtkPixelFormatTypeUtilities::ToString(this->InputPixelFormat)); + return false; + } + + // Ask the compression session to hand us IOSurface-backed pixel buffers of + // the source format, which is what the hardware encoder path expects. + const void* keys[] = { kCVPixelBufferPixelFormatTypeKey, kCVPixelBufferWidthKey, + kCVPixelBufferHeightKey, kCVPixelBufferIOSurfacePropertiesKey }; + CFHandle formatValue( + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &cvFormat)); + CFHandle widthValue(CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &width)); + CFHandle heightValue(CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &height)); + CFHandle ioSurfaceProps(CFDictionaryCreate(kCFAllocatorDefault, nullptr, nullptr, + 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks)); + const void* values[] = { formatValue.Ref, widthValue.Ref, heightValue.Ref, ioSurfaceProps.Ref }; + CFHandle sourceAttrs(CFDictionaryCreate(kCFAllocatorDefault, keys, values, 4, + &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks)); + + // Request a hardware-accelerated encoder. + const void* encKeys[] = { kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder }; + const void* encValues[] = { kCFBooleanTrue }; + CFHandle encoderSpec(CFDictionaryCreate(kCFAllocatorDefault, encKeys, encValues, + 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks)); + + OSStatus status = + VTCompressionSessionCreate(kCFAllocatorDefault, width, height, internals.CodecType, encoderSpec, + sourceAttrs, nullptr, &CompressionOutputCallback, &internals, &internals.Session); + if (status != noErr || internals.Session == nullptr) + { + vtkLogF(ERROR, "VTCompressionSessionCreate failed with OSStatus %d", static_cast(status)); + return false; + } + + VTCompressionSessionRef session = internals.Session; + SetSessionProperty(session, kVTCompressionPropertyKey_RealTime, this->LowDelayMode); + SetSessionProperty( + session, kVTCompressionPropertyKey_AllowFrameReordering, this->MaximumBFrames > 0); + SetSessionProperty( + session, kVTCompressionPropertyKey_MaxKeyFrameInterval, this->GroupOfPicturesSize); + if (this->TimeBaseEnd > 0) + { + SetSessionProperty(session, kVTCompressionPropertyKey_ExpectedFrameRate, + static_cast(this->TimeBaseEnd)); + } + // Rate control. VideoToolbox is bitrate-driven and, unlike NVENC/libvpx, has + // no public constant-QP mode, so the QP-based modes are approximated. + switch (this->BitRateControlMode) + { + case vtkVideoEncoder::BRCType::CBR: + { + // True CBR needs kVTCompressionPropertyKey_ConstantBitRate (macOS 13+), + // and not every hardware encoder honors it; fall back to an average + // bitrate hard-capped by a one-second data-rate limit. + bool cbrSet = false; + if (__builtin_available(macOS 13.0, *)) + { + cbrSet = this->BitRate > 0 && + SetSessionPropertyChecked(session, kVTCompressionPropertyKey_ConstantBitRate, + static_cast(this->BitRate)) == noErr; + } + if (!cbrSet) + { + if (this->BitRate > 0) + { + SetSessionProperty( + session, kVTCompressionPropertyKey_AverageBitRate, static_cast(this->BitRate)); + } + SetDataRateLimit(session, this->BitRate); + } + break; + } + case vtkVideoEncoder::BRCType::CQP: + case vtkVideoEncoder::BRCType::QP: + { + // No fixed-QP mode exists; map the base-class quantizer to a quality hint + // (higher QP -> lower quality) and, where available, bound the frame QP. + vtkLog(TRACE, + "VideoToolbox has no constant-QP mode; approximating with a quality hint " + "derived from QuantizationParameter."); + const double quality = + 1.0 - (vtkMath::ClampValue(this->QuantizationParameter, 1u, 63u) / 63.0); + SetSessionProperty(session, kVTCompressionPropertyKey_Quality, quality); + if (__builtin_available(macOS 13.0, *)) + { + SetSessionPropertyChecked(session, kVTCompressionPropertyKey_MinAllowedFrameQP, + static_cast(this->MinQuantizationParameter)); + SetSessionPropertyChecked(session, kVTCompressionPropertyKey_MaxAllowedFrameQP, + static_cast(this->MaxQuantizationParameter)); + } + break; + } + case vtkVideoEncoder::BRCType::VBR: + default: + if (this->BitRate > 0) + { + SetSessionProperty( + session, kVTCompressionPropertyKey_AverageBitRate, static_cast(this->BitRate)); + } + // A higher ceiling than the average allows the rate to fluctuate (VBR). + if (this->MaxBitRate > this->BitRate) + { + SetDataRateLimit(session, this->MaxBitRate); + } + break; + } + + VTCompressionSessionPrepareToEncodeFrames(session); + internals.Initialized = true; + return true; +} + +//------------------------------------------------------------------------------ +void vtkVideoToolboxEncoder::TearDownEncoderFrame() +{ + auto& internals = (*this->Internals); + if (internals.Session != nullptr) + { + VTCompressionSessionCompleteFrames(internals.Session, kCMTimeInvalid); + VTCompressionSessionInvalidate(internals.Session); + CFRelease(internals.Session); + internals.Session = nullptr; + } +} + +//------------------------------------------------------------------------------ +VTKVideoEncoderResultType vtkVideoToolboxEncoder::EncodeInternal( + vtkSmartPointer frame) +{ + auto& internals = (*this->Internals); + if (frame == nullptr || internals.Session == nullptr) + { + return {}; + } + + // Obtain an IOSurface-backed pixel buffer from the session pool. + CVPixelBufferPoolRef pool = VTCompressionSessionGetPixelBufferPool(internals.Session); + if (pool == nullptr) + { + vtkLog(ERROR, "VideoToolbox pixel buffer pool is unavailable."); + return { VTKVideoProcessingStatusType::VTKVPStatus_UnknownError, {} }; + } + CVPixelBufferRef pixelBuffer = nullptr; + if (CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &pixelBuffer) != + kCVReturnSuccess || + pixelBuffer == nullptr) + { + vtkLog(ERROR, "Failed to allocate a VideoToolbox pixel buffer."); + return { VTKVideoProcessingStatusType::VTKVPStatus_UnknownError, {} }; + } + + // Copy the (CPU-resident) frame pixels into the pixel buffer plane by plane. + auto tc1 = std::chrono::high_resolution_clock::now(); + unsigned char* src = nullptr; + const unsigned int srcSize = frame->GetData(src); + const int* strides = frame->GetStrides(); + const VTKPixelFormatType format = frame->GetPixelFormat(); + const int height = frame->GetHeight(); + + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + if (format == VTKPixelFormatType::VTKPF_RGBA32) + { + // Single packed plane; swizzle RGBA -> BGRA to match kCVPixelFormatType_32BGRA. + auto* dst = static_cast(CVPixelBufferGetBaseAddress(pixelBuffer)); + const size_t dstStride = CVPixelBufferGetBytesPerRow(pixelBuffer); + const int srcStride = strides[0]; + const int copyBytes = std::min(srcStride, static_cast(dstStride)); + for (int row = 0; row < height; ++row) + { + const unsigned char* s = src + (static_cast(row) * srcStride); + unsigned char* d = dst + (static_cast(row) * dstStride); + for (int x = 0; x + 3 < copyBytes; x += 4) + { + d[x + 0] = s[x + 2]; // B + d[x + 1] = s[x + 1]; // G + d[x + 2] = s[x + 0]; // R + d[x + 3] = s[x + 3]; // A + } + } + } + else + { + // Planar YUV (NV12: 2 planes, IYUV: 3 planes). + const size_t planeCount = CVPixelBufferGetPlaneCount(pixelBuffer); + for (size_t plane = 0; plane < planeCount; ++plane) + { + auto* dst = + static_cast(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, plane)); + const size_t dstStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, plane); + const size_t planeHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, plane); + const int srcStride = strides[plane]; + const unsigned char* s = src + frame->GetPlanePointerIdx(static_cast(plane)); + const int copyBytes = std::min(srcStride, static_cast(dstStride)); + for (size_t row = 0; row < planeHeight; ++row) + { + const unsigned char* srcRow = s + (row * srcStride); + std::copy(srcRow, srcRow + copyBytes, dst + (row * dstStride)); + } + } + } + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + delete[] src; + (void)srcSize; + auto tc2 = std::chrono::high_resolution_clock::now(); + internals.dtCopy = tc2 - tc1; + + // Optionally force a key frame. + CFHandle frameProps; + if (this->ForceIFrame || this->KeyFramesOnly) + { + const void* keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame }; + const void* values[] = { kCFBooleanTrue }; + frameProps.Ref = CFDictionaryCreate(kCFAllocatorDefault, keys, values, 1, + &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + } + + const int32_t timescale = this->TimeBaseEnd > 0 ? this->TimeBaseEnd : 30; + const CMTime pts = CMTimeMake(internals.FrameCounter++, timescale); + const CMTime duration = CMTimeMake(1, timescale); + + auto te1 = std::chrono::high_resolution_clock::now(); + internals.PendingPackets.clear(); + internals.LastStatus = VTKVideoProcessingStatusType::VTKVPStatus_Success; + VTEncodeInfoFlags infoFlags = 0; + OSStatus status = VTCompressionSessionEncodeFrame( + internals.Session, pixelBuffer, pts, duration, frameProps, nullptr, &infoFlags); + CVPixelBufferRelease(pixelBuffer); + if (status != noErr) + { + vtkLogF( + ERROR, "VTCompressionSessionEncodeFrame failed with OSStatus %d", static_cast(status)); + return { VTKVideoProcessingStatusType::VTKVPStatus_UnknownError, {} }; + } + // Force synchronous delivery so packets are available to return now. + VTCompressionSessionCompleteFrames(internals.Session, kCMTimeInvalid); + auto te2 = std::chrono::high_resolution_clock::now(); + internals.dtEncode = te2 - te1; + + VTKVideoEncoderResultType result; + result.first = internals.LastStatus; + result.second = internals.PendingPackets; + internals.PendingPackets.clear(); + return result; +} + +//------------------------------------------------------------------------------ +VTKVideoEncoderResultType vtkVideoToolboxEncoder::SendEOS() +{ + auto& internals = (*this->Internals); + if (internals.Session == nullptr) + { + return {}; + } + internals.PendingPackets.clear(); + internals.LastStatus = VTKVideoProcessingStatusType::VTKVPStatus_Success; + VTCompressionSessionCompleteFrames(internals.Session, kCMTimeInvalid); + VTKVideoEncoderResultType result; + result.first = internals.LastStatus; + result.second = internals.PendingPackets; + internals.PendingPackets.clear(); + return result; +} + +//------------------------------------------------------------------------------ +bool vtkVideoToolboxEncoder::CheckAvailability() noexcept +{ + // Try to create a small hardware-accelerated H.264 session; success means a + // usable VideoToolbox encoder is present. + const void* encKeys[] = { kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder }; + const void* encValues[] = { kCFBooleanTrue }; + CFHandle encoderSpec(CFDictionaryCreate(kCFAllocatorDefault, encKeys, encValues, + 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks)); + VTCompressionSessionRef session = nullptr; + OSStatus status = VTCompressionSessionCreate(kCFAllocatorDefault, 640, 480, + kCMVideoCodecType_H264, encoderSpec, nullptr, nullptr, nullptr, nullptr, &session); + if (status == noErr && session != nullptr) + { + VTCompressionSessionInvalidate(session); + CFRelease(session); + return true; + } + return false; +} + +//------------------------------------------------------------------------------ +// TEMP (VTK 9.6): self-register this backend with vtkEncoderFactory so it can be +// selected by preferences. Delete this block for VTK 9.7 and instead return these +// attributes from vtkVideoToolboxEncoder::CreateOverrideAttributes(). +#include "vtkEncoderFactory.h" +namespace +{ +vtkVideoEncoder* CreateVideoToolboxEncoder() +{ + return vtkVideoToolboxEncoder::New(); +} + +struct vtkVideoToolboxEncoderRegistrar +{ + vtkVideoToolboxEncoderRegistrar() + { + vtkEncoderFactory::BackendDescriptor d; + d.SubclassName = "vtkVideoToolboxEncoder"; + d.Create = &CreateVideoToolboxEncoder; + d.Available = &vtkVideoToolboxEncoder::CheckAvailability; + d.Hardware = true; + d.Codecs = { VTKVideoCodecType::VTKVC_H264, VTKVideoCodecType::VTKVC_H265 }; + d.Attributes = { { "Platform", "macOS" }, { "Hardware", "true" } }; + vtkEncoderFactory::RegisterBackend(d); + } +}; +// Runs when the vtkStreamingVTEncode library is loaded (e.g. on `import vtk_streaming`). +const vtkVideoToolboxEncoderRegistrar sVideoToolboxEncoderRegistrar; +} // anonymous namespace diff --git a/Streaming/VTEncode/vtkVideoToolboxEncoder.h b/Streaming/VTEncode/vtkVideoToolboxEncoder.h new file mode 100644 index 0000000..19bb12c --- /dev/null +++ b/Streaming/VTEncode/vtkVideoToolboxEncoder.h @@ -0,0 +1,83 @@ +/*========================================================================= + + Program: Visualization Toolkit + Module: vtkVideoToolboxEncoder.h + + Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen + All rights reserved. + See Copyright.txt or http://www.kitware.com/Copyright.htm for details. + + This software is distributed WITHOUT ANY WARRANTY; without even + the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + PURPOSE. See the above copyright notice for more information. + +=========================================================================*/ +/** + * @class vtkVideoToolboxEncoder + * @brief hardware-accelerated H.264/H.265 encoder for Apple platforms + * + * vtkVideoToolboxEncoder encodes raw video frames on Apple Silicon (and Intel + * Macs) using the system VideoToolbox framework's `VTCompressionSession`, which + * drives the dedicated hardware media engine. It supports the H.264 and H.265 + * (HEVC) codecs. + * + * The incoming vtkRawVideoFrame pixels are uploaded (CPU copy) into an + * IOSurface-backed CVPixelBuffer obtained from the compression session's own + * pool, then submitted to the hardware encoder. Emitted packets carry an Annex B + * elementary stream (NAL units prefixed with `00 00 00 01` start codes, with the + * codec parameter sets prepended on key frames), matching the NVENC backend and + * directly playable by tools such as ffplay. + * + * @sa vtkVideoEncoder, vtkNvEncoderGL, vtkVpxEncoder + */ + +#ifndef vtkVideoToolboxEncoder_h +#define vtkVideoToolboxEncoder_h + +#include "vtkVideoEncoder.h" + +#include "vtkStreamingVTEncodeModule.h" // for export macro + +#include // for ivar + +struct vtkVideoToolboxEncoderInternals; + +class VTKSTREAMINGVTENCODE_EXPORT vtkVideoToolboxEncoder : public vtkVideoEncoder +{ +public: + vtkTypeMacro(vtkVideoToolboxEncoder, vtkVideoEncoder); + void PrintSelf(ostream& os, vtkIndent indent) override; + static vtkVideoToolboxEncoder* New(); + + bool IsHardwareAccelerated() const noexcept override { return true; } + vtkIdType GetLastEncodeTimeNS() const noexcept override; + vtkIdType GetLastScaleTimeNS() const noexcept override; + bool SupportsCodec(VTKVideoCodecType codec) const noexcept override; + + /** + * Returns true when a hardware-accelerated VideoToolbox encoder for H.264 is + * available on this machine. Mirrors vtkNvEncoderGL::CheckAvailability. + */ + static bool CheckAvailability() noexcept; + +protected: + vtkVideoToolboxEncoder(); + ~vtkVideoToolboxEncoder() override; + + bool InitializeInternal() override; + void ShutdownInternal() override; + + bool SetupEncoderFrame(int width, int height) override; + void TearDownEncoderFrame() override; + + VTKVideoEncoderResultType EncodeInternal(vtkSmartPointer frame) override; + VTKVideoEncoderResultType SendEOS() override; + +private: + vtkVideoToolboxEncoder(const vtkVideoToolboxEncoder&) = delete; + void operator=(const vtkVideoToolboxEncoder&) = delete; + + std::unique_ptr Internals; +}; + +#endif // vtkVideoToolboxEncoder_h diff --git a/Streaming/VpxEncode/vtkVpxEncoder.cxx b/Streaming/VpxEncode/vtkVpxEncoder.cxx index 8edc0a2..77b6a22 100644 --- a/Streaming/VpxEncode/vtkVpxEncoder.cxx +++ b/Streaming/VpxEncode/vtkVpxEncoder.cxx @@ -60,6 +60,10 @@ vtkStandardNewMacro(vtkVpxEncoder); vtkVpxEncoder::vtkVpxEncoder() : Internals(new vtkInternals()) { + // Default to a codec this backend supports (VP9, matching Internals->Interface). This + // happens to equal the base class default, but set it explicitly so the default tracks + // the internals rather than the base. + this->Codec = VTKVideoCodecType::VTKVC_VP9; } //------------------------------------------------------------------------------ @@ -374,3 +378,34 @@ VTKVideoEncoderResultType vtkVpxEncoder::SendEOS() { return this->EncodeInternal(nullptr); } + +//------------------------------------------------------------------------------ +// TEMP (VTK 9.6): self-register this backend with vtkEncoderFactory so it can be +// selected by preferences. Delete this block for VTK 9.7 and instead return these +// attributes from vtkVpxEncoder::CreateOverrideAttributes(). +#include "vtkEncoderFactory.h" +namespace +{ +vtkVideoEncoder* CreateVpxEncoder() +{ + return vtkVpxEncoder::New(); +} + +struct vtkVpxEncoderRegistrar +{ + vtkVpxEncoderRegistrar() + { + vtkEncoderFactory::BackendDescriptor d; + d.SubclassName = "vtkVpxEncoder"; + d.Create = &CreateVpxEncoder; + d.Available = nullptr; // software encoder: always available + d.Hardware = false; + d.Codecs = { VTKVideoCodecType::VTKVC_VP9 }; + // Cross-platform software backend: no Platform attribute (wildcard). + d.Attributes = { { "Hardware", "false" } }; + vtkEncoderFactory::RegisterBackend(d); + } +}; +// Runs when the vtkStreamingVpxEncode library is loaded (e.g. on `import vtk_streaming`). +const vtkVpxEncoderRegistrar sVpxEncoderRegistrar; +} // anonymous namespace diff --git a/examples/simple_nvenc_record.py b/examples/simple_hardware_encoder_record.py similarity index 71% rename from examples/simple_nvenc_record.py rename to examples/simple_hardware_encoder_record.py index aa8fb81..5e69f93 100644 --- a/examples/simple_nvenc_record.py +++ b/examples/simple_hardware_encoder_record.py @@ -1,8 +1,12 @@ -"""Live VP9 encode/decode round-trip with two render windows side by side. +"""Record a render window to an H.264 file using a hardware video encoder. -This uses an NVENC encoder to record the render window to a .h264 file. +The concrete backend is chosen by vtkEncoderFactory from a preference string rather than +hard-coded, so the same example drives Apple's VideoToolbox on macOS and NVENC on NVIDIA +GPUs (whichever hardware H.264 encoder is available on this machine). -Tip: After running this, quit it and playback the recording with `ffplay recording.h264` +Tip: After running this, quit it and play back the recording with: + ffplay recording.h264 +The encoder emits an H.264 Annex B stream. """ from datetime import datetime @@ -28,13 +32,19 @@ vtkCompressedVideoPacket, vtkRawVideoFrame, ) -from vtk_streaming.vtkStreamingEncode import vtkVideoEncoder +from vtk_streaming.vtkStreamingEncode import vtkEncoderFactory, vtkVideoEncoder from vtk_streaming.vtkStreamingOpenGL2 import vtkOpenGLVideoFrame -from vtk_streaming.vtkStreamingNvEncode import vtkNvEncoderGL + +# Ask the encoder factory for a hardware H.264 encoder. On macOS this resolves to the +# VideoToolbox backend, on NVIDIA GPUs to NVENC; the factory keeps the backend choice out of +# this script. (This interim helper stands in for VTK 9.7's vtkObjectFactory::SetPreferences +# + New().) +if not vtkEncoderFactory.CheckAvailability(VTKVC_H264): + raise SystemExit("No hardware H.264 encoder is available on this machine.") width, height = 640, 480 # codecs prefer sizes aligned to %4 or %8 -# Left window: the scene that gets encoded. +# The scene that gets encoded. cylinder = vtkCylinderSource() mapper = vtkPolyDataMapper() mapper.SetInputConnection(cylinder.GetOutputPort()) @@ -65,7 +75,7 @@ def current_time_text() -> str: renderer.AddViewProp(frame_text) scene_window = vtkRenderWindow() -scene_window.SetWindowName("Input scene (VP9 encode)") +scene_window.SetWindowName("Input scene (hardware H.264 encode)") scene_window.AddRenderer(renderer) scene_window.SetSize(width, height) scene_window.SetPosition(50, 50) @@ -75,7 +85,11 @@ def current_time_text() -> str: interactor.Initialize() scene_window.Render() -encoder = vtkNvEncoderGL() +vtkEncoderFactory.SetPreferences("Codec=H264;Hardware=true") +encoder = vtkEncoderFactory.CreateEncoder() +if encoder is None: + raise SystemExit("The encoder factory could not create a hardware H.264 encoder.") +print(f"Selected encoder backend: {encoder.GetClassName()}") encoder.SetGraphicsContext(scene_window) encoder.SetCodec(VTKVC_H264) encoder.SetWidth(width) @@ -111,8 +125,6 @@ def update_time_text(_window: vtkRenderWindow, _event: int): def encode_frame(window: vtkRenderWindow, _event: int): picture.Capture(window) encoder.Encode(picture) # fires EncodedVideoChunkEvent per packet - # Decoding rendered into the other window; hand the context back. - window.MakeCurrent() # StartEvent fires at the start of every vtkRenderWindow::Render, so the @@ -130,8 +142,8 @@ def spin(_interactor: vtkRenderWindowInteractor, _event: int): interactor.AddObserver(vtkCommand.TimerEvent, spin) interactor.CreateRepeatingTimer(33) -print("Interact with the left window; the right window shows the decoded stream.") -print("Press 'q' or 'e' in the left window to quit.") +print("Interact with the window; frames are encoded to ./recording.h264.") +print("Press 'q' or 'e' in the window to quit.") interactor.Start() encoder.Drain() diff --git a/pyproject.toml b/pyproject.toml index dc7b68a..62f790d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ build-dir = "build/{wheel_tag}" [project] name = "vtk-streaming" -version = "0.3.5" +version = "0.4.0" readme = "README.md" description = "This module provides classes to encode and stream frames from a VTK OpenGL render window using video codecs." license = { text = "Apache License" } diff --git a/tests/test_videotoolbox_encode_simple.py b/tests/test_videotoolbox_encode_simple.py new file mode 100644 index 0000000..723aea5 --- /dev/null +++ b/tests/test_videotoolbox_encode_simple.py @@ -0,0 +1,110 @@ +"""Exercise the VideoToolbox H.264 encoder with IYUV, NV12 and RGBA32 inputs. + +Moving color bars are generated in memory and uploaded straight to the +vtkOpenGLVideoFrame. There is no VideoToolbox decoder module, so instead of an +image round-trip the test checks that the emitted bitstream is a plausible +H.264 Annex B stream and not all zeros. +""" + +import pytest + +import vtkmodules.vtkRenderingOpenGL2 # noqa: F401 (register the OpenGL factory) +from vtkmodules.util.misc import calldata_type +from vtkmodules.util.numpy_support import vtk_to_numpy +from vtkmodules.util.vtkConstants import VTK_OBJECT + +from vtk_streaming.vtkStreamingCore import ( + VTKPF_IYUV, + VTKPF_NV12, + VTKPF_RGBA32, + VTKVC_H264, + vtkRawVideoFrame, +) +from vtk_streaming.vtkStreamingEncode import vtkVideoEncoder +from vtk_streaming.vtkStreamingOpenGL2 import vtkOpenGLVideoFrame + +from tests import video_test_utils + +pytestmark = [ + video_test_utils.requires_rendering(), + video_test_utils.requires_videotoolbox(), +] + +# Import guarded behind the skip marker: the module only exists on macOS. +vtkVideoToolboxEncoder = pytest.importorskip( + "vtk_streaming.vtkStreamingVTEncode" +).vtkVideoToolboxEncoder + +WIDTH = 320 +HEIGHT = 240 +NUM_FRAMES = 30 + +_FORMATS = { + "iyuv": ( + VTKPF_IYUV, + video_test_utils.iyuv_frame_bytes, + video_test_utils.upload_iyuv_frame, + ), + "nv12": ( + VTKPF_NV12, + video_test_utils.nv12_frame_bytes, + video_test_utils.upload_nv12_frame, + ), + "rgba32": ( + VTKPF_RGBA32, + video_test_utils.rgba32_frame_bytes, + video_test_utils.upload_rgba32_frame, + ), +} + + +@pytest.mark.parametrize("format_name", sorted(_FORMATS)) +def test_videotoolbox_encode_simple(format_name): + pixel_format, pack_frame, upload = _FORMATS[format_name] + + input_window = video_test_utils.make_offscreen_window(WIDTH, HEIGHT) + + packets = [] + + @calldata_type(VTK_OBJECT) + def receive_packet(_encoder, _event, packet): + packets.append(vtk_to_numpy(packet.GetData()).tobytes()) + + encoder = vtkVideoToolboxEncoder() + encoder.AddObserver(vtkVideoEncoder.EncodedVideoChunkEvent, receive_packet) + encoder.SetGraphicsContext(input_window) + encoder.SetWidth(WIDTH) + encoder.SetHeight(HEIGHT) + encoder.SetCodec(VTKVC_H264) + encoder.SetInputPixelFormat(pixel_format) + + picture = vtkOpenGLVideoFrame() + picture.SetContext(input_window) + picture.SetWidth(WIDTH) + picture.SetHeight(HEIGHT) + picture.SetPixelFormat(pixel_format) + picture.SetSliceOrderType(vtkRawVideoFrame.TopDown) + picture.ComputeDefaultStrides() + picture.AllocateDataStore() + + est_size = vtkRawVideoFrame.GetEstimatedSize(WIDTH, HEIGHT, pixel_format) + + for shift in range(NUM_FRAMES): + frame_bytes = pack_frame( + video_test_utils.generate_rgba32_color_bars(WIDTH, HEIGHT, shift) + ) + assert len(frame_bytes) == est_size # plane offsets rely on this + upload(picture, frame_bytes, WIDTH, HEIGHT) + picture.Render(input_window) + encoder.Encode(picture) + # drain out remaining packets + encoder.Drain() + + assert len(packets) == NUM_FRAMES + assert all(len(packet) > 10 for packet in packets[1:]) + # A valid H.264 Annex B stream opens with a start code, and every packet + # must carry real payload rather than zero filler. + assert packets[0][:4] == b"\x00\x00\x00\x01" or packets[0][:3] == b"\x00\x00\x01" + assert all(any(packet) for packet in packets) + + encoder.Shutdown() diff --git a/tests/video_test_utils.py b/tests/video_test_utils.py index 893f15b..d0f942a 100644 --- a/tests/video_test_utils.py +++ b/tests/video_test_utils.py @@ -207,6 +207,36 @@ def requires_nvenc(): ) +# vtkVideoToolboxEncoder is only built (and only functional) on macOS. Probe in +# a subprocess so a missing module or missing hardware encoder cleanly skips. +_VIDEOTOOLBOX_PROBE = """ +from vtk_streaming.vtkStreamingVTEncode import vtkVideoToolboxEncoder +raise SystemExit(0 if vtkVideoToolboxEncoder.CheckAvailability() else 1) +""" + + +@functools.lru_cache(maxsize=1) +def videotoolbox_available(): + """Whether VideoToolbox hardware encoding is available on this machine.""" + try: + probe = subprocess.run( + [sys.executable, "-c", _VIDEOTOOLBOX_PROBE], + capture_output=True, + timeout=120, + ) + except (OSError, subprocess.TimeoutExpired): + return False + return probe.returncode == 0 + + +def requires_videotoolbox(): + return pytest.mark.skipif( + not videotoolbox_available(), + reason="VideoToolbox hardware encoding is not available (non-macOS " + "platform, or no usable hardware encoder)", + ) + + def make_offscreen_window(width, height): """Create and initialize an offscreen render window.