[Cocoa] WebCodecs H265 decoder should reorder frames according presentation time

youennf · philn · commit c5269d8a5100 · 2024-03-11T14:40:51.000Z
https://bugs.webkit.org/show_bug.cgi?id=265389 rdar://problem/118836800 Reviewed by Eric Carlson. We compute the reorder queue size by parsing SPS. We reuse the same queue as RTCVideoDecoderH264 and, for that purpose, refactor the code in a RTCVideoFrameWithOrder class. * LayoutTests/http/tests/webcodecs/hevc-reordering-expected.txt: Added. * LayoutTests/http/tests/webcodecs/hevc-reordering.html: Added. * LayoutTests/platform/glib/TestExpectations: * Source/ThirdParty/libwebrtc/Source/webrtc/common_video/h265/h265_sps_parser.cc: * Source/ThirdParty/libwebrtc/Source/webrtc/common_video/h265/h265_sps_parser.h: * Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitDecoder.mm: (-[WK_RTCLocalVideoH264H265VP9Decoder setFormat:size:width:height:]): * Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH264.mm: (RTCFrameDecodeParams::RTCFrameDecodeParams): (decompressionOutputCallback): (-[RTCVideoDecoderH264 init]): (-[RTCVideoDecoderH264 decodeData:size:timeStamp:]): (-[RTCVideoDecoderH264 setAVCFormat:size:width:height:]): (-[RTCVideoDecoderH264 flush]): (-[RTCVideoDecoderH264 processFrame:reorderSize:]): (RTCVideoFrameWithOrder::RTCVideoFrameWithOrder): Deleted. (RTCVideoFrameWithOrder::~RTCVideoFrameWithOrder): Deleted. (RTCVideoFrameWithOrder::take): Deleted. * Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.h: * Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm: (RTCH265FrameDecodeParams::RTCH265FrameDecodeParams): (spsDataFromHvcc): (ComputeH265ReorderSizeFromSPS): (ComputeH265ReorderSizeFromHVCC): (ComputeH265ReorderSizeFromAnnexB): (h265DecompressionOutputCallback): (-[RTCVideoDecoderH265 decodeData:size:timeStamp:]): (-[RTCVideoDecoderH265 setHVCCFormat:size:width:height:]): (-[RTCVideoDecoderH265 resetDecompressionSession]): (-[RTCVideoDecoderH265 flush]): (-[RTCVideoDecoderH265 processFrame:reorderSize:]): (-[RTCVideoDecoderH265 setAVCFormat:size:width:height:]): Deleted. * Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoFrameReorderQueue.h: Added. (webrtc::RTCVideoFrameReorderQueue::RTCVideoFrameWithOrder::RTCVideoFrameWithOrder): (webrtc::RTCVideoFrameReorderQueue::RTCVideoFrameWithOrder::~RTCVideoFrameWithOrder): (webrtc::RTCVideoFrameReorderQueue::RTCVideoFrameWithOrder::take): * Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoFrameReorderQueue.mm: Added. (webrtc::RTCVideoFrameReorderQueue::isEmpty): (webrtc::RTCVideoFrameReorderQueue::reorderSize const): (webrtc::RTCVideoFrameReorderQueue::setReorderSize): (webrtc::RTCVideoFrameReorderQueue::append): (webrtc::RTCVideoFrameReorderQueue::takeIfAvailable): (webrtc::RTCVideoFrameReorderQueue::takeIfAny): * Source/ThirdParty/libwebrtc/libwebrtc.xcodeproj/project.pbxproj: Canonical link: https://commits.webkit.org/271242@main
diff --git a/LayoutTests/http/tests/webcodecs/hevc-reordering-expected.txt b/LayoutTests/http/tests/webcodecs/hevc-reordering-expected.txt
@@ -0,0 +1,3 @@
+
+PASS Test HEVC reordering
+
diff --git a/LayoutTests/http/tests/webcodecs/hevc-reordering.html b/LayoutTests/http/tests/webcodecs/hevc-reordering.html
@@ -0,0 +1,57 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <script src="../../resources/testharness.js"></script>
+    <script src="../../resources/testharnessreport.js"></script>
+</head>
+<body>
+    <script>
+let decoder;
+promise_test(async () => {
+     const response = await fetch("/media-resources/media-source/content/test-bframes-hevc.mp4");
+     const buffer = await response.arrayBuffer();
+     const hvcCOffset = [705, 130];
+     const frames = [
+         [99, 3973, 46304],
+         [366, 50277, 4531],
+         [233, 54808, 1371],
+         [167, 56179, 379],
+         [133, 56558, 172],
+         [199, 56730, 120],
+         [300, 56850, 313],
+         [266, 57163, 112],
+         [333, 57275, 150],
+         [633, 57425, 6427]
+    ];
+
+    let frameTimestamps = [];
+    decoder =  new VideoDecoder({
+        output(frame) {
+            frameTimestamps.push(frame.timestamp);
+            frame.close();
+        },
+        error(e) {
+            console.log(e);
+        }
+    });
+    decoder.configure({
+        codec: 'hev1.1.6.L120.90',
+        codedWidth: 852,
+        codedHeight: 480,
+        visibleRect: {x: 0, y: 0, width: 852, height: 480},
+        displayWidth: 852,
+        displayHeight: 480,
+        format: 'hevc',
+        description: new Uint8Array(buffer, hvcCOffset[0], hvcCOffset[1])
+    });
+
+    chunks = frames.map((frame, i) => new EncodedVideoChunk({type: i == 0 ? 'key' : 'delta', timestamp: frame[0], duration: 1, data: new Uint8Array(buffer, frame[1], frame[2])}));
+
+    chunks.forEach(chunk => decoder.decode(chunk));
+    await decoder.flush();
+
+    assert_array_equals(frameTimestamps, frames.map(frame => frame[0]).sort((a,b) => a - b), "timestamps are ordered");
+}, "Test HEVC reordering");
+    </script>
+</body>
+</html>
diff --git a/Source/ThirdParty/libwebrtc/Source/webrtc/common_video/h265/h265_sps_parser.cc b/Source/ThirdParty/libwebrtc/Source/webrtc/common_video/h265/h265_sps_parser.cc
@@ -207,6 +207,9 @@ absl::optional<H265SpsParser::SpsState> H265SpsParser::ParseSpsInternal(
   sps_max_sub_layers_minus1 = reader.ReadBits(3);
   sps.sps_max_sub_layers_minus1 = sps_max_sub_layers_minus1;
   sps.sps_max_dec_pic_buffering_minus1.resize(sps_max_sub_layers_minus1 + 1, 0);
+#if WEBRTC_WEBKIT_BUILD
+  sps.sps_max_num_reorder_pics.resize(sps_max_sub_layers_minus1 + 1, 0);
+#endif
   // sps_temporal_id_nesting_flag: u(1)
   reader.ConsumeBits(1);
   // profile_tier_level(1, sps_max_sub_layers_minus1). We are acutally not
@@ -310,7 +313,9 @@ absl::optional<H265SpsParser::SpsState> H265SpsParser::ParseSpsInternal(
     // sps_max_dec_pic_buffering_minus1: ue(v)
     sps.sps_max_dec_pic_buffering_minus1[i] = reader.ReadExponentialGolomb();
     // sps_max_num_reorder_pics: ue(v)
-    reader.ReadExponentialGolomb();
+#if WEBRTC_WEBKIT_BUILD
+    sps.sps_max_num_reorder_pics[i] = reader.ReadExponentialGolomb();
+#endif
     // sps_max_latency_increase_plus1: ue(v)
     reader.ReadExponentialGolomb();
   }
diff --git a/Source/ThirdParty/libwebrtc/Source/webrtc/common_video/h265/h265_sps_parser.h b/Source/ThirdParty/libwebrtc/Source/webrtc/common_video/h265/h265_sps_parser.h
@@ -52,6 +52,9 @@ class H265SpsParser {
     uint32_t pic_height_in_luma_samples = 0;
     uint32_t log2_max_pic_order_cnt_lsb_minus4 = 0;
     std::vector<uint32_t> sps_max_dec_pic_buffering_minus1;
+#if WEBRTC_WEBKIT_BUILD
+    std::vector<uint32_t> sps_max_num_reorder_pics;
+#endif
     uint32_t log2_min_luma_coding_block_size_minus3 = 0;
     uint32_t log2_diff_max_min_luma_coding_block_size = 0;
     uint32_t sample_adaptive_offset_enabled_flag = 0;
diff --git a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoFrameReorderQueue.h b/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoFrameReorderQueue.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2023 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#import "base/RTCVideoFrame.h"
+#include <deque>
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+
+class RTCVideoFrameReorderQueue {
+public:
+    RTCVideoFrameReorderQueue() = default;
+
+    struct RTCVideoFrameWithOrder {
+        RTCVideoFrameWithOrder(RTCVideoFrame* frame, uint64_t reorderSize)
+            : frame((__bridge_retained void*)frame)
+            , timeStamp(frame.timeStamp)
+            , reorderSize(reorderSize)
+        {
+        }
+
+        ~RTCVideoFrameWithOrder()
+        {
+            if (frame)
+                take();
+        }
+
+        RTCVideoFrame* take()
+        {
+            auto* rtcFrame = (__bridge_transfer RTCVideoFrame *)frame;
+            frame = nullptr;
+            return rtcFrame;
+        }
+
+        void* frame;
+        uint64_t timeStamp;
+        uint64_t reorderSize;
+    };
+
+    bool isEmpty();
+    uint8_t reorderSize() const;
+    void setReorderSize(uint8_t);
+    void append(RTCVideoFrame*, uint8_t);
+    RTCVideoFrame *takeIfAvailable();
+    RTCVideoFrame *takeIfAny();
+
+private:
+    std::deque<std::unique_ptr<RTCVideoFrameWithOrder>> _reorderQueue;
+    uint8_t _reorderSize { 0 };
+    mutable webrtc::Mutex _reorderQueueLock;
+};
+
+}
diff --git a/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoFrameReorderQueue.mm b/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoFrameReorderQueue.mm
@@ -0,0 +1,64 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ *
+ */
+
+#import "RTCVideoFrameReorderQueue.h"
+
+namespace webrtc {
+
+bool RTCVideoFrameReorderQueue::isEmpty()
+{
+    return _reorderQueue.empty();
+}
+
+uint8_t RTCVideoFrameReorderQueue::reorderSize() const
+{
+    webrtc::MutexLock lock(&_reorderQueueLock);
+    return _reorderSize;
+}
+
+void RTCVideoFrameReorderQueue::setReorderSize(uint8_t size)
+{
+    webrtc::MutexLock lock(&_reorderQueueLock);
+    _reorderSize = size;
+}
+
+void RTCVideoFrameReorderQueue::append(RTCVideoFrame* frame, uint8_t reorderSize)
+{
+    webrtc::MutexLock lock(&_reorderQueueLock);
+    _reorderQueue.push_back(std::make_unique<RTCVideoFrameWithOrder>(frame, reorderSize));
+    std::sort(_reorderQueue.begin(), _reorderQueue.end(), [](auto& a, auto& b) {
+        return a->timeStamp < b->timeStamp;
+    });
+}
+
+RTCVideoFrame* RTCVideoFrameReorderQueue::takeIfAvailable()
+{
+    webrtc::MutexLock lock(&_reorderQueueLock);
+    if (_reorderQueue.size() && _reorderQueue.size() > _reorderQueue.front()->reorderSize) {
+        auto *frame = _reorderQueue.front()->take();
+        _reorderQueue.pop_front();
+        return frame;
+    }
+    return nil;
+}
+
+RTCVideoFrame* RTCVideoFrameReorderQueue::takeIfAny()
+{
+    webrtc::MutexLock lock(&_reorderQueueLock);
+    if (_reorderQueue.size()) {
+        auto *frame = _reorderQueue.front()->take();
+        _reorderQueue.pop_front();
+        return frame;
+    }
+    return nil;
+}
+
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+`
	`2`	`+PASS Test HEVC reordering`
	`3`	`+`