From 470dd2a5ded201735831228db9fb03cc9de3f78b Mon Sep 17 00:00:00 2001 From: Andrey Dmitriyev <33136303+Nebr88@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:47:07 +0200 Subject: [PATCH 1/2] fix(muxer): drop incomplete HEVC headers without discarding valid samples --- .../AetherEngine/Video/MP4SegmentMuxer.swift | 43 ++++++++++++++++++ Sources/AetherEngine/Video/NALUnitChain.swift | 31 ++++++++++++- .../ShortHEVCHeaderTests.swift | 45 +++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 Tests/AetherEngineTests/ShortHEVCHeaderTests.swift diff --git a/Sources/AetherEngine/Video/MP4SegmentMuxer.swift b/Sources/AetherEngine/Video/MP4SegmentMuxer.swift index 10e847983..a8ed43d12 100644 --- a/Sources/AetherEngine/Video/MP4SegmentMuxer.swift +++ b/Sources/AetherEngine/Video/MP4SegmentMuxer.swift @@ -205,6 +205,8 @@ final class MP4SegmentMuxer { /// lands in the sample entry, and the AE#561 sanitizer walks every video sample with it. private let videoNALLengthPrefixSize: Int? /// Latched from `VideoConfig.convertsAnnexBSamples`. + private let isHEVCVideo: Bool + private var shortHEVCHeadersRemoved = 0 private let convertsAnnexBVideoSamples: Bool /// AE#561 harness switch: the sanitizer removes the only shape that reproduces a segment Apple's /// parser refuses, so the rung underneath it (the software-path escalation) would have nothing to @@ -328,6 +330,7 @@ final class MP4SegmentMuxer { audio.map { Self.audioNeedsParsedPacketForMoov($0.codecpar.pointee.codec_id) } ?? false self.videoNALFraming = video.nalFramingLatch ?? NALFramingLatch() self.convertsAnnexBVideoSamples = video.convertsAnnexBSamples + self.isHEVCVideo = video.codecpar.pointee.codec_id == AV_CODEC_ID_HEVC // AE#561: the override, when there is one, is the record that reaches the sample entry. Both // carry the same width (the #19 rebuild keeps the source header's first 22 bytes), so this // only matters for a source whose own extradata is missing or Annex B. @@ -778,6 +781,46 @@ final class MP4SegmentMuxer { av_shrink_packet(packet, Int32(complete)) } + if streamIndex == videoOutputStreamIndex, isHEVCVideo, + !Self.nalChainSanitizerDisabled, let width = videoNALLengthPrefixSize, + let data = packet.pointee.data, packet.pointee.size > 0 + { + let ranges = NALUnitChain.shortHEVCHeaderRanges( + UnsafeRawBufferPointer(start: data, count: Int(packet.pointee.size)), + lengthPrefixSize: width, + framingConfirmed: convertsAnnexBVideoSamples || videoNALFraming.isConfirmed) + if !ranges.isEmpty { + let result = av_packet_make_writable(packet) + guard result >= 0, let writable = packet.pointee.data else { return (result, .none) } + let count = Int(packet.pointee.size) + var read = 0 + var write = 0 + for range in ranges { + let kept = range.lowerBound - read + if kept > 0 { + memmove(writable + write, writable + read, kept) + write += kept + } + read = range.upperBound + } + if read < count { + memmove(writable + write, writable + read, count - read) + write += count - read + } + shortHEVCHeadersRemoved += ranges.count + if shortHEVCHeadersRemoved <= 5 || shortHEVCHeadersRemoved % 100 == 0 { + EngineLog.emit( + "[MP4SegmentMuxer] removed \(ranges.count) incomplete HEVC NAL header(s) " + + "at dts=\(packet.pointee.dts), preserving \(write) sample bytes", category: .session) + } + av_shrink_packet(packet, Int32(write)) + if write == 0 { + av_packet_unref(packet) + return (0, .none) + } + } + } + // The packet is consumed by the write below. let writtenDuration = max(packet.pointee.duration, 0) diff --git a/Sources/AetherEngine/Video/NALUnitChain.swift b/Sources/AetherEngine/Video/NALUnitChain.swift index 24042142a..367fce5d2 100644 --- a/Sources/AetherEngine/Video/NALUnitChain.swift +++ b/Sources/AetherEngine/Video/NALUnitChain.swift @@ -1,6 +1,6 @@ -import Foundation import AetherLibavcodec import AetherLibavutil +import Foundation /// Audit BIT-104: whether a video sample of the session's track has walked exactly as a /// length-prefixed chain (BIT-1). Held by the session rather than by one muxer, because every seek, @@ -92,4 +92,33 @@ enum NALUnitChain { guard let base = bytes.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return false } return VideoConfigRecord.walksAsLengthPrefixed(base, size: bytes.count, lengthSize: lengthPrefixSize) } + + /// HEVC's NAL header is two bytes, even for EOS/EOB. A one-byte EOS from a + /// truncated transport-stream tail can have a perfectly valid length prefix, + /// so the overrun guard above cannot catch it. Apple's parser refuses the + /// entire HLS segment, while libavcodec ignores the damaged delimiter. + /// Return only the malformed units, preserving valid pictures and RPUs after + /// them. Never treat an unconfirmed Annex-B start code as a length of one. + static func shortHEVCHeaderRanges( + _ bytes: UnsafeRawBufferPointer, lengthPrefixSize: Int, framingConfirmed: Bool + ) -> [Range] { + guard (1...4).contains(lengthPrefixSize), bytes.count >= lengthPrefixSize else { return [] } + if !framingConfirmed, bytes.count >= 3, bytes[0] == 0, bytes[1] == 0, + bytes[2] == 1 || (bytes.count >= 4 && bytes[2] == 0 && bytes[3] == 1) + { + return [] + } + var ranges: [Range] = [] + var offset = 0 + while offset + lengthPrefixSize <= bytes.count { + var length = 0 + for index in 0.. 0, length <= bytes.count - payload else { return [] } + let end = payload + length + if length < 2 { ranges.append(offset.. [UInt8] { + (0..> ($0 * 8)) & 0xff) } + bytes + } + private func ranges(_ bytes: [UInt8], width: Int = 4, confirmed: Bool = true) -> [Range] { + bytes.withUnsafeBytes { + NALUnitChain.shortHEVCHeaderRanges($0, lengthPrefixSize: width, framingConfirmed: confirmed) + } + } + + @Test("A complete length chain can still contain a truncated EOS header") + func damagedEOSIsRemovedWithoutLosingThePictureOrFollowingRPU() { + let picture = unit([0x26, 0x01, 0x80]) + let eos = unit([0x48]) + let rpu = unit([0x7c, 0x01, 0x80]) + let sample = picture + eos + rpu + #expect( + sample.withUnsafeBytes { NALUnitChain.completeRunLength($0, lengthPrefixSize: 4, framingConfirmed: true) } + == nil) + #expect(ranges(sample) == [picture.count..<(picture.count + eos.count)]) + let removed = ranges(sample) + let cleaned = sample.enumerated().filter { entry in + !removed.contains { $0.contains(entry.offset) } + }.map(\.element) + #expect(cleaned == picture + rpu) + } + + @Test("Valid HEVC delimiters and pictures remain byte-for-byte unchanged") + func validUnitsSurvive() { + #expect(ranges(unit([0x26, 0x01, 0x80]) + unit([0x48, 0x01, 0x80])) == []) + } + + @Test("Framing and bounds fail closed, and all prefix widths are supported") + func framingBoundsAndWidths() { + #expect(ranges([0, 0, 0, 1, 0x48], confirmed: false) == []) + #expect(ranges(unit([0x48]) + [0, 0, 2]) == []) + for width in 1...4 { #expect(ranges(unit([0x48], width: width), width: width) == [0..<(width + 1)]) } + } +} From d8b9a775e7f170aef4ad49dc9132a911f552273c Mon Sep 17 00:00:00 2001 From: Andrey Dmitriyev <33136303+Nebr88@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:18:05 +0200 Subject: [PATCH 2/2] test(muxer): verify HEVC compaction through written fragments --- CHANGELOG.md | 4 + .../AetherEngine/Video/MP4SegmentMuxer.swift | 2 +- Sources/AetherEngine/Video/NALUnitChain.swift | 2 +- .../ShortHEVCHeaderTests.swift | 102 ++++++++++++++++++ docs/formats.md | 5 + 5 files changed, 113 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6131b2db5..52ed925fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ the public-API contract. ### Fixed +- **A complete HEVC length chain with an incomplete two-byte NAL header no longer poisons an fMP4 + segment** (#960). The muxer removes only short headers, preserves the surrounding pictures/RPUs and + timestamps, and drops a sample consisting entirely of short headers. Packet compaction is copy-on-write. + - **WMV1/2 software packet caches retain continuous presentation coverage.** Their I/P-only streams use the existing successor-PTS model with depth one, avoiding millisecond holes from rounded Matroska durations. Disk and memory stores share the qualification; missing/late diff --git a/Sources/AetherEngine/Video/MP4SegmentMuxer.swift b/Sources/AetherEngine/Video/MP4SegmentMuxer.swift index a8ed43d12..4fdadc8ca 100644 --- a/Sources/AetherEngine/Video/MP4SegmentMuxer.swift +++ b/Sources/AetherEngine/Video/MP4SegmentMuxer.swift @@ -204,9 +204,9 @@ final class MP4SegmentMuxer { /// length-prefixed at all. Latched at init: it is a property of the configuration record that /// lands in the sample entry, and the AE#561 sanitizer walks every video sample with it. private let videoNALLengthPrefixSize: Int? - /// Latched from `VideoConfig.convertsAnnexBSamples`. private let isHEVCVideo: Bool private var shortHEVCHeadersRemoved = 0 + /// Latched from `VideoConfig.convertsAnnexBSamples`. private let convertsAnnexBVideoSamples: Bool /// AE#561 harness switch: the sanitizer removes the only shape that reproduces a segment Apple's /// parser refuses, so the rung underneath it (the software-path escalation) would have nothing to diff --git a/Sources/AetherEngine/Video/NALUnitChain.swift b/Sources/AetherEngine/Video/NALUnitChain.swift index 367fce5d2..2cf6193c6 100644 --- a/Sources/AetherEngine/Video/NALUnitChain.swift +++ b/Sources/AetherEngine/Video/NALUnitChain.swift @@ -1,6 +1,6 @@ +import Foundation import AetherLibavcodec import AetherLibavutil -import Foundation /// Audit BIT-104: whether a video sample of the session's track has walked exactly as a /// length-prefixed chain (BIT-1). Held by the session rather than by one muxer, because every seek, diff --git a/Tests/AetherEngineTests/ShortHEVCHeaderTests.swift b/Tests/AetherEngineTests/ShortHEVCHeaderTests.swift index 57bdeacd6..e202dfe07 100644 --- a/Tests/AetherEngineTests/ShortHEVCHeaderTests.swift +++ b/Tests/AetherEngineTests/ShortHEVCHeaderTests.swift @@ -1,5 +1,7 @@ import Foundation import Testing +import AetherLibavcodec +import AetherLibavutil @testable import AetherEngine @@ -42,4 +44,104 @@ struct ShortHEVCHeaderTests { #expect(ranges(unit([0x48]) + [0, 0, 2]) == []) for width in 1...4 { #expect(ranges(unit([0x48], width: width), width: width) == [0..<(width + 1)]) } } + private final class InitBytes: @unchecked Sendable { + private let lock = NSLock() + private var bytes = Data() + func set(_ data: Data) { lock.withLock { bytes = data } } + func get() -> Data { lock.withLock { bytes } } + } + + @Test("The muxer compacts a shared packet and writes the correct picture/RPU size") + func muxerCompactsWithoutChangingTheSharedOriginal() throws { + try muxerRoundTrip(dropsFirstSample: false) + } + + @Test("A packet containing only short headers is dropped before the next valid sample") + func muxerDropsAnAllShortSample() throws { + try muxerRoundTrip(dropsFirstSample: true) + } + + private func muxerRoundTrip(dropsFirstSample: Bool) throws { + let fixture = try #require(Data(base64Encoded: Self.hevcFixture, options: .ignoreUnknownCharacters)) + let demuxer = Demuxer() + defer { demuxer.close() } + try demuxer.open(reader: DataIOReader(data: fixture), formatHint: "mp4") + let stream = try #require(demuxer.stream(at: demuxer.videoStreamIndex)) + #expect(stream.pointee.codecpar.pointee.codec_id == AV_CODEC_ID_HEVC) + var sourcePacket: UnsafeMutablePointer? = try #require(try demuxer.readPacket()) + defer { trackedPacketFree(&sourcePacket) } + let picture = Data(bytes: sourcePacket!.pointee.data!, count: Int(sourcePacket!.pointee.size)) + let rpu = Data(unit([0x7c, 0x01, 0x80])) + let expected = picture + rpu + let damaged = dropsFirstSample ? Data(unit([0x48]) + unit([0x4a])) : picture + Data(unit([0x48])) + rpu + let directory = FileManager.default.temporaryDirectory.appendingPathComponent("short-hevc-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let initial = InitBytes() + let muxer = try MP4SegmentMuxer(initialSegmentIndex: 0, sessionDir: directory, + video: .init(codecpar: UnsafePointer(stream.pointee.codecpar), timeBase: stream.pointee.time_base, + codecTagOverride: nil, nalFramingLatch: NALFramingLatch(confirmed: true)), + audio: nil, onInitCaptured: { initial.set($0) }) + + var packet: UnsafeMutablePointer? = try #require(trackedPacketAlloc()) + var shared: UnsafeMutablePointer? = try #require(trackedPacketAlloc()) + defer { trackedPacketFree(&packet); trackedPacketFree(&shared) } + #expect(av_new_packet(packet!, Int32(damaged.count)) >= 0) + damaged.withUnsafeBytes { _ = memcpy(packet!.pointee.data!, $0.baseAddress!, $0.count) } + packet!.pointee.pts = 0; packet!.pointee.dts = 0; packet!.pointee.duration = 512 + packet!.pointee.flags = AV_PKT_FLAG_KEY + packet!.pointee.stream_index = muxer.videoOutputStreamIndex + #expect(av_packet_ref(shared!, packet!) >= 0) + let outcome = muxer.writePacket(packet!) + #expect(outcome.rc >= 0) + #expect(Data(bytes: shared!.pointee.data!, count: Int(shared!.pointee.size)) == damaged) + if dropsFirstSample { + #expect(outcome.written.pts == Int64.min && packet!.pointee.size == 0) + #expect(muxer.writtenVideoEndPts == Int64.min) + #expect(av_new_packet(packet!, Int32(expected.count)) >= 0) + expected.withUnsafeBytes { _ = memcpy(packet!.pointee.data!, $0.baseAddress!, $0.count) } + packet!.pointee.pts = 512; packet!.pointee.dts = 512; packet!.pointee.duration = 512 + packet!.pointee.flags = AV_PKT_FLAG_KEY + packet!.pointee.stream_index = muxer.videoOutputStreamIndex + #expect(muxer.writePacket(packet!).rc >= 0) + } else { + #expect(outcome.written.pts == 0 && outcome.written.dts == 0) + } + guard case .completed(let path, let count) = muxer.cutFragmentForNextSegment(1) else { + Issue.record("The valid sample did not produce a complete fragment") + return + } + let fragment = try Data(contentsOf: path) + #expect(fragment.count == count) + let result = Demuxer() + defer { result.close() } + try result.open(reader: DataIOReader(data: initial.get() + fragment), formatHint: "mp4") + var clean: UnsafeMutablePointer? = try #require(try result.readPacket()) + defer { trackedPacketFree(&clean) } + #expect(clean!.pointee.size == expected.count) + #expect(Data(bytes: clean!.pointee.data!, count: Int(clean!.pointee.size)) == expected) + if let unexpected = try result.readPacket() { + var owned: UnsafeMutablePointer? = unexpected + trackedPacketFree(&owned) + Issue.record("The dropped header-only sample survived in the fragment") + } + } + + // Own synthetic two-frame 64x64 black HEVC MP4; x265 info=0, no private sample. + private static let hevcFixture = """ + AAAAHGZ0eXBpc29tAAACAGlzb21pc28ybXA0MQAAA3ttb292AAAAbG12aGQAAAAAAAAAAAAAAAAAAAPoAAAAVAABAAABAAAAAAAA + AAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAACpnRy + YWsAAABcdGtoZAAAAAMAAAAAAAAAAAAAAAEAAAAAAAAAVAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAA + AAAAAAAAAEAAAAAAQAAAAEAAAAAAACRlZHRzAAAAHGVsc3QAAAAAAAAAAQAAAFQAAAAAAAEAAAAAAh5tZGlhAAAAIG1kaGQAAAAA + AAAAAAAAAAAAADAAAAAEAFXEAAAAAAAtaGRscgAAAAAAAAAAdmlkZQAAAAAAAAAAAAAAAFZpZGVvSGFuZGxlcgAAAAHJbWluZgAA + ABR2bWhkAAAAAQAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAABiXN0YmwAAAEJc3RzZAAAAAAA + AAABAAAA+Wh2YzEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAQABAAEgAAABIAAAAAAAAAAEUTGF2YzYzLjEuMTAyIGxpYngyNjUA + AAAAAAAAAAAAAAAY//8AAAB1aHZjQwEBYAAAAJAAAAAAAB7wAPz9+PgAAA8DoAABABhAAQwB//8BYAAAAwCQAAADAAADAB6VmAmh + AAEAKEIBAQFgAAADAJAAAAMAAAMAHqAggQWWVmkkyvAWgIAAAAMAgAAADASiAAEAB0QBwXK0IkAAAAAKZmllbAEAAAAAEHBhc3AA + AAABAAAAAQAAABRidHJ0AAAAAAAADAAAAAAAAAAAGHN0dHMAAAAAAAAAAQAAAAIAAAIAAAAAFHN0c3MAAAAAAAAAAQAAAAEAAAAc + c3RzYwAAAAAAAAABAAAAAQAAAAIAAAABAAAAHHN0c3oAAAAAAAAAAAAAAAIAAAASAAAADgAAABRzdGNvAAAAAAAAAAEAAAOnAAAA + YXVkdGEAAABZbWV0YQAAAAAAAAAhaGRscgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAsaWxzdAAAACSpdG9vAAAAHGRhdGEA + AAABAAAAAExhdmY2My4xLjEwMgAAAAhmcmVlAAAAKG1kYXQAAAAOKAGvHYD3AbL//SMvszwAAAAKAgHQCX4Qxjv6hA== + """ + } diff --git a/docs/formats.md b/docs/formats.md index 2097e241b..fbf591b6e 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -282,6 +282,11 @@ Non-DV HEVC derives its primary `CODECS` string from the source `hvcC` profile_t Whatever the profile, the `hvcC` shipped in `init.mp4` must carry the parameter sets out of band. Sources authored with in-band VPS/SPS/PPS only (`hev1`, `numOfArrays = 0`, what `MP4Box ...:xps_inband` and the common Dolby Vision MP4 recipes produce) are normalized at load by scanning the head of the stream for the parameter sets and rebuilding the record; a record that arrives with extra non-parameter-set arrays (libx265's user-data SEI) is stripped down to VPS/SPS/PPS instead. Shipping the source record verbatim in either case leaves AVPlayer with no usable format description: it buffers the whole forward window and never renders a frame. H.264 gets the same rebuild (#938): an `avcC` with zero SPS or zero PPS (mkvmerge writes the 7-byte `01 64 00 20 ff e0 00` for such a track) is rebuilt from the first in-band SPS and PPS, by movenc so the High-profile extension comes out right, with the source's NAL length size kept. Shipped verbatim it fails differently from HEVC and more quietly: the item reaches `readyToPlay`, the clock runs and audio plays, and no frame is ever ready for display. +A chain can also walk exactly while a unit contains only one byte of HEVC's two-byte header +(for example a truncated EOS). In confirmed length-prefixed HEVC samples the muxer removes those +units, retaining valid pictures and any following Dolby Vision RPU. Shared packets are made writable +before compaction; a sample with no complete header left is not written (#960). + ### Dolby Vision signaling For DV streams the demuxer surfaces the source's `AVDOVIDecoderConfigurationRecord`, and the route depends on the profile's base-layer compatibility: