// // VideoToolboxDecode.swift // KSPlayer // // Created by kintan on 2018/3/10. // import FFmpegKit import Libavformat #if canImport(VideoToolbox) import VideoToolbox class VideoToolboxDecode: DecodeProtocol { private var session: DecompressionSession { didSet { VTDecompressionSessionInvalidate(oldValue.decompressionSession) } } private let options: KSOptions private var startTime = Int64(0) private var lastPosition = Int64(0) private var needReconfig = false init(options: KSOptions, session: DecompressionSession) { self.options = options self.session = session } func decodeFrame(from packet: Packet, completionHandler: @escaping (Result) -> Void) { if needReconfig { // 解决从后台切换到前台,解码失败的问题 session = DecompressionSession(assetTrack: session.assetTrack, options: options)! doFlushCodec() needReconfig = false } guard let corePacket = packet.corePacket?.pointee, let data = corePacket.data else { return } do { let sampleBuffer = try session.formatDescription.getSampleBuffer(isConvertNALSize: session.assetTrack.isConvertNALSize, data: data, size: Int(corePacket.size)) let flags: VTDecodeFrameFlags = [ ._EnableAsynchronousDecompression, ] var flagOut = VTDecodeInfoFlags.frameDropped let timestamp = packet.timestamp let packetFlags = corePacket.flags let duration = corePacket.duration let size = corePacket.size let status = VTDecompressionSessionDecodeFrame(session.decompressionSession, sampleBuffer: sampleBuffer, flags: flags, infoFlagsOut: &flagOut) { [weak self] status, infoFlags, imageBuffer, _, _ in guard let self, !infoFlags.contains(.frameDropped) else { return } guard status == noErr else { if status == kVTInvalidSessionErr || status == kVTVideoDecoderMalfunctionErr || status == kVTVideoDecoderBadDataErr { if packet.isKeyFrame { completionHandler(.failure(NSError(errorCode: .codecVideoReceiveFrame, avErrorCode: status))) } else { // 解决从后台切换到前台,解码失败的问题 self.needReconfig = true } } return } let frame = VideoVTBFrame(fps: session.assetTrack.nominalFrameRate, isDovi: session.assetTrack.dovi != nil) frame.corePixelBuffer = imageBuffer frame.timebase = session.assetTrack.timebase if packet.isKeyFrame, packetFlags & AV_PKT_FLAG_DISCARD != 0, self.lastPosition > 0 { self.startTime = self.lastPosition - timestamp } self.lastPosition = max(self.lastPosition, timestamp) frame.position = packet.position frame.timestamp = self.startTime + timestamp frame.duration = duration frame.size = size self.lastPosition += frame.duration completionHandler(.success(frame)) } if status == noErr { if !flags.contains(._EnableAsynchronousDecompression) { VTDecompressionSessionWaitForAsynchronousFrames(session.decompressionSession) } } else if status == kVTInvalidSessionErr || status == kVTVideoDecoderMalfunctionErr || status == kVTVideoDecoderBadDataErr { if packet.isKeyFrame { throw NSError(errorCode: .codecVideoReceiveFrame, avErrorCode: status) } else { // 解决从后台切换到前台,解码失败的问题 needReconfig = true } } } catch { completionHandler(.failure(error)) } } func doFlushCodec() { lastPosition = 0 startTime = 0 } func shutdown() { VTDecompressionSessionInvalidate(session.decompressionSession) } func decode() { lastPosition = 0 startTime = 0 } } class DecompressionSession { fileprivate let formatDescription: CMFormatDescription fileprivate let decompressionSession: VTDecompressionSession fileprivate var assetTrack: FFmpegAssetTrack init?(assetTrack: FFmpegAssetTrack, options: KSOptions) { self.assetTrack = assetTrack guard let pixelFormatType = assetTrack.pixelFormatType, let formatDescription = assetTrack.formatDescription else { return nil } self.formatDescription = formatDescription #if os(macOS) VTRegisterProfessionalVideoWorkflowVideoDecoders() if #available(macOS 11.0, *) { VTRegisterSupplementalVideoDecoderIfAvailable(formatDescription.mediaSubType.rawValue) } #endif // VTDecompressionSessionCanAcceptFormatDescription(<#T##session: VTDecompressionSession##VTDecompressionSession#>, formatDescription: <#T##CMFormatDescription#>) let attributes: NSMutableDictionary = [ kCVPixelBufferPixelFormatTypeKey: pixelFormatType, kCVPixelBufferMetalCompatibilityKey: true, kCVPixelBufferWidthKey: assetTrack.codecpar.width, kCVPixelBufferHeightKey: assetTrack.codecpar.height, kCVPixelBufferIOSurfacePropertiesKey: NSDictionary(), ] var session: VTDecompressionSession? // swiftlint:disable line_length let status = VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, formatDescription: formatDescription, decoderSpecification: CMFormatDescriptionGetExtensions(formatDescription), imageBufferAttributes: attributes, outputCallback: nil, decompressionSessionOut: &session) // swiftlint:enable line_length guard status == noErr, let decompressionSession = session else { return nil } if #available(iOS 14.0, tvOS 14.0, macOS 11.0, *) { VTSessionSetProperty(decompressionSession, key: kVTDecompressionPropertyKey_PropagatePerFrameHDRDisplayMetadata, value: kCFBooleanTrue) } if let destinationDynamicRange = options.availableDynamicRange(nil) { let pixelTransferProperties = [kVTPixelTransferPropertyKey_DestinationColorPrimaries: destinationDynamicRange.colorPrimaries, kVTPixelTransferPropertyKey_DestinationTransferFunction: destinationDynamicRange.transferFunction, kVTPixelTransferPropertyKey_DestinationYCbCrMatrix: destinationDynamicRange.yCbCrMatrix] VTSessionSetProperty(decompressionSession, key: kVTDecompressionPropertyKey_PixelTransferProperties, value: pixelTransferProperties as CFDictionary) } self.decompressionSession = decompressionSession } } #endif extension CMFormatDescription { fileprivate func getSampleBuffer(isConvertNALSize: Bool, data: UnsafeMutablePointer, size: Int) throws -> CMSampleBuffer { if isConvertNALSize { var ioContext: UnsafeMutablePointer? let status = avio_open_dyn_buf(&ioContext) if status == 0 { var nalSize: UInt32 = 0 let end = data + size var nalStart = data while nalStart < end { nalSize = UInt32(nalStart[0]) << 16 | UInt32(nalStart[1]) << 8 | UInt32(nalStart[2]) avio_wb32(ioContext, nalSize) nalStart += 3 avio_write(ioContext, nalStart, Int32(nalSize)) nalStart += Int(nalSize) } var demuxBuffer: UnsafeMutablePointer? let demuxSze = avio_close_dyn_buf(ioContext, &demuxBuffer) return try createSampleBuffer(data: demuxBuffer, size: Int(demuxSze)) } else { throw NSError(errorCode: .codecVideoReceiveFrame, avErrorCode: status) } } else { return try createSampleBuffer(data: data, size: size) } } private func createSampleBuffer(data: UnsafeMutablePointer?, size: Int) throws -> CMSampleBuffer { var blockBuffer: CMBlockBuffer? var sampleBuffer: CMSampleBuffer? // swiftlint:disable line_length var status = CMBlockBufferCreateWithMemoryBlock(allocator: kCFAllocatorDefault, memoryBlock: data, blockLength: size, blockAllocator: kCFAllocatorNull, customBlockSource: nil, offsetToData: 0, dataLength: size, flags: 0, blockBufferOut: &blockBuffer) if status == noErr { status = CMSampleBufferCreate(allocator: kCFAllocatorDefault, dataBuffer: blockBuffer, dataReady: true, makeDataReadyCallback: nil, refcon: nil, formatDescription: self, sampleCount: 1, sampleTimingEntryCount: 0, sampleTimingArray: nil, sampleSizeEntryCount: 0, sampleSizeArray: nil, sampleBufferOut: &sampleBuffer) if let sampleBuffer { return sampleBuffer } } throw NSError(errorCode: .codecVideoReceiveFrame, avErrorCode: status) // swiftlint:enable line_length } } extension CMVideoCodecType { var avc: String { switch self { case kCMVideoCodecType_MPEG4Video: return "esds" case kCMVideoCodecType_H264: return "avcC" case kCMVideoCodecType_HEVC: return "hvcC" case kCMVideoCodecType_VP9: return "vpcC" default: return "avcC" } } }