How to achieve better performance in converting UIView to CVPixelBuffer?

Question

I'm wondering if it's possible to achieve a better performance in converting the UIView into CVPixelBuffer.

My app converts a sequence of UIViews first into UIImages and then into CVPixelBuffers as shown below. In the end, I record all these images/frames into an AVAssetWriterInput and save the result as a movie file.

Thank you in advance!

Best, Aibek

func viewToImage(view: UIView) -> CGImage {
  let rect: CGRect = container.frame

  UIGraphicsBeginImageContextWithOptions(rect.size, true, 1)

  let context: CGContext = UIGraphicsGetCurrentContext()!
  view.layer.render(in: context)
  let img = UIGraphicsGetImageFromCurrentImageContext()

  UIGraphicsEndImageContext()

  return img!.cgImage
}

func imageToBuffer(image: CGImage) -> CVPixelBuffer? {
  let frameSize = CGSize(width: image.width, height: image.height)

  var pixelBuffer: CVPixelBuffer?
  let status = CVPixelBufferCreate(kCFAllocatorDefault, Int(frameSize.width), Int(frameSize.height), kCVPixelFormatType_32BGRA, nil, &pixelBuffer)

  if status != kCVReturnSuccess {
    return nil
  }

  CVPixelBufferLockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))
  let data = CVPixelBufferGetBaseAddress(pixelBuffer!)
  let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
  let bitmapInfo = CGBitmapInfo(rawValue: CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.premultipliedFirst.rawValue)
  let context = CGContext(data: data, width: Int(frameSize.width), height: Int(frameSize.height), bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer!), space: rgbColorSpace, bitmapInfo: bitmapInfo.rawValue)

  context?.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))

  CVPixelBufferUnlockBaseAddress(pixelBuffer!, CVPixelBufferLockFlags(rawValue: 0))

  return pixelBuffer
}

Answer 1

You'd better to see this. https://stackoverflow.com/a/61862728/13680955

In short, this sample converts UIView to MTLTexture in 12ms.

Sure you can use CVPixelBuffer directly, but I used MTLTexture to make video and no issue was on it.

If you are struggling with the performance, too slow or weird to use, try to do this.

With MTLTexture

import AVFoundation
import MetalKit

class VideoRecorder {
    let assetWriter: AVAssetWriter
    let assetWriterVideoInput: AVAssetWriterInput
    let assetWriterInputPixelBufferAdapter: AVAssetWriterInputPixelBufferAdaptor
    var recordingStartTime = TimeInterval(0)
    var recordingElapsedTime = TimeInterval(0)
    let url: URL = {
        let fileName = "exported_video.mp4"
        return FileManager.default.temporaryDirectory.appendingPathComponent(fileName)
    }()

    init(outputSize: CGSize) throws {
        if FileManager.default.fileExists(atPath: url.path) {
            try FileManager.default.removeItem(at: url)
        }
        let fileType: AVFileType = .mov
        assetWriter = try AVAssetWriter(outputURL: url, fileType: fileType)
        let mediaType: AVMediaType = .video
        let outputSettings: [String: Any] = [
            AVVideoCodecKey: AVVideoCodecType.h264,
            AVVideoWidthKey: outputSize.width,
            AVVideoHeightKey: outputSize.height
        ]
        assetWriterVideoInput = AVAssetWriterInput(mediaType: mediaType, outputSettings: outputSettings)
        assetWriterVideoInput.expectsMediaDataInRealTime = false
        let sourcePixelBufferAttributes: [String: Any] = [
            kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
            kCVPixelBufferWidthKey as String: outputSize.width,
            kCVPixelBufferHeightKey as String: outputSize.height
        ]
        assetWriterInputPixelBufferAdapter = AVAssetWriterInputPixelBufferAdaptor(
            assetWriterInput: assetWriterVideoInput, sourcePixelBufferAttributes: sourcePixelBufferAttributes)
        assetWriter.add(assetWriterVideoInput)
    }

    private static func currentTimestampString() -> String {
        let date = Date()
        let dateFormatter = DateFormatter()
        dateFormatter.dateFormat = "yyyy-MM-dd HH:mm:ss"
        return dateFormatter.string(from: date)
    }

    public func start() {
        print("videoRecorder.start")
        assetWriter.startWriting()
        assetWriter.startSession(atSourceTime: .zero)
        recordingStartTime = CACurrentMediaTime()
    }

    public func cancel() {
        #if DEBUG
        print("videoRecorder.cancel")
        #endif

        assetWriterVideoInput.markAsFinished()
        assetWriter.cancelWriting()
    }

    public func finish(_ callback: @escaping () -> Void) {
        print("videoRecorder.finish")
        assetWriterVideoInput.markAsFinished()
        assetWriter.finishWriting {
            self.recordingElapsedTime = CACurrentMediaTime() - self.recordingStartTime
            print("videoRecorder.finish elapsedTime: \(self.recordingElapsedTime)")
            callback()
        }
    }

    private var pixelBuffer: CVPixelBuffer?

    public func writeFrame(texture: MTLTexture, at presentationTime: CMTime) {
        print("videoRecorder.writeFrame: \(presentationTime)")
        if pixelBuffer == nil {
            guard let pixelBufferPool = assetWriterInputPixelBufferAdapter.pixelBufferPool else {
                print("Pixel buffer asset writer input did not have a pixel buffer pool available;")
                print("cannot retrieve frame")
                return
            }

            var maybePixelBuffer: CVPixelBuffer?
            let status  = CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &maybePixelBuffer)
            if status != kCVReturnSuccess {
                print("Could not get pixel buffer from asset writer input; dropping frame...")
                return
            }
            pixelBuffer = maybePixelBuffer
            print("videoRecorder.writeFrame: pixelBuffer was created: \(String(describing: pixelBuffer))")
        }

        guard let pixelBuffer = pixelBuffer else {
            print("videoRecorder.writeFrame: NO pixelBuffer")
            return
        }

        writeFrame(texture: texture, at: presentationTime, with: pixelBuffer)
    }

    private func writeFrame(texture: MTLTexture, at presentationTime: CMTime, with pixelBuffer: CVPixelBuffer) {
        while !assetWriterVideoInput.isReadyForMoreMediaData {
            //
            print("NOT ready for more media data at: \(presentationTime)")
        }
        CVPixelBufferLockBaseAddress(pixelBuffer, [])
        let pixelBufferBytes = CVPixelBufferGetBaseAddress(pixelBuffer)!

        // Use the bytes per row value from the pixel buffer since its stride may be rounded up to be 16-byte aligned
        let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
        let region = MTLRegionMake2D(0, 0, texture.width, texture.height)

        texture.getBytes(pixelBufferBytes, bytesPerRow: bytesPerRow, from: region, mipmapLevel: 0)

        assetWriterInputPixelBufferAdapter.append(pixelBuffer, withPresentationTime: presentationTime)

        CVPixelBufferUnlockBaseAddress(pixelBuffer, [])
    }
}

Answer 2

Converting the UIViews into MTLTextures and recording them into a video file using the Recorder provided by Mayo didn't increase the performance actually.

However, the recorder is able to write MTLTextures in real-time. That meant for me that I can re-write all the animations using Metal and use the recorder.