如何裁剪和翻转 CVPixelBuffer 并返回 CVPixelBuffer？

Question

我正在制作一个快速的视频应用程序。

在我的应用程序中，我需要裁剪和水平翻转 CVPixelBuffer 并返回类型也是 CVPixelBuffer 的结果。

我尝试了几件事。

首先，我使用了 'CVPixelBufferCreateWithBytes'

func resizePixelBuffer(_ pixelBuffer: CVPixelBuffer, destSize: CGSize) 
-> CVPixelBuffer? 
{

  CVPixelBufferLockAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: O))

  let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
  let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
  let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
  let width = CVPixelBufferGetWidth(pixelBuffer)
  let height = CVPixelBufferGetHeight(pixelBuffer)

  var destPixelBuffer: CVPixelBuffer?

  let topMargin = (height - destsize.height) / 2
  let leftMargin = (width - destsize.width) / 2 * 4   // bytesPerPixel
  let offset = topMargin * bytesPerRow + leftMargin

  CVPixelBufferCreateWithBytes(kCFAllocatorDefault, 
                               destSize.width, 
                               destSize.height, 
                               pixelFormat, 
                               baseAddress.advanced(by: offset),
                               bytesPerRow, 
                               nil, nil, nil, 
                               &destPixelBuffer)

  CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: O))

  return destPixelBuffer
)

使用此代码，我可以直接裁剪 CVPixelBuffer 并返回 CVPixelBuffer。但是，我不知道如何水平翻转 CVPlxelBuffer。

所以我尝试了其他解决方案。

其次，我将CVPixelBuffer转换为CIImage，然后，返回CVPixelBuffer

func resizePixelBuffer(_ pixelBuffer, destSize: CGSize) 
-> CVPixelBuffer?
{
  let bufferWidth = CVPixelBufferGetWidth(pixelBuffer)
  let bufferHeight = CVPixelBufferGetHeight(pixelBuffer)

  let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
  let rect = CGRect(x: (bufferWidth - destSize.width)/2, y: (bufferHeight - destSize.height)/2, width: destSize.width, height: destSize.height)
  let croppedImage = ciImage.cropped(to: rect)

  croppedImage.transformed(by: CGAffineTransform(translateX: -1, y: 0))

  var destPixelBuffer: CVPixelBuffer?
  CVPixelBufferCreate(kCFAllocatorDefault, destSize.width, destSize.height,
                      CVPixelBufferGetPixelFormatType(pixelBuffer), nil, 
                      &destPixelBuffer)

  CIContext().render(croppedImage, to: destPixelBuffer!, bounds: croppedImage.extent, croppedImage.colorSpace)

  return destPixelBuffer
}

但结果不是我所期望的。图像的某些部分是黑色的，我认为 CGAffineTransform 不起作用。

最后，我尝试转换为 CGImage

func resizePixelBuffer(_ pixelBuffer: CVPixelBuffer, destSize: CGSize)
-> CVPixelBuffer? 
{
  let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
  let cgImage = CIContext().createCGImage(ciImage, from: ciImage.extent)
  let rect = CGRect(x: (bufferWidth - destSize.width)/2, y: (bufferHeight - destSize.height)/2, width: destSize.width, height: destSize.height)

  let croppedImage = cgImage.cropping(to: rect)

  let width = croppedImage.width
  let height = croppedImage.height
  let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)

  var destPixelBuffer: CVPixelBuffer?
  CVPixelBufferCreate(kCFAllocatorDefault, width, height, pixelFormat, &destPixelBuffer)

  CVPixelBufferLockBaseAddress(destPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))

  let destBaseAddress = CVPixelBufferGetBaseAddress(destPixelBuffer)
  let destBytesPerRow = CVPixelBufferGetBytesPerRow(destPixelBuffer)

  let context = CGContext(data: destBaseAddress, 
                          width: width, 
                          height: height, 
                          bitsPerComponent: 8, 
                          bytesPerRow: destBytesPerRow, 
                          space: croppedImage.colorSpace, 
                          bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue)

  context?.concatenate(__CGAffineTransformMake( 1, 0, 0, -1, 0, CGFloat(height)))

  context?.draw(croppedCgImage, in: CGRect(x: 0, y: 0, width: CGFloat(width), height: CGFloat(height)))

  CVPixelBufferUnlockBaseAddress(srcPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))

  return destPixelBuffer
}

此时，输出的像素缓冲区是全黑的。

我不知道如何裁剪和翻转 CVPixelBuffer 并返回 CVPixelBuffer。

我认为转换 CIImage 或 CGImage 是更好的方法，因为我可以用这些格式做很多事情。

但我不知道如何将这些格式转换回 CVPixelBuffer。

请告诉我该怎么做。

Answer 1

这是CVPixelBuffer 中使用vImage 的流。

1. 检查 iOS - Scale and crop CMSampleBufferRef/CVImageBufferRef 以了解作物对缓冲区的作用
2. 查看https://developer.apple.com/documentation/accelerate/reading_from_and_writing_to_core_video_pixel_buffers 了解 vImage 的基本用法。基本上，您需要创建源缓冲区和目标缓冲区并对其应用转换。
3. vImageBuffer 到 CVPixelBuffer 部分引用自https://github.com/tensorflow/examples/blob/master/lite/examples/object_detection/ios/ObjectDetection/Extensions/CVPixelBufferExtension.swift

import Cocoa
import Accelerate

extension CVPixelBuffer {
    func crop(to rect: CGRect) -> CVPixelBuffer? {
        CVPixelBufferLockBaseAddress(self, .readOnly)
        defer { CVPixelBufferUnlockBaseAddress(self, .readOnly) }

        guard let baseAddress = CVPixelBufferGetBaseAddress(self) else {
            return nil
        }

        let inputImageRowBytes = CVPixelBufferGetBytesPerRow(self)

        let imageChannels = 4
        let startPos = Int(rect.origin.y) * inputImageRowBytes + imageChannels * Int(rect.origin.x)
        let outWidth = UInt(rect.width)
        let outHeight = UInt(rect.height)
        let croppedImageRowBytes = Int(outWidth) * imageChannels

        var inBuffer = vImage_Buffer()
        inBuffer.height = outHeight
        inBuffer.width = outWidth
        inBuffer.rowBytes = inputImageRowBytes

        inBuffer.data = baseAddress + UnsafeMutableRawPointer.Stride(startPos)

        guard let croppedImageBytes = malloc(Int(outHeight) * croppedImageRowBytes) else {
            return nil
        }

        var outBuffer = vImage_Buffer(data: croppedImageBytes, height: outHeight, width: outWidth, rowBytes: croppedImageRowBytes)

        let scaleError = vImageScale_ARGB8888(&inBuffer, &outBuffer, nil, vImage_Flags(0))

        guard scaleError == kvImageNoError else {
            free(croppedImageBytes)
            return nil
        }

        return croppedImageBytes.toCVPixelBuffer(pixelBuffer: self, targetWith: Int(outWidth), targetHeight: Int(outHeight), targetImageRowBytes: croppedImageRowBytes)
    }
    
    func flip() -> CVPixelBuffer? {
        CVPixelBufferLockBaseAddress(self, .readOnly)
        defer { CVPixelBufferUnlockBaseAddress(self, .readOnly) }

        guard let baseAddress = CVPixelBufferGetBaseAddress(self) else {
            return nil
        }
        
        let width = UInt(CVPixelBufferGetWidth(self))
        let height = UInt(CVPixelBufferGetHeight(self))
        let inputImageRowBytes = CVPixelBufferGetBytesPerRow(self)
        let outputImageRowBytes = inputImageRowBytes
        
        var inBuffer = vImage_Buffer(
            data: baseAddress,
            height: height,
            width: width,
            rowBytes: inputImageRowBytes)
        
        guard let targetImageBytes = malloc(Int(height) * outputImageRowBytes) else {
            return nil
        }
        var outBuffer = vImage_Buffer(data: targetImageBytes, height: height, width: width, rowBytes: outputImageRowBytes)
        
        // See https://developer.apple.com/documentation/accelerate/vimage/vimage_operations/image_reflection for other transformations
        let reflectError = vImageHorizontalReflect_ARGB8888(&inBuffer, &outBuffer, vImage_Flags(0))
        // let reflectError = vImageVerticalReflect_ARGB8888(&inBuffer, &outBuffer, vImage_Flags(0))
        
        guard reflectError == kvImageNoError else {
            free(targetImageBytes)
            return nil
        }

        return targetImageBytes.toCVPixelBuffer(pixelBuffer: self, targetWith: Int(width), targetHeight: Int(height), targetImageRowBytes: outputImageRowBytes)
    }
}

extension UnsafeMutableRawPointer {
    // Converts the vImage buffer to CVPixelBuffer
    func toCVPixelBuffer(pixelBuffer: CVPixelBuffer, targetWith: Int, targetHeight: Int, targetImageRowBytes: Int) -> CVPixelBuffer? {
        let pixelBufferType = CVPixelBufferGetPixelFormatType(pixelBuffer)
        let releaseCallBack: CVPixelBufferReleaseBytesCallback = {mutablePointer, pointer in
            if let pointer = pointer {
                free(UnsafeMutableRawPointer(mutating: pointer))
            }
        }

        var targetPixelBuffer: CVPixelBuffer?
        let conversionStatus = CVPixelBufferCreateWithBytes(nil, targetWith, targetHeight, pixelBufferType, self, targetImageRowBytes, releaseCallBack, nil, nil, &targetPixelBuffer)

        guard conversionStatus == kCVReturnSuccess else {
            free(self)
            return nil
        }

        return targetPixelBuffer
    }
}

// Change this to your input pixelBuffer
var pixelBuffer: CVPixelBuffer?

// The result would be stored in resultPixelBuffer
let resultPixelBuffer = pixelBuffer?.crop(to: CGRect(x: 50, y: 50, width: 100, height: 100))?.flip()