我正在使用一个名为 NextLevel 的媒体捕获库，它会吐出一个 CMSampleBuffer在每一帧上。我想获取这个缓冲区并通过 rawDataInput 将它提供给 GPUImage2并通过一些过滤器并从 rawDataOutput 读取它在链的末端...
CMSampleBuffer bytes -> rawDataInput -> someFilter -> someotherFilter -> rawDataOutput -> 为其他东西制作一个 CVPixelBuffer 。
问题是，如何将一个 CMSampleBuffer 转换为 UInt8 的数组
以便 rawDataInput 可以接收它。
我有以下代码，但它的速度非常慢......框架一直穿过链条并到达 rawDataOuput . dataAvailableCallback但慢到每秒 1 帧。我在网上找到了这段代码，不知道它在数学上做了什么，但我猜它效率低下。
let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)!
CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
let lumaBaseAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)
let chromaBaseAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1)
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
let lumaBytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0)
let chromaBytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1)
let lumaBuffer = lumaBaseAddress?.assumingMemoryBound(to: UInt8.self)
let chromaBuffer = chromaBaseAddress?.assumingMemoryBound(to: UInt8.self)
var rgbaImage = [UInt8](repeating: 0, count: 4*width*height)
for x in 0 ..< width {
for y in 0 ..< height {
let lumaIndex = x+y*lumaBytesPerRow
let chromaIndex = (y/2)*chromaBytesPerRow+(x/2)*2
let yp = lumaBuffer?[lumaIndex]
let cb = chromaBuffer?[chromaIndex]
let cr = chromaBuffer?[chromaIndex+1]
let ri = Double(yp!) + 1.402 * (Double(cr!) - 128)
let gi = Double(yp!) - 0.34414 * (Double(cb!) - 128) - 0.71414 * (Double(cr!) - 128)
let bi = Double(yp!) + 1.772 * (Double(cb!) - 128)
let r = UInt8(min(max(ri,0), 255))
let g = UInt8(min(max(gi,0), 255))
let b = UInt8(min(max(bi,0), 255))
rgbaImage[(x + y * width) * 4] = b
rgbaImage[(x + y * width) * 4 + 1] = g
rgbaImage[(x + y * width) * 4 + 2] = r
rgbaImage[(x + y * width) * 4 + 3] = 255
}
}
self.rawInput.uploadBytes(rgbaImage, size: Size.init(width: Float(width), height: Float(height)), pixelFormat: PixelFormat.rgba)
CVPixelBufferUnlockBaseAddress( pixelBuffer, CVPixelBufferLockFlags(rawValue: 0) );
更新 1
我使用名为 NextLevel 的相机库来检索相机帧 (CMSampleBuffer) 并将它们提供给过滤器链，在这种情况下，RawDataInput 通过 UInt8 字节数组。因为 NextLevel 尽可能使用亮度/色度，我在 https://github.com/NextLevel/NextLevel/blob/master/Sources/NextLevel.swift#L1106 中注释了 5 行正如@rythmic 鱼人评论的那样。但是上面的代码会中断，所以我用以下代码替换了它。
let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)!
CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0));
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
let int8Buffer = CVPixelBufferGetBaseAddress(pixelBuffer)?.assumingMemoryBound(to: UInt8.self)
var rgbaImage = [UInt8](repeating: 0, count: 4*width*height)
for i in 0 ..< (width*height*4){
rgbaImage[i] = UInt8((int8Buffer?[i])!)
}
self.rawInput.uploadBytes(rgbaImage, size: Size.init(width: Float(width), height: Float(height)), pixelFormat: PixelFormat.rgba)
CVPixelBufferUnlockBaseAddress(pixelBuffer,CVPixelBufferLockFlags(rawValue: 0))
当 NextLevel 不使用亮度/色度时，此代码有效，但在使用 GPUImage RenderView 在过滤器链的末尾显示时，帧仍然非常慢。
更新 2
所以我决定根据 GPUImage2 的 Camera.swift 制作一个自定义的 RawDataInput.swift。因为 Camera 类从 CMSampleBuffer 格式的 native 相机中获取帧，我想.. NextLevel 正在抛出完全相同的缓冲区，我可以复制 GPUImage2 Camera 类的实现并删除我不需要的所有内容，只留下 1 个方法接收一个 CMSampleBuffer 并处理它。事实证明它完美无缺。
除了......有一个滞后(没有掉帧，只是滞后)。我不知道瓶颈在哪里，我正在阅读处理/修改来自 native 相机的 CMSampleBuffers 然后显示它们.. 可能会导致延迟，如本问题所述:How to keep low latency during the preview of video coming from AVFoundation?
我制作了一段我遇到的滞后的视频......
https://www.youtube.com/watch?v=5DQRnOTi4wk
顶角预览来自 NextLevel 的“
previewLayer: AVCaptureVideoPreviewLayer” ' 并且过滤后的预览是链末端的 GPUImage2 Renderview.. 在 iPhone 6 中以 1920 像素分辨率和 7 个过滤器运行。 GPUImage2 Camera 类不会发生这种滞后。这是我放在一起的自定义 RawDataInput。
#if os(Linux)
#if GLES
import COpenGLES.gles2
#else
import COpenGL
#endif
#else
#if GLES
import OpenGLES
#else
import OpenGL.GL3
#endif
#endif
import AVFoundation
public enum PixelFormat {
case bgra
case rgba
case rgb
case luminance
func toGL() -> Int32 {
switch self {
case .bgra: return GL_BGRA
case .rgba: return GL_RGBA
case .rgb: return GL_RGB
case .luminance: return GL_LUMINANCE
}
}
}
// TODO: Replace with texture caches where appropriate
public class RawDataInput: ImageSource {
public let targets = TargetContainer()
let frameRenderingSemaphore = DispatchSemaphore(value:1)
let cameraProcessingQueue = DispatchQueue.global(priority:DispatchQueue.GlobalQueuePriority.default)
let captureAsYUV:Bool = true
let yuvConversionShader:ShaderProgram?
var supportsFullYUVRange:Bool = false
public init() {
if captureAsYUV {
supportsFullYUVRange = false
let videoOutput = AVCaptureVideoDataOutput()
let supportedPixelFormats = videoOutput.availableVideoCVPixelFormatTypes
for currentPixelFormat in supportedPixelFormats! {
if ((currentPixelFormat as! NSNumber).int32Value == Int32(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)) {
supportsFullYUVRange = true
}
}
if (supportsFullYUVRange) {
yuvConversionShader = crashOnShaderCompileFailure("Camera"){try sharedImageProcessingContext.programForVertexShader(defaultVertexShaderForInputs(2), fragmentShader:YUVConversionFullRangeFragmentShader)}
} else {
yuvConversionShader = crashOnShaderCompileFailure("Camera"){try sharedImageProcessingContext.programForVertexShader(defaultVertexShaderForInputs(2), fragmentShader:YUVConversionVideoRangeFragmentShader)}
}
} else {
yuvConversionShader = nil
}
}
public func uploadPixelBuffer(_ cameraFrame: CVPixelBuffer ) {
guard (frameRenderingSemaphore.wait(timeout:DispatchTime.now()) == DispatchTimeoutResult.success) else { return }
let bufferWidth = CVPixelBufferGetWidth(cameraFrame)
let bufferHeight = CVPixelBufferGetHeight(cameraFrame)
CVPixelBufferLockBaseAddress(cameraFrame, CVPixelBufferLockFlags(rawValue:CVOptionFlags(0)))
sharedImageProcessingContext.runOperationAsynchronously{
let cameraFramebuffer:Framebuffer
let luminanceFramebuffer:Framebuffer
let chrominanceFramebuffer:Framebuffer
if sharedImageProcessingContext.supportsTextureCaches() {
var luminanceTextureRef:CVOpenGLESTexture? = nil
let _ = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, sharedImageProcessingContext.coreVideoTextureCache, cameraFrame, nil, GLenum(GL_TEXTURE_2D), GL_LUMINANCE, GLsizei(bufferWidth), GLsizei(bufferHeight), GLenum(GL_LUMINANCE), GLenum(GL_UNSIGNED_BYTE), 0, &luminanceTextureRef)
let luminanceTexture = CVOpenGLESTextureGetName(luminanceTextureRef!)
glActiveTexture(GLenum(GL_TEXTURE4))
glBindTexture(GLenum(GL_TEXTURE_2D), luminanceTexture)
glTexParameteri(GLenum(GL_TEXTURE_2D), GLenum(GL_TEXTURE_WRAP_S), GL_CLAMP_TO_EDGE)
glTexParameteri(GLenum(GL_TEXTURE_2D), GLenum(GL_TEXTURE_WRAP_T), GL_CLAMP_TO_EDGE)
luminanceFramebuffer = try! Framebuffer(context:sharedImageProcessingContext, orientation:.portrait, size:GLSize(width:GLint(bufferWidth), height:GLint(bufferHeight)), textureOnly:true, overriddenTexture:luminanceTexture)
var chrominanceTextureRef:CVOpenGLESTexture? = nil
let _ = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, sharedImageProcessingContext.coreVideoTextureCache, cameraFrame, nil, GLenum(GL_TEXTURE_2D), GL_LUMINANCE_ALPHA, GLsizei(bufferWidth / 2), GLsizei(bufferHeight / 2), GLenum(GL_LUMINANCE_ALPHA), GLenum(GL_UNSIGNED_BYTE), 1, &chrominanceTextureRef)
let chrominanceTexture = CVOpenGLESTextureGetName(chrominanceTextureRef!)
glActiveTexture(GLenum(GL_TEXTURE5))
glBindTexture(GLenum(GL_TEXTURE_2D), chrominanceTexture)
glTexParameteri(GLenum(GL_TEXTURE_2D), GLenum(GL_TEXTURE_WRAP_S), GL_CLAMP_TO_EDGE)
glTexParameteri(GLenum(GL_TEXTURE_2D), GLenum(GL_TEXTURE_WRAP_T), GL_CLAMP_TO_EDGE)
chrominanceFramebuffer = try! Framebuffer(context:sharedImageProcessingContext, orientation:.portrait, size:GLSize(width:GLint(bufferWidth / 2), height:GLint(bufferHeight / 2)), textureOnly:true, overriddenTexture:chrominanceTexture)
} else {
glActiveTexture(GLenum(GL_TEXTURE4))
luminanceFramebuffer = sharedImageProcessingContext.framebufferCache.requestFramebufferWithProperties(orientation:.portrait, size:GLSize(width:GLint(bufferWidth), height:GLint(bufferHeight)), textureOnly:true)
luminanceFramebuffer.lock()
glBindTexture(GLenum(GL_TEXTURE_2D), luminanceFramebuffer.texture)
glTexImage2D(GLenum(GL_TEXTURE_2D), 0, GL_LUMINANCE, GLsizei(bufferWidth), GLsizei(bufferHeight), 0, GLenum(GL_LUMINANCE), GLenum(GL_UNSIGNED_BYTE), CVPixelBufferGetBaseAddressOfPlane(cameraFrame, 0))
glActiveTexture(GLenum(GL_TEXTURE5))
chrominanceFramebuffer = sharedImageProcessingContext.framebufferCache.requestFramebufferWithProperties(orientation:.portrait, size:GLSize(width:GLint(bufferWidth / 2), height:GLint(bufferHeight / 2)), textureOnly:true)
chrominanceFramebuffer.lock()
glBindTexture(GLenum(GL_TEXTURE_2D), chrominanceFramebuffer.texture)
glTexImage2D(GLenum(GL_TEXTURE_2D), 0, GL_LUMINANCE_ALPHA, GLsizei(bufferWidth / 2), GLsizei(bufferHeight / 2), 0, GLenum(GL_LUMINANCE_ALPHA), GLenum(GL_UNSIGNED_BYTE), CVPixelBufferGetBaseAddressOfPlane(cameraFrame, 1))
}
cameraFramebuffer = sharedImageProcessingContext.framebufferCache.requestFramebufferWithProperties(orientation:.portrait, size:luminanceFramebuffer.sizeForTargetOrientation(.portrait), textureOnly:false)
let conversionMatrix:Matrix3x3
if (self.supportsFullYUVRange) {
conversionMatrix = colorConversionMatrix601FullRangeDefault
} else {
conversionMatrix = colorConversionMatrix601Default
}
convertYUVToRGB(shader:self.yuvConversionShader!, luminanceFramebuffer:luminanceFramebuffer, chrominanceFramebuffer:chrominanceFramebuffer, resultFramebuffer:cameraFramebuffer, colorConversionMatrix:conversionMatrix)
//ONLY RGBA
//let cameraFramebuffer:Framebuffer = sharedImageProcessingContext.framebufferCache.requestFramebufferWithProperties(orientation:.portrait, size:GLSize(width:GLint(bufferWidth), height:GLint(bufferHeight)), textureOnly:true)
//glBindTexture(GLenum(GL_TEXTURE_2D), cameraFramebuffer.texture)
//glTexImage2D(GLenum(GL_TEXTURE_2D), 0, GL_RGBA, GLsizei(bufferWidth), GLsizei(bufferHeight), 0, GLenum(GL_BGRA), GLenum(GL_UNSIGNED_BYTE), CVPixelBufferGetBaseAddress(cameraFrame))
CVPixelBufferUnlockBaseAddress(cameraFrame, CVPixelBufferLockFlags(rawValue:CVOptionFlags(0)))
self.updateTargetsWithFramebuffer(cameraFramebuffer)
self.frameRenderingSemaphore.signal()
}
}
public func uploadBytes(_ bytes:[UInt8], size:Size, pixelFormat:PixelFormat, orientation:ImageOrientation = .portrait) {
let dataFramebuffer = sharedImageProcessingContext.framebufferCache.requestFramebufferWithProperties(orientation:orientation, size:GLSize(size), textureOnly:true, internalFormat:pixelFormat.toGL(), format:pixelFormat.toGL())
glActiveTexture(GLenum(GL_TEXTURE1))
glBindTexture(GLenum(GL_TEXTURE_2D), dataFramebuffer.texture)
glTexImage2D(GLenum(GL_TEXTURE_2D), 0, GL_RGBA, size.glWidth(), size.glHeight(), 0, GLenum(pixelFormat.toGL()), GLenum(GL_UNSIGNED_BYTE), bytes)
updateTargetsWithFramebuffer(dataFramebuffer)
}
public func transmitPreviousImage(to target:ImageConsumer, atIndex:UInt) {
// TODO: Determine if this is necessary for the raw data uploads
// if let buff = self.dataFramebuffer {
// buff.lock()
// target.newFramebufferAvailable(buff, fromSourceIndex:atIndex)
// }
}
}
我只是不明白为什么会出现这种滞后，如果它与 GPUImage2 Camera 类没有什么不同。 NextLevel 不会对这些帧进行任何其他处理，它只是将它们传递过去，那么为什么会延迟呢？
最佳答案
我遇到了同样的问题，花了太多时间来解决它。终于找到了解决办法。视频帧延迟问题与视频稳定有关。只需使用这一行:NextLevel.shared.videoStabilizationMode = .off它的默认值是 .auto，这就是问题出现的原因。
关于ios - 如何在 GPUImage2 中使用 RawDataInput，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/42921565/