java - JNI YUV_420_888 到 RGBA_8888 的转换

Question

目前我正在构建一个应用程序来进行实时图像处理然后显示。第一步是尝试使用 Camera2 API 和 ANativeWindow API 显示原始预览。我分别通过 JNI 传递 y、u、v channel ，并按照 Wikipedia article 进行 YUV2RGB 转换。 ，但在 Google Pixel - 7.1.0 - API 25 - 1080x1920 和 Genymotion 上运行时颜色输出错误:

ImageReader.OnImageAvailableListener 的实现:

private ImageReader.OnImageAvailableListener mOnImageAvailableListener = new ImageReader.OnImageAvailableListener() {

    @Override
    public void onImageAvailable(ImageReader reader) {
        // get the newest frame
        Image image = reader.acquireNextImage();

        if (image == null) {
            return;
        }

        Image.Plane Y_plane = image.getPlanes()[0];
        int Y_rowStride = Y_plane.getRowStride();
        Image.Plane U_plane = image.getPlanes()[1];
        int U_rowStride = U_plane.getRowStride();
        Image.Plane V_plane = image.getPlanes()[2];
        int V_rowStride = V_plane.getRowStride();
        JNIUtils.RGBADisplay(image.getWidth(), image.getHeight(), Y_rowStride, Y_plane.getBuffer(), U_rowStride, U_plane.getBuffer(), V_rowStride, V_plane.getBuffer(), surface);
        image.close();
    }
};

JNI:

public static native void RGBADisplay(int srcWidth, int srcHeight, int Y_rowStride, ByteBuffer Y_Buffer, int U_rowStride, ByteBuffer U_Buffer, int V_rowStride, ByteBuffer V_Buffer, Surface surface);

C++:

const uint8_t NUM_128 = 128;
const uint8_t NUM_255 = 255;

JNIEXPORT void JNICALL Java_tau_camera2demo_JNIUtils_RGBADisplay(
        JNIEnv *env,
        jobject obj,
        jint srcWidth,
        jint srcHeight,
        jint Y_rowStride,
        jobject Y_Buffer,
        jint U_rowStride,
        jobject U_Buffer,
        jint V_rowStride,
        jobject V_Buffer,
        jobject surface) {


    uint8_t *srcYPtr = reinterpret_cast<uint8_t *>(env->GetDirectBufferAddress(Y_Buffer));
    uint8_t *srcUPtr = reinterpret_cast<uint8_t *>(env->GetDirectBufferAddress(U_Buffer));
    uint8_t *srcVPtr = reinterpret_cast<uint8_t *>(env->GetDirectBufferAddress(V_Buffer));

    ANativeWindow * window = ANativeWindow_fromSurface(env, surface);
    ANativeWindow_acquire(window);
    ANativeWindow_Buffer buffer;

    //set output size and format
    //only 3 formats are available:
    //WINDOW_FORMAT_RGBA_8888(DEFAULT), WINDOW_FORMAT_RGBX_8888, WINDOW_FORMAT_RGB_565
    ANativeWindow_setBuffersGeometry(window, 0, 0, WINDOW_FORMAT_RGBA_8888);
    if (int32_t err = ANativeWindow_lock(window, &buffer, NULL)) {
        LOGE("ANativeWindow_lock failed with error code: %d\n", err);
        ANativeWindow_release(window);
    }

    //convert YUV_420_888 to RGBA_8888 and display
    uint8_t * outPtr = reinterpret_cast<uint8_t *>(buffer.bits);
    for (size_t y = 0; y < srcHeight; y++)
    {
        uint8_t * Y_rowPtr = srcYPtr + y * Y_rowStride;
        uint8_t * U_rowPtr = srcUPtr + (y >> 1) * U_rowStride;
        uint8_t * V_rowPtr = srcVPtr + (y >> 1) * V_rowStride;
        for (size_t x = 0; x < srcWidth; x++)
        {
            //from Wikipedia article YUV:
            //Integer operation of ITU-R standard for YCbCr(8 bits per channel) to RGB888
            //Y-Y, U-Cb, V-Cr
            //R = Y + V + (V >> 2) + (V >> 3) + (V >> 5);
            //G = Y - ((U >> 2) + (U >> 4) + (U >> 5)) - ((V >> 1) + (V >> 3) + (V >> 4) + (V >> 5));
            //B = Y + U + (U >> 1) + (U >> 2) + (U >> 6);
            uint8_t Y = Y_rowPtr[x];
            uint8_t U = U_rowPtr[(x >> 1)] - NUM_128;
            uint8_t V = V_rowPtr[(x >> 1)] - NUM_128;
            *(outPtr++) = Y + V + (V >> 2) + (V >> 3) + (V >> 5); //R
            *(outPtr++) = Y - ((U >> 2) + (U >> 4) + (U >> 5)) - ((V >> 1) + (V >> 3) + (V >> 4) + (V >> 5)); //G
            *(outPtr++) = Y + U + (U >> 1) + (U >> 2) + (U >> 6); //B
            *(outPtr++) = NUM_255; // gamma for RGBA_8888
        }
    }

    ANativeWindow_unlockAndPost(window);
    ANativeWindow_release(window);
}

整个演示可以在 Github 上找到:https://github.com/Fung-yuantao/android-camera2demo

更新:

在调用 JNIUtils.RGBADisplay 行后添加了以下代码:

        Log.d(TAG, "Y plane pixel stride: " + Y_plane.getPixelStride());
        Log.d(TAG, "U plane pixel stride: " + U_plane.getPixelStride());
        Log.d(TAG, "V plane pixel stride: " + V_plane.getPixelStride());

在 Logcat 中:

09-07 06:40:02.576 5376-5392/tau.camera2demo D/Camera2Demo: Y plane pixel stride: 1
09-07 06:40:02.576 5376-5392/tau.camera2demo D/Camera2Demo: U plane pixel stride: 1
09-07 06:40:02.576 5376-5392/tau.camera2demo D/Camera2Demo: V plane pixel stride: 1

根据alijandro的回答，图像格式应该是平面的。

Answer 1

YUV_420_888 的图像输出格式可能是平面(I420、YV12)或半平面(NV12、NV21)格式，来自文档 here .

那么如何知道它是平面格式还是半平面格式呢？

我猜你可以通过 image.getPlanes()[1].getPixelStride() 找到。如果是2，则图像格式为半平面格式，具有以下位模式。

YYYYYYYY UVUVUVUV ...

在我的测试环境中，ImageReader 的输出图像格式是半平面的。

对于半平面，我们只需要处理前两个平面。

像下面这样更改您的代码。

ANativeWindow_setBuffersGeometry(window, srcWidth, srcHeight, WINDOW_FORMAT_RGBA_8888);
if (int32_t err = ANativeWindow_lock(window, &buffer, NULL)) {
    LOGE("ANativeWindow_lock failed with error code: %d\n", err);
    ANativeWindow_release(window);
}

//convert YUV_420_888 to RGBA_888 and display
uint8_t * outPtr = reinterpret_cast<uint8_t *>(buffer.bits);
for (size_t y = 0; y < srcHeight; y++)
{
    uint8_t * Y_rowPtr = srcYPtr + y * Y_rowStride;
    uint8_t * UV_rowPtr = srcUPtr + (y >> 1) * Y_rowStride;
    // uint8_t * V_rowPtr = srcVPtr + (y >> 1) * Y_rowStride / 4;
    for (size_t x = 0; x < srcWidth; x++)
    {
        uint8_t Y = Y_rowPtr[x];
        size_t uIndex = x & 0xfffffffe;
        uint8_t U = UV_rowPtr[uIndex];
        uint8_t V = UV_rowPtr[uIndex + 1];
        double R = ((Y-16) * 1.164 + (V-128) * 1.596);
        double G = ((Y-16) * 1.164 - (U-128) * 0.392 - (V-128) * 0.813);
        double B = ((Y-16) * 1.164 + (U-128) * 2.017);
        *(outPtr++) = (uint8_t) (R > 255 ? 255 : (R < 0 ? 0 : R));
        *(outPtr++) = (uint8_t) (G > 255 ? 255 : (G < 0 ? 0 : G));
        *(outPtr++) = (uint8_t) (B > 255 ? 255 : (B < 0 ? 0 : B));
        *(outPtr++) = NUM_255; // gamma for RGBA_8888
    }
}

对于平面输出格式，请尝试使用下面的 YUV2RGB 转换方法。

for (size_t y = 0; y < srcHeight; y++)
{
    uint8_t * Y_rowPtr = srcYPtr + y * Y_rowStride;
    uint8_t * U_rowPtr = srcUPtr + (y >> 1) * Y_rowStride / 2;
    uint8_t * V_rowPtr = srcVPtr + (y >> 1) * Y_rowStride / 2;
    for (size_t x = 0; x < srcWidth; x++)
    {
        uint8_t Y = Y_rowPtr[x];
        uint8_t U = U_rowPtr[(x >> 1)];
        uint8_t V = V_rowPtr[(x >> 1)];
        double R = ((Y-16) * 1.164 + (V-128) * 1.596);
        double G = ((Y-16) * 1.164 - (U-128) * 0.392 - (V-128) * 0.813);
        double B = ((Y-16) * 1.164 + (U-128) * 2.017);
        *(outPtr++) = (uint8_t) (R > 255 ? 255 : (R < 0 ? 0 : R));
        *(outPtr++) = (uint8_t) (G > 255 ? 255 : (G < 0 ? 0 : G));
        *(outPtr++) = (uint8_t) (B > 255 ? 255 : (B < 0 ? 0 : B));
        *(outPtr++) = NUM_255; // gamma for RGBA_8888
    }
}