c++ - 双簧管异步音频提取

Question

我正在尝试构建一个基于 NDK 的 C++ 低延迟音频播放器，它将遇到多个音频的三个操作。

1. 利用 Assets 。
2. 从在线资源流式传输。
3. 从本地设备存储播放。

从 Google 提供的一个双簧管示例中，我向类 NDKExtractor.cpp 添加了另一个函数提取基于 URL 的音频并将其呈现给音频设备，同时从源中读取。

int32_t NDKExtractor::decode(char *file, uint8_t *targetData, AudioProperties targetProperties) {

    LOGD("Using NDK decoder: %s",file);

    // Extract the audio frames
    AMediaExtractor *extractor = AMediaExtractor_new();
//using this method instead of AMediaExtractor_setDataSourceFd() as used for asset files in the rythem game example
    media_status_t amresult = AMediaExtractor_setDataSource(extractor, file);


    if (amresult != AMEDIA_OK) {
        LOGE("Error setting extractor data source, err %d", amresult);
        return 0;
    }
    // Specify our desired output format by creating it from our source
    AMediaFormat *format = AMediaExtractor_getTrackFormat(extractor, 0);

    int32_t sampleRate;
    if (AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_SAMPLE_RATE, &sampleRate)) {
        LOGD("Source sample rate %d", sampleRate);
        if (sampleRate != targetProperties.sampleRate) {
            LOGE("Input (%d) and output (%d) sample rates do not match. "
                 "NDK decoder does not support resampling.",
                 sampleRate,
                 targetProperties.sampleRate);
            return 0;
        }
    } else {
        LOGE("Failed to get sample rate");
        return 0;
    };

    int32_t channelCount;
    if (AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_CHANNEL_COUNT, &channelCount)) {
        LOGD("Got channel count %d", channelCount);
        if (channelCount != targetProperties.channelCount) {
            LOGE("NDK decoder does not support different "
                 "input (%d) and output (%d) channel counts",
                 channelCount,
                 targetProperties.channelCount);
        }
    } else {
        LOGE("Failed to get channel count");
        return 0;
    }

    const char *formatStr = AMediaFormat_toString(format);
    LOGD("Output format %s", formatStr);

    const char *mimeType;
    if (AMediaFormat_getString(format, AMEDIAFORMAT_KEY_MIME, &mimeType)) {
        LOGD("Got mime type %s", mimeType);
    } else {
        LOGE("Failed to get mime type");
        return 0;
    }

    // Obtain the correct decoder
    AMediaCodec *codec = nullptr;
    AMediaExtractor_selectTrack(extractor, 0);
    codec = AMediaCodec_createDecoderByType(mimeType);
    AMediaCodec_configure(codec, format, nullptr, nullptr, 0);
    AMediaCodec_start(codec);

    // DECODE

    bool isExtracting = true;
    bool isDecoding = true;
    int32_t bytesWritten = 0;

    while (isExtracting || isDecoding) {

        if (isExtracting) {

            // Obtain the index of the next available input buffer
            ssize_t inputIndex = AMediaCodec_dequeueInputBuffer(codec, 2000);
            //LOGV("Got input buffer %d", inputIndex);

            // The input index acts as a status if its negative
            if (inputIndex < 0) {
                if (inputIndex == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
                    // LOGV("Codec.dequeueInputBuffer try again later");
                } else {
                    LOGE("Codec.dequeueInputBuffer unknown error status");
                }
            } else {

                // Obtain the actual buffer and read the encoded data into it
                size_t inputSize;
                uint8_t *inputBuffer = AMediaCodec_getInputBuffer(codec, inputIndex,
                                                                  &inputSize);
                //LOGV("Sample size is: %d", inputSize);

                ssize_t sampleSize = AMediaExtractor_readSampleData(extractor, inputBuffer,
                                                                    inputSize);
                auto presentationTimeUs = AMediaExtractor_getSampleTime(extractor);

                if (sampleSize > 0) {

                    // Enqueue the encoded data
                    AMediaCodec_queueInputBuffer(codec, inputIndex, 0, sampleSize,
                                                 presentationTimeUs,
                                                 0);
                    AMediaExtractor_advance(extractor);

                } else {
                    LOGD("End of extractor data stream");
                    isExtracting = false;

                    // We need to tell the codec that we've reached the end of the stream
                    AMediaCodec_queueInputBuffer(codec, inputIndex, 0, 0,
                                                 presentationTimeUs,
                                                 AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM);
                }
            }
        }

        if (isDecoding) {
            // Dequeue the decoded data
            AMediaCodecBufferInfo info;
            ssize_t outputIndex = AMediaCodec_dequeueOutputBuffer(codec, &info, 0);

            if (outputIndex >= 0) {

                // Check whether this is set earlier
                if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
                    LOGD("Reached end of decoding stream");
                    isDecoding = false;
                } else {
                    // Valid index, acquire buffer
                    size_t outputSize;
                    uint8_t *outputBuffer = AMediaCodec_getOutputBuffer(codec, outputIndex,
                                                                        &outputSize);

                    /*LOGV("Got output buffer index %d, buffer size: %d, info size: %d writing to pcm index %d",
                         outputIndex,
                         outputSize,
                         info.size,
                         m_writeIndex);*/

                    // copy the data out of the buffer
                    memcpy(targetData + bytesWritten, outputBuffer, info.size);
                    bytesWritten += info.size;
                    AMediaCodec_releaseOutputBuffer(codec, outputIndex, false);
                }

            } else {

                // The outputIndex doubles as a status return if its value is < 0
                switch (outputIndex) {
                    case AMEDIACODEC_INFO_TRY_AGAIN_LATER:
                        LOGD("dequeueOutputBuffer: try again later");
                        break;
                    case AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED:
                        LOGD("dequeueOutputBuffer: output buffers changed");
                        break;
                    case AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED:
                        LOGD("dequeueOutputBuffer: output outputFormat changed");
                        format = AMediaCodec_getOutputFormat(codec);
                        LOGD("outputFormat changed to: %s", AMediaFormat_toString(format));
                        break;
                }
            }
        }
    }

    // Clean up
    AMediaFormat_delete(format);
    AMediaCodec_delete(codec);
    AMediaExtractor_delete(extractor);
    return bytesWritten;
}

现在我面临的问题是这段代码首先提取所有音频数据并将其保存到缓冲区中，然后缓冲区成为我从同一示例中的 DataSource 类派生的 AFileDataSource 的一部分。 在完成提取整个文件后，它通过调用 Oboe AudioStreamBuilder 的 onAudioReady() 来播放。 我需要的是在流式传输音频缓冲区 block 时播放。

可选查询:除了问题之外，即使我创建了一个前台服务来与 NDK 函数通信以执行此代码，它也会阻止 UI。对此有什么想法吗？

Answer 1

您可能已经解决了这个问题，但对于 future 的读者... 您需要一个 FIFO 缓冲区来存储解码后的音频。您可以使用双簧管的 FIFO 缓冲区，例如双簧管::FifoBuffer。 您可以为缓冲区和状态机设置低/高水位线，以便在缓冲区几乎为空时开始解码，在缓冲区满时停止解码(您会找出所需的其他状态)。 作为旁注，我实现了这样的播放器只是在稍后发现，AAC 编解码器在某些设备上被破坏了(我想到了小米和亚马逊)，所以我不得不扔掉 AMediaCodec/AMediaExtractor 部分并使用 AAC取而代之的是图书馆。