我正在学习如何从 this 中创建 MP4 视频示例。问题是该示例演示了动态生成的一些虚拟源数据的音频编码。我需要对文件中的音频进行编码。我检查了很多示例，其中大多数显示相同或只是单独的音频编码。 在我的试错过程中，我对音频和视频帧使用相同的 AVFormatContext。我不确定这样做是否正确，或者我应该有 2 个单独的上下文？到目前为止，我的视频编码正常，但音频流失败因为 AVPacket 无法找到正确的音频流索引。 这是我设置音频流的方式:
void open_audio(AVFormatContext *oc, AVCodec **codec, AVStream **st ,enum AVCodecID codec_id){
// AVCodecContext *c;
int ret;
// c = st->codec;
*codec = avcodec_find_encoder(codec_id);
if (!(*codec)) {
fprintf(stderr, "Could not find encoder for '%s'\n",avcodec_get_name(codec_id));
}
/* open it */
if(avformat_open_input(&oc,_audioInName.c_str(),NULL,NULL) !=0){
Msg::PrintErrorMsg("Error opening audio file");
}
AVStream* audioStream = NULL;
// Find the audio stream (some container files can have multiple streams in them)
for (uint32_t i = 0; i < oc->nb_streams; ++i)
{
if (oc->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
{
audioStream = oc->streams[i];
break;
}
}
if (audioStream == NULL)
{
Msg::PrintErrorMsg("Could not find any audio stream in the file");
}
*st =audioStream;
AVCodecContext *c = audioStream->codec;
c->codec = *codec;//avcodec_find_decoder(c->codec_id);
audioStream->id = 1;
c->sample_fmt = AV_SAMPLE_FMT_S16;
c->bit_rate = 64000;
c->sample_rate = 44100;
c->channels = 1;
if (oc->oformat->flags & AVFMT_GLOBALHEADER){
c->flags |= CODEC_FLAG_GLOBAL_HEADER;
}
if (c->codec == NULL)
{
Msg::PrintErrorMsg("Couldn't find a proper decoder");
}
ret = avcodec_open2(c, *codec, NULL);
if (ret < 0) {
Msg::PrintErrorMsg("Could not open audio codec\n");
}
}
这里的“oc”也是用于初始化视频流的上下文。
然后我尝试像这样编写音频帧:
void write_audio_frame(AVFormatContext *oc, AVStream *st){
AVCodecContext *c;
AVPacket pkt = { 0 }; // data and size must be 0;
AVFrame *frame = avcodec_alloc_frame();
int got_packet, ret;
av_init_packet(&pkt);
c = st->codec;
/////
// get_audio_frame(samples, audio_input_frame_size, c->channels);
////Read the packet:
while(av_read_frame(oc,&pkt) == 0 ){
if(pkt.stream_index ==st->index){
// Try to decode the packet into a frame
int frameFinished = 0;
avcodec_decode_audio4(c, frame, &frameFinished, &pkt);
// Some frames rely on multiple packets, so we have to make sure the frame is finished before
// we can use it
if (frameFinished){
assert(frameFinished);
ret = avcodec_encode_audio2(c, &pkt, frame, &got_packet);
if (ret < 0) {
Msg::PrintErrorMsg("Error encoding audio frame\n");
}
if (!got_packet){
printf("failed to aquire packet");
}
pkt.stream_index = st->index;
/* Write the compressed frame to the media file. */
ret = av_interleaved_write_frame(oc, &pkt);
if (ret != 0) {
Msg::PrintErrorMsg("Error while writing audio frame.");
}
}
}
}
}
av_free_packet(&pkt);
avcodec_free_frame(&frame);
}
问题是我从来没有通过这个声明:“if(pkt.stream_index ==st->index)”。数据包流索引永远不会等于音频流索引。任何人都可以指出我错在哪里？
更新:
我确实设法打开了用于编码的输入音频流，但我无法将音频和视频流编码为单个输出。据我所知，PTS 和 DTS 可能是问题的根源。目前，我基于 muxing 计算 pts。 c 示例，但它根本不适用于音频。
下面是我的使用方法:
while(frame_count < _streamDurationNBFrames-1){
uint8_t *frameToWrite =_frames.front();
// Compute current audio and video time. ///
if (audio_st){
audio_pts = (double)audioIn_st->pts.val * audioIn_st->time_base.num / audioIn_st->time_base.den;
}
else{
audio_pts = 0.0;
}
if (video_st){
video_pts = (double)video_st->pts.val * video_st->time_base.num / video_st->time_base.den;
}else{
video_pts = 0.0;
}
if ((!audio_st || audio_pts >= _streamDuration) && (!video_st || video_pts >= _streamDuration)){
break;
}
if (audio_st && audio_pts < video_pts) {
av_read_frame(informat, &pkt);//read audio from input stream
Msg::PrintMsg("Encode audio here...");
//================== AUDIO ENCODE HERE
outpkt.data = pkt.data;
outpkt.size = pkt.size;
outpkt.stream_index = pkt.stream_index;
outpkt.flags |= AV_PKT_FLAG_KEY;
outpkt.pts = pkt.pts;
outpkt.dts =pkt.dts;
if(av_interleaved_write_frame(oc, &outpkt) < 0)
{
Msg::PrintErrorMsg("Fail Audio Write ");
}
else
{
audio_st->codec->frame_number++;
}
av_free_packet(&outpkt);
av_free_packet(&pkt);
}else{
//================== VIDEO ENCODE HERE
write_video_frame(oc, video_st,frameToWrite);
frame->pts += av_rescale_q(1, video_st->codec->time_base, video_st->time_base);
}
///at last delete this frame:
_frames.pop();
delete frameToWrite; ///deallocate the written frame!
}
不知何故，一旦我进入音频编码循环，audio_pts 永远不会到达 video_pts 并且始终为零:
audio_pts = (double)audio_st->pts.val * audio_st->time_base.num / audio_st->time_base.den; is always zero because (double)audio_st->pts.val returns zero.
所以基本上我又问了同样的问题:当音频来自外部文件时如何进行混流？
顺便说一句，下面的答案没有帮助，因为它假定音频和视频流都来自同一个文件，而在我的例子中只有音频来自外部源。
最佳答案
您可以使您拥有相同的上下文而不需要两个单独的上下文。如果您同时对视频和音频进行编码。然后您首先需要创建视频流，然后是音频流。但是，如果您只想对音频进行编码，那么您将只需要创建音频流。 if(pkt.stream_index ==st->index) 通常在您进行转码时需要，即当您更改容器格式时。您可以在其中从视频文件中读取帧并写入另一个文件，因此您需要知道帧是来自音频流还是视频流。 但是，如果您正在获取解码的音频数据包，那么您需要在执行 av_interleaved_write 之前在音频数据包中设置正确的流索引。
在您的代码中，您没有设置正确编码所需的音频数据包的 pts 和 dts。
前段时间我也写过一个和你类似的程序，你可以研究一下，以供引用。
int VideoClipper::Init(const wxString& filename)
{
int ret = 0;
char errbuf[64];
av_register_all();
if ((ret = avformat_open_input( &m_informat, filename.mb_str(), 0, 0)) != 0 )
{
av_strerror(ret,errbuf,sizeof(errbuf));
PRINT_VAL("Not able to Open file;; ", errbuf)
ret = -1;
return ret;
}
else
{
PRINT_MSG("Opened File ")
}
if ((ret = avformat_find_stream_info(m_informat, 0))< 0 )
{
av_strerror(ret,errbuf,sizeof(errbuf));
PRINT_VAL("Not Able to find stream info:: ", errbuf)
ret = -1;
return ret;
}
else
{
PRINT_MSG("Got stream Info ")
}
for(unsigned int i = 0; i<m_informat->nb_streams; i++)
{
if(m_informat->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
{
PRINT_MSG("Found Video Stream ")
m_in_vid_strm_idx = i;
m_in_vid_strm = m_informat->streams[i];
}
if(m_informat->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
{
PRINT_MSG("Found Audio Stream ")
m_in_aud_strm_idx = i;
m_in_aud_strm = m_informat->streams[i];
}
}
if(m_in_aud_strm_idx == -1 && m_in_vid_strm_idx == -1)
{
ret = -1;
}
if(m_informat->duration == AV_NOPTS_VALUE)
{
if(m_in_vid_strm_idx != -1 && m_informat->streams[m_in_vid_strm_idx])
{
if(m_informat->streams[m_in_vid_strm_idx]->duration != AV_NOPTS_VALUE)
{
//m_in_end_time = (m_informat->streams[m_in_vid_strm_idx]->duration)/(AV_TIME_BASE);
m_in_end_time = (m_informat->streams[m_in_vid_strm_idx]->duration)/(m_informat->streams[m_in_vid_strm_idx]->time_base.den/m_informat->streams[m_in_vid_strm_idx]->time_base.num);
}
}
else if(m_in_aud_strm_idx != -1 && m_informat->streams[m_in_aud_strm_idx])
{
if(m_informat->streams[m_in_aud_strm_idx]->duration != AV_NOPTS_VALUE)
{
m_in_end_time = (m_informat->streams[m_in_aud_strm_idx]->duration)/(AV_TIME_BASE);
}
}
}
else
{
m_in_end_time = (m_informat->duration)/(AV_TIME_BASE);
}
if(m_in_vid_strm_idx != -1 && m_informat->streams[m_in_vid_strm_idx])
{
if(m_informat->streams[m_in_vid_strm_idx]->r_frame_rate.num != AV_NOPTS_VALUE && m_informat->streams[m_in_vid_strm_idx]->r_frame_rate.den != 0)
{
m_fps = (m_informat->streams[m_in_vid_strm_idx]->r_frame_rate.num)/ (m_informat->streams[m_in_vid_strm_idx]->r_frame_rate.den);
}
}
else
{
m_fps = 25;
}
AVOutputFormat *outfmt = NULL;
std::string outfile = std::string(filename) + "clip_out.avi";
outfmt = av_guess_format(NULL,outfile.c_str(),NULL);
if(outfmt == NULL)
{
ret = -1;
return ret;
}
else
{
m_outformat = avformat_alloc_context();
if(m_outformat)
{
m_outformat->oformat = outfmt;
_snprintf(m_outformat->filename, sizeof(m_outformat->filename), "%s", outfile.c_str());
}
else
{
ret = -1;
return ret;
}
}
AVCodec *out_vid_codec,*out_aud_codec;
out_vid_codec = out_aud_codec = NULL;
if(outfmt->video_codec != AV_CODEC_ID_NONE && m_in_vid_strm != NULL)
{
out_vid_codec = avcodec_find_encoder(outfmt->video_codec);
if(NULL == out_vid_codec)
{
PRINT_MSG("Could Not Find Vid Encoder")
ret = -1;
return ret;
}
else
{
PRINT_MSG("Found Out Vid Encoder ")
m_out_vid_strm = avformat_new_stream(m_outformat, out_vid_codec);
if(NULL == m_out_vid_strm)
{
PRINT_MSG("Failed to Allocate Output Vid Strm ")
ret = -1;
return ret;
}
else
{
PRINT_MSG("Allocated Video Stream ")
if(avcodec_copy_context(m_out_vid_strm->codec, m_informat->streams[m_in_vid_strm_idx]->codec) != 0)
{
PRINT_MSG("Failed to Copy Context ")
ret = -1;
return ret;
}
else
{
m_out_vid_strm->sample_aspect_ratio.den = m_out_vid_strm->codec->sample_aspect_ratio.den;
m_out_vid_strm->sample_aspect_ratio.num = m_in_vid_strm->codec->sample_aspect_ratio.num;
PRINT_MSG("Copied Context ")
m_out_vid_strm->codec->codec_id = m_in_vid_strm->codec->codec_id;
m_out_vid_strm->codec->time_base.num = 1;
m_out_vid_strm->codec->time_base.den = m_fps*(m_in_vid_strm->codec->ticks_per_frame);
m_out_vid_strm->time_base.num = 1;
m_out_vid_strm->time_base.den = 1000;
m_out_vid_strm->r_frame_rate.num = m_fps;
m_out_vid_strm->r_frame_rate.den = 1;
m_out_vid_strm->avg_frame_rate.den = 1;
m_out_vid_strm->avg_frame_rate.num = m_fps;
m_out_vid_strm->duration = (m_out_end_time - m_out_start_time)*1000;
}
}
}
}
if(outfmt->audio_codec != AV_CODEC_ID_NONE && m_in_aud_strm != NULL)
{
out_aud_codec = avcodec_find_encoder(outfmt->audio_codec);
if(NULL == out_aud_codec)
{
PRINT_MSG("Could Not Find Out Aud Encoder ")
ret = -1;
return ret;
}
else
{
PRINT_MSG("Found Out Aud Encoder ")
m_out_aud_strm = avformat_new_stream(m_outformat, out_aud_codec);
if(NULL == m_out_aud_strm)
{
PRINT_MSG("Failed to Allocate Out Vid Strm ")
ret = -1;
return ret;
}
else
{
if(avcodec_copy_context(m_out_aud_strm->codec, m_informat->streams[m_in_aud_strm_idx]->codec) != 0)
{
PRINT_MSG("Failed to Copy Context ")
ret = -1;
return ret;
}
else
{
PRINT_MSG("Copied Context ")
m_out_aud_strm->codec->codec_id = m_in_aud_strm->codec->codec_id;
m_out_aud_strm->codec->codec_tag = 0;
m_out_aud_strm->pts = m_in_aud_strm->pts;
m_out_aud_strm->duration = m_in_aud_strm->duration;
m_out_aud_strm->time_base.num = m_in_aud_strm->time_base.num;
m_out_aud_strm->time_base.den = m_in_aud_strm->time_base.den;
}
}
}
}
if (!(outfmt->flags & AVFMT_NOFILE))
{
if (avio_open2(&m_outformat->pb, outfile.c_str(), AVIO_FLAG_WRITE,NULL, NULL) < 0)
{
PRINT_VAL("Could Not Open File ", outfile)
ret = -1;
return ret;
}
}
/* Write the stream header, if any. */
if (avformat_write_header(m_outformat, NULL) < 0)
{
PRINT_VAL("Error Occurred While Writing Header ", outfile)
ret = -1;
return ret;
}
else
{
PRINT_MSG("Written Output header ")
m_init_done = true;
}
return ret;
}
int VideoClipper::GenerateClip(void)
{
AVPacket pkt, outpkt;
int aud_pts = 0, vid_pts = 0, aud_dts = 0, vid_dts = 0;
int last_vid_pts = 0;
if(m_good_clip)
{
SeekFrame();
while(av_read_frame(m_informat, &pkt) >= 0 && (m_num_frames-- > 0))
{
if(pkt.stream_index == m_in_vid_strm_idx)
{
PRINT_VAL("ACTUAL VID Pkt PTS ",av_rescale_q(pkt.pts,m_in_vid_strm->time_base, m_in_vid_strm->codec->time_base))
PRINT_VAL("ACTUAL VID Pkt DTS ", av_rescale_q(pkt.dts, m_in_vid_strm->time_base, m_in_vid_strm->codec->time_base ))
av_init_packet(&outpkt);
if(pkt.pts != AV_NOPTS_VALUE)
{
if(last_vid_pts == vid_pts)
{
vid_pts++;
last_vid_pts = vid_pts;
}
outpkt.pts = vid_pts;
PRINT_VAL("ReScaled VID Pts ", outpkt.pts)
}
else
{
outpkt.pts = AV_NOPTS_VALUE;
}
if(pkt.dts == AV_NOPTS_VALUE)
{
outpkt.dts = AV_NOPTS_VALUE;
}
else
{
outpkt.dts = vid_pts;
PRINT_VAL("ReScaled VID Dts ", outpkt.dts)
PRINT_MSG("=======================================")
}
outpkt.data = pkt.data;
outpkt.size = pkt.size;
outpkt.stream_index = pkt.stream_index;
outpkt.flags |= AV_PKT_FLAG_KEY;
last_vid_pts = vid_pts;
if(av_interleaved_write_frame(m_outformat, &outpkt) < 0)
{
PRINT_MSG("Failed Video Write ")
}
else
{
m_out_vid_strm->codec->frame_number++;
}
av_free_packet(&outpkt);
av_free_packet(&pkt);
}
else if(pkt.stream_index == m_in_aud_strm_idx)
{
PRINT_VAL("ACTUAL AUD Pkt PTS ", av_rescale_q(pkt.pts, m_in_aud_strm->time_base, m_in_aud_strm->codec->time_base))
PRINT_VAL("ACTUAL AUD Pkt DTS ", av_rescale_q(pkt.dts, m_in_aud_strm->time_base, m_in_aud_strm->codec->time_base))
//num_aud_pkt++;
av_init_packet(&outpkt);
if(pkt.pts != AV_NOPTS_VALUE)
{
outpkt.pts = aud_pts;
PRINT_VAL("ReScaled AUD PTS ", outpkt.pts)
}
else
{
outpkt.pts = AV_NOPTS_VALUE;
}
if(pkt.dts == AV_NOPTS_VALUE)
{
outpkt.dts = AV_NOPTS_VALUE;
}
else
{
outpkt.dts = aud_pts;
PRINT_VAL("ReScaled AUD DTS ", outpkt.dts)
PRINT_MSG("====================================")
if( outpkt.pts >= outpkt.dts)
{
outpkt.dts = outpkt.pts;
}
if(outpkt.dts == aud_dts)
{
outpkt.dts++;
}
if(outpkt.pts < outpkt.dts)
{
outpkt.pts = outpkt.dts;
aud_pts = outpkt.pts;
}
}
outpkt.data = pkt.data;
outpkt.size = pkt.size;
outpkt.stream_index = pkt.stream_index;
outpkt.flags |= AV_PKT_FLAG_KEY;
vid_pts = aud_pts;
aud_pts++;
if(av_interleaved_write_frame(m_outformat, &outpkt) < 0)
{
PRINT_MSG("Faile Audio Write ")
}
else
{
m_out_aud_strm->codec->frame_number++;
}
av_free_packet(&outpkt);
av_free_packet(&pkt);
}
else
{
PRINT_MSG("Got Unknown Pkt ")
//num_unkwn_pkt++;
}
//num_total_pkt++;
}
av_write_trailer(m_outformat);
av_free_packet(&outpkt);
av_free_packet(&pkt);
return 0;
}
return -1;
}
关于c++ - 使用 FFmpeg 从音频和视频文件中混合，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/16768794/