ffmpeg # 转码过程固定帧率时的音画同步策略

本文主要适用于输出为固定帧率的情况。
包括帧率转换但输出帧率是固定帧率的情况。

1.png 2.png

ost->sync_opts

3.png

delta0 & delta

// delta0 is the "drift" between the input frame (next_picture) 
// and where it would fall in the output.
delta0 = sync_ipts - ost->sync_opts;
delta  = delta0 + duration;    
/* by default, we output a single frame */
nb0_frames = 0; // tracks the number of times the PREVIOUS frame should be duplicated, 
                           // mostly for variable framerate (VFR)
nb_frames = 1;

4.png

    // duration的计算
    // 通过原素材的帧率计算。
    frame_rate = av_buffersink_get_frame_rate(filter);
    if (frame_rate.num > 0 && frame_rate.den > 0)
    {
        duration = 1/(av_q2d(frame_rate) * av_q2d(enc->time_base));
    }
    // 通过目标帧率进行计算。
    if(ist && ist->st->start_time != AV_NOPTS_VALUE && ist->st->first_dts != AV_NOPTS_VALUE && ost->frame_rate.num)
    {
        duration = FFMIN(duration, 1/(av_q2d(ost->frame_rate) * av_q2d(enc->time_base)));
    }

   // 通过当前帧的next_picture->pkt_duration进行计算。
    if (!ost->filters_script &&
        !ost->filters &&
        next_picture &&
        ist &&
        lrintf(next_picture->pkt_duration * av_q2d(ist->st->time_base) / av_q2d(enc->time_base)) > 0) {
        duration = lrintf(next_picture->pkt_duration * av_q2d(ist->st->time_base) / av_q2d(enc->time_base));
    }

nb0_frames & nb_frames

5.png

调整

6.png

// 进行调整
if (delta0 < 0 &&
            delta > 0 &&
            format_video_sync != VSYNC_PASSTHROUGH &&
            format_video_sync != VSYNC_DROP) {
            if (delta0 < -0.6) {
                av_log(NULL, AV_LOG_VERBOSE, "Past duration %f too large\n", -delta0);
            } else
                av_log(NULL, AV_LOG_ERROR, "Clipping frame in rate conversion by %f\n", -delta0);
            sync_ipts = ost->sync_opts;
            duration += delta0;
            delta0 = 0;
        }

计算nb_frames和nb0_frames

       // 计算nb_frames和nb0_frames
        switch (format_video_sync) {
        case VSYNC_VSCFR:
            if (ost->frame_number == 0 && delta0 >= 0.5) {
                av_log(NULL, AV_LOG_DEBUG, "Not duplicating %d initial frames\n", (int)lrintf(delta0));
                delta = duration;
                delta0 = 0;
                ost->sync_opts = lrint(sync_ipts);
            }
        case VSYNC_CFR:
            // FIXME set to 0.5 after we fix some dts/pts bugs like in avidec.c
            if (frame_drop_threshold && delta < frame_drop_threshold && ost->frame_number) {
                nb_frames = 0;
            } else if (delta < -1.1)
            {
                nb_frames = 0;
            }
            else if (delta > 1.1) {
                nb_frames = lrintf(delta);
                if (delta0 > 1.1)
                {
                    nb0_frames = lrintf(delta0 - 0.6);
                }
            }
            break;
        case VSYNC_VFR:
            if (delta <= -0.6)
                nb_frames = 0;
            else if (delta > 0.6)
                ost->sync_opts = lrint(sync_ipts);
            break;
        case VSYNC_DROP:
        case VSYNC_PASSTHROUGH:
            ost->sync_opts = lrint(sync_ipts);
            break;
        default:
            av_assert0(0);
        }

丢帧策略

7.png

-frame_drop_threshold parameter
Frame drop threshold, which specifies how much behind video frames can be before they are dropped.
In frame rate units, so 1.0 is one frame.
The default is -1.1.
One possible usecase is to avoid framedrops in case of noisy timestamps or to increase frame drop precision in case of exact timestamps.

8 9.png

补帧策略

10.png

保存当前帧的数据

11.png

保存当前帧(Frame0)的数据，因为下一帧(Frame1)处理的时候，会判断是否编码Frame0还是彻底丢弃。

编码

6.png

References:

https://blog.csdn.net/sidumqz/article/details/53102623
https://www.jianshu.com/p/a1ad543a0d0e
https://www.cnblogs.com/jingzhishen/p/3767342.html
http://www.rosoo.net/a/201107/14663.html
http://www.ishenping.com/ArtInfo/353415.html
http://ffmpeg.org/pipermail/ffmpeg-cvslog/2014-November/083387.html
https://blog.csdn.net/newchenxf/article/details/51777891
https://www.jianshu.com/p/27279255f67e
https://codertw.com/%E4%BA%BA%E5%B7%A5%E6%99%BA%E6%85%A7/130033/
https://www.jianshu.com/p/27279255f67e