阿里P7移动互联网架构师进阶视频（每日更新中）免费学习请点击：https://space.bilibili.com/474380680
1.变速入口分析
从jni层的_setPropertyFloat函数
static void ijkMediaPlayer_setPropertyFloat(JNIEnv *env, jobject thiz, jint id, jfloat value)
{
IjkMediaPlayer *mp = jni_get_media_player(env, thiz);
JNI_CHECK_GOTO(mp, env, NULL, "mpjni: setPropertyFloat: null mp", LABEL_RETURN);
ijkmp_set_property_float(mp, id, value);
LABEL_RETURN:
ijkmp_dec_ref_p(&mp);
return;
}
到ff_ffplay.c中的ffp_set_property_float函数来设置速度
void ffp_set_property_float(FFPlayer *ffp, int id, float value)
{
switch (id) {
case FFP_PROP_FLOAT_PLAYBACK_RATE:
ffp_set_playback_rate(ffp, value);
break;
case FFP_PROP_FLOAT_PLAYBACK_VOLUME:
ffp_set_playback_volume(ffp, value);
break;
default:
return;
}
}
跟踪ffp_set_playback_rate函数，我们可以看到这里主要把速度变量设置给了ffp->pf_playback_rate以及把ffp->pf_playback_rate_changed置1.
void ffp_set_playback_rate(FFPlayer *ffp, float rate)
{
if (!ffp)
return;
av_log(ffp, AV_LOG_INFO, "Playback rate: %f\n", rate);
ffp->pf_playback_rate = rate;
ffp->pf_playback_rate_changed = 1;
}
音频变速实现
跟踪这俩个变量，我们可以看到在audio_decode_frame中，我们新增了音频的变速变调算法来处理音频变速。
#if defined(__ANDROID__)
if (ffp->soundtouch_enable && ffp->pf_playback_rate != 1.0f && !is->abort_request) {
av_fast_malloc(&is->audio_new_buf, &is->audio_new_buf_size, out_size * translate_time);
for (int i = 0; i < (resampled_data_size / 2); i++)
{
is->audio_new_buf[i] = (is->audio_buf1[i * 2] | (is->audio_buf1[i * 2 + 1] << 8));
}
int ret_len = ijk_soundtouch_translate(is->handle, is->audio_new_buf, (float)(ffp->pf_playback_rate), (float)(1.0f/ffp->pf_playback_rate),
resampled_data_size / 2, bytes_per_sample, is->audio_tgt.channels, af->frame->sample_rate);
if (ret_len > 0) {
is->audio_buf = (uint8_t*)is->audio_new_buf;
resampled_data_size = ret_len;
} else {
translate_time++;
goto reload;
}
} else if (ffp->sonic_enabled && ffp->pf_playback_rate != 1.0f && !is->abort_request) {
av_fast_malloc(&is->audio_new_buf, &is->audio_new_buf_size, out_size * translate_time * 2);
for (int i = 0; i < (resampled_data_size / 2); i++)
{
is->audio_new_buf[i] = (is->audio_buf1[i * 2] | (is->audio_buf1[i * 2 + 1] << 8));
}
int ret_len = sonicStream_translate(is->sonic_handle,is->audio_new_buf,ffp->pf_playback_rate,
(float)(1.0f/ffp->pf_playback_rate),is->audio_tgt.channels, af->frame->sample_rate,resampled_data_size / 2,bytes_per_sample);
if (ret_len > 0) {
is->audio_buf = (uint8_t*)is->audio_new_buf;
resampled_data_size = ret_len;
} else {
translate_time++;
goto reload;
}
}
#endif
sonic变速是加的，下面在音频回调sdl_audio_callback函数中。
if (ffp->pf_playback_rate_changed) {
ffp->pf_playback_rate_changed = 0;
#if defined(__ANDROID__)
if (!ffp->soundtouch_enable && !ffp->sonic_enabled) {
SDL_AoutSetPlaybackRate(ffp->aout, ffp->pf_playback_rate);
}
#else
SDL_AoutSetPlaybackRate(ffp->aout, ffp->pf_playback_rate);
#endif
}
if (ffp->pf_playback_volume_changed) {
ffp->pf_playback_volume_changed = 0;
SDL_AoutSetPlaybackVolume(ffp->aout, ffp->pf_playback_volume);
}
上面主要处理对ffp->pf_playback_rate_changed的判断，如果没有开启soundtouch变速或sonic变速，直接走audiotrack的变速来处理。在android6.0系统之下，audiotrack的变速存在变调问题。
从上面我们可以看到这里处理了音频的变速，但是没有处理只有视频流的视频（也就是没有音频流）。
3.视频变速实现
ijk中默认选择音频流作为时间基准的，我们看下视频是如何来做变速的。视频上很容易就可以做到倍速播放，一般的视频格式都是每秒固定的帧数，按比例跳帧就可以了。
static void video_refresh(FFPlayer *opaque, double *remaining_time)
{
FFPlayer *ffp = opaque;
VideoState *is = ffp->is;
double time;
Frame *sp, *sp2;
if (!is->paused && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK && is->realtime)
check_external_clock_speed(is);
if (!ffp->display_disable && is->show_mode != SHOW_MODE_VIDEO && is->audio_st) {
time = av_gettime_relative() / 1000000.0;
if (is->force_refresh || is->last_vis_time + ffp->rdftspeed < time) {
video_display2(ffp);
is->last_vis_time = time;
}
*remaining_time = FFMIN(*remaining_time, is->last_vis_time + ffp->rdftspeed - time);
}
if (is->video_st) {
retry:
//add by hxk,support only video change speed
if(!is->audio_st && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK) {
if(ffp->pf_playback_rate != 1.0f){
change_external_clock_speed(is,ffp->pf_playback_rate);
}
}
//add end
if (frame_queue_nb_remaining(&is->pictq) == 0) {
// nothing to do, no picture to display in the queue
} else {
double last_duration, duration, delay;
Frame *vp, *lastvp;
/* dequeue the picture */
lastvp = frame_queue_peek_last(&is->pictq);
vp = frame_queue_peek(&is->pictq);
if (vp->serial != is->videoq.serial) {
frame_queue_next(&is->pictq);
goto retry;
}
if (lastvp->serial != vp->serial)
is->frame_timer = av_gettime_relative() / 1000000.0;
if (is->paused)
goto display;
/* compute nominal last_duration */
last_duration = vp_duration(is, lastvp, vp);
delay = compute_target_delay(ffp, last_duration, is);
time= av_gettime_relative()/1000000.0;
if (isnan(is->frame_timer) || time < is->frame_timer)
is->frame_timer = time;
if (time < is->frame_timer + delay) {
*remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);
goto display;
}
is->frame_timer += delay;
if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX)
is->frame_timer = time;
SDL_LockMutex(is->pictq.mutex);
if (!isnan(vp->pts))
update_video_pts(is, vp->pts, vp->pos, vp->serial);
SDL_UnlockMutex(is->pictq.mutex);
if (frame_queue_nb_remaining(&is->pictq) > 1) {
Frame *nextvp = frame_queue_peek_next(&is->pictq);
duration = vp_duration(is, vp, nextvp);
if(!is->step && (ffp->framedrop > 0 || (ffp->framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration) {
frame_queue_next(&is->pictq);
goto retry;
}
}
if (is->subtitle_st) {
while (frame_queue_nb_remaining(&is->subpq) > 0) {
sp = frame_queue_peek(&is->subpq);
if (frame_queue_nb_remaining(&is->subpq) > 1)
sp2 = frame_queue_peek_next(&is->subpq);
else
sp2 = NULL;
if (sp->serial != is->subtitleq.serial
|| (is->vidclk.pts > (sp->pts + ((float) sp->sub.end_display_time / 1000)))
|| (sp2 && is->vidclk.pts > (sp2->pts + ((float) sp2->sub.start_display_time / 1000))))
{
if (sp->uploaded) {
ffp_notify_msg4(ffp, FFP_MSG_TIMED_TEXT, 0, 0, "", 1);
}
frame_queue_next(&is->subpq);
} else {
break;
}
}
}
frame_queue_next(&is->pictq);
is->force_refresh = 1;
SDL_LockMutex(ffp->is->play_mutex);
if (is->step) {
is->step = 0;
if (!is->paused)
stream_update_pause_l(ffp);
}
SDL_UnlockMutex(ffp->is->play_mutex);
}
display:
/* display picture */
if (!ffp->display_disable && is->force_refresh && is->show_mode == SHOW_MODE_VIDEO && is->pictq.rindex_shown)
video_display2(ffp);
}
is->force_refresh = 0;
if (ffp->show_status) {
static int64_t last_time;
int64_t cur_time;
int aqsize, vqsize, sqsize __unused;
double av_diff;
cur_time = av_gettime_relative();
if (!last_time || (cur_time - last_time) >= 30000) {
aqsize = 0;
vqsize = 0;
sqsize = 0;
if (is->audio_st)
aqsize = is->audioq.size;
if (is->video_st)
vqsize = is->videoq.size;
#ifdef FFP_MERGE
if (is->subtitle_st)
sqsize = is->subtitleq.size;
#else
sqsize = 0;
#endif
av_diff = 0;
if (is->audio_st && is->video_st)
av_diff = get_clock(&is->audclk) - get_clock(&is->vidclk);
else if (is->video_st)
av_diff = get_master_clock(is) - get_clock(&is->vidclk);
else if (is->audio_st)
av_diff = get_master_clock(is) - get_clock(&is->audclk);
av_log(NULL, AV_LOG_INFO,
"%7.2f %s:%7.3f fd=%4d aq=%5dKB vq=%5dKB sq=%5dB f=%"PRId64"/%"PRId64" \r",
get_master_clock(is),
(is->audio_st && is->video_st) ? "A-V" : (is->video_st ? "M-V" : (is->audio_st ? "M-A" : " ")),
av_diff,
is->frame_drops_early + is->frame_drops_late,
aqsize / 1024,
vqsize / 1024,
sqsize,
is->video_st ? is->viddec.avctx->pts_correction_num_faulty_dts : 0,
is->video_st ? is->viddec.avctx->pts_correction_num_faulty_pts : 0);
fflush(stdout);
last_time = cur_time;
}
}
}
其实这里就是视频变速的实现，也是视频同步音频的实现，来实现视频变速。
3.1.没有音频的视频变速
通过前面的分析，我们可以知道ijk没有处理只有视频流的视频变速(没有音频流的视频)。当没有音频流的时候，在read_thread函数中
/* open the streams */
if (st_index[AVMEDIA_TYPE_AUDIO] >= 0) {
stream_component_open(ffp, st_index[AVMEDIA_TYPE_AUDIO]);
} else {//如果没有音频流
ffp->av_sync_type = AV_SYNC_VIDEO_MASTER;
is->av_sync_type = ffp->av_sync_type;
}
我们可以看到如果没有音频流，ijk中选择视频流作为时间基准。个人觉得还是外部时钟作为时间基准比较好，比较准。
那么我们可以如何修改呢？
3.1.1.修改没有音频的视频的同步方式为外部时钟
/* open the streams */
if (st_index[AVMEDIA_TYPE_AUDIO] >= 0) {
stream_component_open(ffp, st_index[AVMEDIA_TYPE_AUDIO]);
} else {
ffp->av_sync_type = AV_SYNC_EXTERNAL_CLOCK;
is->av_sync_type = ffp->av_sync_type;
}
当没有音频流的时候，选择外部时钟作为时间基准。
3.1.2.修改外部时钟变速
修改video_refresh函数
/* called to display each frame */
static void video_refresh(FFPlayer *opaque, double *remaining_time)
{
FFPlayer *ffp = opaque;
VideoState *is = ffp->is;
double time;
Frame *sp, *sp2;
if (!is->paused && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK && is->realtime)
check_external_clock_speed(is);
if (!ffp->display_disable && is->show_mode != SHOW_MODE_VIDEO && is->audio_st) {
time = av_gettime_relative() / 1000000.0;
if (is->force_refresh || is->last_vis_time + ffp->rdftspeed < time) {
video_display2(ffp);
is->last_vis_time = time;
}
*remaining_time = FFMIN(*remaining_time, is->last_vis_time + ffp->rdftspeed - time);
}
if (is->video_st) {
retry:
//当没有音频流的，有视频流时，且时间基准为外部时钟
//add by hxk,support only video change speed
if(!is->audio_st && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK) {
//如果速度不等于1，改变外部时钟速度
if(ffp->pf_playback_rate != 1.0f){
change_external_clock_speed(is,ffp->pf_playback_rate);
}
}
//add end
if (frame_queue_nb_remaining(&is->pictq) == 0) {
// nothing to do, no picture to display in the queue
} else {
double last_duration, duration, delay;
Frame *vp, *lastvp;
/* dequeue the picture */
lastvp = frame_queue_peek_last(&is->pictq);
vp = frame_queue_peek(&is->pictq);
if (vp->serial != is->videoq.serial) {
frame_queue_next(&is->pictq);
goto retry;
}
if (lastvp->serial != vp->serial)
is->frame_timer = av_gettime_relative() / 1000000.0;
if (is->paused)
goto display;
/* compute nominal last_duration */
last_duration = vp_duration(is, lastvp, vp);
delay = compute_target_delay(ffp, last_duration, is);
time= av_gettime_relative()/1000000.0;
if (isnan(is->frame_timer) || time < is->frame_timer)
is->frame_timer = time;
if (time < is->frame_timer + delay) {
*remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);
goto display;
}
is->frame_timer += delay;
if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX)
is->frame_timer = time;
SDL_LockMutex(is->pictq.mutex);
if (!isnan(vp->pts))
update_video_pts(is, vp->pts, vp->pos, vp->serial);
SDL_UnlockMutex(is->pictq.mutex);
if (frame_queue_nb_remaining(&is->pictq) > 1) {
Frame *nextvp = frame_queue_peek_next(&is->pictq);
duration = vp_duration(is, vp, nextvp);
if(!is->step && (ffp->framedrop > 0 || (ffp->framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration) {
frame_queue_next(&is->pictq);
goto retry;
}
}
if (is->subtitle_st) {
while (frame_queue_nb_remaining(&is->subpq) > 0) {
sp = frame_queue_peek(&is->subpq);
if (frame_queue_nb_remaining(&is->subpq) > 1)
sp2 = frame_queue_peek_next(&is->subpq);
else
sp2 = NULL;
if (sp->serial != is->subtitleq.serial
|| (is->vidclk.pts > (sp->pts + ((float) sp->sub.end_display_time / 1000)))
|| (sp2 && is->vidclk.pts > (sp2->pts + ((float) sp2->sub.start_display_time / 1000))))
{
if (sp->uploaded) {
ffp_notify_msg4(ffp, FFP_MSG_TIMED_TEXT, 0, 0, "", 1);
}
frame_queue_next(&is->subpq);
} else {
break;
}
}
}
frame_queue_next(&is->pictq);
is->force_refresh = 1;
SDL_LockMutex(ffp->is->play_mutex);
if (is->step) {
is->step = 0;
if (!is->paused)
stream_update_pause_l(ffp);
}
SDL_UnlockMutex(ffp->is->play_mutex);
}
display:
/* display picture */
if (!ffp->display_disable && is->force_refresh && is->show_mode == SHOW_MODE_VIDEO && is->pictq.rindex_shown)
video_display2(ffp);
}
is->force_refresh = 0;
if (ffp->show_status) {
static int64_t last_time;
int64_t cur_time;
int aqsize, vqsize, sqsize __unused;
double av_diff;
cur_time = av_gettime_relative();
if (!last_time || (cur_time - last_time) >= 30000) {
aqsize = 0;
vqsize = 0;
sqsize = 0;
if (is->audio_st)
aqsize = is->audioq.size;
if (is->video_st)
vqsize = is->videoq.size;
#ifdef FFP_MERGE
if (is->subtitle_st)
sqsize = is->subtitleq.size;
#else
sqsize = 0;
#endif
av_diff = 0;
if (is->audio_st && is->video_st)
av_diff = get_clock(&is->audclk) - get_clock(&is->vidclk);
else if (is->video_st)
av_diff = get_master_clock(is) - get_clock(&is->vidclk);
else if (is->audio_st)
av_diff = get_master_clock(is) - get_clock(&is->audclk);
av_log(NULL, AV_LOG_INFO,
"%7.2f %s:%7.3f fd=%4d aq=%5dKB vq=%5dKB sq=%5dB f=%"PRId64"/%"PRId64" \r",
get_master_clock(is),
(is->audio_st && is->video_st) ? "A-V" : (is->video_st ? "M-V" : (is->audio_st ? "M-A" : " ")),
av_diff,
is->frame_drops_early + is->frame_drops_late,
aqsize / 1024,
vqsize / 1024,
sqsize,
is->video_st ? is->viddec.avctx->pts_correction_num_faulty_dts : 0,
is->video_st ? is->viddec.avctx->pts_correction_num_faulty_pts : 0);
fflush(stdout);
last_time = cur_time;
}
}
}
新增外部时钟变速函数。
//add by hxk
static void change_external_clock_speed(VideoState *is,float speed) {
if (speed != 1.0f){
set_clock_speed(&is->extclk, speed + EXTERNAL_CLOCK_SPEED_STEP * (1.0 - speed) / fabs(1.0 - speed));
}
}
//add end
这样我们就可以解决没有音频的视频流变速问题了。当然如果还是选择视频流作为时间基准的话，我们可以修改刷帧速度来实现视频变速。
原文链接：https://blog.csdn.net/u013470102/article/details/89485488
阿里P7移动互联网架构师进阶视频（每日更新中）免费学习请点击：https://space.bilibili.com/474380680
网友评论