iOS在线识别：https://www.jianshu.com/u/3c2a0bd52ebc
因为GStreamer的操作几乎都是在一个“黑盒”里面进行操作，所以它的这一套操作也有它自己的一个标准，就是一个个插件，我们要用到GStreamer，所以我们需要把解码器那部分也做成一个GStreamer插件，加入到管道（pipeline）中，这样就让GStreamer自己去处理传输和识别解码。
接下来我们就说说该项目最难的一部分了。
因为有些内容涉及到GStreamer自己的东西，比如制作模板要求和规范，我这里就不展开阐述了，也不是本内容的重点，这里只会讲解解码器在插件中的使用。
插件模板
插件名称kaldidecoder
初始化
/* the capabilities of the inputs and outputs.
*
*/
static GstStaticPadTemplate sink_template =
GST_STATIC_PAD_TEMPLATE("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS(
"audio/x-raw, "
"format = (string) S16LE, "
"channels = (int) 1, "
"rate = (int) [ 1, MAX ]"));
static GstStaticPadTemplate src_template =
GST_STATIC_PAD_TEMPLATE("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS("text/x-raw, format= { utf8 }"));
static guint gst_ kaldidecoder_signals[LAST_SIGNAL];
#define gst_ kaldidecoder_parent_class parent_class
G_DEFINE_TYPE(Gst kaldidecoder, gst_ kaldidecoder,
GST_TYPE_ELEMENT);
static void gst_ kaldidecoder_load_phone_syms(Gst kaldidecoder * filter,
const GValue * value);
static void gst_kaldidecoder_load_word_syms(Gst kaldidecoder * filter,
const GValue * value);
static void gst_ kaldidecoder_load_model(Gst kaldidecoder * filter,
const GValue * value);
static void gst_ kaldidecoder_load_fst(Gst kaldidecoder * filter,
const GValue * value);
static void gst_ kaldidecoder_load_lm_fst(Gst kaldidecoder * filter,
const GValue * value);
static void gst_ kaldidecoder_load_big_lm(Gst kaldidecoder * filter,
const GValue * value);
static void gst_ kaldidecoder_load_word_boundary_info(Gst kaldidecoder * filter,
const GValue * value);
static void gst_ kaldidecoder_set_property(GObject * object,
guint prop_id,
const GValue * value,
GParamSpec * pspec);
static void gst_ kaldidecoder_get_property(GObject * object,
guint prop_id,
GValue * value,
GParamSpec * pspec);
static gboolean gst_ kaldidecoder_sink_event(GstPad * pad,
GstObject * parent,
GstEvent * event);
static GstFlowReturn gst_ kaldidecoder_chain(GstPad * pad,
GstObject * parent,
GstBuffer * buf);
static GstStateChangeReturn gst_ kaldidecoder_change_state(
GstElement *element, GstStateChange transition);
static gboolean gst_ kaldidecoder_query(GstPad *pad, GstObject * parent, GstQuery * query);
static void gst_ kaldidecoder_finalize(GObject * object);
这些pad模板都需要通过gst_element_class_add_pad_template ()在_class_init方法里面注册。
_class_init：
/* GObject vmethod implementations */
/* initialize the kaldidecoder's class */
static void gst_kaldidecoder_class_init(
GstkaldidecoderClass * klass) {
GObjectClass *gobject_class;
GstElementClass *gstelement_class;
gobject_class = (GObjectClass *) klass;
gstelement_class = (GstElementClass *) klass;
gobject_class->set_property = gst_kaldidecoder_set_property;
gobject_class->get_property = gst_kaldidecoder_get_property;
gobject_class->finalize = gst_kaldidecoder_finalize;
gstelement_class->change_state = gst_kaldidecoder_change_state;
g_object_class_install_property(
...
);
...
gst_kaldidecoder_signals[PARTIAL_RESULT_SIGNAL] = g_signal_new(
"partial-result", G_TYPE_FROM_CLASS(klass), G_SIGNAL_RUN_LAST,
G_STRUCT_OFFSET(GstkaldidecoderClass, partial_result),
NULL,
NULL, kaldi_marshal_VOID__STRING, G_TYPE_NONE, 1,
G_TYPE_STRING);
gst_kaldidecoder_signals[FINAL_RESULT_SIGNAL] = g_signal_new(
"final-result", G_TYPE_FROM_CLASS(klass), G_SIGNAL_RUN_LAST,
G_STRUCT_OFFSET(GstkaldidecoderClass, final_result),
NULL,
NULL, kaldi_marshal_VOID__STRING, G_TYPE_NONE, 1,
G_TYPE_STRING);
gst_kaldidecoder_signals[FULL_FINAL_RESULT_SIGNAL] = g_signal_new(
"full-final-result", G_TYPE_FROM_CLASS(klass), G_SIGNAL_RUN_LAST,
G_STRUCT_OFFSET(GstkaldidecoderClass, full_final_result),
NULL,
NULL, kaldi_marshal_VOID__STRING, G_TYPE_NONE, 1,
G_TYPE_STRING);
gst_element_class_set_details_simple(
gstelement_class, "KaldiDecoder", "Speech/Audio",
"Convert speech to text", "changfengfuyun");
gst_element_class_add_pad_template(gstelement_class,
gst_static_pad_template_get(&src_template));
gst_element_class_add_pad_template(
gstelement_class, gst_static_pad_template_get(&sink_template));
}
我们定义插件所有部分的代码，我们需要有_init()方法。
/* entry point to initialize the plug-in
* initialize the plug-in itself
* register the element factories and other features
*/
static gboolean kaldidecoder_init(
GstPlugin * kaldidecoder) {
/* debug category for fltering log messages
*
* exchange the string 'Template kaldidecoder' with your description
*/
GST_DEBUG_CATEGORY_INIT(gst_kaldidecoder_debug,
"kaldidecoder", 0,
"Template kaldidecoder");
return gst_element_register(kaldidecoder
"kaldidecoder", GST_RANK_NONE,
GST_TYPE_KALDIDECODER);
}
至此，就完成了解码器模板的初始化
指定pads
pads是数据进出元素的端口，这使得它们在元素创建过程中成为非常重要的项。在模板代码中，我们已经看到了静态pad模板如何负责将pad模板注册到元素类中。在这里，我们将看到如何创建实际的元素，如何使用_event()-函数来配置特定的格式，以及如何注册函数来让数据流经元素。
创建pad：
/* initialize the new element
* instantiate pads and add them to element
* set pad calback functions
* initialize instance structure
*/
static void gst_kaldidecoder_init(
Gstkaldidecoder * filter) {
...
filter->sinkpad = NULL;
filter->sinkpad = gst_pad_new_from_static_template(&sink_template, "sink");
gst_pad_set_event_function(
filter->sinkpad,
GST_DEBUG_FUNCPTR(gst_kaldidecoder_sink_event));
gst_pad_set_chain_function(
filter->sinkpad, GST_DEBUG_FUNCPTR(gst_kaldidecoder_chain));
gst_pad_set_query_function(
filter->sinkpad, GST_DEBUG_FUNCPTR(gst_kaldidecoder_query));
gst_pad_use_fixed_caps(filter->sinkpad);
gst_element_add_pad(GST_ELEMENT(filter), filter->sinkpad);
filter->srcpad = gst_pad_new_from_static_template(&src_template, "src");
gst_pad_use_fixed_caps(filter->srcpad);
gst_element_add_pad(GST_ELEMENT(filter), filter->srcpad);
// 解码相关的初始化
...
}
这里也是每次生成元件的时候都会调用的方法
_chain方法：
- 用于接收和处理sinkpad上的输入数据。
/* chain function
* this function does the actual processing
*/
static GstFlowReturn gst_kaldidecoder_chain(GstPad * pad,
GstObject * parent,
GstBuffer * buf) {
Gstkaldidecoder *filter = GST_KALDIDECODER(parent);
if (G_UNLIKELY(!filter->audio_source))
goto not_negotiated;
if (!filter->silent) {
filter->audio_source->PushBuffer(buf);
}
gst_buffer_unref(buf);
return GST_FLOW_OK;
/* special cases */
not_negotiated: {
GST_ELEMENT_ERROR(filter, CORE, NEGOTIATION, (NULL),
("decoder wasn't allocated before chain function"));
gst_buffer_unref(buf);
return GST_FLOW_NOT_NEGOTIATED;
}
}
_event方法：
- 该方法通知你在传输数据流中发生的特殊事件(如caps, end-of-stream, newsegment, tags等)。
/* this function handles sink events */
static gboolean gst_kaldidecoder_sink_event(GstPad * pad,
GstObject * parent,
GstEvent * event) {
gboolean ret;
Gstkaldidecoder *filter;
filter = GST_KALDIDECODER(parent);
GST_DEBUG_OBJECT(filter, "Handling %s event", GST_EVENT_TYPE_NAME(event));
switch (GST_EVENT_TYPE(event)) {
case GST_EVENT_SEGMENT: {
GST_DEBUG_OBJECT(filter, "Starting decoding task");
filter->decoding = true;
gst_pad_start_task(filter->srcpad,
(GstTaskFunction) gst_ kaldidecoder_loop,
filter, NULL);
GST_DEBUG_OBJECT(filter, "Started decoding task");
ret = TRUE;
break;
}
case GST_EVENT_CAPS: {
ret = TRUE;
break;
}
case GST_EVENT_EOS: {
/* end-of-stream, we should close down all stream leftovers here */
GST_DEBUG_OBJECT(filter, "EOS received");
if (filter->decoding) {
filter->audio_source->SetEnded(true);
} else {
GST_DEBUG_OBJECT(filter, "EOS received while not decoding, pushing EOS out");
gst_pad_push_event(filter->srcpad, gst_event_new_eos());
}
ret = TRUE;
break;
}
default:
ret = gst_pad_event_default(pad, parent, event);
break;
}
return ret;
}
_query方法：
- 元件接收queries必须回复的内容
/* GstElement vmethod implementations */
static gboolean
gst_kaldidecoder_query (GstPad *pad, GstObject * parent, GstQuery * query) {
gboolean ret;
Gstkaldidecoder *filter;
filter = GST_KALDIDECODER(parent);
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_CAPS: {
if (filter->feature_info == NULL) {
filter->feature_info = new OnlineNnet2FeaturePipelineInfo(*(filter->feature_config));
if (strcmp((filter->feature_config->feature_type).c_str(), "plp") == 0)
filter->sample_rate = (int) filter->feature_info->plp_opts.frame_opts.samp_freq;
else
filter->sample_rate = (int) filter->feature_info->mfcc_opts.frame_opts.samp_freq;
}
GstCaps *new_caps = gst_caps_new_simple ("audio/x-raw",
"format", G_TYPE_STRING, "S16LE",
"rate", G_TYPE_INT, filter->sample_rate,
"channels", G_TYPE_INT, 1, NULL);
GST_DEBUG_OBJECT (filter, "Setting caps query result: %" GST_PTR_FORMAT, new_caps);
gst_query_set_caps_result (query, new_caps);
gst_caps_unref (new_caps);
ret = TRUE;
break;
}
default:
ret = gst_pad_query_default (pad, parent, query);
break;
}
return ret;
}
接下来讲解kaldi的解码。
网友评论