使用Gstreamer 作为数据源输出视频数据

作者: Charles_linzc | 来源:发表于2021-11-04 15:40 被阅读0次

在虚拟摄像头程序的设计中，我们将用gstreamer作为数据源，为directshow注册摄像头程序提供数据源。

创建数据源类型

我们创建一个数据源vcamsource, 这个vcamsource将包含gststream的实例，在VcamSource的私有属性_VcamSourcePriv 中我们声明了四个gstElement元素，分别代表pipeline，pipeline，capsfilter ，sink：

/**** 注释掉这部分，因为G_DECLARE_DERIVABLE_TYPE 中已经定义了这个类
struct  _VcamSource {
    GObject parent_instance;
};
***/
struct _VcamSourcePrivate {
    gboolean status;
    GstElement* pipeline, * source,   *capsfilter ， * sink;
    
};

pipeline时streamer的容器，source时streamer的产生源，capsfilter是媒体格式过滤去，sink是stream的最终渲染器。我们将使用videotestsrc插件作为source，他会随机生成stream, 同时使用appsink插件作为sink，它可以提供外部程序访问stream中的数据。
需要注意的是在.h头文件中，G_DECLARE_DERIVABLE_TYPE已经包含了对_VcamSource 的定义，所以不要重复定义，另外_VcamSourcePrivate是一个私有属性，我们将用下面的宏将它加入，并且它的名字不可以改变必须是ModuleObjName##Private：

G_DEFINE_TYPE_WITH_CODE(VcamSource, vcam_source, G_TYPE_OBJECT, G_ADD_PRIVATE(VcamSource) )


#define VCAM_SOURCE_GET_PRIVATE(obj) ( G_TYPE_INSTANCE_GET_PRIVATE((obj), VCAM_TYPE_SOURCE, VcamSourcePrivate))

新的私有函数的定义，需要使用G_ADD_PRIVATE宏，他被作为额外的执行代码放在G_DEFINE_TYPE_WITH_CODE中，所以这里要注意，使用G_DEFINE_TYPE_WITH_CODE替换G_DEFINE_TYPE；并在在class_init文件中不在需要, VCAM_SOURCE_GET_PRIVATE宏仍然保留，用于获取私有的属性，仍旧需要注意第三个参数，VcamSourcePrivate，他的格式是ModuleObjName##Private，不要更改。

vcam_source_class_init(VcamSourceClass *kclass) {
    //使用G_ADD_PRIVATE配合G_DEFINE_TYPE_WITH_CODE宏，不在需要g_type_class_add_private自己添加私有属性， commnet out it
    //g_type_class_add_private(kclass, sizeof(VcamSourcePriv));

gstreamer在VcamSource的工作步骤

在VcamSource实现gstreamer流程需要以下的步骤：

Gstreamer在执行前需要初始化（gst_init (&argc, &argv);），但这个初始化操作最好在系统入口处，所以在vcamsource中我们不进行gstreamer的初始化，而是在初次引入VcamSource的时候，先对gist系统做初始化。
分别创建source, pipleline, sink等元素
我们将为每一个VcamSource实例分别创建一个gstreamer实例，有多个VcamSource时，就会对应多个实例。所以这些元素的创建，被我们放在object_init方法里：

 static void
vcam_source_init(VcamSource *d) {
      ......
      /* Create the elements */
      VcamSourcePriv* priv = VCAM_SOURCE_GET_PRIVATE(d);
      priv ->source= gst_element_factory_make("videotestsrc", "source");
      priv ->capsfilter = gst_element_factory_make("capsfilter ", "caps");
      priv->sink = gst_element_factory_make ("appsink", "app_sink");
      /* Create the empty pipeline */
      priv->pipeline = gst_pipeline_new("test-pipeline");
}

source在gstreamer中代表的是数据源元素，我们使用videotestsrc来作为初始验证源（测试方案用），他将产生测试用vedio数据。同时，我们将采用GstAppSink作为gststream视频的render(实际只输出数据，并不渲染窗口)。
appSink的不同之处在于它提供了额外的API，可以允许外部程序提取stream内的数据（sample). appSink使用queue来储存来自streamer线程的数据，可以设置drop属性，当queue满了的时候，就及时得drop掉数据；相对于自己写一个满足需要的sink插件，appsink还是很方便的。
capsfilter 作为过滤器，并不能修改数据，但是能够限制数据的格式，在这里可以限制videotestsrc 产生的数据格式。

组装pipleLine
在使用pipleLine前我们需要将所有的gstreamer 元素组装起来：

 static void
vcam_source_init(VcamSource *d) {
     ......
      /* Create the elements */
      VcamSourcePrivate* priv = VCAM_SOURCE_GET_PRIVATE(d);
      priv ->source= gst_element_factory_make("videotestsrc", "source");
      priv->sink = gst_element_factory_make("appsink", "app_sink");
      priv->capsfilter = gst_element_factory_make("capsfilter ", "caps");
      /* Create the empty pipeline */
      priv->pipeline = gst_pipeline_new("test-pipeline");

      if (!priv->pipeline || !priv->source || !priv->sink) {
          g_printerr("Not all elements could be created.\n");
          return -1;
      }

      /* Build the pipeline */
    gst_bin_add_many(GST_BIN(priv->pipeline), priv->source, priv->capsfilter, priv->sink, NULL);
    //if (gst_element_link(priv->source,  priv->sink) != TRUE) {
    if (gst_element_link(priv->source, priv->capsfilter) != TRUE || gst_element_link(priv->capsfilter, priv->sink) != TRUE) {
         g_printerr("Elements could not be linked.\n");
        gst_object_unref(priv->pipeline);
        return -1;
    }

}

gstreamer 元素的组装也是在 vcam_source对象初始化的函数中来完成，连接的路线是 videotestsrc | capsfilter | appsink.

配置filter和appsink
capsfilter主要用来限制testsrc的输出格式，我们需要在pipleline调用前，先给提供一个默认的格式：
video/x-raw, width=640, height=480, format=RGB //表示640X480 RGB frame
filter 和appsink的设置也是在初始化函数中完成：

static void
vcam_source_init(VcamSource *d) {
      ....
      /* Create the elements */
      ....
      /**设置 fitler caps**/ 
      GstCaps* caps = gst_caps_new_simple("video/x-raw",
          "format", G_TYPE_STRING, "RGB",
          "framerate", GST_TYPE_FRACTION, 25, 1,
          "width", G_TYPE_INT, 640,
          "height", G_TYPE_INT, 480,
          NULL);
      g_object_set(priv->capsfilter, "caps", caps, NULL);
      /**设置appsink的max-buffers来限制buffer大小，
         设置drop属性，允许queue满意后自动blockthread或drop旧的buffer**/
      gst_app_sink_set_max_buffers(priv->sink, 500); // limit number of buffers queued
      gst_app_sink_set_drop(priv->sink, TRUE); // drop old buffers in queue when full
      /* Build the pipeline */
    ....
}

启动pipleLine
pipleline的启动就是奖状台设置为GST_STATE_PLAYING，但是并不是创建完就启动的，相反VcamSource中我们需要提供一个启动函数，让用户或调用程序，显示的启动pipleLine.
在.h头文件中，我们声明一个start函数，以及一个帮助函数：

struct _VcamSourceClass {
    GObjectClass parent_class;
   .......
   gshort (*start)(VcamSource* self);
  .....
};

void vcam_source_getMediaTyp(VcamSource *self, VcamIMediaType* typeptr);

gshort vcam_source_start(VcamSource* self);

在.c文件中，我们需要创建start的默认实现函数，以及vcam_source_start帮助函数，并且将start指针指向默认实现函数：

//start pipleline
 static gshort startimpl(VcamSource* self) {
     
     VcamSourcePrivate* priv = VCAM_SOURCE_GET_PRIVATE(self);
     GstStateChangeReturn ret;
     /* Start playing */
     ret = gst_element_set_state(priv->pipeline, GST_STATE_PLAYING);
     if (ret == GST_STATE_CHANGE_FAILURE) {
         g_printerr("Unable to set the pipeline to the playing state (check the bus for error messages).\n");
         return -1;  //启动失败
     }
     else {
         priv->status = TRUE;
         return 0;
     }
 }

 gshort vcam_source_start(VcamSource* self) {
     VcamSourceClass *klass = VCAM_SOURCE_GET_CLASS(self);
     if (klass->start == NULL) return -2; //函数未初始化
      return klass->start(self);
 }



static void
vcam_source_class_init(VcamSourceClass *kclass) {
    //不在需要手动注册私有属性
   // g_type_class_add_private(kclass, sizeof(VcamSourcePriv));
     .....
    kclass->start = startimpl;
    .......
}

在pipleline成功start以后，我们会将vcamsource的状态设置为true.

获取来自gst appsink的数据
appsink可以提供外部程序访问stream中数据的能力。通过appsink访问视频samples的方法通常包括 GstApp.AppSink.prototype.pull_sample和 GstApp.AppSink.prototype.pull_preroll，它们是同步方式，这两个方法会一致出去block状态，除非新的samples已经可以被访问，或者sink关闭里，stream流到了末尾，它们才会返回。
当然， appsink对这两个函数也提供了timeout的版本，GstApp.AppSink.prototype.try_pull_sample 和 GstApp.AppSink.prototype.try_pull_preroll，它们连个额外接收一个timeout的参数，用于限制等待的时间。
pull_sample主要用于appsink在PlAYING状态下获取sample, 而pull_preroll则是当PlAYING处于PAUSED状态时，获取最近的预拨（preroll）sample。特别是当发生seek后，调用该方法可以立刻获得seek位置的数据。
vcamsource将同样提供两个方法，让使用者可以方便的访问视频数据，同时忽略内部的实现：
void vcam_source_pull_sample(VcamSource self, Gst.Sample sample)
void vcam_source_pull_preroll(VcamSource self, Gst.Sample sample)

并在.c文件中完成它们的实现，在当前阶段，我们只是简单的返回gstsample:

  void  vcam_source_pull_sample(VcamSource* self, GstSample* sample) {
      

      VcamSourcePriv* private = VCAM_SOURCE_GET_PRIVATE(self);

      g_return_if_fail(priv->sink != NULL);

      sample = gst_app_sink_pull_sample(priv->sink);

      if (sample == NULL && gst_app_sink_is_eos(priv->sink)) {
          g_print("the stream reach the ond \n");
          //send out singal for eof of stream
      }
      else
      {
          g_print("gst_app_sink_pull_sample returned null\n");
          return NULL;
      }
  }

  void  vcam_source_pull_preroll(VcamSource* self, GstSample* sample) {
      
      VcamSourcePrivate* priv = VCAM_SOURCE_GET_PRIVATE(self);

      g_return_if_fail(priv->sink != NULL);
      
      sample = gst_app_sink_pull_preroll(priv->sink);
      
      if (sample == NULL && gst_app_sink_is_eos(priv->sink)) {
          g_print("the stream reach the ond \n");
          //send out singal for eof of stream
      }else{
          g_print("gst_app_sink_pull_preroll returned null\n");
         
      }
  }

gststreamer元素的销毁
每一个vcamsource元素，包含一个stream pipleline, 当vcamsource由于任何原因走向销毁得时候，我们需要同时销毁stream得所有元素。
在VcamSource中我们增肌内存管理所需的函数，这些函数将帮我们最终销毁VcamSource中创建的所有gstreamer 元素：

 static void
      vcam_source_dispose(GObject* gobject)
  {
      VcamSourcePriv* priv = VCAM_SOURCE_GET_PRIVATE(gobject);

      /* In dispose(), you are supposed to free all types referenced from this
       * object which might themselves hold a reference to self. Generally,
       * the most simple solution is to unref all members on which you own a
       * reference.
       */

       /* dispose() might be called multiple times, so we must guard against
        * calling g_object_unref() on an invalid GObject by setting the member
        * NULL; g_clear_object() does this for us.
        */
      gst_element_set_state(priv->pipeline, GST_STATE_NULL);
      g_clear_object(&priv->pipeline);

      /* Always chain up to the parent class; there is no need to check if
       * the parent class implements the dispose() virtual function: it is
       * always guaranteed to do so
       */
      G_OBJECT_CLASS(vcam_source_parent_class)->dispose(gobject);
  }


  static void
      vcam_source_finalize(GObject* gobject)
  {
      /* Always chain up to the parent class; as with dispose(), finalize()
       * is guaranteed to exist on the parent's class virtual function table
       */
      G_OBJECT_CLASS(vcam_source_parent_class)->finalize(gobject);
  }

以上的代码中，不需要组个unreference source, sink, capsfilter. 由于它们都在pipleline中，pipleline会自己处理好。如果手动处理，会报错误。

最后，在修改class_init函数，将上边的函数与gobject对象中定义的dispose与finalize函数bind起来：

static void
vcam_source_class_init(VcamSourceClass *kclass) {
    //注册私有属性
   ......
    /**memory manage**/
    base_class->dispose = vcam_source_dispose;
    base_class->finalize = vcam_source_finalize;
    /* install properties */
  ......
}

测试代码：
在.c文件中我们写一个测试程序，创建一个新的vcamsource对象，并且启动它，然后获取preroll sample, 如果获取成功，打印 ****：

int main(int argc, char* argv) {
    gst_init(&argc, &argv);
    g_print("start gst test\n");
    VcamSource * source = g_object_new(VCAM_TYPE_SOURCE,NULL);
    vcam_source_start(source);
    VcamSourcePrivate* priv = VCAM_SOURCE_GET_PRIVATE(source);
    GstSample* sample;
    sample = gst_app_sink_pull_preroll(priv->sink);
    if(sample!=NULL)
        g_print("*********\n");
    g_print("start unrefercene \n");
    g_object_unref(source);
    g_print("program stop!!!!!!! \n");
}

上面的代码包含了一行 gst_init(&argc, &argv);，用于启动gst系统，它是我们上文提到的第一步的工作。测试打印结果如下：

image.png

完成代码：
VcamSource.h

#ifndef VCAM_SOURCE_H
#define VCAM_SOURCE_H
#include <gst/gst.h>

#define VCAM_TYPE_SOURCE (vcam_source_get_type())
G_DECLARE_DERIVABLE_TYPE(VcamSource, vcam_source, VCAM, SOURCE, GObject)

typedef struct _VcamSourcePrivate VcamSourcePrivate;



struct _VcamSourceClass {
    GObjectClass parent_class;

    gshort(*start)(VcamSource* self);
};


gshort vcam_source_start(VcamSource* self);

void  vcam_source_pull_sample(VcamSource* self, GstSample* sample);

void  vcam_source_pull_preroll(VcamSource* self, GstSample* sample);

#endif /* __VCAM_SOURCE_H__ */

VcamSource.c


#include "VcamSource.h"
#include <gst/app/gstappsink.h>

struct _VcamSourcePrivate {
    gboolean status;
    GstElement* pipeline;
    GstElement* source;
    GstElement* capsfilter;
    GstElement* sink;
};


G_DEFINE_TYPE_WITH_CODE(VcamSource, vcam_source, G_TYPE_OBJECT, G_ADD_PRIVATE(VcamSource) )


#define VCAM_SOURCE_GET_PRIVATE(obj) ( G_TYPE_INSTANCE_GET_PRIVATE((obj), VCAM_TYPE_SOURCE, VcamSourcePrivate))



enum _VCAM_SOURCE_PROPERTY {
    VCAM_SOURCE_PROPERTY0,
    VCAM_SOURCE_STATUS,
    VCAM_SOURCE_VEDIO
};




static void  vcam_source_finalize(GObject* gobject);
static void  vcam_source_dispose(GObject* gobject);
static gshort startimpl(VcamSource* self);
static void   vcam_source_set_property(GObject* object, guint property_id,
                                       const GValue* value, GParamSpec* pspec);
static void    vcam_source_get_property(GObject* object, guint property_id,
                                         GValue* value, GParamSpec* pspec);

static void
vcam_source_class_init(VcamSourceClass* kclass) {
    g_print("start the class init \n");
    //注册私有属性
    //g_type_class_add_private(kclass, sizeof(VcamSourcePriv));

    kclass->start = startimpl;

    /* override base object methods */
    GObjectClass* base_class = G_OBJECT_CLASS(kclass);
    base_class->set_property = vcam_source_set_property;
    base_class->get_property = vcam_source_get_property;
    /**memory manage**/
    base_class->dispose = vcam_source_dispose;
    base_class->finalize = vcam_source_finalize;

    /* install properties */
   // g_object_class_install_property(base_class, VCAM_SOURCE_STATUS,
  //      g_param_spec_uint("status", "source status", "desscribe server's status", \
   //         0, 10, 0, G_PARAM_READWRITE));
 
}

static void
vcam_source_init(VcamSource* d) {
    g_print("start the object init \n");
    /* Create the elements */
    VcamSourcePrivate * priv = VCAM_SOURCE_GET_PRIVATE(d);
    priv->source = gst_element_factory_make("videotestsrc", "source");
    priv->sink = gst_element_factory_make("appsink", "app_sink");
    priv->capsfilter = gst_element_factory_make("capsfilter", "caps_filter");
    /* Create the empty pipeline */
    priv->pipeline = gst_pipeline_new("test-pipeline");
    if (!priv->pipeline || !priv->source || !priv->sink) {
        g_printerr("Not all elements could be created.\n");
        return;
    }
    /**设置 fitler caps**/
    GstCaps* caps = gst_caps_new_simple("video/x-raw",
        "format", G_TYPE_STRING, "I420",
        "framerate", GST_TYPE_FRACTION, 25, 1,
        "width", G_TYPE_INT, 320,
        "height", G_TYPE_INT, 240, 
        NULL);
    g_object_set(priv->capsfilter, "caps", caps, NULL);
    /**设置appsink的max-buffers来限制buffer大小，设置drop属性，允许queue满意后自动blockthread或drop旧的buffer**/
    gst_app_sink_set_max_buffers(priv->sink, 500); // limit number of buffers queued
    gst_app_sink_set_drop(priv->sink , TRUE); // drop old buffers in queue when full
   

    /* Build the pipeline */
    gst_bin_add_many(GST_BIN(priv->pipeline), priv->source, priv->capsfilter, priv->sink, NULL);
    //if (gst_element_link(priv->source,  priv->sink) != TRUE) {
    if (gst_element_link(priv->source, priv->capsfilter) != TRUE || gst_element_link(priv->capsfilter, priv->sink) != TRUE) {
         g_printerr("Elements could not be linked.\n");
        gst_object_unref(priv->pipeline);
        return -1;
    }

}


//start pipleline
static gshort startimpl(VcamSource* self) {

    VcamSourcePrivate * priv = VCAM_SOURCE_GET_PRIVATE(self);
    GstStateChangeReturn ret;
    /* Start playing */
    ret = gst_element_set_state(priv->pipeline, GST_STATE_PLAYING);
    if (ret == GST_STATE_CHANGE_FAILURE) {
        g_printerr("Unable to set the pipeline to the playing state (check the bus for error messages).\n");
        return -1;  //启动失败
    }
    else {
        priv->status = TRUE;
        return 0;
    }
    g_object_unref(priv); //结束引用
}

gshort vcam_source_start(VcamSource* self) {
    VcamSourceClass* klass = VCAM_SOURCE_GET_CLASS(self);
    if (klass->start == NULL) return -2; //函数未初始化
    return klass->start(self);
    g_object_unref(klass); //结束引用
}

                         

static void
vcam_source_set_property(GObject* object, guint property_id,
    const GValue* value, GParamSpec* pspec)
{
    VcamSource* self = VCAM_SOURCE(object);
    VcamSourcePrivate * priv = VCAM_SOURCE_GET_PRIVATE(self);
    switch (property_id) {
    case VCAM_SOURCE_STATUS:

        priv->status = g_value_get_boolean(value);
        break;
    default:
        G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec);
        break;
    }
    g_object_unref(self); //结束引用
    g_object_unref(priv); //结束引用
}

static void
vcam_source_get_property(GObject* object, guint property_id,
    GValue* value, GParamSpec* pspec)
{
    VcamSource* self = VCAM_SOURCE(object);
    VcamSourcePrivate * priv = VCAM_SOURCE_GET_PRIVATE(self);
    switch (property_id) {
    case VCAM_SOURCE_STATUS:
        g_value_set_boolean(value, priv->status);
        break;
    default:
        G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec);
        break;
    }
    g_object_unref(self); //结束引用
    g_object_unref(priv); //结束引用
}


void  vcam_source_pull_sample(VcamSource* self, GstSample* sample) {


    VcamSourcePrivate * priv = VCAM_SOURCE_GET_PRIVATE(self);

    g_return_if_fail(priv->sink != NULL);

    sample = gst_app_sink_pull_sample(priv->sink);

    if (sample == NULL && gst_app_sink_is_eos(priv->sink)) {
        g_print("the stream reach the ond \n");
        sample = NULL;
        //send out singal for eof of stream
    }
    else
    {
        g_print("gst_app_sink_pull_sample returned null\n");
        sample = NULL;
        return NULL;
    }
    g_object_unref(priv); //结束引用
}

void  vcam_source_pull_preroll(VcamSource* self, GstSample* sample) {

    VcamSourcePrivate* priv = VCAM_SOURCE_GET_PRIVATE(self);

    g_return_if_fail(priv->sink != NULL);

    sample = gst_app_sink_pull_preroll(priv->sink);

    if (sample == NULL && gst_app_sink_is_eos(priv->sink)) {
        g_print("the stream reach the ond \n");
        //send out singal for eof of stream
    }
    else {
        g_print("gst_app_sink_pull_preroll returned null\n");

    }
    g_object_unref(priv); //结束引用
}


static void
vcam_source_dispose(GObject* gobject)
{
    VcamSourcePrivate * priv = VCAM_SOURCE_GET_PRIVATE(gobject);

    /* In dispose(), you are supposed to free all types referenced from this
     * object which might themselves hold a reference to self. Generally,
     * the most simple solution is to unref all members on which you own a
     * reference.
     */

     /* dispose() might be called multiple times, so we must guard against
      * calling g_object_unref() on an invalid GObject by setting the member
      * NULL; g_clear_object() does this for us.
      */
    gst_element_set_state(priv->pipeline, GST_STATE_NULL);
    //source, capsfitler, sink 都有pipeline 管理， 清除工作有pipleline来完成
    g_clear_object(&priv->pipeline);
    /* Always chain up to the parent class; there is no need to check if
     * the parent class implements the dispose() virtual function: it is
     * always guaranteed to do so
     */
    G_OBJECT_CLASS(vcam_source_parent_class)->dispose(gobject);
}


static void
vcam_source_finalize(GObject* gobject)
{
    VcamSourcePrivate * priv = VCAM_SOURCE_GET_PRIVATE(gobject);

    /* Always chain up to the parent class; as with dispose(), finalize()
     * is guaranteed to exist on the parent's class virtual function table
     */
    G_OBJECT_CLASS(vcam_source_parent_class)->finalize(gobject);
}


int main(int argc, char* argv) {
    gst_init(&argc, &argv);
    g_print("start gst test\n");
    VcamSource * source = g_object_new(VCAM_TYPE_SOURCE,NULL);
    vcam_source_start(source);
    VcamSourcePrivate* priv = VCAM_SOURCE_GET_PRIVATE(source);
    GstSample* sample;
    sample = gst_app_sink_pull_preroll(priv->sink);
    if(sample!=NULL)
        g_print("*********\n");
    g_print("start unrefercene \n");
    g_object_unref(source);
    g_print("program stop!!!!!!! \n");
}

使用Gstreamer 作为数据源输出视频数据

创建数据源类型

gstreamer在VcamSource的工作步骤

相关文章

网友评论

延伸阅读

深度阅读

栏目导航

热点阅读