对OpenXR 的一些理解

参考官方文档openxr-10-reference-guide

• 一个标准的openxr应用包含方法调用、对象创建、Session状态变化、渲染循环等

  
  image.png

• 以monon的openxr runtime为例，主要涉及以下几个大的模块

1、连接方面，有蓝牙设备，和usb设备，涉及蓝牙通信和HID通信
2、语言方面设计openxr 、vulkan、opengl等
3、api方面有交换链、session会话、event队列
4、ipc通信方面有soctet通信、匿名共享内存
5、渲染方面主要是vulkan 语法和 shader

• monon的数据来源主要是获取手机自带的加速度和陀螺仪 模拟成3dof数据

while (ASensorEventQueue_getEvents(d->event_queue, &event, 1) > 0) {

        switch (event.type) {
        case ASENSOR_TYPE_ACCELEROMETER: {
            accel.x = event.acceleration.y;
            accel.y = -event.acceleration.x;
            accel.z = event.acceleration.z;

            ANDROID_TRACE(d, "accel %ld %.2f %.2f %.2f", event.timestamp, accel.x, accel.y, accel.z);
            break;
        }
        case ASENSOR_TYPE_GYROSCOPE: {
            gyro.x = -event.data[1];
            gyro.y = event.data[0];
            gyro.z = event.data[2];

            ANDROID_TRACE(d, "gyro %ld %.2f %.2f %.2f", event.timestamp, gyro.x, gyro.y, gyro.z);

            // TODO: Make filter handle accelerometer
            struct xrt_vec3 null_accel;

            // Lock last and the fusion.
            os_mutex_lock(&d->lock);

            m_imu_3dof_update(&d->fusion, event.timestamp, &null_accel, &gyro);

            // Now done.
            os_mutex_unlock(&d->lock);
        }
        default: ANDROID_TRACE(d, "Unhandled event type %d", event.type);
        }
    }

• monon的openxr Runingtime Session创建过程，
  1、有个宏控XR_USE_GRAPHICS_API_VULKAN 表示用vulkan作为图形库的语言。

//src/xrt/state_trackers/oxr/oxr_api_session.c
XrResult
oxr_xrCreateSession(XrInstance instance, const XrSessionCreateInfo *createInfo, XrSession *out_session)
{
    OXR_TRACE_MARKER();

      ret = oxr_session_create(&log, &inst->system, createInfo, &sess);
}
//src/xrt/state_trackers/oxr/oxr_session.c

XrResult
oxr_session_create(struct oxr_logger *log,
                   struct oxr_system *sys,
                   const XrSessionCreateInfo *createInfo,
                   struct oxr_session **out_session)
{
XrResult ret = oxr_session_create_impl(log, sys, createInfo, &xsi, &sess);

}

/* Just the allocation and populate part, so we can use early-returns to
 * simplify code flow and avoid weird if/else */
static XrResult
oxr_session_create_impl(struct oxr_logger *log,
                        struct oxr_system *sys,
                        const XrSessionCreateInfo *createInfo,
                        const struct xrt_session_info *xsi,
                        struct oxr_session **out_session){
#ifdef XR_USE_GRAPHICS_API_VULKAN
    XrGraphicsBindingVulkanKHR const *vulkan =
        OXR_GET_INPUT_FROM_CHAIN(createInfo, XR_TYPE_GRAPHICS_BINDING_VULKAN_KHR, XrGraphicsBindingVulkanKHR);
    if (vulkan != NULL) {
        OXR_VERIFY_ARG_NOT_ZERO(log, vulkan->instance);
        OXR_VERIFY_ARG_NOT_ZERO(log, vulkan->physicalDevice);
        if (vulkan->device == VK_NULL_HANDLE) {
            return oxr_error(log, XR_ERROR_GRAPHICS_DEVICE_INVALID, "VkDevice must not be VK_NULL_HANDLE");
        }

        if (!sys->gotten_requirements) {
            return oxr_error(log, XR_ERROR_GRAPHICS_REQUIREMENTS_CALL_MISSING,
                             "Has not called "
                             "xrGetVulkanGraphicsRequirementsKHR");
        }

        if (sys->suggested_vulkan_physical_device == VK_NULL_HANDLE) {
            char *fn = sys->inst->extensions.KHR_vulkan_enable ? "xrGetVulkanGraphicsDeviceKHR"
                                                               : "xrGetVulkanGraphicsDevice2KHR";
            return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "Has not called %s", fn);
        }

        if (sys->suggested_vulkan_physical_device != vulkan->physicalDevice) {
            char *fn = sys->inst->extensions.KHR_vulkan_enable ? "xrGetVulkanGraphicsDeviceKHR"
                                                               : "xrGetVulkanGraphicsDevice2KHR";
            return oxr_error(
                log, XR_ERROR_VALIDATION_FAILURE,
                "XrGraphicsBindingVulkanKHR::physicalDevice %p must match device %p specified by %s",
                (void *)vulkan->physicalDevice, (void *)sys->suggested_vulkan_physical_device, fn);
        }

        OXR_SESSION_ALLOCATE(log, sys, *out_session);
        OXR_ALLOCATE_NATIVE_COMPOSITOR(log, xsi, *out_session);
        return oxr_session_populate_vk(log, sys, vulkan, *out_session);//这里表示用的是vk，还可以选择用opengl
    }
#endif
}

2、XrGraphicsBindingVulkanKHR 是xr图形绑定vulkan语言支持扩展

//src/xrt/state_trackers/oxr/oxr_session_gfx_vk.c
XrResult
oxr_session_populate_vk(struct oxr_logger *log,
                        struct oxr_system *sys,
                        XrGraphicsBindingVulkanKHR const *next,
                        struct oxr_session *sess)
{
struct xrt_compositor_native *xcn = sess->xcn;
    struct xrt_compositor_vk *xcvk = xrt_gfx_vk_provider_create( //
        xcn,                                                     //
        next->instance,                                          //
        vkGetInstanceProcAddr,                                   //
        next->physicalDevice,                                    //
        next->device,                                            //
        sess->sys->vk.timeline_semaphore_enabled,                //
        next->queueFamilyIndex,                                  //
        next->queueIndex);                                       //

    if (xcvk == NULL) {
        return oxr_error(log, XR_ERROR_INITIALIZATION_FAILED, "Failed to create an vk client compositor");
    }

    sess->compositor = &xcvk->base;
    sess->create_swapchain = oxr_swapchain_vk_create;  //指定交换链创建

    return XR_SUCCESS;
}
//src/xrt/compositor/client/comp_vk_glue.c 
client_vk_compositor_create

//src/xrt/compositor/client/comp_vk_client.c
struct client_vk_compositor *
client_vk_compositor_create(struct xrt_compositor_native *xcn,
                            VkInstance instance,
                            PFN_vkGetInstanceProcAddr getProc,
                            VkPhysicalDevice physicalDevice,
                            VkDevice device,
                            bool timeline_semaphore_enabled,
                            uint32_t queueFamilyIndex,
                            uint32_t queueIndex)
{
//主要是初始化渲染相关的一些东西了
    c->base.base.create_swapchain = client_vk_swapchain_create;
    c->base.base.begin_session = client_vk_compositor_begin_session;
    c->base.base.end_session = client_vk_compositor_end_session;
    c->base.base.wait_frame = client_vk_compositor_wait_frame;
    c->base.base.begin_frame = client_vk_compositor_begin_frame;
    c->base.base.discard_frame = client_vk_compositor_discard_frame;
    c->base.base.layer_begin = client_vk_compositor_layer_begin;
    c->base.base.layer_stereo_projection = client_vk_compositor_layer_stereo_projection;
    c->base.base.layer_stereo_projection_depth = client_vk_compositor_layer_stereo_projection_depth;
    c->base.base.layer_quad = client_vk_compositor_layer_quad;
    c->base.base.layer_cube = client_vk_compositor_layer_cube;
    c->base.base.layer_cylinder = client_vk_compositor_layer_cylinder;
    c->base.base.layer_equirect1 = client_vk_compositor_layer_equirect1;
    c->base.base.layer_equirect2 = client_vk_compositor_layer_equirect2;
    c->base.base.layer_commit = client_vk_compositor_layer_commit;
    c->base.base.destroy = client_vk_compositor_destroy;
    c->base.base.poll_events = client_vk_compositor_poll_events;
}

3、client 里面的client_vk_compositor_create 中的c->base.base.begin_session = client_vk_compositor_begin_session; 可以看出 调用到 xc->begin_session ，这里的xc 就是传过来的
xrt_compositor_native 结构体中的base -->struct xrt_compositor base;

static xrt_result_t
client_vk_compositor_begin_session(struct xrt_compositor *xc, enum xrt_view_type type)
{
    COMP_TRACE_MARKER();

    struct client_vk_compositor *c = client_vk_compositor(xc);

    // Pipe down call into native compositor.
    return xrt_comp_begin_session(&c->xcn->base, type);//这里传过去的xrt_compositor_native 中的     
 xrt_compositor 
}

static inline xrt_result_t
xrt_comp_begin_session(struct xrt_compositor *xc, enum xrt_view_type view_type)
{
    return xc->begin_session(xc, view_type);
}

4、xrt_compositor_native 的xrt_compositor 的begin_session 函数指针在 src/xrt/compositor/main/comp_compositor.c 的xrt_gfx_provider_create_system 方法中被赋值

xrt_result_t
xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compositor **out_xsysc)
{
    struct comp_compositor *c = U_TYPED_CALLOC(struct comp_compositor);

    c->base.base.base.begin_session = compositor_begin_session;  //这里赋值，其他的几个方法类似
    c->base.base.base.end_session = compositor_end_session;
    c->base.base.base.predict_frame = compositor_predict_frame;
    c->base.base.base.mark_frame = compositor_mark_frame;
    c->base.base.base.begin_frame = compositor_begin_frame;
    c->base.base.base.discard_frame = compositor_discard_frame;
    c->base.base.base.layer_commit = compositor_layer_commit;
    c->base.base.base.poll_events = compositor_poll_events;
    c->base.base.base.destroy = compositor_destroy;
    c->frame.waited.id = -1;
    c->frame.rendering.id = -1;
    c->xdev = xdev;

5、至此compositor_begin_session 创建完成 ，特别记录compositor_layer_commit 函数是应用提交一帧后怎么绘制到屏幕的流程,下面是用perfetto抓取的trace。

image.png

• monon的openxr渲染提交命令缓冲过程

src/xrt/compositor/main/comp_compositor.c
static xrt_result_t
compositor_layer_commit(struct xrt_compositor *xc, int64_t frame_id, xrt_graphics_sync_handle_t sync_handle)
{
comp_renderer_draw(c->r);
}

//src/xrt/compositor/main/comp_renderer.c
void
comp_renderer_draw(struct comp_renderer *r)
{

comp_target_mark_begin(ct, c->frame.rendering.id, os_monotonic_get_ns());
if (use_compute) {
        dispatch_compute(r, &crc);
    } else {
        dispatch_graphics(r, &rr);
    }

    renderer_present_swapchain_image(r, c->frame.rendering.desired_present_time_ns,
                                     c->frame.rendering.present_slop_ns);

}


static void
dispatch_graphics(struct comp_renderer *r, struct comp_rendering *rr)
{
renderer_get_view_projection(r);
comp_layer_renderer_draw(r->lr);

renderer_build_rendering(r, rr, rtr, src_samplers, src_image_views, src_norm_rects);
renderer_submit_queue(r, rr->cmd, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
}
//src/xrt/compositor/main/comp_layer_renderer.c
void
comp_layer_renderer_draw(struct comp_layer_renderer *self)
{
    COMP_TRACE_MARKER();

    struct vk_bundle *vk = self->vk;

    VkCommandBuffer cmd_buffer;
    if (vk_init_cmd_buffer(vk, &cmd_buffer) != VK_SUCCESS)
        return;
    os_mutex_lock(&vk->cmd_pool_mutex);
    if (self->layer_count == 0) {
        _render_stereo(self, vk, cmd_buffer, &background_color_idle);
    } else {
        _render_stereo(self, vk, cmd_buffer, &background_color_active);
    }
    os_mutex_unlock(&vk->cmd_pool_mutex);

    VkResult res = vk_submit_cmd_buffer(vk, cmd_buffer);
    vk_check_error("vk_submit_cmd_buffer", res, );
}

vk_submit_cmd_buffer 在 src/xrt/auxiliary/vk/vk_helpers.c 中
使用vkQueueSubmit函数向图像队列提交命令缓冲区

• runtime 迷惑的结构体定义，我们看到这种comp_compositor ，c->base.base.base.begin_session 有三个base 分别代表啥呢

xrt_result_t
xrt_gfx_provider_create_system(struct xrt_device *xdev, struct xrt_system_compositor **out_xsysc)
{
    struct comp_compositor *c = U_TYPED_CALLOC(struct comp_compositor);

    c->base.base.base.begin_session = compositor_begin_session;
    c->base.base.base.end_session = compositor_end_session;

1、comp_compositor 结构体

struct comp_compositor
{
    struct comp_base base;

    //! Renderer helper.
    struct comp_renderer *r;

    //! The target we are displaying to.
    struct comp_target *target;

    //! The device we are displaying to.
    struct xrt_device *xdev;

    //! The settings.
    struct comp_settings settings;

    //! Vulkan shaders that the compositor uses.
    struct comp_shaders shaders;

    //! Timestamp of last-rendered (immersive) frame.
    int64_t last_frame_time_ns;

    //! State for generating the correct set of events.
    enum comp_state state;

    /*!
     * @brief Data exclusive to the begin_frame/end_frame for computing an
     * estimate of the app's needs.
     */
    struct
    {
        int64_t last_begin;
        int64_t last_end;
    } app_profiling;

    struct
    {
        //! Current Index for times_ns.
        int index;

        //! Timestamps of last-rendered (immersive) frames.
        int64_t times_ns[NUM_FRAME_TIMES];

        //! Frametimes between last-rendered (immersive) frames.
        float timings_ms[NUM_FRAME_TIMES];

        //! Average FPS of last NUM_FRAME_TIMES rendered frames.
        float fps;

        struct u_var_timing *debug_var;
    } compositor_frame_times;

    struct
    {
        struct comp_frame waited;
        struct comp_frame rendering;
    } frame;

    struct
    {
        //! Temporarily disable ATW
        bool atw_off;
    } debug;

    struct comp_resources nr;
};

2、struct comp_base base; 主要封装了xrt_compositor_native 和 vulkan

struct comp_base
{
    //! Base native compositor.
    struct xrt_compositor_native base;

    //! Vulkan bundle of useful things, used by swapchain and fence.
    struct vk_bundle vk;

    //! For default @ref xrt_compositor::wait_frame.
    struct os_precise_sleeper sleeper;

    //! Swapchain garbage collector, used by swapchain, child class needs to call.
    struct comp_swapchain_gc cscgc;

    //! We only need to track a single slot.
    struct comp_layer_slot slot;
};

3、struct xrt_compositor_native base; 封装了xrt_compositor

struct xrt_compositor_native
{
    //! @public Base
    struct xrt_compositor base;
};

4、struct xrt_compositor base; 里面主要是定义了一些函数指针

struct xrt_compositor
{
    /*!
     * Capabilities and recommended values information.
     */
    struct xrt_compositor_info info;

    /*!
     * Create a swapchain with a set of images.
     *
     * The pointer pointed to by @p out_xsc has to either be NULL or a valid
     * @ref xrt_swapchain pointer. If there is a valid @ref xrt_swapchain
     * pointed by the pointed pointer it will have it reference decremented.
     */
    xrt_result_t (*create_swapchain)(struct xrt_compositor *xc,
                                     const struct xrt_swapchain_create_info *info,
                                     struct xrt_swapchain **out_xsc);

• 因为openxr的渲染部分是用vulkan实现的，最后可以再看下vulkan绘制出一个三角形流程可以从大的方面理解openxr的渲染流程

init_vulkan_instance();//创建Vulkan实例
 enumerate_vulkan_phy_devices();//获取物理设备列表
create_vulkan_devices();//创建逻辑设备
create_vulkan_CommandBuffer();//创建命令缓冲
init_queue();//获取设备中支持图形工作的队列
create_vulkan_swapChain();//初始化交换链
create_vulkan_DepthBuffer();//创建深度缓冲
create_render_pass();//创建渲染通道
create_frame_buffer();//创建帧缓冲
createDrawableObject();//创建绘制用的物体
initPipeline();//初始化渲染管线
createFence();//创建栅栏
initPresentInfo();//初始化呈现信息
initMatrix();//初始化基本变换矩阵、摄像机矩阵、投影矩阵
drawObject();//执行绘制
destroyFence();//销毁栅栏
destroyPipeline();//销毁管线
destroyDrawableObject();//销毁绘制用物体
destroy_frame_buffer();//销毁帧缓冲
destroy_render_pass();//销毁渲染通道相关
destroy_vulkan_DepthBuffer();//销毁深度缓冲相关
destroy_vulkan_swapChain();//销毁交换链相关
destroy_vulkan_CommandBuffer();//销毁命令缓冲
destroy_vulkan_devices();//销毁逻辑设备
destroy_vulkan_instance();//销毁Vulkan 实例

• 绘制一帧画面要经过
  1、从交换链获取当前帧索引
  2、为渲染通道设置当前帧索引
  3、初始化命令缓冲，然后启动命令缓冲
  4、将当前帧送入一直变量缓冲
  5、更新绘制用描述集
  6、绘制
  7、结束渲染通道 结束命令缓冲 等待信号量
  8、提交命令缓冲
  9、等待渲染完毕
  10、为呈现信息指定当前交换链索引
  11、呈现