face3d例子解析1: pipeline

作者: 霞客环肥 | 来源:发表于2019-03-25 14:27 被阅读0次

【准备】载入需要的库

from time import time
import matplotlib.pyplot as plt

sys.path.append('..')
import face3d
from face3d import mesh

载入网格数据example1.mat

网格数据example1.mat,由点，三角，颜色，纹理组成。

这里用颜色来代表面部纹理。

C = sio.loadmat('Data/example1')

vertices = C['vertices']
colors = C['colors']
triangles = C['triangles']

colors = colors/np.max(colors)#归一化

查看exmple1.mat文件的keys

>example.keys()
>>dict_keys(['__header__',
'__version__', '__globals__', 'vertices', 'colors', 'triangles', 'full_triangles'])

数据以字典形式存储。

各个key的shape：

vertices	colors	triangles	full_triangles
(53215, 3)	(53215, 3)	(105840, 3)	(105954, 3)

vertices为53215的原因？
通过BFM形状模型和表情模型，可以得到最终的3D点云的图像坐标(共53215个)，每个点有x,y,z 3个坐标，共有53215x3个值，这些点云中的68个点x,y坐标即为常用的68个人脸关键点，约40k个点的x，y坐标即为密集人脸关键点。点云预测的目标就是从单张2D人脸RGB图像中直接预测这约53k的点的3维坐标值。

顶点变换

在世界坐标下做相似变换。


#目标范围尺寸180
s = 180/(np.max(vertices[:, 1]) - np.min(vertices[:, 1]))

#样本旋转30度
R = mesh.transform.angle2matrix([0, 30, 0])

#无位移
t = [0, 0, 0]

#相似变换
transformed_vertices = mesh.transform.similarity_transform(vertices, s, R, t)

一般 $X_{projection} = s * P * R * (\bar S + \sum_{i=1}^{m-1}\alpha_i s_i + \sum_{i=1}^{n-1} \beta_i * e_i) + t_{2d}$

其中， $X_{projection}$ 是三维模型投影到二维平面的的点；

$P = \begin{bmatrix} 1 & 0 & 0\\ 0 & 1 & 0 \\ \end{bmatrix}$ 为正交投影矩阵；

$R$ 为shape为(3, 3)的旋转矩阵；

$t_{2d}$ 为位移矩阵。

此处为正交投影，P省略。

旋转矩阵 $R$ 由下述函数计算。

def def angle2matrix(angles):
    
    ''' 
    根据右手系三个旋转角，
    得到三个旋转矩阵。
    Args:
        angles: [3,]. x, y, z angles
        x: pitch. positive for looking down.
        y: yaw. positive for looking left. 
        z: roll. positive for tilting head right. 
    Returns:
        R: [3, 3]. rotation matrix.
    
    '''
    
    #np.deg2rad将角度变为弧度pi
    x, y, z = np.deg2rad(angles[0]), np.deg2rad(angles[1]), np.deg2rad(angles[2])
    # x
    Rx=np.array([[1,      0,       0],
                 [0, cos(x),  -sin(x)],
                 [0, sin(x),   cos(x)]])
    # y
    Ry=np.array([[ cos(y), 0, sin(y)],
                 [      0, 1,      0],
                 [-sin(y), 0, cos(y)]])
    # z
    Rz=np.array([[cos(z), -sin(z), 0],
                 [sin(z),  cos(z), 0],
                 [     0,       0, 1]])
    
    R=Rz.dot(Ry.dot(Rx))
    return R.astype(np.float32)

介绍一个概念：相似变换。
今天《流浪地球》上映了，我坐在第一排看电影，而你坐在最后一排看电影，我们看的是同一部电影，但是我们各自眼中看到的电影却因为位置不同而有所不同（比如清晰度啊、角度啊），所以说，“第一排看到的电影”和“最后一排看到的电影”是“相似”的。
同一个线性变换，不同基底下的矩阵，成为相似矩阵。

def similarity_transform(vertices, s, R, t3d):
    ''' similarity transform. dof = 7.
    3D: s*R.dot(X) + t
    Homo: M = [[sR, t],[0^T, 1]].  M.dot(X)
    Args:(float32)
        vertices: [nver, 3]. 
        s: [1,]. scale factor.
        R: [3,3]. rotation matrix.
        t3d: [3,]. 3d translation vector.
    Returns:
        transformed vertices: [nver, 3]
    '''
    t3d = np.squeeze(np.array(t3d, dtype = np.float32))
    transformed_vertices = s * vertices.dot(R.T) + t3d[np.newaxis, :]

    return transformed_vertices

颜色，纹理变换
增加点光（光从某一点发出）。光的位置定义在世界坐标里。

#点光在世界坐标系的坐标
light_positions = np.array([[-128, -128, 300]])
#点光的强度
light_intensities = np.array([1, 1, 1])
#在已定义的点光下，变换颜色
lit_colors = mesh.light.add_light(transformed_vertices, triangles, colors, light_positions, light_intensities)

在已定义的点光下，变换颜色的函数如下，

def add_light(vertices, triangles, colors, light_positions = 0, light_intensities = 0):
   ''' Gouraud shading. add point lights.
   In 3d face, usually assume:
   1. The surface of face is Lambertian(reflect only the low frequencies of lighting)
   2. Lighting can be an arbitrary combination of point sources
   3. No specular (unless skin is oil, 23333)

   Ref: https://cs184.eecs.berkeley.edu/lecture/pipeline    
   Args:
       vertices: [nver, 3]
       triangles: [ntri, 3]
       light_positions: [nlight, 3] 
       light_intensities: [nlight, 3]
   Returns:
       lit_colors: [nver, 3]
   '''
   nver = vertices.shape[0]
   normals = get_normal(vertices, triangles) # [nver, 3]

   # ambient
   # La = ka*Ia

   # diffuse
   # Ld = kd*(I/r^2)max(0, nxl)
   direction_to_lights = vertices[np.newaxis, :, :] - light_positions[:, np.newaxis, :] # [nlight, nver, 3]
   direction_to_lights_n = np.sqrt(np.sum(direction_to_lights**2, axis = 2)) # [nlight, nver]
   direction_to_lights = direction_to_lights/direction_to_lights_n[:, :, np.newaxis]
   normals_dot_lights = normals[np.newaxis, :, :]*direction_to_lights # [nlight, nver, 3]
   normals_dot_lights = np.sum(normals_dot_lights, axis = 2) # [nlight, nver]
   diffuse_output = colors[np.newaxis, :, :]*normals_dot_lights[:, :, np.newaxis]*light_intensities[:, np.newaxis, :]
   diffuse_output = np.sum(diffuse_output, axis = 0) # [nver, 3]
   
   # specular
   # h = (v + l)/(|v + l|) bisector
   # Ls = ks*(I/r^2)max(0, nxh)^p
   # increasing p narrows the reflectionlob

   lit_colors = diffuse_output # only diffuse part here.
   lit_colors = np.minimum(np.maximum(lit_colors, 0), 1)
   return lit_colors

运用高氏着色法。

其中需要求拓扑图下各个点的法向量，如下

def get_normal(vertices, triangles):
    ''' calculate normal direction in each vertex
    Args:
        vertices: [nver, 3]
        triangles: [ntri, 3]
    Returns:
        normal: [nver, 3]
    '''
    pt0 = vertices[triangles[:, 0], :] # [ntri, 3]
    pt1 = vertices[triangles[:, 1], :] # [ntri, 3]
    pt2 = vertices[triangles[:, 2], :] # [ntri, 3]
    tri_normal = np.cross(pt0 - pt1, pt0 - pt2) # [ntri, 3]. normal of each triangle

    normal = np.zeros_like(vertices) # [nver, 3]
    for i in range(triangles.shape[0]):
        normal[triangles[i, 0], :] = normal[triangles[i, 0], :] + tri_normal[i, :]
        normal[triangles[i, 1], :] = normal[triangles[i, 1], :] + tri_normal[i, :]
        normal[triangles[i, 2], :] = normal[triangles[i, 2], :] + tri_normal[i, :]
    
    # normalize to unit length
    mag = np.sum(normal**2, 1) # [nver]
    zero_ind = (mag == 0)
    mag[zero_ind] = 1;
    normal[zero_ind, 0] = np.ones((np.sum(zero_ind)))

    normal = normal/np.sqrt(mag[:,np.newaxis])

    return normal

顶点变换
将世界坐标下的顶点变换到投影面下。

#这一步如果用规范相机可以忽略。
camera_vertices = mesh.transform.lookat_camera(transformed_vertices,  eye = [0, 0, 200], at = np.array([0, 0, 0]), up = None))
#将物体从3D世界坐标投影到2D投影面，可以是正交投影，也可以透视投影。
projected_vertices = mesh.transform.orthographic_project(camera_vertices)

假设使用规范相机，忽略第一步。
将物体从3D世界坐标投影到2D投影面，可以是正交投影，也可以透视投影。此时其实仅剩下X，Y坐标，保留Z坐标，只是为了后面使用方便。
此处使用正交投影，

def orthographic_project(vertices):
   ''' scaled orthographic projection(just delete z)
       assumes: variations in depth over the object is small relative to the mean distance from camera to object
       x -> x*f/z, y -> x*f/z, z -> f.
       for point i,j. zi~=zj. so just delete z
       ** often used in face
       Homo: P = [[1,0,0,0], [0,1,0,0], [0,0,1,0]]
   Args:
       vertices: [nver, 3]
   Returns:
       projected_vertices: [nver, 3] if isKeepZ=True. [nver, 2] if isKeepZ=False.
   '''
   return vertices.copy()

渲染

渲染为d图片。

h = w = 256
image_vertices = mesh.transform.to_image(projected_vertices, h, w)
#变换为uv坐标
rendering = mesh.render.render_colors(image_vertices, triangles, lit_colors, h, w)

怎么将投影坐标下的2D数据（虽然这是点的坐标依然是3D的，但是Z坐标其实并不会用到）投影到uv坐标呢？如下：

def to_image(vertices, h, w, is_perspective = False):
    ''' change vertices to image coord system
    3d system: XYZ, center(0, 0, 0)
    2d image: x(u), y(v). center(w/2, h/2), flip y-axis. 
    Args:
        vertices: [nver, 3]
        h: height of the rendering
        w : width of the rendering
    Returns:
        projected_vertices: [nver, 3]  
    '''
    image_vertices = vertices.copy()
    if is_perspective:
        # if perspective, the projected vertices are normalized to [-1, 1]. so change it to image size first.
        image_vertices[:,0] = image_vertices[:,0]*w/2
        image_vertices[:,1] = image_vertices[:,1]*h/2
    # move to center of image
    image_vertices[:,0] = image_vertices[:,0] + w/2
    image_vertices[:,1] = image_vertices[:,1] + h/2
    # flip vertices along y-axis.
    image_vertices[:,1] = h - image_vertices[:,1] - 1
    return image_vertices

对应图片的每个像素点都有3个色道RGB，如下：

def render_colors(vertices, triangles, colors, h, w, c = 3):
    ''' render mesh with colors
    Args:
        vertices: [nver, 3]
        triangles: [ntri, 3] 
        colors: [nver, 3]
        h: height
        w: width    
    Returns:
        image: [h, w, c]. 
    '''
    assert vertices.shape[0] == colors.shape[0]
    
    # initial 
    image = np.zeros((h, w, c))
    depth_buffer = np.zeros([h, w]) - 999999.

    for i in range(triangles.shape[0]):
        tri = triangles[i, :] # 3 vertex indices

        # the inner bounding box
        umin = max(int(np.ceil(np.min(vertices[tri, 0]))), 0)
        umax = min(int(np.floor(np.max(vertices[tri, 0]))), w-1)

        vmin = max(int(np.ceil(np.min(vertices[tri, 1]))), 0)
        vmax = min(int(np.floor(np.max(vertices[tri, 1]))), h-1)

        if umax<umin or vmax<vmin:
            continue

        for u in range(umin, umax+1):
            for v in range(vmin, vmax+1):
                if not isPointInTri([u,v], vertices[tri, :2]): 
                    continue
                w0, w1, w2 = get_point_weight([u, v], vertices[tri, :2])
                point_depth = w0*vertices[tri[0], 2] + w1*vertices[tri[1], 2] + w2*vertices[tri[2], 2]

                if point_depth > depth_buffer[v, u]:
                    depth_buffer[v, u] = point_depth
                    image[v, u, :] = w0*colors[tri[0], :] + w1*colors[tri[1], :] + w2*colors[tri[2], :]
    return image

返回image: [h, w, c]。

最后一步展示我们渲染好的图片

save_folder = 'results/pipline'
if not os.path.exits(save_folder):
    os.mkdir(save_folder)
io.imsave('{}/rendering.jpg'.format(save_folder), rendering)

face3d例子解析1: pipeline

相关文章

网友评论

延伸阅读

深度阅读

栏目导航

热点阅读