一、WebRTC

浏览器API

• MediaStream：通过MediaStream的API能够通过设备的摄像头及话筒获得视频、音频的同步流。
• RTCPeerConnection：RTCPeerConnection是WebRTC用于构建点对点之间稳定、高效的流传输的组件。
• RTCDataChannel：RTCDataChannel使得浏览器之间（点对点）建立一个高吞吐量、低延时的信道，用于传输任意数据。

WebRTC-internal-structure-cn.jpg image.png

先用ajax和ws传递信息，建立信道
再用RTC点对点传输

视频压缩技术：H264/H265、VP8/VP9、AV1
webRTC：集成回音消除，视频编解码，跨平台
支持WebRTC的浏览器：Chrome、Firefox、Safari、Edge。支持情况具体：https://cloud.tencent.com/document/product/647/16863

image.png

终端：音视频采集，编解码，NAT穿越，音视频传输
signal服务器：信令处理
STUN /TURN：获取终端在公网的IP，以及NAT穿越失败后数据中转

track：视频轨，音频轨
stream：媒体流存放0音频视频轨，数据流存放0数据轨

API

var promise = navigator.mediaDevices.getUserMedia(constraints);
const mediaStreamContrains = {
    video: true,
    audio: true
};

const mediaStreamContrains = {
    video: {
        frameRate: {min: 20}, 
        width: {min: 640, ideal: 1280},
        height: {min: 360, ideal: 720},
      aspectRatio: 16/9 // 宽高比
    },
    audio: {
        echoCancellation: true, // 回音消除
        noiseSuppression: true, // 降噪
        autoGainControl: true // 自动增益
    }
};

image.png

音频原理-采样率：人的听觉是20-20kHZ，大于最高频率两倍就算是无损所以平时采样率都是44.1k\48k等；
采样大小：8位（255）,16位

每个设备信息：id,label,kind
编码帧：压缩后的帧
H264：I帧（关键帧），P帧（参考帧），B帧（前后参考帧）

RTP协议

image.png

拆包组合
序号，时间戳，PT(音频和视频不一样)

image.png

RTP可能发生丢包、乱序、抖动，使用RTCP
RTCP:RR和SR收和发交换报文，来看网络质量

image.png

header：标识报文类型
sender info：发送多少包
report block： 接收包的情况

SDP(Session Description Protocal)

在webRTC中

image.png
image.png

会话元数据
网络描述
流描述
安全描述
服务质量描述

媒体协商

RTCPeerConnection

image.png

var pcConfig = null;
var pc = new RTCPeerConnection(pcConfig); // 创建peer对象
// 呼叫方创建offer
   function doCall() {
       console.log('Sending offer to peer');
       pc.createOffer(setLocalAndSendMessage, handleCreateOfferError);
  }  
// 发送给被呼叫方
  function setLocalAndSendMessage(sessionDescription) {
      pc.setLocalDescription(sessionDescription);
      sendMessage(sessionDescription);
  }
// 呼叫方收到anser
  socket.on('message', function(message) {
      ...
      } else if (message.type === 'answer') {
    
          pc.setRemoteDescription(new RTCSessionDescription(message));
      } else if (...) {
          ...
      }
      ....
  });

ICE Candidate
本地IP/端口、候选者类型（host/srflx/relay）、优先级、传输协议、访问服务的用户名

{
  IP: xxx.xxx.xxx.xxx,
  port: number,
  type: host/srflx/relay, //本机，内网主机映射的外网地址和端口，中继候选者
  priority: number,
  protocol: UDP/TCP,
  usernameFragment: string
  ...
}

先对host类型进行联通测试，按照优先级试

ICE

本机收集所有host类型
STUN拿到公网IP，收集srflx
再用TURN收集relay

NAT

WeChatWorkScreenshot_9dc62a12-9267-4c9a-aaad-9dc9ce7dcac3.png

流程梳理

选择类型

• 登入rtc manager：返回streamID和信令服务器地址（可返回ws地址）
  demo参数

config  设置相关配置
localVideo 本地流播放标签null
remoteVideo  远程流播放标签
nickname streamID
recevieDataCallback  receiveMessageData方法

• 登入信令服务器，get方法，返回peers（可以用new ws建立连接）

  
  WXWorkCapture_15935094097369.png

• 构造demo
  构造函数里面:构造PeerConnSignaling,构造call

异步等待两个事件 newMessage，newNotification

ice过程

开发经验

改造谷歌原生代码

代码地址：https://github.com/webrtc/apprtc
先跑起来然后魔改源码，简化代码，抽取信令，改造信令。源码解读有空再开别的文章。

信令设计

msg_type字段定义

sign_in_event 前端登入
accept_in_event 后端登入
recv_src_id 接收源id
recv_dst_id 接收目标id
offer_sdp_event 转发sdp
answer_sdp_event 转发sdp
ice_event 转发ice
bye 发送断开请求
error 信令服务器发送错误信息
ping  向信令服务器发ping
pong  信令服务器返回pong
reconn_event 重新连接
update_session_event 切换事件

交互过程中前端状态机

1、建立连接

var ws = new WebSocket(url);

2、登入,message里面放原来的字段，结构不变，加一个msgType字段表示发送消息类型

signMsg = {
    msg_type:'sign_in_event',
    message:{
        protocol:  "webRTC",
        media: {
            audio: {
                ...
            },
            video: {
                ...
            }
        },
        svrType: 0
    }
}
ws.send(JSON.stringify(signMsg))

3、登入后返回自身id

{
    msg_type: 'recv_src_id',
    message:{
      stream_id:'1594014948578356499vdptx',
      src_id:1
    }
}

4、等待返回对端id

// 成功
{
    msg_type: 'recv_dst_id',
    message:{
      stream_id:'1594014948578356499vdptx',
      dst_id:2
    }
}

5、发送前端sdp,message里面和原sdp字段相同

sdpMsg = {
    msg_type:'offer_sdp_event',
    src_id:1,
    dst_id:2,
    message:{
        type:'offer',
        sdp:''
    }
}
ws.send(JSON.stringify(sdpMsg))

6、发送anser sdp

{
    msg_type: 'answer_sdp_event',
    src_id: 2,
    dst_id: 1,
    message:{
      type:'answer',
      sdp:'...'
    }
}

5、发送前端candidate,message里面和原candidate字段相同

iceMsg = {
    msg_type:'ice_event',
    src_id:1,
    dst_id:2,
    message:{
      type:'candidate',
      candidate:'...',
      id:'...',
      label:'...'
    }
}
ws.send(JSON.stringify(iceMsg))

8、等待接收对端candidate

{
    msg_type: "ice_event",
    src_id: 2,
    dst_id: 1,
    message:{
      type:'candidate',
      candidate:'...',
      id:0,
      label:'...'
    }
}

9、结束连接,先发送一个断开的消息，再断开ws

byeMsg = {
    msg_type:'bye',
    src_id:1,
    dst_id:2,
}
ws.send(JSON.stringify(byeMsg))

10、错误消息

{
    msg_type: "error",
    src_id:2/null,
    dst_id:1,
    message:{
      // 失败原因
    }
}

11、ping

pingMsg = {
    msg_type:'ping',
    src_id:1,
    dst_id:2,
}
ws.send(JSON.stringify(pingMsg))

12、回复ping

{
    msg_type:'pong',
    src_id:2,
    dst_id:1/null,
}

13、重连

reconnMsg = {
    msg_type:'reconn_event',
    message: {
      stream_id,
      src_id:1,
      dst_id:2,
    }
}
ws.send(JSON.stringify(reconnMsg))

14、更新

updateMsg = {
    msg_type:'update_session_event',
    src_id:1,
    dst_id:2,
    message: {
        protocol:  "webRTC",
        media: {
            audio: {
                ...
            },
            video: {
                ...
            }
        },
        svrType: 0
    }
}
ws.send(JSON.stringify(updateMsg))

信令单独走websocket，关于WS的限制，浏览器无法ping pong

  0                   1                   2                   3
  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 +-+-+-+-+-------+-+-------------+-------------------------------+
 |F|R|R|R| opcode|M| Payload len |    Extended payload length    |
 |I|S|S|S|  (4)  |A|     (7)     |             (16/64)           |
 |N|V|V|V|       |S|             |   (if payload len==126/127)   |
 | |1|2|3|       |K|             |                               |
 +-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - +
 |     Extended payload length continued, if payload len == 127  |
 + - - - - - - - - - - - - - - - +-------------------------------+
 |                               |Masking-key, if MASK set to 1  |
 +-------------------------------+-------------------------------+
 | Masking-key (continued)       |          Payload Data         |
 +-------------------------------- - - - - - - - - - - - - - - - +
 :                     Payload Data continued ...                :
 + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +
 |                     Payload Data continued ...                |
 +---------------------------------------------------------------+

具体每一bit的意思
FIN      1bit 表示信息的最后一帧
RSV 1-3  1bit each 以后备用的 默认都为 0
Opcode   4bit 帧类型，操作码
Mask     1bit 掩码，是否加密数据，默认必须置为1
Payload  7bit 数据的长度
Masking-key      1 or 4 bit 掩码
Payload data     (x + y) bytes 数据
Extension data   x bytes  扩展数据
Application data y bytes  程序数据

OPCODE字段：4位
解释PayloadData，如果接收到未知的opcode，接收端必须关闭连接。
0x0表示附加数据帧
0x1表示文本数据帧
0x2表示二进制数据帧
0x3-7暂时无定义，为以后的非控制帧保留
0x8表示连接关闭
0x9表示ping
0xA表示pong
0xB-F暂时无定义，为以后的控制帧保留

验证浏览器自动回复pong

ws = require('ws');
wss = new ws.Server({port: 7777});
wss.on('connection', conn_ws => {  console.log(conn_ws);
conn_ws.on('pong', msg => console.log('get cli pong', msg))
conn_ws.ping("clq");})

简单的get请求

const express = require('express')
const app = express()
const port = 3000
app.get('/v1/get123/', (req, res) => res.send('Hello World!'))
app.listen(port, () => console.log(`Example app listening on port ${port}!`))

错误码

http://blog.sina.com.cn/s/blog_145f07e7b0102x3x1.html

原生API再写一版最简版本

基础上看了一门课

image.png
代码地址： https://github.com/avdance/webrtc_web
找到官方文档API，跟着api写了一版SDK，可以自由嵌入任何前端框架，大致思路，此项目现在已废弃脱敏。

import { sendBye, offLine, sendPing, onLine, notifyIce, sendCallResponse, sendCall, sendCancelCall } from './wsSignal'

function webrtcPlayer(option) {
    /**
     * 私有变量，无需加this，模块变量外部无法获取
     * 参数：内部websocket连接，本地流，远程流，远端连接，重新计数次数，websocket是否打开标识
     */
    let _ws
    let _localStream
    let _remoteStream
    let _remotePeerConnection
    let _reconnCount = 0
    let _wsOpen = true
    let _iceCandidateCache = []

    /**
     * 传入参数
     * readme有详细解释
     */
    let handleMessage = option.handleMessage || null
    let wsHost = option.wsHost
    let stunHost = option.stunHost || ""
    let pingTime = option.pingTime || 5
    let reconnTimes = option.reconnTimes || 3
    //set constraints
    const mediaStreamConstraints = option.mediaStreamConstraints || {
        video: true,
        audio: true,
    }
    // Set up to exchange only video.
    const offerOptions = option.offerOptions || {
        offerToReceiveVideo: 1,
        offerToReceiveAudio: 1,
    }
    // Define peer connections, streams and video elements.
    let localVideo = option.localVideo || null
    let remoteVideo = option.remoteVideo || null
    let localId = option.localId || ''
    let remoteIds = option.remoteId || []
    let remoteId = ''

    let sessionId = Date.parse(new Date()) + 's'

    // Sets the MediaStream as the video element src.
    const gotLocalMediaStream = (mediaStream) => {
        return new Promise(function (resolve) {
            // @ts-ignore
            if (localVideo) {
                localVideo.srcObject = mediaStream;
            }
            _localStream = mediaStream;
            resolve();
        })
    }

    function callAction() {
        // Add local stream to connection and create offer to connect.
        _remotePeerConnection.addStream(_localStream);
        _remotePeerConnection.createOffer(offerOptions).then(createdOffer).catch();
    }

    // Handles remote MediaStream success by adding it as the remoteVideo src.
    function gotRemoteMediaStream(event) {
        const mediaStream = event.stream;
        remoteVideo.srcObject = mediaStream;
        _remoteStream = mediaStream;
    }


    function reconnect() {
        if (_reconnCount >= reconnTimes - 1) {
            _reconnCount = 0
            wsClose()
        } else {
            _reconnCount++
            wsOnline()
        }
    }

    let sdp = ''
    function receiveCall() {
        console.log('receiveCall');
        const servers = null; // Allows for RTC server configuration.
        _remotePeerConnection = new RTCPeerConnection(servers)
        navigator.mediaDevices
            .getUserMedia(mediaStreamConstraints)
            .then(gotLocalMediaStream)
            .then(() => {
                _remotePeerConnection.addEventListener('icecandidate', handleConnection)
                _remotePeerConnection.addEventListener('addstream', gotRemoteMediaStream)
                // 获取对端sdp,返回callResponse
                let description = new RTCSessionDescription({
                    type: 'offer',
                    sdp: sdp,
                });
                _remotePeerConnection.addStream(_localStream);
                _remotePeerConnection.setRemoteDescription(description)
                _remotePeerConnection.createAnswer().then(createdAnswer)
            })
    }

    function cancelCall() {
        _ws.send(sendCancelCall(sessionId, localId, remoteId))
    }

    const wsClose = value => {
        if (remoteVideo && remoteVideo.srcObject) {
            remoteVideo.srcObject.getTracks().forEach(track => track.stop())
        }
        if (localVideo && localVideo.srcObject) {
            localVideo.srcObject.getTracks().forEach(track => track.stop())
        }
        if (_remotePeerConnection) {
            _remotePeerConnection.close()
            _remotePeerConnection = null
        }
        if (value) {
            _ws.send(sendBye(sessionId, localId, remoteId))
        }
    }

    const wsOffline = () => {
        // wsClose(true)
        _wsOpen = false
        _ws.send(offLine(localId))
        _ws.close()
    }

    const wsOnline = (local_id, remote_id, token, extra_data) => {
        localId = local_id
        remoteIds = remote_id
        console.log('connect')
        _ws = new WebSocket(wsHost)
        _ws.onopen = () => {
            console.info('注册连接成功')
            _wsOpen = true
            // setInterval(() => {
            //     _ws.send(sendPing(localId));
            // }, pingTime * 1000);
            _ws.send(onLine(localId, token, pingTime + 2, extra_data))
        };
        _ws.onclose = () => {
            if (_wsOpen) {
                reconnect()
            } else {
                handleMessage({evt: 'onClose'})
                console.log('连接已关闭')
            }
        };
        _ws.onmessage = (evt) => {
            let message = JSON.parse(evt.data);
            console.log('---evt.data---', message)
            if (message.msg_id === 'onlineResponse') {
                // 获取对端sdp，收集ice
                console.log('登录成功');
            }
            if (message.msg_id === 'callResponse') {
                handleMessage({evt: 'onCallResponse'})
                // 获取对端sdp，收集ice
                let description = new RTCSessionDescription({
                    type: 'answer',
                    sdp: message.data.sdp,
                });
                _remotePeerConnection.setRemoteDescription(description);
                remoteId = message.data.callee_ids[0]
                _iceCandidateCache.forEach(iceCandidate => {
                    _ws.send(notifyIce(sessionId, localId, remoteId, iceCandidate))
                })
            }
            if (message.msg_id === 'call') {
                sdp = message.data.sdp
                sessionId = message.data.session_id
                remoteId = message.data.caller_id
                handleMessage({evt: 'onCall', remoteId: remoteId})
            }
            if (message.msg_id === 'cancelCall') {
                sessionId = message.data.session_id
                handleMessage({evt: 'onCancelCall'})
            }
            if (message.msg_id === 'notifyIceCandidate') {
                let iceCandidate = new RTCIceCandidate({
                    candidate: message.data.candidate,
                    sdpMid: message.data.sdp_mid,
                    sdpMLineIndex: message.data.sdp_mline_index,
                });
                if (iceCandidate && _remotePeerConnection && _remotePeerConnection.remoteDescription) {
                    console.log(iceCandidate)
                    console.log(_remotePeerConnection)
                    _remotePeerConnection.addIceCandidate(iceCandidate).catch(e => {
                        console.log("Failure during addIceCandidate(): " + e.name);
                    });
                }
            }
            if (message.msg_id === 'bye') {
                handleMessage({evt: 'onBye'})
                wsClose()
            }
        };
    };

    const wsCall = (remote_id) => {
        console.log('call');
        if (remote_id) {
            remoteIds = remote_id
        }
        _iceCandidateCache = []
        const servers = null; // Allows for RTC server configuration.
        _remotePeerConnection = new RTCPeerConnection(servers)
        navigator.mediaDevices
            .getUserMedia(mediaStreamConstraints)
            .then(gotLocalMediaStream)
            .then(() => {
                _remotePeerConnection.addEventListener('icecandidate', handleConnectionCall)
                _remotePeerConnection.addEventListener('addstream', gotRemoteMediaStream)
                callAction()
            })
    };

    function handleConnection(event) {
        const peerConnection = event.target;
        const iceCandidate = event.candidate;
        if (iceCandidate && iceCandidate.candidate) {
            _ws.send(notifyIce(sessionId, localId, remoteId, iceCandidate));
        }
    }

    function handleConnectionCall(event) {
        const peerConnection = event.target;
        const iceCandidate = event.candidate;
        if (iceCandidate && iceCandidate.candidate) {
            _iceCandidateCache.push(iceCandidate)
        }
    }

    // Logs offer creation and sets peer connection session descriptions.
    function createdOffer(description) {
        _ws.send(sendCall(sessionId, localId, remoteIds, description.sdp));
        _remotePeerConnection.setLocalDescription(description);
    }

    // Logs answer to offer creation and sets peer connection session descriptions.
    function createdAnswer(description) {
        _ws.send(sendCallResponse(sessionId, localId, remoteId, description.sdp))
        _remotePeerConnection.setLocalDescription(description);
    }

    return {
        wsCall: wsCall,
        wsClose: wsClose,
        receiveCall: receiveCall,
        cancelCall: cancelCall,
        wsOnline: wsOnline,
        wsOffline: wsOffline,
    }
}

export default webrtcPlayer;

其它方面

1、用kurento写一版
2、端上只有小程序无法实现，需要用腾讯云TRTC，需要云上翻译才能对接，无法直接接webRTC
3、关于保活，很复杂也很重要
4、建立音视频，时序很重要，缓存做不好不行

项目已经不做一年了，记录下当时的一些过程

对于新技术

1、源码很重要，看懂入口出口，各种函数流转过程
2、官方API很重要，浏览器是一个黑盒子，封装了很多神奇的API，直接用。当然第三方库的思路差不多，区别就是可以看源码，白盒子。
3、快速入门能花钱的就要花，有前人总结过最好，当然是随缘，最近也做到很多无人涉及的板块。
4、所以英语好很重要，无中文资料的项目太多了。