1. 简介
开源项目：
https://github.com/Genymobile/scrcpy
项目简介:
通过在手机端使用虚拟显示器进行录屏, 并直接使用手机自带的视频编码器将屏幕数据编码成视频流(格式H264), 并将其发送成PC端, 使用ffmpeg对视频流进行解码, 并通过SDL将手机屏幕镜像显示到电脑屏幕, 并且再通过控制流将PC端的鼠标手势等操作发送给APP端对手机进行远程遥控.
技术点:
该项目使用的技术和云游戏或手机直播使用的技术类似, 包括录屏, 视频流编码, 推流, 视频流解码, 控制流远程操控等. 扩展内容需查看其他笔记: ffmpeg, WebRTC
目录结构：
- app：PC端，纯C语言开发, 基于ffmpeg和SDL开发, 作为client端.
- server，APP端，Java语言开发, adb命令行下执行的Java进程(Dex格式)
2. APP端 (Java)
2.1 Server
Server.main()
- createOptions()
- maxSize // 最大尺寸
- bitRate // 比特率
- maxFps // 限帧
- lockedVideoOrientation // 锁定视频方向
- tunnelForward // 默认false，app作为server，监听unix端口，adb forward到PC端口，等待PC端连接。true则反之，adb tunnel。
- crop // 视频截取尺寸
- sendFrameMeta // 是否发送FrameMeta（和视频流一起）
- control // 是否控制
- displayId // 屏幕ID
- scrcpy(opts)
- final Device device = new Device(options);
- DesktopConnection connection = DesktopConnection.open(device, tunnelForward) // 建立与PC的连接， 详见 2.2节
- ScreenEncoder screenEncoder = new ScreenEncoder(options.getSendFrameMeta(), options.getBitRate(), options.getMaxFps());
- if control:
- Controller controller = new Controller(device, connection);
- sender = new DeviceMessageSender(connection);
- startController(controller); // 控制流
- controller.control(); // 开线程调用, 详见 2.4节
- startDeviceMessageSender(controller.getSender());
- sender.loop(); // 开线程调用
- while true:
- connection.sendDeviceMessage(clipboardTextEvent); // 发送剪贴板内容给PC端
- Controller controller = new Controller(device, connection);
- screenEncoder.streamScreen(device, connection.getVideoFd()); // 发送视频流（阻塞）, 详见 2.3节
2.2 DesktopConnection
DesktopConnection.open(device, tunnelForward)
- if tunnelForward:
- LocalServerSocket localServerSocket = new LocalServerSocket("scrcpy"); // app作为server，监听在unix端口，adb forward到PC端口，等待PC端来连接
- videoSocket = localServerSocket.accept(); // 视频流
- controlSocket = localServerSocket.accept(); // 控制流
- else:
- videoSocket = connect("scrcpy"); // app作为client，连接unix “scrcpy”端 <- adb reverse PC端口
- controlSocket = connect("scrcpy");
- DesktopConnection connection = new DesktopConnection(videoSocket, controlSocket);
- connection.send(Device.getDeviceName(), videoSize.getWidth(), videoSize.getHeight()); // 像PC发送设备名称，视频长宽尺寸, PC端读取的代码在device_read_info
- buffer = new byte[64 + 4]
- 64: deviceNames.getBytes()
- 2: width
- 2: height
DesktopConnection.receiveControlMessage()
- msg = controlMessageReader.next()
- controlMessageReader.readFrom(controlInputStream)
- controlInputStream.read(rawBuffer, head, rawBuffer.length - head); // byte[] rawBuffer = new byte[1024];
2.3 ScreenEncoder
ScreenEncoder.streamScreen(device, videoFd)
- Looper.prepareMainLooper();
- Workarounds.fillAppInfo();
- new android.app.ActivityThread()
- Application app = Instrumentation.newApplication(Application.class, ctx);
- .....
- createFormat()
- MediaFormat format = new MediaFormat();
- format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
- format.setFloat(KEY_MAX_FPS_TO_ENCODER, maxFps);
- MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC); // 创建视频编码器
- IBinder display = SurfaceControl.createDisplay("scrcpy", true); // 创建虚拟屏幕 ， 详见 2.5小节
- setSize()
- format.setInteger(MediaFormat.KEY_WIDTH, width);
- format.setInteger(MediaFormat.KEY_HEIGHT, height);
- codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
- Surface surface = codec.createInputSurface();
- setDisplaySurface()
- SurfaceControl.openTransaction();
- SurfaceControl.setDisplaySurface(display, surface);
- SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect);
- SurfaceControl.setDisplayLayerStack(display, layerStack);
- SurfaceControl.closeTransaction();
- codec.start();
- alive = encode(codec, fd);
- int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1); // 获取输出的编码bufferID
- ByteBuffer codecBuffer = codec.getOutputBuffer(outputBufferId); // 获取编码后的视频buffer
- if sendFrameMeta: // 总是true
- writeFrameMeta(fd, bufferInfo, codecBuffer.remaining()); // 发送FrameMeta
- IO.writeFully(fd, codecBuffer); // 发送buffer到视频流
- codec.stop();
2.4 Controller
Controller.control()
- if not device.isScreenOn(): // 如果屏幕没亮，则点击电源键点亮屏幕
- injectKeyCode(KeyEvent.KEYCODE_POWER)
- while true:
- handleEvent();
- msg = connection.receiveControlMessage()
- switch msg.type:
- case TYPE_INJECT_KEYCODE:
- if device.supportsInputEvents():
- injectKeycode(msg.getAction(), msg.getKeycode(), msg.getMetaState());
- KeyEvent event = new KeyEvent();
- device.injectInputEvent(event)
- injectKeycode(msg.getAction(), msg.getKeycode(), msg.getMetaState());
- if device.supportsInputEvents():
- case TYPE_INJECT_TEXT
- case TYPE_INJECT_TOUCH_EVENT
- case TYPE_INJECT_SCROLL_EVENT
- case TYPE_BACK_OR_SCREEN_ON
- case TYPE_EXPAND_NOTIFICATION_PANEL
- case TYPE_COLLAPSE_NOTIFICATION_PANEL
- case TYPE_GET_CLIPBOARD
- sender.pushClipboardText(serviceManager.getClipboardManager().getText());
- case TYPE_SET_CLIPBOARD
- device.setClipboardText(msg.getText());
- serviceManager.getClipboardManager().setText(text);
- device.setClipboardText(msg.getText());
- case TYPE_SET_SCREEN_POWER_MODE
- device.setScreenPowerMode(msg.getAction());
- SurfaceControl.setDisplayPowerMode()
- device.setScreenPowerMode(msg.getAction());
- case TYPE_ROTATE_DEVICE
- case TYPE_INJECT_KEYCODE:
- handleEvent();
2.5 SurfaceControl
SurfaceControl
- createDisplay() // 创建虚拟显示器
- android.view.SurfaceControl.createDisplay(name, secure)
- setDisplaySurface()
- android.view.SurfaceControl.setDisplaySurface(display, surface)
- setDisplayProjection()
- android.view.SurfaceControl.setDisplayProjection(display, orientaion, layerStackRect, displayRect)
- setDisplayLayerStack()
- android.view.SurfaceControl.setDisplayLayerStack(display, layerStack)
- openTransaction()
- android.view.SurfaceControl.openTransaction()
- closeTransaction()
- android.view.SurfaceControl.closeTransaction()
- getBuiltInDisplay()
- android.view.SurfaceControl.getBuiltInDisplay() // or getInternalDisplayToken() if sdk >= android Q
- setDisplayPowerMode()
- android.view.SurfaceControl.setDisplayPowerMode()
- destroyDisplay()
- android.view.SurfaceControl.destroyDisplay()
为什么用反射去调用 android.view.SurfaceControl 接口，而不是使用如下的接口：
import android.hardware.display.DisplayManager;
import android.hardware.display.VirtualDisplay;
import android.media.projection.MediaProjection;
import android.media.projection.MediaProjectionManager;
virtualDisplay = mediaProjection. **createVirtualDisplay** ("WebRTC_ScreenCapture", width, height,
VIRTUAL_DISPLAY_DPI, DISPLAY_FLAGS, new Surface(surfaceTextureHelper.getSurfaceTexture()),
null /* callback */, null /* callback handler */);
这些接口可能需要权限，以及Context，而在命令行运行的dex没有这些。
3. PC端 (C语言)
3.1 main
main()
- scrcpy_parse_args() // 解析参数, TODO
- serial // 多台adb device时指定需要连接的serial
- av_register_all() // ffmpeg注册所有视频编码格式
- avformat_network_init() // ffmpeg初始化网络格式, TODO
- scrcpy(args.opts)
-
server_start() // 开启本地服务
- push_server() // 将APP端的server文件推送(adb push)到手机
- enable_tunnel_any_port()
- enable_tunnel_reverse_any_port() // PC端作为server，监听在local_port, 等待APP端来连接
- adb reverse tcp:<local_port> localabstract:scrcpy
- server_socket = listen_on_port(port)
- net_connect("localhost", port)
- sock = socket(AF_INET, SOCK_STREAM, 0)
- setsockopt(sock, ...)
- bind(sock)
- listen(sock)
- net_connect("localhost", port)
- or enable_tunnel_forward_any_port(） // PC端作为client，通过adb forward去连接监听在scrcpy端口的APP端的server
- adb forward tcp:<local_port> localabstract:scrcpy
- enable_tunnel_reverse_any_port() // PC端作为server，监听在local_port, 等待APP端来连接
- server->process = execute_server(server, params) // 拉起PC端的server
- adb shell CLASSPATH=/data/local/tmp/scrcpy-server.jar app_process
- SDL_CreateThread(run_wait_server)
- cmd_simple_wait(server->process)
-
sdl_init_and_configure(display, render_driver) // SDL初始化
- SDL_Init()
- SDL_SetHint(SDL_HINT_RENER_DRIVER,options->render_driver) // "direct3d", "opengl", "opengles2", "opengles", "metal" and "software"
- SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "1")
- SDL_SetHint(SDL_HINT_MOUSE_FOCUS_CLICKTHROUGH, "1")
- SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0")
- SDL_SetHint(SDL_HINT_VIDEO_MINIMIZE_ON_FOCUS_LOSS, "0")
- SDL_EnableScreenSaver()
-
server_connect_to() // 连接APP端
- if tunnel_forward:
- erver->video_socket = net_accept(server->server_socket);
- server->control_socket = net_accept(server->server_socket);
- else:
- server->video_socket = connect_to_server(server->local_port, attempts, delay);
- server->control_socket = net_connect(IPV4_LOCALHOST, server->local_port);
- if tunnel_forward:
-
device_read_info(server.video_socket) // 读取手机设备名, 对应 2.2小节
-
fps_counter_init() // ALT+I 在控制台显示FPS
-
video_buffer_init(video_buffer) // 初始化视频buffer
-
if control:
- file_handler_init() // 初始化控制流
-
decoder_init() // 初始化解码器
-
recorder_init() // 初始化录制器(直接录制视频到文件)
-
av_log_set_callback(av_log_callback); // ffmpeg日志回调, TODO
-
stream_init() // 初始化视频流
-
stream_start()
-
controller_init() // 初始化控制流流
-
controller_start()
-
screen_init_rendering() // 详见 screen.c, 初始化渲染
-
if opts.turn_screen_off()
- controller_push_msg(screen_power_mode_off_msg) // 关闭手机屏幕
-
if opts.fullscreen():
- screen_switch_fullscreen() // 切换PC端全屏
-
if opts.show_touches:
- wait_show_touches() // 显示触摸
-
event_loop()
- while SDL_WaitEvent(&event): // SDL事件驱动主循环
- handle_event(&event, control)
- switch event.type:
- case EVENT_NEW_FRAME
- case SDL_WINDOWEVENT
- case SDL_TEXTINPUT
- case SDL_KEYDOWN
- case SDL_KEYUP
- case SDL_MOUSEMOTION
- case SDL_MOUSEWHEEL
- case SDL_MOUSEBUTTONDOWN
- case SDL_MOUSEBUTTONUP
- case SDL_FINGERMOTION
- case SDL_FINGERDOWN
- case SDL_FINGERUP
- case SDL_DROPFILE
- switch event.type:
- handle_event(&event, control)
- while SDL_WaitEvent(&event): // SDL事件驱动主循环
-
screen_destroy()
-
- avformat_network_deinit()
execute_server: 拉起PC端的命令及参数说明：
adb shell CLASSPATH=/data/local/tmp/scrcpy-server.jar app_process
/ // unused
com.genymobile.scrcpy.Server // java main class
1.13 // version
0 // max_size
8000000 // bit_rate
0 // max_fps
-1 // lock_video_orientation
false // trunel_forward
- // crop
true // send frame meta
true // iscontrol
NOTE ATTRIBUTES
Created Date: 2020-05-18 04:41:29
Last Evernote Update Date: 2020-05-20 03:22:46
网友评论