美文网首页
Protocol Buffer学习笔记(Java&NodeJS)

Protocol Buffer学习笔记(Java&NodeJS)

作者: 飘逸峰 | 来源:发表于2018-03-02 23:35 被阅读0次

    什么是Protocol Buffer

    Protocol Buffers(也称protobuf)是Google公司出品的一种独立于开发语言，独立于平台的可扩展的结构化数据序列机制。通俗点来讲它跟xml和json是一类。是一种数据交互格式协议。
    主要优点是它是基于二进制的，所以比起结构化的xml协议来说，它的体积很少，数据在传输过程中会更快。另外它也支持c++、java、python、php、javascript等主流开发语言。

    官网地址：https://developers.google.com/protocol-buffers/

    Proto3安装

    下载地址：3.x.x的版本基本都按照操作系统和语言进行了区分，系统包里只包含了protoc命令，语言包则是用于编译后使用，比如java需要生成对应的jar包。这里可以根据需要下载对应的操作系统和语言包，比如这里我下载的是protoc-3.5.1-osx-x86_64.zip（苹果系统）和protobuf-java-3.5.1.tar.gz（java语言）。

    • unzip protoc-3.5.1-osx-x86_64.zip
    • 在/etc/profile中添加环境变量PROTOCTL_BUFFER_HOME（protoc-3.5.1-osx-x86_64.zip解压后目录），并在PATH中添加$PROTOCTL_BUFFER_HOME/bin
    • 查看版本：protoc --version ：输出 libprotoc 3.5.1

    以下部分只为自行编译生成对应的jar包，实际上maven中央仓库中已经存在了

    • tar -zxcf protobuf-java-3.5.1.tar.gz，解压后目录名称为protobuf-3.5.1
    • cd protobuf-3.5.1/src，创建软连接 ln -s $PROTOCTL_BUFFER_HOME/bin/protoc protoc
    • cd protobuf-3.5.1/java，mvn package（maven请自行安装），成功后会在protobuf-3.5.1/java/code/target下生成protobuf-java-3.5.1.jar
    • 然后将protobuf-java-3.5.1.jar上传到maven私服或者安装到本地仓库就可以使用了
    mvn install:install-file -Dfile=protobuf-java-3.5.1.jar -DgroupId=com.google.protobuf -DartifactId=protobuf-java -Dversion=3.5.1 -Dpackaging=jar
    • pom中添加依赖
    <!-- protocol buffer -->
<dependency>
    <groupId>com.google.protobuf</groupId>
    <artifactId>protobuf-java</artifactId>
    <version>3.5.1</version>
</dependency>

    Proto2安装

    下载地址：这里只是操作系统包，比如这里我下载的是protoc-2.6.1-osx-x86_64.exe，语言包protobuf-2.6.1.tar.gz。

    • mv protoc-2.6.1-osx-x86_64.exe protoc
    • 将上面重命名后的protoc文件所在目录加到系统环境变量PATH中
    • 查看版本：protoc --version ：输出 libprotoc 2.6.1

    以下部分只为自行编译生成对应的jar包，实际上maven中央仓库中已经存在了

    • tar -zxcf protobuf-2.6.1.tar.gz，解压后目录名称为protobuf-2.6.1
    • cd protobuf-2.6.1/src，创建软连接 ln -s $PROTOCTL_BUFFER_HOME/bin/protoc protoc
    • cd protobuf-2.6.1/java，mvn package（maven请自行安装），成功后会在protobuf-2.6.1/java/target下生成protobuf-java-2.6.1.jar
    • 然后将protobuf-java-2.6.1.jar上传到maven私服或者安装到本地仓库就可以使用了
    mvn install:install-file -Dfile=protobuf-java-2.6.1.jar -DgroupId=com.google.protobuf -DartifactId=protobuf-java -Dversion=2.6.1 -Dpackaging=jar
    • pom中添加依赖
    <!-- protocol buffer -->
<dependency>
    <groupId>com.google.protobuf</groupId>
    <artifactId>protobuf-java</artifactId>
    <version>2.6.1</version>
</dependency>

    Proto使用

    • 先编写proto文件，具体语法请参考通信协议之Protocol buffer(Java篇)
    • 生成java文件：protoc --java_out=. XXXX.proto
    • 生成js文件：protoc --js_out=import_style=commonjs,binary:. XXXX.proto 『只有proto3支持该命令』
    • proto2与proto3语法上有一些不同，但是在使用时却没有特别的不同之处，此外proto3向下兼容proto2，所以可以只安装proto3，然后通过在proto文件中声明『syntax = "proto2";或者syntax = "proto3";』来指定类型

    proto例子

    //syntax = "proto2";
package com.data.upload.proto;

// 4.1 网约车平台公司基本信息接口
message BaseInfoCompany
{
    // 公司标识
    required string CompanyId       = 1;

    // 公司名称
    required string CompanyName     = 2;

    // 统一社会信用代码
    required string Identifier      = 3;

    // 注册地行政区划代码
    required uint32 Address         = 4;

    // 经营范围
    required string BusinessScope   = 5;

    // 通讯地址
    required string ContactAddress  = 6;

    // 经营业户经济类型
    required string EconomicType    = 7;

    // 注册资本
    required string RegCapital      = 8;

    // 法人代表姓名
    required string LegalName       = 9;

    // 法人代表身份证号
    required string LegalID         = 10;

    // 法人代表电话
    required string LegalPhone      = 11;

    // 法人代表身份证扫描件文件编号
    optional string LegalPhoto      = 12;

    // 状态
    required uint32 State           = 13;

    // 操作标识
    required uint32 Flag            = 14;

    // 更新时间
    required uint64 UpdateTime      = 15;

    // 保留字段
    optional string Reserved        = 16;
}

// 4.2 网约车平台公司营运规模信息信息接口
message BaseInfoCompanyStat
{
    // 公司标识
    required string CompanyId       = 1;

    // 平台注册网约车辆数
    required uint32 VehicleNum      = 2;

    // 平台注册驾驶员数
    required uint32 DriverNum       = 3;

    // 操作标识
    required uint32 Flag            = 4;

    // 更新时间
    required uint64 UpdateTime      = 5;

    // 保留字段
    optional string Reserved        = 6;
}

enum IpcType
{
    // 4.1 网约车平台公司基本信息接口
    baseInfoCompany                             = 0x1001;

    // 4.2 网约车平台公司营运规模信息信息接口
    baseInfoCompanyStat                         = 0x1002;
}

message OTIpc
{
     // 公司标识
     required string CompanyId                                  = 1;

     // 消息来源标识
     required string Source                                     = 2;

     // 业务接口代码
     required IpcType IPCType                                   = 3;

    // 4.1 网约车平台公司基本信息接口
    repeated BaseInfoCompany baseInfoCompany                    = 0x1001;

    // 4.2 网约车平台公司营运规模信息信息接口
    repeated BaseInfoCompanyStat baseInfoCompanyStat            = 0x1002;
}

message OTIpcList
{
     repeated OTIpc otpic                     = 1;
}

    java中使用Protocol Buffer

    • 添加依赖
    <!-- protocol buffer -->
<dependency>
    <groupId>com.google.protobuf</groupId>
    <artifactId>protobuf-java</artifactId>
    <version>2.6.1</version>
</dependency>
    • Client端
      //创建对象
  OTIpcDef.BaseInfoCompany.Builder baseInfoCompanyBuilder = OTIpcDef.BaseInfoCompany.newBuilder();
  baseInfoCompanyBuilder.setAddress(110011);
  baseInfoCompanyBuilder.setCompanyId("companyId");
  baseInfoCompanyBuilder.setCompanyName("companyName");
  baseInfoCompanyBuilder.setIdentifier("identifier");
  baseInfoCompanyBuilder.setBusinessScope("BusinessScope");
  baseInfoCompanyBuilder.setContactAddress("ContactAddress");
  baseInfoCompanyBuilder.setEconomicType("EconomicType");
  baseInfoCompanyBuilder.setRegCapital("RegCapital");
  baseInfoCompanyBuilder.setLegalName("LegalName");
  baseInfoCompanyBuilder.setLegalID("LegalID");
  baseInfoCompanyBuilder.setLegalPhone("LegalPhone");
  baseInfoCompanyBuilder.setState(0);
  baseInfoCompanyBuilder.setFlag(1);
  baseInfoCompanyBuilder.setUpdateTime(20180226121212l);

  OTIpcDef.BaseInfoCompany baseInfoCompany = baseInfoCompanyBuilder.build();

  OTIpcDef.OTIpc.Builder otIpcBuilder = OTIpcDef.OTIpc.newBuilder();
  otIpcBuilder.setCompanyId("companyId");
  otIpcBuilder.setSource("Source");
  otIpcBuilder.setIPCType(OTIpcDef.IpcType.baseInfoCompany);

  //如果一次传递多条记录可以使用list
  //List<OTIpcDef.BaseInfoCompany> list  = new ArrayList<OTIpcDef.BaseInfoCompany>();
  //list.add(baseInfoCompany);
  //otIpcBuilder.addAllBaseInfoCompany(list);

  //也可以用add方法一个一个的添加
  otIpcBuilder.addBaseInfoCompany(baseInfoCompany);
  otIpcBuilder.addBaseInfoCompany(baseInfoCompany);

  OTIpcDef.OTIpc otIpc = otIpcBuilder.build();

  OTIpcDef.BaseInfoCompanyStat.Builder baseInfoCompanyStatBuilder = OTIpcDef.BaseInfoCompanyStat.newBuilder();
  baseInfoCompanyStatBuilder.setCompanyId("companyId");
  baseInfoCompanyStatBuilder.setDriverNum(10);
  baseInfoCompanyStatBuilder.setFlag(0);
  baseInfoCompanyStatBuilder.setUpdateTime(20180226121212l);
  baseInfoCompanyStatBuilder.setVehicleNum(5);

  OTIpcDef.BaseInfoCompanyStat baseInfoCompanyStat = baseInfoCompanyStatBuilder.build();

  OTIpcDef.OTIpc.Builder otIpcBuilder2 = OTIpcDef.OTIpc.newBuilder();
  otIpcBuilder2.setCompanyId("companyId");
  otIpcBuilder2.setSource("Source");
  otIpcBuilder2.setIPCType(OTIpcDef.IpcType.baseInfoCompanyStat);

  otIpcBuilder2.addBaseInfoCompanyStat(baseInfoCompanyStat);

  OTIpcDef.OTIpc otIpc2 = otIpcBuilder2.build();

  OTIpcDef.OTIpcList.Builder oTIpcListBuilder = OTIpcDef.OTIpcList.newBuilder();
  oTIpcListBuilder.addOtpic(otIpc);
  oTIpcListBuilder.addOtpic(otIpc2);

  OTIpcDef.OTIpcList otIpcList = oTIpcListBuilder.build();
  //序列话数据
  byte[] array = otIpcList.toByteArray();

  HttpClientUtils httpClientUtils = new HttpClientUtils();
  httpClientUtils.doPost4ProtocleBuffer("http://localhost:3000/demo/protoc",array);

    HttpClientUtils.java

    import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

public class HttpClientUtils {

    private CloseableHttpClient httpClient;

    private RequestConfig requestConfig;

    public HttpClientUtils(){
        init();
    }


    public void init(){

        PoolingHttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager();
        // 创建全局的requestConfig
        this.requestConfig = RequestConfig.custom().build();
        // 声明重定向策略对象
        LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy();

        this.httpClient = HttpClients.custom().setConnectionManager(httpClientConnectionManager)
                .setDefaultRequestConfig(requestConfig)
                .setRedirectStrategy(redirectStrategy)
                .build();
    }

    public void doPost4ProtocleBuffer(String url, byte[] bytes) throws Exception {


        // 创建http POST请求
        HttpPost httpPost = new HttpPost(url);
        httpPost.setConfig(this.requestConfig);
        httpPost.setHeader("Connection", "keep-alive");
        httpPost.setHeader("Content-type", "application/x-protobuf");
        httpPost.setHeader("Accept-Encoding", "gzip");
        httpPost.setHeader("Accept-Charset", "utf-8");

        if (bytes != null) {
            // 构造一个请求实体
            ByteArrayEntity byteArrayEntity = new ByteArrayEntity(bytes);
            byteArrayEntity.setContentType("application/x-protobuf");
            // 将请求实体设置到httpPost对象中
            httpPost.setEntity(byteArrayEntity);
        }
        CloseableHttpResponse response = null;
        try {
            // 执行请求
            response = this.httpClient.execute(httpPost);
           
        } finally {
            if (response != null) {
                response.close();
            }
        }
    }
}
    • server端
    InputStream in = request.getInputStream();
OTIpcDef.OTIpcList otIpcList = OTIpcDef.OTIpcList.parseFrom(in);
List<OTIpcDef.OTIpc> list= otIpcList.getOtpicList();
for(OTIpcDef.OTIpc otIpc : list){
    String companyid = otIpc.getCompanyId();
    String source = otIpc.getSource();
    OTIpcDef.IpcType ipcType = otIpc.getIPCType();
    if(ipcType == OTIpcDef.IpcType.baseInfoCompany){
        List<OTIpcDef.BaseInfoCompany> baseInfoCompanyList = otIpc.getBaseInfoCompanyList();
        for(OTIpcDef.BaseInfoCompany baseInfoCompany : baseInfoCompanyList){
            String companyName = baseInfoCompany.getCompanyName();
        }
    }else if(ipcType == OTIpcDef.IpcType.baseInfoCompanyStat){
        List<OTIpcDef.BaseInfoCompanyStat> baseInfoCompanyStatList = otIpc.getBaseInfoCompanyStatList();
        for(OTIpcDef.BaseInfoCompanyStat baseInfoCompanyStat : baseInfoCompanyStatList){
            int driverNum = baseInfoCompanyStat.getDriverNum();
        }
    }
}

    nodejs中使用Protocol Buffer

    • 安装依赖
      npm install google-protobuf --save
      npm install bufferhelper --save

    • Client端

    var OTIpcDefProto = require('../protocbuf/OTIpcDef_pb');
var http = require('http');

//业务对象封装
var baseInfoCompany = new OTIpcDefProto.BaseInfoCompany();
baseInfoCompany.setAddress(110011);
baseInfoCompany.setCompanyid("companyId");
baseInfoCompany.setIdentifier("identifier");
baseInfoCompany.setCompanyname("companyName公司名称");
baseInfoCompany.setBusinessscope("BusinessScope");
baseInfoCompany.setContactaddress("ContactAddress");
baseInfoCompany.setEconomictype("EconomicType");
baseInfoCompany.setRegcapital("RegCapital");
baseInfoCompany.setLegalname("LegalName");
baseInfoCompany.setLegalid("LegalID");
baseInfoCompany.setLegalphone("LegalPhone");
baseInfoCompany.setState(0);
baseInfoCompany.setFlag(1);
baseInfoCompany.setUpdatetime(20180226121212);

//业务类型封装
var otIpc = new OTIpcDefProto.OTIpc();
otIpc.setCompanyid("companyId");
otIpc.setSource("Source");
otIpc.setIpctype(OTIpcDefProto.IpcType.BASEINFOCOMPANY);
//可以多次调用add方法添加多条业务对象数据
otIpc.addBaseinfocompany(baseInfoCompany);

//统一封装为list传输
var otIpcList = new OTIpcDefProto.OTIpcList();
//可以通过add方法条件多条业务类型数据
otIpcList.addOtpic(otIpc);

//序列化对象
var contents = otIpcList.serializeBinary();


var options = {
    host: 'localhost',
    port: 3000,
    path: '/demo2/protoc',
    method: 'POST',
    headers: {
        'Content-Type': 'application/x-protobuf'
    }
};

//发送请求
var req = http.request(options, function(res){
    // res.setEncoding('uft8');
    res.on('data', function(data){
        console.log(data);
    });
});

//转成buffer
var buffer = new Buffer(contents);
//只支持string和buffer类型
req.write(buffer);
req.end();
    • Server端（express）
    var express = require('express');
var router = express.Router();
var OTIpcDefProto = require('../protocbuf/OTIpcDef_pb');
var BufferHelper = require('bufferhelper');


// http://localhost:3000/demo/
router.post('/protoc', function(req, res, next) {
    //数据接收，可以使用bufferHelper接收protocolbuffer数据
    var bufferHelper = new BufferHelper();
    req.on("data", function (chunk) {
        bufferHelper.concat(chunk);
    });
    req.on('end', function () {
        var buffer = bufferHelper.toBuffer();
        //buffer转换为proto对象
        var otIpcList = OTIpcDefProto.OTIpcList.deserializeBinary(new Uint8Array(buffer));

        for(var i=0;i<otIpcList.getOtpicList().length;i++) {
            console.log(i+"========================================");
            var otIpc = otIpcList.getOtpicList()[i];
            var companyid = otIpc.getCompanyid();
            var source = otIpc.getSource();
            var iPCType = otIpc.getIpctype();
            console.log(companyid);
            console.log(source);
            console.log(iPCType);
            if(iPCType == OTIpcDefProto.IpcType.BASEINFOCOMPANY){
                var baseInfoCompanyList = otIpc.getBaseinfocompanyList();
                for(var j=0;j<baseInfoCompanyList.length;j++){
                    console.log(j+"===============baseInfoCompanyList=================");
                    var baseInfoCompany = baseInfoCompanyList[j];
                    console.log(baseInfoCompany.toObject());
                    console.log(baseInfoCompany.getCompanyid());
                    console.log(baseInfoCompany.getCompanyname());
                }

            }else if(iPCType == OTIpcDefProto.IpcType.BASEINFOCOMPANYSTAT){
                var baseInfoCompanyStatList = otIpc.getBaseinfocompanystatList();
                for(var j=0;j<baseInfoCompanyStatList.length;j++){
                    console.log(j+"===============baseInfoCompanyStatList=================");
                    var baseInfoCompanyStat = baseInfoCompanyStatList[j];
                    console.log(baseInfoCompanyStat.toObject());
                    console.log(baseInfoCompanyStat.getCompanyid());
                    console.log(baseInfoCompanyStat.getDrivernum());
                }
            }

        }

        console.log(otIpcList.toObject());

        res.send(otIpcList.toObject());
    });

});

module.exports = router;

    这里可以将protocolbuffer数据的接收过程封装到app.js中

    //以下代码要在路由映射的最上方声明，以保证其先被执行
app.use('/*',function(req, res, next) {

  var contentType = req.get('Content-Type');
  //判断contentType，如果是protobuf类型则将数据封装到req.body中
  if(contentType=='application/x-protobuf') {
      var bufferHelper = new BufferHelper();
      req.on("data", function (chunk) {
          bufferHelper.concat(chunk);
      });
      req.on('end', function () {
          var buffer = bufferHelper.toBuffer();
          req.body = buffer;
          console.log(req.body);
          next();
      });
  }else{
      next();
  }

});

    然后在路由js中只需要按照如下方式接收数据即可

    var otIpcList = OTIpcDefProto.OTIpcList.deserializeBinary(new Uint8Array(req.body));
    • Server端（restify）
      restify中接收proto数据比较简单，因为proto数据已经被封装到req.body中了，所以使用方式类似于上面express的第二种方法
    var otIpcList = OTIpcDefProto.OTIpcList.deserializeBinary(new Uint8Array(req.body));

    JSON与Protobuf相互转换

    JAVA

    <!-- protocol buffer format -->
<dependency>
    <groupId>com.googlecode.protobuf-java-format</groupId>
    <artifactId>protobuf-java-format</artifactId>
    <version>1.4</version>
</dependency>
    • json to proto
    com.googlecode.protobuf.format.JsonFormat jsonFormat = new JsonFormat();
com.google.protobuf.Message.Builder builder = OTIpcDef.BaseInfoCompany.newBuilder();
//这里实际上需要提供一个json字符串，这里假设这个json是从某个对象转换而来的
String json = com.alibaba.fastjson.JSON.toJSONString(myObject);
//该方法会将json中与builder所代表的对象中的属性做merge，也就是说只要字段名称和类型一致即可进行封装，对于字段名称和类型匹配不上的属性不予处理，方法成功后builder对象会完成属性值的封装。
jsonFormat.merge(new ByteArrayInputStream(json.getBytes()), builder);
    • proto to json
    OTIpcDef.OTIpcList otIpcList = oTIpcListBuilder.build();
//proto对象转json
com.googlecode.protobuf.format.JsonFormat jsonFormat = new JsonFormat();
String json =jsonFormat.printToString(otIpcList);

    nodejs

    • json to proto
      编写json2Proto.js，里面就一个方法，用于将json字符串转换为封装好的proto对象
    var json2proto = function (json_str,protoObject) {
    Array.prototype.contains = function ( needle ) {
        for (i in this) {
            if (this[i] == needle) return true;
        }
        return false;
    }
    var p_json_str = json_str;
    var p_json = eval("(" + p_json_str + ")");
    var p_json_key_array = [];
    var i = 0;
    for(var p in p_json){//遍历json对象的每个key/value对,p为key
        p_json_key_array[i] = p;
        i++;
    }
    var s_json = protoObject.toObject();
    for(var p in s_json){//遍历json对象的每个key/value对,p为key
        if (p_json_key_array.contains(p)) {
            var setMethod = "set"+p.charAt(0).toUpperCase() + p.slice(1);
            protoObject[setMethod](p_json[p]);
        }
    }
    return protoObject;
}
module.exports = json2proto;

    调用方法

    var OTIpcDefProto = require('../protocbuf/OTIpcDef_pb');
var json2proto = require('../json2Proto');
//json字符串
var p_json_str = "{ companyid: '公司ID'," +
    "companyname: 'companyId'," +
    "identifier : 'identifier'," +
    "address : 111111," +
    "businessscope : 'businessscope'," +
    "contactaddress : 'contactaddress'," +
    "economictype : 'economictype'," +
    "regcapital : 'regcapital'," +
    "legalname : 'legalname'," +
    "legalid : 'legalid'," +
    "legalphone : 'legalphone'," +
    "legalphoto : 'legalphoto'," +
    "state : 0," +
    "flag : 1," +
    "updatetime: 20180226121212}";
var baseInfoCompany = json2proto(p_json_str,new OTIpcDefProto.BaseInfoCompany());

console.log(baseInfoCompany.toObject());

    相关文章

      网友评论

          本文标题：Protocol Buffer学习笔记(Java&NodeJS)

          本文链接：https://www.haomeiwen.com/subject/uxizxftx.html

          延伸阅读

          深度阅读

          • 您也可以注册成为美文阅读网的作者，发表您的原创作品、分享您的心情！

          栏目导航

          热点阅读

          关于我们|服务条款|联系我们|Protocol Buffer学习笔记(Java&NodeJS)|投稿指南|网站地图|RSS订阅|排版工具|手机版

          提供经典美文摘抄,优美散文欣赏,现代诗歌精选,短篇小说,心情随笔,表白情书范文,故事会在线阅读欣赏

          Copyright © 2014-2023 Haomeiwen.com All Rights Reserved. 好美文阅读网 版权所有

          备案信息：桂公网安备 45052102000051号 · 桂ICP备13007215号-3

          本站所收录作品、热点评论等信息部分来源互联网，目的只是为了系统归纳学习和传递资讯

          所有作品版权归原创作者所有，与本站立场无关，如不慎侵犯了你的权益，请联系我们告知，我们将做删除处理！