Skip to content

Protocol Buffers: 谷歌开源的数据交换格式协议

引言

仓库:https://github.com/protocolbuffers/protobuf

文档:https://protobuf.dev/

protobuf(Protocal Buffers)是广泛使用的序列化、数据交换开源库,在RPC框架brpc、grpc和TensorFlow中都有使用到。

  • 直接安装软件包:
$ apt-get install libprotobuf-dev libprotoc-dev protobuf-compiler

$ protoc --version
libprotoc 3.6.1

定义数据格式协议

addressbook.proto:

syntax = "proto2";

package tutorial;

message Person {
  optional string name = 1;
  optional int32 id = 2;
  optional string email = 3;

  enum PhoneType {
    MOBILE = 0;
    HOME = 1;
    WORK = 2;
  }

  message PhoneNumber {
    optional string number = 1;
    optional PhoneType type = 2 [default = HOME];
  }

  repeated PhoneNumber phones = 4;
}

message AddressBook {
  repeated Person people = 1;
}

编译生成编码解码接口文件

$ protoc -I=$SRC_DIR --cpp_out=$DST_DIR $SRC_DIR/addressbook.proto
$ protoc -I=. --cpp_out=. addressbook.proto

$ ls -1
addressbook.pb.cc
addressbook.pb.h
addressbook.proto

不需要每次手动生成接口文件,可以在CMakeLists.txt中通过CMake提供的命令自动生成接口文件。

写入一条数据

writer.cpp:

#include <iostream>
#include <fstream>
#include <string>
#include "addressbook.pb.h"
using namespace std;

// This function fills in a Person message based on user input.
void PromptForAddress(tutorial::Person* person) {
  cout << "Enter person ID number: ";
  int id;
  cin >> id;
  person->set_id(id);
  cin.ignore(256, '\n');

  cout << "Enter name: ";
  getline(cin, *person->mutable_name());

  cout << "Enter email address (blank for none): ";
  string email;
  getline(cin, email);
  if (!email.empty()) {
    person->set_email(email);
  }

  while (true) {
    cout << "Enter a phone number (or leave blank to finish): ";
    string number;
    getline(cin, number);
    if (number.empty()) {
      break;
    }

    tutorial::Person::PhoneNumber* phone_number = person->add_phones();
    phone_number->set_number(number);

    cout << "Is this a mobile, home, or work phone? ";
    string type;
    getline(cin, type);
    if (type == "mobile") {
      phone_number->set_type(tutorial::Person::MOBILE);
    } else if (type == "home") {
      phone_number->set_type(tutorial::Person::HOME);
    } else if (type == "work") {
      phone_number->set_type(tutorial::Person::WORK);
    } else {
      cout << "Unknown phone type.  Using default." << endl;
    }
  }
}

// Main function:  Reads the entire address book from a file,
//   adds one person based on user input, then writes it back out to the same
//   file.
int main(int argc, char* argv[]) {
  // Verify that the version of the library that we linked against is
  // compatible with the version of the headers we compiled against.
  GOOGLE_PROTOBUF_VERIFY_VERSION;

  if (argc != 2) {
    cerr << "Usage:  " << argv[0] << " ADDRESS_BOOK_FILE" << endl;
    return -1;
  }

  tutorial::AddressBook address_book;

  {
    // Read the existing address book.
    fstream input(argv[1], ios::in | ios::binary);
    if (!input) {
      cout << argv[1] << ": File not found.  Creating a new file." << endl;
    } else if (!address_book.ParseFromIstream(&input)) {
      cerr << "Failed to parse address book." << endl;
      return -1;
    }
  }

  // Add an address.
  PromptForAddress(address_book.add_people());

  {
    // Write the new address book back to disk.
    fstream output(argv[1], ios::out | ios::trunc | ios::binary);
    if (!address_book.SerializeToOstream(&output)) {
      cerr << "Failed to write address book." << endl;
      return -1;
    }
  }

  // Optional:  Delete all global objects allocated by libprotobuf.
  google::protobuf::ShutdownProtobufLibrary();

  return 0;
}

读取一条数据

reader.cpp:

#include <iostream>
#include <fstream>
#include <string>
#include "addressbook.pb.h"
using namespace std;

// Iterates though all people in the AddressBook and prints info about them.
void ListPeople(const tutorial::AddressBook& address_book) {
  for (int i = 0; i < address_book.people_size(); i++) {
    const tutorial::Person& person = address_book.people(i);

    cout << "Person ID: " << person.id() << endl;
    cout << "  Name: " << person.name() << endl;
    if (person.has_email()) {
      cout << "  E-mail address: " << person.email() << endl;
    }

    for (int j = 0; j < person.phones_size(); j++) {
      const tutorial::Person::PhoneNumber& phone_number = person.phones(j);

      switch (phone_number.type()) {
        case tutorial::Person::MOBILE:
          cout << "  Mobile phone #: ";
          break;
        case tutorial::Person::HOME:
          cout << "  Home phone #: ";
          break;
        case tutorial::Person::WORK:
          cout << "  Work phone #: ";
          break;
      }
      cout << phone_number.number() << endl;
    }
  }
}

// Main function:  Reads the entire address book from a file and prints all
//   the information inside.
int main(int argc, char* argv[]) {
  // Verify that the version of the library that we linked against is
  // compatible with the version of the headers we compiled against.
  GOOGLE_PROTOBUF_VERIFY_VERSION;

  if (argc != 2) {
    cerr << "Usage:  " << argv[0] << " ADDRESS_BOOK_FILE" << endl;
    return -1;
  }

  tutorial::AddressBook address_book;

  {
    // Read the existing address book.
    fstream input(argv[1], ios::in | ios::binary);
    if (!address_book.ParseFromIstream(&input)) {
      cerr << "Failed to parse address book." << endl;
      return -1;
    }
  }

  ListPeople(address_book);

  // Optional:  Delete all global objects allocated by libprotobuf.
  google::protobuf::ShutdownProtobufLibrary();

  return 0;
}

CMake编译程序

CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12)
PROJECT(TestProtcolBuffers)

find_package(Protobuf REQUIRED)
if(PROTOBUF_FOUND)
    message(STATUS "library protobuf found")
else()
    message(FATAL_ERROR "library protobuf can not found")
endif()

include_directories(${PROTOBUF_INCLUDE_DIRS})
protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS addressbook.proto)
message(${PROTO_SRCS})
message(${PROTO_HDRS})

message(${CMAKE_CURRENT_BINARY_DIR})
include_directories(${CMAKE_CURRENT_BINARY_DIR})

add_executable(writer writer.cpp ${PROTO_SRCS} ${PROTO_HDRS})
target_link_libraries(writer ${PROTOBUF_LIBRARIES})

add_executable(reader reader.cpp ${PROTO_SRCS} ${PROTO_HDRS})
target_link_libraries(reader ${PROTOBUF_LIBRARIES})

编译运行:

$ cmake ..
-- library protobuf found
/root/src/person/build/addressbook.pb.cc
/root/src/person/build/addressbook.pb.h
/root/src/person/build
-- Configuring done
-- Generating done
-- Build files have been written to: /root/src/person/build

$ make
[ 50%] Built target reader
[100%] Built target writer

$ ./writer test.data
test.data: File not found.  Creating a new file.
Enter person ID number: 101
Enter name: leslie
Enter email address (blank for none):
Enter a phone number (or leave blank to finish):

$ ./reader test.data
Person ID: 101
  Name: leslie

$ ./writer test.data
Enter person ID number: 102
Enter name: lesliezhu
Enter email address (blank for none): x@y.com
Enter a phone number (or leave blank to finish): 191
Is this a mobile, home, or work phone? home
Enter a phone number (or leave blank to finish):

$ ./reader test.data
Person ID: 101
  Name: leslie
Person ID: 102
  Name: lesliezhu
  E-mail address: x@y.com
  Home phone #: 191

改变数据格式协议

在扩展数据格式协议时,如果要保持向前兼容(旧程序可以处理新格式数据)向后兼容(新API可以处理旧格式数据),则有以下原则:

  • 不能改变现有字段的名字,否则新接口文件里面的API会对不上,无法兼容旧格式数据
  • 不能添加或删除任何required字段
  • 可以删除optional或repeated字段
  • 可以添加optional或repeated字段,但不能使用之前使用过(即使删除了)的字段名字

最佳实践原则

  • 不重复使用tag(字段)名字,只要曾经使用过哪怕后来删除了,也不要再次使用
  • 不要改变字段的类型
  • 不要添加required字段
  • 不要让一个Message含有大量的字段,会占用很大内存和降低效率
  • 枚举里面不要包含不确定的值
  • 枚举里面不要使用定义的宏来代替具体的值
  • 不要改变字段的默认值