Protobuf Guide.txt

## https://developers.google.com/protocol-buffers/docs/pythontutorial
## https://developers.google.com/protocol-buffers/docs/proto
## https://developers.google.com/protocol-buffers/docs/proto3
## https://github.com/google/protobuf/blob/master/src/google/protobuf/api.proto
## https://github.com/grpc-ecosystem/grpc-gateway/blob/master/examples/examplepb/a_bit_of_everything.proto
## https://github.com/grpc/grpc/tree/master/examples
## https://grpc.io/
##
##
## https://developers.google.com/protocol-buffers/docs/reference/python-generated
##

## grpc testing!
## https://github.com/grpc/grpc/tree/master/src/python/grpcio_tests/tests/testing

## node grpc
## https://github.com/grpc/grpc/issues/8339
## https://github.com/grpc/grpc-node/pull/204

# install protoc
mkdir -p ~/.local/bin
mkdir -p ~/.local/include

cd ~/Downloads
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.12.3/protoc-3.12.3-linux-x86_64.zip
unzip protoc-3.7.1-linux-x86_64.zip -d protoc3
mv protoc3/bin/* ~/.local/bin/
mv protoc3/include/* ~/.local/include/
rm -rfv protoc3

# configure profile
vim ~/.profile
..........................................
export PATH="$HOME/bin:$HOME/.local/bin:$PATH"
..........................................


# types
bool, int32, float, double, string

(repeated) (type|<enum>|<message>) (<name>) = (<tag>) (([default = <value>]))


.................................................................
syntax = "proto2";

package tutorial;

// comment
/*
multi line comment
*/

message Person {
  string name = 1;
  int32 id = 2;
  string email = 3;

  enum PhoneType {
    MOBILE = 0;
    HOME = 1;
    WORK = 2;
  }

  message PhoneNumber {
    string number = 1;
    PhoneType type = 2 [default = HOME];
  }

  PhoneNumber phones = 4;
}

message AddressBook {
  Person people = 1;
}
.................................................................
.................................................................

# call compiler
protoc -I=$SRC_DIR --python_out=$DST_DIR $SRC_DIR/addressbook.proto
# install python dependencies

# using pipenv
pipenv install protobuf
pipenv install grpcio
pipenv install grpcio-tools --dev
pipenv run python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. helloworld.proto

# using go
go get google.golang.org/grpc

# standard message methods
	IsInitialized()
	__str__()
	CopyFrom(other_msg)
	Clear()
	SerializeToString()
	HasField(field_name)
	ClearField(field_name)
	ListFields() -> array of (field_name, value)
	WhichOneof(oneof_group) -> returns name of field set by oneof group
	ByteSize()

# standard message fields
	DESCRIPTOR.full_name

# static methods
	SerializeToString()
	ParseFromString(data) OR ParseFromString(bytearray(data, 'ascii'))
	FromString(str)



####################################################################

# reserving fields
message Foo {
  reserved 2, 15, 9 to 11;
  reserved "foo", "bar";
}

# enums
## REMEMBER: the first value should be 0
enum EnumAllowingAlias {
  option allow_alias = true;
  UNKNOWN = 0;
  STARTED = 1;
  RUNNING = 1;
}

enum Foo {
  reserved 2, 15, 9 to 11, 40 to max;
  reserved "FOO", "BAR";
}

# importing definitions
import "myproject/other_protos.proto";
# importing and re-exporting definitions (forwarded importing!)
import public "new.proto";

# any!
# for example, in Java, the Any type will have special pack() and unpack() accessors, while in C++ there are PackFrom() and UnpackTo() methods
import "google/protobuf/any.proto";
message ErrorStatus {
  string message = 1;
  repeated google.protobuf.Any details = 2;
}

# struct!
# to hold unstructured data like JSON and dicts
# value of a key could be: struct, listvalue, bool, string, double, null
import "google/protobuf/struct.proto";
message MyMessage {
  google.protobuf.Struct more = 1;
}

# timstamp
# has "int64 seconds" and "int32 nanos"
import "google/protobuf/timestamp.proto";

# duration
# has "int64 seconds" and "int32 nanos"
import "google/protobuf/duration.proto";

# oneof!
# CAN NOT USE REPEATED!
#Be careful when adding or removing oneof fields. If checking the value of a oneof returns None/NOT_SET, it could mean that the oneof has not been set or it has been set to a field in a different version of the oneof. 
message SampleMessage {
  oneof test_oneof {
    string name = 4;
    SubMessage sub_message = 9;
  }
}

# extend
extend google.protobuf.FieldOptions {
    MyFieldOptions my_field = 1000;
}

# custom options
import "google/protobuf/descriptor.proto";
message MyFieldOptions {
	bool nullable = 4;
}
extend google.protobuf.FieldOptions {
    MyFieldOptions my_field = 1000;
}
message MyMessage {
    string token = 1 [(my_field).nullable = true];
}

# maps!
# key type should be scalar or string, NOT enum!
# value can not be map!
# can not be repeated!
map<string, Project> projects = 3;
# is backward compatible with a hack!
message MapFieldEntry {
  key_type key = 1;
  value_type value = 2;
}
repeated MapFieldEntry map_field = N;

# empty!
import "google/protobuf/empty.proto";
service SomeService {
  rpc ParamLess (google.protobuf.Empty) returns (Something);
}

# service declaration
service SearchService {
  rpc Search (SearchRequest) returns (SearchResponse);
}

############################################
Updating proto
############################################

you must not change the tag numbers of any existing fields.
you must not add or delete any required fields.
you may delete optional or repeated fields.
you may add new optional or repeated fields but you must use fresh tag numbers (i.e. tag numbers that were never used in this protocol buffer, not even by deleted fields).

you may change default value of optional

optional can be converted to extension while tag and type remains same

optional is compatible with repeated

Compatible types:
{int32,uint32,int64,uint64,bool}
{sint32,sint64}
{fixed32,sfixed32}
{fixed64,sfixed64}
{string,bytes}
{enum -> int32, uint32, int64, uint64}

############################################
PYTHON GUIDE
############################################

# any!
any_message.Pack(message)
any_message.Unpack(message)
assert any_message.Is(message.DESCRIPTOR)

from google.protobuf.any_pb2 import Any


# timestamp
msg.ToJsonString()
msg.FromJsonString(str)
msg.ToDatetime()
msg.FromDatetime(dt)
msg.To{Seconds,Milliseconds,Microseconds,Nanoseconds}
msg.From{Seconds,Milliseconds,Microseconds,Nanoseconds}

from google.protobuf.timestamp_pb2 import Timestamp

# duration
msg.ToJsonString()
msg.FromJsonString(str)
msg.ToDatetime()
msg.FromDatetime(dt)
msg.To{Seconds,Milliseconds,Microseconds,Nanoseconds}
msg.From{Seconds,Milliseconds,Microseconds,Nanoseconds}

# repeated!
.append(x)
.extend([x,y])
[0] = x
print([0])
[:] = [x, y]
del [:]
for x in ...:
  print(x)
len(field)

# maps
print (f[3])
f[3] = 5
for key in f:
  print f[key]
if key in f:
  pass
del f[key]
f.get_or_create(key)
dict(f)

# enum
self.assertEqual('VALUE_A', myproto_pb2.SomeEnum.Name(myproto_pb2.VALUE_A))
self.assertEqual(5, myproto_pb2.SomeEnum.Value('VALUE_B'))

# struct
from google.protobuf.struct_pb2 import Struct
msg["key"] = None
msg["key"] = True
msg["key"] = 5.0
msg["key"] = "hello"
msg.get_or_create_struct("key")
msg.get_or_create_list("list")
dict(msg)

list.extend([1, "hello", True])
list.append("omg!")
len(list)
list[0]
list.add_struct()
list.add_list()

############################################
GRPC SERVER
############################################

class Greeter(helloworld_pb2_grpc.GreeterServicer):
    def SayHello(self, request, context):
        return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name)

def serve():
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
    helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server)
    server.add_insecure_port('[::]:50051')
    server.start()

if __name__ == '__main__':
    serve()

############################################
GRPC CLIENT
############################################

def run():
    channel = grpc.insecure_channel('app:8000')
    stub = MathServiceStub(channel)

    try:
        print(stub.add(AddRequest(x=3, y=6)))
    except Exception as e:
        code = e.code()
        details = e.details()
        print(e)


if __name__ == '__main__':
    run()

############################################

### REMEMBER! ALWAYS RETURN EMPTY DEFAULT RESULT WHEN USING set_code AND set_details FOR RETURNING ERRORS!

## NOTE:
## connection failed is an exception
### -> code -> grpc.StatusCode.UNAVAILABLE
### -> details -> 'Connect Failed'


############################################
@enum.unique
class StatusCode(enum.Enum):
    """Mirrors grpc_status_code in the gRPC Core."""
    OK = (_cygrpc.StatusCode.ok, 'ok')
    CANCELLED = (_cygrpc.StatusCode.cancelled, 'cancelled')
    UNKNOWN = (_cygrpc.StatusCode.unknown, 'unknown')
    INVALID_ARGUMENT = (_cygrpc.StatusCode.invalid_argument, 'invalid argument')
    DEADLINE_EXCEEDED = (_cygrpc.StatusCode.deadline_exceeded,
                         'deadline exceeded')
    NOT_FOUND = (_cygrpc.StatusCode.not_found, 'not found')
    ALREADY_EXISTS = (_cygrpc.StatusCode.already_exists, 'already exists')
    PERMISSION_DENIED = (_cygrpc.StatusCode.permission_denied,
                         'permission denied')
    RESOURCE_EXHAUSTED = (_cygrpc.StatusCode.resource_exhausted,
                          'resource exhausted')
    FAILED_PRECONDITION = (_cygrpc.StatusCode.failed_precondition,
                           'failed precondition')
    ABORTED = (_cygrpc.StatusCode.aborted, 'aborted')
    OUT_OF_RANGE = (_cygrpc.StatusCode.out_of_range, 'out of range')
    UNIMPLEMENTED = (_cygrpc.StatusCode.unimplemented, 'unimplemented')
    INTERNAL = (_cygrpc.StatusCode.internal, 'internal')
    UNAVAILABLE = (_cygrpc.StatusCode.unavailable, 'unavailable')
    DATA_LOSS = (_cygrpc.StatusCode.data_loss, 'data loss')
    UNAUTHENTICATED = (_cygrpc.StatusCode.unauthenticated, 'unauthenticated')
############################################


# Raise exception and handing
@@@ Server
class MathService(MathServiceServicer):
    def add(self, request, context):
        raise Exception("Msg1", "Msg2")
@@@ Client
    try:
        print(stub.add(AddRequest(x=3, y=6)))
    except Exception as e:
        code = e.code()
        details = e.details()
@@@ Values:::
code = grpc.StatusCode.UNKNOWN
details = 'Exception calling application: (\\'Msg1\\', \\'Msg2\\')'

# Return Status code and details
@@@ Server
class MathService(MathServiceServicer):
    def add(self, request, context):
        context.set_code(grpc.StatusCode.NOT_FOUND)
        context.set_details("nop!")
	return AddResponse()
@@@ Client
    try:
        print(stub.add(AddRequest(x=3, y=6)))
    except Exception as e:
        code = e.code()
        details = e.details()
@@@ Values:::
code = grpc.StatusCode.NOT_FOUND
details = 'nop!'

# Return Custom object with status code and details
@@@ Server
class MathService(MathServiceServicer):
    def add(self, request, context):
        context.set_code(grpc.StatusCode.UNAVAILABLE)
        context.set_details(MathError(message="salam!").SerializeToString())
	return AddResponse()
@@@ Client
    try:
        print(stub.add(AddRequest(x=3, y=6)))
    except Exception as e:
        code = e.code()
        details = MathError.FromString(bytearray(e.details(), 'ascii'))

# Return custom error object with Any
@@@ Server
from google.protobuf.any_pb2 import Any
class MathService(MathServiceServicer):
    def add(self, request, context):
        details = Any()
        details.Pack(MathError(message="salam!"))
        context.set_code(grpc.StatusCode.UNAVAILABLE)
        context.set_details(details.SerializeToString())
	return AddResponse()
@@@ Client
    try:
        print(stub.add(AddRequest(x=3, y=6)))
    except Exception as e:
        code = e.code()
        details = Any.FromString(bytearray(e.details(), 'ascii'))

        if (details.Is(MathError.DESCRIPTOR)):
            math_error = MathError()
            details.Unpack(math_error)
            print(math_error)
        else:
            print(e)


# Client -> Server streaming
@@@ Proto
rpc add(stream Value) returns (Result);
@@@ Server
    def add(self, request_iterator, context):
        result = 0.0
        for request in request_iterator:
            if not context.is_active():
                break
            result = result + request.value
        return Result(result=result)
@@@ Client
    try:
        print(stub.add(iter([Value(value=x) for x in range(10)])))
    except Exception as e:
        print(e)
@@@ Node Client (grpcc)
MathService@app:8000> ch = client.add(pr)
MathService@app:8000> ch.write(2)
MathService@app:8000> ch.write(5)
MathService@app:8000> ch.end()

# Client <-> Server streaming
@@@ Proto
rpc fibonachi_stream(stream More) returns (stream Result);
@@@ Server
    def fibonachi_stream(self, request_iterator, context):
        a, b = 0, 1
        logger.info('fibonachi stream called!')
        for request in request_iterator:
            if not context.is_active():
                break
            logger.info('request to write {}!'.format(request.count))
            for _ in range(request.count):
                if not context.is_active():
                    break
                logger.info('returning result: {}'.format(a))
                yield Result(result=a)
                a, b = b, a + b
        logger.info('end fibonachi stream!')
@@@ Client
@@@ Node Client (grpcc)
Math@app:8000> x = client.fibonachiStream()
EventEmitter {}
Math@app:8000> x.on('data', sr)
EventEmitter {}
Math@app:8000> x.write({count:3})
true
Math@app:8000> 
{
  "result": 0
}
Math@app:8000> 
{
  "result": 1
}
Math@app:8000> 
{
  "result": 1
}
Math@app:8000> x.write({count:2})
true
Math@app:8000> 
{
  "result": 2
}
Math@app:8000> 
{
  "result": 3
}
Math@app:8000> x.write({count:3})
true
Math@app:8000> 
{
  "result": 5
}
Math@app:8000> 
{
  "result": 8
}
Math@app:8000> 
{
  "result": 13
}
Math@app:8000> x.end()

####################################################

# Python iterate over fields
for desc in msg.DESCRIPTOR.fields:
  options = desc.GetOptions()
  type = desc.type
  name = desc.name

####################################################
HAPROXY LOAD BALANCING
####################################################
global
  tune.ssl.default-dh-param 1024
 
defaults
  timeout connect 10000ms
  timeout client 60000ms
  timeout server 60000ms
 
frontend fe_http
  mode http
  bind *:8000
  # Redirect to https
  redirect scheme https code 301
 
frontend fe_https
  mode tcp
  bind *:8443 npn spdy/2 alpn h2,http/1.1
  default_backend be_grpc

# gRPC servers running on port 8083-8084
backend be_grpc
  mode tcp
  balance roundrobin
  server srv01 127.0.0.1:8083
  server srv02 127.0.0.1:8084