diff --git a/.editorconfig b/.editorconfig index 79eed820e3e0..62357920aa2e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -50,3 +50,4 @@ indent_style = unset indent_style = unset insert_final_newline = unset trim_trailing_whitespace = unset +end_of_line = unset diff --git a/apisix/plugins/grpc-transcode/util.lua b/apisix/plugins/grpc-transcode/util.lua index dc4526195639..4a27c1d3b9df 100644 --- a/apisix/plugins/grpc-transcode/util.lua +++ b/apisix/plugins/grpc-transcode/util.lua @@ -22,6 +22,7 @@ local ngx = ngx local string = string local table = table local ipairs = ipairs +local pairs = pairs local tonumber = tonumber local type = type @@ -131,7 +132,7 @@ local function get_from_request(request_table, name, kind) end -function _M.map_message(field, default_values, request_table) +function _M.map_message(field, default_values, request_table, real_key) if not pb.type(field) then return nil, "Field " .. field .. " is not defined" end @@ -164,15 +165,34 @@ function _M.map_message(field, default_values, request_table) end sub = sub_array else - sub, err = _M.map_message(field_type, default_values and default_values[name], - request_table[name]) - if err then - return nil, err + if ty == "map" then + for k, v in pairs(request_table[name]) do + local tbl, err = _M.map_message(field_type, + default_values and default_values[name], + request_table[name], k) + if err then + return nil, err + end + if not sub then + sub = {} + end + sub[k] = tbl[k] + end + else + sub, err = _M.map_message(field_type, + default_values and default_values[name], + request_table[name]) + if err then + return nil, err + end end end request[name] = sub else + if real_key then + name = real_key + end request[name] = get_from_request(request_table, name, field_type) or (default_values and default_values[name]) end diff --git a/t/grpc_server_example/echo.pb b/t/grpc_server_example/echo.pb new file mode 100644 index 000000000000..3f82f2554acc Binary files /dev/null and b/t/grpc_server_example/echo.pb differ diff --git a/t/grpc_server_example/go.mod b/t/grpc_server_example/go.mod index 8d36bae682b7..9b95e63fdeaa 100644 --- a/t/grpc_server_example/go.mod +++ b/t/grpc_server_example/go.mod @@ -3,6 +3,7 @@ module github.com/api7/grpc_server_example go 1.11 require ( + github.com/golang/protobuf v1.5.0 golang.org/x/net v0.2.0 google.golang.org/grpc v1.32.0 google.golang.org/protobuf v1.27.1 diff --git a/t/grpc_server_example/main.go b/t/grpc_server_example/main.go index fdb8dedd730a..6eb4a9b32ae0 100644 --- a/t/grpc_server_example/main.go +++ b/t/grpc_server_example/main.go @@ -18,6 +18,8 @@ //go:generate protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative proto/helloworld.proto //go:generate protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative proto/import.proto //go:generate protoc --include_imports --descriptor_set_out=proto.pb --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative proto/src.proto +//go:generate protoc --descriptor_set_out=echo.pb --include_imports --proto_path=$PWD/proto echo.proto +//go:generate protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative proto/echo.proto // Package main implements a server for Greeter service. package main @@ -76,6 +78,7 @@ type server struct { // Embed the unimplemented server pb.UnimplementedGreeterServer pb.UnimplementedTestImportServer + pb.UnimplementedEchoServer } // SayHello implements helloworld.GreeterServer @@ -141,6 +144,14 @@ func (s *server) Plus(ctx context.Context, in *pb.PlusRequest) (*pb.PlusReply, e return &pb.PlusReply{Result: in.A + in.B}, nil } +func (s *server) EchoStruct(ctx context.Context, in *pb.StructRequest) (*pb.StructReply, error) { + log.Printf("Received: %+v", in) + + return &pb.StructReply{ + Data: in.Data, + }, nil +} + // SayHelloServerStream streams HelloReply back to the client. func (s *server) SayHelloServerStream(req *pb.HelloRequest, stream pb.Greeter_SayHelloServerStreamServer) error { log.Printf("Received server side stream req: %v\n", req) @@ -251,6 +262,7 @@ func main() { reflection.Register(s) pb.RegisterGreeterServer(s, &server{}) pb.RegisterTestImportServer(s, &server{}) + pb.RegisterEchoServer(s, &server{}) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) diff --git a/t/grpc_server_example/proto/echo.pb.go b/t/grpc_server_example/proto/echo.pb.go new file mode 100644 index 000000000000..cafcdc24c581 --- /dev/null +++ b/t/grpc_server_example/proto/echo.pb.go @@ -0,0 +1,236 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.6.1 +// source: proto/echo.proto + +package proto + +import ( + _struct "github.com/golang/protobuf/ptypes/struct" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type StructRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Data *_struct.Struct `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *StructRequest) Reset() { + *x = StructRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_echo_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StructRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StructRequest) ProtoMessage() {} + +func (x *StructRequest) ProtoReflect() protoreflect.Message { + mi := &file_proto_echo_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StructRequest.ProtoReflect.Descriptor instead. +func (*StructRequest) Descriptor() ([]byte, []int) { + return file_proto_echo_proto_rawDescGZIP(), []int{0} +} + +func (x *StructRequest) GetData() *_struct.Struct { + if x != nil { + return x.Data + } + return nil +} + +type StructReply struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Data *_struct.Struct `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *StructReply) Reset() { + *x = StructReply{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_echo_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StructReply) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StructReply) ProtoMessage() {} + +func (x *StructReply) ProtoReflect() protoreflect.Message { + mi := &file_proto_echo_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StructReply.ProtoReflect.Descriptor instead. +func (*StructReply) Descriptor() ([]byte, []int) { + return file_proto_echo_proto_rawDescGZIP(), []int{1} +} + +func (x *StructReply) GetData() *_struct.Struct { + if x != nil { + return x.Data + } + return nil +} + +var File_proto_echo_proto protoreflect.FileDescriptor + +var file_proto_echo_proto_rawDesc = []byte{ + 0x0a, 0x10, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x04, 0x65, 0x63, 0x68, 0x6f, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3c, 0x0a, 0x0d, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x22, 0x3a, 0x0a, 0x0b, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x12, 0x2b, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, + 0x32, 0x3e, 0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x36, 0x0a, 0x0a, 0x45, 0x63, 0x68, 0x6f, + 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x12, 0x13, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x53, 0x74, + 0x72, 0x75, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x65, 0x63, + 0x68, 0x6f, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, + 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_proto_echo_proto_rawDescOnce sync.Once + file_proto_echo_proto_rawDescData = file_proto_echo_proto_rawDesc +) + +func file_proto_echo_proto_rawDescGZIP() []byte { + file_proto_echo_proto_rawDescOnce.Do(func() { + file_proto_echo_proto_rawDescData = protoimpl.X.CompressGZIP(file_proto_echo_proto_rawDescData) + }) + return file_proto_echo_proto_rawDescData +} + +var file_proto_echo_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_proto_echo_proto_goTypes = []interface{}{ + (*StructRequest)(nil), // 0: echo.StructRequest + (*StructReply)(nil), // 1: echo.StructReply + (*_struct.Struct)(nil), // 2: google.protobuf.Struct +} +var file_proto_echo_proto_depIdxs = []int32{ + 2, // 0: echo.StructRequest.data:type_name -> google.protobuf.Struct + 2, // 1: echo.StructReply.data:type_name -> google.protobuf.Struct + 0, // 2: echo.Echo.EchoStruct:input_type -> echo.StructRequest + 1, // 3: echo.Echo.EchoStruct:output_type -> echo.StructReply + 3, // [3:4] is the sub-list for method output_type + 2, // [2:3] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_proto_echo_proto_init() } +func file_proto_echo_proto_init() { + if File_proto_echo_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_proto_echo_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StructRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_echo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StructReply); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_proto_echo_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_proto_echo_proto_goTypes, + DependencyIndexes: file_proto_echo_proto_depIdxs, + MessageInfos: file_proto_echo_proto_msgTypes, + }.Build() + File_proto_echo_proto = out.File + file_proto_echo_proto_rawDesc = nil + file_proto_echo_proto_goTypes = nil + file_proto_echo_proto_depIdxs = nil +} diff --git a/t/grpc_server_example/proto/echo.proto b/t/grpc_server_example/proto/echo.proto new file mode 100644 index 000000000000..144c261b13f7 --- /dev/null +++ b/t/grpc_server_example/proto/echo.proto @@ -0,0 +1,35 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package echo; +option go_package = "./proto"; + +import "google/protobuf/struct.proto"; + +service Echo { + rpc EchoStruct (StructRequest) returns (StructReply) {} +} + +message StructRequest { + google.protobuf.Struct data = 1; +} + +message StructReply { + google.protobuf.Struct data = 1; +} diff --git a/t/grpc_server_example/proto/echo_grpc.pb.go b/t/grpc_server_example/proto/echo_grpc.pb.go new file mode 100644 index 000000000000..a546e7f3ae96 --- /dev/null +++ b/t/grpc_server_example/proto/echo_grpc.pb.go @@ -0,0 +1,105 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.6.1 +// source: proto/echo.proto + +package proto + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// EchoClient is the client API for Echo service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type EchoClient interface { + EchoStruct(ctx context.Context, in *StructRequest, opts ...grpc.CallOption) (*StructReply, error) +} + +type echoClient struct { + cc grpc.ClientConnInterface +} + +func NewEchoClient(cc grpc.ClientConnInterface) EchoClient { + return &echoClient{cc} +} + +func (c *echoClient) EchoStruct(ctx context.Context, in *StructRequest, opts ...grpc.CallOption) (*StructReply, error) { + out := new(StructReply) + err := c.cc.Invoke(ctx, "/echo.Echo/EchoStruct", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// EchoServer is the server API for Echo service. +// All implementations must embed UnimplementedEchoServer +// for forward compatibility +type EchoServer interface { + EchoStruct(context.Context, *StructRequest) (*StructReply, error) + mustEmbedUnimplementedEchoServer() +} + +// UnimplementedEchoServer must be embedded to have forward compatible implementations. +type UnimplementedEchoServer struct { +} + +func (UnimplementedEchoServer) EchoStruct(context.Context, *StructRequest) (*StructReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method EchoStruct not implemented") +} +func (UnimplementedEchoServer) mustEmbedUnimplementedEchoServer() {} + +// UnsafeEchoServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to EchoServer will +// result in compilation errors. +type UnsafeEchoServer interface { + mustEmbedUnimplementedEchoServer() +} + +func RegisterEchoServer(s grpc.ServiceRegistrar, srv EchoServer) { + s.RegisterService(&Echo_ServiceDesc, srv) +} + +func _Echo_EchoStruct_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StructRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(EchoServer).EchoStruct(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/echo.Echo/EchoStruct", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(EchoServer).EchoStruct(ctx, req.(*StructRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Echo_ServiceDesc is the grpc.ServiceDesc for Echo service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Echo_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "echo.Echo", + HandlerType: (*EchoServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "EchoStruct", + Handler: _Echo_EchoStruct_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "proto/echo.proto", +} diff --git a/t/plugin/grpc-transcode3.t b/t/plugin/grpc-transcode3.t index 6ce4cc943323..0cdbe6e16bdd 100644 --- a/t/plugin/grpc-transcode3.t +++ b/t/plugin/grpc-transcode3.t @@ -435,3 +435,93 @@ failed to call pb.decode to decode details in grpc-status-details-bin --- error_log transform response error: failed to call pb.decode to decode details in grpc-status-details-bin, err: --- error_code: 503 + + + +=== TEST 11: set binary rule for EchoStruct +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + + local content = t.read_file("t/grpc_server_example/echo.pb") + local data = {content = ngx.encode_base64(content)} + local code, body = t.test('/apisix/admin/protos/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/grpctest", + "plugins": { + "grpc-transcode": { + "proto_id": "1", + "service": "echo.Echo", + "method": "EchoStruct" + } + }, + "upstream": { + "scheme": "grpc", + "type": "roundrobin", + "nodes": { + "127.0.0.1:50051": 1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 12: hit route to test EchoStruct +--- config +location /t { + content_by_lua_block { + local core = require "apisix.core" + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/grpctest" + local body = [[{"data":{"fields":{"foo":{"string_value":"xxx"},"bar":{"number_value":666}}}}]] + local opt = {method = "POST", body = body, headers = {["Content-Type"] = "application/json"}, keepalive = false} + local httpc = http.new() + local res, err = httpc:request_uri(uri, opt) + if not res then + ngx.log(ngx.ERR, err) + return ngx.exit(500) + end + if res.status > 300 then + return ngx.exit(res.status) + else + local req = core.json.decode(body) + local rsp = core.json.decode(res.body) + for k, v in pairs(req.data.fields) do + if rsp.data.fields[k] == nil then + ngx.log(ngx.ERR, "rsp missing field=", k, ", rsp: ", res.body) + else + for k1, v1 in pairs(v) do + if v1 ~= rsp.data.fields[k][k1] then + ngx.log(ngx.ERR, "rsp mismatch: k=", k1, + ", req=", v1, ", rsp=", rsp.data.fields[k][k1]) + end + end + end + end + end + } +}