From 6aa03090e62d88e38fd6d8f501e78cc3c6d8beef Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Thu, 3 Dec 2020 19:34:29 +0530 Subject: [PATCH 01/24] init --- server/config/config.go | 58 +++++++++++++++++++++++++++++++++++++++++ server/config/toml.go | 8 ++++++ 2 files changed, 66 insertions(+) diff --git a/server/config/config.go b/server/config/config.go index 80e79f04558d..58619333c77e 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -105,6 +105,40 @@ type GRPCConfig struct { // Address defines the API server to listen on Address string `mapstructure:"address"` + + // GRPCWebProxy defines the list of flags for grpc-proxy. + GRPCWebProxy GRPCProxy `mapstructure:grpc-proxy` +} + +// GRPProxy defines configuration for the gRPC server. +type GRPCProxy struct { + // Enable defines if the proxy should be enabled. + Enable bool `mapstructure:"enable"` + + // AllowAllOrigins + AllowAllOrigins bool `mapstructure:"allow-all-origins"` + + AllowedOrigins []string `mapstructure:"allowed-origins"` + + AllowedHeaders []string `mapstructure:"allowed-headers"` + + // EnableHTTPServer defines if the HTTP should be enabled. + EnableHTTPServer bool `mapstructure:"enable-http-server"` + + // EnableTLSServer defines if the TLS should be enabled. + EnableTLSServer bool `mapstructure:"enable-tls-server"` + + // TlsServerCert defines the path to the PEM certificate for server use. + TlsServerCert string `mapstructure:"tls-server-cert"` + + // TlsServerKey defines the path to the PEM key for the certificate for the server use. + TlsServerKey string `mapstructure:"tls-server-key"` + + // TlsServerClientCertVerification defines controls whether a client certificate is on. Values: none, verify_if_given, require. + TlsServerClientCertVerification string `mapstructure:"tls-server-client-cert-verification"` + + // TlsServerClientCAFiles defines Paths (comma separated) to PEM certificate chains used for client-side verification. If empty, host CA chain will be used. + TlsServerClientCAFiles string `mapstructure:"tls-server-client-ca-files"` } // StateSyncConfig defines the state sync snapshot configuration. @@ -184,6 +218,18 @@ func DefaultConfig() *Config { GRPC: GRPCConfig{ Enable: true, Address: DefaultGRPCAddress, + GRPCWebProxy: GRPCProxy{ + Enable: true, + AllowAllOrigins: true, + EnableHTTPServer: true, + EnableTLSServer: false, + AllowedOrigins: []string{"*"}, + TlsServerCert: "", + AllowedHeaders: []string{}, + TlsServerClientCAFiles: "", + TlsServerClientCertVerification: "", + TlsServerKey: "", + }, }, StateSync: StateSyncConfig{ SnapshotInterval: 0, @@ -238,6 +284,18 @@ func GetConfig(v *viper.Viper) Config { GRPC: GRPCConfig{ Enable: v.GetBool("grpc.enable"), Address: v.GetString("grpc.address"), + GRPCWebProxy: GRPCProxy{ + Enable: v.GetBool("grpc.grpc-proxy.enable"), + AllowAllOrigins: v.GetBool("grpc.grpc-proxy.allow-all-origins"), + AllowedHeaders: v.GetStringSlice("grpc.grpc-proxy.allowed-headers"), + AllowedOrigins: v.GetStringSlice("grpc.grpc-proxy.allowed-origins"), + EnableHTTPServer: v.GetBool("grpc.grpc-proxy.enable-http-server"), + EnableTLSServer: v.GetBool("grpc.grpc-proxy.enable-tls-server"), + TlsServerCert: v.GetString("grpc.grpc-proxy.tls-server-cert"), + TlsServerClientCAFiles: v.GetString("grpc.grpc-proxy.tls-server-client-ca-files"), + TlsServerClientCertVerification: v.GetString("grpc.grpc-proxy.tls-server-client-cert-verification"), + TlsServerKey: v.GetString("grpc.grpc-proxy.tls-server-key"), + }, }, StateSync: StateSyncConfig{ SnapshotInterval: v.GetUint64("state-sync.snapshot-interval"), diff --git a/server/config/toml.go b/server/config/toml.go index b042da74293d..38e44151d449 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -146,7 +146,15 @@ enable = {{ .GRPC.Enable }} # Address defines the gRPC server address to bind to. address = "{{ .GRPC.Address }}" +[grpc-proxy] + # GRPCWebProxyEnable defines if the gRPC-web-proxy should be enabled. + enable = {{ .GRPC.GRPCWebProxy.Enable }} + # AllowOrigins + allow-all-origin = {{ .GRPC.GRPCWebProxy.AllowAllOrigins }} + + # AllowedOrigins + allowed-origins = {{ .GRPC.GRPCWebProxy.AllowedOrigins }} ############################################################################### ### State Sync Configuration ### ############################################################################### From efaa0b43a7e879c63eba76eadc65ac61145a5a94 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Fri, 4 Dec 2020 12:38:44 +0530 Subject: [PATCH 02/24] WIP config --- server/config/config.go | 8 +++++--- server/config/toml.go | 31 +++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 58619333c77e..25994e573bfb 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -107,7 +107,7 @@ type GRPCConfig struct { Address string `mapstructure:"address"` // GRPCWebProxy defines the list of flags for grpc-proxy. - GRPCWebProxy GRPCProxy `mapstructure:grpc-proxy` + GRPCWebProxy GRPCProxy `mapstructure:"grpc-proxy"` } // GRPProxy defines configuration for the gRPC server. @@ -115,11 +115,13 @@ type GRPCProxy struct { // Enable defines if the proxy should be enabled. Enable bool `mapstructure:"enable"` - // AllowAllOrigins + // AllowAllOrigins defines allow requests from any origin. AllowAllOrigins bool `mapstructure:"allow-all-origins"` + // AllowedOrigins defines list of origin URLs which are allowed to make cross-origin requests. AllowedOrigins []string `mapstructure:"allowed-origins"` + // AllowedHeaders defines list of headers which are allowed to propagate to the gRPC backend. AllowedHeaders []string `mapstructure:"allowed-headers"` // EnableHTTPServer defines if the HTTP should be enabled. @@ -225,7 +227,7 @@ func DefaultConfig() *Config { EnableTLSServer: false, AllowedOrigins: []string{"*"}, TlsServerCert: "", - AllowedHeaders: []string{}, + AllowedHeaders: make([]string, 0), TlsServerClientCAFiles: "", TlsServerClientCertVerification: "", TlsServerKey: "", diff --git a/server/config/toml.go b/server/config/toml.go index 38e44151d449..1fad8ab63256 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -146,15 +146,38 @@ enable = {{ .GRPC.Enable }} # Address defines the gRPC server address to bind to. address = "{{ .GRPC.Address }}" -[grpc-proxy] + +[grpc.grpc-proxy] # GRPCWebProxyEnable defines if the gRPC-web-proxy should be enabled. enable = {{ .GRPC.GRPCWebProxy.Enable }} - # AllowOrigins + # AllowAllOrigins defines allow requests from any origin. allow-all-origin = {{ .GRPC.GRPCWebProxy.AllowAllOrigins }} - # AllowedOrigins - allowed-origins = {{ .GRPC.GRPCWebProxy.AllowedOrigins }} + # AllowedOrigins defines list of origin URLs which are allowed to make cross-origin requests. + allowed-origins = [{{ range $k, $v := .GRPC.GRPCWebProxy.AllowedOrigins }} "{{ $v }}", {{ end }}] + + # AllowedHeaders defines list of headers which are allowed to propagate to the gRPC backend. + allowed-headers = [{{ range $k, $v := .GRPC.GRPCWebProxy.AllowedHeaders }} "{{ $v }}", {{ end }}] + + # EnableHTTPServer defines if the HTTP should be enabled. + enable-http-server = {{ .GRPC.GRPCWebProxy.EnableHTTPServer }} + + # EnableTLSServer defines if the TLS should be enabled. + enable-tls-server = {{ .GRPC.GRPCWebProxy.EnableTLSServer }} + + # TlsServerCert defines the path to the PEM certificate for server use. + tls-server-cert = "{{ .GRPC.GRPCWebProxy.TlsServerCert}}" + + # TlsServerKey defines the path to the PEM key for the certificate for the server use. + tls-server-key = "{{ .GRPC.GRPCWebProxy.TlsServerKey }}" + + # TlsServerClientCertVerification defines controls whether a client certificate is on. Values: none, verify_if_given, require. + tls-server-client-cert-verification = "{{ .GRPC.GRPCWebProxy.TlsServerClientCertVerification }}" + + # TlsServerClientCAFiles defines Paths (comma separated) to PEM certificate chains used for client-side verification. If empty, host CA chain will be used. + tls-server-client-ca-files = "{{ .GRPC.GRPCWebProxy.TlsServerClientCAFiles }}" + ############################################################################### ### State Sync Configuration ### ############################################################################### From 939eb799cb33b46a2eb8b3329096442e69624a17 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Fri, 4 Dec 2020 17:53:34 +0530 Subject: [PATCH 03/24] WIP add proxy server --- go.mod | 4 + go.sum | 10 ++ server/config/config.go | 69 +++++++------- server/config/toml.go | 22 ++--- server/grpc/proxy_server.go | 180 ++++++++++++++++++++++++++++++++++++ server/start.go | 25 ++++- 6 files changed, 263 insertions(+), 47 deletions(-) create mode 100644 server/grpc/proxy_server.go diff --git a/go.mod b/go.mod index b437a1006342..6b51e507a97c 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d github.com/cosmos/iavl v0.15.0-rc5 github.com/cosmos/ledger-cosmos-go v0.11.1 + github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/enigmampc/btcutil v1.0.3-0.20200723161021-e2fb6adb2a25 @@ -26,8 +27,11 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/golang-lru v0.5.4 + github.com/improbable-eng/grpc-web v0.13.0 github.com/magiconair/properties v1.8.4 github.com/mattn/go-isatty v0.0.12 + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f + github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76 github.com/otiai10/copy v1.2.0 github.com/pelletier/go-toml v1.8.0 // indirect github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 4542327b0e4b..2551b26ff867 100644 --- a/go.sum +++ b/go.sum @@ -121,6 +121,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= +github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/dgraph-io/badger/v2 v2.2007.1/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= @@ -295,6 +297,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/improbable-eng/grpc-web v0.13.0 h1:7XqtaBWaOCH0cVGKHyvhtcuo6fgW32Y10yRKrDHFHOc= +github.com/improbable-eng/grpc-web v0.13.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= @@ -304,6 +308,7 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -323,6 +328,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kkdai/bstream v1.0.0/go.mod h1:FDnDOHt5Yx4p3FaHcioFT0QjDOtgUpvjeZqAs+NVZZA= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -368,7 +374,10 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76 h1:0xuRacu/Zr+jX+KyLLPPktbwXqyOvnOPUQmMLzX1jxU= +github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76/go.mod h1:x5OoJHDHqxHS801UIuhqGl6QdSAEJvtausosHSdazIo= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= @@ -494,6 +503,7 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= diff --git a/server/config/config.go b/server/config/config.go index 25994e573bfb..01bb2006956b 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -3,8 +3,10 @@ package config import ( "fmt" "strings" + "time" "github.com/spf13/viper" + "google.golang.org/grpc" storetypes "github.com/cosmos/cosmos-sdk/store/types" "github.com/cosmos/cosmos-sdk/telemetry" @@ -16,6 +18,12 @@ const ( // DefaultGRPCAddress is the default address the gRPC server binds to. DefaultGRPCAddress = "0.0.0.0:9090" + + // DefaultGRPCAddress is the address to bind the server to. + DefaultBindAddress = "0.0.0.0" + + // TCP port to listen on for HTTP1.1 debug calls. + DefaultGRPCProxyPort = 7890 ) // BaseConfig defines the server's basic configuration @@ -115,6 +123,12 @@ type GRPCProxy struct { // Enable defines if the proxy should be enabled. Enable bool `mapstructure:"enable"` + // BindAddress defines address to bind the server to. + BindAddress string `mapstructure:"bind-address"` + + // HTTPPort defines TCP port to listen on for HTTP1.1 debug calls. + HTTPPort int `mapstructure:"http-port"` + // AllowAllOrigins defines allow requests from any origin. AllowAllOrigins bool `mapstructure:"allow-all-origins"` @@ -127,20 +141,11 @@ type GRPCProxy struct { // EnableHTTPServer defines if the HTTP should be enabled. EnableHTTPServer bool `mapstructure:"enable-http-server"` - // EnableTLSServer defines if the TLS should be enabled. - EnableTLSServer bool `mapstructure:"enable-tls-server"` - - // TlsServerCert defines the path to the PEM certificate for server use. - TlsServerCert string `mapstructure:"tls-server-cert"` - - // TlsServerKey defines the path to the PEM key for the certificate for the server use. - TlsServerKey string `mapstructure:"tls-server-key"` - - // TlsServerClientCertVerification defines controls whether a client certificate is on. Values: none, verify_if_given, require. - TlsServerClientCertVerification string `mapstructure:"tls-server-client-cert-verification"` + // MaxCallRecvMsgSize defines maximum receive message size limit. If not specified, the default of 4MB will be used. + MaxCallRecvMsgSize int `mapstructure:"max-call-recv-msg-size"` - // TlsServerClientCAFiles defines Paths (comma separated) to PEM certificate chains used for client-side verification. If empty, host CA chain will be used. - TlsServerClientCAFiles string `mapstructure:"tls-server-client-ca-files"` + // BackendBackoffMaxDelay defines maximum delay when backing off after failed connection attempts to the backend. + BackendBackoffMaxDelay time.Duration `mapstructure:"backend-backoff-max-delay"` } // StateSyncConfig defines the state sync snapshot configuration. @@ -221,16 +226,15 @@ func DefaultConfig() *Config { Enable: true, Address: DefaultGRPCAddress, GRPCWebProxy: GRPCProxy{ - Enable: true, - AllowAllOrigins: true, - EnableHTTPServer: true, - EnableTLSServer: false, - AllowedOrigins: []string{"*"}, - TlsServerCert: "", - AllowedHeaders: make([]string, 0), - TlsServerClientCAFiles: "", - TlsServerClientCertVerification: "", - TlsServerKey: "", + Enable: true, + AllowAllOrigins: true, + EnableHTTPServer: true, + AllowedOrigins: []string{"*"}, + AllowedHeaders: make([]string, 0), + BackendBackoffMaxDelay: grpc.DefaultBackoffConfig.MaxDelay, + MaxCallRecvMsgSize: 1024 * 1024 * 4, + BindAddress: DefaultBindAddress, + HTTPPort: DefaultGRPCProxyPort, }, }, StateSync: StateSyncConfig{ @@ -287,16 +291,15 @@ func GetConfig(v *viper.Viper) Config { Enable: v.GetBool("grpc.enable"), Address: v.GetString("grpc.address"), GRPCWebProxy: GRPCProxy{ - Enable: v.GetBool("grpc.grpc-proxy.enable"), - AllowAllOrigins: v.GetBool("grpc.grpc-proxy.allow-all-origins"), - AllowedHeaders: v.GetStringSlice("grpc.grpc-proxy.allowed-headers"), - AllowedOrigins: v.GetStringSlice("grpc.grpc-proxy.allowed-origins"), - EnableHTTPServer: v.GetBool("grpc.grpc-proxy.enable-http-server"), - EnableTLSServer: v.GetBool("grpc.grpc-proxy.enable-tls-server"), - TlsServerCert: v.GetString("grpc.grpc-proxy.tls-server-cert"), - TlsServerClientCAFiles: v.GetString("grpc.grpc-proxy.tls-server-client-ca-files"), - TlsServerClientCertVerification: v.GetString("grpc.grpc-proxy.tls-server-client-cert-verification"), - TlsServerKey: v.GetString("grpc.grpc-proxy.tls-server-key"), + Enable: v.GetBool("grpc.grpc-proxy.enable"), + AllowAllOrigins: v.GetBool("grpc.grpc-proxy.allow-all-origins"), + AllowedHeaders: v.GetStringSlice("grpc.grpc-proxy.allowed-headers"), + AllowedOrigins: v.GetStringSlice("grpc.grpc-proxy.allowed-origins"), + EnableHTTPServer: v.GetBool("grpc.grpc-proxy.enable-http-server"), + MaxCallRecvMsgSize: v.GetInt("grpc.grpc-proxy.max-call-recv-msg-size"), + BackendBackoffMaxDelay: v.GetDuration("grpc.grpc-proxy.backend-backoff-max-delay"), + BindAddress: v.GetString("grpc.grpc-proxy.bind-address"), + HTTPPort: v.GetInt("grpc.grpc-proxy.http-port"), }, }, StateSync: StateSyncConfig{ diff --git a/server/config/toml.go b/server/config/toml.go index 1fad8ab63256..4e2fa8e300f6 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -151,6 +151,12 @@ address = "{{ .GRPC.Address }}" # GRPCWebProxyEnable defines if the gRPC-web-proxy should be enabled. enable = {{ .GRPC.GRPCWebProxy.Enable }} + # BindAddress defines address to bind the server to. + bind-address = "{{ .GRPC.GRPCWebProxy.BindAddress }}" + + # HTTPPort defines TCP port to listen on for HTTP1.1 debug calls. + http-port = {{ .GRPC.GRPCWebProxy.HTTPPort }} + # AllowAllOrigins defines allow requests from any origin. allow-all-origin = {{ .GRPC.GRPCWebProxy.AllowAllOrigins }} @@ -163,20 +169,12 @@ address = "{{ .GRPC.Address }}" # EnableHTTPServer defines if the HTTP should be enabled. enable-http-server = {{ .GRPC.GRPCWebProxy.EnableHTTPServer }} - # EnableTLSServer defines if the TLS should be enabled. - enable-tls-server = {{ .GRPC.GRPCWebProxy.EnableTLSServer }} - - # TlsServerCert defines the path to the PEM certificate for server use. - tls-server-cert = "{{ .GRPC.GRPCWebProxy.TlsServerCert}}" - - # TlsServerKey defines the path to the PEM key for the certificate for the server use. - tls-server-key = "{{ .GRPC.GRPCWebProxy.TlsServerKey }}" + # MaxCallRecvMsgSize defines maximum receive message size limit. If not specified, the default of 4MB will be used. + max-call-recv-msg-size = {{ .GRPC.GRPCWebProxy.MaxCallRecvMsgSize }} - # TlsServerClientCertVerification defines controls whether a client certificate is on. Values: none, verify_if_given, require. - tls-server-client-cert-verification = "{{ .GRPC.GRPCWebProxy.TlsServerClientCertVerification }}" + # BackendBackoffMaxDelay defines maximum delay when backing off after failed connection attempts to the backend. + backend-backoff-max-delay = "{{ .GRPC.GRPCWebProxy.BackendBackoffMaxDelay }}" - # TlsServerClientCAFiles defines Paths (comma separated) to PEM certificate chains used for client-side verification. If empty, host CA chain will be used. - tls-server-client-ca-files = "{{ .GRPC.GRPCWebProxy.TlsServerClientCAFiles }}" ############################################################################### ### State Sync Configuration ### diff --git a/server/grpc/proxy_server.go b/server/grpc/proxy_server.go new file mode 100644 index 000000000000..c4438b8cab93 --- /dev/null +++ b/server/grpc/proxy_server.go @@ -0,0 +1,180 @@ +package grpc + +import ( + "context" + "fmt" + "net" + "net/http" + "strings" + "time" + + "github.com/cosmos/cosmos-sdk/server/config" + "github.com/improbable-eng/grpc-web/go/grpcweb" + "github.com/mwitkow/go-conntrack" + "github.com/mwitkow/grpc-proxy/proxy" + "github.com/prometheus/client_golang/prometheus/promhttp" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +var ( + flagHttpMaxReadTimeout = 10 * time.Second + flagHttpMaxWriteTimeout = 10 * time.Second +) + +// StartGRPCProxyServer starts a gRPC-proxy server on the given config. +func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { + + proxyFlags := grpcConfig.GRPCWebProxy + + grpcSrv, err := buildGrpcProxyServer(proxyFlags, grpcConfig.Address) + if err != nil { + return nil, err + } + + allowedOrigins := makeAllowedOrigins(proxyFlags.AllowedOrigins) + options := []grpcweb.Option{ + grpcweb.WithCorsForRegisteredEndpointsOnly(false), + grpcweb.WithOriginFunc(makeHttpOriginFunc(allowedOrigins, proxyFlags.AllowAllOrigins)), + } + + if len(proxyFlags.AllowedHeaders) > 0 { + options = append( + options, + grpcweb.WithAllowedRequestHeaders(proxyFlags.AllowedHeaders), + ) + } + + wrappedGrpc := grpcweb.WrapServer(grpcSrv, options...) + + if !proxyFlags.EnableHTTPServer { + return nil, fmt.Errorf("run_http_server is set to false. Enable for grpcweb proxy to function correctly.") + } + + // Debug server. + debugServer := buildServer(wrappedGrpc, proxyFlags) + http.Handle("/metrics", promhttp.Handler()) + listener, err := buildListenerOrFail("http", proxyFlags.HTTPPort) + + if err != nil { + return nil, err + } + errCh := make(chan error) + + go func() { + err = debugServer.Serve(listener) + if err != nil { + errCh <- fmt.Errorf("failed to serve: %w", err) + } + }() + + select { + case err := <-errCh: + return nil, err + case <-time.After(5 * time.Second): // assume server started successfully + return debugServer, nil + } + +} + +func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer, proxyFlags config.GRPCProxy) *http.Server { + return &http.Server{ + // TODO update flags + WriteTimeout: flagHttpMaxWriteTimeout, + ReadTimeout: flagHttpMaxReadTimeout, + Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + wrappedGrpc.ServeHTTP(resp, req) + }), + } +} + +func buildGrpcProxyServer(proxyFlags config.GRPCProxy, host string) (*grpc.Server, error) { + // gRPC-wide changes. + grpc.EnableTracing = true + + // gRPC proxy logic. + backendConn, err := dialBackendOrFail(proxyFlags, host) + if err != nil { + return nil, err + } + director := func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error) { + md, _ := metadata.FromIncomingContext(ctx) + outCtx, _ := context.WithCancel(ctx) + mdCopy := md.Copy() + delete(mdCopy, "user-agent") + // If this header is present in the request from the web client, + // the actual connection to the backend will not be established. + // https://github.com/improbable-eng/grpc-web/issues/568 + delete(mdCopy, "connection") + outCtx = metadata.NewOutgoingContext(outCtx, mdCopy) + return outCtx, backendConn, nil + } + // Server with logging and monitoring enabled. + return grpc.NewServer( + grpc.CustomCodec(proxy.Codec()), // needed for proxy to function. + grpc.UnknownServiceHandler(proxy.TransparentHandler(director)), + ), nil +} + +func buildListenerOrFail(name string, port int) (net.Listener, error) { + addr := fmt.Sprintf("%s:%d", "0.0.0.0", port) + listener, err := net.Listen("tcp", addr) + if err != nil { + return nil, fmt.Errorf("failed listening for '%v' on %v: %v", name, port, err) + } + return conntrack.NewListener(listener, + conntrack.TrackWithName(name), + conntrack.TrackWithTcpKeepAlive(20*time.Second), + conntrack.TrackWithTracing(), + ), nil +} + +func makeHttpOriginFunc(allowedOrigins *allowedOrigins, allowAllOrigins bool) func(origin string) bool { + if allowAllOrigins { + return func(origin string) bool { + return true + } + } + return allowedOrigins.IsAllowed +} + +func makeAllowedOrigins(origins []string) *allowedOrigins { + o := map[string]struct{}{} + for _, allowedOrigin := range origins { + o[allowedOrigin] = struct{}{} + } + return &allowedOrigins{ + origins: o, + } +} + +type allowedOrigins struct { + origins map[string]struct{} +} + +func (a *allowedOrigins) IsAllowed(origin string) bool { + _, ok := a.origins[origin] + return ok +} + +func dialBackendOrFail(proxyFlags config.GRPCProxy, host string) (*grpc.ClientConn, error) { + ip := strings.Split(host, ":") + if len(ip) != 2 { + return nil, fmt.Errorf("host address must be 0.0.0.0:0000 this format") + } + opt := []grpc.DialOption{} + opt = append(opt, grpc.WithCodec(proxy.Codec())) + + opt = append(opt, grpc.WithInsecure()) + + opt = append(opt, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(proxyFlags.MaxCallRecvMsgSize)), + grpc.WithBackoffMaxDelay(proxyFlags.BackendBackoffMaxDelay), + ) + + cc, err := grpc.Dial(ip[1], opt...) + if err != nil { + return nil, fmt.Errorf("failed dialing backend: %v", err) + } + return cc, nil +} diff --git a/server/start.go b/server/start.go index d3af7db42ca4..3c6a40e5304e 100644 --- a/server/start.go +++ b/server/start.go @@ -4,6 +4,7 @@ package server import ( "fmt" + "net/http" "os" "runtime/pprof" "time" @@ -53,8 +54,15 @@ const ( // GRPC-related flags. const ( - flagGRPCEnable = "grpc.enable" - flagGRPCAddress = "grpc.address" + flagGRPCEnable = "grpc.enable" + flagGRPCAddress = "grpc.address" + flagGRPCProxyEnable = "grpc.grpc-proxy.enable" + flagGRPCProxyAllowAllOrigins = "grpc.grpc-proxy.allow-all-origins" + flagGRPCProxyAllowedHeaders = "grpc.grpc-proxy.allowed-headers" + flagGRPCProxyAllowedOrigins = "grpc.grpc-proxy.allowed-origins" + flagGRPCProxyEnableHTTPServer = "grpc.grpc-proxy.enable-http-server" + flagGRPCProxyMaxCallRecvMsgSize = "grpc.grpc-proxy.max-call-recv-msg-size" + flagGRPCProxyBackendBackoffMaxDelay = "grpc.grpc-proxy.backend-backoff-max-delay" ) // State sync-related flags. @@ -140,6 +148,7 @@ which accepts a path for the resulting pprof file. cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled") cmd.Flags().String(flagGRPCAddress, config.DefaultGRPCAddress, "the gRPC server address to listen on") + cmd.Flags().Bool(flagGRPCProxyEnable, true, "Define if the gRPC-Proxy server should be enabled") cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval") cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep") @@ -294,11 +303,19 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App } var grpcSrv *grpc.Server + var proxySrv *http.Server if config.GRPC.Enable { grpcSrv, err = servergrpc.StartGRPCServer(app, config.GRPC.Address) if err != nil { return err } + if config.GRPC.GRPCWebProxy.Enable { + fmt.Println("I'm calllllled") + proxySrv, err = servergrpc.StartGRPCProxyServer(config.GRPC) + if err != nil { + return err + } + } } defer func() { @@ -318,6 +335,10 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App grpcSrv.Stop() } + if proxySrv != nil { + _ = proxySrv.Close() + } + ctx.Logger.Info("exiting...") }() From f4f98eb7a20349c49ad424b4e323c2051ebee04f Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Fri, 4 Dec 2020 18:10:44 +0530 Subject: [PATCH 04/24] fmt --- server/grpc/proxy_server.go | 1 - server/start.go | 1 - 2 files changed, 2 deletions(-) diff --git a/server/grpc/proxy_server.go b/server/grpc/proxy_server.go index c4438b8cab93..a781d1602d95 100644 --- a/server/grpc/proxy_server.go +++ b/server/grpc/proxy_server.go @@ -24,7 +24,6 @@ var ( // StartGRPCProxyServer starts a gRPC-proxy server on the given config. func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { - proxyFlags := grpcConfig.GRPCWebProxy grpcSrv, err := buildGrpcProxyServer(proxyFlags, grpcConfig.Address) diff --git a/server/start.go b/server/start.go index 3c6a40e5304e..5371bbcce3aa 100644 --- a/server/start.go +++ b/server/start.go @@ -310,7 +310,6 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App return err } if config.GRPC.GRPCWebProxy.Enable { - fmt.Println("I'm calllllled") proxySrv, err = servergrpc.StartGRPCProxyServer(config.GRPC) if err != nil { return err From 758b4bbb2b8802402c4aedd2efedc4d0ebfea45b Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Sat, 5 Dec 2020 10:35:26 +0530 Subject: [PATCH 05/24] WIP --- go.mod | 2 ++ go.sum | 1 + server/grpc/proxy_server.go | 34 +++++++++++++++++++++++++--------- server/start.go | 12 +++--------- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 6b51e507a97c..1ece36bd13bc 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/golang-lru v0.5.4 github.com/improbable-eng/grpc-web v0.13.0 @@ -40,6 +41,7 @@ require ( github.com/rakyll/statik v0.1.7 github.com/regen-network/cosmos-proto v0.3.0 github.com/rs/zerolog v1.20.0 + github.com/sirupsen/logrus v1.6.0 github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cast v1.3.1 github.com/spf13/cobra v1.1.1 diff --git a/go.sum b/go.sum index 2551b26ff867..6b5a945f3a0f 100644 --- a/go.sum +++ b/go.sum @@ -251,6 +251,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-middleware v1.2.1/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 h1:FlFbCRLd5Jr4iYXZufAvgWN6Ao0JrI5chLINnUXDDr0= github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= diff --git a/server/grpc/proxy_server.go b/server/grpc/proxy_server.go index a781d1602d95..9831dbd91cdd 100644 --- a/server/grpc/proxy_server.go +++ b/server/grpc/proxy_server.go @@ -5,14 +5,18 @@ import ( "fmt" "net" "net/http" - "strings" + "os" "time" "github.com/cosmos/cosmos-sdk/server/config" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/improbable-eng/grpc-web/go/grpcweb" "github.com/mwitkow/go-conntrack" "github.com/mwitkow/grpc-proxy/proxy" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) @@ -25,13 +29,17 @@ var ( // StartGRPCProxyServer starts a gRPC-proxy server on the given config. func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { proxyFlags := grpcConfig.GRPCWebProxy + logrus.SetOutput(os.Stdout) - grpcSrv, err := buildGrpcProxyServer(proxyFlags, grpcConfig.Address) + logEntry := logrus.NewEntry(logrus.StandardLogger()) + + grpcSrv, err := buildGrpcProxyServer(logEntry, proxyFlags, grpcConfig.Address) if err != nil { return nil, err } allowedOrigins := makeAllowedOrigins(proxyFlags.AllowedOrigins) + fmt.Println("Allowed origins = ", allowedOrigins) options := []grpcweb.Option{ grpcweb.WithCorsForRegisteredEndpointsOnly(false), grpcweb.WithOriginFunc(makeHttpOriginFunc(allowedOrigins, proxyFlags.AllowAllOrigins)), @@ -40,7 +48,7 @@ func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { if len(proxyFlags.AllowedHeaders) > 0 { options = append( options, - grpcweb.WithAllowedRequestHeaders(proxyFlags.AllowedHeaders), + grpcweb.WithAllowedRequestHeaders([]string{"*"}), ) } @@ -78,7 +86,6 @@ func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer, proxyFlags config.GRPCProxy) *http.Server { return &http.Server{ - // TODO update flags WriteTimeout: flagHttpMaxWriteTimeout, ReadTimeout: flagHttpMaxReadTimeout, Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { @@ -87,9 +94,10 @@ func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer, proxyFlags config.GRPCP } } -func buildGrpcProxyServer(proxyFlags config.GRPCProxy, host string) (*grpc.Server, error) { +func buildGrpcProxyServer(logger *logrus.Entry, proxyFlags config.GRPCProxy, host string) (*grpc.Server, error) { // gRPC-wide changes. grpc.EnableTracing = true + grpc_logrus.ReplaceGrpcLogger(logger) // gRPC proxy logic. backendConn, err := dialBackendOrFail(proxyFlags, host) @@ -112,6 +120,14 @@ func buildGrpcProxyServer(proxyFlags config.GRPCProxy, host string) (*grpc.Serve return grpc.NewServer( grpc.CustomCodec(proxy.Codec()), // needed for proxy to function. grpc.UnknownServiceHandler(proxy.TransparentHandler(director)), + grpc_middleware.WithUnaryServerChain( + grpc_logrus.UnaryServerInterceptor(logger), + grpc_prometheus.UnaryServerInterceptor, + ), + grpc_middleware.WithStreamServerChain( + grpc_logrus.StreamServerInterceptor(logger), + grpc_prometheus.StreamServerInterceptor, + ), ), nil } @@ -157,10 +173,10 @@ func (a *allowedOrigins) IsAllowed(origin string) bool { } func dialBackendOrFail(proxyFlags config.GRPCProxy, host string) (*grpc.ClientConn, error) { - ip := strings.Split(host, ":") - if len(ip) != 2 { - return nil, fmt.Errorf("host address must be 0.0.0.0:0000 this format") + if host == "" { + return nil, fmt.Errorf("host cannot be empty") } + opt := []grpc.DialOption{} opt = append(opt, grpc.WithCodec(proxy.Codec())) @@ -171,7 +187,7 @@ func dialBackendOrFail(proxyFlags config.GRPCProxy, host string) (*grpc.ClientCo grpc.WithBackoffMaxDelay(proxyFlags.BackendBackoffMaxDelay), ) - cc, err := grpc.Dial(ip[1], opt...) + cc, err := grpc.Dial(host, opt...) if err != nil { return nil, fmt.Errorf("failed dialing backend: %v", err) } diff --git a/server/start.go b/server/start.go index 5371bbcce3aa..1701d064a09e 100644 --- a/server/start.go +++ b/server/start.go @@ -54,15 +54,9 @@ const ( // GRPC-related flags. const ( - flagGRPCEnable = "grpc.enable" - flagGRPCAddress = "grpc.address" - flagGRPCProxyEnable = "grpc.grpc-proxy.enable" - flagGRPCProxyAllowAllOrigins = "grpc.grpc-proxy.allow-all-origins" - flagGRPCProxyAllowedHeaders = "grpc.grpc-proxy.allowed-headers" - flagGRPCProxyAllowedOrigins = "grpc.grpc-proxy.allowed-origins" - flagGRPCProxyEnableHTTPServer = "grpc.grpc-proxy.enable-http-server" - flagGRPCProxyMaxCallRecvMsgSize = "grpc.grpc-proxy.max-call-recv-msg-size" - flagGRPCProxyBackendBackoffMaxDelay = "grpc.grpc-proxy.backend-backoff-max-delay" + flagGRPCEnable = "grpc.enable" + flagGRPCAddress = "grpc.address" + flagGRPCProxyEnable = "grpc.grpc-proxy.enable" ) // State sync-related flags. From 34c4eb6f5439a2e7ae70e4994fee456c92e79868 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Mon, 7 Dec 2020 18:16:39 +0530 Subject: [PATCH 06/24] setup proxy server --- server/config/config.go | 2 +- server/config/toml.go | 2 +- server/grpc/proxy_server.go | 58 ++++++++++--------------------------- server/start.go | 1 + 4 files changed, 19 insertions(+), 44 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 01bb2006956b..6d11f163b037 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -229,7 +229,7 @@ func DefaultConfig() *Config { Enable: true, AllowAllOrigins: true, EnableHTTPServer: true, - AllowedOrigins: []string{"*"}, + AllowedOrigins: []string{DefaultGRPCAddress}, AllowedHeaders: make([]string, 0), BackendBackoffMaxDelay: grpc.DefaultBackoffConfig.MaxDelay, MaxCallRecvMsgSize: 1024 * 1024 * 4, diff --git a/server/config/toml.go b/server/config/toml.go index 4e2fa8e300f6..cdffb2be9cf3 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -158,7 +158,7 @@ address = "{{ .GRPC.Address }}" http-port = {{ .GRPC.GRPCWebProxy.HTTPPort }} # AllowAllOrigins defines allow requests from any origin. - allow-all-origin = {{ .GRPC.GRPCWebProxy.AllowAllOrigins }} + allow-all-origins = {{ .GRPC.GRPCWebProxy.AllowAllOrigins }} # AllowedOrigins defines list of origin URLs which are allowed to make cross-origin requests. allowed-origins = [{{ range $k, $v := .GRPC.GRPCWebProxy.AllowedOrigins }} "{{ $v }}", {{ end }}] diff --git a/server/grpc/proxy_server.go b/server/grpc/proxy_server.go index 9831dbd91cdd..69f5fcc74a71 100644 --- a/server/grpc/proxy_server.go +++ b/server/grpc/proxy_server.go @@ -5,18 +5,13 @@ import ( "fmt" "net" "net/http" - "os" "time" "github.com/cosmos/cosmos-sdk/server/config" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/improbable-eng/grpc-web/go/grpcweb" "github.com/mwitkow/go-conntrack" "github.com/mwitkow/grpc-proxy/proxy" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) @@ -26,41 +21,33 @@ var ( flagHttpMaxWriteTimeout = 10 * time.Second ) +type allowedOrigins struct { + origins map[string]struct{} +} + // StartGRPCProxyServer starts a gRPC-proxy server on the given config. func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { proxyFlags := grpcConfig.GRPCWebProxy - logrus.SetOutput(os.Stdout) - logEntry := logrus.NewEntry(logrus.StandardLogger()) - - grpcSrv, err := buildGrpcProxyServer(logEntry, proxyFlags, grpcConfig.Address) + grpcSrv, err := buildGrpcProxyServer(proxyFlags, grpcConfig.Address) if err != nil { return nil, err } allowedOrigins := makeAllowedOrigins(proxyFlags.AllowedOrigins) - fmt.Println("Allowed origins = ", allowedOrigins) + options := []grpcweb.Option{ grpcweb.WithCorsForRegisteredEndpointsOnly(false), grpcweb.WithOriginFunc(makeHttpOriginFunc(allowedOrigins, proxyFlags.AllowAllOrigins)), } - if len(proxyFlags.AllowedHeaders) > 0 { - options = append( - options, - grpcweb.WithAllowedRequestHeaders([]string{"*"}), - ) - } - wrappedGrpc := grpcweb.WrapServer(grpcSrv, options...) if !proxyFlags.EnableHTTPServer { return nil, fmt.Errorf("run_http_server is set to false. Enable for grpcweb proxy to function correctly.") } - // Debug server. - debugServer := buildServer(wrappedGrpc, proxyFlags) - http.Handle("/metrics", promhttp.Handler()) + proxyServer := buildServer(wrappedGrpc, proxyFlags) listener, err := buildListenerOrFail("http", proxyFlags.HTTPPort) if err != nil { @@ -69,7 +56,7 @@ func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { errCh := make(chan error) go func() { - err = debugServer.Serve(listener) + err = proxyServer.Serve(listener) if err != nil { errCh <- fmt.Errorf("failed to serve: %w", err) } @@ -79,7 +66,7 @@ func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { case err := <-errCh: return nil, err case <-time.After(5 * time.Second): // assume server started successfully - return debugServer, nil + return proxyServer, nil } } @@ -94,11 +81,7 @@ func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer, proxyFlags config.GRPCP } } -func buildGrpcProxyServer(logger *logrus.Entry, proxyFlags config.GRPCProxy, host string) (*grpc.Server, error) { - // gRPC-wide changes. - grpc.EnableTracing = true - grpc_logrus.ReplaceGrpcLogger(logger) - +func buildGrpcProxyServer(proxyFlags config.GRPCProxy, host string) (*grpc.Server, error) { // gRPC proxy logic. backendConn, err := dialBackendOrFail(proxyFlags, host) if err != nil { @@ -120,18 +103,13 @@ func buildGrpcProxyServer(logger *logrus.Entry, proxyFlags config.GRPCProxy, hos return grpc.NewServer( grpc.CustomCodec(proxy.Codec()), // needed for proxy to function. grpc.UnknownServiceHandler(proxy.TransparentHandler(director)), - grpc_middleware.WithUnaryServerChain( - grpc_logrus.UnaryServerInterceptor(logger), - grpc_prometheus.UnaryServerInterceptor, - ), - grpc_middleware.WithStreamServerChain( - grpc_logrus.StreamServerInterceptor(logger), - grpc_prometheus.StreamServerInterceptor, - ), + grpc_middleware.WithUnaryServerChain(), + grpc_middleware.WithStreamServerChain(), ), nil } func buildListenerOrFail(name string, port int) (net.Listener, error) { + addr := fmt.Sprintf("%s:%d", "0.0.0.0", port) listener, err := net.Listen("tcp", addr) if err != nil { @@ -163,10 +141,6 @@ func makeAllowedOrigins(origins []string) *allowedOrigins { } } -type allowedOrigins struct { - origins map[string]struct{} -} - func (a *allowedOrigins) IsAllowed(origin string) bool { _, ok := a.origins[origin] return ok @@ -177,11 +151,11 @@ func dialBackendOrFail(proxyFlags config.GRPCProxy, host string) (*grpc.ClientCo return nil, fmt.Errorf("host cannot be empty") } - opt := []grpc.DialOption{} + opt := []grpc.DialOption{ + grpc.WithInsecure(), + } opt = append(opt, grpc.WithCodec(proxy.Codec())) - opt = append(opt, grpc.WithInsecure()) - opt = append(opt, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(proxyFlags.MaxCallRecvMsgSize)), grpc.WithBackoffMaxDelay(proxyFlags.BackendBackoffMaxDelay), diff --git a/server/start.go b/server/start.go index 1701d064a09e..7d46776a15b5 100644 --- a/server/start.go +++ b/server/start.go @@ -305,6 +305,7 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App } if config.GRPC.GRPCWebProxy.Enable { proxySrv, err = servergrpc.StartGRPCProxyServer(config.GRPC) + proxySrv.ListenAndServe() if err != nil { return err } From 8ff60e3a9b9476f31848631a5e1aa616a1fbb3f0 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Mon, 7 Dec 2020 18:18:08 +0530 Subject: [PATCH 07/24] clean go.mod --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 1ece36bd13bc..6b51e507a97c 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/golang-lru v0.5.4 github.com/improbable-eng/grpc-web v0.13.0 @@ -41,7 +40,6 @@ require ( github.com/rakyll/statik v0.1.7 github.com/regen-network/cosmos-proto v0.3.0 github.com/rs/zerolog v1.20.0 - github.com/sirupsen/logrus v1.6.0 github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cast v1.3.1 github.com/spf13/cobra v1.1.1 From d2cabcd27253af4258b867757b460e8343cfbc67 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Mon, 7 Dec 2020 18:48:56 +0530 Subject: [PATCH 08/24] lint --- server/grpc/proxy_server.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/grpc/proxy_server.go b/server/grpc/proxy_server.go index 69f5fcc74a71..a76b65afab2a 100644 --- a/server/grpc/proxy_server.go +++ b/server/grpc/proxy_server.go @@ -17,8 +17,8 @@ import ( ) var ( - flagHttpMaxReadTimeout = 10 * time.Second - flagHttpMaxWriteTimeout = 10 * time.Second + flagHTTPMaxReadTimeout = 10 * time.Second + flagHTTPMaxWriteTimeout = 10 * time.Second ) type allowedOrigins struct { @@ -38,7 +38,7 @@ func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { options := []grpcweb.Option{ grpcweb.WithCorsForRegisteredEndpointsOnly(false), - grpcweb.WithOriginFunc(makeHttpOriginFunc(allowedOrigins, proxyFlags.AllowAllOrigins)), + grpcweb.WithOriginFunc(makeHTTPOriginFunc(allowedOrigins, proxyFlags.AllowAllOrigins)), } wrappedGrpc := grpcweb.WrapServer(grpcSrv, options...) @@ -73,8 +73,8 @@ func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer, proxyFlags config.GRPCProxy) *http.Server { return &http.Server{ - WriteTimeout: flagHttpMaxWriteTimeout, - ReadTimeout: flagHttpMaxReadTimeout, + ReadTimeout: flagHTTPMaxReadTimeout, + WriteTimeout: flagHTTPMaxWriteTimeout, Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { wrappedGrpc.ServeHTTP(resp, req) }), @@ -82,7 +82,7 @@ func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer, proxyFlags config.GRPCP } func buildGrpcProxyServer(proxyFlags config.GRPCProxy, host string) (*grpc.Server, error) { - // gRPC proxy logic. + backendConn, err := dialBackendOrFail(proxyFlags, host) if err != nil { return nil, err @@ -122,7 +122,7 @@ func buildListenerOrFail(name string, port int) (net.Listener, error) { ), nil } -func makeHttpOriginFunc(allowedOrigins *allowedOrigins, allowAllOrigins bool) func(origin string) bool { +func makeHTTPOriginFunc(allowedOrigins *allowedOrigins, allowAllOrigins bool) func(origin string) bool { if allowAllOrigins { return func(origin string) bool { return true From a5d90bc9936a90bc72573220e8631ee3519070ed Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Mon, 7 Dec 2020 19:41:57 +0530 Subject: [PATCH 09/24] lint --- server/config/config.go | 40 +++++++++++++------------------------ server/config/toml.go | 7 ------- server/grpc/proxy_server.go | 23 ++++++++++----------- 3 files changed, 24 insertions(+), 46 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 6d11f163b037..cd721663dc96 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -3,10 +3,8 @@ package config import ( "fmt" "strings" - "time" "github.com/spf13/viper" - "google.golang.org/grpc" storetypes "github.com/cosmos/cosmos-sdk/store/types" "github.com/cosmos/cosmos-sdk/telemetry" @@ -140,12 +138,6 @@ type GRPCProxy struct { // EnableHTTPServer defines if the HTTP should be enabled. EnableHTTPServer bool `mapstructure:"enable-http-server"` - - // MaxCallRecvMsgSize defines maximum receive message size limit. If not specified, the default of 4MB will be used. - MaxCallRecvMsgSize int `mapstructure:"max-call-recv-msg-size"` - - // BackendBackoffMaxDelay defines maximum delay when backing off after failed connection attempts to the backend. - BackendBackoffMaxDelay time.Duration `mapstructure:"backend-backoff-max-delay"` } // StateSyncConfig defines the state sync snapshot configuration. @@ -226,15 +218,13 @@ func DefaultConfig() *Config { Enable: true, Address: DefaultGRPCAddress, GRPCWebProxy: GRPCProxy{ - Enable: true, - AllowAllOrigins: true, - EnableHTTPServer: true, - AllowedOrigins: []string{DefaultGRPCAddress}, - AllowedHeaders: make([]string, 0), - BackendBackoffMaxDelay: grpc.DefaultBackoffConfig.MaxDelay, - MaxCallRecvMsgSize: 1024 * 1024 * 4, - BindAddress: DefaultBindAddress, - HTTPPort: DefaultGRPCProxyPort, + Enable: true, + AllowAllOrigins: true, + EnableHTTPServer: true, + AllowedOrigins: []string{DefaultGRPCAddress}, + AllowedHeaders: make([]string, 0), + BindAddress: DefaultBindAddress, + HTTPPort: DefaultGRPCProxyPort, }, }, StateSync: StateSyncConfig{ @@ -291,15 +281,13 @@ func GetConfig(v *viper.Viper) Config { Enable: v.GetBool("grpc.enable"), Address: v.GetString("grpc.address"), GRPCWebProxy: GRPCProxy{ - Enable: v.GetBool("grpc.grpc-proxy.enable"), - AllowAllOrigins: v.GetBool("grpc.grpc-proxy.allow-all-origins"), - AllowedHeaders: v.GetStringSlice("grpc.grpc-proxy.allowed-headers"), - AllowedOrigins: v.GetStringSlice("grpc.grpc-proxy.allowed-origins"), - EnableHTTPServer: v.GetBool("grpc.grpc-proxy.enable-http-server"), - MaxCallRecvMsgSize: v.GetInt("grpc.grpc-proxy.max-call-recv-msg-size"), - BackendBackoffMaxDelay: v.GetDuration("grpc.grpc-proxy.backend-backoff-max-delay"), - BindAddress: v.GetString("grpc.grpc-proxy.bind-address"), - HTTPPort: v.GetInt("grpc.grpc-proxy.http-port"), + Enable: v.GetBool("grpc.grpc-proxy.enable"), + AllowAllOrigins: v.GetBool("grpc.grpc-proxy.allow-all-origins"), + AllowedHeaders: v.GetStringSlice("grpc.grpc-proxy.allowed-headers"), + AllowedOrigins: v.GetStringSlice("grpc.grpc-proxy.allowed-origins"), + EnableHTTPServer: v.GetBool("grpc.grpc-proxy.enable-http-server"), + BindAddress: v.GetString("grpc.grpc-proxy.bind-address"), + HTTPPort: v.GetInt("grpc.grpc-proxy.http-port"), }, }, StateSync: StateSyncConfig{ diff --git a/server/config/toml.go b/server/config/toml.go index cdffb2be9cf3..c94f5c651c68 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -169,13 +169,6 @@ address = "{{ .GRPC.Address }}" # EnableHTTPServer defines if the HTTP should be enabled. enable-http-server = {{ .GRPC.GRPCWebProxy.EnableHTTPServer }} - # MaxCallRecvMsgSize defines maximum receive message size limit. If not specified, the default of 4MB will be used. - max-call-recv-msg-size = {{ .GRPC.GRPCWebProxy.MaxCallRecvMsgSize }} - - # BackendBackoffMaxDelay defines maximum delay when backing off after failed connection attempts to the backend. - backend-backoff-max-delay = "{{ .GRPC.GRPCWebProxy.BackendBackoffMaxDelay }}" - - ############################################################################### ### State Sync Configuration ### ############################################################################### diff --git a/server/grpc/proxy_server.go b/server/grpc/proxy_server.go index a76b65afab2a..a4d7c4e48efc 100644 --- a/server/grpc/proxy_server.go +++ b/server/grpc/proxy_server.go @@ -26,10 +26,11 @@ type allowedOrigins struct { } // StartGRPCProxyServer starts a gRPC-proxy server on the given config. +// https://github.com/improbable-eng/grpc-web/tree/master/go/grpcwebproxy used as a reference. func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { proxyFlags := grpcConfig.GRPCWebProxy - grpcSrv, err := buildGrpcProxyServer(proxyFlags, grpcConfig.Address) + grpcSrv, err := buildGrpcProxyServer(grpcConfig.Address) if err != nil { return nil, err } @@ -44,10 +45,10 @@ func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { wrappedGrpc := grpcweb.WrapServer(grpcSrv, options...) if !proxyFlags.EnableHTTPServer { - return nil, fmt.Errorf("run_http_server is set to false. Enable for grpcweb proxy to function correctly.") + return nil, fmt.Errorf("run_http_server is set to false. enable for grpcweb proxy to function correctly") } - proxyServer := buildServer(wrappedGrpc, proxyFlags) + proxyServer := buildServer(wrappedGrpc) listener, err := buildListenerOrFail("http", proxyFlags.HTTPPort) if err != nil { @@ -71,7 +72,8 @@ func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { } -func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer, proxyFlags config.GRPCProxy) *http.Server { +// nolint:interfacer +func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer) *http.Server { return &http.Server{ ReadTimeout: flagHTTPMaxReadTimeout, WriteTimeout: flagHTTPMaxWriteTimeout, @@ -81,15 +83,15 @@ func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer, proxyFlags config.GRPCP } } -func buildGrpcProxyServer(proxyFlags config.GRPCProxy, host string) (*grpc.Server, error) { +func buildGrpcProxyServer(host string) (*grpc.Server, error) { - backendConn, err := dialBackendOrFail(proxyFlags, host) + backendConn, err := dialBackendOrFail(host) if err != nil { return nil, err } director := func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error) { md, _ := metadata.FromIncomingContext(ctx) - outCtx, _ := context.WithCancel(ctx) + outCtx, _ := context.WithCancel(ctx) //nolint mdCopy := md.Copy() delete(mdCopy, "user-agent") // If this header is present in the request from the web client, @@ -146,7 +148,7 @@ func (a *allowedOrigins) IsAllowed(origin string) bool { return ok } -func dialBackendOrFail(proxyFlags config.GRPCProxy, host string) (*grpc.ClientConn, error) { +func dialBackendOrFail(host string) (*grpc.ClientConn, error) { if host == "" { return nil, fmt.Errorf("host cannot be empty") } @@ -156,11 +158,6 @@ func dialBackendOrFail(proxyFlags config.GRPCProxy, host string) (*grpc.ClientCo } opt = append(opt, grpc.WithCodec(proxy.Codec())) - opt = append(opt, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(proxyFlags.MaxCallRecvMsgSize)), - grpc.WithBackoffMaxDelay(proxyFlags.BackendBackoffMaxDelay), - ) - cc, err := grpc.Dial(host, opt...) if err != nil { return nil, fmt.Errorf("failed dialing backend: %v", err) From dd6a00069a7b2101d73cfd6fb43767bfaf2fee58 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Tue, 8 Dec 2020 11:28:56 +0530 Subject: [PATCH 10/24] lint --- server/config/config.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index cd721663dc96..3c1568234975 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -121,23 +121,23 @@ type GRPCProxy struct { // Enable defines if the proxy should be enabled. Enable bool `mapstructure:"enable"` - // BindAddress defines address to bind the server to. - BindAddress string `mapstructure:"bind-address"` + // EnableHTTPServer defines if the HTTP should be enabled. + EnableHTTPServer bool `mapstructure:"enable-http-server"` + + // AllowAllOrigins defines allow requests from any origin. + AllowAllOrigins bool `mapstructure:"allow-all-origins"` // HTTPPort defines TCP port to listen on for HTTP1.1 debug calls. HTTPPort int `mapstructure:"http-port"` - // AllowAllOrigins defines allow requests from any origin. - AllowAllOrigins bool `mapstructure:"allow-all-origins"` + // BindAddress defines address to bind the server to. + BindAddress string `mapstructure:"bind-address"` // AllowedOrigins defines list of origin URLs which are allowed to make cross-origin requests. AllowedOrigins []string `mapstructure:"allowed-origins"` // AllowedHeaders defines list of headers which are allowed to propagate to the gRPC backend. AllowedHeaders []string `mapstructure:"allowed-headers"` - - // EnableHTTPServer defines if the HTTP should be enabled. - EnableHTTPServer bool `mapstructure:"enable-http-server"` } // StateSyncConfig defines the state sync snapshot configuration. From 222d1cf47f6e87a0ade73103863acbfd5a603598 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Thu, 10 Dec 2020 12:41:23 +0530 Subject: [PATCH 11/24] custom codec --- server/grpc/proto_codec.go | 88 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 server/grpc/proto_codec.go diff --git a/server/grpc/proto_codec.go b/server/grpc/proto_codec.go new file mode 100644 index 000000000000..2b44d3f09fff --- /dev/null +++ b/server/grpc/proto_codec.go @@ -0,0 +1,88 @@ +package grpc + +import ( + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/encoding" +) + +// Name is the name by which the proxy codec is registered in the encoding codec registry +// We have to say that we are the "proto" codec otherwise marshaling will fail! +const Name = "proto" + +// Register manually registers the codec +func RegisterCodec() { + encoding.RegisterCodec(codec()) +} + +// codec returns a proxying grpc.codec with the default protobuf codec as parent. +// +// See CodecWithParent. +func codec() encoding.Codec { + // since we have registered the default codec by importing it, + // we can fetch it from the registry and use it as our parent + // and overwrite the existing codec in the registry + return codecWithParent(&protoCodec{}) +} + +// CodecWithParent returns a proxying grpc.Codec with a user provided codec as parent. +// +// This codec is *crucial* to the functioning of the proxy. It allows the proxy server to be oblivious +// to the schema of the forwarded messages. It basically treats a gRPC message frame as raw bytes. +// However, if the server handler, or the client caller are not proxy-internal functions it will fall back +// to trying to decode the message using a fallback codec. +func codecWithParent(fallback encoding.Codec) encoding.Codec { + return &Proxy{parentCodec: fallback} +} + +// Proxy satisfies the encoding.Codec interface +type Proxy struct { + parentCodec encoding.Codec +} + +// Frame holds the proxy transported data +type Frame struct { + payload []byte +} + +// ProtoMessage tags a frame as valid proto message +func (f *Frame) ProtoMessage() {} + +// Marshal implents the encoding.Codec interface method +func (p *Proxy) Marshal(v interface{}) ([]byte, error) { + out, ok := v.(*Frame) + if !ok { + return p.parentCodec.Marshal(v) + } + return out.payload, nil + +} + +// Unmarshal implents the encoding.Codec interface method +func (p *Proxy) Unmarshal(data []byte, v interface{}) error { + dst, ok := v.(*Frame) + if !ok { + return p.parentCodec.Unmarshal(data, v) + } + dst.payload = data + return nil +} + +// Name implents the encoding.Codec interface method +func (*Proxy) Name() string { + return Name +} + +// protoCodec is a Codec implementation with protobuf. It is the default rawCodec for gRPC. +type protoCodec struct{} + +func (*protoCodec) Marshal(v interface{}) ([]byte, error) { + return proto.Marshal(v.(proto.Message)) +} + +func (*protoCodec) Unmarshal(data []byte, v interface{}) error { + return proto.Unmarshal(data, v.(proto.Message)) +} + +func (*protoCodec) Name() string { + return "proxy>proto" +} From a5e7dbfcc1ef94b91bd086f8460e23240c6cb0c4 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Mon, 14 Dec 2020 11:57:53 +0530 Subject: [PATCH 12/24] lint --- .../{proxy_server.go => grpcweb_proxy.go} | 17 ++-- server/grpc/proto_codec.go | 88 ------------------- server/start.go | 7 +- 3 files changed, 14 insertions(+), 98 deletions(-) rename server/grpc/{proxy_server.go => grpcweb_proxy.go} (92%) delete mode 100644 server/grpc/proto_codec.go diff --git a/server/grpc/proxy_server.go b/server/grpc/grpcweb_proxy.go similarity index 92% rename from server/grpc/proxy_server.go rename to server/grpc/grpcweb_proxy.go index a4d7c4e48efc..608e61d57d32 100644 --- a/server/grpc/proxy_server.go +++ b/server/grpc/grpcweb_proxy.go @@ -49,7 +49,7 @@ func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { } proxyServer := buildServer(wrappedGrpc) - listener, err := buildListenerOrFail("http", proxyFlags.HTTPPort) + listener, err := buildListenerOrFail("http", proxyFlags.BindAddress, proxyFlags.HTTPPort) if err != nil { return nil, err @@ -84,11 +84,11 @@ func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer) *http.Server { } func buildGrpcProxyServer(host string) (*grpc.Server, error) { - backendConn, err := dialBackendOrFail(host) if err != nil { return nil, err } + director := func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error) { md, _ := metadata.FromIncomingContext(ctx) outCtx, _ := context.WithCancel(ctx) //nolint @@ -101,7 +101,8 @@ func buildGrpcProxyServer(host string) (*grpc.Server, error) { outCtx = metadata.NewOutgoingContext(outCtx, mdCopy) return outCtx, backendConn, nil } - // Server with logging and monitoring enabled. + // TODO: handle grpc.CustomCodec is deprecated. + // nolint:staticcheck return grpc.NewServer( grpc.CustomCodec(proxy.Codec()), // needed for proxy to function. grpc.UnknownServiceHandler(proxy.TransparentHandler(director)), @@ -110,13 +111,13 @@ func buildGrpcProxyServer(host string) (*grpc.Server, error) { ), nil } -func buildListenerOrFail(name string, port int) (net.Listener, error) { - - addr := fmt.Sprintf("%s:%d", "0.0.0.0", port) +func buildListenerOrFail(name string, host string, port int) (net.Listener, error) { + addr := fmt.Sprintf("%s:%d", host, port) listener, err := net.Listen("tcp", addr) if err != nil { return nil, fmt.Errorf("failed listening for '%v' on %v: %v", name, port, err) } + return conntrack.NewListener(listener, conntrack.TrackWithName(name), conntrack.TrackWithTcpKeepAlive(20*time.Second), @@ -130,6 +131,7 @@ func makeHTTPOriginFunc(allowedOrigins *allowedOrigins, allowAllOrigins bool) fu return true } } + return allowedOrigins.IsAllowed } @@ -138,6 +140,7 @@ func makeAllowedOrigins(origins []string) *allowedOrigins { for _, allowedOrigin := range origins { o[allowedOrigin] = struct{}{} } + return &allowedOrigins{ origins: o, } @@ -156,6 +159,8 @@ func dialBackendOrFail(host string) (*grpc.ClientConn, error) { opt := []grpc.DialOption{ grpc.WithInsecure(), } + // TODO: handle grpc.WithCodec is deprecated. + // nolint:staticcheck opt = append(opt, grpc.WithCodec(proxy.Codec())) cc, err := grpc.Dial(host, opt...) diff --git a/server/grpc/proto_codec.go b/server/grpc/proto_codec.go deleted file mode 100644 index 2b44d3f09fff..000000000000 --- a/server/grpc/proto_codec.go +++ /dev/null @@ -1,88 +0,0 @@ -package grpc - -import ( - "github.com/golang/protobuf/proto" - "google.golang.org/grpc/encoding" -) - -// Name is the name by which the proxy codec is registered in the encoding codec registry -// We have to say that we are the "proto" codec otherwise marshaling will fail! -const Name = "proto" - -// Register manually registers the codec -func RegisterCodec() { - encoding.RegisterCodec(codec()) -} - -// codec returns a proxying grpc.codec with the default protobuf codec as parent. -// -// See CodecWithParent. -func codec() encoding.Codec { - // since we have registered the default codec by importing it, - // we can fetch it from the registry and use it as our parent - // and overwrite the existing codec in the registry - return codecWithParent(&protoCodec{}) -} - -// CodecWithParent returns a proxying grpc.Codec with a user provided codec as parent. -// -// This codec is *crucial* to the functioning of the proxy. It allows the proxy server to be oblivious -// to the schema of the forwarded messages. It basically treats a gRPC message frame as raw bytes. -// However, if the server handler, or the client caller are not proxy-internal functions it will fall back -// to trying to decode the message using a fallback codec. -func codecWithParent(fallback encoding.Codec) encoding.Codec { - return &Proxy{parentCodec: fallback} -} - -// Proxy satisfies the encoding.Codec interface -type Proxy struct { - parentCodec encoding.Codec -} - -// Frame holds the proxy transported data -type Frame struct { - payload []byte -} - -// ProtoMessage tags a frame as valid proto message -func (f *Frame) ProtoMessage() {} - -// Marshal implents the encoding.Codec interface method -func (p *Proxy) Marshal(v interface{}) ([]byte, error) { - out, ok := v.(*Frame) - if !ok { - return p.parentCodec.Marshal(v) - } - return out.payload, nil - -} - -// Unmarshal implents the encoding.Codec interface method -func (p *Proxy) Unmarshal(data []byte, v interface{}) error { - dst, ok := v.(*Frame) - if !ok { - return p.parentCodec.Unmarshal(data, v) - } - dst.payload = data - return nil -} - -// Name implents the encoding.Codec interface method -func (*Proxy) Name() string { - return Name -} - -// protoCodec is a Codec implementation with protobuf. It is the default rawCodec for gRPC. -type protoCodec struct{} - -func (*protoCodec) Marshal(v interface{}) ([]byte, error) { - return proto.Marshal(v.(proto.Message)) -} - -func (*protoCodec) Unmarshal(data []byte, v interface{}) error { - return proto.Unmarshal(data, v.(proto.Message)) -} - -func (*protoCodec) Name() string { - return "proxy>proto" -} diff --git a/server/start.go b/server/start.go index 7d46776a15b5..38df2329fe49 100644 --- a/server/start.go +++ b/server/start.go @@ -327,10 +327,9 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App if grpcSrv != nil { grpcSrv.Stop() - } - - if proxySrv != nil { - _ = proxySrv.Close() + if proxySrv != nil { + _ = proxySrv.Close() + } } ctx.Logger.Info("exiting...") From e2e1704dc464836ef71d2987ff2dd573d12d395c Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Tue, 15 Dec 2020 14:33:46 +0530 Subject: [PATCH 13/24] add comments --- server/grpc/grpcweb_proxy.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/grpc/grpcweb_proxy.go b/server/grpc/grpcweb_proxy.go index 608e61d57d32..8c33baa5aa32 100644 --- a/server/grpc/grpcweb_proxy.go +++ b/server/grpc/grpcweb_proxy.go @@ -91,7 +91,7 @@ func buildGrpcProxyServer(host string) (*grpc.Server, error) { director := func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error) { md, _ := metadata.FromIncomingContext(ctx) - outCtx, _ := context.WithCancel(ctx) //nolint + outCtx, _ := context.WithCancel(ctx) //nolint:govet mdCopy := md.Copy() delete(mdCopy, "user-agent") // If this header is present in the request from the web client, @@ -101,7 +101,7 @@ func buildGrpcProxyServer(host string) (*grpc.Server, error) { outCtx = metadata.NewOutgoingContext(outCtx, mdCopy) return outCtx, backendConn, nil } - // TODO: handle grpc.CustomCodec is deprecated. + // TODO: handle grpc.CustomCodec is deprecated. Waiting on https://github.com/mwitkow/grpc-proxy/pull/36. // nolint:staticcheck return grpc.NewServer( grpc.CustomCodec(proxy.Codec()), // needed for proxy to function. @@ -159,7 +159,7 @@ func dialBackendOrFail(host string) (*grpc.ClientConn, error) { opt := []grpc.DialOption{ grpc.WithInsecure(), } - // TODO: handle grpc.WithCodec is deprecated. + // TODO: handle grpc.WithCodec is deprecated. Waiting on https://github.com/mwitkow/grpc-proxy/pull/36 // nolint:staticcheck opt = append(opt, grpc.WithCodec(proxy.Codec())) From 3e1e49519415d82874e26c65c65038f4baf1fff9 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Tue, 15 Dec 2020 14:37:04 +0530 Subject: [PATCH 14/24] change grpc-proxy port --- server/config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 3c1568234975..a69816e7be1e 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -20,8 +20,8 @@ const ( // DefaultGRPCAddress is the address to bind the server to. DefaultBindAddress = "0.0.0.0" - // TCP port to listen on for HTTP1.1 debug calls. - DefaultGRPCProxyPort = 7890 + // TCP port to listen on for HTTP1.1 calls. + DefaultGRPCProxyPort = 9091 ) // BaseConfig defines the server's basic configuration From aaddecab85d32213ec0725fc52487fbc045d8615 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Wed, 16 Dec 2020 16:23:48 +0530 Subject: [PATCH 15/24] add grpc-web --- go.mod | 2 - go.sum | 2 - server/config/config.go | 64 ++++--------- server/config/toml.go | 26 ++---- server/grpc/grpcweb_proxy.go | 171 ----------------------------------- server/start.go | 26 ++++-- 6 files changed, 43 insertions(+), 248 deletions(-) delete mode 100644 server/grpc/grpcweb_proxy.go diff --git a/go.mod b/go.mod index d2a5369288f7..a4b6f879010a 100644 --- a/go.mod +++ b/go.mod @@ -30,8 +30,6 @@ require ( github.com/improbable-eng/grpc-web v0.13.0 github.com/magiconair/properties v1.8.4 github.com/mattn/go-isatty v0.0.12 - github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f - github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76 github.com/otiai10/copy v1.2.0 github.com/pelletier/go-toml v1.8.0 // indirect github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 33ff9430231a..4ad8a6bcc2f3 100644 --- a/go.sum +++ b/go.sum @@ -378,8 +378,6 @@ github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ib github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76 h1:0xuRacu/Zr+jX+KyLLPPktbwXqyOvnOPUQmMLzX1jxU= -github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76/go.mod h1:x5OoJHDHqxHS801UIuhqGl6QdSAEJvtausosHSdazIo= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= diff --git a/server/config/config.go b/server/config/config.go index a69816e7be1e..b6ae2d50e2d6 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -14,14 +14,11 @@ import ( const ( defaultMinGasPrices = "" - // DefaultGRPCAddress is the default address the gRPC server binds to. + // DefaultGRPCAddress defines the default address to bind the gRPC server to. DefaultGRPCAddress = "0.0.0.0:9090" - // DefaultGRPCAddress is the address to bind the server to. - DefaultBindAddress = "0.0.0.0" - - // TCP port to listen on for HTTP1.1 calls. - DefaultGRPCProxyPort = 9091 + // DefaultGRPCWebAddress defines the default address to bind the gRPC-web server to. + DefaultGRPCWebAddress = "0.0.0.0:9091" ) // BaseConfig defines the server's basic configuration @@ -111,33 +108,15 @@ type GRPCConfig struct { // Address defines the API server to listen on Address string `mapstructure:"address"` - - // GRPCWebProxy defines the list of flags for grpc-proxy. - GRPCWebProxy GRPCProxy `mapstructure:"grpc-proxy"` } -// GRPProxy defines configuration for the gRPC server. -type GRPCProxy struct { - // Enable defines if the proxy should be enabled. +// GRPCWeb defines configuration for the gRPC-web server. +type GRPCWeb struct { + // Enable defines if the gRPC-web should be enabled. Enable bool `mapstructure:"enable"` - // EnableHTTPServer defines if the HTTP should be enabled. - EnableHTTPServer bool `mapstructure:"enable-http-server"` - - // AllowAllOrigins defines allow requests from any origin. - AllowAllOrigins bool `mapstructure:"allow-all-origins"` - - // HTTPPort defines TCP port to listen on for HTTP1.1 debug calls. - HTTPPort int `mapstructure:"http-port"` - - // BindAddress defines address to bind the server to. - BindAddress string `mapstructure:"bind-address"` - - // AllowedOrigins defines list of origin URLs which are allowed to make cross-origin requests. - AllowedOrigins []string `mapstructure:"allowed-origins"` - - // AllowedHeaders defines list of headers which are allowed to propagate to the gRPC backend. - AllowedHeaders []string `mapstructure:"allowed-headers"` + // Address defines the gRPC-web server to listen on + Address string `mapstructure:"address"` } // StateSyncConfig defines the state sync snapshot configuration. @@ -159,6 +138,7 @@ type Config struct { Telemetry telemetry.Config `mapstructure:"telemetry"` API APIConfig `mapstructure:"api"` GRPC GRPCConfig `mapstructure:"grpc"` + GRPCWeb GRPCWeb `mapstructure:"grpc-web"` StateSync StateSyncConfig `mapstructure:"state-sync"` } @@ -217,15 +197,10 @@ func DefaultConfig() *Config { GRPC: GRPCConfig{ Enable: true, Address: DefaultGRPCAddress, - GRPCWebProxy: GRPCProxy{ - Enable: true, - AllowAllOrigins: true, - EnableHTTPServer: true, - AllowedOrigins: []string{DefaultGRPCAddress}, - AllowedHeaders: make([]string, 0), - BindAddress: DefaultBindAddress, - HTTPPort: DefaultGRPCProxyPort, - }, + }, + GRPCWeb: GRPCWeb{ + Enable: true, + Address: DefaultGRPCWebAddress, }, StateSync: StateSyncConfig{ SnapshotInterval: 0, @@ -280,15 +255,10 @@ func GetConfig(v *viper.Viper) Config { GRPC: GRPCConfig{ Enable: v.GetBool("grpc.enable"), Address: v.GetString("grpc.address"), - GRPCWebProxy: GRPCProxy{ - Enable: v.GetBool("grpc.grpc-proxy.enable"), - AllowAllOrigins: v.GetBool("grpc.grpc-proxy.allow-all-origins"), - AllowedHeaders: v.GetStringSlice("grpc.grpc-proxy.allowed-headers"), - AllowedOrigins: v.GetStringSlice("grpc.grpc-proxy.allowed-origins"), - EnableHTTPServer: v.GetBool("grpc.grpc-proxy.enable-http-server"), - BindAddress: v.GetString("grpc.grpc-proxy.bind-address"), - HTTPPort: v.GetInt("grpc.grpc-proxy.http-port"), - }, + }, + GRPCWeb: GRPCWeb{ + Enable: v.GetBool("grpc-web.enable"), + Address: v.GetString("grpc-web.address"), }, StateSync: StateSyncConfig{ SnapshotInterval: v.GetUint64("state-sync.snapshot-interval"), diff --git a/server/config/toml.go b/server/config/toml.go index c94f5c651c68..42e9884840c3 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -147,27 +147,17 @@ enable = {{ .GRPC.Enable }} # Address defines the gRPC server address to bind to. address = "{{ .GRPC.Address }}" -[grpc.grpc-proxy] - # GRPCWebProxyEnable defines if the gRPC-web-proxy should be enabled. - enable = {{ .GRPC.GRPCWebProxy.Enable }} - - # BindAddress defines address to bind the server to. - bind-address = "{{ .GRPC.GRPCWebProxy.BindAddress }}" - - # HTTPPort defines TCP port to listen on for HTTP1.1 debug calls. - http-port = {{ .GRPC.GRPCWebProxy.HTTPPort }} - - # AllowAllOrigins defines allow requests from any origin. - allow-all-origins = {{ .GRPC.GRPCWebProxy.AllowAllOrigins }} +############################################################################### +### gRPC Web Configuration ### +############################################################################### - # AllowedOrigins defines list of origin URLs which are allowed to make cross-origin requests. - allowed-origins = [{{ range $k, $v := .GRPC.GRPCWebProxy.AllowedOrigins }} "{{ $v }}", {{ end }}] +[grpc-web] - # AllowedHeaders defines list of headers which are allowed to propagate to the gRPC backend. - allowed-headers = [{{ range $k, $v := .GRPC.GRPCWebProxy.AllowedHeaders }} "{{ $v }}", {{ end }}] +# GRPCWebEnable defines if the gRPC-web should be enabled. +enable = {{ .GRPCWeb.Enable }} - # EnableHTTPServer defines if the HTTP should be enabled. - enable-http-server = {{ .GRPC.GRPCWebProxy.EnableHTTPServer }} +# Address defines the gRPC-web server address to bind to. +address = "{{ .GRPCWeb.Address }}" ############################################################################### ### State Sync Configuration ### diff --git a/server/grpc/grpcweb_proxy.go b/server/grpc/grpcweb_proxy.go deleted file mode 100644 index 8c33baa5aa32..000000000000 --- a/server/grpc/grpcweb_proxy.go +++ /dev/null @@ -1,171 +0,0 @@ -package grpc - -import ( - "context" - "fmt" - "net" - "net/http" - "time" - - "github.com/cosmos/cosmos-sdk/server/config" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - "github.com/improbable-eng/grpc-web/go/grpcweb" - "github.com/mwitkow/go-conntrack" - "github.com/mwitkow/grpc-proxy/proxy" - "google.golang.org/grpc" - "google.golang.org/grpc/metadata" -) - -var ( - flagHTTPMaxReadTimeout = 10 * time.Second - flagHTTPMaxWriteTimeout = 10 * time.Second -) - -type allowedOrigins struct { - origins map[string]struct{} -} - -// StartGRPCProxyServer starts a gRPC-proxy server on the given config. -// https://github.com/improbable-eng/grpc-web/tree/master/go/grpcwebproxy used as a reference. -func StartGRPCProxyServer(grpcConfig config.GRPCConfig) (*http.Server, error) { - proxyFlags := grpcConfig.GRPCWebProxy - - grpcSrv, err := buildGrpcProxyServer(grpcConfig.Address) - if err != nil { - return nil, err - } - - allowedOrigins := makeAllowedOrigins(proxyFlags.AllowedOrigins) - - options := []grpcweb.Option{ - grpcweb.WithCorsForRegisteredEndpointsOnly(false), - grpcweb.WithOriginFunc(makeHTTPOriginFunc(allowedOrigins, proxyFlags.AllowAllOrigins)), - } - - wrappedGrpc := grpcweb.WrapServer(grpcSrv, options...) - - if !proxyFlags.EnableHTTPServer { - return nil, fmt.Errorf("run_http_server is set to false. enable for grpcweb proxy to function correctly") - } - - proxyServer := buildServer(wrappedGrpc) - listener, err := buildListenerOrFail("http", proxyFlags.BindAddress, proxyFlags.HTTPPort) - - if err != nil { - return nil, err - } - errCh := make(chan error) - - go func() { - err = proxyServer.Serve(listener) - if err != nil { - errCh <- fmt.Errorf("failed to serve: %w", err) - } - }() - - select { - case err := <-errCh: - return nil, err - case <-time.After(5 * time.Second): // assume server started successfully - return proxyServer, nil - } - -} - -// nolint:interfacer -func buildServer(wrappedGrpc *grpcweb.WrappedGrpcServer) *http.Server { - return &http.Server{ - ReadTimeout: flagHTTPMaxReadTimeout, - WriteTimeout: flagHTTPMaxWriteTimeout, - Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - wrappedGrpc.ServeHTTP(resp, req) - }), - } -} - -func buildGrpcProxyServer(host string) (*grpc.Server, error) { - backendConn, err := dialBackendOrFail(host) - if err != nil { - return nil, err - } - - director := func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error) { - md, _ := metadata.FromIncomingContext(ctx) - outCtx, _ := context.WithCancel(ctx) //nolint:govet - mdCopy := md.Copy() - delete(mdCopy, "user-agent") - // If this header is present in the request from the web client, - // the actual connection to the backend will not be established. - // https://github.com/improbable-eng/grpc-web/issues/568 - delete(mdCopy, "connection") - outCtx = metadata.NewOutgoingContext(outCtx, mdCopy) - return outCtx, backendConn, nil - } - // TODO: handle grpc.CustomCodec is deprecated. Waiting on https://github.com/mwitkow/grpc-proxy/pull/36. - // nolint:staticcheck - return grpc.NewServer( - grpc.CustomCodec(proxy.Codec()), // needed for proxy to function. - grpc.UnknownServiceHandler(proxy.TransparentHandler(director)), - grpc_middleware.WithUnaryServerChain(), - grpc_middleware.WithStreamServerChain(), - ), nil -} - -func buildListenerOrFail(name string, host string, port int) (net.Listener, error) { - addr := fmt.Sprintf("%s:%d", host, port) - listener, err := net.Listen("tcp", addr) - if err != nil { - return nil, fmt.Errorf("failed listening for '%v' on %v: %v", name, port, err) - } - - return conntrack.NewListener(listener, - conntrack.TrackWithName(name), - conntrack.TrackWithTcpKeepAlive(20*time.Second), - conntrack.TrackWithTracing(), - ), nil -} - -func makeHTTPOriginFunc(allowedOrigins *allowedOrigins, allowAllOrigins bool) func(origin string) bool { - if allowAllOrigins { - return func(origin string) bool { - return true - } - } - - return allowedOrigins.IsAllowed -} - -func makeAllowedOrigins(origins []string) *allowedOrigins { - o := map[string]struct{}{} - for _, allowedOrigin := range origins { - o[allowedOrigin] = struct{}{} - } - - return &allowedOrigins{ - origins: o, - } -} - -func (a *allowedOrigins) IsAllowed(origin string) bool { - _, ok := a.origins[origin] - return ok -} - -func dialBackendOrFail(host string) (*grpc.ClientConn, error) { - if host == "" { - return nil, fmt.Errorf("host cannot be empty") - } - - opt := []grpc.DialOption{ - grpc.WithInsecure(), - } - // TODO: handle grpc.WithCodec is deprecated. Waiting on https://github.com/mwitkow/grpc-proxy/pull/36 - // nolint:staticcheck - opt = append(opt, grpc.WithCodec(proxy.Codec())) - - cc, err := grpc.Dial(host, opt...) - if err != nil { - return nil, fmt.Errorf("failed dialing backend: %v", err) - } - return cc, nil -} diff --git a/server/start.go b/server/start.go index 574ae5fe9ba2..d50496a822d4 100644 --- a/server/start.go +++ b/server/start.go @@ -9,6 +9,7 @@ import ( "runtime/pprof" "time" + "github.com/improbable-eng/grpc-web/go/grpcweb" "github.com/spf13/cobra" "github.com/tendermint/tendermint/abci/server" tcmd "github.com/tendermint/tendermint/cmd/tendermint/commands" @@ -305,17 +306,26 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App } } - var grpcSrv *grpc.Server - var proxySrv *http.Server + var ( + grpcSrv *grpc.Server + grpcWebSrv *http.Server + ) if config.GRPC.Enable { grpcSrv, err = servergrpc.StartGRPCServer(app, config.GRPC.Address) if err != nil { return err } - if config.GRPC.GRPCWebProxy.Enable { - proxySrv, err = servergrpc.StartGRPCProxyServer(config.GRPC) - proxySrv.ListenAndServe() - if err != nil { + if config.GRPCWeb.Enable { + wrappedServer := grpcweb.WrapServer(grpcSrv) + handler := func(resp http.ResponseWriter, req *http.Request) { + wrappedServer.ServeHTTP(resp, req) + } + grpcWebSrv = &http.Server{ + Addr: config.GRPCWeb.Address, + Handler: http.HandlerFunc(handler), + } + if err := grpcWebSrv.ListenAndServe(); err != nil { + ctx.Logger.Error("failed starting http server: ", err) return err } } @@ -336,8 +346,8 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App if grpcSrv != nil { grpcSrv.Stop() - if proxySrv != nil { - _ = proxySrv.Close() + if grpcWebSrv != nil { + grpcWebSrv.Close() } } From a29f2a4b537e923c992d35e357fcc4f8153ac3cd Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Mon, 21 Dec 2020 09:42:13 +0530 Subject: [PATCH 16/24] update server/start.go --- server/start.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/server/start.go b/server/start.go index d50496a822d4..262458cfdf9e 100644 --- a/server/start.go +++ b/server/start.go @@ -55,9 +55,10 @@ const ( // GRPC-related flags. const ( - flagGRPCEnable = "grpc.enable" - flagGRPCAddress = "grpc.address" - flagGRPCProxyEnable = "grpc.grpc-proxy.enable" + flagGRPCEnable = "grpc.enable" + flagGRPCAddress = "grpc.address" + flagGRPCWebEnable = "grpc.grpc-web.enable" + flagGRPCWebAddress = "grpc.grpc-web.address" ) // State sync-related flags. @@ -152,7 +153,8 @@ which accepts a path for the resulting pprof file. cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled") cmd.Flags().String(flagGRPCAddress, config.DefaultGRPCAddress, "the gRPC server address to listen on") - cmd.Flags().Bool(flagGRPCProxyEnable, true, "Define if the gRPC-Proxy server should be enabled") + cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled") + cmd.Flags().String(flagGRPCWebAddress, config.DefaultGRPCAddress, "the gRPC-Web server address to listen on") cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval") cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep") @@ -325,7 +327,7 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App Handler: http.HandlerFunc(handler), } if err := grpcWebSrv.ListenAndServe(); err != nil { - ctx.Logger.Error("failed starting http server: ", err) + ctx.Logger.Error("failed starting grpc-web http server: ", err) return err } } From 72f0ce1e2ed4013b60c6789afb2a50cb4d05d461 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Tue, 22 Dec 2020 10:24:25 +0530 Subject: [PATCH 17/24] add tests --- server/grpc/grpc_web.go | 25 +++++++++++++++++++++++++ server/start.go | 12 ++---------- testutil/network/network.go | 17 ++++++++++++++--- testutil/network/util.go | 18 ++++++++++++++++++ 4 files changed, 59 insertions(+), 13 deletions(-) create mode 100644 server/grpc/grpc_web.go diff --git a/server/grpc/grpc_web.go b/server/grpc/grpc_web.go new file mode 100644 index 000000000000..30c71808e43e --- /dev/null +++ b/server/grpc/grpc_web.go @@ -0,0 +1,25 @@ +package grpc + +import ( + "net/http" + + "github.com/cosmos/cosmos-sdk/server/config" + "github.com/improbable-eng/grpc-web/go/grpcweb" + "google.golang.org/grpc" +) + +// StartGRPCWeb starts a gRPC-Web server on the given address. +func StartGRPCWeb(grpcSrv *grpc.Server, config config.Config) (*http.Server, error) { + wrappedServer := grpcweb.WrapServer(grpcSrv) + handler := func(resp http.ResponseWriter, req *http.Request) { + wrappedServer.ServeHTTP(resp, req) + } + grpcWebSrv := &http.Server{ + Addr: config.GRPCWeb.Address, + Handler: http.HandlerFunc(handler), + } + if err := grpcWebSrv.ListenAndServe(); err != nil { + return nil, err + } + return grpcWebSrv, nil +} diff --git a/server/start.go b/server/start.go index 262458cfdf9e..a50b621b0898 100644 --- a/server/start.go +++ b/server/start.go @@ -9,7 +9,6 @@ import ( "runtime/pprof" "time" - "github.com/improbable-eng/grpc-web/go/grpcweb" "github.com/spf13/cobra" "github.com/tendermint/tendermint/abci/server" tcmd "github.com/tendermint/tendermint/cmd/tendermint/commands" @@ -318,15 +317,8 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App return err } if config.GRPCWeb.Enable { - wrappedServer := grpcweb.WrapServer(grpcSrv) - handler := func(resp http.ResponseWriter, req *http.Request) { - wrappedServer.ServeHTTP(resp, req) - } - grpcWebSrv = &http.Server{ - Addr: config.GRPCWeb.Address, - Handler: http.HandlerFunc(handler), - } - if err := grpcWebSrv.ListenAndServe(); err != nil { + grpcWebSrv, err = servergrpc.StartGRPCWeb(grpcSrv, config) + if err != nil { ctx.Logger.Error("failed starting grpc-web http server: ", err) return err } diff --git a/testutil/network/network.go b/testutil/network/network.go index 23706c12ca80..2c10772253ff 100644 --- a/testutil/network/network.go +++ b/testutil/network/network.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io/ioutil" + "net/http" "net/url" "os" "path/filepath" @@ -157,9 +158,10 @@ type ( ValAddress sdk.ValAddress RPCClient tmclient.Client - tmNode *node.Node - api *api.Server - grpc *grpc.Server + tmNode *node.Node + api *api.Server + grpc *grpc.Server + grpcWeb *http.Server } ) @@ -212,6 +214,7 @@ func New(t *testing.T, cfg Config) *Network { apiAddr := "" tmCfg.RPC.ListenAddress = "" appCfg.GRPC.Enable = false + appCfg.GRPCWeb.Enable = false if i == 0 { apiListenAddr, _, err := server.FreeTCPAddr() require.NoError(t, err) @@ -229,6 +232,11 @@ func New(t *testing.T, cfg Config) *Network { require.NoError(t, err) appCfg.GRPC.Address = fmt.Sprintf("0.0.0.0:%s", grpcPort) appCfg.GRPC.Enable = true + + _, grpcWebPort, err := server.FreeTCPAddr() + require.NoError(t, err) + appCfg.GRPCWeb.Address = fmt.Sprintf("0.0.0.0:%s", grpcWebPort) + appCfg.GRPCWeb.Enable = true } logger := log.NewNopLogger() @@ -466,6 +474,9 @@ func (n *Network) Cleanup() { if v.grpc != nil { v.grpc.Stop() + if v.grpcWeb != nil { + _ = v.grpcWeb.Close() + } } } diff --git a/testutil/network/util.go b/testutil/network/util.go index 8906c929e093..3a66039179ef 100644 --- a/testutil/network/util.go +++ b/testutil/network/util.go @@ -99,6 +99,24 @@ func startInProcess(cfg Config, val *Validator) error { } val.grpc = grpcSrv + + if val.AppConfig.GRPCWeb.Enable { + errCh1 := make(chan error) + go func() { + grpcWeb, err := servergrpc.StartGRPCWeb(grpcSrv, *val.AppConfig) + if err != nil { + errCh1 <- err + } + + val.grpcWeb = grpcWeb + }() + select { + case err := <-errCh1: + return err + case <-time.After(5 * time.Second): // assume server started successfully + } + + } } return nil From d57513dde33f249696cc612b807d9352292c10f1 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Tue, 22 Dec 2020 14:14:40 +0530 Subject: [PATCH 18/24] add test with client --- server/grpc/grpc_web_test.go | 313 +++++++++++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100644 server/grpc/grpc_web_test.go diff --git a/server/grpc/grpc_web_test.go b/server/grpc/grpc_web_test.go new file mode 100644 index 000000000000..8f563becc666 --- /dev/null +++ b/server/grpc/grpc_web_test.go @@ -0,0 +1,313 @@ +package grpc_test + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/textproto" + "strconv" + "strings" + "testing" + + "github.com/golang/protobuf/proto" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc/codes" + + "github.com/cosmos/cosmos-sdk/client/grpc/tmservice" + "github.com/cosmos/cosmos-sdk/codec" + cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" + "github.com/cosmos/cosmos-sdk/testutil/network" + banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" +) + +// https://github.com/improbable-eng/grpc-web/blob/master/go/grpcweb/wrapper_test.go used as a reference +// to setup grpcRequest config. + +const grpcWebContentType = "application/grpc-web" + +type GRPCWebTestSuite struct { + suite.Suite + + cfg network.Config + network *network.Network + protoCdc *codec.ProtoCodec +} + +func (s *GRPCWebTestSuite) SetupSuite() { + s.T().Log("setting up integration test suite") + + cfg := network.DefaultConfig() + cfg.NumValidators = 1 + s.cfg = cfg + s.network = network.New(s.T(), s.cfg) + s.Require().NotNil(s.network) + + _, err := s.network.WaitForHeight(2) + s.Require().NoError(err) + + s.protoCdc = codec.NewProtoCodec(s.cfg.InterfaceRegistry) +} + +func (s *GRPCWebTestSuite) TearDownSuite() { + s.T().Log("tearing down integration test suite") + s.network.Cleanup() +} + +func (s *GRPCWebTestSuite) Test_Latest_Validators() { + val := s.network.Validators[0] + for _, contentType := range []string{grpcWebContentType} { + headers, trailers, responses, err := s.makeGrpcRequest( + "/cosmos.base.tendermint.v1beta1.Service/GetLatestValidatorSet", + headerWithFlag(), + serializeProtoMessages([]proto.Message{&tmservice.GetLatestValidatorSetRequest{}}), false) + + s.Require().NoError(err) + s.Require().Equal(1, len(responses)) + s.assertTrailerGrpcCode(trailers, codes.OK, "") + s.assertContentTypeSet(headers, contentType) + var valsSet tmservice.GetLatestValidatorSetResponse + err = s.protoCdc.UnmarshalBinaryBare(responses[0], &valsSet) + s.Require().NoError(err) + pubKey, ok := valsSet.Validators[0].PubKey.GetCachedValue().(cryptotypes.PubKey) + s.Require().Equal(true, ok) + s.Require().Equal(pubKey, val.PubKey) + } +} + +func (s *GRPCWebTestSuite) Test_Total_Supply() { + for _, contentType := range []string{grpcWebContentType} { + headers, trailers, responses, err := s.makeGrpcRequest( + "/cosmos.bank.v1beta1.Query/TotalSupply", + headerWithFlag(), + serializeProtoMessages([]proto.Message{&banktypes.QueryTotalSupplyRequest{}}), false) + + s.Require().NoError(err) + s.Require().Equal(1, len(responses)) + s.assertTrailerGrpcCode(trailers, codes.OK, "") + s.assertContentTypeSet(headers, contentType) + var totalSupply banktypes.QueryTotalSupplyResponse + _ = s.protoCdc.UnmarshalBinaryBare(responses[0], &totalSupply) + } +} + +func (s *GRPCWebTestSuite) assertContentTypeSet(headers http.Header, contentType string) { + s.Require().Equal(contentType, headers.Get("content-type"), `Expected there to be content-type=%v`, contentType) +} + +func (s *GRPCWebTestSuite) assertTrailerGrpcCode(trailers Trailer, code codes.Code, desc string) { + s.Require().NotEmpty(trailers.Get("grpc-status"), "grpc-status must not be empty in trailers") + statusCode, err := strconv.Atoi(trailers.Get("grpc-status")) + s.Require().NoError(err, "no error parsing grpc-status") + s.Require().EqualValues(code, statusCode, "grpc-status must match expected code") + s.Require().EqualValues(desc, trailers.Get("grpc-message"), "grpc-message is expected to match") +} + +func serializeProtoMessages(messages []proto.Message) [][]byte { + out := [][]byte{} + for _, m := range messages { + b, _ := proto.Marshal(m) + out = append(out, b) + } + return out +} + +func (s *GRPCWebTestSuite) makeRequest( + verb string, method string, headers http.Header, body io.Reader, isText bool, +) (*http.Response, error) { + val := s.network.Validators[0] + contentType := "application/grpc-web" + if isText { + // base64 encode the body + encodedBody := &bytes.Buffer{} + encoder := base64.NewEncoder(base64.StdEncoding, encodedBody) + _, err := io.Copy(encoder, body) + if err != nil { + return nil, err + } + err = encoder.Close() + if err != nil { + return nil, err + } + body = encodedBody + contentType = "application/grpc-web-text" + } + + url := fmt.Sprintf("http://%s%s", val.AppConfig.GRPCWeb.Address, method) + req, err := http.NewRequest(verb, url, body) + s.Require().NoError(err, "failed creating a request") + req.Header = headers + + req.Header.Set("Content-Type", contentType) + client := &http.Client{} + resp, err := client.Do(req) + return resp, err +} + +func decodeMultipleBase64Chunks(b []byte) ([]byte, error) { + // grpc-web allows multiple base64 chunks: the implementation may send base64-encoded + // "chunks" with potential padding whenever the runtime needs to flush a byte buffer. + // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-WEB.md + output := make([]byte, base64.StdEncoding.DecodedLen(len(b))) + outputEnd := 0 + + for inputEnd := 0; inputEnd < len(b); { + chunk := b[inputEnd:] + paddingIndex := bytes.IndexByte(chunk, '=') + if paddingIndex != -1 { + // find the consecutive = + for { + paddingIndex += 1 + if paddingIndex >= len(chunk) || chunk[paddingIndex] != '=' { + break + } + } + chunk = chunk[:paddingIndex] + } + inputEnd += len(chunk) + + n, err := base64.StdEncoding.Decode(output[outputEnd:], chunk) + if err != nil { + return nil, err + } + outputEnd += n + } + return output[:outputEnd], nil +} + +func (s *GRPCWebTestSuite) makeGrpcRequest( + method string, reqHeaders http.Header, requestMessages [][]byte, isText bool, +) (headers http.Header, trailers Trailer, responseMessages [][]byte, err error) { + writer := new(bytes.Buffer) + for _, msgBytes := range requestMessages { + grpcPreamble := []byte{0, 0, 0, 0, 0} + binary.BigEndian.PutUint32(grpcPreamble[1:], uint32(len(msgBytes))) + writer.Write(grpcPreamble) + writer.Write(msgBytes) + } + resp, err := s.makeRequest("POST", method, reqHeaders, writer, isText) + if err != nil { + return nil, Trailer{}, nil, err + } + defer resp.Body.Close() + contents, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, Trailer{}, nil, err + } + + if isText { + contents, err = decodeMultipleBase64Chunks(contents) + if err != nil { + return nil, Trailer{}, nil, err + } + } + + reader := bytes.NewReader(contents) + for { + grpcPreamble := []byte{0, 0, 0, 0, 0} + readCount, err := reader.Read(grpcPreamble) + if err == io.EOF { + break + } + if readCount != 5 || err != nil { + return nil, Trailer{}, nil, fmt.Errorf("Unexpected end of body in preamble: %v", err) + } + payloadLength := binary.BigEndian.Uint32(grpcPreamble[1:]) + payloadBytes := make([]byte, payloadLength) + + readCount, err = reader.Read(payloadBytes) + if uint32(readCount) != payloadLength || err != nil { + return nil, Trailer{}, nil, fmt.Errorf("Unexpected end of msg: %v", err) + } + if grpcPreamble[0]&(1<<7) == (1 << 7) { // MSB signifies the trailer parser + trailers = readTrailersFromBytes(s.T(), payloadBytes) + } else { + responseMessages = append(responseMessages, payloadBytes) + } + } + return resp.Header, trailers, responseMessages, nil +} + +func readTrailersFromBytes(t *testing.T, dataBytes []byte) Trailer { + bufferReader := bytes.NewBuffer(dataBytes) + tp := textproto.NewReader(bufio.NewReader(bufferReader)) + + // First, read bytes as MIME headers. + // However, it normalizes header names by textproto.CanonicalMIMEHeaderKey. + // In the next step, replace header names by raw one. + mimeHeader, err := tp.ReadMIMEHeader() + if err == nil { + return Trailer{} + } + + trailers := make(http.Header) + bufferReader = bytes.NewBuffer(dataBytes) + tp = textproto.NewReader(bufio.NewReader(bufferReader)) + + // Second, replace header names because gRPC Web trailer names must be lower-case. + for { + line, err := tp.ReadLine() + if err == io.EOF { + break + } + require.NoError(t, err, "failed to read header line") + + i := strings.IndexByte(line, ':') + if i == -1 { + require.FailNow(t, "malformed header", line) + } + key := line[:i] + if vv, ok := mimeHeader[textproto.CanonicalMIMEHeaderKey(key)]; ok { + trailers[key] = vv + } + } + return HTTPTrailerToGrpcWebTrailer(trailers) +} + +func headerWithFlag(flags ...string) http.Header { + h := http.Header{} + for _, f := range flags { + h.Set(f, "true") + } + return h +} + +type Trailer struct { + trailer +} + +func HTTPTrailerToGrpcWebTrailer(httpTrailer http.Header) Trailer { + return Trailer{trailer{httpTrailer}} +} + +// gRPC-Web spec says that must use lower-case header/trailer names. +// See "HTTP wire protocols" section in +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-WEB.md#protocol-differences-vs-grpc-over-http2 +type trailer struct { + http.Header +} + +func (t trailer) Add(key, value string) { + key = strings.ToLower(key) + t.Header[key] = append(t.Header[key], value) +} + +func (t trailer) Get(key string) string { + if t.Header == nil { + return "" + } + v := t.Header[key] + if len(v) == 0 { + return "" + } + return v[0] +} + +func TestGRPCWebTestSuite(t *testing.T) { + suite.Run(t, new(GRPCWebTestSuite)) +} From 4debd218722f4fb175e6f97bced03d9fff35ba2f Mon Sep 17 00:00:00 2001 From: MD Aleem <72057206+aleem1314@users.noreply.github.com> Date: Wed, 23 Dec 2020 21:00:00 +0530 Subject: [PATCH 19/24] Update server/start.go Co-authored-by: Anil Kumar Kammari --- server/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/start.go b/server/start.go index a50b621b0898..4fbf8ca014e3 100644 --- a/server/start.go +++ b/server/start.go @@ -319,7 +319,7 @@ func startInProcess(ctx *Context, clientCtx client.Context, appCreator types.App if config.GRPCWeb.Enable { grpcWebSrv, err = servergrpc.StartGRPCWeb(grpcSrv, config) if err != nil { - ctx.Logger.Error("failed starting grpc-web http server: ", err) + ctx.Logger.Error("failed to start grpc-web http server: ", err) return err } } From b8bb16b89f64545b59171ac2fbcf10194ac2788a Mon Sep 17 00:00:00 2001 From: MD Aleem <72057206+aleem1314@users.noreply.github.com> Date: Mon, 4 Jan 2021 22:09:23 +0530 Subject: [PATCH 20/24] Update server/start.go Co-authored-by: Amaury --- server/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/start.go b/server/start.go index 4fbf8ca014e3..228162fe1114 100644 --- a/server/start.go +++ b/server/start.go @@ -56,7 +56,7 @@ const ( const ( flagGRPCEnable = "grpc.enable" flagGRPCAddress = "grpc.address" - flagGRPCWebEnable = "grpc.grpc-web.enable" + flagGRPCWebEnable = "grpc-web.enable" flagGRPCWebAddress = "grpc.grpc-web.address" ) From f3bc9c64adfec7d2f03bb85e9da70d2cc3ae1855 Mon Sep 17 00:00:00 2001 From: MD Aleem <72057206+aleem1314@users.noreply.github.com> Date: Mon, 4 Jan 2021 22:09:33 +0530 Subject: [PATCH 21/24] Update server/start.go Co-authored-by: Amaury --- server/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/start.go b/server/start.go index 228162fe1114..891e511cc010 100644 --- a/server/start.go +++ b/server/start.go @@ -57,7 +57,7 @@ const ( flagGRPCEnable = "grpc.enable" flagGRPCAddress = "grpc.address" flagGRPCWebEnable = "grpc-web.enable" - flagGRPCWebAddress = "grpc.grpc-web.address" + flagGRPCWebAddress = "grpc-web.address" ) // State sync-related flags. From 6e2f0706860a871466f24613d3033024d3d0d320 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Mon, 4 Jan 2021 22:21:08 +0530 Subject: [PATCH 22/24] review changes --- server/config/config.go | 10 +++++----- server/config/toml.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index b6ae2d50e2d6..68f6cf4352f6 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -110,8 +110,8 @@ type GRPCConfig struct { Address string `mapstructure:"address"` } -// GRPCWeb defines configuration for the gRPC-web server. -type GRPCWeb struct { +// GRPCWebConfig defines configuration for the gRPC-web server. +type GRPCWebConfig struct { // Enable defines if the gRPC-web should be enabled. Enable bool `mapstructure:"enable"` @@ -138,7 +138,7 @@ type Config struct { Telemetry telemetry.Config `mapstructure:"telemetry"` API APIConfig `mapstructure:"api"` GRPC GRPCConfig `mapstructure:"grpc"` - GRPCWeb GRPCWeb `mapstructure:"grpc-web"` + GRPCWeb GRPCWebConfig `mapstructure:"grpc-web"` StateSync StateSyncConfig `mapstructure:"state-sync"` } @@ -198,7 +198,7 @@ func DefaultConfig() *Config { Enable: true, Address: DefaultGRPCAddress, }, - GRPCWeb: GRPCWeb{ + GRPCWeb: GRPCWebConfig{ Enable: true, Address: DefaultGRPCWebAddress, }, @@ -256,7 +256,7 @@ func GetConfig(v *viper.Viper) Config { Enable: v.GetBool("grpc.enable"), Address: v.GetString("grpc.address"), }, - GRPCWeb: GRPCWeb{ + GRPCWeb: GRPCWebConfig{ Enable: v.GetBool("grpc-web.enable"), Address: v.GetString("grpc-web.address"), }, diff --git a/server/config/toml.go b/server/config/toml.go index 42e9884840c3..556b95126470 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -148,7 +148,7 @@ enable = {{ .GRPC.Enable }} address = "{{ .GRPC.Address }}" ############################################################################### -### gRPC Web Configuration ### +### gRPC Web Configuration ### ############################################################################### [grpc-web] From 5468ca091f3a0fb1b11e72a0707ccd9cf9a85661 Mon Sep 17 00:00:00 2001 From: aleem1413 Date: Wed, 13 Jan 2021 11:15:38 +0530 Subject: [PATCH 23/24] review changes --- server/config/toml.go | 1 + server/start.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/server/config/toml.go b/server/config/toml.go index 556b95126470..58dc7bfed2b4 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -154,6 +154,7 @@ address = "{{ .GRPC.Address }}" [grpc-web] # GRPCWebEnable defines if the gRPC-web should be enabled. +# NOTE: gRPC must also be enabled, otherwise, this configuration is a no-op. enable = {{ .GRPCWeb.Enable }} # Address defines the gRPC-web server address to bind to. diff --git a/server/start.go b/server/start.go index 891e511cc010..351b2fdd980b 100644 --- a/server/start.go +++ b/server/start.go @@ -152,7 +152,8 @@ which accepts a path for the resulting pprof file. cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled") cmd.Flags().String(flagGRPCAddress, config.DefaultGRPCAddress, "the gRPC server address to listen on") - cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled") + + cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled. (Note: gRPC must also be enabled.)") cmd.Flags().String(flagGRPCWebAddress, config.DefaultGRPCAddress, "the gRPC-Web server address to listen on") cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval") From 24e8f9ed3f681a298399396b7bacbf3d3c66fc3c Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Thu, 14 Jan 2021 08:44:00 -0500 Subject: [PATCH 24/24] Update server/start.go --- server/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/start.go b/server/start.go index 351b2fdd980b..84c4dc858651 100644 --- a/server/start.go +++ b/server/start.go @@ -154,7 +154,7 @@ which accepts a path for the resulting pprof file. cmd.Flags().String(flagGRPCAddress, config.DefaultGRPCAddress, "the gRPC server address to listen on") cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled. (Note: gRPC must also be enabled.)") - cmd.Flags().String(flagGRPCWebAddress, config.DefaultGRPCAddress, "the gRPC-Web server address to listen on") + cmd.Flags().String(flagGRPCWebAddress, config.DefaultGRPCAddress, "The gRPC-Web server address to listen on") cmd.Flags().Uint64(FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval") cmd.Flags().Uint32(FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep")