Merge branch 'master' into sds_api

DEPRECATED.md

@@ -18,6 +18,11 @@ A logged warning is expected for each deprecated item that is in deprecation win
   `use_data_plane_proto` boolean flag in the [ratelimit configuration](https://github.com/envoyproxy/envoy/blob/master/api/envoy/config/ratelimit/v2/rls.proto).
   However, when using the deprecated client a warning is logged.
 * Use of the --v2-config-only flag.
+* Use of both `use_websocket` and `websocket_config` in
+  [route.proto](https://github.com/envoyproxy/envoy/blob/master/api/envoy/api/v2/route/route.proto)
+  is deprecated. Please use the new `upgrade_configs` in the
+  [HttpConnectionManager](https://github.com/envoyproxy/envoy/blob/master/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto)
+  instead.
 
 ## Version 1.7.0
 

api/envoy/api/v2/core/base.proto

@@ -133,7 +133,7 @@ enum RequestMethod {
 // Header name/value pair.
 message HeaderValue {
   // Header name.
-  string key = 1;
+  string key = 1 [(validate.rules).string.min_bytes = 1];
 
   // Header value.
   //

api/envoy/api/v2/core/health_check.proto

@@ -152,16 +152,17 @@ message HealthCheck {
     // TCP health check.
     TcpHealthCheck tcp_health_check = 9;
 
-    // Redis health check.
-    RedisHealthCheck redis_health_check = 10;
-
     // gRPC health check.
     GrpcHealthCheck grpc_health_check = 11;
 
     // Custom health check.
     CustomHealthCheck custom_health_check = 13;
   }
 
+  reserved 10; // redis_health_check is deprecated by :ref:`custom_health_check
+               // <envoy_api_field_core.HealthCheck.custom_health_check>`
+  reserved "redis_health_check";
+
   // The "no traffic interval" is a special health check interval that is used when a cluster has
   // never had traffic routed to it. This lower interval allows cluster information to be kept up to
   // date, without sending a potentially large amount of active health checking traffic for no

api/envoy/api/v2/endpoint/load_report.proto

@@ -4,6 +4,8 @@ package envoy.api.v2.endpoint;
 
 import "envoy/api/v2/core/base.proto";
 
+import "google/protobuf/duration.proto";
+
 import "validate/validate.proto";
 import "gogoproto/gogo.proto";
 
@@ -93,4 +95,10 @@ message ClusterStats {
   // The total number of dropped requests. This covers requests
   // deliberately dropped by the drop_overload policy and circuit breaking.
   uint64 total_dropped_requests = 3;
+
+  // Period over which the actual load report occurred. This will be guaranteed to include every
+  // request reported. Due to system load and delays between the *LoadStatsRequest* sent from Envoy
+  // and the *LoadStatsResponse* message sent from the management server, this may be longer than
+  // the requested load reporting interval in the *LoadStatsResponse*.
+  google.protobuf.Duration load_report_interval = 4;
 }

api/envoy/api/v2/route/route.proto

@@ -313,7 +313,7 @@ message CorsPolicy {
   google.protobuf.BoolValue enabled = 7;
 }
 
-// [#comment:next free field: 24]
+// [#comment:next free field: 25]
 message RouteAction {
   oneof cluster_specifier {
     option (validate.required) = true;
@@ -401,7 +401,9 @@ message RouteAction {
     google.protobuf.BoolValue auto_host_rewrite = 7;
   }
 
-  // Specifies the timeout for the route. If not specified, the default is 15s.
+  // Specifies the upstream timeout for the route. If not specified, the default is 15s. This
+  // spans between the point at which the entire downstream request (i.e. end-of-stream) has been
+  // processed and when the upstream response has been completely processed.
   //
   // .. note::
   //
@@ -423,8 +425,8 @@ message RouteAction {
     // :ref:`config_http_filters_router_x-envoy-max-retries`.
     google.protobuf.UInt32Value num_retries = 2;
 
-    // Specifies a non-zero timeout per retry attempt. This parameter is optional.
-    // The same conditions documented for
+    // Specifies a non-zero upstream timeout per retry attempt. This parameter is optional. The
+    // same conditions documented for
     // :ref:`config_http_filters_router_x-envoy-upstream-rq-per-try-timeout-ms` apply.
     //
     // .. note::
@@ -437,6 +439,28 @@ message RouteAction {
     google.protobuf.Duration per_try_timeout = 3 [(gogoproto.stdduration) = true];
   }
 
+  // Specifies the idle timeout for the route. If not specified, this defaults
+  // to 5 minutes. The default value was select so as not to interfere with any
+  // smaller configured timeouts that may have existed in configurations prior
+  // to the introduction of this feature, while introducing robustness to TCP
+  // connections that terminate without FIN. A value of 0 will completely
+  // disable the idle timeout.
+  //
+  // The idle timeout is distinct to :ref:`timeout
+  // <envoy_api_field_route.RouteAction.timeout>`, which provides an upper bound
+  // on the upstream response time; :ref:`idle_timeout
+  // <envoy_api_field_route.RouteAction.idle_timeout>` instead bounds the amount
+  // of time the request's stream may be idle.
+  //
+  // After header decoding, the idle timeout will apply on downstream and
+  // upstream request events. Each time an encode/decode event for headers or
+  // data is processed for the stream, the timer will be reset. If the timeout
+  // fires, the stream is terminated with a 408 Request Timeout error code if no
+  // upstream response header has been received, otherwise a stream reset
+  // occurs.
+  google.protobuf.Duration idle_timeout = 24
+      [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true];
+
   // Indicates that the route has a retry policy.
   RetryPolicy retry_policy = 9;
 
@@ -581,7 +605,7 @@ message RouteAction {
   //   proxy data from the client to the upstream server.
   //
   //   Redirects are not supported on routes where WebSocket upgrades are allowed.
-  google.protobuf.BoolValue use_websocket = 16;
+  google.protobuf.BoolValue use_websocket = 16 [deprecated = true];
 
   message WebSocketProxyConfig {
     // See :ref:`stat_prefix
@@ -611,7 +635,7 @@ message RouteAction {
 
   // Proxy configuration used for WebSocket connections. If unset, the default values as specified
   // in :ref:`TcpProxy <envoy_api_msg_config.filter.network.tcp_proxy.v2.TcpProxy>` are used.
-  WebSocketProxyConfig websocket_config = 22;
+  WebSocketProxyConfig websocket_config = 22 [deprecated = true];
 
   // Indicates that the route has a CORS policy.
   CorsPolicy cors = 17;

api/envoy/config/rbac/v2alpha/rbac.proto

@@ -16,12 +16,11 @@ option go_package = "v2alpha";
 //
 // Here is an example of RBAC configuration. It has two policies:
 //
-// * Service account "cluster.local/ns/default/sa/admin" has full access (empty permission entry
-//   means full access) to the service.
+// * Service account "cluster.local/ns/default/sa/admin" has full access to the service, and so
+//   does "cluster.local/ns/default/sa/superuser".
 //
-// * Any user (empty principal entry means any user) can read ("GET") the service at paths with
-//   prefix "/products" or suffix "/reviews" when request header "version" set to either "v1" or
-//   "v2".
+// * Any user can read ("GET") the service at paths with prefix "/products", so long as the
+//   destination port is either 80 or 443.
 //
 //  .. code-block:: yaml
 //

api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD

@@ -0,0 +1,8 @@
+load("//bazel:api_build_system.bzl", "api_proto_library_internal")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_library_internal(
+    name = "fixed_heap",
+    srcs = ["fixed_heap.proto"],
+)

api/envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.proto

@@ -0,0 +1,10 @@
+syntax = "proto3";
+
+package envoy.config.resource_monitor.fixed_heap.v2alpha;
+option go_package = "v2alpha";
+
+message FixedHeapConfig {
+  // Limit of the Envoy process heap size. This is used to calculate heap memory pressure which
+  // is defined as (current heap size)/max_heap_size_bytes.
+  uint64 max_heap_size_bytes = 1;
+}

api/envoy/service/load_stats/v2/lrs.proto

@@ -63,6 +63,12 @@ message LoadStatsResponse {
   // Clusters to report stats for.
   repeated string clusters = 1 [(validate.rules).repeated .min_items = 1];
 
-  // The interval of time to collect stats. The default is 10 seconds.
+  // The minimum interval of time to collect stats over. This is only a minimum for two reasons:
+  // 1. There may be some delay from when the timer fires until stats sampling occurs.
+  // 2. For clusters that were already feature in the previous *LoadStatsResponse*, any traffic
+  //    that is observed in between the corresponding previous *LoadStatsRequest* and this
+  //    *LoadStatsResponse* will also be accumulated and billed to the cluster. This avoids a period
+  //    of inobservability that might otherwise exists between the messages. New clusters are not
+  //    subject to this consideration.
   google.protobuf.Duration load_reporting_interval = 2;
 }

bazel/README.md

@@ -34,7 +34,9 @@ On Ubuntu, run the following commands:
  apt-get install cmake
  apt-get install realpath
  apt-get install clang-format-5.0
+ apt-get install autoconf
  apt-get install automake
+ apt-get install pkg-config
 ```
 
 On Fedora (maybe also other red hat distros), run the following:
@@ -50,7 +52,9 @@ brew install cmake
 brew install libtool
 brew install go
 brew install bazel
+brew install autoconf
 brew install automake
+brew install pkg-config
 ```
 
 Envoy compiles and passes tests with the version of clang installed by XCode 9.3.0:

bazel/repository_locations.bzl

@@ -68,9 +68,11 @@ REPOSITORY_LOCATIONS = dict(
         remote = "https://github.com/google/jwt_verify_lib",
     ),
     com_github_nodejs_http_parser = dict(
-        # 2018-05-30 snapshot to pick up a performance fix, nodejs/http-parser PR 422
+        # 2018-07-20 snapshot to pick up:
+        # A performance fix, nodejs/http-parser PR 422.
+        # A bug fix, nodejs/http-parser PR 432.
         # TODO(brian-pane): Upgrade to the next http-parser release once it's available
-        commit = "cf69c8eda9fe79e4682598a7b3d39338dea319a3",
+        commit = "77310eeb839c4251c07184a5db8885a572a08352",
         remote = "https://github.com/nodejs/http-parser",
     ),
     com_github_pallets_jinja = dict(

ci/build_container/build_container_ubuntu.sh

@@ -6,7 +6,7 @@ set -e
 apt-get update
 export DEBIAN_FRONTEND=noninteractive
 apt-get install -y wget software-properties-common make cmake git python python-pip \
-  bc libtool automake zip time golang g++ gdb strace wireshark tshark
+  bc libtool autoconf automake zip time golang g++ gdb strace wireshark tshark
 # clang head (currently 5.0)
 wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
 apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main"

ci/build_container/build_recipes/nghttp2.sh

@@ -2,10 +2,16 @@
 
 set -e
 
-VERSION=1.32.0
+# Use master branch, which contains a fix for the spurious limit of 100 concurrent streams:
+# https://github.com/nghttp2/nghttp2/commit/2ba1389993729fcb6ee5794ac512f2b67b29952e
+# TODO(PiotrSikora): switch back to releases once v1.33.0 is out.
+VERSION=e5b3f9addd49bca27e2f99c5c65a564eb5c0cf6d  # 2018-06-09
 
-wget -O nghttp2-"$VERSION".tar.gz https://github.com/nghttp2/nghttp2/releases/download/v"$VERSION"/nghttp2-"$VERSION".tar.gz
+wget -O nghttp2-"$VERSION".tar.gz https://github.com/nghttp2/nghttp2/archive/"$VERSION".tar.gz
 tar xf nghttp2-"$VERSION".tar.gz
 cd nghttp2-"$VERSION"
+autoreconf -i
+automake
+autoconf
 ./configure --prefix="$THIRDPARTY_BUILD" --enable-shared=no --enable-lib-only
 make V=1 install

ci/build_setup.sh

@@ -87,7 +87,7 @@ if [ "$1" != "-nofetch" ]; then
   fi
 
   # This is the hash on https://github.com/envoyproxy/envoy-filter-example.git we pin to.
-  (cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" && git fetch origin && git checkout -f 92307d723a1ead25c39f025a734fa091443efdbc)
+  (cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" && git fetch origin && git checkout -f 3e5b73305b961526ffcee7584251692a9a3ce4b3)
   cp -f "${ENVOY_SRCDIR}"/ci/WORKSPACE.filter.example "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/WORKSPACE
 fi
 

docs/root/configuration/health_checkers/redis.rst

@@ -3,12 +3,22 @@
 Redis
 =====
 
-The Redis health checker is a custom health checker which checks Redis upstream hosts. It sends
-a Redis PING command and expect a PONG response. The upstream Redis server can respond with
-anything other than PONG to cause an immediate active health check failure. Optionally, Envoy can
-perform EXISTS on a user-specified key. If the key does not exist it is considered a passing healthcheck.
-This allows the user to mark a Redis instance for maintenance by setting the specified
-:ref:`key <envoy_api_field_config.health_checker.redis.v2.Redis.key>` to any value and waiting for
-traffic to drain.
+The Redis health checker is a custom health checker (with :code:`envoy.health_checkers.redis` as name)
+which checks Redis upstream hosts. It sends a Redis PING command and expect a PONG response. The upstream
+Redis server can respond with anything other than PONG to cause an immediate active health check failure.
+Optionally, Envoy can perform EXISTS on a user-specified key. If the key does not exist it is considered a
+passing healthcheck. This allows the user to mark a Redis instance for maintenance by setting the
+specified :ref:`key <envoy_api_field_config.health_checker.redis.v2.Redis.key>` to any value and waiting
+for traffic to drain.
+
+An example setting for :ref:`custom_health_check <envoy_api_msg_core.HealthCheck.CustomHealthCheck>` as a
+Redis health checker is shown below:
+
+.. code-block:: yaml
+
+  custom_health_check:
+    name: envoy.health_checkers.redis
+      config:
+        key: foo
 
 * :ref:`v2 API reference <envoy_api_msg_core.HealthCheck.CustomHealthCheck>`

docs/root/configuration/http_conn_man/stats.rst

@@ -52,6 +52,7 @@ statistics:
    downstream_rq_5xx, Counter, Total 5xx responses
    downstream_rq_ws_on_non_ws_route, Counter, Total WebSocket upgrade requests rejected by non WebSocket routes
    downstream_rq_time, Histogram, Request time milliseconds
+   downstream_rq_idle_timeout, Counter, Total requests closed due to idle timeout
    rs_too_large, Counter, Total response errors due to buffering an overly large body
 
 Per user agent statistics

docs/root/intro/arch_overview/redis.rst

@@ -43,8 +43,10 @@ For filter configuration details, see the Redis proxy filter
 The corresponding cluster definition should be configured with
 :ref:`ring hash load balancing <config_cluster_manager_cluster_lb_type>`.
 
-If active healthchecking is desired, the cluster should be configured with a
-:ref:`Redis healthcheck <config_cluster_manager_cluster_hc>`.
+If :ref:`active health checking <arch_overview_health_checking>` is desired, the
+cluster should be configured with a :ref:`custom health check
+<envoy_api_field_core.HealthCheck.custom_health_check>` which configured as a
+:ref:`Redis health checker <config_health_checkers_redis>`.
 
 If passive healthchecking is desired, also configure
 :ref:`outlier detection <config_cluster_manager_cluster_outlier_detection_summary>`.

docs/root/intro/version_history.rst

@@ -14,16 +14,22 @@ Version history
 * health check: added support for :ref:`custom health check <envoy_api_field_core.HealthCheck.custom_health_check>`.
 * health check: added support for :ref:`specifying jitter as a percentage <envoy_api_field_core.HealthCheck.interval_jitter_percent>`.
 * health_check: added support for :ref:`health check event logging <arch_overview_health_check_logging>`.
+* http: added support for a :ref:`per-stream idle timeout
+  <envoy_api_field_route.RouteAction.idle_timeout>`. This defaults to 5 minutes; if you have
+  other timeouts (e.g. connection idle timeout, upstream response per-retry) that are longer than
+  this in duration, you may want to consider setting a non-default per-stream idle timeout.
+* http: added generic :ref:`Upgrade support 
+  <envoy_api_field_config.filter.network.http_connection_manager.v2.HttpConnectionManager.upgrade_configs>`.
 * http: better handling of HEAD requests. Now sending transfer-encoding: chunked rather than content-length: 0.
 * http: response filters not applied to early error paths such as http_parser generated 400s.
-* proxy_protocol: added support for HAProxy Proxy Protocol v2 (AF_INET/AF_INET6 only).
-* http: added generic +:ref:`Upgrade support 
-  <envoy_api_field_config.filter.network.http_connection_manager.v2.HttpConnectionManager.upgrade_configs>`
+* http: :ref:`hpack_table_size <envoy_api_field_core.Http2ProtocolOptions.hpack_table_size>` now controls
+  dynamic table size of both: encoder and decoder.
 * listeners: added the ability to match :ref:`FilterChain <envoy_api_msg_listener.FilterChain>` using
   :ref:`destination_port <envoy_api_field_listener.FilterChainMatch.destination_port>` and
   :ref:`prefix_ranges <envoy_api_field_listener.FilterChainMatch.prefix_ranges>`.
 * lua: added :ref:`connection() <config_http_filters_lua_connection_wrapper>` wrapper and *ssl()* API.
 * lua: added :ref:`requestInfo() <config_http_filters_lua_request_info_wrapper>` wrapper and *protocol()* API.
+* proxy_protocol: added support for HAProxy Proxy Protocol v2 (AF_INET/AF_INET6 only).
 * ratelimit: added support for :repo:`api/envoy/service/ratelimit/v2/rls.proto`.
   Lyft's reference implementation of the `ratelimit <https://github.com/lyft/ratelimit>`_ service also supports the data-plane-api proto as of v1.1.0.
   Envoy can use either proto to send client requests to a ratelimit server with the use of the

examples/grpc-bridge/Dockerfile-python

@@ -3,7 +3,7 @@ FROM envoyproxy/envoy:latest
 RUN apt-get update
 RUN apt-get -q install -y python-dev \
     python-pip
-RUN pip install -q grpcio requests
+RUN pip install -q grpcio protobuf requests
 ADD ./client /client
 RUN chmod a+x /client/client.py
 RUN mkdir /var/log/envoy/

include/envoy/buffer/buffer.h

@@ -4,6 +4,7 @@
 #include <functional>
 #include <memory>
 #include <string>
+#include <tuple>
 
 #include "envoy/common/pure.h"
 
@@ -142,9 +143,11 @@ class Instance {
    * Read from a file descriptor directly into the buffer.
    * @param fd supplies the descriptor to read from.
    * @param max_length supplies the maximum length to read.
-   * @return the number of bytes read or -1 if there was an error.
+   * @return a tuple with the number of bytes read and the errno. If an error occurred, the
+   *   number of bytes read would indicate -1 and the errno would be non-zero. Otherwise, if
+   *   bytes were read, errno shouldn't be used.
    */
-  virtual int read(int fd, uint64_t max_length) PURE;
+  virtual std::tuple<int, int> read(int fd, uint64_t max_length) PURE;
 
   /**
    * Reserve space in the buffer.
@@ -173,9 +176,11 @@ class Instance {
   /**
    * Write the buffer out to a file descriptor.
    * @param fd supplies the descriptor to write to.
-   * @return the number of bytes written or -1 if there was an error.
+   * @return a tuple with the number of bytes written and the errno. If an error occurred, the
+   *   number of bytes written would indicate -1 and the errno would be non-zero. Otherwise, if
+   *   bytes were written, errno shouldn't be used.
    */
-  virtual int write(int fd) PURE;
+  virtual std::tuple<int, int> write(int fd) PURE;
 };
 
 typedef std::unique_ptr<Instance> InstancePtr;

include/envoy/router/router.h

@@ -472,6 +472,11 @@ class RouteEntry : public ResponseEntry {
    */
   virtual std::chrono::milliseconds timeout() const PURE;
 
+  /**
+   * @return absl::optional<std::chrono::milliseconds> the route's idle timeout.
+   */
+  virtual absl::optional<std::chrono::milliseconds> idleTimeout() const PURE;
+
   /**
    * @return absl::optional<std::chrono::milliseconds> the maximum allowed timeout value derived
    * from 'grpc-timeout' header of a gRPC request. Non-present value disables use of 'grpc-timeout'