From b39e52a9c6f80b760c295f4aef914a6afe52dcdf Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 21 May 2021 17:26:24 -0400 Subject: [PATCH] Add healthcheck endpoint for rpc-proxy Added ethereum-nginx-proxy source updated README and docker image build --- .github/workflows/release.yml | 8 ++ ops/README.md | 2 + ops/docker-compose-rpc-proxy.yml | 17 ++++ ops/docker-compose.yml | 1 + ops/docker/Dockerfile.rpc-proxy | 21 +++++ ops/docker/Dockerfile.rpc-proxy-old | 25 ++++++ ops/docker/rpc-proxy/docker-entrypoint.sh | 20 +++++ ops/docker/rpc-proxy/eth-jsonrpc-access.lua | 91 +++++++++++++++++++++ ops/docker/rpc-proxy/nginx.template.conf | 80 ++++++++++++++++++ 9 files changed, 265 insertions(+) create mode 100644 ops/docker-compose-rpc-proxy.yml create mode 100644 ops/docker/Dockerfile.rpc-proxy create mode 100644 ops/docker/Dockerfile.rpc-proxy-old create mode 100755 ops/docker/rpc-proxy/docker-entrypoint.sh create mode 100644 ops/docker/rpc-proxy/eth-jsonrpc-access.lua create mode 100644 ops/docker/rpc-proxy/nginx.template.conf diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2bdc59c027b4..65bfc5216d1f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -92,6 +92,14 @@ jobs: push: true tags: ethereumoptimism/l2geth:${{ needs.release.outputs.l2geth }} + - name: Publish rpc-proxy + uses: docker/build-push-action@v2 + with: + context: . + file: ./ops/docker/Dockerfile.rpc-proxy + push: true + tags: ethereumoptimism/rpc-proxy:${{ needs.release.outputs.l2geth }} + # pushes the base builder image to dockerhub builder: name: Prepare the base builder image for the services diff --git a/ops/README.md b/ops/README.md index df841dfc61ea..9a791c7ca2d5 100644 --- a/ops/README.md +++ b/ops/README.md @@ -14,6 +14,8 @@ The base `docker-compose.yml` file will start the required components for a full Supplementing the base configuration is an additional metric enabling file, `docker-compose-metrics.yml`. Adding this configuration to the stack will enable metric emission for l2geth and start grafana (for metrics visualisation) and influxdb (for metric collection) instances. +Also available for testing is the `rpc-proxy` service in the `docker-compose-rpc-proxy.yml` file. It can be used to restrict what RPC methods are allowed to the Sequencer. + The base stack can be started and stopped with a command like this (there is no need to specify the default docker-compose.yml) ``` docker-compose \ diff --git a/ops/docker-compose-rpc-proxy.yml b/ops/docker-compose-rpc-proxy.yml new file mode 100644 index 000000000000..dccd31c1cb07 --- /dev/null +++ b/ops/docker-compose-rpc-proxy.yml @@ -0,0 +1,17 @@ +version: "3" +services: + rpc-proxy: + depends_on: + - l1_chain + - deployer + - l2geth + image: rpc-proxy + build: + context: .. + dockerfile: ./ops/docker/Dockerfile.rpc-proxy + environment: + SEQUENCER: l2geth:8545 + ETH_CALLS_ALLOWED: eth_blockNumber,eth_sendRawTransaction + ports: + - 9546:8080 + - 9145:9145 diff --git a/ops/docker-compose.yml b/ops/docker-compose.yml index 940069375bec..0af6cd3b4245 100644 --- a/ops/docker-compose.yml +++ b/ops/docker-compose.yml @@ -143,3 +143,4 @@ services: URL: http://deployer:8081/addresses.json ENABLE_GAS_REPORT: 1 NO_NETWORK: 1 + diff --git a/ops/docker/Dockerfile.rpc-proxy b/ops/docker/Dockerfile.rpc-proxy new file mode 100644 index 000000000000..c0e03b93bd14 --- /dev/null +++ b/ops/docker/Dockerfile.rpc-proxy @@ -0,0 +1,21 @@ +FROM openresty/openresty:buster +LABEL maintainer="Optimistic Systems " +ARG GOTEMPLATE_VERSION=v3.9.0 + +RUN DEBIAN_FRONTEND=noninteractive apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + openresty-opm \ + && opm get knyar/nginx-lua-prometheus + +RUN curl -o /usr/local/bin/gomplate \ + -sSL https://github.com/hairyhenderson/gomplate/releases/download/$GOTEMPLATE_VERSION/gomplate_linux-amd64-slim \ + && chmod +x /usr/local/bin/gomplate + +RUN mkdir -p /var/log/nginx/ \ + && ln -sf /dev/stdout /var/log/nginx/access.log \ + && ln -sf /dev/stderr /var/log/nginx/error.log + +COPY ./ops/docker/rpc-proxy/eth-jsonrpc-access.lua /usr/local/openresty/nginx/eth-jsonrpc-access.lua +COPY ./ops/docker/rpc-proxy/nginx.template.conf /docker-entrypoint.d/nginx.template.conf +COPY ./ops/docker/rpc-proxy/docker-entrypoint.sh /docker-entrypoint.sh +ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/ops/docker/Dockerfile.rpc-proxy-old b/ops/docker/Dockerfile.rpc-proxy-old new file mode 100644 index 000000000000..e0dd12cc818a --- /dev/null +++ b/ops/docker/Dockerfile.rpc-proxy-old @@ -0,0 +1,25 @@ +FROM nginx:stable + +ARG GOTEMPLATE_VERSION=v3.9.0 + +RUN curl -o /usr/local/bin/gomplate \ + -sSL https://github.com/hairyhenderson/gomplate/releases/download/$GOTEMPLATE_VERSION/gomplate_linux-amd64-slim \ + && chmod +x /usr/local/bin/gomplate + +RUN apt-get update \ + && apt-get -y install --no-install-recommends wget gnupg ca-certificates \ + && wget -O - https://openresty.org/package/pubkey.gpg | apt-key add - \ + && codename=$(grep -Po 'VERSION="[0-9]+ \(\K[^)]+' /etc/os-release) \ + && echo "deb http://openresty.org/package/debian $codename openresty" \ + | tee /etc/apt/sources.list.d/openresty.list \ + && apt-get update \ + && apt-get -y install openresty + +# RUN NGINX=$(which nginx) \ +# && mv $NGINX $NGINX-old \ +# && ln -s $(which openresty) $NGINX + +RUN opm get knyar/nginx-lua-prometheus + +COPY ./ops/docker/rpc-proxy/nginx.template.conf /docker-entrypoint.d/nginx.template.conf +COPY ./ops/docker/rpc-proxy/start.sh /docker-entrypoint.d/start.sh diff --git a/ops/docker/rpc-proxy/docker-entrypoint.sh b/ops/docker/rpc-proxy/docker-entrypoint.sh new file mode 100755 index 000000000000..627f7c485e30 --- /dev/null +++ b/ops/docker/rpc-proxy/docker-entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -eo pipefail + +if [ -z "$SEQUENCER" ];then + echo "SEQUENCER env must be set, exiting" + exit 1 +fi + +# if [ -z "$REPLICAS" ];then +# echo "REPLICAS env must be set, exiting" +# exit 1 +# fi + +# REPLICAS=$REPLICAS \ +gomplate -f /docker-entrypoint.d/nginx.template.conf > /usr/local/openresty/nginx/conf/nginx.conf + +cat /usr/local/openresty/nginx/conf/nginx.conf + +exec openresty "$@" diff --git a/ops/docker/rpc-proxy/eth-jsonrpc-access.lua b/ops/docker/rpc-proxy/eth-jsonrpc-access.lua new file mode 100644 index 000000000000..3f2280ec815f --- /dev/null +++ b/ops/docker/rpc-proxy/eth-jsonrpc-access.lua @@ -0,0 +1,91 @@ +-- Source: https://github.com/adetante/ethereum-nginx-proxy +local cjson = require('cjson') + +local function empty(s) + return s == nil or s == '' +end + +local function split(s) + local res = {} + local i = 1 + for v in string.gmatch(s, "([^,]+)") do + res[i] = v + i = i + 1 + end + return res +end + +local function contains(arr, val) + for i, v in ipairs (arr) do + if v == val then + return true + end + end + return false +end + +-- parse conf +local blacklist, whitelist = nil +if not empty(ngx.var.jsonrpc_blacklist) then + blacklist = split(ngx.var.jsonrpc_blacklist) +end +if not empty(ngx.var.jsonrpc_whitelist) then + whitelist = split(ngx.var.jsonrpc_whitelist) +end + +-- check conf +if blacklist ~= nil and whitelist ~= nil then + ngx.log(ngx.ERR, 'invalid conf: jsonrpc_blacklist and jsonrpc_whitelist are both set') + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + return +end + +-- get request content +ngx.req.read_body() + +-- try to parse the body as JSON +local success, body = pcall(cjson.decode, ngx.var.request_body); +if not success then + ngx.log(ngx.ERR, 'invalid JSON request') + ngx.exit(ngx.HTTP_BAD_REQUEST) + return +end + +local method = body['method'] +local version = body['jsonrpc'] + +-- check we have a method and a version +if empty(method) or empty(version) then + ngx.log(ngx.ERR, 'no method and/or jsonrpc attribute') + ngx.exit(ngx.HTTP_BAD_REQUEST) + return +end + +metric_sequencer_requests:inc(1, {method, ngx.var.server_name, ngx.var.status}) + +-- check the version is supported +if version ~= "2.0" then + ngx.log(ngx.ERR, 'jsonrpc version not supported: ' .. version) + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + return +end + +-- if whitelist is configured, check that the method is whitelisted +if whitelist ~= nil then + if not contains(whitelist, method) then + ngx.log(ngx.ERR, 'jsonrpc method is not whitelisted: ' .. method) + ngx.exit(ngx.HTTP_FORBIDDEN) + return + end +end + +-- if blacklist is configured, check that the method is not blacklisted +if blacklist ~= nil then + if contains(blacklist, method) then + ngx.log(ngx.ERR, 'jsonrpc method is blacklisted: ' .. method) + ngx.exit(ngx.HTTP_FORBIDDEN) + return + end +end + +return diff --git a/ops/docker/rpc-proxy/nginx.template.conf b/ops/docker/rpc-proxy/nginx.template.conf new file mode 100644 index 000000000000..4530b5ddb849 --- /dev/null +++ b/ops/docker/rpc-proxy/nginx.template.conf @@ -0,0 +1,80 @@ +worker_processes 5; +daemon off; +error_log /var/log/nginx/error.log; +worker_rlimit_nofile 8192; +pcre_jit on; + +events { + worker_connections 4096; +} + +http { + include mime.types; + index index.html; + + # See Move default writable paths to a dedicated directory (#119) + # https://github.com/openresty/docker-openresty/issues/119 + client_body_temp_path /var/run/openresty/nginx-client-body; + proxy_temp_path /var/run/openresty/nginx-proxy; + fastcgi_temp_path /var/run/openresty/nginx-fastcgi; + uwsgi_temp_path /var/run/openresty/nginx-uwsgi; + scgi_temp_path /var/run/openresty/nginx-scgi; + + keepalive_timeout 0; + + default_type application/octet-stream; + log_format main '$remote_addr - $remote_user [$time_local] $status ' + '"$request" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + sendfile on; + tcp_nopush on; + + lua_shared_dict prometheus_metrics 10M; + init_worker_by_lua_block { + prometheus = require("prometheus").init("prometheus_metrics") + metric_requests = prometheus:counter( + "nginx_http_requests_total", "Number of HTTP requests", {"host", "status"}) + metric_sequencer_requests = prometheus:counter( + "nginx_eth_sequencer_requests", "Number of requests going to the sequencer", {"method", "host", "status"}) + metric_replica_requests = prometheus:counter( + "nginx_eth_replica_requests", "Number of requests going to the replicas", {"host", "status"}) + metric_latency = prometheus:histogram( + "nginx_http_request_duration_seconds", "HTTP request latency", {"host"}) + metric_connections = prometheus:gauge( + "nginx_http_connections", "Number of HTTP connections", {"state"}) + } + log_by_lua_block { + metric_requests:inc(1, {ngx.var.server_name, ngx.var.status}) + metric_latency:observe(tonumber(ngx.var.request_time), {ngx.var.server_name}) + } + + upstream sequencer { + server {{env.Getenv "SEQUENCER"}}; + } + + server { # RPC proxy server + listen 8080; + location = /healthz { + return 200 'healthz'; + } + location / { + set $jsonrpc_whitelist {{env.Getenv "ETH_CALLS_ALLOWED"}}; + access_by_lua_file 'eth-jsonrpc-access.lua'; + proxy_pass http://sequencer; + } + } + + server { # Metrics server + listen 9145; + location /metrics { + content_by_lua_block { + metric_connections:set(ngx.var.connections_reading, {"reading"}) + metric_connections:set(ngx.var.connections_waiting, {"waiting"}) + metric_connections:set(ngx.var.connections_writing, {"writing"}) + prometheus:collect() + } + } + } + +} \ No newline at end of file