option to drop post_action

.luacheckrc

@@ -3,4 +3,4 @@ std = 'ngx_lua+lua52' -- lua52 has table.pack
 busted = {std = "+busted"}
 files["**/spec/**/*_spec.lua"] = busted
 
-globals = { 'ngx', 'unpack', 'rawlen' }
+globals = { 'rawlen' }

CHANGELOG.md

@@ -6,15 +6,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
-### Added
+## Added
 
+- Experimental option for true out of band reporting (`APICAST_REPORTING_WORKERS`) [PR #290](https://github.com/3scale/apicast/pull/290)
+- `/status/info` endpoint to the Management API [PR #290](https://github.com/3scale/apicast/pull/290)
 - `/_threescale/healthz` endpoint returns a success status code, this is used for health checking in kubernetes environments [PR #285](https://github.com/3scale/apicast/pull/285)
 
 ## Changed
 
 - Upgraded to OpenResty 1.11.2.5-1 [PR #428](https://github.com/3scale/apicast/pull/428)
 - `/oauth/token` endpoint returns an error status code, when the access token couldn't be stored in 3scale backend [PR #436](https://github.com/3scale/apicast/pull/436)]
 - URI params in POST requests are now taken into account when matching mapping rules [PR #437](https://github.com/3scale/apicast/pull/437)
+- Increased number of background timers and connections in the cosocket pool [PR #290](https://github.com/3scale/apicast/pull/290)
 - Make OAuth tokens TTL configurable [PR #448](https://github.com/3scale/apicast/pull/448)
 
 ### Fixed

apicast/conf.d/apicast.conf

@@ -16,6 +16,7 @@ location = /threescale_authrep {
     local log = ngx.var.arg_log;
     if log then return '&' .. ngx.unescape_uri(log) end
   }
+  set_by_lua_block $backend_endpoint { return ngx.ctx.backend_endpoint }
   set $path /transactions/authrep.xml?$backend_authentication_type=$backend_authentication_value&service_id=$service_id&$usage&$credentials$log;
   proxy_pass_request_headers off;
   proxy_http_version 1.1;
@@ -31,23 +32,25 @@ location = /threescale_authrep {
   }
 }
 
-
 location @out_of_band_authrep_action {
   internal;
 
   proxy_pass_request_headers off;
 
   set_by_lua $original_request_time 'return ngx.var.request_time';
+  set_by_lua $backend_endpoint 'return ngx.var.post_action_backend_endpoint';
 
   content_by_lua_block { require('module'):post_action() }
 
   log_by_lua_block {
     ngx.var.post_action_impact = ngx.var.request_time - ngx.var.original_request_time
-    ngx.log(ngx.INFO, '[authrep] ', ngx.var.request_uri, ' ', ngx.var.status)
     require('module'):log()
   }
 }
 
+set $backend_endpoint 'http://127.0.0.1:8081';
+
+
 location / {
   set $cached_key null;
   set $credentials null;
@@ -62,7 +65,7 @@ location / {
   set $redirect_url null;
 
   set $backend_host 'backend';
-  set $backend_endpoint 'http://127.0.0.1:8081';
+  set $post_action_backend_endpoint '';
   set $backend_authentication_type null;
   set $backend_authentication_value null;
   set $version null;

apicast/conf/nginx.conf

@@ -7,6 +7,8 @@ env OPENSSL_VERIFY;
 
 include ../main.d/*.conf;
 
+env APICAST_REPORTING_THREADS;
+
 error_log /dev/null emerg;
 
 events {
@@ -29,6 +31,8 @@ http {
   # Enabling the Lua code cache is strongly encouraged for production use
   # Disabling it should only be done for testing and development purposes
   lua_code_cache on;
+  lua_max_running_timers 2048;
+  lua_socket_pool_size 512;
 
   include ../http.d/*.conf;
 

apicast/src/backend/cache_handler.lua

@@ -1,4 +1,5 @@
 local setmetatable = setmetatable
+local pcall = pcall
 
 local _M = {
   handlers = setmetatable({}, { __index = { default = 'strict' } })
@@ -23,12 +24,23 @@ function _M.new(handler)
   return setmetatable({ handler = name }, mt)
 end
 
+local function cached_key_var()
+  return ngx.var.cached_key
+end
+
+local function fetch_cached_key()
+  local ok, stored = pcall(cached_key_var)
+
+  return ok and stored
+end
+
 function _M.handlers.strict(cache, cached_key, response, ttl)
   if response.status == 200 then
     -- cached_key is set in post_action and it is in in authorize
     -- so to not write the cache twice lets write it just in authorize
-    if ngx.var.cached_key ~= cached_key then
-      ngx.log(ngx.INFO, 'apicast cache write key: ', cached_key, ', ttl: ', ttl )
+
+    if fetch_cached_key(cached_key) ~= cached_key then
+      ngx.log(ngx.INFO, 'apicast cache write key: ', cached_key, ', ttl: ', ttl, ' sub: ')
       cache:set(cached_key, 200, ttl or 0)
     end
 

apicast/src/backend_client.lua

@@ -11,6 +11,8 @@
 
 local setmetatable = setmetatable
 local concat = table.concat
+local insert = table.insert
+local len = string.len
 
 local http_ng = require('resty.http_ng')
 local user_agent = require('user_agent')
@@ -84,11 +86,15 @@ local function call_backend_transaction(self, path, ...)
 
   local args = { self.authentication, ... }
 
+  local query = {}
   for i=1, #args do
-    args[i] = ngx.encode_args(args[i])
+    local arg = ngx.encode_args(args[i])
+    if len(arg) > 0 then
+      insert(query, arg)
+    end
   end
 
-  local url = resty_url.join(endpoint, '/transactions', path .. '?' .. concat(args, '&'))
+  local url = resty_url.join(endpoint, '/transactions', path .. '?' .. concat(query, '&'))
 
   local res = http_client.get(url)
 

apicast/src/management.lua

@@ -110,13 +110,28 @@ function _M.disabled()
   ngx.exit(ngx.HTTP_FORBIDDEN)
 end
 
+function _M.info()
+  return json_response({
+    timers = {
+      pending = ngx.timer.pending_count(),
+      running = ngx.timer.running_count()
+    },
+    worker = {
+      exiting = ngx.worker.exiting(),
+      count = ngx.worker.count(),
+      id = ngx.worker.id()
+    }
+  })
+end
+
 local routes = {}
 
 function routes.disabled(r)
   r:get('/', _M.disabled)
 end
 
 function routes.status(r)
+  r:get('/status/info', _M.info)
   r:get('/status/ready', _M.ready)
   r:get('/status/live', _M.live)
 end

apicast/src/proxy.lua

@@ -9,8 +9,8 @@
 local env = require 'resty.env'
 local custom_config = env.get('APICAST_CUSTOM_CONFIG')
 local configuration_store = require 'configuration_store'
+local util = require('util')
 local resty_lrucache = require('resty.lrucache')
-
 local backend_cache_handler = require('backend.cache_handler')
 
 local resty_url = require 'resty.url'
@@ -23,13 +23,22 @@ local concat = table.concat
 local gsub = string.gsub
 local tonumber = tonumber
 local setmetatable = setmetatable
-local exit = ngx.exit
 local encode_args = ngx.encode_args
 local resty_resolver = require 'resty.resolver'
+local semaphore = require('ngx.semaphore')
+local backend_client = require('backend_client')
+local timers = semaphore.new(tonumber(env.get('APICAST_REPORTING_THREADS') or 0))
+
 local empty = {}
 
 local response_codes = env.enabled('APICAST_RESPONSE_CODES')
 
+local using_post_action = response_codes or timers:count() < 1
+
+if not using_post_action then
+  ngx.log(ngx.WARN, 'using experimental asynchronous reporting threads: ', timers:count())
+end
+
 local _M = { }
 
 local mt = {
@@ -158,7 +167,7 @@ local http = {
     local backend_upstream = ngx.ctx.backend_upstream
     local previous_real_url = ngx.var.real_url
     ngx.log(ngx.DEBUG, '[ctx] copying backend_upstream of size: ', #backend_upstream)
-    local res = ngx.location.capture(assert(url), { share_all_vars = true, ctx = { backend_upstream = backend_upstream } })
+    local res = ngx.location.capture(assert(url), { share_all_vars = true, ctx = { backend_upstream = backend_upstream, backend_endpoint = ngx.var.backend_endpoint } })
 
     local real_url = ngx.var.real_url
 
@@ -217,6 +226,10 @@ function _M:authorize(service, usage, credentials, ttl)
       error_authorization_failed(service)
     end
   end
+
+  if not using_post_action then
+    self:post_action(true)
+  end
 end
 
 function _M:set_service(host)
@@ -267,8 +280,8 @@ function _M:set_backend_upstream(service)
   ngx.var.backend_authentication_value = backend_authentication.value
   ngx.var.version = self.configuration.version
 
-  -- set backend
-  local url = resty_url.split(backend.endpoint or ngx.var.backend_endpoint)
+  local backend_endpoint = backend.endpoint  or ngx.var.backend_endpoint
+  local url = resty_url.split(backend_endpoint)
   local scheme, _, _, server, port, path =
     url[1], url[2], url[3], url[4], url[5] or resty_url.default_port(url[1]), url[6] or ''
 
@@ -278,6 +291,7 @@ function _M:set_backend_upstream(service)
 
   ngx.var.backend_endpoint = scheme .. '://backend_upstream' .. path
   ngx.var.backend_host = backend.host or server or ngx.var.backend_host
+  ngx.var.post_action_backend_endpoint = backend_endpoint
 end
 
 -----
@@ -368,22 +382,62 @@ function _M:access(service)
   return self:authorize(service, usage, credentials, ttl)
 end
 
-
-local function response_codes_encoded_data()
+local function response_codes_data()
   local params = {}
 
   if not response_codes then
-    return ''
+    return params
   end
 
   if response_codes then
     params["log[code]"] = ngx.var.status
   end
 
-  return ngx.escape_uri(ngx.encode_args(params))
+  return params
+end
+
+local function response_codes_encoded_data()
+  return ngx.escape_uri(ngx.encode_args(response_codes_data()))
+end
+
+local function post_action(_, self, cached_key, backend, ...)
+  local res = util.timer('backend post_action', backend.authrep, backend, ...)
+
+  if not using_post_action then
+    timers:post(1)
+  end
+
+  self:handle_backend_response(cached_key, res)
 end
 
-function _M:post_action()
+local function capture_post_action(self, cached_key, service)
+  self:set_backend_upstream(service)
+
+  local auth_uri = service.backend_version == 'oauth' and 'threescale_oauth_authrep' or 'threescale_authrep'
+  local res = http.get("/".. auth_uri .."?log=" .. response_codes_encoded_data())
+
+  self:handle_backend_response(cached_key, res)
+end
+
+local function timer_post_action(self, cached_key, service)
+  local backend = assert(backend_client:new(service), 'missing backend')
+
+  local ok, err = timers:wait(10)
+
+  if ok then
+    -- TODO: try to do this in different phase and use semaphore to limit number of background threads
+    -- TODO: Also it is possible to use sets in shared memory to enqueue work
+    ngx.timer.at(0, post_action, self, cached_key, backend, ngx.ctx.usage, ngx.ctx.credentials, response_codes_data())
+  else
+    ngx.log(ngx.ERR, 'failed to acquire timer: ', err)
+    return capture_post_action(self, cached_key, service)
+  end
+end
+
+function _M:post_action(force)
+  if not using_post_action and not force then
+    return nil, 'post action not needed'
+  end
 
   local cached_key = ngx.var.cached_key
 
@@ -392,17 +446,15 @@ function _M:post_action()
 
     local service_id = ngx.var.service_id
     local service = ngx.ctx.service or self.configuration:find_by_id(service_id)
-    self:set_backend_upstream(service)
 
-    local auth_uri = service.backend_version == 'oauth' and 'threescale_oauth_authrep' or 'threescale_authrep'
-    local res = http.get("/".. auth_uri .."?log=" .. response_codes_encoded_data())
-
-    self:handle_backend_response(cached_key, res)
+    if using_post_action then
+      capture_post_action(self, cached_key, service)
+    else
+      timer_post_action(self, cached_key, service)
+    end
   else
     ngx.log(ngx.INFO, '[async] skipping after action, no cached key')
   end
-
-  exit(ngx.HTTP_OK)
 end
 
 function _M:handle_backend_response(cached_key, response, ttl)

doc/management-api.md

@@ -82,3 +82,13 @@ Available endpoints:
   ```json
   { "status": "live", "success": true }
   ```
+
+- `GET /status/info`
+  Returns internal information about timers and workers.
+
+  ```json
+  {
+    "timers": { "pending": 0, "running": 0 },
+    "worker": { "count":1, "exiting": false, "id": 0 }
+  }
+  ```

doc/parameters.md

@@ -171,3 +171,14 @@ It requires custom certificate bundle and adding it to trusted certificates.
 
 It is recommended to use https://github.com/openresty/lua-nginx-module#lua_ssl_trusted_certificate and point to to
 certificate bundle generated by [export-builtin-trusted-certs](https://github.com/openresty/openresty-devel-utils/blob/master/export-builtin-trusted-certs).
+
+### `APICAST_REPORTING_THREADS`
+
+**Default**: 0
+**Value:**: integer >= 0
+
+Value greater than 0 is going to enable out-of-band reporting to backend.
+This is new **experimental** feature for increasing performance. Client
+Won't see the backend latency and everything will be processed asynchronously.
+This value determines how many asynchronous reports can be running simultainesly
+before client starts being throttled by adding latency.

openshift/apicast-template.yml

@@ -58,6 +58,8 @@ objects:
             value: "${MANAGEMENT_API}"
           - name: OPENSSL_VERIFY
             value: "${OPENSSL_VERIFY}"
+          - name: APICAST_REPORTING_THREADS
+            value: "${REPORTING_THREADS}"
           image: "${IMAGE_NAME}"
           imagePullPolicy: Always
           name: "${APICAST_NAME}"
@@ -166,3 +168,7 @@ parameters:
   description: "Turn on/off the OpenSSL peer verification. Can be set to true/false."
   required: true
   value: "false"
+- name: REPORTING_THREADS
+  description: "Number of asynchronous reporting threads. Experimental feature."
+  required: false
+  value: "0"