Merge pull request #196 from dadi/feature/performance

🚀 Performance enhancements

Author: David Longworth

config.js

@@ -234,8 +234,13 @@ var conf = convict({
   },
   headers: {
     useGzipCompression: {
+      doc: 'Depricated: use `useCompression` instead.',
+      format: Boolean,
+      default: true
+    },
+    useCompression: {
       doc:
-        "If true, uses gzip compression and adds a 'Content-Encoding:gzip' header to the response.",
+        "If true, uses br or gzip compression where available and adds a 'Content-Encoding: [br|gzip]' header to the response.",
       format: Boolean,
       default: true
     },
@@ -244,9 +249,12 @@ var conf = convict({
         "A set of custom cache control headers for different content types. For example 'cacheControl': { 'text/css': 'public, max-age=1000' }",
       format: Object,
       default: {
+        'image/png': 'public, max-age=86400',
+        'image/jpeg': 'public, max-age=86400',
         'text/css': 'public, max-age=86400',
         'text/javascript': 'public, max-age=86400',
-        'application/javascript': 'public, max-age=86400'
+        'application/javascript': 'public, max-age=86400',
+        'image/x-icon': 'public, max-age=31536000000'
       }
     }
   },
@@ -288,6 +296,14 @@ var conf = convict({
         format: String,
         default: ''
       }
+    },
+    sentry: {
+      dsn: {
+        doc:
+          "The 'DSN' to use for logging errors and events to a Sentry server. It should be similar to 'https://693ef18da3184cffa82144fde2979cbc:[email protected]/59524'.",
+        format: String,
+        default: ''
+      }
     }
   },
   globalEvents: {
@@ -306,7 +322,6 @@ var conf = convict({
     default: {
       datasources: path.join(__dirname, '/workspace/datasources'),
       events: path.join(__dirname, '/workspace/events'),
-      media: path.join(__dirname, '/workspace/media'),
       middleware: path.join(__dirname, '/workspace/middleware'),
       pages: path.join(__dirname, '/workspace/pages'),
       public: path.join(__dirname, '/workspace/public'),

dadi/lib/api/index.js

@@ -30,9 +30,7 @@ var Api = function () {
   this.protocol = config.get('server.protocol') || 'http'
   this.redirectPort = config.get('server.redirectPort')
 
-  if (this.protocol === 'http') {
-    this.httpInstance = http.createServer(this.listener)
-  } else if (this.protocol === 'https') {
+  if (this.protocol === 'https') {
     // Redirect http to https
     if (this.redirectPort > 0) {
       this.redirectInstance = http.createServer(this.redirectListener)
@@ -84,6 +82,8 @@ var Api = function () {
           throw new Error(exPrefix + ex.message)
       }
     }
+  } else {
+    this.httpInstance = http.createServer(this.listener)
   }
 }
 
@@ -219,7 +219,7 @@ Api.prototype.listener = function (req, res) {
         // if end of the stack, no middleware could handle the current
         // request, so get matching routes from the loaded page components and
         // add them to the stack just before the 404 handler, then continue the loop
-        // console.log(pathsLoaded, this.stack[stackIdx].name, this.stack[stackIdx+1] ? this.stack[stackIdx+1].name : 'nothing left!')
+
         if (
           this.stack[stackIdx + 1] &&
           this.stack[stackIdx + 1].name === 'notFound' &&
@@ -242,7 +242,6 @@ Api.prototype.listener = function (req, res) {
             _.each(matches, match => {
               this.stack.splice(-1, 0, match)
             })
-          } else {
           }
 
           pathsLoaded = true
@@ -281,7 +280,7 @@ Api.prototype.redirectListener = function (req, res) {
   var location = 'https://' + hostname + ':' + port + req.url
 
   res.setHeader('Location', location)
-  res.statusCode = 301
+  res.statusCode = 302
   res.end()
 }
 

dadi/lib/auth/bearer.js

@@ -28,7 +28,8 @@ BearerAuthStrategy.prototype.getToken = function (authStrategy, done) {
     credentials: strategy.credentials,
     wallet: 'file',
     walletOptions: {
-      path: config.get('paths.tokenWallets') +
+      path:
+        config.get('paths.tokenWallets') +
         '/' +
         help.generateTokenWalletFilename(
           strategy.host,

dadi/lib/cache/datasource.js

@@ -21,9 +21,6 @@ var DatasourceCache = function () {
   this.cacheOptions = config.get('caching')
   this.cache = new DadiCache(this.cacheOptions)
 
-  DatasourceCache.numInstances = (DatasourceCache.numInstances || 0) + 1
-  // console.log('DatasourceCache:', DatasourceCache.numInstances)
-
   var directoryEnabled = this.cacheOptions.directory.enabled
   var redisEnabled = this.cacheOptions.redis.enabled
 

dadi/lib/cache/index.js

@@ -6,8 +6,14 @@ var crypto = require('crypto')
 var debug = require('debug')('web:cache')
 var path = require('path')
 var url = require('url')
+var fs = require('fs')
+
+var compressible = require('compressible')
+var mime = require('mime-types')
+var etag = require('etag')
 
 var config = require(path.join(__dirname, '/../../../config.js'))
+var help = require(path.join(__dirname, '/../help'))
 
 var DadiCache = require('@dadi/cache')
 
@@ -20,9 +26,6 @@ var Cache = function (server) {
   this.server = server
   this.cache = new DadiCache(config.get('caching'))
 
-  Cache.numInstances = (Cache.numInstances || 0) + 1
-  // console.log('Cache:', Cache.numInstances)
-
   var directoryEnabled = config.get('caching.directory.enabled')
   var redisEnabled = config.get('caching.redis.enabled')
 
@@ -46,31 +49,29 @@ module.exports = function (server) {
  * @returns {Boolean}
  */
 Cache.prototype.cachingEnabled = function (req) {
+  // Check it is not a json view
   var query = url.parse(req.url, true).query
-  if (query.json && query.json !== 'false') {
-    return false
-  }
+  if (query.json && query.json !== 'false') return false
 
-  if (config.get('debug')) {
-    return false
-  }
+  // Disable cache for debug mode
+  if (config.get('debug')) return false
 
-  var endpoint = this.getEndpointMatchingRequest(req)
+  // if it's in the endpoint and caching is enabled
+  var endpoint = this.getEndpoint(req)
 
-  if (!endpoint) {
-    endpoint = this.getEndpointMatchingLoadedPaths(req)
-  }
+  if (endpoint) {
+    this.options.cache =
+      typeof endpoint.page.settings.cache !== 'undefined'
+        ? endpoint.page.settings.cache
+        : this.enabled
 
-  // not found in the loaded routes, let's not bother caching
-  if (!endpoint) return false
-
-  if (endpoint.page && endpoint.page.settings) {
-    this.options = endpoint.page.settings
+    return this.enabled && this.options.cache
   } else {
-    this.options.cache = false
-  }
+    // Otherwise it might be in the public folder
+    var file = url.parse(req.url).pathname
 
-  return this.enabled && (this.options.cache || false)
+    return compressible(mime.lookup(file))
+  }
 }
 
 /**
@@ -80,12 +81,7 @@ Cache.prototype.cachingEnabled = function (req) {
  */
 Cache.prototype.getEndpointMatchingRequest = function (req) {
   var endpoints = this.server.components
-  var requestUrl = url.parse(req.url, true).pathname
-
-  // strip trailing slash before testing
-  if (requestUrl !== '/' && requestUrl[requestUrl.length - 1] === '/') {
-    requestUrl = requestUrl.substring(0, requestUrl.length - 1)
-  }
+  var requestUrl = url.parse(req.url, true).pathname.replace(/\/+$/, '')
 
   // get the host key that matches the request's host header
   var virtualHosts = config.get('virtualHosts')
@@ -96,7 +92,7 @@ Cache.prototype.getEndpointMatchingRequest = function (req) {
     }) || ''
 
   // check if there is a match in the loaded routes for the current request URL
-  var endpoint = _.find(endpoints, endpoint => {
+  return _.find(endpoints, endpoint => {
     var paths = _.pluck(endpoint.page.routes, 'path')
     return (
       _.contains(paths, requestUrl) &&
@@ -105,9 +101,8 @@ Cache.prototype.getEndpointMatchingRequest = function (req) {
         : true)
     )
   })
-
-  return endpoint
 }
+
 /**
  * Retrieves the page component that best matches the paths loaded in api/index.js
  * @param {IncomingMessage} req - the current HTTP request
@@ -130,14 +125,23 @@ Cache.prototype.getEndpointMatchingLoadedPaths = function (req) {
  * @param {IncomingMessage} req - the current HTTP request
  * @returns {string}
  */
-Cache.prototype.getEndpointContentType = function (req) {
-  var endpoint = this.getEndpointMatchingRequest(req)
+Cache.prototype.getReqContentType = function (req) {
+  // Check for content-type in the page json
+  var endpoint = this.getEndpoint(req)
 
-  if (!endpoint) {
-    endpoint = this.getEndpointMatchingLoadedPaths(req)
-  }
+  return endpoint && endpoint.page && endpoint.page.contentType
+    ? endpoint.page.contentType
+    : false
+}
+
+/**
+ * Adds the Cache middleware to the stack
+ */
+Cache.prototype.getEndpoint = function (req) {
+  var endpoint = this.getEndpointMatchingRequest(req)
+  if (!endpoint) endpoint = this.getEndpointMatchingLoadedPaths(req)
 
-  return endpoint.page.contentType
+  return endpoint
 }
 
 /**
@@ -154,9 +158,15 @@ Cache.prototype.init = function () {
   this.server.app.use(function cache (req, res, next) {
     var enabled = self.cachingEnabled(req)
 
+    if (!enabled) return next()
+
     debug('%s%s, cache enabled: %s', req.headers.host, req.url, enabled)
 
-    if (!enabled) return next()
+    // Check it's a page
+    if (!self.getEndpoint(req)) return next()
+
+    // get contentType that current endpoint requires
+    var contentType = self.getReqContentType(req)
 
     // only cache GET requests
     if (req.method && req.method.toLowerCase() !== 'get') return next()
@@ -183,33 +193,63 @@ Cache.prototype.init = function () {
     var noCache =
       query.cache && query.cache.toString().toLowerCase() === 'false'
 
-    // get contentType that current endpoint requires
-    var contentType = self.getEndpointContentType(req)
+    // File extension for cache file
+    var cacheExt =
+      compressible(contentType) && help.canCompress(req.headers)
+        ? '.' + help.canCompress(req.headers)
+        : null
+
+    var opts = {
+      directory: { extension: mime.extension(contentType) + cacheExt }
+    }
+
+    // Compression settings
+    var shouldCompress = compressible(contentType)
+      ? help.canCompress(req.headers)
+      : false
 
     // attempt to get from the cache
     self.cache
-      .get(filename)
+      .get(filename, opts)
       .then(stream => {
         debug('serving %s%s from cache', req.headers.host, req.url)
 
-        res.setHeader('X-Cache-Lookup', 'HIT')
-
         if (noCache) {
+          res.setHeader('X-Cache-Lookup', 'HIT')
           res.setHeader('X-Cache', 'MISS')
           return next()
         }
 
+        var headers = {
+          'X-Cache-Lookup': 'HIT',
+          'X-Cache': 'HIT',
+          'Content-Type': contentType,
+          'Cache-Control':
+            config.get('headers.cacheControl')[contentType] ||
+            'public, max-age=86400'
+        }
+
+        // Add compression headers
+        if (shouldCompress) headers['Content-Encoding'] = shouldCompress
+
+        // Add extra headers
+        stream.on('open', fd => {
+          fs.fstat(fd, (_, stats) => {
+            res.setHeader('Content-Length', stats.size)
+            res.setHeader('ETag', etag(stats))
+          })
+        })
+
         res.statusCode = 200
-        res.setHeader('X-Cache', 'HIT')
-        res.setHeader('Content-Type', contentType)
+        Object.keys(headers).map(i => res.setHeader(i, headers[i]))
 
-        // send cached content back
         stream.pipe(res)
       })
       .catch(() => {
         // not found in cache
         res.setHeader('X-Cache', 'MISS')
         res.setHeader('X-Cache-Lookup', 'MISS')
+
         return cacheResponse()
       })
 
@@ -222,24 +262,31 @@ Cache.prototype.init = function () {
       var _end = res.end
       var _write = res.write
 
-      var data = ''
+      var data = []
 
       res.write = function (chunk) {
+        if (chunk) data.push(chunk)
+
         _write.apply(res, arguments)
       }
 
       res.end = function (chunk) {
         // respond before attempting to cache
         _end.apply(res, arguments)
 
-        if (chunk) data += chunk
+        if (chunk && !data.length) data.push(chunk)
 
         // if response is not 200 don't cache
         if (res.statusCode !== 200) return
 
-        // cache the content
-        self.cache.set(filename, data).then(() => {})
+        // cache the content, with applicable file extension
+        try {
+          self.cache.set(filename, Buffer.concat(data), opts).then(() => {})
+        } catch (e) {
+          console.log('Could not cache content: ' + requestUrl)
+        }
       }
+
       return next()
     }
   })