feat: agent pooling

src/agents.ts

@@ -0,0 +1,62 @@
+/*!
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Agent as HTTPAgent} from 'http';
+import {Agent as HTTPSAgent} from 'https';
+import {Options} from './';
+
+const pool = new Map<string, HTTPAgent>();
+
+/**
+ * Returns a custom request Agent if one is found, otherwise returns undefined
+ * which will result in the global http(s) Agent being used.
+ * @private
+ * @param {string} uri The request uri
+ * @param {object} reqOpts The request options
+ * @returns {Agent|undefined}
+ */
+export function getAgent(uri: string, reqOpts: Options): HTTPAgent | undefined {
+  const isHttp = uri.startsWith('http://');
+  const proxy =
+    reqOpts.proxy ||
+    process.env.HTTP_PROXY ||
+    process.env.http_proxy ||
+    process.env.HTTPS_PROXY ||
+    process.env.https_proxy;
+
+  if (proxy) {
+    // tslint:disable-next-line variable-name
+    const Agent = isHttp
+      ? require('http-proxy-agent')
+      : require('https-proxy-agent');
+
+    return new Agent(proxy) as HTTPAgent;
+  }
+
+  let key = isHttp ? 'http' : 'https';
+
+  if (reqOpts.forever) {
+    key += ':forever';
+
+    if (!pool.has(key)) {
+      // tslint:disable-next-line variable-name
+      const Agent = isHttp ? HTTPAgent : HTTPSAgent;
+      pool.set(key, new Agent({keepAlive: true}));
+    }
+  }
+
+  return pool.get(key);
+}

src/index.ts

@@ -14,12 +14,11 @@
  * limitations under the License.
  */
 
-import {Agent as HTTPAgent} from 'http';
 import {Agent} from 'https';
-
 import fetch, * as f from 'node-fetch';
 import {PassThrough, Readable} from 'stream';
 import * as uuid from 'uuid';
+import {getAgent} from './agents';
 const streamEvents = require('stream-events');
 
 export interface CoreOptions {
@@ -119,28 +118,7 @@ function requestToFetchOptions(reqOpts: Options) {
     uri = uri + '?' + params;
   }
 
-  const isHttp = uri.startsWith('http://');
-  const proxy =
-    reqOpts.proxy ||
-    process.env.HTTP_PROXY ||
-    process.env.http_proxy ||
-    process.env.HTTPS_PROXY ||
-    process.env.https_proxy;
-  if (proxy) {
-    if (isHttp) {
-      // tslint:disable-next-line variable-name
-      const HttpProxyAgent = require('http-proxy-agent');
-      options.agent = new HttpProxyAgent(proxy);
-    } else {
-      // tslint:disable-next-line variable-name
-      const HttpsProxyAgent = require('https-proxy-agent');
-      options.agent = new HttpsProxyAgent(proxy);
-    }
-  } else if (reqOpts.forever) {
-    options.agent = isHttp
-      ? new HTTPAgent({keepAlive: true})
-      : new Agent({keepAlive: true});
-  }
+  options.agent = getAgent(uri, reqOpts);
 
   return {uri, options};
 }

test/agents.ts

@@ -0,0 +1,126 @@
+/*!
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as assert from 'assert';
+import * as http from 'http';
+import * as https from 'https';
+import * as sinon from 'sinon';
+import {getAgent} from '../src/agents';
+
+// tslint:disable-next-line variable-name
+const HttpProxyAgent = require('http-proxy-agent');
+// tslint:disable-next-line variable-name
+const HttpsProxyAgent = require('https-proxy-agent');
+
+describe('agents', () => {
+  const httpUri = 'http://example.com';
+  const httpsUri = 'https://example.com';
+  const sandbox = sinon.createSandbox();
+
+  afterEach(() => sandbox.restore());
+
+  describe('getAgent', () => {
+    const defaultOptions = {uri: httpUri};
+
+    it('should return undefined by default', () => {
+      const agent = getAgent(httpUri, defaultOptions);
+      assert.strictEqual(agent, undefined);
+    });
+
+    describe('proxy', () => {
+      const envVars = [
+        'http_proxy',
+        'https_proxy',
+        'HTTP_PROXY',
+        'HTTPS_PROXY',
+      ];
+
+      describe('http', () => {
+        const uri = httpUri;
+        const proxy = 'http://hello.there:8080';
+
+        it('should respect the proxy option', () => {
+          const options = Object.assign({proxy}, defaultOptions);
+          const agent = getAgent(uri, options);
+          assert(agent instanceof HttpProxyAgent);
+        });
+
+        envVars.forEach(envVar => {
+          it(`should respect the ${envVar} env var`, () => {
+            process.env[envVar] = proxy;
+            const agent = getAgent(uri, defaultOptions);
+            assert(agent instanceof HttpProxyAgent);
+            delete process.env[envVar];
+          });
+        });
+      });
+
+      describe('https', () => {
+        const uri = httpsUri;
+        const proxy = 'https://hello.there:8080';
+
+        it('should respect the proxy option', () => {
+          const options = Object.assign({proxy}, defaultOptions);
+          const agent = getAgent(uri, options);
+          assert(agent instanceof HttpsProxyAgent);
+        });
+
+        envVars.forEach(envVar => {
+          it(`should respect the ${envVar} env var`, () => {
+            process.env[envVar] = proxy;
+            const agent = getAgent(uri, defaultOptions);
+            assert(agent instanceof HttpsProxyAgent);
+            delete process.env[envVar];
+          });
+        });
+      });
+    });
+
+    describe('forever', () => {
+      describe('http', () => {
+        const uri = httpUri;
+        const options = Object.assign({forever: true}, defaultOptions);
+
+        it('should return an http Agent', () => {
+          const agent = getAgent(uri, options)!;
+          assert(agent instanceof http.Agent);
+        });
+
+        it('should cache the agent', () => {
+          const agent1 = getAgent(uri, options);
+          const agent2 = getAgent(uri, options);
+          assert.strictEqual(agent1, agent2);
+        });
+      });
+
+      describe('https', () => {
+        const uri = httpUri;
+        const options = Object.assign({forever: true}, defaultOptions);
+
+        it('should return an http Agent', () => {
+          const agent = getAgent(uri, options)!;
+          assert(agent instanceof http.Agent);
+        });
+
+        it('should cache the agent', () => {
+          const agent1 = getAgent(uri, options);
+          const agent2 = getAgent(uri, options);
+          assert.strictEqual(agent1, agent2);
+        });
+      });
+    });
+  });
+});