Skip to content

Commit

Permalink
🚢 v1.4.3
Browse files Browse the repository at this point in the history 
- 👨‍💻 Minor overall codebase enhancements in stability and durability
- 👨‍💻 Better debugging logs and hints
- 📋 Minor documentation update
- 📦 Update `request-libcurl` to `v2.0.0` (*was `1.0.3`*)
- 📦 Update dev-dependency (*test-suite*) `mocha` to `v6.2.1` (*was
`v6.1.4`*)
  • Loading branch information
@dr-dimitru
dr-dimitru committed Oct 11, 2019
1 parent 9fdffa3 commit b3251e3
Show file tree
Hide file tree
Showing 5 changed files with 282 additions and 450 deletions.
4 changes: 2 additions & 2 deletions .versions
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ [email protected]
[email protected]
[email protected]
[email protected]
local-test:ostrio:[email protected].2
local-test:ostrio:[email protected].3
[email protected]
[email protected]
[email protected]
Expand All @@ -35,7 +35,7 @@ [email protected]
[email protected]
[email protected]
[email protected]
ostrio:[email protected].2
ostrio:[email protected].3
[email protected]
[email protected]
[email protected]
Expand Down
65 changes: 37 additions & 28 deletions lib/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ var re = {
staticExt: /\.(?:3ds|3g2|3gp|3gpp|7z|a|aac|aaf|adp|ai|aif|aiff|alz|ape|apk|appcache|ar|arj|asf|asx|atom|au|avchd|avi|bak|bbaw|bh|bin|bk|bmp|btif|bz2|bzip2|cab|caf|cco|cgm|class|cmx|cpio|cr2|crt|crx|css|csv|cur|dat|deb|der|dex|djvu|dll|dmg|dng|doc|docm|docx|dot|dotm|dra|drc|DS_Store|dsk|dts|dtshd|dvb|dwg|dxf|ear|ecelp4800|ecelp7470|ecelp9600|egg|eol|eot|eps|epub|exe|f4a|f4b|f4p|f4v|fbs|fh|fla|flac|fli|flv|fpx|fst|fvt|g3|geojson|gif|graffle|gz|gzip|h261|h263|h264|hqx|htc|ico|ief|img|ipa|iso|jad|jar|jardiff|jng|jnlp|jpeg|jpg|jpgv|jpm|js|jxr|key|kml|kmz|ktx|less|lha|lvp|lz|lzh|lzma|lzo|m2v|m3u|m4a|m4p|m4v|map|manifest|mar|markdown|md|mdi|mdown|mdwn|mht|mid|midi|mj2|mka|mkd|mkdn|mkdown|mkv|mml|mmr|mng|mobi|mov|movie|mp2|mp3|mp4|mp4a|mpe|mpeg|mpg|mpga|mpv|msi|msm|msp|mxf|mxu|nef|npx|nsv|numbers|o|oex|oga|ogg|ogv|opus|otf|pages|pbm|pcx|pdb|pdf|pea|pem|pgm|pic|pl|pm|png|pnm|pot|potm|potx|ppa|ppam|ppm|pps|ppsm|ppsx|ppt|pptm|pptx|prc|ps|psd|pya|pyc|pyo|pyv|qt|ra|rar|ras|raw|rdf|rgb|rip|rlc|rm|rmf|rmvb|ron|roq|rpm|rss|rtf|run|rz|s3m|s7z|safariextz|scpt|sea|sgi|shar|sil|sit|slk|smv|so|sub|svg|svgz|svi|swf|tar|tbz|tbz2|tcl|tga|tgz|thmx|tif|tiff|tk|tlz|topojson|torrent|ttc|ttf|txt|txz|udf|uvh|uvi|uvm|uvp|uvs|uvu|vcard|vcf|viv|vob|vtt|war|wav|wax|wbmp|wdp|weba|webapp|webm|webmanifest|webp|whl|wim|wm|wma|wml|wmlc|wmv|wmx|woff|woff2|wvx|xbm|xif|xla|xlam|xloc|xls|xlsb|xlsm|xlsx|xlt|xltm|xltx|xm|xmind|xml|xpi|xpm|xsl|xwd|xz|yuv|z|zip|zipx)(?:\?[a-zA-Z0-9\-\.\_\~\:\/\#\[\]\@\!\$\&\'\(\)\*\+\,\;\=]*)?$/i
};

var _debug = function debug () {
console.trace();
console.warn.apply(null, arguments);
var _debug = function debug (...args) {
console.info.call(console, '[DEBUG] [Spiderable-Middleware]', ...args);
};

request.defaultOptions.debug = false;
request.defaultOptions.timeout = 51200;

module.exports = (function () {
function Spiderable(_opts) {
Expand All @@ -39,27 +39,27 @@ module.exports = (function () {
this.ignoredHeaders = opts.ignoredHeaders || Spiderable.prototype.ignoredHeaders;

if (Object.prototype.toString.call(this.staticExt) !== '[object RegExp]') {
_debug('[Spiderable-Middleware] `opts.staticExt` must be instance of RegExp, falling back to defaults.');
_debug('`opts.staticExt` must be instance of RegExp, falling back to defaults.');
this.staticExt = re.staticExt;
}

if (this.onlyRE && Object.prototype.toString.call(this.onlyRE) !== '[object RegExp]') {
_debug('[Spiderable-Middleware] `opts.onlyRE` must be instance of RegExp, rules are ignored!');
_debug('`opts.onlyRE` must be instance of RegExp, rules are ignored!');
this.onlyRE = false;
}

if (Object.prototype.toString.call(this.botsUA) !== '[object Array]') {
_debug('[Spiderable-Middleware] `opts.botsUA` must be instance of Array, falling back to defaults.');
_debug('`opts.botsUA` must be instance of Array, falling back to defaults.');
this.botsUA = this.prototype.botsUA;
}

if (Object.prototype.toString.call(this.ignoredHeaders) !== '[object Array]') {
_debug('[Spiderable-Middleware] `opts.ignoredHeaders` must be instance of Array, falling back to defaults.');
_debug('`opts.ignoredHeaders` must be instance of Array, falling back to defaults.');
this.ignoredHeaders = this.prototype.ignoredHeaders;
}

if (this.only && Object.prototype.toString.call(this.only) !== '[object Array]') {
_debug('[Spiderable-Middleware] `opts.only` must be instance of Array, rules are ignored!');
_debug('`opts.only` must be instance of Array, rules are ignored!');
this.only = false;
}

Expand All @@ -78,7 +78,7 @@ module.exports = (function () {
}

if (ignore && Object.prototype.toString.call(ignore) !== '[object Array]') {
_debug('[Spiderable-Middleware] `opts.ignore` must be instance of Array, rules are ignored!');
_debug('`opts.ignore` must be instance of Array, rules are ignored!');
ignore = false;
}

Expand All @@ -91,19 +91,19 @@ module.exports = (function () {
}

if (!this.rootURL) {
throw new Error('[Spiderable-Middleware] {rootURL} or env variable ROOT_URL is not detected! But must be specified!');
throw new Error('{rootURL} or env variable ROOT_URL is not detected! But must be specified!');
}

if (!this.serviceURL) {
throw new Error('[Spiderable-Middleware] {serviceURL} or env variable SPIDERABLE_SERVICE_URL or PRERENDER_SERVICE_URL is not detected! But must be specified!');
throw new Error('{serviceURL} or env variable SPIDERABLE_SERVICE_URL or PRERENDER_SERVICE_URL is not detected! But must be specified!');
}

if (!re.proto.test(this.rootURL)) {
throw new Error('[Spiderable-Middleware] {rootURL} is malformed! Must start with protocol http or https');
throw new Error('{rootURL} is malformed! Must start with protocol http or https');
}

if (!re.proto.test(this.serviceURL)) {
throw new Error('[Spiderable-Middleware] {serviceURL} is malformed! Must start with protocol http or https');
throw new Error('{serviceURL} is malformed! Must start with protocol http or https');
}

this.rootURL = this.rootURL.replace(re.trailingSlash, '').replace(re.beginningSlash, '');
Expand Down Expand Up @@ -156,7 +156,7 @@ module.exports = (function () {
break;
}
} else {
_debug('[Spiderable-Middleware] `opts.only` {' + this.only[i] + '} rule isn\'t instance of {String} nor {RegExp}, rule ignored!');
_debug('`opts.only` {' + this.only[i] + '} rule isn\'t instance of {String} nor {RegExp}, rule ignored!');
}
}
}
Expand Down Expand Up @@ -184,28 +184,29 @@ module.exports = (function () {
wait: true,
auth: this.auth || false,
headers: {
'User-Agent': 'spiderable-middleware/1.4.2'
'User-Agent': 'spiderable-middleware/1.4.3'
},
rawBody: true
rawBody: true,
noStorage: true
};

try {
var usedHeaders = [];
var _headersRE = this.headersRE;
var serviceReq = request(opts, function (error, resp) {
if (error) {
_debug('[Spiderable-Middleware] [request.get] Error while connecting to external service:', error);
_debug('Error while connecting to external service:', error);
next();
} else {
if (resp.statusCode === 401 || resp.statusCode === 403) {
_debug('[Spiderable-Middleware] Can\'t authenticate! Please check you "auth" parameter and other settings.');
_debug('Can\'t authenticate! Please check you "auth" parameter and other settings.');
}
res.statusCode = resp.statusCode || 200;
res.end();
}
});

serviceReq.request.on('header', function (header) {
serviceReq.onHeader(function (header) {
var h = header.toString('utf8');
if (!!~h.indexOf(':')) {
h = h.split(re.semi);
Expand All @@ -219,39 +220,47 @@ module.exports = (function () {
}
});

serviceReq.request.on('data', function (data) {
res.write(data.toString('utf8'), 'utf8');
});
serviceReq.onData(res.write.bind(res));

req.on('aborted', function () {
_debug('[Spiderable-Middleware] [REQ] ["aborted" event]:', arguments);
// No need to log this event as nothing bad happened
// this simply means host which sent this request
// has aborted the connection or got disconnected
// _debug('[REQ] ["aborted" event]:', arguments); // TODO: comment out
req.aborted = true;
serviceReq.abort();
res.end();
try {
res.end();
} catch (e) {
// We assume res.end() to throw an error
// but still will call for it, as we'd like
// to make sure memory, socket and session
// are freed-up and closed
}
});

req.on('error', function (error) {
_debug('[Spiderable-Middleware] [REQ] ["error" event] Unexpected error:', error);
_debug('[REQ] ["error" event] Unexpected error:', error);
serviceReq.abort();
next();
});

res.on('error', function (error) {
_debug('[Spiderable-Middleware] [RES] ["error" event] Unexpected error:', error);
_debug('[RES] ["error" event] Unexpected error:', error);
serviceReq.abort();
next();
});

res.on('close', function () {
_debug('[Spiderable-Middleware] [RES] ["close" event]:', arguments);
_debug('[RES] ["close" event]:', arguments);
serviceReq.abort();
next();
});

res.sendDate = false;
serviceReq.send();
} catch (e) {
_debug('[Spiderable-Middleware] Exception while connecting to external service:', e);
_debug('Exception while connecting to external service:', e);
next();
}
return true;
Expand Down
Loading

0 comments on commit b3251e3

Please sign in to comment.