feat(algorithm): Adds Bezkrovny's SSIM algorithm (#85)

- Adds Bezkrovny's algorithm for SSIM and validation analysis. This is a good approximation to the original results with a better performance profile - Consolidates error logic (makes all errors asynchronous) - Forces valid ssim parameter (instead of falling back to 'original' which is slow) - Decouples resizing logic. It was kept within the ssim scripts since that matched the original scripts better but it's harder to maintain when adding additional comparison algorithms. BREAKING CHANGES: All errors are asynchrnous and removes fallback when wrong ssim algorithm is specified
obartra · Jan 24, 2017 · 3c62674 · 3c62674
1 parent 24ac5bd
commit 3c62674
Show file tree

Hide file tree

Showing 32 changed files with 786 additions and 204 deletions.
diff --git a/LICENSES/bezkrovny b/LICENSES/bezkrovny
@@ -0,0 +1,49 @@
+# image-quantization (https://github.com/igor-bezkrovny/image-quantization)
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Igor Bezkrovny
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+# http://members.ozemail.com.au/~dekker/NEUQUANT.HTML
+
+NeuQuant Neural-Net Quantization Algorithm
+------------------------------------------
+
+Copyright (c) 1994 Anthony Dekker
+
+NEUQUANT Neural-Net quantization algorithm by Anthony Dekker, 1994. See
+"Kohonen neural networks for optimal colour quantization" in "Network:
+Computation in Neural Systems" Vol. 5 (1994) pp 351-367. for a discussion of
+the algorithm.
+
+Any party obtaining a copy of these files from the author, directly or
+indirectly, is granted, free of charge, a full and unrestricted irrevocable,
+world-wide, paid up, royalty-free, nonexclusive right and license to deal in
+this software and documentation files (the "Software"), including without
+limitation the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit persons who
+receive copies from any such party to do so, with the only requirement being
+that this copyright notice remain intact.
+
+# https://github.com/leeoniya/RgbQuant.js
+
+Copyright (c) 2015, Leon Sorokin
+All rights reserved. (MIT Licensed)
diff --git a/LICENSES/rhys-e b/LICENSES/rhys-e
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 rhys-e
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/cli.js b/cli.js
@@ -10,11 +10,16 @@ const argv = yargs
 	.example(`${name} img1.png img2.png`)
 	.example(`${name} img1.png img2.png --quiet`)
 	.example(`${name} img1.png img2.png --threshold 0.95`)
+	.example(`${name} img1.png img2.png --algorithm bezkrovny`)
 	.example(`${name} https://url.jpg https://url2.jpg`)
 	.nargs('threshold', 1)
 	.number('threshold')
 	.alias('t', 'threshold')
 	.describe('threshold', 'exit 0 if mssim >= threshold, exit 1 otherwise')
+	.nargs('algorithm', 1)
+	.alias('a', 'algorithm')
+	.describe('algorithm', 'SSIM algorithm to use: fast/original/bezkrovny')
+	.default('algorithm', 'fast')
 	.nargs('quiet', 0)
 	.alias('q', 'quiet')
 	.describe('quiet', 'Prints only the mean ssim value')
@@ -76,7 +81,7 @@ function onError(error) {
 }
 
 if (validate()) {
-	ssim(argv._[0], argv._[1])
+	ssim(argv._[0], argv._[1], { ssim: argv.algorithm })
 		.then(onComplete)
 		.catch(onError);
 } else {

diff --git a/generate/README.md b/generate/README.md
@@ -20,6 +20,7 @@ The generated csv files can then be processed by `LIVEresults.js` to generate th
 Note that you don't have to regenerate this file to validate the results. If you want to though, you would run (in Matlab / Octave):
 
 ```Matlab
+>> pkg load image
 >> compareLIVESSIM
 ```
 

diff --git a/generate/bezkrovny/README.md b/generate/bezkrovny/README.md
@@ -0,0 +1,25 @@
+# Bezkrovny
+
+This folder contains scripts related to the comparison of Mean Opinion Score (MOS) and the mean ssim values with those generated by Bezkrovny's implementation of the SSIM algorithm.
+
+While the difference between the algorithm and reported results is greater in Bezkrovny's case, the algorithm is significantly faster and correlates better with MOS values.
+
+The following graph illustrates these differences by looking at all results from the LIVE database ("Release 2"):
+
+![LIVE](all.png)
+
+## Matlab Comparison
+
+To reproduce these results you first need to run `compareLIVESSIM.m` which will generate the csv files needed for `genresults.js`. Then, running: `node genresults.js` will generate `out.dat` which is needed by the `compare.m` matlab script
+
+in Matlab / Octave you can reproduce the different plots by running:
+
+```Matlab
+>> pkg load image
+>> compare('fastfading', 809, 982);
+>> compare('gblur', 635, 808);
+>> compare('wn', 461, 634);
+>> compare('jpeg', 228, 460);
+>> compare('jp2k', 1, 227);
+>> compare('all', 1, 982);
+```
diff --git a/generate/bezkrovny/all.png b/generate/bezkrovny/all.png
diff --git a/generate/bezkrovny/compare.m b/generate/bezkrovny/compare.m
@@ -0,0 +1,47 @@
+function compare(name, start, end)
+	[parentdir,~,~]=fileparts(pwd);
+	load(fullfile(parentdir,'LIVE_SSIM_results'))
+	load(fullfile(parentdir,'computed_ssim'))
+
+	nmos = 100 - (dmos_all - min(dmos_all)) / (max(dmos_all) - min(dmos_all)) * 100;
+	fid = fopen('out.dat');
+	out = textscan(fid, '%s %s %s %f %f %f');
+	fclose(fid);
+	total = size(out{1}, 1);
+
+	index = 1;
+	set = zeros(1, end - start);
+	for i = 1:total
+		if (strcmp(out{2}(i), name) || strcmp(name, 'all'))
+			set(index) = out{6}(i);
+			index += 1;
+		end
+	end
+	clf;
+	color1 = [62 150 81] ./ 255;
+	color2 = [146 36 40] ./ 255;
+	color3 = [0 0 0] ./ 255;
+	scatter(LIVE_ssim_all(start:end), nmos(start:end), 6, color1, 's', 'filled');
+	hold on;
+	scatter(computed_ssim(start:end), nmos(start:end), 6, color2, 'p', 'filled');
+	hold on;
+	scatter(set, nmos(start:end), 6, color3, 'h', 'filled');
+	grid on;
+	legend('original', 'ssim.js', 'Bezkrovny', 'location', 'northeastoutside');
+	xlabel('mssim score');
+	ylabel('subjective rating');
+	title(name);
+	print(strcat(name, '.png'));
+
+	corr_original = corr(LIVE_ssim_all(start:end), nmos(start:end))
+	corr_ssimjs = corr(computed_ssim(start:end), nmos(start:end))
+	corr_bezkrovny = corr(set, nmos(start:end))
+
+	mse_original = sqrt(mean((nmos(start:end)/100 - LIVE_ssim_all(start:end)).^2));
+	mse_ssimjs = sqrt(mean((nmos(start:end)/100 - computed_ssim(start:end)).^2));
+	mse_bezkrovny = sqrt(mean((nmos(start:end)/100 - set).^2));''
+
+	max_original = max(abs(nmos(start:end)/100 - LIVE_ssim_all(start:end)));
+	max_ssimjs = max(abs(nmos(start:end)/100 - computed_ssim(start:end)));
+	max_bezkrovny = max(abs(nmos(start:end)/100 - set));
+end
diff --git a/generate/bezkrovny/fastfading.png b/generate/bezkrovny/fastfading.png
diff --git a/generate/bezkrovny/gblur.png b/generate/bezkrovny/gblur.png
diff --git a/generate/bezkrovny/genresults.js b/generate/bezkrovny/genresults.js
@@ -0,0 +1,65 @@
+const { readFileSync, writeFileSync } = require('fs');
+const ssim = require('../../dist/ssim');
+const { resolve } = require('path');
+
+const imgPath = resolve(__dirname, '../../spec/samples/LIVE/');
+// valid formats are *.json and *.dat
+const targetPath = resolve(__dirname, './out.dat');
+const outputFormat = targetPath.split('.').pop();
+
+function genBezkrovnySSIM([name, csv], format) {
+	return csv
+		.split('\n')
+		.map(line => line.split(','))
+		.map(([reference, file, mssim, reportedMssim]) => {
+			const refImg = resolve(imgPath, 'refimgs', reference);
+			const targetImg = resolve(imgPath, name, `${file}.bmp`);
+
+			return ssim(refImg, targetImg, { ssim: 'bezkrovny' })
+				.then(({ mssim: bezkrovnyMssim }) => {
+					bezkrovnyMssim = parseFloat(bezkrovnyMssim.toPrecision(5));
+
+					if (format === 'dat') {
+						return `${reference} ${name} ${file} ${mssim} ${reportedMssim} ${bezkrovnyMssim}`;
+					}
+
+					return {
+						file: `${file}.bmp`,
+						mssim,
+						reference,
+						bezkrovnyMssim,
+						reportedMssim,
+						type: name
+					};
+				});
+		});
+}
+
+function accumulate(val, pre = [], format = 'dat') {
+	process.stdout.write('.');
+	return Promise.all(genBezkrovnySSIM(val, format)).then((resp) => {
+		if (format === 'dat') {
+			return `${pre}\n${resp.join('\n')}`;
+		}
+		return [...pre, ...resp];
+	});
+}
+
+const files = ['jp2k', 'jpeg', 'wn', 'gblur', 'fastfading']
+	.map(file => [file, resolve(__dirname, '..', `${file}.csv`)])
+	.map(([name, path]) => [name, readFileSync(path)])
+	.map(([name, buffer]) => [name, buffer.toString()]);
+
+files.reduce((promise, val) =>
+	promise.then(pre => accumulate(val, pre, outputFormat))
+, Promise.resolve())
+	.then((out) => {
+		process.stdout.write(`\nSaving ${targetPath}...\n`);
+		if (outputFormat === 'dat') {
+			writeFileSync(targetPath, out.trim());
+		} else {
+			const jsonContent = [].concat.apply([], out); // eslint-disable-line prefer-spread
+
+			writeFileSync(targetPath, JSON.stringify(jsonContent, undefined, '\t'));
+		}
+	});
diff --git a/generate/bezkrovny/jp2k.png b/generate/bezkrovny/jp2k.png
diff --git a/generate/bezkrovny/jpeg.png b/generate/bezkrovny/jpeg.png
diff --git a/generate/bezkrovny/wn.png b/generate/bezkrovny/wn.png
diff --git a/index.js b/index.js
@@ -3,11 +3,18 @@ const { rgb2gray } = require('./src/matlab');
 const { mean2d } = require('./src/math');
 const { ssim } = require('./src/ssim');
 const { originalSsim } = require('./src/originalSsim');
-const { force } = require('./src/util');
+const { bezkrovnySsim } = require('./src/bezkrovnySsim');
+const { downsample } = require('./src/downsample');
 const defaults = require('./src/defaults.json');
 const { version } = require('./version.js');
 const promiz = require('promiz');
 
+const ssimTargets = {
+	fast: ssim,
+	original: originalSsim,
+	bezkrovny: bezkrovnySsim
+};
+
 function validateOptions(options) {
 	Object.keys(options).forEach((option) => {
 		if (!(option in defaults)) {
@@ -20,43 +27,68 @@ function validateOptions(options) {
 	if ('k2' in options && (typeof options.k2 !== 'number' || options.k2 < 0)) {
 		throw new Error(`Invalid k2 value. Default is ${defaults.k2}`);
 	}
+	if (!(options.ssim in ssimTargets)) {
+		throw new Error(`Invalid ssim option (use: ${Object.keys(ssimTargets).join(', ')})`);
+	}
 }
 
-function getOptions(options) {
-	validateOptions(options);
-	return Object.assign({}, defaults, options);
+function getOptions(userOptions) {
+	return new singleSSIM.Promise((resolve) => {
+		const options = Object.assign({}, defaults, userOptions);
+
+		validateOptions(options);
+
+		resolve(options);
+	});
 }
 
-function validateDimensions(pixels) {
-	if (pixels[0].width !== pixels[1].width || pixels[0].height !== pixels[1].height) {
+function validateDimensions([pixels1, pixels2, options]) {
+	if (pixels1.width !== pixels2.width || pixels1.height !== pixels2.height) {
 		throw new Error('Image dimensions do not match');
 	}
 
-	return pixels;
+	return [pixels1, pixels2, options];
+}
+
+function toGrayScale([pixels1, pixels2, options]) {
+	return [rgb2gray(pixels1), rgb2gray(pixels2), options];
 }
 
-function toGrayScale(pixels) {
-	return [rgb2gray(pixels[0]), rgb2gray(pixels[1])];
+function toResize([pixels1, pixels2, options]) {
+	const pixels = downsample([pixels1, pixels2], options);
+
+	return [pixels[0], pixels[1], options];
+}
+
+function comparison([pixels1, pixels2, options]) {
+	return ssimTargets[options.ssim](pixels1, pixels2, options);
 }
 
 function readImage(image, options) {
-	if (options.downsample === 'fast') {
+	if (!image) {
+		throw new Error('Missing image parameter');
+	} else if (options.downsample === 'fast') {
 		return readpixels(image, singleSSIM.Promise, options.maxSize);
 	}
 	return readpixels(image, singleSSIM.Promise);
 }
 
-function singleSSIM(image1 = force('image1'), image2 = force('image2'), options = {}) {
-	const start = new Date().getTime();
-
-	options = getOptions(options);
+function readImages(image1, image2, options) {
+	return singleSSIM.Promise.all([
+		readImage(image1, options),
+		readImage(image2, options)
+	]).then(images => [images[0], images[1], options]);
+}
 
-	const ssimImpl = options.ssim === 'fast' ? ssim : originalSsim;
+function singleSSIM(image1, image2, userOptions) {
+	const start = new Date().getTime();
 
-	return singleSSIM.Promise.all([readImage(image1, options), readImage(image2, options)])
+	return getOptions(userOptions)
+		.then(options => readImages(image1, image2, options))
 		.then(validateDimensions)
 		.then(toGrayScale)
-		.then(pixels => ssimImpl(pixels[0], pixels[1], options))
+		.then(toResize)
+		.then(comparison)
 		.then(ssimMap => ({
 			ssim_map: ssimMap,
 			mssim: mean2d(ssimMap),
@@ -78,7 +110,7 @@ singleSSIM.Promise = this.Promise || promiz;
  * mod('/img1.jpg', '/img2.jpg');
  * mod.ssim('/img1.jpg', '/img2.jpg');
  */
-singleSSIM.ssim = ssim;
+singleSSIM.ssim = singleSSIM;
 /**
  * @property {String} version - The SSIM package version
  * @public