diff --git a/extension/scripts/Sitemap.js b/extension/scripts/Sitemap.js index 3ea38cb3..94b62a57 100644 --- a/extension/scripts/Sitemap.js +++ b/extension/scripts/Sitemap.js @@ -179,11 +179,15 @@ Sitemap.prototype = { getDataColumns: function () { var columns = []; this.selectors.forEach(function (selector) { - columns = columns.concat(selector.getDataColumns()); }); - return columns; + var uniqueColumns = []; + $.each(columns, function (i, e) { + if ($.inArray(e, result) == -1) result.push(e); + }); + + return uniqueColumns; }, getDataExportCsvBlob: function (data, option) { diff --git a/extension/scripts/Store.js b/extension/scripts/Store.js index 9a287c7d..0ef0cdb4 100644 --- a/extension/scripts/Store.js +++ b/extension/scripts/Store.js @@ -9,32 +9,39 @@ var StoreScrapeResultWriter = function(db) { this.db = db; }; +/** +* Make sure all obj have the same properties +* They can differe if table selector retrieves dynamic columns +*/ +var normalizeProperties = function (docs) { + // get all keys of the objects + var keys = []; + docs.forEach(function (doc) { + for (var key in doc) { + if (doc.hasOwnProperty(key) && keys.indexOf(key) === -1) { keys.push(key); } + }; + }); + + // add missing keys to objects + docs.forEach(function (doc) { + var objKeys = Object.keys(doc) + keys.forEach(function (key) { + if (!(key in doc)) { + doc[key] = ""; + } + }); + }); +}; + StoreScrapeResultWriter.prototype = { + writeDocs: function(docs, callback) { if(docs.length === 0) { callback(); } else { - // get all keys of the objects - var keys = []; - docs.forEach(function (doc) { - for (var key in doc) { - if (doc.hasOwnProperty(key) && keys.indexOf(key) === -1) { keys.push(key); } - }; - }); - - // add missing keys to objects - // This can happen if same element containing different properties