From 09073d53b66aac48b9b7a2a669d7bbf0153d2de0 Mon Sep 17 00:00:00 2001 From: RajPratim Date: Sun, 9 Apr 2017 21:24:49 +0530 Subject: [PATCH 1/2] integrated levi without testing --- .babelrc | 4 ++ README.md | 1 - WorldBrain-Memex-Extension- | 1 + package.json | 2 + src/js/background.js | 74 +++++++++++++++++++++++++++++++++++-- 5 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 .babelrc create mode 160000 WorldBrain-Memex-Extension- diff --git a/.babelrc b/.babelrc new file mode 100644 index 0000000..1b94970 --- /dev/null +++ b/.babelrc @@ -0,0 +1,4 @@ +{ + "plugins": ["transform-regenerator", "transform-runtime"], + "presets": ["es2015", "stage-1", "react"] +} diff --git a/README.md b/README.md index 1320837..2096885 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ - ## IMPORTANT We started collaborating with the [WebMemex project](https://github.com/WebMemex/memextension) and are in the mode of porting all the features of WorldBrain there. The WebMemex will power our software in the future and will lend its name to the WorldBrain tool as well. Therefore be not surprised, if you see no action in this repo. diff --git a/WorldBrain-Memex-Extension- b/WorldBrain-Memex-Extension- new file mode 160000 index 0000000..ba52a84 --- /dev/null +++ b/WorldBrain-Memex-Extension- @@ -0,0 +1 @@ +Subproject commit ba52a84b80bff84f1969809eafdbb3d250049cba diff --git a/package.json b/package.json index 4645ba5..547c3f5 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "npm-run-all": "^4.0.0" }, "devDependencies": { + "babel-core": "^6.24.1", + "babel-loader": "^6.4.1", "cpx": "^1.5.0", "watchify": "^3.8.0" }, diff --git a/src/js/background.js b/src/js/background.js index b552638..dc7b077 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -1,3 +1,4 @@ + var MILLIS_BEFORE_CLEAR = 1000 * 60; // 60 seconds var CLEAR_DELAY = 20000; var MAX_URL_LEN_SHOWN = 50; @@ -8,6 +9,12 @@ var LT_OBJ = function(a,b) { } var db +//import levi from 'levi' something like that +var levidb = levi.levi('db') +.use(levi.tokenizer()) +.use(levi.stemmer()) +.use(levi.stopword()) + function ValidURL(text) { var valid = /((https?):\/\/)?(([w|W]{3}\.)+)?[a-zA-Z0-9\-\.]{3,}\.[a-zA-Z]{2,}(\.[a-zA-Z]{2,})?/ @@ -46,7 +53,8 @@ function acceptInput(text, disposition) { } function init() { - + + //levi = new indexedDB('levi') db = new PouchDB('main') // db.plugin() var remoteCouch = false; @@ -89,7 +97,7 @@ function transferToPouch() { keys[i] != "blacklist" && keys[i] != "preferences" && keys[i] != "shouldOpenTab") - count += store_url(results[keys[i]]); + count += store_url(results[keys[i]]); } ); console.log('Successfully stored ' + count.toString() + ' / ' + (keys.length - 4).toString() + ' items to PDB'); @@ -97,7 +105,7 @@ function transferToPouch() { function handleMessage(data, sender, sendRespones) { // data is from message - if (data.msg === 'pageContent' && shouldArchive(data)) { + if (data.msg === 'pageContent' && shouldArchive(data) ) { delete data.msg; data.text = processPageText(data.text); //console.log("TEST"+ data.text) @@ -154,15 +162,75 @@ function store_url(data) { return 0; } }); + levidb.put('indexing_key',item ) + console.log('#############Build') } function search_pouch(query, text, cb, suggestCb) { + //levi search implementation + levidb.searchStream(query, { + fields: { 'title': 10, 'url': 5, 'text':1 } // title field boost. then url then text as mentioned by Oliver. + }).toArray(function (res) { console.log(res) + /*levi search returns the search results into an Array. We parse that array along with the conditions + applied in the below original search of Research Engine checking for date time compatiblities and + negative query length . + */ + var results = []; + for(var i = 0; i < res.length ; i++) { + var doc = res[i]; + if (query.before != false) { + if (doc.doc.LastVisitTime <= query.before.getTime() && doc.doc.LastVisitTime >= query.after.getTime()) + results.push(doc); + } + else if (doc.doc.LastVisitTime >= query.after.getTime()) + results.push(doc); + } + // If Minuswords are entered in the query + var final_results = []; + if(query.negative.length > 0) { + + //calculating the precision, if more than 1 minus word is entered: https://github.com/nolanlawson/pouchdb-quick-search#minimum-should-match-mm + var full_percent = 100 + divisor = query.negative.length + var mm = 100 / divisor + "%" + // searching for two minus words + query.negative= query.negative.join(" ") + db.search({ + query: query.negative, + fields: ['title','url','text'], + mm: mm, + include_docs: true, + build: false + }).then(function (res_negative) { + for(var i = 0 ; i < res.length ; i++) { + var flag = 0; + for(var j = 0 ; j < res_negative.rows.length ; j++) { + if(res[i].id === res_negative.rows[j].id) + flag = 1; + } + if(flag === 0) + final_results.push(res[i]); + } + return suggestionsComplete(final_results, query.shouldDate, suggestCb); + }); + } else { + final_results = results; + } + if(final_results.length > 0) + return suggestionsComplete(final_results, query.shouldDate, suggestCb); + }) + + + + for(var i = 0 ; i < query.keywords.length ; i++) if(query.keywords[i].length == 0) query.keywords.splice(i, 1) // Regular search through PouchDB + + db.search({ query: query.text, fields: ['title','url','text'], From e3fb4cb02fcd7afbc3bdd1d5005dc571ea9a0796 Mon Sep 17 00:00:00 2001 From: RajPratim Date: Sun, 9 Apr 2017 21:47:45 +0530 Subject: [PATCH 2/2] removed babel --- .babelrc | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 .babelrc diff --git a/.babelrc b/.babelrc deleted file mode 100644 index 1b94970..0000000 --- a/.babelrc +++ /dev/null @@ -1,4 +0,0 @@ -{ - "plugins": ["transform-regenerator", "transform-runtime"], - "presets": ["es2015", "stage-1", "react"] -}