add pennTest sample to unit tests

.eslintrc

@@ -11,7 +11,7 @@
   },
   "rules": {
     "no-cond-assign": 2,
-    "no-var": 1,
+    "no-var": 0,
     "prefer-const": 0,
     "no-extra-parens": 0,
     "no-dupe-keys": 2,

test/stress/penn-treebank/test.js

@@ -1,8 +1,9 @@
+// d/l file from here:
+// https://raw.githubusercontent.com/nlp-compromise/nlp_compromise/2e657cd4564fc25b5e271d008c1032610e586a33/test/perf_tests/pos_test/pennTreebank.js
 const data = require('./pennTreebank').data;
 const nlp = require('../../../src');
 
-
-const mapping = {
+const softMapping = {
   CC: 'Conjunction',
   CD: 'Cardinal',
   DT: 'Determiner',
@@ -40,4 +41,42 @@ const mapping = {
   WRB: 'Adverb',
 };
 
-console.log(data[3]);
+const worst = {};
+
+let good = data.filter((o) => {
+  let terms = nlp(o.text).terms();
+  if (terms.length !== o.pos.length) {
+    return false;
+  }
+  for(let i = 0; i < o.pos.length; i++) {
+    let want = o.pos[i].pos;
+    want = softMapping[want];
+    let term = terms.list[i].terms[0];
+    if (!term.tags[want]) {
+      let key = term.normal; //+ ' - ' + want;
+      worst[key] = worst[key] || 1;
+      worst[key] += 1;
+      // console.log(term.normal, want, Object.keys(term.tags));
+      return false;
+    }
+  }
+  return true;
+});
+
+let keys = Object.keys(worst);
+keys = keys.sort((a, b) => {
+  if (worst[a] > worst[b]) {
+    return -1;
+  }
+  return 1;
+});
+keys.slice(0, 100).forEach((k) => {
+  console.log(k, worst[k]);
+});
+// console.log(keys);
+// good = good.map((o) => {
+//   o.pos = o.pos.map((p) => p.pos);
+//   o.pos = o.pos.join(', ');
+//   return o;
+// });
+// console.log(JSON.stringify(good, null, 2));

test/unit/lib/pennSample.js

@@ -0,0 +1,319 @@
+//(very) small subset of the Penn-treebank that should always pass
+module.exports = [
+  {
+    'text': 'Dr. Talcott led a team of researchers from the National Cancer Institute and the medical schools of Harvard University and Boston University.',
+    'pos': 'NNP, NNP, VBD, DT, NN, IN, NNS, IN, DT, NNP, NNP, NNP, CC, DT, JJ, NNS, IN, NNP, NNP, CC, NNP, NNP'
+  },
+  {
+    'text': 'The monthly sales have been setting records every month since March.',
+    'pos': 'DT, JJ, NNS, VBP, VBN, VBG, NNS, DT, NN, IN, NNP'
+  },
+  {
+    'text': ' Cray Computer will be a concept stock, he said.',
+    'pos': 'NNP, NNP, MD, VB, DT, NN, NN, PRP, VBD'
+  },
+  {
+    'text': 'Esso said the Whiting field started production Tuesday.',
+    'pos': 'NNP, VBD, DT, NNP, NN, VBD, NN, NNP'
+  },
+  {
+    'text': 'Pick a country, any country.',
+    'pos': 'VB, DT, NN, DT, NN'
+  },
+  {
+    'text': 'They fell into oblivion after the 1929 crash.',
+    'pos': 'PRP, VBD, IN, NN, IN, DT, CD, NN'
+  },
+  {
+    'text': 'Political and currency gyrations can whipsaw the funds.',
+    'pos': 'JJ, CC, NN, NNS, MD, VB, DT, NNS'
+  },
+  {
+    'text': 'They cite a lack of imbalances that provide early warning signals of a downturn.',
+    'pos': 'PRP, VBP, DT, NN, IN, NNS, WDT, VBP, JJ, NN, NNS, IN, DT, NN'
+  },
+  {
+    'text': 'Characters drink Salty Dogs, whistle Johnny B. Goode and watch Bugs Bunny reruns.',
+    'pos': 'NNS, VBP, NNP, NNP, VBP, NNP, NNP, NNP, CC, VBP, NNP, NNP, NNS'
+  },
+  {
+    'text': 'They read Mickey Spillane and talk about Groucho and Harpo.',
+    'pos': 'PRP, VBP, NNP, NNP, CC, VBP, IN, NNP, CC, NNP'
+  },
+  {
+    'text': ' Consider Jim Courter.',
+    'pos': 'VB, NNP, NNP'
+  },
+  {
+    'text': 'But it resists yielding political ground.',
+    'pos': 'CC, PRP, VBZ, VBG, JJ, NN'
+  },
+  {
+    'text': ' In Asia, as in Europe, a new order is taking shape, Mr. Baker said.',
+    'pos': 'IN, NNP, IN, IN, NNP, DT, JJ, NN, VBZ, VBG, NN, NNP, NNP, VBD'
+  },
+  {
+    'text': 'And South Carolina says it is getting results.',
+    'pos': 'CC, NNP, NNP, VBZ, PRP, VBZ, VBG, NNS'
+  },
+  {
+    'text': ' It was full of violence and gangs and kids cutting class, says Linda Ward, the school\'s principal.',
+    'pos': 'PRP, VBD, JJ, IN, NN, CC, NNS, CC, NNS, VBG, NN, VBZ, NNP, NNP, DT, NN, NN'
+  },
+  {
+    'text': 'I believe in the system.',
+    'pos': 'PRP, VBP, IN, DT, NN'
+  },
+  {
+    'text': 'Mrs. Yeargin declined.',
+    'pos': 'NNP, NNP, VBD'
+  },
+  {
+    'text': 'Yeargin won widespread local support.',
+    'pos': 'NNP, VBD, JJ, JJ, NN'
+  },
+  {
+    'text': 'But Learning Materials matched on 66.5 of 69 subskills.',
+    'pos': 'CC, NNP, NNPS, VBD, IN, CD, IN, CD, NNS'
+  },
+  {
+    'text': 'The two banks merged in 1985.',
+    'pos': 'DT, CD, NNS, VBD, IN, CD'
+  },
+  {
+    'text': 'He said the company\'s core business remains strong.',
+    'pos': 'PRP, VBD, DT, NN, NN, NN, VBZ, JJ'
+  },
+  {
+    'text': 'Estimated volume was a moderate 3.5 million ounces.',
+    'pos': 'VBN, NN, VBD, DT, JJ, CD, CD, NNS'
+  },
+  {
+    'text': 'Mr. Gillespie at Viacom says the ratings are rising.',
+    'pos': 'NNP, NNP, IN, NNP, VBZ, DT, NNS, VBP, VBG'
+  },
+  {
+    'text': 'Ad Notes....',
+    'pos': 'NNP, NNPS'
+  },
+  {
+    'text': 'The business had been handled by VanSant Dugdale, Baltimore.',
+    'pos': 'DT, NN, VBD, VBN, VBN, IN, NNP, NNP, NNP'
+  },
+  {
+    'text': ' The economy is clearly slowing, says Robert Black, president of the Richmond Federal Reserve Bank.',
+    'pos': 'DT, NN, VBZ, RB, VBG, VBZ, NNP, NNP, NN, IN, DT, NNP, NNP, NNP, NNP'
+  },
+  {
+    'text': 'They will mature Dec. 21.',
+    'pos': 'PRP, MD, VB, NNP, CD'
+  },
+  {
+    'text': 'Lancaster Colony Corp. said it acquired Reames Foods Inc. in a cash transaction.',
+    'pos': 'NNP, NNP, NNP, VBD, PRP, VBD, NNP, NNP, NNP, IN, DT, NN, NN'
+  },
+  {
+    'text': 'NL is officially making the offer.',
+    'pos': 'NNP, VBZ, RB, VBG, DT, NN'
+  },
+  {
+    'text': 'The Japanese fret openly about the U.S. public\'s rancor.',
+    'pos': 'DT, NNP, NN, RB, IN, DT, NNP, NN, NN'
+  },
+  {
+    'text': 'They operate ships and banks.',
+    'pos': 'PRP, VBP, NNS, CC, NNS'
+  },
+  {
+    'text': 'Adds Takeshi Kondo, senior vice president of C. Itoh America Inc.: We have a great interest in making investments, particularly in new ventures.',
+    'pos': 'VBZ, NNP, NNP, JJ, NN, NN, IN, NNP, NNP, NNP, NNP, PRP, VBP, DT, JJ, NN, IN, VBG, NNS, RB, IN, JJ, NNS'
+  },
+  {
+    'text': 'But many banks are turning away from strict price competition.',
+    'pos': 'CC, JJ, NNS, VBP, VBG, RB, IN, JJ, NN, NN'
+  },
+  {
+    'text': 'One big reason: thin margins.',
+    'pos': 'CD, JJ, NN, JJ, NNS'
+  },
+  {
+    'text': 'Buy a new Chevrolet.',
+    'pos': 'VB, DT, JJ, NNP'
+  },
+  {
+    'text': 'Buy a diamond necklace.',
+    'pos': 'VB, DT, NN, NN'
+  },
+  {
+    'text': 'They are keeping a close watch on the yield on the S&P 500.',
+    'pos': 'PRP, VBP, VBG, DT, JJ, NN, IN, DT, NN, IN, DT, NNP, CD'
+  },
+  {
+    'text': 'In fact, the market has always tanked.',
+    'pos': 'IN, NN, DT, NN, VBZ, RB, VBN'
+  },
+  {
+    'text': 'Always.',
+    'pos': 'RB'
+  },
+  {
+    'text': 'China pulled out of the program in July.',
+    'pos': 'NNP, VBD, IN, IN, DT, NN, IN, NNP'
+  },
+  {
+    'text': 'But regulators are wary.',
+    'pos': 'CC, NNS, VBP, JJ'
+  },
+  {
+    'text': 'He also is a consensus manager, insiders say.',
+    'pos': 'PRP, RB, VBZ, DT, NN, NN, NNS, VBP'
+  },
+  {
+    'text': 'Compromises are possible.',
+    'pos': 'NNS, VBP, JJ'
+  },
+  {
+    'text': 'The company acknowledges some problems.',
+    'pos': 'DT, NN, VBZ, DT, NNS'
+  },
+  {
+    'text': 'A number of cities including Minneapolis, Philadelphia and Houston have vacant grain elevators, Eggers says.',
+    'pos': 'DT, NN, IN, NNS, VBG, NNP, NNP, CC, NNP, VBP, JJ, NN, NNS, NNP, VBZ'
+  },
+  {
+    'text': 'They suffered from malnutrition, chest diseases, cardiovascular disorders, skin problems, infectious diseases and the aftereffects of assaults and rape.',
+    'pos': 'PRP, VBD, IN, NN, NN, NNS, JJ, NNS, NN, NNS, JJ, NNS, CC, DT, NNS, IN, NNS, CC, NN'
+  },
+  {
+    'text': 'That was the law.',
+    'pos': 'DT, VBD, DT, NN'
+  },
+  {
+    'text': 'It was censorship.',
+    'pos': 'PRP, VBD, NN'
+  },
+  {
+    'text': 'It was outrageous.',
+    'pos': 'PRP, VBD, JJ'
+  },
+  {
+    'text': 'But the court disagreed.',
+    'pos': 'CC, DT, NN, VBD'
+  },
+  {
+    'text': 'The man was Charles Z. Wick.',
+    'pos': 'DT, NN, VBD, NNP, NNP, NNP'
+  },
+  {
+    'text': ' Bob has handled the extraordinary growth of the company quite brilliantly, said Mr. Newhouse.',
+    'pos': 'NNP, VBZ, VBN, DT, JJ, NN, IN, DT, NN, RB, RB, VBD, NNP, NNP'
+  },
+  {
+    'text': 'This species of congressional action is predicated on an interpretation of the appropriations clause that is erroneous and unconstitutional.',
+    'pos': 'DT, NN, IN, JJ, NN, VBZ, VBN, IN, DT, NN, IN, DT, NNS, NN, WDT, VBZ, JJ, CC, JJ'
+  },
+  {
+    'text': 'President Reagan learned that lesson.',
+    'pos': 'NNP, NNP, VBD, DT, NN'
+  },
+  {
+    'text': 'Mr. Sidak served as an attorney in the Reagan administration.',
+    'pos': 'NNP, NNP, VBD, IN, DT, NN, IN, DT, NNP, NN'
+  },
+  {
+    'text': 'The death of the Herald, a newsstand paper in a freeway town, was perhaps inevitable.',
+    'pos': 'DT, NN, IN, DT, NNP, DT, NN, NN, IN, DT, NN, NN, VBD, RB, JJ'
+  },
+  {
+    'text': ' The Herald was a survivor from a bygone age, said J. Kendrick Noble, a media analyst with PaineWebber Inc.',
+    'pos': 'DT, NNP, VBD, DT, NN, IN, DT, JJ, NN, VBD, NNP, NNP, NNP, DT, NNS, NN, IN, NNP, NNP'
+  },
+  {
+    'text': 'The reaction in the newsroom was emotional.',
+    'pos': 'DT, NN, IN, DT, NN, VBD, JJ'
+  },
+  {
+    'text': 'The program traders, on the other hand, portray old-fashioned stock pickers as the Neanderthals of the industry.',
+    'pos': 'DT, NN, NNS, IN, DT, JJ, NN, VBP, JJ, NN, NNS, IN, DT, NNS, IN, DT, NN'
+  },
+  {
+    'text': 'Reducing volatility.',
+    'pos': 'VBG, NN'
+  },
+  {
+    'text': 'Ballot watchers say attention already is focused on the 1990 elections.',
+    'pos': 'NN, NNS, VBP, NN, RB, VBZ, VBN, IN, DT, CD, NNS'
+  },
+  {
+    'text': 'Colleges, she says, are eyeing registration through 900 service.',
+    'pos': 'NNS, PRP, VBZ, VBP, VBG, NN, IN, CD, NN'
+  },
+  {
+    'text': 'FAMILY PETS are improving recovery rates of patients at Columbia Hospital, Milwaukee.',
+    'pos': 'NN, NNS, VBP, VBG, NN, NNS, IN, NNS, IN, NNP, NNP, NNP'
+  },
+  {
+    'text': 'The appointment takes effect Nov. 13.',
+    'pos': 'DT, NN, VBZ, NN, NNP, CD'
+  },
+  {
+    'text': 'Heiwado Co.',
+    'pos': 'NNP, NNP'
+  },
+  {
+    'text': 'Guaranteed by Svenska Handelsbanken.',
+    'pos': 'VBN, IN, NNP, NNP'
+  },
+  {
+    'text': 'Mitsubishi Pencil Co.',
+    'pos': 'NNP, NNP, NNP'
+  },
+  {
+    'text': 'Koizumi Sangyo Corp.',
+    'pos': 'NNP, NNP, NNP'
+  },
+  {
+    'text': 'A stadium craze is sweeping the country.',
+    'pos': 'DT, NN, NN, VBZ, VBG, DT, NN'
+  },
+  {
+    'text': 'Stock prices closed higher in Stockholm, Amsterdam and Frankfurt and lower in Zurich.',
+    'pos': 'NN, NNS, VBD, JJR, IN, NNP, NNP, CC, NNP, CC, JJR, IN, NNP'
+  },
+  {
+    'text': 'A faster version, the SuperDot, was launched in 1984.',
+    'pos': 'DT, JJR, NN, DT, NNP, VBD, VBN, IN, CD'
+  },
+  {
+    'text': 'Valley Federal is currently being examined by regulators.',
+    'pos': 'NNP, NNP, VBZ, RB, VBG, VBN, IN, NNS'
+  },
+  {
+    'text': 'Columbia has only about 10 million common shares in public hands.',
+    'pos': 'NNP, VBZ, RB, IN, CD, CD, JJ, NNS, IN, JJ, NNS'
+  },
+  {
+    'text': 'But the concept is workable.',
+    'pos': 'CC, DT, NN, VBZ, JJ'
+  },
+  {
+    'text': 'Mr. Spiegel\'s next career move is a subject of speculation on Wall Street.',
+    'pos': 'NNP, NNP, JJ, NN, NN, VBZ, DT, NN, IN, NN, IN, NNP, NNP'
+  },
+  {
+    'text': 'Wedtech management used the merit system.',
+    'pos': 'NNP, NN, VBD, DT, NN, NN'
+  },
+  {
+    'text': 'Numerous other scandals, among them the ones at HUD, have the same characteristics as Wedtech.',
+    'pos': 'JJ, JJ, NNS, IN, PRP, DT, NNS, IN, NNP, VBP, DT, JJ, NNS, IN, NNP'
+  },
+  {
+    'text': 'Railroad companies and some ports are reaping a sudden windfall of business.',
+    'pos': 'NN, NNS, CC, DT, NNS, VBP, VBG, DT, JJ, NN, IN, NN'
+  },
+  {
+    'text': ' The recent rally in precious metals was a result of uncertainty and volatility in equities, he said.',
+    'pos': 'DT, JJ, NN, IN, JJ, NNS, VBD, DT, NN, IN, NN, CC, NN, IN, NNS, PRP, VBD'
+  }
+];