Draft

CHANGELOG.md

@@ -1,9 +1,10 @@
 # Changelog
 
 ## Canary
+*   Right click a word and select 'Look up...' to see its dictionary definition
 *   Skip button for auto-post ads
 *   Image tab on the character profile view now loads up faster 
-*   Search results view is more responsive while still scoring profiles
+*   Search results view is now more responsive while scoring profiles
 *   Fixed 'unicorn match' badge color
 
 

assets/wordnet-dictionary/adv.exc

@@ -0,0 +1,7 @@
+best well
+better well
+deeper deeply
+farther far
+further far
+harder hard
+hardest hard

electron/Window.vue

@@ -94,7 +94,7 @@
         async mounted(): Promise<void> {
             log.debug('init.window.mounting');
             // top bar devtools
-            // browserWindow.webContents.openDevTools({ mode: 'detach' });
+            browserWindow.webContents.openDevTools({ mode: 'detach' });
 
             updateSupportedLanguages(browserWindow.webContents.session.availableSpellCheckerLanguages);
 
@@ -255,7 +255,7 @@
             );
 
             // tab devtools
-            // view.webContents.openDevTools();
+            view.webContents.openDevTools();
 
             // console.log('ADD TAB LANGUAGES', getSafeLanguages(this.settings.spellcheckLang), this.settings.spellcheckLang);
             view.webContents.session.setSpellCheckerLanguages(getSafeLanguages(this.settings.spellcheckLang));

electron/chat.ts

@@ -52,7 +52,8 @@ import {Logs, SettingsStore} from './filesystem';
 import Notifications from './notifications';
 import * as SlimcatImporter from './importer';
 import Index from './Index.vue';
-import log from 'electron-log'; // tslint:disable-line: match-default-export-name
+import log from 'electron-log';
+import { DefinitionDictionary } from '../learn/dictionary/definition-dictionary'; // tslint:disable-line: match-default-export-name
 
 
 log.debug('init.chat');
@@ -74,6 +75,7 @@ const sc = nativeRequire<{
 }>('spellchecker/build/Release/spellchecker.node');
 const spellchecker = new sc.Spellchecker();*/
 
+
 Axios.defaults.params = {__fchat: `desktop/${electron.remote.app.getVersion()}`};
 
 if(process.env.NODE_ENV === 'production') {
@@ -105,6 +107,8 @@ function openIncognito(url: string): void {
 }
 
 const webContents = electron.remote.getCurrentWebContents();
+const wordDef = new DefinitionDictionary(electron.remote.app.getAppPath());
+
 webContents.on('context-menu', (_, props) => {
     const hasText = props.selectionText.trim().length > 0;
     const can = (type: string) => (<Electron.EditFlags & {[key: string]: boolean}>props.editFlags)[`can${type}`] && hasText;
@@ -181,7 +185,23 @@ webContents.on('context-menu', (_, props) => {
             click: () => electron.ipcRenderer.send('dictionary-remove', props.selectionText)
         }, {type: 'separator'});
 
+    if (props.selectionText) {
+        menuTemplate.unshift(
+          { type: 'separator' },
+          {
+            label: `Look up '${props.selectionText}'`,
+            click: async() => {
+                const words = await wordDef.getDefinition(props.selectionText);
+
+                // console.log('WORDS', words);
+            }
+          }
+        );
+    }
+
     if(menuTemplate.length > 0) electron.remote.Menu.buildFromTemplate(menuTemplate).popup({});
+
+    log.debug('context.text', { linkText: props.linkText, misspelledWord: props.misspelledWord, selectionText: props.selectionText, titleText: props.titleText });
 });
 
 let dictDir = path.join(electron.remote.app.getPath('userData'), 'spellchecker');

electron/webpack.config.js

@@ -126,6 +126,11 @@ const mainConfig = {
                         from: path.resolve(__dirname, '..', 'node_modules', '@cliqz', 'adblocker-electron-preload', 'dist', 'preload.cjs.js').replace(/\\/g, '/'),
                         to: path.join('preview', 'assets', 'adblocker'),
                         context: path.resolve(__dirname, '..', 'node_modules', '@cliqz', 'adblocker-electron-preload', 'dist')
+                    },
+                    {
+                        from: path.resolve(__dirname, '..', 'assets', '**', '*').replace(/\\/g, '/'),
+                        to: path.join('assets'),
+                        context: path.resolve(__dirname, '..', 'assets')
                     }
                 ]
             }

learn/dictionary/definition-dictionary.ts

@@ -0,0 +1,81 @@
+import _ from 'lodash';
+import path from 'path';
+
+// tslint:disable-next-line ban-ts-ignore
+// @ts-ignore
+import anyAscii from 'any-ascii';
+
+// tslint:disable-next-line ban-ts-ignore
+// @ts-ignore
+import WordNet from './wordnet/wordnet';
+
+
+export interface Definition {
+  definition: string;
+  synonyms: string[];
+}
+
+
+export class DefinitionDictionary {
+  private wordnet: any;
+  private extendedWords: Record<string, Definition[]> = {};
+
+  constructor(basePath: string) {
+    const dataDir = path.join(basePath, 'assets', 'wordnet-dictionary');
+
+    console.log('MOO MOO', process.cwd(), dataDir);
+
+    // tslint:disable-next-line:ban-ts-ignore
+    // @ts-ignore
+    this.wordnet = new WordNet(
+      {
+        dataDir
+      }
+    );
+  }
+
+
+  async getDefinition(expression: string): Promise<Definition[]> {
+    const exp = this.cleanExpression(expression);
+    const forms = await this.getWordNetValidForms(exp);
+
+    const results = await Promise.all(
+      [
+        this.getExtendedDefinitions(forms),
+        this.getWordNetDefinitions(forms)
+      ]
+    );
+
+    return _.flatten(results);
+  }
+
+
+  private async getExtendedDefinitions(forms: string[]) {
+    return _.flatten(_.map(forms, (f) => this.extendedWords[f] || []));
+  }
+
+
+  private async getWordNetValidForms(expression: string): Promise<string[]> {
+    return  [...(await this.wordnet.validFormsAsync(expression)), expression];
+  }
+
+
+  private async getWordNetDefinitions(forms: string[]): Promise<Definition[]> {
+    const definitions = await Promise.all(_.map(forms, (f: string) => this.wordnet.lookupAsync(f).catch(() => [])));
+
+    return _.map(
+      _.reverse(_.flatten(definitions)),
+      (r) => (
+        {
+          definition: r.def,
+          synonyms: r.synonyms
+        } as any as Definition
+      )
+    );
+  }
+
+
+  private cleanExpression(expression: string): string {
+    return anyAscii(expression).toLowerCase().replace(/[^a-z0-9\-]/g, ' ').replace(/  +/g, ' ').trim();
+  }
+}

learn/dictionary/wordnet/data_file.js

@@ -0,0 +1,79 @@
+var DataFile, WordNetFile, fs, util,
+  extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; },
+  hasProp = {}.hasOwnProperty;
+
+WordNetFile = require('./wordnet_file');
+
+fs = require('fs');
+
+util = require('util');
+
+module.exports = DataFile = (function(superClass) {
+  extend(DataFile, superClass);
+
+  function DataFile(dataDir, name) {
+    DataFile.__super__.constructor.call(this, dataDir, 'data.' + name);
+  }
+
+  DataFile.prototype.get = function(location, callback) {
+    var buff, self;
+    self = this;
+    buff = new Buffer(4096);
+    return this.open(function(err, fd) {
+      if (err != null) {
+        return callback.call(self, err, null);
+      }
+      return this.appendLineChar(fd, location, 0, buff, function(err, line) {
+        var base, data, definition, element, examples, glossArray, i, j, k, l, len, m, ptrOffset, ptrs, ref, ref1, synonyms, synsetOffset, tokens, wCnt;
+        if (err != null) {
+          return callback.call(self, err, null);
+        }
+        data = line.split('| ');
+        tokens = data[0].split(/\s+/);
+        ptrs = [];
+        wCnt = parseInt(tokens[3], 16);
+        synonyms = [];
+        for (i = j = 0, ref = wCnt - 1; j <= ref; i = j += 1) {
+          synonyms.push(tokens[4 + i * 2]);
+        }
+        ptrOffset = (wCnt - 1) * 2 + 6;
+        for (i = l = 0, ref1 = parseInt(tokens[ptrOffset], 10) - 1; l <= ref1; i = l += 1) {
+          base = i * 4 + ptrOffset;
+          ptrs.push({
+            pointerSymbol: tokens[base + 1],
+            synsetOffset: parseInt(tokens[base + 2], 10),
+            pos: tokens[base + 3],
+            sourceTarget: tokens[base + 4]
+          });
+        }
+        glossArray = data[1].split("; ");
+        definition = glossArray[0];
+        examples = glossArray.slice(1);
+        for (k = m = 0, len = examples.length; m < len; k = ++m) {
+          element = examples[k];
+          examples[k] = examples[k].replace(/\"/g, '').replace(/\s\s+/g, '');
+        }
+        synsetOffset = parseInt(tokens[0], 10);
+        if (synsetOffset !== location) {
+          return callback.call(self, "Invalid synsetOffset: " + location, null);
+        }
+        return callback.call(self, null, {
+          synsetOffset: parseInt(tokens[0], 10),
+          lexFilenum: parseInt(tokens[1], 10),
+          pos: tokens[2],
+          wCnt: wCnt,
+          lemma: tokens[4],
+          synonyms: synonyms,
+          lexId: tokens[5],
+          ptrs: ptrs,
+          gloss: data[1],
+          def: definition,
+          exp: examples
+        });
+      });
+    });
+  };
+
+  return DataFile;
+
+})(WordNetFile);

learn/dictionary/wordnet/index_file.js

@@ -0,0 +1,143 @@
+var IndexFile, WordNetFile, fs, util,
+  extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; },
+  hasProp = {}.hasOwnProperty;
+
+WordNetFile = require('./wordnet_file');
+
+fs = require('fs');
+
+util = require('util');
+
+module.exports = IndexFile = (function(superClass) {
+  var _findAt, _findPrevEOL, _getFileSize, _readLine;
+
+  extend(IndexFile, superClass);
+
+  function IndexFile(dataDir, name) {
+    IndexFile.__super__.constructor.call(this, dataDir, 'index.' + name);
+  }
+
+  _findPrevEOL = function(self, fd, pos, callback) {
+    var buff;
+    buff = new Buffer(1024);
+    if (pos === 0) {
+      return callback(null, 0);
+    } else {
+      return fs.read(fd, buff, 0, 1, pos, function(err, count) {
+        if (err != null) {
+          return callback(err, count);
+        }
+        if (buff[0] === 10) {
+          return callback(null, pos + 1);
+        } else {
+          return _findPrevEOL(self, fd, pos - 1, callback);
+        }
+      });
+    }
+  };
+
+  _readLine = function(self, fd, pos, callback) {
+    var buff;
+    buff = new Buffer(1024);
+    return _findPrevEOL(self, fd, pos, function(err, pos) {
+      if (err != null) {
+        return callback(err, pos);
+      }
+      return self.appendLineChar(fd, pos, 0, buff, callback);
+    });
+  };
+
+  _findAt = function(self, fd, size, pos, lastPos, adjustment, searchKey, callback, lastKey) {
+    if (lastPos === pos || pos >= size) {
+      return callback(null, {
+        status: 'miss'
+      });
+    } else {
+      return _readLine(self, fd, pos, function(err, line) {
+        var key, tokens;
+        if (err != null) {
+          return callback(err);
+        }
+        tokens = line.split(/\s+/);
+        key = tokens[0];
+        if (key === searchKey) {
+          return callback(null, {
+            status: 'hit',
+            key: key,
+            'line': line,
+            tokens: tokens
+          });
+        } else if (adjustment === 1 || key === lastKey) {
+          return callback(null, {
+            status: 'miss'
+          });
+        } else {
+          adjustment = Math.ceil(adjustment * 0.5);
+          if (key < searchKey) {
+            return _findAt(self, fd, size, pos + adjustment, pos, adjustment, searchKey, callback, key);
+          } else {
+            return _findAt(self, fd, size, pos - adjustment, pos, adjustment, searchKey, callback, key);
+          }
+        }
+      });
+    }
+  };
+
+  _getFileSize = function(path) {
+    var stat;
+    stat = fs.statSync(path);
+    return stat.size;
+  };
+
+  IndexFile.prototype.find = function(searchKey, callback) {
+    var self;
+    self = this;
+    return this.open(function(err, fd) {
+      var pos, size;
+      if (err != null) {
+        return callback(err, null);
+      }
+      size = _getFileSize(this.filePath) - 1;
+      pos = Math.ceil(size / 2);
+      return _findAt(self, fd, size, pos, null, pos, searchKey, function(err, result) {
+        return callback.call(self, err, result);
+      });
+    });
+  };
+
+  IndexFile.prototype.lookupFromFile = function(word, callback) {
+    return this.find(word, function(err, record) {
+      var i, indexRecord, j, k, offsets, ptrs, ref, ref1;
+      if (err != null) {
+        return callback.call(this, err, null);
+      }
+      indexRecord = null;
+      if (record.status === 'hit') {
+        ptrs = [];
+        offsets = [];
+        for (i = j = 0, ref = parseInt(record.tokens[3]) - 1; j <= ref; i = j += 1) {
+          ptrs.push(record.tokens[i]);
+        }
+        for (i = k = 0, ref1 = parseInt(record.tokens[2]) - 1; k <= ref1; i = k += 1) {
+          offsets.push(parseInt(record.tokens[ptrs.length + 6 + i], 10));
+        }
+        indexRecord = {
+          lemma: record.tokens[0],
+          pos: record.tokens[1],
+          ptrSymbol: ptrs,
+          senseCnt: parseInt(record.tokens[ptrs.length + 4], 10),
+          tagsenseCnt: parseInt(record.tokens[ptrs.length + 5], 10),
+          synsetOffset: offsets
+        };
+      }
+      return callback.call(this, null, indexRecord);
+    });
+  };
+
+  IndexFile.prototype.lookup = function(word, callback) {
+    return this.lookupFromFile(word, callback);
+  };
+
+  return IndexFile;
+
+})(WordNetFile);

learn/dictionary/wordnet/wordnet.js

@@ -0,0 +1,681 @@
+var DataFile, IndexFile, LRU, Promise, WordNet, async, fs, path,
+  slice = [].slice;
+
+IndexFile = require('./index_file');
+
+DataFile = require('./data_file');
+
+async = require('async');
+
+Promise = require('bluebird');
+
+path = require('path');
+
+fs = require('fs');
+
+LRU = require('lru-cache');
+
+WordNet = (function() {
+  var _forms, _loadExceptions, _validForms, _validFormsWithExceptions, exceptions, forms, tokenDetach, unique;
+
+  function WordNet(options) {
+    var WNdb, e, error;
+    if (typeof options === 'string') {
+      options = {
+        dataDir: options
+      };
+    } else {
+      if (options == null) {
+        options = {};
+      }
+    }
+    if (options.dataDir == null) {
+      try {
+        WNdb = require('wndb-with-exceptions');
+      } catch (error) {
+        e = error;
+        console.error("Please 'npm install wndb-with-exceptions' before using WordNet module or specify a dict directory.");
+        throw e;
+      }
+      options.dataDir = WNdb.path;
+    }
+    if (!options.cache) {
+      this.cache = null;
+    } else {
+      if (options.cache === true) {
+        options.cache = {
+          max: 2000
+        };
+      }
+      if (typeof options.cache === 'object' && typeof options.cache.get === 'function') {
+        this.cache = options.cache;
+      } else {
+        this.cache = LRU(options.cache);
+      }
+    }
+    this.path = options.dataDir;
+    this.nounIndex = new IndexFile(this.path, 'noun');
+    this.verbIndex = new IndexFile(this.path, 'verb');
+    this.adjIndex = new IndexFile(this.path, 'adj');
+    this.advIndex = new IndexFile(this.path, 'adv');
+    this.nounData = new DataFile(this.path, 'noun');
+    this.verbData = new DataFile(this.path, 'verb');
+    this.adjData = new DataFile(this.path, 'adj');
+    this.advData = new DataFile(this.path, 'adv');
+    this.allFiles = [
+      {
+        index: this.nounIndex,
+        data: this.nounData,
+        pos: 'n'
+      }, {
+        index: this.verbIndex,
+        data: this.verbData,
+        pos: 'v'
+      }, {
+        index: this.adjIndex,
+        data: this.adjData,
+        pos: 'a'
+      }, {
+        index: this.advIndex,
+        data: this.advData,
+        pos: 'r'
+      }
+    ];
+  }
+
+  WordNet.prototype.get = function(synsetOffset, pos, callback) {
+    var dataFile, hit, query, wordnet;
+    wordnet = this;
+    if (this.cache) {
+      query = "get:" + synsetOffset + ":" + pos;
+      if (hit = wordnet.cache.get(query)) {
+        if (callback.length === 1) {
+          return callback.call(wordnet, hit);
+        } else {
+          return callback.call(wordnet, null, hit);
+        }
+      }
+    }
+    dataFile = wordnet.getDataFile(pos);
+    return dataFile.get(synsetOffset, function(err, result) {
+      if (query && (err == null)) {
+        wordnet.cache.set(query, result);
+      }
+      if (callback.length === 1) {
+        return callback.call(wordnet, result);
+      } else {
+        return callback.call(wordnet, err, result);
+      }
+    });
+  };
+
+  WordNet.prototype.getAsync = function(synsetOffset, pos) {
+    var wordnet;
+    wordnet = this;
+    return new Promise(function(resolve, reject) {
+      return wordnet.get(synsetOffset, pos, function(err, data) {
+        if (err != null) {
+          return reject(err);
+        } else {
+          return resolve(data);
+        }
+      });
+    });
+  };
+
+  WordNet.prototype.lookup = function(input, callback) {
+    var hit, lword, pos, query, ref, selectedFiles, word, wordnet;
+    wordnet = this;
+    ref = input.split('#'), word = ref[0], pos = ref[1];
+    lword = word.toLowerCase().replace(/\s+/g, '_');
+    if (this.cache) {
+      query = "lookup:" + input;
+      if (hit = wordnet.cache.get(query)) {
+        if (callback.length === 1) {
+          return callback.call(wordnet, hit);
+        } else {
+          return callback.call(wordnet, null, hit);
+        }
+      }
+    }
+    selectedFiles = !pos ? wordnet.allFiles.slice() : wordnet.allFiles.filter(function(file) {
+      return file.pos === pos;
+    });
+    return wordnet.lookupFromFiles(selectedFiles, [], lword, function(err, results) {
+      if (err != null) {
+        return callback.call(wordnet, err);
+      }
+      if (query) {
+        wordnet.cache.set(query, results);
+      }
+      if (callback.length === 1) {
+        return callback.call(wordnet, results);
+      } else {
+        return callback.call(wordnet, null, results);
+      }
+    });
+  };
+
+  WordNet.prototype.lookupAsync = function(input, callback) {
+    var wordnet;
+    wordnet = this;
+    return new Promise(function(resolve, reject) {
+      return wordnet.lookup(input, function(err, data) {
+        if (err != null) {
+          return reject(err);
+        } else {
+          return resolve(data);
+        }
+      });
+    });
+  };
+
+  WordNet.prototype.findSense = function(input, callback) {
+    var hit, lword, pos, query, ref, selectedFiles, sense, senseNumber, word, wordnet;
+    wordnet = this;
+    ref = input.split('#'), word = ref[0], pos = ref[1], senseNumber = ref[2];
+    if (this.cache) {
+      query = "findSense:" + input;
+      if (hit = wordnet.cache.get(query)) {
+        if (callback.length === 1) {
+          return callback.call(wordnet, hit);
+        } else {
+          return callback.call(wordnet, null, hit);
+        }
+      }
+    }
+    sense = parseInt(senseNumber);
+    if (Number.isNaN(sense)) {
+      throw new Error("Sense number should be an integer");
+    } else if (sense < 1) {
+      throw new Error("Sense number should be a positive integer");
+    }
+    lword = word.toLowerCase().replace(/\s+/g, '_');
+    selectedFiles = wordnet.allFiles.filter(function(file) {
+      return file.pos === pos;
+    });
+    return wordnet.lookupFromFiles(selectedFiles, [], lword, function(err, response) {
+      var result;
+      if (err != null) {
+        return callback.call(wordnet, err);
+      }
+      result = response[sense - 1];
+      if (query) {
+        wordnet.cache.set(query, result);
+      }
+      if (callback.length === 1) {
+        return callback.call(wordnet, result);
+      } else {
+        return callback.call(wordnet, null, result);
+      }
+    });
+  };
+
+  WordNet.prototype.findSenseAsync = function(input) {
+    var wordnet;
+    wordnet = this;
+    return new Promise(function(resolve, reject) {
+      return wordnet.findSense(input, function(err, data) {
+        if (err != null) {
+          return reject(err);
+        } else {
+          return resolve(data);
+        }
+      });
+    });
+  };
+
+  WordNet.prototype.querySense = function(input, callback) {
+    var hit, pos, query, ref, word, wordnet;
+    wordnet = this;
+    ref = input.split('#'), word = ref[0], pos = ref[1];
+    if (this.cache) {
+      query = "querySense:" + input;
+      if (hit = wordnet.cache.get(query)) {
+        if (callback.length === 1) {
+          return callback.call(wordnet, hit);
+        } else {
+          return callback.call(wordnet, null, hit);
+        }
+      }
+    }
+    return wordnet.lookup(input, function(err, results) {
+      var i, sense, senseCounts, senses;
+      if (err != null) {
+        return callback.call(wordnet, err);
+      }
+      senseCounts = {};
+      senses = (function() {
+        var j, len, results1;
+        results1 = [];
+        for (i = j = 0, len = results.length; j < len; i = ++j) {
+          sense = results[i];
+          pos = sense.pos;
+          if (pos === 's') {
+            pos = 'a';
+          }
+          if (senseCounts[pos] == null) {
+            senseCounts[pos] = 1;
+          }
+          results1.push(word + "#" + pos + "#" + senseCounts[pos]++);
+        }
+        return results1;
+      })();
+      if (query) {
+        wordnet.cache.set(query, senses);
+      }
+      if (callback.length === 1) {
+        return callback.call(wordnet, senses);
+      } else {
+        return callback.call(wordnet, null, senses);
+      }
+    });
+  };
+
+  WordNet.prototype.querySenseAsync = function(input) {
+    var wordnet;
+    wordnet = this;
+    return new Promise(function(resolve, reject) {
+      return wordnet.querySense(input, function(err, data) {
+        if (err != null) {
+          return reject(err);
+        } else {
+          return resolve(data);
+        }
+      });
+    });
+  };
+
+  WordNet.prototype.lookupFromFiles = function(files, results, word, callback) {
+    var file, wordnet;
+    wordnet = this;
+    if (files.length === 0) {
+      return callback.call(wordnet, null, results);
+    } else {
+      file = files.pop();
+      return file.index.lookup(word, function(err, record) {
+        if (record) {
+          return wordnet.pushResults(file.data, results, record.synsetOffset, function() {
+            return wordnet.lookupFromFiles(files, results, word, callback);
+          });
+        } else {
+          return wordnet.lookupFromFiles(files, results, word, callback);
+        }
+      });
+    }
+  };
+
+  WordNet.prototype.pushResults = function(data, results, offsets, callback) {
+    var wordnet;
+    wordnet = this;
+    if (offsets.length === 0) {
+      return callback(results);
+    } else {
+      return data.get(offsets.pop(), function(err, record) {
+        results.push(record);
+        return wordnet.pushResults(data, results, offsets, callback);
+      });
+    }
+  };
+
+  WordNet.prototype.loadResultSynonyms = function(synonyms, results, callback) {
+    var result, wordnet;
+    wordnet = this;
+    if (results.length > 0) {
+      result = results.pop();
+      return wordnet.loadSynonyms(synonyms, results, result.ptrs, callback);
+    } else {
+      return callback(synonyms);
+    }
+  };
+
+  WordNet.prototype.loadSynonyms = function(synonyms, results, ptrs, callback) {
+    var ptr, wordnet;
+    wordnet = this;
+    if (ptrs.length > 0) {
+      ptr = ptrs.pop();
+      return this.get(ptr.synsetOffset, ptr.pos, function(result) {
+        synonyms.push(result);
+        return wordnet.loadSynonyms(synonyms, results, ptrs, callback);
+      });
+    } else {
+      return wordnet.loadResultSynonyms(synonyms, results, callback);
+    }
+  };
+
+  WordNet.prototype.lookupSynonyms = function(word, callback) {
+    var wordnet;
+    wordnet = this;
+    return wordnet.lookup(word, function(results) {
+      return wordnet.loadResultSynonyms([], results, callback);
+    });
+  };
+
+  WordNet.prototype.getSynonyms = function() {
+    var callback, pos, synsetOffset, wordnet;
+    wordnet = this;
+    callback = arguments[2] ? arguments[2] : arguments[1];
+    pos = arguments[0].pos ? arguments[0].pos : arguments[1];
+    synsetOffset = arguments[0].synsetOffset ? arguments[0].synsetOffset : arguments[0];
+    return this.get(synsetOffset, pos, function(result) {
+      return wordnet.loadSynonyms([], [], result.ptrs, callback);
+    });
+  };
+
+  WordNet.prototype.getDataFile = function(pos) {
+    switch (pos) {
+      case 'n':
+        return this.nounData;
+      case 'v':
+        return this.verbData;
+      case 'a':
+      case 's':
+        return this.adjData;
+      case 'r':
+        return this.advData;
+    }
+  };
+
+  exceptions = [
+    {
+      name: "noun.exc",
+      pos: 'n'
+    }, {
+      name: "verb.exc",
+      pos: 'v'
+    }, {
+      name: "adj.exc",
+      pos: 'a'
+    }, {
+      name: "adv.exc",
+      pos: 'r'
+    }
+  ];
+
+  _loadExceptions = function(wordnet, callback) {
+    var loadFile;
+    WordNet.prototype.exceptions = 'pending';
+    loadFile = function(exception, callback) {
+      var fullPath;
+      fullPath = path.join(wordnet.path, exception.name);
+      return fs.readFile(fullPath, function(err, data) {
+        var j, len, line, lines, ref, temp, term1, term2;
+        if (err) {
+          return callback(err);
+        }
+        temp = {};
+        lines = data.toString().split("\n");
+        for (j = 0, len = lines.length; j < len; j++) {
+          line = lines[j];
+          if (line.length > 0) {
+            ref = line.split(' '), term1 = ref[0], term2 = 2 <= ref.length ? slice.call(ref, 1) : [];
+            if (temp[term1] == null) {
+              temp[term1] = [];
+            }
+            Array.prototype.push.apply(temp[term1], term2);
+          }
+        }
+        return callback(null, {
+          pos: exception.pos,
+          data: temp
+        });
+      });
+    };
+    return async.map(exceptions, loadFile, function(err, results) {
+      var j, len, result;
+      exceptions = {};
+      for (j = 0, len = results.length; j < len; j++) {
+        result = results[j];
+        exceptions[result.pos] = result.data;
+      }
+      WordNet.prototype.exceptions = exceptions;
+      return callback();
+    });
+  };
+
+  WordNet.prototype.close = function() {
+    this.nounIndex.close();
+    this.verbIndex.close();
+    this.adjIndex.close();
+    this.advIndex.close();
+    this.nounData.close();
+    this.verbData.close();
+    this.adjData.close();
+    return this.advData.close();
+  };
+
+  unique = function(a) {
+    var found;
+    found = {};
+    return a.filter(function(item) {
+      if (found[item]) {
+        return false;
+      } else {
+        return found[item] = true;
+      }
+    });
+  };
+
+  tokenDetach = function(string) {
+    var detach, length, pos, ref, sense, word;
+    ref = string.split('#'), word = ref[0], pos = ref[1], sense = ref[2];
+    detach = [word];
+    length = word.length;
+    switch (pos) {
+      case 'n':
+        if (word.endsWith("s")) {
+          detach.push(word.substring(0, length - 1));
+        }
+        if (word.endsWith("ses")) {
+          detach.push(word.substring(0, length - 2));
+        }
+        if (word.endsWith("xes")) {
+          detach.push(word.substring(0, length - 2));
+        }
+        if (word.endsWith("zes")) {
+          detach.push(word.substring(0, length - 2));
+        }
+        if (word.endsWith("ches")) {
+          detach.push(word.substring(0, length - 2));
+        }
+        if (word.endsWith("shes")) {
+          detach.push(word.substring(0, length - 2));
+        }
+        if (word.endsWith("men")) {
+          detach.push(word.substring(0, length - 3) + "man");
+        }
+        if (word.endsWith("ies")) {
+          detach.push(word.substring(0, length - 3) + "y");
+        }
+        break;
+      case 'v':
+        if (word.endsWith("s")) {
+          detach.push(word.substring(0, length - 1));
+        }
+        if (word.endsWith("ies")) {
+          detach.push(word.substring(0, length - 3) + "y");
+        }
+        if (word.endsWith("es")) {
+          detach.push(word.substring(0, length - 2));
+        }
+        if (word.endsWith("ed")) {
+          detach.push(word.substring(0, length - 1));
+        }
+        if (word.endsWith("ed")) {
+          detach.push(word.substring(0, length - 2));
+        }
+        if (word.endsWith("ing")) {
+          detach.push(word.substring(0, length - 3) + "e");
+        }
+        if (word.endsWith("ing")) {
+          detach.push(word.substring(0, length - 3));
+        }
+        break;
+      case 'r':
+        if (word.endsWith("er")) {
+          detach.push(word.substring(0, length - 2));
+        }
+        if (word.endsWith("er")) {
+          detach.push(word.substring(0, length - 1));
+        }
+        if (word.endsWith("est")) {
+          detach.push(word.substring(0, length - 3));
+        }
+        if (word.endsWith("est")) {
+          detach.push(word.substring(0, length - 2));
+        }
+    }
+    return unique(detach);
+  };
+
+  _forms = function(wordnet, word, pos) {
+    var colloc, exception, forms, i, index, j, lword, ref, ref1, rtn, token, tokens;
+    lword = word.toLowerCase();
+    exception = (ref = wordnet.exceptions[pos]) != null ? ref[lword] : void 0;
+    if (exception) {
+      return [word].concat(exception);
+    }
+    tokens = word.split(/[ _]/g);
+    if (tokens.length === 1) {
+      return tokenDetach(tokens[0] + "#" + pos);
+    }
+    forms = tokens.map(function(token) {
+      return _forms(wordnet, token, pos);
+    });
+    rtn = [];
+    index = (function() {
+      var j, len, results1;
+      results1 = [];
+      for (j = 0, len = tokens.length; j < len; j++) {
+        token = tokens[j];
+        results1.push(0);
+      }
+      return results1;
+    })();
+    while (true) {
+      colloc = forms[0][index[0]];
+      for (i = j = 1, ref1 = tokens.length - 1; 1 <= ref1 ? j <= ref1 : j >= ref1; i = 1 <= ref1 ? ++j : --j) {
+        colloc = colloc + '_' + forms[i][index[i]];
+      }
+      rtn.push(colloc);
+      i = 0;
+      while (i < tokens.length) {
+        index[i] = index[i] + 1;
+        if (index[i] < forms[i].length) {
+          break;
+        } else {
+          index[i] = 0;
+        }
+        i = i + 1;
+      }
+      if (i >= tokens.length) {
+        break;
+      }
+    }
+    return rtn;
+  };
+
+  forms = function(wordnet, string) {
+    var element, j, len, pos, ref, results1, rtn, sense, word;
+    ref = string.split('#'), word = ref[0], pos = ref[1], sense = ref[2];
+    rtn = _forms(wordnet, word, pos);
+    results1 = [];
+    for (j = 0, len = rtn.length; j < len; j++) {
+      element = rtn[j];
+      results1.push(element + "#" + pos);
+    }
+    return results1;
+  };
+
+  _validForms = function(wordnet, string, callback) {
+    var eachFn, filteredResults, pos, possibleForms, reducer, ref, sense, word;
+    ref = string.split('#'), word = ref[0], pos = ref[1], sense = ref[2];
+    if (!pos) {
+      reducer = function(previous, current, next) {
+        return _validForms(wordnet, string + "#" + current, function(err, value) {
+          if (value === void 0) {
+            return next(null, previous);
+          } else {
+            return next(null, previous.concat(value));
+          }
+        });
+      };
+      return async.reduce(['n', 'v', 'a', 'r'], [], reducer, function(err, result) {
+        return callback(null, result);
+      });
+    } else {
+      possibleForms = forms(wordnet, word + "#" + pos);
+      filteredResults = [];
+      eachFn = function(term, done) {
+        return wordnet.lookup(term, function(err, data) {
+          if (err != null) {
+            return done(err);
+          }
+          if (data.length > 0) {
+            filteredResults.push(term);
+          }
+          return done();
+        });
+      };
+      return async.each(possibleForms, eachFn, function(err) {
+        return callback(err, filteredResults);
+      });
+    }
+  };
+
+  _validFormsWithExceptions = function(wordnet, string, callback) {
+    if (wordnet.exceptions === void 0) {
+      return _loadExceptions(wordnet, function() {
+        return _validFormsWithExceptions(wordnet, string, callback);
+      });
+    } else if (wordnet.exceptions === 'pending') {
+      return setImmediate(_validFormsWithExceptions, wordnet, string, callback);
+    } else {
+      return _validForms(wordnet, string, callback);
+    }
+  };
+
+  WordNet.prototype.validForms = function(string, callback) {
+    var hit, query, wordnet;
+    wordnet = this;
+    if (this.cache) {
+      query = "validForms:" + string;
+      if (hit = wordnet.cache.get(query)) {
+        if (callback.length === 1) {
+          return callback.call(wordnet, hit);
+        } else {
+          return callback.call(wordnet, null, hit);
+        }
+      }
+    }
+    return _validFormsWithExceptions(this, string, function(err, result) {
+      if (query) {
+        wordnet.cache.set(query, result);
+      }
+      if (callback.length === 1) {
+        return callback.call(wordnet, result);
+      } else {
+        return callback.call(wordnet, null, result);
+      }
+    });
+  };
+
+  WordNet.prototype.validFormsAsync = function(string) {
+    return new Promise((function(_this) {
+      return function(resolve, reject) {
+        return _this.validForms(string, function(err, data) {
+          if (err != null) {
+            return reject(err);
+          } else {
+            return resolve(data);
+          }
+        });
+      };
+    })(this));
+  };
+
+  return WordNet;
+
+})();
+
+module.exports = WordNet;

learn/dictionary/wordnet/wordnet_file.js

@@ -0,0 +1,64 @@
+var WordNetFile, fs, path, util;
+
+fs = require('fs');
+
+path = require('path');
+
+util = require('util');
+
+module.exports = WordNetFile = (function() {
+  function WordNetFile(dataDir, fileName) {
+    this.dataDir = dataDir;
+    this.fileName = fileName;
+    this.filePath = path.join(this.dataDir, this.fileName);
+  }
+
+  WordNetFile.prototype.open = function(callback) {
+    var filePath, self;
+    self = this;
+    if (this.fd) {
+      return callback.call(self, null, this.fd);
+    }
+    filePath = this.filePath;
+    return fs.open(filePath, 'r', null, (function(_this) {
+      return function(err, fd) {
+        if (err != null) {
+          return callback.call(self, err, null);
+        }
+        _this.fd = fd;
+        return callback.call(self, err, fd);
+      };
+    })(this));
+  };
+
+  WordNetFile.prototype.close = function() {
+    if (this.fd != null) {
+      fs.close(this.fd);
+      return delete this.fd;
+    }
+  };
+
+  WordNetFile.prototype.appendLineChar = function(fd, pos, buffPos, buff, callback) {
+    var length, self, space;
+    self = this;
+    length = buff.length;
+    space = length - buffPos;
+    return fs.read(fd, buff, buffPos, space, pos, function(err, count, buffer) {
+      var i, j, newBuff, ref;
+      if (err != null) {
+        return callback.call(self, err, null);
+      }
+      for (i = j = 0, ref = count - 1; 0 <= ref ? j <= ref : j >= ref; i = 0 <= ref ? ++j : --j) {
+        if (buff[i] === 10) {
+          return callback.call(self, null, buff.slice(0, i).toString('ASCII'));
+        }
+      }
+      newBuff = new Buffer(length * 2);
+      buff.copy(newBuff, 0, 0, length);
+      return self.appendLineChar(fd, pos + length, length, newBuff, callback);
+    });
+  };
+
+  return WordNetFile;
+
+})();

package.json

@@ -17,6 +17,7 @@
     "@types/sortablejs": "^1.10.6",
     "@vue/devtools": "^5.3.3",
     "any-ascii": "^0.1.7",
+    "async": "^0.9.0",
     "axios": "^0.21.0",
     "bluebird": "^3.7.2",
     "bootstrap": "^4.5.3",
@@ -30,7 +31,9 @@
     "extract-loader": "^5.1.0",
     "file-loader": "^6.2.0",
     "lodash": "^4.17.20",
+    "lru-cache": "^2.5.0",
     "node-sass": "^4.14.1",
+    "node-wordnet": "^0.1.12",
     "optimize-css-assets-webpack-plugin": "^5.0.4",
     "qs": "^6.9.4",
     "raven-js": "^3.27.2",
@@ -48,7 +51,8 @@
     "vue": "^2.6.12",
     "vue-loader": "^15.9.4",
     "vue-template-compiler": "^2.6.12",
-    "webpack": "^5.3.2"
+    "webpack": "^5.3.2",
+    "wndb-with-exceptions": "^3.0.2"
   },
   "dependencies": {
     "@cliqz/adblocker-electron": "^1.18.3",

yarn.lock

@@ -790,6 +790,11 @@ async-limiter@~1.0.0:
   resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd"
   integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==
 
+async@^0.9.0:
+  version "0.9.2"
+  resolved "https://registry.yarnpkg.com/async/-/async-0.9.2.tgz#aea74d5e61c1f899613bf64bda66d4c78f2fd17d"
+  integrity sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=
+
 async@^1.4.2:
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a"
@@ -1441,6 +1446,11 @@ block-stream@*:
   dependencies:
     inherits "~2.0.0"
 
+bluebird@^2.6.0:
+  version "2.11.0"
+  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-2.11.0.tgz#534b9033c022c9579c56ba3b3e5a5caafbb650e1"
+  integrity sha1-U0uQM8AiyVecVro7Plpcqvu2UOE=
+
 bluebird@^3.1.1, bluebird@^3.5.0, bluebird@^3.7.2:
   version "3.7.2"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
@@ -2799,6 +2809,11 @@ es6-error@^4.1.1:
   resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d"
   integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==
 
+es6-shim@^0.22.1:
+  version "0.22.2"
+  resolved "https://registry.yarnpkg.com/es6-shim/-/es6-shim-0.22.2.tgz#4afe7c54bb92b178e5d464dd5d9c31c5742d7521"
+  integrity sha1-Sv58VLuSsXjl1GTdXZwxxXQtdSE=
+
 escalade@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
@@ -4455,6 +4470,11 @@ lowercase-keys@^2.0.0:
   resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479"
   integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==
 
+lru-cache@^2.5.0:
+  version "2.7.3"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.7.3.tgz#6d4524e8b955f95d4f5b58851ce21dd72fb4e952"
+  integrity sha1-bUUk6LlV+V1PW1iFHOId1y+06VI=
+
 lru-cache@^4.0.1, lru-cache@^4.1.2:
   version "4.1.5"
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd"
@@ -4926,6 +4946,16 @@ node-sass@^4.14.1:
     stdout-stream "^1.4.0"
     "true-case-path" "^1.0.2"
 
+node-wordnet@^0.1.12:
+  version "0.1.12"
+  resolved "https://registry.yarnpkg.com/node-wordnet/-/node-wordnet-0.1.12.tgz#62b0568f5dfc188bd02d136cf730832952fa9c1b"
+  integrity sha1-YrBWj138GIvQLRNs9zCDKVL6nBs=
+  dependencies:
+    async "^0.9.0"
+    bluebird "^2.6.0"
+    es6-shim "^0.22.1"
+    lru-cache "^2.5.0"
+
 noop-logger@^0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/noop-logger/-/noop-logger-0.1.1.tgz#94a2b1633c4f1317553007d8966fd0e841b6a4c2"
@@ -6952,6 +6982,18 @@ tar@^6.0.2, tar@^6.0.5:
     mkdirp "^1.0.3"
     yallist "^4.0.0"
 
+tar@latest:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.0.tgz#d1724e9bcc04b977b18d5c573b333a2207229a83"
+  integrity sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA==
+  dependencies:
+    chownr "^2.0.0"
+    fs-minipass "^2.0.0"
+    minipass "^3.0.0"
+    minizlib "^2.1.1"
+    mkdirp "^1.0.3"
+    yallist "^4.0.0"
+
 temp@^0.9.0:
   version "0.9.4"
   resolved "https://registry.yarnpkg.com/temp/-/temp-0.9.4.tgz#cd20a8580cb63635d0e4e9d4bd989d44286e7620"
@@ -7495,6 +7537,13 @@ wide-align@^1.1.0:
   dependencies:
     string-width "^1.0.2 || 2"
 
+wndb-with-exceptions@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/wndb-with-exceptions/-/wndb-with-exceptions-3.0.2.tgz#7048c1be7e0d13d231a51e408e0b37426000bcbb"
+  integrity sha1-cEjBvn4NE9IxpR5Ajgs3QmAAvLs=
+  dependencies:
+    tar latest
+
 worker-rpc@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/worker-rpc/-/worker-rpc-0.1.1.tgz#cb565bd6d7071a8f16660686051e969ad32f54d5"