This commit is contained in:
Mr. Stallion 2021-01-20 14:22:22 -06:00
parent 28527c1840
commit c168873265
24 changed files with 487203 additions and 5 deletions

View File

@ -1,9 +1,10 @@
# Changelog
## Canary
* Right click a word and select 'Look up...' to see its dictionary definition
* Skip button for auto-post ads
* Image tab on the character profile view now loads up faster
* Search results view is more responsive while still scoring profiles
* Search results view is now more responsive while scoring profiles
* Fixed 'unicorn match' badge color

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,7 @@
best well
better well
deeper deeply
farther far
further far
harder hard
hardest hard

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -94,7 +94,7 @@
async mounted(): Promise<void> {
log.debug('init.window.mounting');
// top bar devtools
// browserWindow.webContents.openDevTools({ mode: 'detach' });
browserWindow.webContents.openDevTools({ mode: 'detach' });
updateSupportedLanguages(browserWindow.webContents.session.availableSpellCheckerLanguages);
@ -255,7 +255,7 @@
);
// tab devtools
// view.webContents.openDevTools();
view.webContents.openDevTools();
// console.log('ADD TAB LANGUAGES', getSafeLanguages(this.settings.spellcheckLang), this.settings.spellcheckLang);
view.webContents.session.setSpellCheckerLanguages(getSafeLanguages(this.settings.spellcheckLang));

View File

@ -52,7 +52,8 @@ import {Logs, SettingsStore} from './filesystem';
import Notifications from './notifications';
import * as SlimcatImporter from './importer';
import Index from './Index.vue';
import log from 'electron-log'; // tslint:disable-line: match-default-export-name
import log from 'electron-log';
import { DefinitionDictionary } from '../learn/dictionary/definition-dictionary'; // tslint:disable-line: match-default-export-name
log.debug('init.chat');
@ -74,6 +75,7 @@ const sc = nativeRequire<{
}>('spellchecker/build/Release/spellchecker.node');
const spellchecker = new sc.Spellchecker();*/
Axios.defaults.params = {__fchat: `desktop/${electron.remote.app.getVersion()}`};
if(process.env.NODE_ENV === 'production') {
@ -105,6 +107,8 @@ function openIncognito(url: string): void {
}
const webContents = electron.remote.getCurrentWebContents();
const wordDef = new DefinitionDictionary(electron.remote.app.getAppPath());
webContents.on('context-menu', (_, props) => {
const hasText = props.selectionText.trim().length > 0;
const can = (type: string) => (<Electron.EditFlags & {[key: string]: boolean}>props.editFlags)[`can${type}`] && hasText;
@ -181,7 +185,23 @@ webContents.on('context-menu', (_, props) => {
click: () => electron.ipcRenderer.send('dictionary-remove', props.selectionText)
}, {type: 'separator'});
if (props.selectionText) {
menuTemplate.unshift(
{ type: 'separator' },
{
label: `Look up '${props.selectionText}'`,
click: async() => {
const words = await wordDef.getDefinition(props.selectionText);
// console.log('WORDS', words);
}
}
);
}
if(menuTemplate.length > 0) electron.remote.Menu.buildFromTemplate(menuTemplate).popup({});
log.debug('context.text', { linkText: props.linkText, misspelledWord: props.misspelledWord, selectionText: props.selectionText, titleText: props.titleText });
});
let dictDir = path.join(electron.remote.app.getPath('userData'), 'spellchecker');

View File

@ -126,6 +126,11 @@ const mainConfig = {
from: path.resolve(__dirname, '..', 'node_modules', '@cliqz', 'adblocker-electron-preload', 'dist', 'preload.cjs.js').replace(/\\/g, '/'),
to: path.join('preview', 'assets', 'adblocker'),
context: path.resolve(__dirname, '..', 'node_modules', '@cliqz', 'adblocker-electron-preload', 'dist')
},
{
from: path.resolve(__dirname, '..', 'assets', '**', '*').replace(/\\/g, '/'),
to: path.join('assets'),
context: path.resolve(__dirname, '..', 'assets')
}
]
}

View File

@ -0,0 +1,81 @@
import _ from 'lodash';
import path from 'path';
// tslint:disable-next-line ban-ts-ignore
// @ts-ignore
import anyAscii from 'any-ascii';
// tslint:disable-next-line ban-ts-ignore
// @ts-ignore
import WordNet from './wordnet/wordnet';
export interface Definition {
definition: string;
synonyms: string[];
}
export class DefinitionDictionary {
private wordnet: any;
private extendedWords: Record<string, Definition[]> = {};
constructor(basePath: string) {
const dataDir = path.join(basePath, 'assets', 'wordnet-dictionary');
console.log('MOO MOO', process.cwd(), dataDir);
// tslint:disable-next-line:ban-ts-ignore
// @ts-ignore
this.wordnet = new WordNet(
{
dataDir
}
);
}
async getDefinition(expression: string): Promise<Definition[]> {
const exp = this.cleanExpression(expression);
const forms = await this.getWordNetValidForms(exp);
const results = await Promise.all(
[
this.getExtendedDefinitions(forms),
this.getWordNetDefinitions(forms)
]
);
return _.flatten(results);
}
private async getExtendedDefinitions(forms: string[]) {
return _.flatten(_.map(forms, (f) => this.extendedWords[f] || []));
}
private async getWordNetValidForms(expression: string): Promise<string[]> {
return [...(await this.wordnet.validFormsAsync(expression)), expression];
}
private async getWordNetDefinitions(forms: string[]): Promise<Definition[]> {
const definitions = await Promise.all(_.map(forms, (f: string) => this.wordnet.lookupAsync(f).catch(() => [])));
return _.map(
_.reverse(_.flatten(definitions)),
(r) => (
{
definition: r.def,
synonyms: r.synonyms
} as any as Definition
)
);
}
private cleanExpression(expression: string): string {
return anyAscii(expression).toLowerCase().replace(/[^a-z0-9\-]/g, ' ').replace(/ +/g, ' ').trim();
}
}

View File

@ -0,0 +1,79 @@
var DataFile, WordNetFile, fs, util,
extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; },
hasProp = {}.hasOwnProperty;
WordNetFile = require('./wordnet_file');
fs = require('fs');
util = require('util');
module.exports = DataFile = (function(superClass) {
extend(DataFile, superClass);
function DataFile(dataDir, name) {
DataFile.__super__.constructor.call(this, dataDir, 'data.' + name);
}
DataFile.prototype.get = function(location, callback) {
var buff, self;
self = this;
buff = new Buffer(4096);
return this.open(function(err, fd) {
if (err != null) {
return callback.call(self, err, null);
}
return this.appendLineChar(fd, location, 0, buff, function(err, line) {
var base, data, definition, element, examples, glossArray, i, j, k, l, len, m, ptrOffset, ptrs, ref, ref1, synonyms, synsetOffset, tokens, wCnt;
if (err != null) {
return callback.call(self, err, null);
}
data = line.split('| ');
tokens = data[0].split(/\s+/);
ptrs = [];
wCnt = parseInt(tokens[3], 16);
synonyms = [];
for (i = j = 0, ref = wCnt - 1; j <= ref; i = j += 1) {
synonyms.push(tokens[4 + i * 2]);
}
ptrOffset = (wCnt - 1) * 2 + 6;
for (i = l = 0, ref1 = parseInt(tokens[ptrOffset], 10) - 1; l <= ref1; i = l += 1) {
base = i * 4 + ptrOffset;
ptrs.push({
pointerSymbol: tokens[base + 1],
synsetOffset: parseInt(tokens[base + 2], 10),
pos: tokens[base + 3],
sourceTarget: tokens[base + 4]
});
}
glossArray = data[1].split("; ");
definition = glossArray[0];
examples = glossArray.slice(1);
for (k = m = 0, len = examples.length; m < len; k = ++m) {
element = examples[k];
examples[k] = examples[k].replace(/\"/g, '').replace(/\s\s+/g, '');
}
synsetOffset = parseInt(tokens[0], 10);
if (synsetOffset !== location) {
return callback.call(self, "Invalid synsetOffset: " + location, null);
}
return callback.call(self, null, {
synsetOffset: parseInt(tokens[0], 10),
lexFilenum: parseInt(tokens[1], 10),
pos: tokens[2],
wCnt: wCnt,
lemma: tokens[4],
synonyms: synonyms,
lexId: tokens[5],
ptrs: ptrs,
gloss: data[1],
def: definition,
exp: examples
});
});
});
};
return DataFile;
})(WordNetFile);

View File

@ -0,0 +1,143 @@
var IndexFile, WordNetFile, fs, util,
extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; },
hasProp = {}.hasOwnProperty;
WordNetFile = require('./wordnet_file');
fs = require('fs');
util = require('util');
module.exports = IndexFile = (function(superClass) {
var _findAt, _findPrevEOL, _getFileSize, _readLine;
extend(IndexFile, superClass);
function IndexFile(dataDir, name) {
IndexFile.__super__.constructor.call(this, dataDir, 'index.' + name);
}
_findPrevEOL = function(self, fd, pos, callback) {
var buff;
buff = new Buffer(1024);
if (pos === 0) {
return callback(null, 0);
} else {
return fs.read(fd, buff, 0, 1, pos, function(err, count) {
if (err != null) {
return callback(err, count);
}
if (buff[0] === 10) {
return callback(null, pos + 1);
} else {
return _findPrevEOL(self, fd, pos - 1, callback);
}
});
}
};
_readLine = function(self, fd, pos, callback) {
var buff;
buff = new Buffer(1024);
return _findPrevEOL(self, fd, pos, function(err, pos) {
if (err != null) {
return callback(err, pos);
}
return self.appendLineChar(fd, pos, 0, buff, callback);
});
};
_findAt = function(self, fd, size, pos, lastPos, adjustment, searchKey, callback, lastKey) {
if (lastPos === pos || pos >= size) {
return callback(null, {
status: 'miss'
});
} else {
return _readLine(self, fd, pos, function(err, line) {
var key, tokens;
if (err != null) {
return callback(err);
}
tokens = line.split(/\s+/);
key = tokens[0];
if (key === searchKey) {
return callback(null, {
status: 'hit',
key: key,
'line': line,
tokens: tokens
});
} else if (adjustment === 1 || key === lastKey) {
return callback(null, {
status: 'miss'
});
} else {
adjustment = Math.ceil(adjustment * 0.5);
if (key < searchKey) {
return _findAt(self, fd, size, pos + adjustment, pos, adjustment, searchKey, callback, key);
} else {
return _findAt(self, fd, size, pos - adjustment, pos, adjustment, searchKey, callback, key);
}
}
});
}
};
_getFileSize = function(path) {
var stat;
stat = fs.statSync(path);
return stat.size;
};
IndexFile.prototype.find = function(searchKey, callback) {
var self;
self = this;
return this.open(function(err, fd) {
var pos, size;
if (err != null) {
return callback(err, null);
}
size = _getFileSize(this.filePath) - 1;
pos = Math.ceil(size / 2);
return _findAt(self, fd, size, pos, null, pos, searchKey, function(err, result) {
return callback.call(self, err, result);
});
});
};
IndexFile.prototype.lookupFromFile = function(word, callback) {
return this.find(word, function(err, record) {
var i, indexRecord, j, k, offsets, ptrs, ref, ref1;
if (err != null) {
return callback.call(this, err, null);
}
indexRecord = null;
if (record.status === 'hit') {
ptrs = [];
offsets = [];
for (i = j = 0, ref = parseInt(record.tokens[3]) - 1; j <= ref; i = j += 1) {
ptrs.push(record.tokens[i]);
}
for (i = k = 0, ref1 = parseInt(record.tokens[2]) - 1; k <= ref1; i = k += 1) {
offsets.push(parseInt(record.tokens[ptrs.length + 6 + i], 10));
}
indexRecord = {
lemma: record.tokens[0],
pos: record.tokens[1],
ptrSymbol: ptrs,
senseCnt: parseInt(record.tokens[ptrs.length + 4], 10),
tagsenseCnt: parseInt(record.tokens[ptrs.length + 5], 10),
synsetOffset: offsets
};
}
return callback.call(this, null, indexRecord);
});
};
IndexFile.prototype.lookup = function(word, callback) {
return this.lookupFromFile(word, callback);
};
return IndexFile;
})(WordNetFile);

View File

@ -0,0 +1,681 @@
var DataFile, IndexFile, LRU, Promise, WordNet, async, fs, path,
slice = [].slice;
IndexFile = require('./index_file');
DataFile = require('./data_file');
async = require('async');
Promise = require('bluebird');
path = require('path');
fs = require('fs');
LRU = require('lru-cache');
WordNet = (function() {
var _forms, _loadExceptions, _validForms, _validFormsWithExceptions, exceptions, forms, tokenDetach, unique;
function WordNet(options) {
var WNdb, e, error;
if (typeof options === 'string') {
options = {
dataDir: options
};
} else {
if (options == null) {
options = {};
}
}
if (options.dataDir == null) {
try {
WNdb = require('wndb-with-exceptions');
} catch (error) {
e = error;
console.error("Please 'npm install wndb-with-exceptions' before using WordNet module or specify a dict directory.");
throw e;
}
options.dataDir = WNdb.path;
}
if (!options.cache) {
this.cache = null;
} else {
if (options.cache === true) {
options.cache = {
max: 2000
};
}
if (typeof options.cache === 'object' && typeof options.cache.get === 'function') {
this.cache = options.cache;
} else {
this.cache = LRU(options.cache);
}
}
this.path = options.dataDir;
this.nounIndex = new IndexFile(this.path, 'noun');
this.verbIndex = new IndexFile(this.path, 'verb');
this.adjIndex = new IndexFile(this.path, 'adj');
this.advIndex = new IndexFile(this.path, 'adv');
this.nounData = new DataFile(this.path, 'noun');
this.verbData = new DataFile(this.path, 'verb');
this.adjData = new DataFile(this.path, 'adj');
this.advData = new DataFile(this.path, 'adv');
this.allFiles = [
{
index: this.nounIndex,
data: this.nounData,
pos: 'n'
}, {
index: this.verbIndex,
data: this.verbData,
pos: 'v'
}, {
index: this.adjIndex,
data: this.adjData,
pos: 'a'
}, {
index: this.advIndex,
data: this.advData,
pos: 'r'
}
];
}
WordNet.prototype.get = function(synsetOffset, pos, callback) {
var dataFile, hit, query, wordnet;
wordnet = this;
if (this.cache) {
query = "get:" + synsetOffset + ":" + pos;
if (hit = wordnet.cache.get(query)) {
if (callback.length === 1) {
return callback.call(wordnet, hit);
} else {
return callback.call(wordnet, null, hit);
}
}
}
dataFile = wordnet.getDataFile(pos);
return dataFile.get(synsetOffset, function(err, result) {
if (query && (err == null)) {
wordnet.cache.set(query, result);
}
if (callback.length === 1) {
return callback.call(wordnet, result);
} else {
return callback.call(wordnet, err, result);
}
});
};
WordNet.prototype.getAsync = function(synsetOffset, pos) {
var wordnet;
wordnet = this;
return new Promise(function(resolve, reject) {
return wordnet.get(synsetOffset, pos, function(err, data) {
if (err != null) {
return reject(err);
} else {
return resolve(data);
}
});
});
};
WordNet.prototype.lookup = function(input, callback) {
var hit, lword, pos, query, ref, selectedFiles, word, wordnet;
wordnet = this;
ref = input.split('#'), word = ref[0], pos = ref[1];
lword = word.toLowerCase().replace(/\s+/g, '_');
if (this.cache) {
query = "lookup:" + input;
if (hit = wordnet.cache.get(query)) {
if (callback.length === 1) {
return callback.call(wordnet, hit);
} else {
return callback.call(wordnet, null, hit);
}
}
}
selectedFiles = !pos ? wordnet.allFiles.slice() : wordnet.allFiles.filter(function(file) {
return file.pos === pos;
});
return wordnet.lookupFromFiles(selectedFiles, [], lword, function(err, results) {
if (err != null) {
return callback.call(wordnet, err);
}
if (query) {
wordnet.cache.set(query, results);
}
if (callback.length === 1) {
return callback.call(wordnet, results);
} else {
return callback.call(wordnet, null, results);
}
});
};
WordNet.prototype.lookupAsync = function(input, callback) {
var wordnet;
wordnet = this;
return new Promise(function(resolve, reject) {
return wordnet.lookup(input, function(err, data) {
if (err != null) {
return reject(err);
} else {
return resolve(data);
}
});
});
};
WordNet.prototype.findSense = function(input, callback) {
var hit, lword, pos, query, ref, selectedFiles, sense, senseNumber, word, wordnet;
wordnet = this;
ref = input.split('#'), word = ref[0], pos = ref[1], senseNumber = ref[2];
if (this.cache) {
query = "findSense:" + input;
if (hit = wordnet.cache.get(query)) {
if (callback.length === 1) {
return callback.call(wordnet, hit);
} else {
return callback.call(wordnet, null, hit);
}
}
}
sense = parseInt(senseNumber);
if (Number.isNaN(sense)) {
throw new Error("Sense number should be an integer");
} else if (sense < 1) {
throw new Error("Sense number should be a positive integer");
}
lword = word.toLowerCase().replace(/\s+/g, '_');
selectedFiles = wordnet.allFiles.filter(function(file) {
return file.pos === pos;
});
return wordnet.lookupFromFiles(selectedFiles, [], lword, function(err, response) {
var result;
if (err != null) {
return callback.call(wordnet, err);
}
result = response[sense - 1];
if (query) {
wordnet.cache.set(query, result);
}
if (callback.length === 1) {
return callback.call(wordnet, result);
} else {
return callback.call(wordnet, null, result);
}
});
};
WordNet.prototype.findSenseAsync = function(input) {
var wordnet;
wordnet = this;
return new Promise(function(resolve, reject) {
return wordnet.findSense(input, function(err, data) {
if (err != null) {
return reject(err);
} else {
return resolve(data);
}
});
});
};
WordNet.prototype.querySense = function(input, callback) {
var hit, pos, query, ref, word, wordnet;
wordnet = this;
ref = input.split('#'), word = ref[0], pos = ref[1];
if (this.cache) {
query = "querySense:" + input;
if (hit = wordnet.cache.get(query)) {
if (callback.length === 1) {
return callback.call(wordnet, hit);
} else {
return callback.call(wordnet, null, hit);
}
}
}
return wordnet.lookup(input, function(err, results) {
var i, sense, senseCounts, senses;
if (err != null) {
return callback.call(wordnet, err);
}
senseCounts = {};
senses = (function() {
var j, len, results1;
results1 = [];
for (i = j = 0, len = results.length; j < len; i = ++j) {
sense = results[i];
pos = sense.pos;
if (pos === 's') {
pos = 'a';
}
if (senseCounts[pos] == null) {
senseCounts[pos] = 1;
}
results1.push(word + "#" + pos + "#" + senseCounts[pos]++);
}
return results1;
})();
if (query) {
wordnet.cache.set(query, senses);
}
if (callback.length === 1) {
return callback.call(wordnet, senses);
} else {
return callback.call(wordnet, null, senses);
}
});
};
WordNet.prototype.querySenseAsync = function(input) {
var wordnet;
wordnet = this;
return new Promise(function(resolve, reject) {
return wordnet.querySense(input, function(err, data) {
if (err != null) {
return reject(err);
} else {
return resolve(data);
}
});
});
};
WordNet.prototype.lookupFromFiles = function(files, results, word, callback) {
var file, wordnet;
wordnet = this;
if (files.length === 0) {
return callback.call(wordnet, null, results);
} else {
file = files.pop();
return file.index.lookup(word, function(err, record) {
if (record) {
return wordnet.pushResults(file.data, results, record.synsetOffset, function() {
return wordnet.lookupFromFiles(files, results, word, callback);
});
} else {
return wordnet.lookupFromFiles(files, results, word, callback);
}
});
}
};
WordNet.prototype.pushResults = function(data, results, offsets, callback) {
var wordnet;
wordnet = this;
if (offsets.length === 0) {
return callback(results);
} else {
return data.get(offsets.pop(), function(err, record) {
results.push(record);
return wordnet.pushResults(data, results, offsets, callback);
});
}
};
WordNet.prototype.loadResultSynonyms = function(synonyms, results, callback) {
var result, wordnet;
wordnet = this;
if (results.length > 0) {
result = results.pop();
return wordnet.loadSynonyms(synonyms, results, result.ptrs, callback);
} else {
return callback(synonyms);
}
};
WordNet.prototype.loadSynonyms = function(synonyms, results, ptrs, callback) {
var ptr, wordnet;
wordnet = this;
if (ptrs.length > 0) {
ptr = ptrs.pop();
return this.get(ptr.synsetOffset, ptr.pos, function(result) {
synonyms.push(result);
return wordnet.loadSynonyms(synonyms, results, ptrs, callback);
});
} else {
return wordnet.loadResultSynonyms(synonyms, results, callback);
}
};
WordNet.prototype.lookupSynonyms = function(word, callback) {
var wordnet;
wordnet = this;
return wordnet.lookup(word, function(results) {
return wordnet.loadResultSynonyms([], results, callback);
});
};
WordNet.prototype.getSynonyms = function() {
var callback, pos, synsetOffset, wordnet;
wordnet = this;
callback = arguments[2] ? arguments[2] : arguments[1];
pos = arguments[0].pos ? arguments[0].pos : arguments[1];
synsetOffset = arguments[0].synsetOffset ? arguments[0].synsetOffset : arguments[0];
return this.get(synsetOffset, pos, function(result) {
return wordnet.loadSynonyms([], [], result.ptrs, callback);
});
};
WordNet.prototype.getDataFile = function(pos) {
switch (pos) {
case 'n':
return this.nounData;
case 'v':
return this.verbData;
case 'a':
case 's':
return this.adjData;
case 'r':
return this.advData;
}
};
exceptions = [
{
name: "noun.exc",
pos: 'n'
}, {
name: "verb.exc",
pos: 'v'
}, {
name: "adj.exc",
pos: 'a'
}, {
name: "adv.exc",
pos: 'r'
}
];
_loadExceptions = function(wordnet, callback) {
var loadFile;
WordNet.prototype.exceptions = 'pending';
loadFile = function(exception, callback) {
var fullPath;
fullPath = path.join(wordnet.path, exception.name);
return fs.readFile(fullPath, function(err, data) {
var j, len, line, lines, ref, temp, term1, term2;
if (err) {
return callback(err);
}
temp = {};
lines = data.toString().split("\n");
for (j = 0, len = lines.length; j < len; j++) {
line = lines[j];
if (line.length > 0) {
ref = line.split(' '), term1 = ref[0], term2 = 2 <= ref.length ? slice.call(ref, 1) : [];
if (temp[term1] == null) {
temp[term1] = [];
}
Array.prototype.push.apply(temp[term1], term2);
}
}
return callback(null, {
pos: exception.pos,
data: temp
});
});
};
return async.map(exceptions, loadFile, function(err, results) {
var j, len, result;
exceptions = {};
for (j = 0, len = results.length; j < len; j++) {
result = results[j];
exceptions[result.pos] = result.data;
}
WordNet.prototype.exceptions = exceptions;
return callback();
});
};
WordNet.prototype.close = function() {
this.nounIndex.close();
this.verbIndex.close();
this.adjIndex.close();
this.advIndex.close();
this.nounData.close();
this.verbData.close();
this.adjData.close();
return this.advData.close();
};
unique = function(a) {
var found;
found = {};
return a.filter(function(item) {
if (found[item]) {
return false;
} else {
return found[item] = true;
}
});
};
tokenDetach = function(string) {
var detach, length, pos, ref, sense, word;
ref = string.split('#'), word = ref[0], pos = ref[1], sense = ref[2];
detach = [word];
length = word.length;
switch (pos) {
case 'n':
if (word.endsWith("s")) {
detach.push(word.substring(0, length - 1));
}
if (word.endsWith("ses")) {
detach.push(word.substring(0, length - 2));
}
if (word.endsWith("xes")) {
detach.push(word.substring(0, length - 2));
}
if (word.endsWith("zes")) {
detach.push(word.substring(0, length - 2));
}
if (word.endsWith("ches")) {
detach.push(word.substring(0, length - 2));
}
if (word.endsWith("shes")) {
detach.push(word.substring(0, length - 2));
}
if (word.endsWith("men")) {
detach.push(word.substring(0, length - 3) + "man");
}
if (word.endsWith("ies")) {
detach.push(word.substring(0, length - 3) + "y");
}
break;
case 'v':
if (word.endsWith("s")) {
detach.push(word.substring(0, length - 1));
}
if (word.endsWith("ies")) {
detach.push(word.substring(0, length - 3) + "y");
}
if (word.endsWith("es")) {
detach.push(word.substring(0, length - 2));
}
if (word.endsWith("ed")) {
detach.push(word.substring(0, length - 1));
}
if (word.endsWith("ed")) {
detach.push(word.substring(0, length - 2));
}
if (word.endsWith("ing")) {
detach.push(word.substring(0, length - 3) + "e");
}
if (word.endsWith("ing")) {
detach.push(word.substring(0, length - 3));
}
break;
case 'r':
if (word.endsWith("er")) {
detach.push(word.substring(0, length - 2));
}
if (word.endsWith("er")) {
detach.push(word.substring(0, length - 1));
}
if (word.endsWith("est")) {
detach.push(word.substring(0, length - 3));
}
if (word.endsWith("est")) {
detach.push(word.substring(0, length - 2));
}
}
return unique(detach);
};
_forms = function(wordnet, word, pos) {
var colloc, exception, forms, i, index, j, lword, ref, ref1, rtn, token, tokens;
lword = word.toLowerCase();
exception = (ref = wordnet.exceptions[pos]) != null ? ref[lword] : void 0;
if (exception) {
return [word].concat(exception);
}
tokens = word.split(/[ _]/g);
if (tokens.length === 1) {
return tokenDetach(tokens[0] + "#" + pos);
}
forms = tokens.map(function(token) {
return _forms(wordnet, token, pos);
});
rtn = [];
index = (function() {
var j, len, results1;
results1 = [];
for (j = 0, len = tokens.length; j < len; j++) {
token = tokens[j];
results1.push(0);
}
return results1;
})();
while (true) {
colloc = forms[0][index[0]];
for (i = j = 1, ref1 = tokens.length - 1; 1 <= ref1 ? j <= ref1 : j >= ref1; i = 1 <= ref1 ? ++j : --j) {
colloc = colloc + '_' + forms[i][index[i]];
}
rtn.push(colloc);
i = 0;
while (i < tokens.length) {
index[i] = index[i] + 1;
if (index[i] < forms[i].length) {
break;
} else {
index[i] = 0;
}
i = i + 1;
}
if (i >= tokens.length) {
break;
}
}
return rtn;
};
forms = function(wordnet, string) {
var element, j, len, pos, ref, results1, rtn, sense, word;
ref = string.split('#'), word = ref[0], pos = ref[1], sense = ref[2];
rtn = _forms(wordnet, word, pos);
results1 = [];
for (j = 0, len = rtn.length; j < len; j++) {
element = rtn[j];
results1.push(element + "#" + pos);
}
return results1;
};
_validForms = function(wordnet, string, callback) {
var eachFn, filteredResults, pos, possibleForms, reducer, ref, sense, word;
ref = string.split('#'), word = ref[0], pos = ref[1], sense = ref[2];
if (!pos) {
reducer = function(previous, current, next) {
return _validForms(wordnet, string + "#" + current, function(err, value) {
if (value === void 0) {
return next(null, previous);
} else {
return next(null, previous.concat(value));
}
});
};
return async.reduce(['n', 'v', 'a', 'r'], [], reducer, function(err, result) {
return callback(null, result);
});
} else {
possibleForms = forms(wordnet, word + "#" + pos);
filteredResults = [];
eachFn = function(term, done) {
return wordnet.lookup(term, function(err, data) {
if (err != null) {
return done(err);
}
if (data.length > 0) {
filteredResults.push(term);
}
return done();
});
};
return async.each(possibleForms, eachFn, function(err) {
return callback(err, filteredResults);
});
}
};
_validFormsWithExceptions = function(wordnet, string, callback) {
if (wordnet.exceptions === void 0) {
return _loadExceptions(wordnet, function() {
return _validFormsWithExceptions(wordnet, string, callback);
});
} else if (wordnet.exceptions === 'pending') {
return setImmediate(_validFormsWithExceptions, wordnet, string, callback);
} else {
return _validForms(wordnet, string, callback);
}
};
WordNet.prototype.validForms = function(string, callback) {
var hit, query, wordnet;
wordnet = this;
if (this.cache) {
query = "validForms:" + string;
if (hit = wordnet.cache.get(query)) {
if (callback.length === 1) {
return callback.call(wordnet, hit);
} else {
return callback.call(wordnet, null, hit);
}
}
}
return _validFormsWithExceptions(this, string, function(err, result) {
if (query) {
wordnet.cache.set(query, result);
}
if (callback.length === 1) {
return callback.call(wordnet, result);
} else {
return callback.call(wordnet, null, result);
}
});
};
WordNet.prototype.validFormsAsync = function(string) {
return new Promise((function(_this) {
return function(resolve, reject) {
return _this.validForms(string, function(err, data) {
if (err != null) {
return reject(err);
} else {
return resolve(data);
}
});
};
})(this));
};
return WordNet;
})();
module.exports = WordNet;

View File

@ -0,0 +1,64 @@
var WordNetFile, fs, path, util;
fs = require('fs');
path = require('path');
util = require('util');
module.exports = WordNetFile = (function() {
function WordNetFile(dataDir, fileName) {
this.dataDir = dataDir;
this.fileName = fileName;
this.filePath = path.join(this.dataDir, this.fileName);
}
WordNetFile.prototype.open = function(callback) {
var filePath, self;
self = this;
if (this.fd) {
return callback.call(self, null, this.fd);
}
filePath = this.filePath;
return fs.open(filePath, 'r', null, (function(_this) {
return function(err, fd) {
if (err != null) {
return callback.call(self, err, null);
}
_this.fd = fd;
return callback.call(self, err, fd);
};
})(this));
};
WordNetFile.prototype.close = function() {
if (this.fd != null) {
fs.close(this.fd);
return delete this.fd;
}
};
WordNetFile.prototype.appendLineChar = function(fd, pos, buffPos, buff, callback) {
var length, self, space;
self = this;
length = buff.length;
space = length - buffPos;
return fs.read(fd, buff, buffPos, space, pos, function(err, count, buffer) {
var i, j, newBuff, ref;
if (err != null) {
return callback.call(self, err, null);
}
for (i = j = 0, ref = count - 1; 0 <= ref ? j <= ref : j >= ref; i = 0 <= ref ? ++j : --j) {
if (buff[i] === 10) {
return callback.call(self, null, buff.slice(0, i).toString('ASCII'));
}
}
newBuff = new Buffer(length * 2);
buff.copy(newBuff, 0, 0, length);
return self.appendLineChar(fd, pos + length, length, newBuff, callback);
});
};
return WordNetFile;
})();

View File

@ -17,6 +17,7 @@
"@types/sortablejs": "^1.10.6",
"@vue/devtools": "^5.3.3",
"any-ascii": "^0.1.7",
"async": "^0.9.0",
"axios": "^0.21.0",
"bluebird": "^3.7.2",
"bootstrap": "^4.5.3",
@ -30,7 +31,9 @@
"extract-loader": "^5.1.0",
"file-loader": "^6.2.0",
"lodash": "^4.17.20",
"lru-cache": "^2.5.0",
"node-sass": "^4.14.1",
"node-wordnet": "^0.1.12",
"optimize-css-assets-webpack-plugin": "^5.0.4",
"qs": "^6.9.4",
"raven-js": "^3.27.2",
@ -48,7 +51,8 @@
"vue": "^2.6.12",
"vue-loader": "^15.9.4",
"vue-template-compiler": "^2.6.12",
"webpack": "^5.3.2"
"webpack": "^5.3.2",
"wndb-with-exceptions": "^3.0.2"
},
"dependencies": {
"@cliqz/adblocker-electron": "^1.18.3",

View File

@ -790,6 +790,11 @@ async-limiter@~1.0.0:
resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd"
integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==
async@^0.9.0:
version "0.9.2"
resolved "https://registry.yarnpkg.com/async/-/async-0.9.2.tgz#aea74d5e61c1f899613bf64bda66d4c78f2fd17d"
integrity sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=
async@^1.4.2:
version "1.5.2"
resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a"
@ -1441,6 +1446,11 @@ block-stream@*:
dependencies:
inherits "~2.0.0"
bluebird@^2.6.0:
version "2.11.0"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-2.11.0.tgz#534b9033c022c9579c56ba3b3e5a5caafbb650e1"
integrity sha1-U0uQM8AiyVecVro7Plpcqvu2UOE=
bluebird@^3.1.1, bluebird@^3.5.0, bluebird@^3.7.2:
version "3.7.2"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
@ -2799,6 +2809,11 @@ es6-error@^4.1.1:
resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d"
integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==
es6-shim@^0.22.1:
version "0.22.2"
resolved "https://registry.yarnpkg.com/es6-shim/-/es6-shim-0.22.2.tgz#4afe7c54bb92b178e5d464dd5d9c31c5742d7521"
integrity sha1-Sv58VLuSsXjl1GTdXZwxxXQtdSE=
escalade@^3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
@ -4455,6 +4470,11 @@ lowercase-keys@^2.0.0:
resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479"
integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==
lru-cache@^2.5.0:
version "2.7.3"
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.7.3.tgz#6d4524e8b955f95d4f5b58851ce21dd72fb4e952"
integrity sha1-bUUk6LlV+V1PW1iFHOId1y+06VI=
lru-cache@^4.0.1, lru-cache@^4.1.2:
version "4.1.5"
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd"
@ -4926,6 +4946,16 @@ node-sass@^4.14.1:
stdout-stream "^1.4.0"
"true-case-path" "^1.0.2"
node-wordnet@^0.1.12:
version "0.1.12"
resolved "https://registry.yarnpkg.com/node-wordnet/-/node-wordnet-0.1.12.tgz#62b0568f5dfc188bd02d136cf730832952fa9c1b"
integrity sha1-YrBWj138GIvQLRNs9zCDKVL6nBs=
dependencies:
async "^0.9.0"
bluebird "^2.6.0"
es6-shim "^0.22.1"
lru-cache "^2.5.0"
noop-logger@^0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/noop-logger/-/noop-logger-0.1.1.tgz#94a2b1633c4f1317553007d8966fd0e841b6a4c2"
@ -6952,6 +6982,18 @@ tar@^6.0.2, tar@^6.0.5:
mkdirp "^1.0.3"
yallist "^4.0.0"
tar@latest:
version "6.1.0"
resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.0.tgz#d1724e9bcc04b977b18d5c573b333a2207229a83"
integrity sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA==
dependencies:
chownr "^2.0.0"
fs-minipass "^2.0.0"
minipass "^3.0.0"
minizlib "^2.1.1"
mkdirp "^1.0.3"
yallist "^4.0.0"
temp@^0.9.0:
version "0.9.4"
resolved "https://registry.yarnpkg.com/temp/-/temp-0.9.4.tgz#cd20a8580cb63635d0e4e9d4bd989d44286e7620"
@ -7495,6 +7537,13 @@ wide-align@^1.1.0:
dependencies:
string-width "^1.0.2 || 2"
wndb-with-exceptions@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/wndb-with-exceptions/-/wndb-with-exceptions-3.0.2.tgz#7048c1be7e0d13d231a51e408e0b37426000bcbb"
integrity sha1-cEjBvn4NE9IxpR5Ajgs3QmAAvLs=
dependencies:
tar latest
worker-rpc@^0.1.0:
version "0.1.1"
resolved "https://registry.yarnpkg.com/worker-rpc/-/worker-rpc-0.1.1.tgz#cb565bd6d7071a8f16660686051e969ad32f54d5"