From e1adc05caeba598c5fc24d8d65658ccc66f4e5cb Mon Sep 17 00:00:00 2001 From: "Mr. Stallion" Date: Sat, 23 Jan 2021 18:52:27 -0600 Subject: [PATCH] Web worker draft --- CHANGELOG.md | 1 + electron/Window.vue | 1 + electron/main.ts | 2 +- electron/webpack.config.js | 43 +++++- learn/cache-manager.ts | 11 +- learn/personal-profiler.ts | 0 learn/profile-cache.ts | 2 +- learn/store/better-sqlite3.ts | 89 ------------ learn/store/indexed.ts | 72 +++++----- learn/store/sql-store.ts | 123 ----------------- learn/store/sql.ts | 55 -------- learn/store/sqlite.ts | 63 --------- learn/store/sqlite3.ts | 144 -------------------- learn/store/types.ts | 56 ++++++++ learn/store/worker.ts | 57 ++++++++ learn/store/worker/client.ts | 98 +++++++++++++ learn/store/worker/store.worker.endpoint.ts | 72 ++++++++++ learn/store/worker/types.ts | 17 +++ 18 files changed, 390 insertions(+), 516 deletions(-) delete mode 100644 learn/personal-profiler.ts delete mode 100644 learn/store/better-sqlite3.ts delete mode 100644 learn/store/sql-store.ts delete mode 100644 learn/store/sql.ts delete mode 100644 learn/store/sqlite.ts delete mode 100644 learn/store/sqlite3.ts create mode 100644 learn/store/types.ts create mode 100644 learn/store/worker.ts create mode 100644 learn/store/worker/client.ts create mode 100644 learn/store/worker/store.worker.endpoint.ts create mode 100644 learn/store/worker/types.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ce83950..baa4868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Canary * Fixed Gelbooru video previews +* Moved database queries to a web worker to gain more responsive UI ## 1.9.0 diff --git a/electron/Window.vue b/electron/Window.vue index 6e8e021..1c533bb 100644 --- a/electron/Window.vue +++ b/electron/Window.vue @@ -254,6 +254,7 @@ webPreferences: { webviewTag: true, nodeIntegration: true, + nodeIntegrationInWorker: true, spellcheck: true, enableRemoteModule: true } diff --git a/electron/main.ts b/electron/main.ts index a9ec4ce..f7f1e8b 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -179,7 +179,7 @@ function createWindow(): Electron.BrowserWindow | undefined { const lastState = windowState.getSavedWindowState(); const windowProperties: Electron.BrowserWindowConstructorOptions & {maximized: boolean} = { ...lastState, center: lastState.x === undefined, show: false, - webPreferences: { webviewTag: true, nodeIntegration: true, spellcheck: true, enableRemoteModule: true } + webPreferences: { webviewTag: true, nodeIntegration: true, nodeIntegrationInWorker: true, spellcheck: true, enableRemoteModule: true } }; if(process.platform === 'darwin') { diff --git a/electron/webpack.config.js b/electron/webpack.config.js index 1ae1930..9b9e520 100644 --- a/electron/webpack.config.js +++ b/electron/webpack.config.js @@ -1,3 +1,4 @@ +const _ = require('lodash'); const path = require('path'); const fs = require('fs'); const ForkTsCheckerWebpackPlugin = require('@f-list/fork-ts-checker-webpack-plugin'); @@ -145,6 +146,44 @@ const mainConfig = { } }; + +const storeWorkerEndpointConfig = _.assign( + _.cloneDeep(mainConfig), + { + entry: [path.join(__dirname, '..', 'learn', 'store', 'worker', 'store.worker.endpoint.ts')], + output: { + path: __dirname + '/app', + filename: 'storeWorkerEndpoint.js', + globalObject: 'this' + }, + target: 'electron-renderer', + + module: { + rules: [ + { + test: /\.ts$/, + loader: 'ts-loader', + options: { + configFile: __dirname + '/tsconfig-renderer.json', + transpileOnly: true, + getCustomTransformers: () => ({before: [vueTransformer]}) + } + }, + ] + }, + + plugins: [ + new ForkTsCheckerWebpackPlugin({ + async: false, + tslint: path.join(__dirname, '../tslint.json'), + tsconfig: './tsconfig-renderer.json', + vue: true + }) + ] + } +); + + module.exports = function(mode) { const themesDir = path.join(__dirname, '../scss/themes/chat'); const themes = fs.readdirSync(themesDir); @@ -185,12 +224,14 @@ module.exports = function(mode) { process.env.NODE_ENV = 'production'; mainConfig.devtool = rendererConfig.devtool = 'source-map'; rendererConfig.plugins.push(new OptimizeCssAssetsPlugin()); + storeWorkerEndpointConfig.devtool = 'source-map'; } else { // mainConfig.devtool = rendererConfig.devtool = 'none'; mainConfig.devtool = 'inline-source-map'; rendererConfig.devtool = 'inline-source-map'; + storeWorkerEndpointConfig.devtool = 'inline-source-map'; } - return [mainConfig, rendererConfig]; + return [mainConfig, rendererConfig, storeWorkerEndpointConfig]; }; diff --git a/learn/cache-manager.ts b/learn/cache-manager.ts index a65d820..0811dfb 100644 --- a/learn/cache-manager.ts +++ b/learn/cache-manager.ts @@ -8,7 +8,6 @@ import { AdCache } from './ad-cache'; import { ChannelConversationCache } from './channel-conversation-cache'; import { CharacterProfiler } from './character-profiler'; import { CharacterCacheRecord, ProfileCache } from './profile-cache'; -import { IndexedStore } from './store/indexed'; import Timer = NodeJS.Timer; import ChannelConversation = Conversation.ChannelConversation; import Message = Conversation.Message; @@ -17,6 +16,10 @@ import Bluebird from 'bluebird'; import ChatMessage = Conversation.ChatMessage; import { GeneralSettings } from '../electron/common'; import { Gender } from './matcher-types'; +import { WorkerStore } from './store/worker'; +import { PermanentIndexedStore } from './store/types'; +import * as path from 'path'; +// import * as electron from 'electron'; export interface ProfileCacheQueueEntry { @@ -41,7 +44,7 @@ export class CacheManager { protected profileTimer: Timer | null = null; protected characterProfiler: CharacterProfiler | undefined; - protected profileStore?: IndexedStore; + protected profileStore?: PermanentIndexedStore; protected lastPost: Date = new Date(); @@ -175,7 +178,9 @@ export class CacheManager { async start(settings: GeneralSettings, skipFlush: boolean): Promise { await this.stop(); - this.profileStore = await IndexedStore.open(); + this.profileStore = await WorkerStore.open( + path.join(/*electron.remote.app.getAppPath(),*/ 'storeWorkerEndpoint.js') + ); // await IndexedStore.open(); this.profileCache.setStore(this.profileStore); diff --git a/learn/personal-profiler.ts b/learn/personal-profiler.ts deleted file mode 100644 index e69de29..0000000 diff --git a/learn/profile-cache.ts b/learn/profile-cache.ts index ae8a74b..4dc159b 100644 --- a/learn/profile-cache.ts +++ b/learn/profile-cache.ts @@ -4,7 +4,7 @@ import core from '../chat/core'; import {Character as ComplexCharacter, CharacterGroup, Guestbook} from '../site/character_page/interfaces'; import { AsyncCache } from './async-cache'; import { Matcher, MatchReport, Scoring } from './matcher'; -import { PermanentIndexedStore } from './store/sql-store'; +import { PermanentIndexedStore } from './store/types'; import { CharacterImage, SimpleCharacter } from '../interfaces'; diff --git a/learn/store/better-sqlite3.ts b/learn/store/better-sqlite3.ts deleted file mode 100644 index 25c5327..0000000 --- a/learn/store/better-sqlite3.ts +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Do not use - */ - -// // import { Database, Statement } from 'better-sqlite3'; -// -// type Database = any; -// type Statement = any; -// -// import { ProfileRecord, SqlStore } from './sql-store'; -// import { Character as ComplexCharacter } from '../../site/character_page/interfaces'; -// import * as SQL from './sql'; -// -// -// export class BetterSqliteStore extends SqlStore { -// protected static Sqlite: any; -// -// protected stmtGetProfile: Statement; -// protected stmtStoreProfile: Statement; -// protected stmtUpdateCounts: Statement; -// -// protected db: Database; -// protected checkpointTimer: NodeJS.Timer | null = null; -// -// constructor(name: string) { -// super(name); -// -// if (!BetterSqliteStore.Sqlite) { -// throw new Error('BetterSqliteStore.setSqlite() must be called before instantiation'); -// } -// -// this.db = new BetterSqliteStore.Sqlite(this.dbFile, {}); -// -// this.stmtGetProfile = this.db.prepare(SQL.ProfileGet); -// this.stmtStoreProfile = this.db.prepare(SQL.ProfileInsert); -// this.stmtUpdateCounts = this.db.prepare(SQL.ProfileUpdateCount); -// } -// -// -// static setSqlite(Sqlite: any) { -// BetterSqliteStore.Sqlite = Sqlite; -// } -// -// -// async getProfile(name: string): Promise { -// const data = this.stmtGetProfile.get(this.toProfileId(name)); -// -// if (!data) { -// return; -// } -// -// // tslint:disable-next-line: no-unsafe-any -// data.profileData = JSON.parse(data.profileData) as ComplexCharacter; -// -// return data as ProfileRecord; -// } -// -// -// protected async run(stmtName: 'stmtStoreProfile' | 'stmtUpdateCounts', data: any[]): Promise { -// if (!(stmtName in this)) { -// throw new Error(`Unknown statement: ${stmtName}`); -// } -// -// this[stmtName].run(data); -// } -// -// -// async start(): Promise { -// await this.stop(); -// -// this.db.pragma('journal_mode = WAL'); -// this.db.exec(SQL.DatabaseMigration); -// -// this.checkpointTimer = setInterval( -// () => this.db.checkpoint(), -// 10 * 60 * 1000 -// ); -// } -// -// -// async stop(): Promise { -// if (this.checkpointTimer) { -// clearInterval(this.checkpointTimer); -// -// this.checkpointTimer = null; -// } -// } -// } -// diff --git a/learn/store/indexed.ts b/learn/store/indexed.ts index 25130de..835e9e4 100644 --- a/learn/store/indexed.ts +++ b/learn/store/indexed.ts @@ -3,7 +3,7 @@ import * as _ from 'lodash'; import {Character as ComplexCharacter, CharacterGroup, Guestbook} from '../../site/character_page/interfaces'; import { CharacterAnalysis } from '../matcher'; -import { PermanentIndexedStore, ProfileRecord } from './sql-store'; +import { PermanentIndexedStore, ProfileRecord } from './types'; import { CharacterImage, SimpleCharacter } from '../../interfaces'; import Bluebird from 'bluebird'; @@ -30,7 +30,7 @@ export class IndexedStore implements PermanentIndexedStore { } static async open(dbName: string = 'flist-ascending-profiles'): Promise { - const request = window.indexedDB.open(dbName, 2); + const request = indexedDB.open(dbName, 2); request.onupgradeneeded = (event) => { const db = request.result; @@ -85,7 +85,7 @@ export class IndexedStore implements PermanentIndexedStore { } - async prepareProfileData(c: ComplexCharacter): Promise { + private async prepareProfileData(c: ComplexCharacter): Promise { const existing = await this.getProfile(c.character.name); const ca = new CharacterAnalysis(c.character); @@ -121,8 +121,8 @@ export class IndexedStore implements PermanentIndexedStore { } - async storeProfile(c: ComplexCharacter): Promise { - const data = await this.prepareProfileData(c); + async storeProfile(character: ComplexCharacter): Promise { + const data = await this.prepareProfileData(character); const tx = this.db.transaction(IndexedStore.STORE_NAME, 'readwrite'); const store = tx.objectStore(IndexedStore.STORE_NAME); @@ -135,37 +135,37 @@ export class IndexedStore implements PermanentIndexedStore { } - async updateProfileCounts( - name: string, - guestbookCount: number | null, - friendCount: number | null, - groupCount: number | null - ): Promise { - const existing = await this.getProfile(name); - - if (!existing) { - return; - } - - const data = _.merge( - existing, - { - lastCounted: Math.round(Date.now() / 1000), - guestbookCount, - friendCount, - groupCount - } - ); - - const tx = this.db.transaction(IndexedStore.STORE_NAME, 'readwrite'); - const store = tx.objectStore(IndexedStore.STORE_NAME); - const putRequest = store.put(data); - - // tslint:disable-next-line no-any - await promisifyRequest(putRequest); - - // console.log('IDX update counts', name, data); - } + // async updateProfileCounts( + // name: string, + // guestbookCount: number | null, + // friendCount: number | null, + // groupCount: number | null + // ): Promise { + // const existing = await this.getProfile(name); + // + // if (!existing) { + // return; + // } + // + // const data = _.merge( + // existing, + // { + // lastCounted: Math.round(Date.now() / 1000), + // guestbookCount, + // friendCount, + // groupCount + // } + // ); + // + // const tx = this.db.transaction(IndexedStore.STORE_NAME, 'readwrite'); + // const store = tx.objectStore(IndexedStore.STORE_NAME); + // const putRequest = store.put(data); + // + // // tslint:disable-next-line no-any + // await promisifyRequest(putRequest); + // + // // console.log('IDX update counts', name, data); + // } async updateProfileMeta( diff --git a/learn/store/sql-store.ts b/learn/store/sql-store.ts deleted file mode 100644 index 445a7e4..0000000 --- a/learn/store/sql-store.ts +++ /dev/null @@ -1,123 +0,0 @@ -// tslint:disable-next-line:no-duplicate-imports -// import * as path from 'path'; -// import core from '../../chat/core'; - -import {Character as ComplexCharacter, CharacterGroup, Guestbook} from '../../site/character_page/interfaces'; -import { CharacterImage, SimpleCharacter } from '../../interfaces'; -import { FurryPreference, Gender, Orientation, Species } from '../matcher-types'; - -// This design should be refactored; it's bad -export interface ProfileRecord { - id: string; - name: string; - profileData: ComplexCharacter; - firstSeen: number; - lastFetched: number; - gender: Gender | null; - orientation: Orientation | null; - furryPreference: FurryPreference | null; - species: Species | null; - age: number | null; - domSubRole: number | null; - position: number | null; - - // lastCounted: number | null; - // guestbookCount: number | null; - // friendCount: number | null; - // groupCount: number | null; - - lastMetaFetched: number | null; - guestbook: Guestbook | null; - images: CharacterImage[] | null; - friends: SimpleCharacter[] | null; - groups: CharacterGroup[] | null; -} - -// export type Statement = any; -// export type Database = any; - -export interface PermanentIndexedStore { - getProfile(name: string): Promise; - storeProfile(c: ComplexCharacter): Promise; - - updateProfileMeta( - name: string, - images: CharacterImage[] | null, - guestbook: Guestbook | null, - friends: SimpleCharacter[] | null, - groups: CharacterGroup[] | null - ): Promise; - - flushProfiles(daysToExpire: number): Promise; - - start(): Promise; - stop(): Promise; -} - - -// export abstract class SqlStore implements PermanentIndexedStore { -// protected dbFile: string; -// protected checkpointTimer: NodeJS.Timer | null = null; -// -// constructor(dbName: string = 'fchat-ascending.sqlite') { -// this.dbFile = path.join(core.state.generalSettings!.logDirectory, dbName); -// } -// -// // tslint:disable-next-line: prefer-function-over-method -// protected toProfileId(name: string): string { -// return name.toLowerCase(); -// } -// -// abstract getProfile(name: string): Promise; -// -// abstract start(): Promise; -// abstract stop(): Promise; -// -// // tslint:disable-next-line no-any -// protected abstract run(statementName: 'stmtStoreProfile' | 'stmtUpdateCounts', data: any[]): Promise; -// -// -// async storeProfile(c: ComplexCharacter): Promise { -// const ca = new CharacterAnalysis(c.character); -// -// const data = [ -// this.toProfileId(c.character.name), -// c.character.name, -// JSON.stringify(c), -// Math.round(Date.now() / 1000), -// Math.round(Date.now() / 1000), -// ca.gender, -// ca.orientation, -// ca.furryPreference, -// ca.species, -// ca.age, -// null, // domSubRole -// null // position -// ]; -// -// await this.run('stmtStoreProfile', data); -// } -// -// async updateProfileMeta( -// name: string, -// images: CharacterImage[] | null, -// guestbook: GuestbookState | null, -// friends: CharacterFriend[] | null, -// groups: CharacterGroup[] | null -// ): Promise { -// throw new Error('Not implemented'); -// } -// -// // async updateProfileCounts( -// // name: string, -// // guestbookCount: number | null, -// // friendCount: number | null, -// // groupCount: number | null -// // ): Promise { -// // await this.run( -// // 'stmtUpdateCounts', -// // [Math.round(Date.now() / 1000), guestbookCount, friendCount, groupCount, this.toProfileId(name)] -// // ); -// // } -// } -// diff --git a/learn/store/sql.ts b/learn/store/sql.ts deleted file mode 100644 index 1db72b0..0000000 --- a/learn/store/sql.ts +++ /dev/null @@ -1,55 +0,0 @@ - - -export const ProfileInsert = - `INSERT INTO profiles - (id, name, profileData, firstSeen, lastFetched, gender, orientation, furryPreference, - species, age, domSubRole, position) - VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(id) DO UPDATE SET - profileData=excluded.profileData, - lastFetched=excluded.lastFetched, - gender=excluded.gender, - orientation=excluded.orientation, - furryPreference=excluded.furryPreference, - species=excluded.species, - age=excluded.age, - domSubRole=excluded.domSubRole, - position=excluded.position - `; - -export const ProfileGet = - 'SELECT * FROM profiles WHERE id = ?'; - -export const ProfileUpdateCount = - 'UPDATE profiles SET lastCounted = ?, guestbookCount = ?, friendCount = ?, groupCount = ? WHERE id = ?'; - - -export const DatabaseMigration = - `CREATE TABLE IF NOT EXISTS "migration" ( - "version" INTEGER NOT NULL - , UNIQUE("version") - ); - - CREATE TABLE IF NOT EXISTS "profiles" ( - "id" TEXT NOT NULL PRIMARY KEY - , "name" TEXT NOT NULL - , "profileData" TEXT NOT NULL - , "firstSeen" INTEGER NOT NULL - , "lastFetched" INTEGER NOT NULL - , "lastCounted" INTEGER - , "gender" INTEGER - , "orientation" INTEGER - , "furryPreference" INTEGER - , "species" INTEGER - , "age" INTEGER - , "domSubRole" INTEGER - , "position" INTEGER - , "guestbookCount" INTEGER - , "friendCount" INTEGER - , "groupCount" INTEGER - , UNIQUE("id") - ); - - INSERT OR IGNORE INTO migration(version) VALUES(1); - `; - diff --git a/learn/store/sqlite.ts b/learn/store/sqlite.ts deleted file mode 100644 index 8793e30..0000000 --- a/learn/store/sqlite.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Do not use - */ - -// import * as Sqlite from 'sqlite'; -// import { Character as ComplexCharacter } from '../../site/character_page/interfaces'; -// -// import * as SQL from './sql'; -// import { ProfileRecord, SqlStore } from './sql-store'; -// -// export class SqliteStore extends SqlStore { -// protected stmtGetProfile: Promise; -// protected stmtStoreProfile: Promise; -// protected stmtUpdateCounts: Promise; -// -// protected db: Promise; -// -// constructor(dbName: string = 'fchat-ascending.sqlite') { -// super(dbName); -// -// this.db = Sqlite.open(this.dbFile); -// -// this.stmtGetProfile = this.db.then((db) => db.prepare(SQL.ProfileGet)); -// this.stmtStoreProfile = this.db.then((db) => db.prepare(SQL.ProfileInsert)); -// this.stmtUpdateCounts = this.db.then((db) => db.prepare(SQL.ProfileUpdateCount)); -// } -// -// -// async getProfile(name: string): Promise { -// const data = await (await this.stmtGetProfile).get(this.toProfileId(name)); -// -// if (!data) { -// return; -// } -// -// // tslint:disable-next-line: no-unsafe-any -// data.profileData = JSON.parse(data.profileData) as ComplexCharacter; -// -// return data as ProfileRecord; -// } -// -// -// protected async run(stmtName: 'stmtStoreProfile' | 'stmtUpdateCounts', data: any[]): Promise { -// await (await this[stmtName]).run(...data); -// } -// -// -// async start(): Promise { -// await this.stop(); -// -// await (await this.db).run(SQL.DatabaseMigration); -// } -// -// -// async stop(): Promise { -// if (this.checkpointTimer) { -// clearInterval(this.checkpointTimer); -// -// this.checkpointTimer = null; -// } -// } -// -// } diff --git a/learn/store/sqlite3.ts b/learn/store/sqlite3.ts deleted file mode 100644 index 513d189..0000000 --- a/learn/store/sqlite3.ts +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Do not use - */ - -// type Sqlite3Statement = any; -// type Sqlite3Database = any; -// -// import { Character as ComplexCharacter } from '../../site/character_page/interfaces'; -// -// import * as SQL from './sql'; -// import { ProfileRecord, SqlStore } from './sql-store'; -// -// export class Sqlite3Store extends SqlStore { -// protected static Sqlite: any; -// -// protected stmtGetProfile: Promise; -// protected stmtStoreProfile: Promise; -// protected stmtUpdateCounts: Promise; -// -// protected db: Promise; -// -// constructor(dbName: string = 'fchat-ascending.sqlite') { -// super(dbName); -// -// if (!Sqlite3Store.Sqlite) { -// throw new Error('Sqlite3Store.setSqlite() must be called before instantiation'); -// } -// -// this.db = this.prepareDatabase(this.dbFile); -// -// this.stmtGetProfile = this.prepareStatement(SQL.ProfileGet); -// this.stmtStoreProfile = this.prepareStatement(SQL.ProfileInsert); -// this.stmtUpdateCounts = this.prepareStatement(SQL.ProfileUpdateCount); -// } -// -// static setSqlite(Sqlite: any) { -// Sqlite3Store.Sqlite = Sqlite; -// } -// -// prepareDatabase(fileName: string): Promise { -// return new Promise( -// (resolve, reject) => { -// // Sqlite3Store.Sqlite.verbose(); -// const db = new Sqlite3Store.Sqlite.Database( -// fileName, -// (err: any) => { -// if (err) { -// reject(err); -// return; -// } -// -// resolve(db); -// } -// ); -// } -// ); -// } -// -// prepareStatement(sql: string): Promise { -// return new Promise( -// async(resolve, reject) => { -// const s = (await this.db).prepare( -// sql, -// (err: any) => { -// if (err) { -// reject(err); -// return; -// } -// -// resolve(s); -// } -// ); -// } -// ); -// } -// -// -// getProfile(name: string): Promise { -// return new Promise( -// async(resolve, reject) => { -// (await this.stmtGetProfile).get( -// this.toProfileId(name), -// (err: any, data: any) => { -// if (err) { -// reject(err); -// return; -// } -// -// if (!data) { -// resolve(); -// return; -// } -// -// // tslint:disable-next-line: no-unsafe-any -// data.profileData = JSON.parse(data.profileData) as ComplexCharacter; -// -// resolve(data as ProfileRecord); -// } -// ); -// } -// ); -// } -// -// -// protected run(stmtName: 'stmtStoreProfile' | 'stmtUpdateCounts', data: any[]): Promise { -// return new Promise( -// async(resolve, reject) => { -// if (!(stmtName in this)) { -// throw new Error(`Unknown statement: ${stmtName}`); -// } -// -// (await this[stmtName]).run( -// ...data, -// (err: any) => { -// if (err) { -// reject(err); -// return; -// } -// -// resolve(); -// } -// ); -// } -// ); -// } -// -// -// async start(): Promise { -// await this.stop(); -// -// await (await this.db).run(SQL.DatabaseMigration); -// } -// -// -// async stop(): Promise { -// if (this.checkpointTimer) { -// clearInterval(this.checkpointTimer); -// -// this.checkpointTimer = null; -// } -// } -// -// } - diff --git a/learn/store/types.ts b/learn/store/types.ts new file mode 100644 index 0000000..c2cc015 --- /dev/null +++ b/learn/store/types.ts @@ -0,0 +1,56 @@ +// tslint:disable-next-line:no-duplicate-imports +// import * as path from 'path'; +// import core from '../../chat/core'; + +import {Character as ComplexCharacter, CharacterGroup, Guestbook} from '../../site/character_page/interfaces'; +import { CharacterImage, SimpleCharacter } from '../../interfaces'; +import { FurryPreference, Gender, Orientation, Species } from '../matcher-types'; + +// This design should be refactored; it's bad +export interface ProfileRecord { + id: string; + name: string; + profileData: ComplexCharacter; + firstSeen: number; + lastFetched: number; + gender: Gender | null; + orientation: Orientation | null; + furryPreference: FurryPreference | null; + species: Species | null; + age: number | null; + domSubRole: number | null; + position: number | null; + + // lastCounted: number | null; + // guestbookCount: number | null; + // friendCount: number | null; + // groupCount: number | null; + + lastMetaFetched: number | null; + guestbook: Guestbook | null; + images: CharacterImage[] | null; + friends: SimpleCharacter[] | null; + groups: CharacterGroup[] | null; +} + +// export type Statement = any; +// export type Database = any; + +export interface PermanentIndexedStore { + getProfile(name: string): Promise; + storeProfile(c: ComplexCharacter): Promise; + + updateProfileMeta( + name: string, + images: CharacterImage[] | null, + guestbook: Guestbook | null, + friends: SimpleCharacter[] | null, + groups: CharacterGroup[] | null + ): Promise; + + flushProfiles(daysToExpire: number): Promise; + + start(): Promise; + stop(): Promise; +} + diff --git a/learn/store/worker.ts b/learn/store/worker.ts new file mode 100644 index 0000000..5510dd6 --- /dev/null +++ b/learn/store/worker.ts @@ -0,0 +1,57 @@ +import {Character as ComplexCharacter, CharacterGroup, Guestbook} from '../../site/character_page/interfaces'; +import { PermanentIndexedStore, ProfileRecord } from './types'; +import { CharacterImage, SimpleCharacter } from '../../interfaces'; + +import { WorkerClient } from './worker/client'; + + +export class WorkerStore implements PermanentIndexedStore { + protected readonly workerClient: WorkerClient; + + constructor(jsEndpointFile: string) { + this.workerClient = new WorkerClient(jsEndpointFile); + } + + + static async open(jsEndpointFile: string, dbName?: string): Promise { + const store = new WorkerStore(jsEndpointFile); + + await store.workerClient.request('init', { dbName }); + + return store; + } + + + async getProfile(name: string): Promise { + return this.workerClient.request('get', { name }); + } + + + async storeProfile(character: ComplexCharacter): Promise { + return this.workerClient.request('store', { character }); + } + + + async updateProfileMeta( + name: string, + images: CharacterImage[] | null, + guestbook: Guestbook | null, + friends: SimpleCharacter[] | null, + groups: CharacterGroup[] | null + ): Promise { + return this.workerClient.request('update-meta', { name, images, guestbook, friends, groups }); + } + + async start(): Promise { + return this.workerClient.request('start'); + } + + async stop(): Promise { + return this.workerClient.request('stop'); + } + + + async flushProfiles(daysToExpire: number): Promise { + return this.workerClient.request('flush', { daysToExpire }); + } +} diff --git a/learn/store/worker/client.ts b/learn/store/worker/client.ts new file mode 100644 index 0000000..2099881 --- /dev/null +++ b/learn/store/worker/client.ts @@ -0,0 +1,98 @@ +import _ from 'lodash'; +import log from 'electron-log'; //tslint:disable-line:match-default-export-name + + +import { IndexedRequest, IndexedResponse, ProfileStoreCommand } from './types'; + +export interface WaiterDef { + id: string; + resolve(result?: any): void; + reject(result?: any): void; +} + +export class WorkerClient { + private readonly worker: Worker; + + private idCounter = 0; + + private waiters: WaiterDef[] = []; + + constructor(jsFile: string) { + this.worker = new Worker(jsFile); + this.worker.onmessage = this.generateMessageProcessor(); + } + + + private generateId(): string { + this.idCounter++; + + return `wc-${this.idCounter}`; + } + + + private when(id: string, resolve: (result?: any) => void, reject: (reason?: any) => void): void { + this.waiters.push({ id, resolve, reject }); + } + + + private generateMessageProcessor(): ((e: Event) => void) { + return (e: Event) => { + const res = (e as any).data as IndexedResponse; + + log.silly('store.worker.client.msg', { res }); + + if (!res) { + log.error('store.worker.client.msg.invalid', { res }); + return; + } + + const waiter = _.find(this.waiters, (w) => (w.id === res.id)); + + if (!waiter) { + log.error('store.worker.client.msg.unknown', { res }); + return; + } + + if (res.state === 'ok') { + waiter.resolve(res.result); + } else { + waiter.reject(new Error(res.msg)); + } + + this.clearWaiter(waiter.id); + }; + } + + + private clearWaiter(id: string): void { + this.waiters = _.filter(this.waiters, (w) => (w.id !== id)); + } + + + async request(cmd: ProfileStoreCommand, params: Record = {}): Promise { + const id = this.generateId(); + + const request: IndexedRequest = { + cmd, + id, + params + }; + + return new Promise( + (resolve, reject) => { + try { + this.when( + id, + resolve, + reject + ); + + this.worker.postMessage(request); + } catch (err) { + reject(err); + this.clearWaiter(id); + } + } + ); + } +} diff --git a/learn/store/worker/store.worker.endpoint.ts b/learn/store/worker/store.worker.endpoint.ts new file mode 100644 index 0000000..7e441b5 --- /dev/null +++ b/learn/store/worker/store.worker.endpoint.ts @@ -0,0 +1,72 @@ +import _ from 'lodash'; +import log from 'electron-log'; //tslint:disable-line:match-default-export-name + +import { IndexedStore } from '../indexed'; +import { IndexedRequest, ProfileStoreCommand } from './types'; + +type IndexedCallback = (params: Record) => Promise; + +let indexed: IndexedStore; + + +const reply = (req: IndexedRequest, result?: any, err?: string | Error): void => { + const res: any = { + type: 'res', + id: req.id, + state: err ? 'err' : 'ok', + result + }; + + if (err) { + console.error(err); + console.error('store.worker.endpoint.error', { err }); + res.msg = _.isString(err) ? err : err.message; + } + + log.debug('store.worker.endpoint.reply', { req, res }); + + postMessage(res); +}; + + +const generateMessageProcessor = () => { + const messageMapper: Record = { + flush: (params: Record) => indexed.flushProfiles(params.daysToExpire), + start: () => indexed.start(), + stop: () => indexed.stop(), + get: (params: Record) => indexed.getProfile(params.name), + store: (params: Record) => indexed.storeProfile(params.character), + + 'update-meta': (params: Record) => + indexed.updateProfileMeta(params.name, params.images, params.guestbook, params.friends, params.groups), + + init: async(params: Record): Promise => { + indexed = await IndexedStore.open(params.dbName); + } + }; + + return async(e: Event) => { + log.silly('store.worker.endpoint.msg', { e }); + + const req = (e as any).data as IndexedRequest; + + if (!req) { + return; + } + + if (!(req.cmd in messageMapper)) { + reply(req, undefined, 'unknown command'); + return; + } + + try { + const result = await messageMapper[req.cmd](req.params); + reply(req, result); + } catch(err) { + reply(req, undefined, err); + } + }; +}; + + +onmessage = generateMessageProcessor(); diff --git a/learn/store/worker/types.ts b/learn/store/worker/types.ts new file mode 100644 index 0000000..c9327ac --- /dev/null +++ b/learn/store/worker/types.ts @@ -0,0 +1,17 @@ + +export type ProfileStoreCommand = 'flush' | 'start' | 'stop' | 'update-meta' | 'store' | 'get' | 'init'; + +export interface IndexedRequest { + cmd: ProfileStoreCommand; + id: string; + params: Record; +} + + +export interface IndexedResponse { + id: string; + type: 'event' | 'res'; + state: 'err' | 'ok'; + result?: any; + msg?: string; +}