From 80dfe32006263e8c77e47dccec875a4e345595c7 Mon Sep 17 00:00:00 2001 From: "Mr. Stallion" Date: Tue, 30 Jun 2020 16:51:06 -0500 Subject: [PATCH] Cache expiration --- CHANGELOG.md | 5 ++-- chat/core.ts | 9 +++--- electron/Index.vue | 60 +++++++++++++++++++++++++++++++++++++--- electron/Window.vue | 9 +++++- electron/chat.ts | 9 +++++- electron/common.ts | 4 +++ electron/main.ts | 50 +++++++++++++++++++++++++++------ fchat/connection.ts | 8 ++++++ learn/cache-manager.ts | 5 +++- learn/store/indexed.ts | 48 ++++++++++++++++++++++++++++++-- learn/store/sql-store.ts | 2 ++ mobile/filesystem.ts | 6 ++-- 12 files changed, 188 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56cf51b..79666c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,11 @@ # Changelog ## Canary -* Fixed caching issue that causes cache misses on charater page metadata -* Fixed ad posting issue that sometimes disconnects characters if multiple characters are in use +* Fixed a caching issue that caused cache misses on character page metadata +* Fixed rate limit issues that sometimes disconnected characters when multiple characters were connected * URL preview fixes for Redgifs, Gelbooru, Tumblr, and Gifmixxx * All dependencies are now up to date +* F-Chat Rising now flushes character profiles out of its cache after 30 days ## 1.0.1 diff --git a/chat/core.ts b/chat/core.ts index 88ba6d8..4c08518 100644 --- a/chat/core.ts +++ b/chat/core.ts @@ -6,6 +6,7 @@ import {Settings as SettingsImpl} from './common'; import Conversations from './conversations'; import {Channel, Character, Connection, Conversation, Logs, Notifications, Settings, State as StateInterface} from './interfaces'; import { AdCoordinatorGuest } from './ads/ad-coordinator-guest'; +import { GeneralSettings } from '../electron/common'; function createBBCodeParser(): BBCodeParser { const parser = new BBCodeParser(); @@ -82,8 +83,9 @@ const data = { } }; -export function init(this: any, connection: Connection, logsClass: new() => Logs, settingsClass: new() => Settings.Store, - notificationsClass: new() => Notifications): void { +export function init( + this: any, connection: Connection, settings: GeneralSettings, logsClass: new() => Logs, + settingsClass: new() => Settings.Store, notificationsClass: new() => Notifications): void { data.connection = connection; data.logs = new logsClass(); data.settingsStore = new settingsClass(); @@ -91,8 +93,7 @@ export function init(this: any, connection: Connection, logsClass: new() => Logs data.cache = new CacheManager(); data.adCoordinator = new AdCoordinatorGuest(); - // tslint:disable-next-line no-floating-promises - data.cache.start(); + (data.state as any).generalSettings = settings; data.register('characters', Characters(connection)); data.register('channels', Channels(connection, core.characters)); diff --git a/electron/Index.vue b/electron/Index.vue index 93ddbe8..0053baa 100644 --- a/electron/Index.vue +++ b/electron/Index.vue @@ -3,8 +3,16 @@
+
+
+ Getting ready, please wait... +
+ +
+

{{l('title')}} + {{l('logs.title')}} @@ -92,7 +100,7 @@ import Vue from 'vue'; import Chat from '../chat/Chat.vue'; import {getKey, Settings} from '../chat/common'; - import core from '../chat/core'; + import core /*, { init as initCore }*/ from '../chat/core'; import l from '../chat/localize'; import Logs from '../chat/Logs.vue'; import Socket from '../chat/WebSocket'; @@ -103,8 +111,10 @@ // import { Sqlite3Store } from '../learn/store/sqlite3'; import CharacterPage from '../site/character_page/character_page.vue'; import {defaultHost, GeneralSettings, nativeRequire} from './common'; - import {fixLogs} from './filesystem'; + import { fixLogs /*SettingsStore, Logs as FSLogs*/ } from './filesystem'; import * as SlimcatImporter from './importer'; + // import Connection from '../fchat/connection'; + // import Notifications from './notifications'; const webContents = electron.remote.getCurrentWebContents(); const parent = electron.remote.getCurrentWindow().webContents; @@ -163,8 +173,17 @@ fixCharacters: ReadonlyArray = []; fixCharacter = ''; + showSpinner = true; + + @Hook('created') - created(): void { + async created(): Promise { + // tslint:disable-next-line no-floating-promises + await core.cache.start(this.settings); + + // await this.prepper; + this.showSpinner = false; + if(this.settings.account.length > 0) this.saveLogin = true; keyStore.getPassword(this.settings.account) .then((value: string) => this.password = value, (err: Error) => this.error = err.message); @@ -347,7 +366,7 @@ } - diff --git a/electron/Window.vue b/electron/Window.vue index 6b4e217..2c10540 100644 --- a/electron/Window.vue +++ b/electron/Window.vue @@ -45,6 +45,7 @@ import l from '../chat/localize'; import {GeneralSettings} from './common'; import { getSafeLanguages, updateSupportedLanguages } from './language'; + import log from 'electron-log'; // tslint:disable-line: match-default-export-name const browserWindow = electron.remote.getCurrentWindow(); @@ -97,7 +98,13 @@ await this.addTab(); - electron.ipcRenderer.on('settings', (_e: Event, settings: GeneralSettings) => this.settings = settings); + electron.ipcRenderer.on('settings', (_e: Event, settings: GeneralSettings) => { + this.settings = settings; + + log.transports.file.level = settings.risingSystemLogLevel; + log.transports.console.level = settings.risingSystemLogLevel; + }); + electron.ipcRenderer.on('allow-new-tabs', (_e: Event, allow: boolean) => this.canOpenTab = allow); electron.ipcRenderer.on('open-tab', () => this.addTab()); electron.ipcRenderer.on('update-available', (_e: Event, available: boolean) => this.hasUpdate = available); diff --git a/electron/chat.ts b/electron/chat.ts index b579009..ac26f7e 100644 --- a/electron/chat.ts +++ b/electron/chat.ts @@ -52,6 +52,7 @@ import {Logs, SettingsStore} from './filesystem'; import Notifications from './notifications'; import * as SlimcatImporter from './importer'; import Index from './Index.vue'; +import log from 'electron-log'; // tslint:disable-line: match-default-export-name document.addEventListener('keydown', (e: KeyboardEvent) => { @@ -192,6 +193,9 @@ if(process.platform === 'win32') //get the path in DOS (8-character) format as s function onSettings(s: GeneralSettings): void { settings = s; + log.transports.file.level = settings.risingSystemLogLevel; + log.transports.console.level = settings.risingSystemLogLevel; + // spellchecker.setDictionary(s.spellcheckLang, dictDir); // for(const word of s.customDictionary) spellchecker.add(word); } @@ -200,6 +204,9 @@ electron.ipcRenderer.on('settings', (_: Event, s: GeneralSettings) => onSettings const params = <{[key: string]: string | undefined}>qs.parse(window.location.search.substr(1)); let settings = JSON.parse(params['settings']!); + +// console.log('SETTINGS', settings); + if(params['import'] !== undefined) try { if(SlimcatImporter.canImportGeneral() && confirm(l('importer.importGeneral'))) { @@ -212,7 +219,7 @@ if(params['import'] !== undefined) onSettings(settings); const connection = new Connection(`F-Chat 3.0 (${process.platform})`, electron.remote.app.getVersion(), Socket); -initCore(connection, Logs, SettingsStore, Notifications); +initCore(connection, settings, Logs, SettingsStore, Notifications); //tslint:disable-next-line:no-unused-expression new Index({ diff --git a/electron/common.ts b/electron/common.ts index 733754d..1d564f7 100644 --- a/electron/common.ts +++ b/electron/common.ts @@ -1,6 +1,8 @@ import * as electron from 'electron'; import * as path from 'path'; +import log from 'electron-log'; //tslint:disable-line:match-default-export-name + export const defaultHost = 'wss://chat.f-list.net/chat2'; export class GeneralSettings { @@ -15,6 +17,8 @@ export class GeneralSettings { beta = false; customDictionary: string[] = []; hwAcceleration = true; + risingCacheExpiryDays = 45; + risingSystemLogLevel: log.LevelOption = 'info'; } //tslint:disable diff --git a/electron/main.ts b/electron/main.ts index d677846..9ffb793 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -134,7 +134,11 @@ async function toggleDictionary(lang: string): Promise { function setGeneralSettings(value: GeneralSettings): void { fs.writeFileSync(path.join(settingsDir, 'settings'), JSON.stringify(value)); for(const w of electron.webContents.getAllWebContents()) w.send('settings', settings); + shouldImportSettings = false; + + log.transports.file.level = settings.risingSystemLogLevel; + log.transports.console.level = settings.risingSystemLogLevel; } async function addSpellcheckerItems(menu: Electron.Menu): Promise { @@ -380,6 +384,12 @@ function onReady(): void { settings.theme = theme; setGeneralSettings(settings); }; + + const setSystemLogLevel = (logLevel: log.LevelOption) => { + settings.risingSystemLogLevel = logLevel; + setGeneralSettings(settings); + }; + electron.Menu.setApplicationMenu(electron.Menu.buildFromTemplate([ { label: `&${l('title')}`, @@ -442,18 +452,40 @@ function onReady(): void { settings.hwAcceleration = item.checked; setGeneralSettings(settings); } - }, { - label: l('settings.beta'), type: 'checkbox', checked: settings.beta, - click: async(item: Electron.MenuItem) => { - settings.beta = item.checked; - setGeneralSettings(settings); - // electron.autoUpdater.setFeedURL({url: updaterUrl + (item.checked ? '?channel=beta' : ''), serverType: 'json'}); - // return electron.autoUpdater.checkForUpdates(); - } - }, { + }, + + // { + // label: l('settings.beta'), type: 'checkbox', checked: settings.beta, + // click: async(item: Electron.MenuItem) => { + // settings.beta = item.checked; + // setGeneralSettings(settings); + // // electron.autoUpdater.setFeedURL({url: updaterUrl+(item.checked ? '?channel=beta' : ''), serverType: 'json'}); + // // return electron.autoUpdater.checkForUpdates(); + // } + // }, + { label: l('fixLogs.action'), click: (_m, window: BrowserWindow) => window.webContents.send('fix-logs') }, + + {type: 'separator'}, + { + label: 'Rising', + submenu: [ + { + label: 'System log level', + submenu: ['error', 'warn', 'info', 'verbose', 'debug', 'silly'].map((level: string) => ( + { + checked: settings.risingSystemLogLevel === level, + label: `${level.substr(0, 1).toUpperCase()}${level.substr(1)}`, + click: () => setSystemLogLevel(level as log.LevelOption), + type: <'radio'>'radio' + } + )) + } + ] + }, + {type: 'separator'}, {role: 'minimize'}, { diff --git a/fchat/connection.ts b/fchat/connection.ts index 8da2f47..10b1c62 100644 --- a/fchat/connection.ts +++ b/fchat/connection.ts @@ -92,6 +92,14 @@ export default class Connection implements Interfaces.Connection { this.socket.onMessage(async(msg: string) => { const type = msg.substr(0, 3); const data = msg.length > 6 ? JSON.parse(msg.substr(4)) : undefined; + + log.silly( + 'socket.message', + { + type, data + } + ); + return this.handleMessage(type, data); }); this.socket.onClose(async(event: CloseEvent) => { diff --git a/learn/cache-manager.ts b/learn/cache-manager.ts index a1dde08..c7823d0 100644 --- a/learn/cache-manager.ts +++ b/learn/cache-manager.ts @@ -16,6 +16,7 @@ import Message = Conversation.Message; import { Character } from '../fchat/interfaces'; import Bluebird from 'bluebird'; import ChatMessage = Conversation.ChatMessage; +import { GeneralSettings } from '../electron/common'; export interface ProfileCacheQueueEntry { @@ -170,13 +171,15 @@ export class CacheManager { } - async start(): Promise { + async start(settings: GeneralSettings): Promise { await this.stop(); this.profileStore = await IndexedStore.open(); this.profileCache.setStore(this.profileStore); + await this.profileStore.flushProfiles(settings.risingCacheExpiryDays); + EventBus.$on( 'character-data', async(data: CharacterDataEvent) => { diff --git a/learn/store/indexed.ts b/learn/store/indexed.ts index 009ae9c..25130de 100644 --- a/learn/store/indexed.ts +++ b/learn/store/indexed.ts @@ -1,3 +1,4 @@ +import log from 'electron-log'; //tslint:disable-line:match-default-export-name import * as _ from 'lodash'; import {Character as ComplexCharacter, CharacterGroup, Guestbook} from '../../site/character_page/interfaces'; @@ -5,6 +6,8 @@ import { CharacterAnalysis } from '../matcher'; import { PermanentIndexedStore, ProfileRecord } from './sql-store'; import { CharacterImage, SimpleCharacter } from '../../interfaces'; +import Bluebird from 'bluebird'; + async function promisifyRequest(req: IDBRequest): Promise { return new Promise((resolve, reject) => { @@ -19,6 +22,7 @@ export class IndexedStore implements PermanentIndexedStore { protected db: IDBDatabase; protected static readonly STORE_NAME = 'profiles'; + protected static readonly LAST_FETCHED_INDEX_NAME = 'idxLastFetched'; constructor(db: IDBDatabase, dbName: string) { this.dbName = dbName; @@ -26,12 +30,27 @@ export class IndexedStore implements PermanentIndexedStore { } static async open(dbName: string = 'flist-ascending-profiles'): Promise { - const request = window.indexedDB.open(dbName, 1); + const request = window.indexedDB.open(dbName, 2); - request.onupgradeneeded = () => { + request.onupgradeneeded = (event) => { const db = request.result; - db.createObjectStore(IndexedStore.STORE_NAME, { keyPath: 'id' }); + if (event.oldVersion < 1) { + db.createObjectStore(IndexedStore.STORE_NAME, { keyPath: 'id' }); + } + + if (event.oldVersion < 2) { + const store = request.transaction!.objectStore(IndexedStore.STORE_NAME); + + store.createIndex( + IndexedStore.LAST_FETCHED_INDEX_NAME, + 'lastFetched', + { + unique: false, + multiEntry: false + } + ); + } }; return new IndexedStore(await promisifyRequest(request), dbName); @@ -190,5 +209,28 @@ export class IndexedStore implements PermanentIndexedStore { async stop(): Promise { // empty } + + + async flushProfiles(daysToExpire: number): Promise { + const tx = this.db.transaction(IndexedStore.STORE_NAME, 'readwrite'); + const store = tx.objectStore(IndexedStore.STORE_NAME); + const idx = store.index(IndexedStore.LAST_FETCHED_INDEX_NAME); + + const totalRecords = await promisifyRequest(store.count()); + + const expirationTime = Math.round(Date.now() / 1000) - (daysToExpire * 24 * 60 * 60); + const getAllKeysRequest = idx.getAllKeys(IDBKeyRange.upperBound(expirationTime)); + const result = await promisifyRequest(getAllKeysRequest); + + log.info('character.cache.expire', {daysToExpire, totalRecords, removableRecords: result.length}); + + await Bluebird.mapSeries( + result, + async(pk: IDBValidKey) => { + log.silly('character.cache.expire.name', { name: pk }); + await promisifyRequest(store.delete(pk)); + } + ); + } } diff --git a/learn/store/sql-store.ts b/learn/store/sql-store.ts index 1d44736..58a8fef 100644 --- a/learn/store/sql-store.ts +++ b/learn/store/sql-store.ts @@ -48,6 +48,8 @@ export interface PermanentIndexedStore { groups: CharacterGroup[] | null ): Promise; + flushProfiles(daysToExpire: number): Promise; + start(): Promise; stop(): Promise; } diff --git a/mobile/filesystem.ts b/mobile/filesystem.ts index 22501ee..ff888e5 100644 --- a/mobile/filesystem.ts +++ b/mobile/filesystem.ts @@ -1,3 +1,5 @@ +import * as _ from 'lodash'; + import {Message as MessageImpl} from '../chat/common'; import core from '../chat/core'; import {Conversation, Logs as Logging, Settings} from '../chat/interfaces'; @@ -135,7 +137,7 @@ export class Logs implements Logging { export async function getGeneralSettings(): Promise { const file = await NativeFile.read('!settings'); if(file === undefined) return undefined; - const settings = JSON.parse(file); + const settings = _.merge(new GeneralSettings(), JSON.parse(file)); if(settings.host === 'wss://chat.f-list.net:9799') settings.host = 'wss://chat.f-list.net/chat2'; return settings; } @@ -158,4 +160,4 @@ export class SettingsStore implements Settings.Store { async getAvailableCharacters(): Promise { return NativeFile.listDirectories('/'); } -} \ No newline at end of file +}