Cache expiration

CHANGELOG.md

@@ -1,10 +1,11 @@
 # Changelog
 
 ## Canary
-*   Fixed caching issue that causes cache misses on charater page metadata
+*   Fixed a caching issue that caused cache misses on character page metadata
-*   Fixed ad posting issue that sometimes disconnects characters if multiple characters are in use
+*   Fixed rate limit issues that sometimes disconnected characters when multiple characters were connected
 *   URL preview fixes for Redgifs, Gelbooru, Tumblr, and Gifmixxx
 *   All dependencies are now up to date
+*   F-Chat Rising now flushes character profiles out of its cache after 30 days
 
 
 ## 1.0.1

chat/core.ts

@@ -6,6 +6,7 @@ import {Settings as SettingsImpl} from './common';
 import Conversations from './conversations';
 import {Channel, Character, Connection, Conversation, Logs, Notifications, Settings, State as StateInterface} from './interfaces';
 import { AdCoordinatorGuest } from './ads/ad-coordinator-guest';
+import { GeneralSettings } from '../electron/common';
 
 function createBBCodeParser(): BBCodeParser {
     const parser = new BBCodeParser();
@@ -82,8 +83,9 @@ const data = {
     }
 };
 
-export function init(this: any, connection: Connection, logsClass: new() => Logs, settingsClass: new() => Settings.Store,
+export function init(
-                     notificationsClass: new() => Notifications): void {
+    this: any, connection: Connection, settings: GeneralSettings, logsClass: new() => Logs,
+    settingsClass: new() => Settings.Store, notificationsClass: new() => Notifications): void {
     data.connection = connection;
     data.logs = new logsClass();
     data.settingsStore = new settingsClass();
@@ -91,8 +93,7 @@ export function init(this: any, connection: Connection, logsClass: new() => Logs
     data.cache = new CacheManager();
     data.adCoordinator = new AdCoordinatorGuest();
 
-    // tslint:disable-next-line no-floating-promises
+    (data.state as any).generalSettings = settings;
-    data.cache.start();
 
     data.register('characters', Characters(connection));
     data.register('channels', Channels(connection, core.characters));

electron/Index.vue

@@ -3,8 +3,16 @@
         <div v-html="styling"></div>
         <div v-if="!characters" style="display:flex; align-items:center; justify-content:center; height: 100%;">
             <div class="card bg-light" style="width: 400px;">
+                <div class="initializer" v-show="showSpinner">
+                    <div class="title">
+                        Getting ready, please wait...
+                    </div>
+                    <i class="fas fa-circle-notch fa-spin search-spinner"></i>
+                </div>
+
                 <h3 class="card-header" style="margin-top:0;display:flex">
                     {{l('title')}}
+
                     <a href="#" @click.prevent="showLogs()" class="btn" style="flex:1;text-align:right">
                         <span class="fa fa-file-alt"></span> <span class="btn-text">{{l('logs.title')}}</span>
                     </a>
@@ -92,7 +100,7 @@
     import Vue from 'vue';
     import Chat from '../chat/Chat.vue';
     import {getKey, Settings} from '../chat/common';
-    import core from '../chat/core';
+    import core /*, { init as initCore }*/ from '../chat/core';
     import l from '../chat/localize';
     import Logs from '../chat/Logs.vue';
     import Socket from '../chat/WebSocket';
@@ -103,8 +111,10 @@
 //     import { Sqlite3Store } from '../learn/store/sqlite3';
     import CharacterPage from '../site/character_page/character_page.vue';
     import {defaultHost, GeneralSettings, nativeRequire} from './common';
-    import {fixLogs} from './filesystem';
+    import { fixLogs /*SettingsStore, Logs as FSLogs*/ } from './filesystem';
     import * as SlimcatImporter from './importer';
+    // import Connection from '../fchat/connection';
+    // import Notifications from './notifications';
 
     const webContents = electron.remote.getCurrentWebContents();
     const parent = electron.remote.getCurrentWindow().webContents;
@@ -163,8 +173,17 @@
         fixCharacters: ReadonlyArray<string> = [];
         fixCharacter = '';
 
+        showSpinner = true;
+
+
         @Hook('created')
-        created(): void {
+        async created(): Promise<void> {
+            // tslint:disable-next-line no-floating-promises
+            await core.cache.start(this.settings);
+
+            // await this.prepper;
+            this.showSpinner = false;
+
             if(this.settings.account.length > 0) this.saveLogin = true;
             keyStore.getPassword(this.settings.account)
                 .then((value: string) => this.password = value, (err: Error) => this.error = err.message);
@@ -347,7 +366,7 @@
     }
 </script>
 
-<style>
+<style lang="scss">
     html, body, #page {
         height: 100%;
     }
@@ -362,4 +381,37 @@
         font-size: 12pt;
         opacity: 0.5;
     }
+
+
+    .initializer {
+        position: absolute;
+        top: 0;
+        left: 0;
+        right: 0;
+        bottom: 0;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        transition: all 0.25s;
+        backdrop-filter: blur(3px) grayscale(35%);
+
+        i {
+            font-size: 130pt;
+            top: 50%;
+            right: 50%;
+            transform: translate(-50%, -50%);
+            width: fit-content;
+        }
+
+        .title {
+            position: absolute;
+            top: 0;
+            background: rgba(19, 19, 19, 0.6);
+            width: 100%;
+            text-align: center;
+            padding-top: 20px;
+            padding-bottom: 20px;
+            font-weight: bold;
+        }
+    }
 </style>

electron/Window.vue

@@ -45,6 +45,7 @@
     import l from '../chat/localize';
     import {GeneralSettings} from './common';
     import { getSafeLanguages, updateSupportedLanguages } from './language';
+    import log from 'electron-log'; // tslint:disable-line: match-default-export-name
 
     const browserWindow = electron.remote.getCurrentWindow();
 
@@ -97,7 +98,13 @@
 
             await this.addTab();
 
-            electron.ipcRenderer.on('settings', (_e: Event, settings: GeneralSettings) => this.settings = settings);
+            electron.ipcRenderer.on('settings', (_e: Event, settings: GeneralSettings) => {
+                this.settings = settings;
+
+                log.transports.file.level = settings.risingSystemLogLevel;
+                log.transports.console.level = settings.risingSystemLogLevel;
+            });
+
             electron.ipcRenderer.on('allow-new-tabs', (_e: Event, allow: boolean) => this.canOpenTab = allow);
             electron.ipcRenderer.on('open-tab', () => this.addTab());
             electron.ipcRenderer.on('update-available', (_e: Event, available: boolean) => this.hasUpdate = available);

electron/chat.ts

@@ -52,6 +52,7 @@ import {Logs, SettingsStore} from './filesystem';
 import Notifications from './notifications';
 import * as SlimcatImporter from './importer';
 import Index from './Index.vue';
+import log from 'electron-log'; // tslint:disable-line: match-default-export-name
 
 
 document.addEventListener('keydown', (e: KeyboardEvent) => {
@@ -192,6 +193,9 @@ if(process.platform === 'win32') //get the path in DOS (8-character) format as s
 function onSettings(s: GeneralSettings): void {
     settings = s;
 
+    log.transports.file.level = settings.risingSystemLogLevel;
+    log.transports.console.level = settings.risingSystemLogLevel;
+
     // spellchecker.setDictionary(s.spellcheckLang, dictDir);
     // for(const word of s.customDictionary) spellchecker.add(word);
 }
@@ -200,6 +204,9 @@ electron.ipcRenderer.on('settings', (_: Event, s: GeneralSettings) => onSettings
 
 const params = <{[key: string]: string | undefined}>qs.parse(window.location.search.substr(1));
 let settings = <GeneralSettings>JSON.parse(params['settings']!);
+
+// console.log('SETTINGS', settings);
+
 if(params['import'] !== undefined)
     try {
         if(SlimcatImporter.canImportGeneral() && confirm(l('importer.importGeneral'))) {
@@ -212,7 +219,7 @@ if(params['import'] !== undefined)
 onSettings(settings);
 
 const connection = new Connection(`F-Chat 3.0 (${process.platform})`, electron.remote.app.getVersion(), Socket);
-initCore(connection, Logs, SettingsStore, Notifications);
+initCore(connection, settings, Logs, SettingsStore, Notifications);
 
 //tslint:disable-next-line:no-unused-expression
 new Index({

electron/common.ts

@@ -1,6 +1,8 @@
 import * as electron from 'electron';
 import * as path from 'path';
 
+import log from 'electron-log'; //tslint:disable-line:match-default-export-name
+
 export const defaultHost = 'wss://chat.f-list.net/chat2';
 
 export class GeneralSettings {
@@ -15,6 +17,8 @@ export class GeneralSettings {
     beta = false;
     customDictionary: string[] = [];
     hwAcceleration = true;
+    risingCacheExpiryDays = 45;
+    risingSystemLogLevel: log.LevelOption = 'info';
 }
 
 //tslint:disable

electron/main.ts

@@ -134,7 +134,11 @@ async function toggleDictionary(lang: string): Promise<void> {
 function setGeneralSettings(value: GeneralSettings): void {
     fs.writeFileSync(path.join(settingsDir, 'settings'), JSON.stringify(value));
     for(const w of electron.webContents.getAllWebContents()) w.send('settings', settings);
+
     shouldImportSettings = false;
+
+    log.transports.file.level = settings.risingSystemLogLevel;
+    log.transports.console.level = settings.risingSystemLogLevel;
 }
 
 async function addSpellcheckerItems(menu: Electron.Menu): Promise<void> {
@@ -380,6 +384,12 @@ function onReady(): void {
         settings.theme = theme;
         setGeneralSettings(settings);
     };
+
+    const setSystemLogLevel = (logLevel: log.LevelOption) => {
+        settings.risingSystemLogLevel = logLevel;
+        setGeneralSettings(settings);
+    };
+
     electron.Menu.setApplicationMenu(electron.Menu.buildFromTemplate([
         {
             label: `&${l('title')}`,
@@ -442,18 +452,40 @@ function onReady(): void {
                         settings.hwAcceleration = item.checked;
                         setGeneralSettings(settings);
                     }
-                }, {
+                },
-                    label: l('settings.beta'), type: 'checkbox', checked: settings.beta,
+
-                    click: async(item: Electron.MenuItem) => {
+                // {
-                        settings.beta = item.checked;
+                //     label: l('settings.beta'), type: 'checkbox', checked: settings.beta,
-                        setGeneralSettings(settings);
+                //     click: async(item: Electron.MenuItem) => {
-                        // electron.autoUpdater.setFeedURL({url: updaterUrl + (item.checked ? '?channel=beta' : ''), serverType: 'json'});
+                //         settings.beta = item.checked;
-                        // return electron.autoUpdater.checkForUpdates();
+                //         setGeneralSettings(settings);
-                    }
+                //         // electron.autoUpdater.setFeedURL({url: updaterUrl+(item.checked ? '?channel=beta' : ''), serverType: 'json'});
-                }, {
+                //         // return electron.autoUpdater.checkForUpdates();
+                //     }
+                // },
+                {
                     label: l('fixLogs.action'),
                     click: (_m, window: BrowserWindow) => window.webContents.send('fix-logs')
                 },
+
+                {type: 'separator'},
+                {
+                    label: 'Rising',
+                    submenu: [
+                        {
+                            label: 'System log level',
+                            submenu: ['error', 'warn', 'info', 'verbose', 'debug', 'silly'].map((level: string) => (
+                                {
+                                    checked: settings.risingSystemLogLevel === level,
+                                    label: `${level.substr(0, 1).toUpperCase()}${level.substr(1)}`,
+                                    click: () => setSystemLogLevel(level as log.LevelOption),
+                                    type: <'radio'>'radio'
+                                }
+                            ))
+                        }
+                    ]
+                },
+
                 {type: 'separator'},
                 {role: 'minimize'},
                 {

fchat/connection.ts

@@ -92,6 +92,14 @@ export default class Connection implements Interfaces.Connection {
         this.socket.onMessage(async(msg: string) => {
             const type = <keyof Interfaces.ServerCommands>msg.substr(0, 3);
             const data = msg.length > 6 ? <object>JSON.parse(msg.substr(4)) : undefined;
+
+            log.silly(
+              'socket.message',
+              {
+                type, data
+              }
+            );
+
             return this.handleMessage(type, data);
         });
         this.socket.onClose(async(event: CloseEvent) => {

learn/cache-manager.ts

@@ -16,6 +16,7 @@ import Message = Conversation.Message;
 import { Character } from '../fchat/interfaces';
 import Bluebird from 'bluebird';
 import ChatMessage = Conversation.ChatMessage;
+import { GeneralSettings } from '../electron/common';
 
 
 export interface ProfileCacheQueueEntry {
@@ -170,13 +171,15 @@ export class CacheManager {
     }
 
 
-    async start(): Promise<void> {
+    async start(settings: GeneralSettings): Promise<void> {
         await this.stop();
 
         this.profileStore = await IndexedStore.open();
 
         this.profileCache.setStore(this.profileStore);
 
+        await this.profileStore.flushProfiles(settings.risingCacheExpiryDays);
+
         EventBus.$on(
             'character-data',
             async(data: CharacterDataEvent) => {

learn/store/indexed.ts

@@ -1,3 +1,4 @@
+import log from 'electron-log'; //tslint:disable-line:match-default-export-name
 import * as _ from 'lodash';
 
 import {Character as ComplexCharacter, CharacterGroup, Guestbook} from '../../site/character_page/interfaces';
@@ -5,6 +6,8 @@ import { CharacterAnalysis } from '../matcher';
 import { PermanentIndexedStore, ProfileRecord } from './sql-store';
 import { CharacterImage, SimpleCharacter } from '../../interfaces';
 
+import Bluebird from 'bluebird';
+
 
 async function promisifyRequest<T>(req: IDBRequest): Promise<T> {
     return new Promise<T>((resolve, reject) => {
@@ -19,6 +22,7 @@ export class IndexedStore implements PermanentIndexedStore {
     protected db: IDBDatabase;
 
     protected static readonly STORE_NAME = 'profiles';
+    protected static readonly LAST_FETCHED_INDEX_NAME = 'idxLastFetched';
 
     constructor(db: IDBDatabase, dbName: string) {
         this.dbName = dbName;
@@ -26,12 +30,27 @@ export class IndexedStore implements PermanentIndexedStore {
     }
 
     static async open(dbName: string = 'flist-ascending-profiles'): Promise<IndexedStore> {
-        const request = window.indexedDB.open(dbName, 1);
+        const request = window.indexedDB.open(dbName, 2);
 
-        request.onupgradeneeded = () => {
+        request.onupgradeneeded = (event) => {
             const db = request.result;
 
-            db.createObjectStore(IndexedStore.STORE_NAME, { keyPath: 'id' });
+            if (event.oldVersion < 1) {
+                db.createObjectStore(IndexedStore.STORE_NAME, { keyPath: 'id' });
+            }
+
+            if (event.oldVersion < 2) {
+                const store = request.transaction!.objectStore(IndexedStore.STORE_NAME);
+
+                store.createIndex(
+                    IndexedStore.LAST_FETCHED_INDEX_NAME,
+                    'lastFetched',
+                  {
+                      unique: false,
+                      multiEntry: false
+                  }
+                );
+            }
         };
 
         return new IndexedStore(await promisifyRequest<IDBDatabase>(request), dbName);
@@ -190,5 +209,28 @@ export class IndexedStore implements PermanentIndexedStore {
     async stop(): Promise<void> {
         // empty
     }
+
+
+    async flushProfiles(daysToExpire: number): Promise<void> {
+        const tx = this.db.transaction(IndexedStore.STORE_NAME, 'readwrite');
+        const store = tx.objectStore(IndexedStore.STORE_NAME);
+        const idx = store.index(IndexedStore.LAST_FETCHED_INDEX_NAME);
+
+        const totalRecords = await promisifyRequest<number>(store.count());
+
+        const expirationTime = Math.round(Date.now() / 1000) - (daysToExpire * 24 * 60 * 60);
+        const getAllKeysRequest = idx.getAllKeys(IDBKeyRange.upperBound(expirationTime));
+        const result = await promisifyRequest<IDBValidKey[]>(getAllKeysRequest);
+
+        log.info('character.cache.expire', {daysToExpire, totalRecords, removableRecords: result.length});
+
+        await Bluebird.mapSeries(
+            result,
+            async(pk: IDBValidKey) => {
+                log.silly('character.cache.expire.name', { name: pk });
+                await promisifyRequest(store.delete(pk));
+            }
+        );
+    }
 }
 

learn/store/sql-store.ts

@@ -48,6 +48,8 @@ export interface PermanentIndexedStore {
         groups: CharacterGroup[] | null
     ): Promise<void>;
 
+    flushProfiles(daysToExpire: number): Promise<void>;
+
     start(): Promise<void>;
     stop(): Promise<void>;
 }

mobile/filesystem.ts

@@ -1,3 +1,5 @@
+import * as _ from 'lodash';
+
 import {Message as MessageImpl} from '../chat/common';
 import core from '../chat/core';
 import {Conversation, Logs as Logging, Settings} from '../chat/interfaces';
@@ -135,7 +137,7 @@ export class Logs implements Logging {
 export async function getGeneralSettings(): Promise<GeneralSettings | undefined> {
     const file = await NativeFile.read('!settings');
     if(file === undefined) return undefined;
-    const settings = <GeneralSettings>JSON.parse(file);
+    const settings = <GeneralSettings>_.merge(new GeneralSettings(),  JSON.parse(file));
     if(settings.host === 'wss://chat.f-list.net:9799') settings.host = 'wss://chat.f-list.net/chat2';
     return settings;
 }
@@ -158,4 +160,4 @@ export class SettingsStore implements Settings.Store {
     async getAvailableCharacters(): Promise<string[]> {
         return NativeFile.listDirectories('/');
     }
-}
+}