kittyfangs
/
fchat-rising
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cache expiration

This commit is contained in:
Mr. Stallion 2020-06-30 16:51:06 -05:00
parent 24aced5b64
commit 80dfe32006
12 changed files with 188 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -1,10 +1,11 @@
# Changelog
## Canary
* Fixed caching issue that causes cache misses on charater page metadata
* Fixed ad posting issue that sometimes disconnects characters if multiple characters are in use
* Fixed a caching issue that caused cache misses on character page metadata
* Fixed rate limit issues that sometimes disconnected characters when multiple characters were connected
* URL preview fixes for Redgifs, Gelbooru, Tumblr, and Gifmixxx
* All dependencies are now up to date
* F-Chat Rising now flushes character profiles out of its cache after 30 days
## 1.0.1

9
chat/core.ts
View File

@ -6,6 +6,7 @@ import {Settings as SettingsImpl} from './common';
import Conversations from './conversations';
import {Channel, Character, Connection, Conversation, Logs, Notifications, Settings, State as StateInterface} from './interfaces';
import { AdCoordinatorGuest } from './ads/ad-coordinator-guest';
import { GeneralSettings } from '../electron/common';
function createBBCodeParser(): BBCodeParser {
const parser = new BBCodeParser();
@ -82,8 +83,9 @@ const data = {
}
};
export function init(this: any, connection: Connection, logsClass: new() => Logs, settingsClass: new() => Settings.Store,
notificationsClass: new() => Notifications): void {
export function init(
this: any, connection: Connection, settings: GeneralSettings, logsClass: new() => Logs,
settingsClass: new() => Settings.Store, notificationsClass: new() => Notifications): void {
data.connection = connection;
data.logs = new logsClass();
data.settingsStore = new settingsClass();
@ -91,8 +93,7 @@ export function init(this: any, connection: Connection, logsClass: new() => Logs
data.cache = new CacheManager();
data.adCoordinator = new AdCoordinatorGuest();
// tslint:disable-next-line no-floating-promises
data.cache.start();
(data.state as any).generalSettings = settings;
data.register('characters', Characters(connection));
data.register('channels', Channels(connection, core.characters));

60
electron/Index.vue
View File

@ -3,8 +3,16 @@
<div v-html="styling"></div>
<div v-if="!characters" style="display:flex; align-items:center; justify-content:center; height: 100%;">
<div class="card bg-light" style="width: 400px;">
<div class="initializer" v-show="showSpinner">
<div class="title">
Getting ready, please wait...
</div>
<i class="fas fa-circle-notch fa-spin search-spinner"></i>
</div>
<h3 class="card-header" style="margin-top:0;display:flex">
{{l('title')}}
<a href="#" @click.prevent="showLogs()" class="btn" style="flex:1;text-align:right">
<span class="fa fa-file-alt"></span> <span class="btn-text">{{l('logs.title')}}</span>
</a>
@ -92,7 +100,7 @@
import Vue from 'vue';
import Chat from '../chat/Chat.vue';
import {getKey, Settings} from '../chat/common';
import core from '../chat/core';
import core /*, { init as initCore }*/ from '../chat/core';
import l from '../chat/localize';
import Logs from '../chat/Logs.vue';
import Socket from '../chat/WebSocket';
@ -103,8 +111,10 @@
// import { Sqlite3Store } from '../learn/store/sqlite3';
import CharacterPage from '../site/character_page/character_page.vue';
import {defaultHost, GeneralSettings, nativeRequire} from './common';
import {fixLogs} from './filesystem';
import { fixLogs /*SettingsStore, Logs as FSLogs*/ } from './filesystem';
import * as SlimcatImporter from './importer';
// import Connection from '../fchat/connection';
// import Notifications from './notifications';
const webContents = electron.remote.getCurrentWebContents();
const parent = electron.remote.getCurrentWindow().webContents;
@ -163,8 +173,17 @@
fixCharacters: ReadonlyArray<string> = [];
fixCharacter = '';
showSpinner = true;
@Hook('created')
created(): void {
async created(): Promise<void> {
// tslint:disable-next-line no-floating-promises
await core.cache.start(this.settings);
// await this.prepper;
this.showSpinner = false;
if(this.settings.account.length > 0) this.saveLogin = true;
keyStore.getPassword(this.settings.account)
.then((value: string) => this.password = value, (err: Error) => this.error = err.message);
@ -347,7 +366,7 @@
}
</script>
<style>
<style lang="scss">
html, body, #page {
height: 100%;
}
@ -362,4 +381,37 @@
font-size: 12pt;
opacity: 0.5;
}
.initializer {
position: absolute;
top: 0;
left: 0;
right: 0;
bottom: 0;
display: flex;
align-items: center;
justify-content: center;
transition: all 0.25s;
backdrop-filter: blur(3px) grayscale(35%);
i {
font-size: 130pt;
top: 50%;
right: 50%;
transform: translate(-50%, -50%);
width: fit-content;
}
.title {
position: absolute;
top: 0;
background: rgba(19, 19, 19, 0.6);
width: 100%;
text-align: center;
padding-top: 20px;
padding-bottom: 20px;
font-weight: bold;
}
}
</style>

9
electron/Window.vue
View File

@ -45,6 +45,7 @@
import l from '../chat/localize';
import {GeneralSettings} from './common';
import { getSafeLanguages, updateSupportedLanguages } from './language';
import log from 'electron-log'; // tslint:disable-line: match-default-export-name
const browserWindow = electron.remote.getCurrentWindow();
@ -97,7 +98,13 @@
await this.addTab();
electron.ipcRenderer.on('settings', (_e: Event, settings: GeneralSettings) => this.settings = settings);
electron.ipcRenderer.on('settings', (_e: Event, settings: GeneralSettings) => {
this.settings = settings;
log.transports.file.level = settings.risingSystemLogLevel;
log.transports.console.level = settings.risingSystemLogLevel;
});
electron.ipcRenderer.on('allow-new-tabs', (_e: Event, allow: boolean) => this.canOpenTab = allow);
electron.ipcRenderer.on('open-tab', () => this.addTab());
electron.ipcRenderer.on('update-available', (_e: Event, available: boolean) => this.hasUpdate = available);

9
electron/chat.ts
View File

@ -52,6 +52,7 @@ import {Logs, SettingsStore} from './filesystem';
import Notifications from './notifications';
import * as SlimcatImporter from './importer';
import Index from './Index.vue';
import log from 'electron-log'; // tslint:disable-line: match-default-export-name
document.addEventListener('keydown', (e: KeyboardEvent) => {
@ -192,6 +193,9 @@ if(process.platform === 'win32') //get the path in DOS (8-character) format as s
function onSettings(s: GeneralSettings): void {
settings = s;
log.transports.file.level = settings.risingSystemLogLevel;
log.transports.console.level = settings.risingSystemLogLevel;
// spellchecker.setDictionary(s.spellcheckLang, dictDir);
// for(const word of s.customDictionary) spellchecker.add(word);
}
@ -200,6 +204,9 @@ electron.ipcRenderer.on('settings', (_: Event, s: GeneralSettings) => onSettings
const params = <{[key: string]: string | undefined}>qs.parse(window.location.search.substr(1));
let settings = <GeneralSettings>JSON.parse(params['settings']!);
// console.log('SETTINGS', settings);
if(params['import'] !== undefined)
try {
if(SlimcatImporter.canImportGeneral() && confirm(l('importer.importGeneral'))) {
@ -212,7 +219,7 @@ if(params['import'] !== undefined)
onSettings(settings);
const connection = new Connection(`F-Chat 3.0 (${process.platform})`, electron.remote.app.getVersion(), Socket);
initCore(connection, Logs, SettingsStore, Notifications);
initCore(connection, settings, Logs, SettingsStore, Notifications);
//tslint:disable-next-line:no-unused-expression
new Index({

4
electron/common.ts
View File

@ -1,6 +1,8 @@
import * as electron from 'electron';
import * as path from 'path';
import log from 'electron-log'; //tslint:disable-line:match-default-export-name
export const defaultHost = 'wss://chat.f-list.net/chat2';
export class GeneralSettings {
@ -15,6 +17,8 @@ export class GeneralSettings {
beta = false;
customDictionary: string[] = [];
hwAcceleration = true;
risingCacheExpiryDays = 45;
risingSystemLogLevel: log.LevelOption = 'info';
}
//tslint:disable

50
electron/main.ts
View File

@ -134,7 +134,11 @@ async function toggleDictionary(lang: string): Promise<void> {
function setGeneralSettings(value: GeneralSettings): void {
fs.writeFileSync(path.join(settingsDir, 'settings'), JSON.stringify(value));
for(const w of electron.webContents.getAllWebContents()) w.send('settings', settings);
shouldImportSettings = false;
log.transports.file.level = settings.risingSystemLogLevel;
log.transports.console.level = settings.risingSystemLogLevel;
}
async function addSpellcheckerItems(menu: Electron.Menu): Promise<void> {
@ -380,6 +384,12 @@ function onReady(): void {
settings.theme = theme;
setGeneralSettings(settings);
};
const setSystemLogLevel = (logLevel: log.LevelOption) => {
settings.risingSystemLogLevel = logLevel;
setGeneralSettings(settings);
};
electron.Menu.setApplicationMenu(electron.Menu.buildFromTemplate([
{
label: `&${l('title')}`,
@ -442,18 +452,40 @@ function onReady(): void {
settings.hwAcceleration = item.checked;
setGeneralSettings(settings);
}
}, {
label: l('settings.beta'), type: 'checkbox', checked: settings.beta,
click: async(item: Electron.MenuItem) => {
settings.beta = item.checked;
setGeneralSettings(settings);
// electron.autoUpdater.setFeedURL({url: updaterUrl + (item.checked ? '?channel=beta' : ''), serverType: 'json'});
// return electron.autoUpdater.checkForUpdates();
}
}, {
},
// {
// label: l('settings.beta'), type: 'checkbox', checked: settings.beta,
// click: async(item: Electron.MenuItem) => {
// settings.beta = item.checked;
// setGeneralSettings(settings);
// // electron.autoUpdater.setFeedURL({url: updaterUrl+(item.checked ? '?channel=beta' : ''), serverType: 'json'});
// // return electron.autoUpdater.checkForUpdates();
// }
// },
{
label: l('fixLogs.action'),
click: (_m, window: BrowserWindow) => window.webContents.send('fix-logs')
},
{type: 'separator'},
{
label: 'Rising',
submenu: [
{
label: 'System log level',
submenu: ['error', 'warn', 'info', 'verbose', 'debug', 'silly'].map((level: string) => (
{
checked: settings.risingSystemLogLevel === level,
label: `${level.substr(0, 1).toUpperCase()}${level.substr(1)}`,
click: () => setSystemLogLevel(level as log.LevelOption),
type: <'radio'>'radio'
}
))
}
]
},
{type: 'separator'},
{role: 'minimize'},
{

8
fchat/connection.ts
View File

@ -92,6 +92,14 @@ export default class Connection implements Interfaces.Connection {
this.socket.onMessage(async(msg: string) => {
const type = <keyof Interfaces.ServerCommands>msg.substr(0, 3);
const data = msg.length > 6 ? <object>JSON.parse(msg.substr(4)) : undefined;
log.silly(
'socket.message',
{
type, data
}
);
return this.handleMessage(type, data);
});
this.socket.onClose(async(event: CloseEvent) => {

5
learn/cache-manager.ts
View File

@ -16,6 +16,7 @@ import Message = Conversation.Message;
import { Character } from '../fchat/interfaces';
import Bluebird from 'bluebird';
import ChatMessage = Conversation.ChatMessage;
import { GeneralSettings } from '../electron/common';
export interface ProfileCacheQueueEntry {
@ -170,13 +171,15 @@ export class CacheManager {
}
async start(): Promise<void> {
async start(settings: GeneralSettings): Promise<void> {
await this.stop();
this.profileStore = await IndexedStore.open();
this.profileCache.setStore(this.profileStore);
await this.profileStore.flushProfiles(settings.risingCacheExpiryDays);
EventBus.$on(
'character-data',
async(data: CharacterDataEvent) => {

48
learn/store/indexed.ts
View File

@ -1,3 +1,4 @@
import log from 'electron-log'; //tslint:disable-line:match-default-export-name
import * as _ from 'lodash';
import {Character as ComplexCharacter, CharacterGroup, Guestbook} from '../../site/character_page/interfaces';
@ -5,6 +6,8 @@ import { CharacterAnalysis } from '../matcher';
import { PermanentIndexedStore, ProfileRecord } from './sql-store';
import { CharacterImage, SimpleCharacter } from '../../interfaces';
import Bluebird from 'bluebird';
async function promisifyRequest<T>(req: IDBRequest): Promise<T> {
return new Promise<T>((resolve, reject) => {
@ -19,6 +22,7 @@ export class IndexedStore implements PermanentIndexedStore {
protected db: IDBDatabase;
protected static readonly STORE_NAME = 'profiles';
protected static readonly LAST_FETCHED_INDEX_NAME = 'idxLastFetched';
constructor(db: IDBDatabase, dbName: string) {
this.dbName = dbName;
@ -26,12 +30,27 @@ export class IndexedStore implements PermanentIndexedStore {
}
static async open(dbName: string = 'flist-ascending-profiles'): Promise<IndexedStore> {
const request = window.indexedDB.open(dbName, 1);
const request = window.indexedDB.open(dbName, 2);
request.onupgradeneeded = () => {
request.onupgradeneeded = (event) => {
const db = request.result;
db.createObjectStore(IndexedStore.STORE_NAME, { keyPath: 'id' });
if (event.oldVersion < 1) {
db.createObjectStore(IndexedStore.STORE_NAME, { keyPath: 'id' });
}
if (event.oldVersion < 2) {
const store = request.transaction!.objectStore(IndexedStore.STORE_NAME);
store.createIndex(
IndexedStore.LAST_FETCHED_INDEX_NAME,
'lastFetched',
{
unique: false,
multiEntry: false
}
);
}
};
return new IndexedStore(await promisifyRequest<IDBDatabase>(request), dbName);
@ -190,5 +209,28 @@ export class IndexedStore implements PermanentIndexedStore {
async stop(): Promise<void> {
// empty
}
async flushProfiles(daysToExpire: number): Promise<void> {
const tx = this.db.transaction(IndexedStore.STORE_NAME, 'readwrite');
const store = tx.objectStore(IndexedStore.STORE_NAME);
const idx = store.index(IndexedStore.LAST_FETCHED_INDEX_NAME);
const totalRecords = await promisifyRequest<number>(store.count());
const expirationTime = Math.round(Date.now() / 1000) - (daysToExpire * 24 * 60 * 60);
const getAllKeysRequest = idx.getAllKeys(IDBKeyRange.upperBound(expirationTime));
const result = await promisifyRequest<IDBValidKey[]>(getAllKeysRequest);
log.info('character.cache.expire', {daysToExpire, totalRecords, removableRecords: result.length});
await Bluebird.mapSeries(
result,
async(pk: IDBValidKey) => {
log.silly('character.cache.expire.name', { name: pk });
await promisifyRequest(store.delete(pk));
}
);
}
}

2
learn/store/sql-store.ts
View File

@ -48,6 +48,8 @@ export interface PermanentIndexedStore {
groups: CharacterGroup[] | null
): Promise<void>;
flushProfiles(daysToExpire: number): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
}

6
mobile/filesystem.ts
View File

@ -1,3 +1,5 @@
import * as _ from 'lodash';
import {Message as MessageImpl} from '../chat/common';
import core from '../chat/core';
import {Conversation, Logs as Logging, Settings} from '../chat/interfaces';
@ -135,7 +137,7 @@ export class Logs implements Logging {
export async function getGeneralSettings(): Promise<GeneralSettings | undefined> {
const file = await NativeFile.read('!settings');
if(file === undefined) return undefined;
const settings = <GeneralSettings>JSON.parse(file);
const settings = <GeneralSettings>_.merge(new GeneralSettings(), JSON.parse(file));
if(settings.host === 'wss://chat.f-list.net:9799') settings.host = 'wss://chat.f-list.net/chat2';
return settings;
}
@ -158,4 +160,4 @@ export class SettingsStore implements Settings.Store {
async getAvailableCharacters(): Promise<string[]> {
return NativeFile.listDirectories('/');
}
}
}