From b052689afcaa8e3ea23bdaed96e04c8d1e508ed2 Mon Sep 17 00:00:00 2001 From: "Mr. Stallion" Date: Tue, 16 Jun 2020 11:27:31 -0500 Subject: [PATCH] Much more optimized fetching strategy for profiles --- chat/conversations.ts | 20 +++---- chat/preview/event-bus.ts | 1 + fchat/connection.ts | 15 ++++- learn/cache-manager.ts | 112 +++++++++++++++++++++++++++++++++++--- learn/profile-cache.ts | 9 ++- webchat/chat.ts | 2 +- 6 files changed, 137 insertions(+), 22 deletions(-) diff --git a/chat/conversations.ts b/chat/conversations.ts index a587a65..5b6724d 100644 --- a/chat/conversations.ts +++ b/chat/conversations.ts @@ -1,6 +1,6 @@ import {queuedJoin} from '../fchat/channels'; import {decodeHTML} from '../fchat/common'; -import { CharacterCacheRecord } from '../learn/profile-cache'; +// import { CharacterCacheRecord } from '../learn/profile-cache'; import { AdManager } from './ads/ad-manager'; import { characterImage, ConversationSettings, EventMessage, Message, messageToString } from './common'; import core from './core'; @@ -496,6 +496,7 @@ class State implements Interfaces.State { this.selectedConversation.onHide(); conversation.unread = Interfaces.UnreadState.None; this.selectedConversation = conversation; + EventBus.$emit('select-conversation', { conversation }); } async reloadSettings(): Promise { @@ -552,6 +553,7 @@ export default function(this: any): Interfaces.State { state.privateMap = {}; } else state.consoleTab.unread = Interfaces.UnreadState.None; state.selectedConversation = state.consoleTab; + EventBus.$emit('select-conversation', { conversation: state.selectedConversation }); await state.reloadSettings(); }); connection.onEvent('connected', (isReconnect) => { @@ -643,16 +645,12 @@ export default function(this: any): Interfaces.State { const msg = new Message(MessageType.Ad, char, decodeHTML(data.message), time); - // this is done here so that the message will be rendered correctly when cache is hit - let p: CharacterCacheRecord | undefined; - - if (core.characters.ownProfile) { - p = await core.cache.profileCache.get(char.name) || undefined; - - if (p) { - msg.score = p.matchScore; - } - } + const p = await core.cache.resolvePScore( + (core.conversations.selectedConversation !== conv), + char, + conv, + msg + ); EventBus.$emit('channel-ad', { message: msg, channel: conv, profile: p }); diff --git a/chat/preview/event-bus.ts b/chat/preview/event-bus.ts index 0a14730..c9ec95d 100644 --- a/chat/preview/event-bus.ts +++ b/chat/preview/event-bus.ts @@ -13,6 +13,7 @@ import ChannelConversation = Conversation.ChannelConversation; * 'private-message': {message: Message} * 'channel-ad': {message: Message, channel: Conversation, profile: ComplexCharacter | undefined} * 'channel-message': {message: Message, channel: Conversation} + * 'select-conversation': { conversation: Conversation } */ diff --git a/fchat/connection.ts b/fchat/connection.ts index 24511b8..9f68890 100644 --- a/fchat/connection.ts +++ b/fchat/connection.ts @@ -6,7 +6,14 @@ import ReadyState = WebSocketConnection.ReadyState; const fatalErrors = [2, 3, 4, 9, 30, 31, 33, 39, 40, 62, -4]; const dieErrors = [9, 30, 31, 39, 40]; +let lastFetch = Date.now(); + async function queryApi(this: void, endpoint: string, data: object): Promise { + if (false) { + console.log(`https://www.f-list.net/json/api/${endpoint}, gap: ${Date.now() - lastFetch}ms`); + lastFetch = Date.now(); + } + return Axios.post(`https://www.f-list.net/json/api/${endpoint}`, qs.stringify(data)); } @@ -194,7 +201,13 @@ export default class Connection implements Interfaces.Connection { //tslint:enable private async getTicket(password: string): Promise { - console.log('GET TICKET GET TICKET GET TICKET GET TICKET'); + console.log('Acquiring new API ticket'); + + if (false) { + console.log(`https://www.f-list.net/json/getApiTicket.php, gap: ${Date.now() - lastFetch}ms`); + lastFetch = Date.now(); + } + const data = <{ticket?: string, error: string}>(await Axios.post('https://www.f-list.net/json/getApiTicket.php', qs.stringify( {account: this.account, password, no_friends: true, no_bookmarks: true, no_characters: true}))).data; if(data.ticket !== undefined) return data.ticket; diff --git a/learn/cache-manager.ts b/learn/cache-manager.ts index 47c9d5d..d88165f 100644 --- a/learn/cache-manager.ts +++ b/learn/cache-manager.ts @@ -1,18 +1,22 @@ import * as _ from 'lodash'; import core from '../chat/core'; import { ChannelAdEvent, ChannelMessageEvent, CharacterDataEvent, EventBus } from '../chat/preview/event-bus'; -import { Conversation } from '../chat/interfaces'; +import { Channel, Conversation } from '../chat/interfaces'; import { methods } from '../site/character_page/data_store'; import { Character as ComplexCharacter } from '../site/character_page/interfaces'; import { Gender } from './matcher'; import { AdCache } from './ad-cache'; import { ChannelConversationCache } from './channel-conversation-cache'; import { CharacterProfiler } from './character-profiler'; -import { ProfileCache } from './profile-cache'; +import { CharacterCacheRecord, ProfileCache } from './profile-cache'; import { IndexedStore } from './store/indexed'; import Timer = NodeJS.Timer; import ChannelConversation = Conversation.ChannelConversation; import Message = Conversation.Message; +import { Character } from '../fchat/interfaces'; +import Bluebird from 'bluebird'; +import ChatMessage = Conversation.ChatMessage; + export interface ProfileCacheQueueEntry { name: string; @@ -20,11 +24,12 @@ export interface ProfileCacheQueueEntry { added: Date; gender?: Gender; score: number; + channelId?: string; } export class CacheManager { - static readonly PROFILE_QUERY_DELAY = 1 * 1000; //1 * 1000; + static readonly PROFILE_QUERY_DELAY = 400; //1 * 1000; adCache: AdCache = new AdCache(); profileCache: ProfileCache = new ProfileCache(); @@ -48,11 +53,12 @@ export class CacheManager { return this.lastPost; } - async queueForFetching(name: string, skipCacheCheck: boolean = false): Promise { + async queueForFetching(name: string, skipCacheCheck: boolean = false, channelId?: string): Promise { if (!core.state.settings.risingAdScore) { return; } + if (!skipCacheCheck) { const c = await this.profileCache.get(name); @@ -70,6 +76,7 @@ export class CacheManager { const entry: ProfileCacheQueueEntry = { name, key, + channelId, added: new Date(), score: 0 }; @@ -203,7 +210,7 @@ export class CacheManager { } ); - await this.addProfile(message.sender.name); + // await this.addProfile(message.sender.name); } ); @@ -222,14 +229,69 @@ export class CacheManager { } ); - if (!data.profile) { - await this.queueForFetching(message.sender.name, true); + if ((!data.profile) && (core.conversations.selectedConversation === data.channel)) { + await this.queueForFetching(message.sender.name, true, data.channel.channel.id); } // this.addProfile(message.sender.name); } ); + + EventBus.$on( + 'select-conversation', + async(data: ChannelAdEvent) => { + const conversation = data.conversation as Conversation; + const channel = _.get(conversation, 'channel') as (Channel.Channel | undefined); + const channelId = _.get(channel, 'id', ''); + + // Remove unfinished fetches related to other channels + this.queue = _.reject( + this.queue, + (q) => (!!q.channelId) && (q.channelId !== channelId) + ); + + if (channel) { + const checkedNames: Record = {}; + + // Add fetchers for unknown profiles in ads + await Bluebird.each( + _.filter( + conversation.messages, + (m) => { + if (m.type !== Message.Type.Ad) { + return false; + } + + const chatMessage = m as unknown as ChatMessage; + + if (chatMessage.sender.name in checkedNames) { + return false; + } + + checkedNames[chatMessage.sender.name] = true; + return true; + } + ), + async(m: Message) => { + const chatMessage: ChatMessage = m as unknown as ChatMessage; + + if (chatMessage.score) { + return; + } + + const p = await this.resolvePScore(false, chatMessage.sender, conversation as ChannelConversation, chatMessage); + + if (!p) { + await this.queueForFetching(chatMessage.sender.name, true, channel.id); + } + } + ); + } + } + ); + + // EventBus.$on( // 'private-message', // (data: any) => {} @@ -243,7 +305,11 @@ export class CacheManager { if (next) { try { - // console.log('Learn fetch', next.name, next.score); + if ((false) && (next)) { + console.log(`Fetch '${next.name}' for channel '${next.channelId}', gap: ${(Date.now() - this.lastFetch)}ms`); + this.lastFetch = Date.now(); + } + await this.fetchProfile(next.name); } catch (err) { console.error('Profile queue error', err); @@ -262,6 +328,36 @@ export class CacheManager { } + protected lastFetch = Date.now(); + + + async resolvePScore( + skipStore: boolean, + char: Character.Character, + conv: ChannelConversation, + msg?: Message + ): Promise { + if (!core.characters.ownProfile) { + return undefined; + } + + // this is done here so that the message will be rendered correctly when cache is hit + let p: CharacterCacheRecord | undefined; + + p = await core.cache.profileCache.get( + char.name, + skipStore, + conv.channel.name + ) || undefined; + + if ((p) && (msg)) { + msg.score = p.matchScore; + } + + return p; + } + + async stop(): Promise { if (this.profileTimer) { clearTimeout(this.profileTimer); diff --git a/learn/profile-cache.ts b/learn/profile-cache.ts index 9dbbdcc..747d4d4 100644 --- a/learn/profile-cache.ts +++ b/learn/profile-cache.ts @@ -36,6 +36,8 @@ export interface CharacterCacheRecord { export class ProfileCache extends AsyncCache { protected store?: PermanentIndexedStore; + protected lastFetch = Date.now(); + setStore(store: PermanentIndexedStore): void { this.store = store; @@ -53,7 +55,7 @@ export class ProfileCache extends AsyncCache { } - async get(name: string, skipStore: boolean = false): Promise { + async get(name: string, skipStore: boolean = false, fromChannel?: string): Promise { const key = AsyncCache.nameKey(name); if (key in this.cache) { @@ -64,6 +66,11 @@ export class ProfileCache extends AsyncCache { return null; } + if (false) { + console.log(`Retrieve '${name}' for channel '${fromChannel}, gap: ${(Date.now() - this.lastFetch)}ms`); + this.lastFetch = Date.now(); + } + const pd = await this.store.getProfile(name); if (!pd) { diff --git a/webchat/chat.ts b/webchat/chat.ts index 8fccc39..c317b00 100644 --- a/webchat/chat.ts +++ b/webchat/chat.ts @@ -53,7 +53,7 @@ if(process.env.NODE_ENV === 'production') declare const chatSettings: {account: string, theme: string, characters: ReadonlyArray, defaultCharacter: number | null}; const ticketProvider = async() => { - // console.log('PROVIDER GET TICKET GET TICKET GET TICKET'); + console.log('TICK TICK TICK TICK'); const data = (await Axios.post<{ticket?: string, error: string}>( '/json/getApiTicket.php?no_friends=true&no_bookmarks=true&no_characters=true')).data;