kittyfangs
/
fchat-rising
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Much more optimized fetching strategy for profiles

This commit is contained in:
Mr. Stallion 2020-06-16 11:27:31 -05:00
parent adacce1e7a
commit b052689afc
6 changed files with 137 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
chat/conversations.ts
View File

@ -1,6 +1,6 @@
import {queuedJoin} from '../fchat/channels';
import {decodeHTML} from '../fchat/common';
import { CharacterCacheRecord } from '../learn/profile-cache';
// import { CharacterCacheRecord } from '../learn/profile-cache';
import { AdManager } from './ads/ad-manager';
import { characterImage, ConversationSettings, EventMessage, Message, messageToString } from './common';
import core from './core';
@ -496,6 +496,7 @@ class State implements Interfaces.State {
this.selectedConversation.onHide();
conversation.unread = Interfaces.UnreadState.None;
this.selectedConversation = conversation;
EventBus.$emit('select-conversation', { conversation });
}
async reloadSettings(): Promise<void> {
@ -552,6 +553,7 @@ export default function(this: any): Interfaces.State {
state.privateMap = {};
} else state.consoleTab.unread = Interfaces.UnreadState.None;
state.selectedConversation = state.consoleTab;
EventBus.$emit('select-conversation', { conversation: state.selectedConversation });
await state.reloadSettings();
});
connection.onEvent('connected', (isReconnect) => {
@ -643,16 +645,12 @@ export default function(this: any): Interfaces.State {
const msg = new Message(MessageType.Ad, char, decodeHTML(data.message), time);
// this is done here so that the message will be rendered correctly when cache is hit
let p: CharacterCacheRecord | undefined;
if (core.characters.ownProfile) {
p = await core.cache.profileCache.get(char.name) || undefined;
if (p) {
msg.score = p.matchScore;
}
}
const p = await core.cache.resolvePScore(
(core.conversations.selectedConversation !== conv),
char,
conv,
msg
);
EventBus.$emit('channel-ad', { message: msg, channel: conv, profile: p });

1
chat/preview/event-bus.ts
View File

@ -13,6 +13,7 @@ import ChannelConversation = Conversation.ChannelConversation;
* 'private-message': {message: Message}
* 'channel-ad': {message: Message, channel: Conversation, profile: ComplexCharacter | undefined}
* 'channel-message': {message: Message, channel: Conversation}
* 'select-conversation': { conversation: Conversation }
*/

15
fchat/connection.ts
View File

@ -6,7 +6,14 @@ import ReadyState = WebSocketConnection.ReadyState;
const fatalErrors = [2, 3, 4, 9, 30, 31, 33, 39, 40, 62, -4];
const dieErrors = [9, 30, 31, 39, 40];
let lastFetch = Date.now();
async function queryApi(this: void, endpoint: string, data: object): Promise<AxiosResponse> {
if (false) {
console.log(`https://www.f-list.net/json/api/${endpoint}, gap: ${Date.now() - lastFetch}ms`);
lastFetch = Date.now();
}
return Axios.post(`https://www.f-list.net/json/api/${endpoint}`, qs.stringify(data));
}
@ -194,7 +201,13 @@ export default class Connection implements Interfaces.Connection {
//tslint:enable
private async getTicket(password: string): Promise<string> {
console.log('GET TICKET GET TICKET GET TICKET GET TICKET');
console.log('Acquiring new API ticket');
if (false) {
console.log(`https://www.f-list.net/json/getApiTicket.php, gap: ${Date.now() - lastFetch}ms`);
lastFetch = Date.now();
}
const data = <{ticket?: string, error: string}>(await Axios.post('https://www.f-list.net/json/getApiTicket.php', qs.stringify(
{account: this.account, password, no_friends: true, no_bookmarks: true, no_characters: true}))).data;
if(data.ticket !== undefined) return data.ticket;

112
learn/cache-manager.ts
View File

@ -1,18 +1,22 @@
import * as _ from 'lodash';
import core from '../chat/core';
import { ChannelAdEvent, ChannelMessageEvent, CharacterDataEvent, EventBus } from '../chat/preview/event-bus';
import { Conversation } from '../chat/interfaces';
import { Channel, Conversation } from '../chat/interfaces';
import { methods } from '../site/character_page/data_store';
import { Character as ComplexCharacter } from '../site/character_page/interfaces';
import { Gender } from './matcher';
import { AdCache } from './ad-cache';
import { ChannelConversationCache } from './channel-conversation-cache';
import { CharacterProfiler } from './character-profiler';
import { ProfileCache } from './profile-cache';
import { CharacterCacheRecord, ProfileCache } from './profile-cache';
import { IndexedStore } from './store/indexed';
import Timer = NodeJS.Timer;
import ChannelConversation = Conversation.ChannelConversation;
import Message = Conversation.Message;
import { Character } from '../fchat/interfaces';
import Bluebird from 'bluebird';
import ChatMessage = Conversation.ChatMessage;
export interface ProfileCacheQueueEntry {
name: string;
@ -20,11 +24,12 @@ export interface ProfileCacheQueueEntry {
added: Date;
gender?: Gender;
score: number;
channelId?: string;
}
export class CacheManager {
static readonly PROFILE_QUERY_DELAY = 1 * 1000; //1 * 1000;
static readonly PROFILE_QUERY_DELAY = 400; //1 * 1000;
adCache: AdCache = new AdCache();
profileCache: ProfileCache = new ProfileCache();
@ -48,11 +53,12 @@ export class CacheManager {
return this.lastPost;
}
async queueForFetching(name: string, skipCacheCheck: boolean = false): Promise<void> {
async queueForFetching(name: string, skipCacheCheck: boolean = false, channelId?: string): Promise<void> {
if (!core.state.settings.risingAdScore) {
return;
}
if (!skipCacheCheck) {
const c = await this.profileCache.get(name);
@ -70,6 +76,7 @@ export class CacheManager {
const entry: ProfileCacheQueueEntry = {
name,
key,
channelId,
added: new Date(),
score: 0
};
@ -203,7 +210,7 @@ export class CacheManager {
}
);
await this.addProfile(message.sender.name);
// await this.addProfile(message.sender.name);
}
);
@ -222,14 +229,69 @@ export class CacheManager {
}
);
if (!data.profile) {
await this.queueForFetching(message.sender.name, true);
if ((!data.profile) && (core.conversations.selectedConversation === data.channel)) {
await this.queueForFetching(message.sender.name, true, data.channel.channel.id);
}
// this.addProfile(message.sender.name);
}
);
EventBus.$on(
'select-conversation',
async(data: ChannelAdEvent) => {
const conversation = data.conversation as Conversation;
const channel = _.get(conversation, 'channel') as (Channel.Channel | undefined);
const channelId = _.get(channel, 'id', '<missing>');
// Remove unfinished fetches related to other channels
this.queue = _.reject(
this.queue,
(q) => (!!q.channelId) && (q.channelId !== channelId)
);
if (channel) {
const checkedNames: Record<string, boolean> = {};
// Add fetchers for unknown profiles in ads
await Bluebird.each(
_.filter(
conversation.messages,
(m) => {
if (m.type !== Message.Type.Ad) {
return false;
}
const chatMessage = m as unknown as ChatMessage;
if (chatMessage.sender.name in checkedNames) {
return false;
}
checkedNames[chatMessage.sender.name] = true;
return true;
}
),
async(m: Message) => {
const chatMessage: ChatMessage = m as unknown as ChatMessage;
if (chatMessage.score) {
return;
}
const p = await this.resolvePScore(false, chatMessage.sender, conversation as ChannelConversation, chatMessage);
if (!p) {
await this.queueForFetching(chatMessage.sender.name, true, channel.id);
}
}
);
}
}
);
// EventBus.$on(
// 'private-message',
// (data: any) => {}
@ -243,7 +305,11 @@ export class CacheManager {
if (next) {
try {
// console.log('Learn fetch', next.name, next.score);
if ((false) && (next)) {
console.log(`Fetch '${next.name}' for channel '${next.channelId}', gap: ${(Date.now() - this.lastFetch)}ms`);
this.lastFetch = Date.now();
}
await this.fetchProfile(next.name);
} catch (err) {
console.error('Profile queue error', err);
@ -262,6 +328,36 @@ export class CacheManager {
}
protected lastFetch = Date.now();
async resolvePScore(
skipStore: boolean,
char: Character.Character,
conv: ChannelConversation,
msg?: Message
): Promise<CharacterCacheRecord | undefined> {
if (!core.characters.ownProfile) {
return undefined;
}
// this is done here so that the message will be rendered correctly when cache is hit
let p: CharacterCacheRecord | undefined;
p = await core.cache.profileCache.get(
char.name,
skipStore,
conv.channel.name
) || undefined;
if ((p) && (msg)) {
msg.score = p.matchScore;
}
return p;
}
async stop(): Promise<void> {
if (this.profileTimer) {
clearTimeout(this.profileTimer);

9
learn/profile-cache.ts
View File

@ -36,6 +36,8 @@ export interface CharacterCacheRecord {
export class ProfileCache extends AsyncCache<CharacterCacheRecord> {
protected store?: PermanentIndexedStore;
protected lastFetch = Date.now();
setStore(store: PermanentIndexedStore): void {
this.store = store;
@ -53,7 +55,7 @@ export class ProfileCache extends AsyncCache<CharacterCacheRecord> {
}
async get(name: string, skipStore: boolean = false): Promise<CharacterCacheRecord | null> {
async get(name: string, skipStore: boolean = false, fromChannel?: string): Promise<CharacterCacheRecord | null> {
const key = AsyncCache.nameKey(name);
if (key in this.cache) {
@ -64,6 +66,11 @@ export class ProfileCache extends AsyncCache<CharacterCacheRecord> {
return null;
}
if (false) {
console.log(`Retrieve '${name}' for channel '${fromChannel}, gap: ${(Date.now() - this.lastFetch)}ms`);
this.lastFetch = Date.now();
}
const pd = await this.store.getProfile(name);
if (!pd) {

2
webchat/chat.ts
View File

@ -53,7 +53,7 @@ if(process.env.NODE_ENV === 'production')
declare const chatSettings: {account: string, theme: string, characters: ReadonlyArray<SimpleCharacter>, defaultCharacter: number | null};
const ticketProvider = async() => {
// console.log('PROVIDER GET TICKET GET TICKET GET TICKET');
console.log('TICK TICK TICK TICK');
const data = (await Axios.post<{ticket?: string, error: string}>(
'/json/getApiTicket.php?no_friends=true&no_bookmarks=true&no_characters=true')).data;