kittyfangs
/
fchat-rising
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Switched to IndexedDb cache for profiles

This commit is contained in:
Mr. Stallion 2019-07-08 14:08:16 -05:00
parent b983f388e1
commit 68097817d5
29 changed files with 3938 additions and 402 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -5,3 +5,7 @@ node_modules/
/webchat/dist
.idea/
.DS_Store

8
chat/conversations.ts
View File

@ -1,5 +1,6 @@
import {queuedJoin} from '../fchat/channels';
import {decodeHTML} from '../fchat/common';
import { CharacterCacheRecord } from '../learn/profile-cache';
import { AdManager } from './ad-manager';
import { characterImage, ConversationSettings, EventMessage, Message, messageToString } from './common';
import core from './core';
@ -591,15 +592,18 @@ export default function(this: void): Interfaces.State {
if((char.isIgnored || core.state.hiddenUsers.indexOf(char.name) !== -1) && !isOp(conv)) return;
const msg = new Message(MessageType.Ad, char, decodeHTML(data.message), time);
// this is done here so that the message will be rendered correctly when cache is hit
let p: CharacterCacheRecord | undefined;
if (core.characters.ownProfile) {
const p = core.cache.profileCache.get(char.name);
p = await core.cache.profileCache.get(char.name) || undefined;
if (p) {
msg.score = p.matchScore;
}
}
EventBus.$emit('channel-ad', { message: msg, channel: conv });
EventBus.$emit('channel-ad', { message: msg, channel: conv, profile: p });
await conv.addMessage(msg);
});
connection.onMessage('RLL', async(data, time) => {

1
chat/core.ts
View File

@ -89,6 +89,7 @@ export function init(this: void, connection: Connection, logsClass: new() => Log
data.notifications = new notificationsClass();
data.cache = new CacheManager();
// tslint:disable-next-line no-floating-promises
data.cache.start();
connection.onEvent('connecting', async() => {

2
chat/event-bus.ts
View File

@ -10,7 +10,7 @@ import ChannelConversation = Conversation.ChannelConversation;
* 'imagepreview-toggle-stickyness': {url: string}
* 'character-data': {character: Character}
* 'private-message': {message: Message}
* 'channel-ad': {message: Message, channel: Conversation}
* 'channel-ad': {message: Message, channel: Conversation, profile: ComplexCharacter | undefined}
* 'channel-message': {message: Message, channel: Conversation}
*/

6
chat/message_view.ts
View File

@ -75,10 +75,10 @@ export default class MessageView extends Vue {
@Hook('beforeDestroy')
onBeforeDestroy(): void {
console.log('onbeforedestroy');
// console.log('onbeforedestroy');
if (this.scoreWatcher) {
console.log('onbeforedestroy killed');
// console.log('onbeforedestroy killed');
this.scoreWatcher(); // stop watching
this.scoreWatcher = null;
@ -96,7 +96,7 @@ export default class MessageView extends Vue {
}
if (this.scoreWatcher) {
console.log('watch killed');
// console.log('watch killed');
this.scoreWatcher(); // stop watching
this.scoreWatcher = null;

2
chat/profile_api.ts
View File

@ -33,6 +33,8 @@ const parserSettings = {
// tslint:disable-next-line: ban-ts-ignore
// @ts-ignore
async function characterData(name: string | undefined, id: number = -1, skipEvent: boolean = false): Promise<Character> {
// console.log('CharacterDataquery', name);
const data = await core.connection.queryApi<CharacterInfo & {
badges: string[]
customs_first: boolean

2
electron/Index.vue
View File

@ -87,6 +87,8 @@
import Modal from '../components/Modal.vue';
import Connection from '../fchat/connection';
import {Keys} from '../keys';
// import { BetterSqliteStore } from '../learn/store/better-sqlite3';
// import { Sqlite3Store } from '../learn/store/sqlite3';
import CharacterPage from '../site/character_page/character_page.vue';
import {defaultHost, GeneralSettings, nativeRequire} from './common';
import {fixLogs, Logs, SettingsStore} from './filesystem';

5
electron/chat.ts
View File

@ -29,6 +29,11 @@
* @version 3.0
* @see {@link https://github.com/f-list/exported|GitHub repo}
*/
// import { DebugLogger } from './debug-logger';
// // @ts-ignore
// const dl = new DebugLogger('chat');
import Axios from 'axios';
import {exec, execSync} from 'child_process';
import * as electron from 'electron';

36
electron/filesystem.ts
View File

@ -66,15 +66,15 @@ export function checkIndex(this: void, index: Index, message: Message, key: stri
index[key] = item = {name, index: {}, offsets: []};
const nameLength = Buffer.byteLength(name);
buffer = Buffer.allocUnsafe(nameLength + 8);
buffer.writeUInt8(nameLength, 0, noAssert);
buffer.writeUInt8(nameLength, 0);
buffer.write(name, 1);
offset = nameLength + 1;
}
const newValue = typeof size === 'function' ? size() : size;
item.index[date] = item.offsets.length;
item.offsets.push(newValue);
buffer.writeUInt16LE(date, offset, noAssert);
buffer.writeUIntLE(newValue, offset + 2, 5, noAssert);
buffer.writeUInt16LE(date, offset);
buffer.writeUIntLE(newValue, offset + 2, 5);
return buffer;
}
@ -83,25 +83,27 @@ export function serializeMessage(message: Message): {serialized: Buffer, size: n
const senderLength = Buffer.byteLength(name);
const messageLength = Buffer.byteLength(message.text);
const buffer = Buffer.allocUnsafe(senderLength + messageLength + 10);
buffer.writeUInt32LE(message.time.getTime() / 1000, 0, noAssert);
buffer.writeUInt8(message.type, 4, noAssert);
buffer.writeUInt8(senderLength, 5, noAssert);
buffer.writeUInt32LE(message.time.getTime() / 1000, 0);
buffer.writeUInt8(message.type, 4);
buffer.writeUInt8(senderLength, 5);
buffer.write(name, 6);
let offset = senderLength + 6;
buffer.writeUInt16LE(messageLength, offset, noAssert);
buffer.writeUInt16LE(messageLength, offset);
buffer.write(message.text, offset += 2);
buffer.writeUInt16LE(offset += messageLength, offset, noAssert);
buffer.writeUInt16LE(offset += messageLength, offset);
return {serialized: buffer, size: offset + 2};
}
function deserializeMessage(buffer: Buffer, offset: number = 0,
characterGetter: (name: string) => Character = (name) => core.characters.get(name),
// tslint:disable-next-line ban-ts-ignore
// @ts-ignore
unsafe: boolean = noAssert): {size: number, message: Conversation.Message} {
const time = buffer.readUInt32LE(offset, unsafe);
const type = buffer.readUInt8(offset += 4, unsafe);
const senderLength = buffer.readUInt8(offset += 1, unsafe);
const time = buffer.readUInt32LE(offset);
const type = buffer.readUInt8(offset += 4);
const senderLength = buffer.readUInt8(offset += 1);
const sender = buffer.toString('utf8', offset += 1, offset += senderLength);
const messageLength = buffer.readUInt16LE(offset, unsafe);
const messageLength = buffer.readUInt16LE(offset);
const text = buffer.toString('utf8', offset += 2, offset + messageLength);
const message = new MessageImpl(type, characterGetter(sender), text, new Date(time * 1000));
return {message, size: senderLength + messageLength + 10};
@ -126,7 +128,7 @@ export function fixLogs(character: string): void {
const indexFd = fs.openSync(indexPath, 'r+');
fs.readSync(indexFd, buffer, 0, 1, 0);
let pos = 0, lastDay = 0;
const nameEnd = buffer.readUInt8(0, noAssert) + 1;
const nameEnd = buffer.readUInt8(0) + 1;
fs.ftruncateSync(indexFd, nameEnd);
fs.readSync(indexFd, buffer, 0, nameEnd, null); //tslint:disable-line:no-null-keyword
const size = (fs.fstatSync(fd)).size;
@ -141,8 +143,8 @@ export function fixLogs(character: string): void {
const time = deserialized.message.time;
const day = Math.floor(time.getTime() / dayMs - time.getTimezoneOffset() / 1440);
if(day > lastDay) {
buffer.writeUInt16LE(day, 0, noAssert);
buffer.writeUIntLE(pos, 2, 5, noAssert);
buffer.writeUInt16LE(day, 0);
buffer.writeUIntLE(pos, 2, 5);
fs.writeSync(indexFd, buffer, 0, 7);
lastDay = day;
}
@ -166,7 +168,7 @@ function loadIndex(name: string): Index {
if(file.substr(-4) === '.idx')
try {
const content = fs.readFileSync(path.join(dir, file));
let offset = content.readUInt8(0, noAssert) + 1;
let offset = content.readUInt8(0) + 1;
const item: IndexItem = {
name: content.toString('utf8', 1, offset),
index: {},
@ -175,7 +177,7 @@ function loadIndex(name: string): Index {
for(; offset < content.length; offset += 7) {
const key = content.readUInt16LE(offset);
item.index[key] = item.offsets.length;
item.offsets.push(content.readUIntLE(offset + 2, 5, noAssert));
item.offsets.push(content.readUIntLE(offset + 2, 5));
}
index[file.slice(0, -4).toLowerCase()] = item;
} catch(e) {

5
electron/main.ts
View File

@ -29,6 +29,11 @@
* @version 3.0
* @see {@link https://github.com/f-list/exported|GitHub repo}
*/
// import { DebugLogger } from './debug-logger';
// // @ts-ignore
// const dl = new DebugLogger('main');
import * as electron from 'electron';
import log from 'electron-log'; //tslint:disable-line:match-default-export-name
import * as fs from 'fs';

16
electron/pack.js
View File

@ -45,16 +45,22 @@ const modules = path.join(__dirname, 'app', 'node_modules');
const includedPaths = [
'spellchecker/build/Release/spellchecker.node',
'keytar/build/Release/keytar.node',
'integer/build/Release/integer.node',
'better-sqlite3/build/Release/better_sqlite3.node'
'keytar/build/Release/keytar.node'
];
_.each(
includedPaths,
(p) => {
mkdir(path.dirname(path.join(modules, p)));
fs.copyFileSync(require.resolve(p), path.join(modules, p));
let from = p;
let to = p;
if (_.isArray(p)) {
from = p[0];
to = p[1];
}
mkdir(path.dirname(path.join(modules, to)));
fs.copyFileSync(require.resolve(from), path.join(modules, to));
}
);

14
learn/async-cache.ts Normal file
View File

@ -0,0 +1,14 @@
import { Cache, CacheCollection } from './cache';
export abstract class AsyncCache<RecordType> {
protected cache: CacheCollection<RecordType> = {};
abstract get(name: string): Promise<RecordType | null>;
// tslint:disable-next-line no-any
abstract register(record: any): void;
static nameKey(name: string): string {
return Cache.nameKey(name);
}
}

49
learn/cache-manager.ts
View File

@ -9,7 +9,7 @@ import { AdCache } from './ad-cache';
import { ChannelConversationCache } from './channel-conversation-cache';
import { CharacterProfiler } from './character-profiler';
import { ProfileCache } from './profile-cache';
import { SqliteStore } from './sqlite-store';
import { IndexedStore } from './store/indexed';
import Timer = NodeJS.Timer;
import ChannelConversation = Conversation.ChannelConversation;
import Message = Conversation.Message;
@ -35,12 +35,14 @@ export class CacheManager {
protected profileTimer: Timer | null = null;
protected characterProfiler: CharacterProfiler | undefined;
protected profileStore = new SqliteStore();
protected profileStore?: IndexedStore;
queueForFetching(name: string): void {
if (this.profileCache.get(name))
return;
queueForFetching(name: string, skipCacheCheck: boolean = false): void {
if (!skipCacheCheck) {
if (this.profileCache.get(name))
return;
}
const key = ProfileCache.nameKey(name);
@ -63,7 +65,7 @@ export class CacheManager {
const c = await methods.characterData(name, -1, true);
const r = this.profileCache.register(c);
const r = await this.profileCache.register(c);
this.updateAdScoringForProfile(c, r.matchScore);
} catch (err) {
@ -90,7 +92,7 @@ export class CacheManager {
}
addProfile(character: string | Character): void {
async addProfile(character: string | Character): Promise<void> {
if (typeof character === 'string') {
// console.log('Learn discover', character);
@ -98,7 +100,7 @@ export class CacheManager {
return;
}
this.profileCache.register(character);
await this.profileCache.register(character);
}
@ -132,22 +134,24 @@ export class CacheManager {
return this.characterProfiler ? this.characterProfiler.calculateInterestScoreForQueueEntry(e) : 0;
}
start(): void {
this.stop();
this.profileStore.start();
async start(): Promise<void> {
await this.stop();
this.profileStore = await IndexedStore.open();
this.profileCache.setStore(this.profileStore);
EventBus.$on(
'character-data',
(data: CharacterDataEvent) => {
this.addProfile(data.character);
async(data: CharacterDataEvent) => {
await this.addProfile(data.character);
}
);
EventBus.$on(
'channel-message',
(data: ChannelMessageEvent) => {
async(data: ChannelMessageEvent) => {
const message = data.message;
const channel = data.channel;
@ -160,7 +164,7 @@ export class CacheManager {
}
);
this.addProfile(message.sender.name);
await this.addProfile(message.sender.name);
}
);
@ -179,7 +183,11 @@ export class CacheManager {
}
);
this.addProfile(message.sender.name);
if (!data.profile) {
this.queueForFetching(message.sender.name, true);
}
// this.addProfile(message.sender.name);
}
);
@ -214,15 +222,18 @@ export class CacheManager {
scheduleNextFetch();
}
stop(): void {
async stop(): Promise<void> {
if (this.profileTimer) {
clearTimeout(this.profileTimer);
this.profileTimer = null;
}
this.profileStore.stop();
if (this.profileStore) {
await this.profileStore.stop();
}
// should do some $off here
// should do some $off here?
}

2
learn/cache.ts
View File

@ -1,4 +1,4 @@
interface CacheCollection<RecordType> {
export interface CacheCollection<RecordType> {
[key: string]: RecordType
}

18
learn/matcher.ts
View File

@ -645,7 +645,23 @@ export class Matcher {
if (!(kinkId in c.kinks))
return null;
return kinkMapping[c.kinks[kinkId] as string];
const kinkVal = c.kinks[kinkId];
if (kinkVal === undefined) {
return null;
}
if (typeof kinkVal === 'string') {
return kinkMapping[c.kinks[kinkId] as string];
}
const custom = c.customs[kinkVal];
if (!custom) {
return null;
}
return kinkMapping[custom.choice];
}
static getKinkGenderPreference(c: Character, gender: Gender): KinkPreference | null {

79
learn/profile-cache.ts
View File

@ -1,50 +1,90 @@
import * as _ from 'lodash';
import core from '../chat/core';
import { Character } from '../site/character_page/interfaces';
import { Character as ComplexCharacter } from '../site/character_page/interfaces';
import { AsyncCache } from './async-cache';
import { Matcher, Score, Scoring } from './matcher';
import { Cache } from './cache';
import { SqliteStore } from './sqlite-store';
import { PermanentIndexedStore } from './store/sql-store';
export interface CountRecord {
groupCount: number | null;
friendCount: number | null;
guestbookCount: number | null;
lastCounted: number | null;
}
export interface CharacterCacheRecord {
character: Character;
character: ComplexCharacter;
lastFetched: Date;
added: Date;
matchScore: number;
counts?: CountRecord;
}
export class ProfileCache extends Cache<CharacterCacheRecord> {
protected store?: SqliteStore;
export class ProfileCache extends AsyncCache<CharacterCacheRecord> {
protected store?: PermanentIndexedStore;
setStore(store: SqliteStore): void {
setStore(store: PermanentIndexedStore): void {
this.store = store;
}
get(name: string, skipStore: boolean = false): CharacterCacheRecord | null {
const v = super.get(name);
async get(name: string, skipStore: boolean = false): Promise<CharacterCacheRecord | null> {
const key = AsyncCache.nameKey(name);
if ((v !== null) || (!this.store) || (skipStore)) {
return v;
if (key in this.cache) {
return this.cache[key];
} else {
if ((!this.store) || (skipStore)) {
return null;
}
}
const pd = this.store.getProfile(name);
const pd = await this.store.getProfile(name);
if (!pd) {
return null;
}
return this.register(pd.profileData, true);
const cacheRecord = await this.register(pd.profileData, true);
cacheRecord.lastFetched = new Date(pd.lastFetched * 1000);
cacheRecord.added = new Date(pd.firstSeen * 1000);
cacheRecord.counts = {
lastCounted: pd.lastCounted,
groupCount: pd.groupCount,
friendCount: pd.friendCount,
guestbookCount: pd.guestbookCount
};
return cacheRecord;
}
register(c: Character, skipStore: boolean = false): CharacterCacheRecord {
const k = Cache.nameKey(c.character.name);
async registerCount(name: string, counts: CountRecord): Promise<void> {
const record = await this.get(name);
if (!record) {
// coward's way out
return;
}
record.counts = counts;
if (this.store) {
await this.store.updateProfileCounts(name, counts.guestbookCount, counts.friendCount, counts.groupCount);
}
}
async register(c: ComplexCharacter, skipStore: boolean = false): Promise<CharacterCacheRecord> {
const k = AsyncCache.nameKey(c.character.name);
const score = ProfileCache.score(c);
if ((this.store) && (!skipStore)) {
this.store.storeProfile(c);
await this.store.storeProfile(c);
}
if (k in this.cache) {
@ -70,8 +110,13 @@ export class ProfileCache extends Cache<CharacterCacheRecord> {
}
static score(c: Character): number {
static score(c: ComplexCharacter): number {
const you = core.characters.ownProfile;
if (!you) {
return 0;
}
const m = Matcher.generateReport(you.character, c.character);
// let mul = Math.sign(Math.min(m.you.total, m.them.total));

187
learn/sqlite-store.ts
View File

@ -1,187 +0,0 @@
// import * as Sqlite from 'better-sqlite3';
// tslint:disable-next-line:no-duplicate-imports
import * as path from 'path';
import { Database, Statement } from 'better-sqlite3';
import core from '../chat/core';
// tslint:disable-next-line: no-require-imports
// const Sqlite = require('better-sqlite3');
import { nativeRequire } from '../electron/common';
// tslint:disable-next-line: no-any
const Sqlite = nativeRequire<any>('better-sqlite3');
import { Orientation, Gender, FurryPreference, Species, CharacterAnalysis } from './matcher';
import { Character as ComplexCharacter } from '../site/character_page/interfaces';
export interface ProfileRecord {
id: string;
name: string;
profileData: ComplexCharacter;
firstSeen: number;
lastFetched: number;
gender: Gender | null;
orientation: Orientation | null;
furryPreference: FurryPreference | null;
species: Species | null;
age: number | null;
domSubRole: number | null;
position: number | null;
lastCounted: number | null;
guestbookCount: number | null;
friendCount: number | null;
groupCount: number | null;
}
// export type Statement = any;
// export type Database = any;
export class SqliteStore {
protected stmtGetProfile: Statement;
protected stmtStoreProfile: Statement;
protected stmtUpdateCounts: Statement;
protected db: Database;
protected checkpointTimer: NodeJS.Timer | null = null;
constructor() {
const dbFile = path.join(core.state.generalSettings!.logDirectory, 'fchat-ascending.sqlite');
// tslint:disable-next-line: no-unsafe-any
this.db = new Sqlite(dbFile, {});
this.init();
this.migrateDatabase();
this.stmtGetProfile = this.db.prepare('SELECT * FROM profiles WHERE id = ?');
this.stmtStoreProfile = this.db.prepare(
`INSERT INTO profiles
(id, name, profileData, firstSeen, lastFetched, gender, orientation, furryPreference,
species, age, domSubRole, position)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
profileData=excluded.profileData,
lastFetched=excluded.lastFetched,
gender=excluded.gender,
orientation=excluded.orientation,
furryPreference=excluded.furryPreference,
species=excluded.species,
age=excluded.age,
domSubRole=excluded.domSubRole,
position=excluded.position
`);
this.stmtUpdateCounts = this.db.prepare(
'UPDATE profiles SET lastCounted = ?, guestbookCount = ?, friendCount = ?, groupCount = ? WHERE id = ? LIMIT 1'
);
}
// tslint:disable-next-line: prefer-function-over-method
protected toProfileId(name: string): string {
return name.toLowerCase();
}
getProfile(name: string): ProfileRecord | undefined {
const data = this.stmtGetProfile.get(this.toProfileId(name));
if (!data) {
return;
}
// tslint:disable-next-line: no-unsafe-any
data.profileData = JSON.parse(data.profileData) as ComplexCharacter;
return data as ProfileRecord;
}
storeProfile(c: ComplexCharacter): void {
const ca = new CharacterAnalysis(c.character);
const data = [
this.toProfileId(c.character.name),
c.character.name,
JSON.stringify(c),
Math.round(Date.now() / 1000),
Math.round(Date.now() / 1000),
ca.gender,
ca.orientation,
ca.furryPreference,
ca.species,
ca.age,
null, // domSubRole
null // position
];
this.stmtStoreProfile.run(data);
}
updateProfileCounts(name: string, guestbookCount: number | null, friendCount: number | null, groupCount: number | null): void {
this.stmtUpdateCounts.run(Math.round(Date.now() / 1000), guestbookCount, friendCount, groupCount, this.toProfileId(name));
}
protected init(): void {
this.db.pragma('journal_mode = WAL');
}
protected migrateDatabase(): void {
this.db.exec(
`CREATE TABLE IF NOT EXISTS "migration" (
"version" INTEGER NOT NULL
, UNIQUE("version")
);
CREATE TABLE IF NOT EXISTS "profiles" (
"id" TEXT NOT NULL PRIMARY KEY
, "name" TEXT NOT NULL
, "profileData" TEXT NOT NULL
, "firstSeen" INTEGER NOT NULL
, "lastFetched" INTEGER NOT NULL
, "lastCounted" INTEGER
, "gender" INTEGER
, "orientation" INTEGER
, "furryPreference" INTEGER
, "species" INTEGER
, "age" INTEGER
, "domSubRole" INTEGER
, "position" INTEGER
, "guestbookCount" INTEGER
, "friendCount" INTEGER
, "groupCount" INTEGER
, UNIQUE("id")
);
INSERT OR IGNORE INTO migration(version) VALUES(1);
`);
}
start(): void {
this.stop();
this.checkpointTimer = setInterval(
() => this.db.checkpoint(),
10 * 60 * 1000
);
}
stop(): void {
if (this.checkpointTimer) {
clearInterval(this.checkpointTimer);
this.checkpointTimer = null;
}
}
}

85
learn/store/better-sqlite3.ts Normal file
View File

@ -0,0 +1,85 @@
// // import { Database, Statement } from 'better-sqlite3';
//
// type Database = any;
// type Statement = any;
//
// import { ProfileRecord, SqlStore } from './sql-store';
// import { Character as ComplexCharacter } from '../../site/character_page/interfaces';
// import * as SQL from './sql';
//
//
// export class BetterSqliteStore extends SqlStore {
// protected static Sqlite: any;
//
// protected stmtGetProfile: Statement;
// protected stmtStoreProfile: Statement;
// protected stmtUpdateCounts: Statement;
//
// protected db: Database;
// protected checkpointTimer: NodeJS.Timer | null = null;
//
// constructor(name: string) {
// super(name);
//
// if (!BetterSqliteStore.Sqlite) {
// throw new Error('BetterSqliteStore.setSqlite() must be called before instantiation');
// }
//
// this.db = new BetterSqliteStore.Sqlite(this.dbFile, {});
//
// this.stmtGetProfile = this.db.prepare(SQL.ProfileGet);
// this.stmtStoreProfile = this.db.prepare(SQL.ProfileInsert);
// this.stmtUpdateCounts = this.db.prepare(SQL.ProfileUpdateCount);
// }
//
//
// static setSqlite(Sqlite: any) {
// BetterSqliteStore.Sqlite = Sqlite;
// }
//
//
// async getProfile(name: string): Promise<ProfileRecord | undefined> {
// const data = this.stmtGetProfile.get(this.toProfileId(name));
//
// if (!data) {
// return;
// }
//
// // tslint:disable-next-line: no-unsafe-any
// data.profileData = JSON.parse(data.profileData) as ComplexCharacter;
//
// return data as ProfileRecord;
// }
//
//
// protected async run(stmtName: 'stmtStoreProfile' | 'stmtUpdateCounts', data: any[]): Promise<void> {
// if (!(stmtName in this)) {
// throw new Error(`Unknown statement: ${stmtName}`);
// }
//
// this[stmtName].run(data);
// }
//
//
// async start(): Promise<void> {
// await this.stop();
//
// this.db.pragma('journal_mode = WAL');
// this.db.exec(SQL.DatabaseMigration);
//
// this.checkpointTimer = setInterval(
// () => this.db.checkpoint(),
// 10 * 60 * 1000
// );
// }
//
//
// async stop(): Promise<void> {
// if (this.checkpointTimer) {
// clearInterval(this.checkpointTimer);
//
// this.checkpointTimer = null;
// }
// }
// }
//

151
learn/store/indexed.ts Normal file
View File

@ -0,0 +1,151 @@
import * as _ from 'lodash';
import { Character as ComplexCharacter } from '../../site/character_page/interfaces';
import { CharacterAnalysis } from '../matcher';
import { PermanentIndexedStore, ProfileRecord } from './sql-store';
async function promisifyRequest<T>(req: IDBRequest): Promise<T> {
return new Promise<T>((resolve, reject) => {
req.onsuccess = () => resolve(<T>req.result);
req.onerror = () => reject(req.error);
});
}
export class IndexedStore implements PermanentIndexedStore {
protected dbName: string;
protected db: IDBDatabase;
protected static readonly STORE_NAME = 'profiles';
constructor(db: IDBDatabase, dbName: string) {
this.dbName = dbName;
this.db = db;
}
static async open(dbName: string = 'flist-ascending-profiles'): Promise<IndexedStore> {
const request = window.indexedDB.open(dbName, 1);
request.onupgradeneeded = () => {
const db = request.result;
db.createObjectStore(IndexedStore.STORE_NAME, { keyPath: 'id' });
};
return new IndexedStore(await promisifyRequest<IDBDatabase>(request), dbName);
}
// tslint:disable-next-line prefer-function-over-method
protected toProfileId(name: string): string {
return name.toLowerCase();
}
async getProfile(name: string): Promise<ProfileRecord | undefined> {
const tx = this.db.transaction(IndexedStore.STORE_NAME, 'readonly');
const store = tx.objectStore(IndexedStore.STORE_NAME);
const getRequest = store.get(this.toProfileId(name));
// tslint:disable-next-line no-any
const data = await promisifyRequest<any>(getRequest);
if (!data) {
// console.info('IDX empty profile', name);
return;
}
// tslint:disable-next-line: no-unsafe-any
data.profileData = data.profileData as ComplexCharacter;
// console.log('IDX profile', name, data);
return data as ProfileRecord;
}
async prepareProfileData(c: ComplexCharacter): Promise<ProfileRecord> {
const existing = await this.getProfile(c.character.name);
const ca = new CharacterAnalysis(c.character);
const data: ProfileRecord = {
id: this.toProfileId(c.character.name),
name: c.character.name,
profileData: c,
firstSeen: Math.round(Date.now() / 1000),
lastFetched: Math.round(Date.now() / 1000),
gender: ca.gender,
orientation: ca.orientation,
furryPreference: ca.furryPreference,
species: ca.species,
age: ca.age,
domSubRole: null, // domSubRole
position: null, // position
lastCounted: null,
guestbookCount: null,
friendCount: null,
groupCount: null
};
return (existing)
? _.merge(existing, data, _.pick(existing, ['firstSeen', 'lastCounted', 'guestbookCount', 'friendCount', 'groupCount']))
: data;
}
async storeProfile(c: ComplexCharacter): Promise<void> {
const data = await this.prepareProfileData(c);
const tx = this.db.transaction(IndexedStore.STORE_NAME, 'readwrite');
const store = tx.objectStore(IndexedStore.STORE_NAME);
const putRequest = store.put(data);
// tslint:disable-next-line no-any
await promisifyRequest<any>(putRequest);
// console.log('IDX store profile', c.character.name, data);
}
async updateProfileCounts(
name: string,
guestbookCount: number | null,
friendCount: number | null,
groupCount: number | null
): Promise<void> {
const existing = await this.getProfile(name);
if (!existing) {
return;
}
const data = _.merge(
existing,
{
lastCounted: Math.round(Date.now() / 1000),
guestbookCount,
friendCount,
groupCount
}
);
const tx = this.db.transaction(IndexedStore.STORE_NAME, 'readwrite');
const store = tx.objectStore(IndexedStore.STORE_NAME);
const putRequest = store.put(data);
// tslint:disable-next-line no-any
await promisifyRequest<any>(putRequest);
// console.log('IDX update counts', name, data);
}
async start(): Promise<void> {
// empty
}
async stop(): Promise<void> {
// empty
}
}

95
learn/store/sql-store.ts Normal file
View File

@ -0,0 +1,95 @@
// tslint:disable-next-line:no-duplicate-imports
import * as path from 'path';
import core from '../../chat/core';
import { Orientation, Gender, FurryPreference, Species, CharacterAnalysis } from '../matcher';
import { Character as ComplexCharacter } from '../../site/character_page/interfaces';
export interface ProfileRecord {
id: string;
name: string;
profileData: ComplexCharacter;
firstSeen: number;
lastFetched: number;
gender: Gender | null;
orientation: Orientation | null;
furryPreference: FurryPreference | null;
species: Species | null;
age: number | null;
domSubRole: number | null;
position: number | null;
lastCounted: number | null;
guestbookCount: number | null;
friendCount: number | null;
groupCount: number | null;
}
// export type Statement = any;
// export type Database = any;
export interface PermanentIndexedStore {
getProfile(name: string): Promise<ProfileRecord | undefined>;
storeProfile(c: ComplexCharacter): Promise<void>;
updateProfileCounts(name: string, guestbookCount: number | null, friendCount: number | null, groupCount: number | null): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
}
export abstract class SqlStore implements PermanentIndexedStore {
protected dbFile: string;
protected checkpointTimer: NodeJS.Timer | null = null;
constructor(dbName: string = 'fchat-ascending.sqlite') {
this.dbFile = path.join(core.state.generalSettings!.logDirectory, dbName);
}
// tslint:disable-next-line: prefer-function-over-method
protected toProfileId(name: string): string {
return name.toLowerCase();
}
abstract getProfile(name: string): Promise<ProfileRecord | undefined>;
abstract start(): Promise<void>;
abstract stop(): Promise<void>;
// tslint:disable-next-line no-any
protected abstract run(statementName: 'stmtStoreProfile' | 'stmtUpdateCounts', data: any[]): Promise<void>;
async storeProfile(c: ComplexCharacter): Promise<void> {
const ca = new CharacterAnalysis(c.character);
const data = [
this.toProfileId(c.character.name),
c.character.name,
JSON.stringify(c),
Math.round(Date.now() / 1000),
Math.round(Date.now() / 1000),
ca.gender,
ca.orientation,
ca.furryPreference,
ca.species,
ca.age,
null, // domSubRole
null // position
];
await this.run('stmtStoreProfile', data);
}
async updateProfileCounts(
name: string,
guestbookCount: number | null,
friendCount: number | null,
groupCount: number | null
): Promise<void> {
await this.run(
'stmtUpdateCounts',
[Math.round(Date.now() / 1000), guestbookCount, friendCount, groupCount, this.toProfileId(name)]
);
}
}

55
learn/store/sql.ts Normal file
View File

@ -0,0 +1,55 @@
export const ProfileInsert =
`INSERT INTO profiles
(id, name, profileData, firstSeen, lastFetched, gender, orientation, furryPreference,
species, age, domSubRole, position)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
profileData=excluded.profileData,
lastFetched=excluded.lastFetched,
gender=excluded.gender,
orientation=excluded.orientation,
furryPreference=excluded.furryPreference,
species=excluded.species,
age=excluded.age,
domSubRole=excluded.domSubRole,
position=excluded.position
`;
export const ProfileGet =
'SELECT * FROM profiles WHERE id = ?';
export const ProfileUpdateCount =
'UPDATE profiles SET lastCounted = ?, guestbookCount = ?, friendCount = ?, groupCount = ? WHERE id = ?';
export const DatabaseMigration =
`CREATE TABLE IF NOT EXISTS "migration" (
"version" INTEGER NOT NULL
, UNIQUE("version")
);
CREATE TABLE IF NOT EXISTS "profiles" (
"id" TEXT NOT NULL PRIMARY KEY
, "name" TEXT NOT NULL
, "profileData" TEXT NOT NULL
, "firstSeen" INTEGER NOT NULL
, "lastFetched" INTEGER NOT NULL
, "lastCounted" INTEGER
, "gender" INTEGER
, "orientation" INTEGER
, "furryPreference" INTEGER
, "species" INTEGER
, "age" INTEGER
, "domSubRole" INTEGER
, "position" INTEGER
, "guestbookCount" INTEGER
, "friendCount" INTEGER
, "groupCount" INTEGER
, UNIQUE("id")
);
INSERT OR IGNORE INTO migration(version) VALUES(1);
`;

59
learn/store/sqlite.ts Normal file
View File

@ -0,0 +1,59 @@
// import * as Sqlite from 'sqlite';
// import { Character as ComplexCharacter } from '../../site/character_page/interfaces';
//
// import * as SQL from './sql';
// import { ProfileRecord, SqlStore } from './sql-store';
//
// export class SqliteStore extends SqlStore {
// protected stmtGetProfile: Promise<Sqlite.Statement>;
// protected stmtStoreProfile: Promise<Sqlite.Statement>;
// protected stmtUpdateCounts: Promise<Sqlite.Statement>;
//
// protected db: Promise<Sqlite.Database>;
//
// constructor(dbName: string = 'fchat-ascending.sqlite') {
// super(dbName);
//
// this.db = Sqlite.open(this.dbFile);
//
// this.stmtGetProfile = this.db.then((db) => db.prepare(SQL.ProfileGet));
// this.stmtStoreProfile = this.db.then((db) => db.prepare(SQL.ProfileInsert));
// this.stmtUpdateCounts = this.db.then((db) => db.prepare(SQL.ProfileUpdateCount));
// }
//
//
// async getProfile(name: string): Promise<ProfileRecord | undefined> {
// const data = await (await this.stmtGetProfile).get(this.toProfileId(name));
//
// if (!data) {
// return;
// }
//
// // tslint:disable-next-line: no-unsafe-any
// data.profileData = JSON.parse(data.profileData) as ComplexCharacter;
//
// return data as ProfileRecord;
// }
//
//
// protected async run(stmtName: 'stmtStoreProfile' | 'stmtUpdateCounts', data: any[]): Promise<void> {
// await (await this[stmtName]).run(...data);
// }
//
//
// async start(): Promise<void> {
// await this.stop();
//
// await (await this.db).run(SQL.DatabaseMigration);
// }
//
//
// async stop(): Promise<void> {
// if (this.checkpointTimer) {
// clearInterval(this.checkpointTimer);
//
// this.checkpointTimer = null;
// }
// }
//
// }

140
learn/store/sqlite3.ts Normal file
View File

@ -0,0 +1,140 @@
// type Sqlite3Statement = any;
// type Sqlite3Database = any;
//
// import { Character as ComplexCharacter } from '../../site/character_page/interfaces';
//
// import * as SQL from './sql';
// import { ProfileRecord, SqlStore } from './sql-store';
//
// export class Sqlite3Store extends SqlStore {
// protected static Sqlite: any;
//
// protected stmtGetProfile: Promise<Sqlite3Statement>;
// protected stmtStoreProfile: Promise<Sqlite3Statement>;
// protected stmtUpdateCounts: Promise<Sqlite3Statement>;
//
// protected db: Promise<Sqlite3Database>;
//
// constructor(dbName: string = 'fchat-ascending.sqlite') {
// super(dbName);
//
// if (!Sqlite3Store.Sqlite) {
// throw new Error('Sqlite3Store.setSqlite() must be called before instantiation');
// }
//
// this.db = this.prepareDatabase(this.dbFile);
//
// this.stmtGetProfile = this.prepareStatement(SQL.ProfileGet);
// this.stmtStoreProfile = this.prepareStatement(SQL.ProfileInsert);
// this.stmtUpdateCounts = this.prepareStatement(SQL.ProfileUpdateCount);
// }
//
// static setSqlite(Sqlite: any) {
// Sqlite3Store.Sqlite = Sqlite;
// }
//
// prepareDatabase(fileName: string): Promise<Sqlite3Database> {
// return new Promise(
// (resolve, reject) => {
// // Sqlite3Store.Sqlite.verbose();
// const db = new Sqlite3Store.Sqlite.Database(
// fileName,
// (err: any) => {
// if (err) {
// reject(err);
// return;
// }
//
// resolve(db);
// }
// );
// }
// );
// }
//
// prepareStatement(sql: string): Promise<Sqlite3Statement> {
// return new Promise(
// async(resolve, reject) => {
// const s = (await this.db).prepare(
// sql,
// (err: any) => {
// if (err) {
// reject(err);
// return;
// }
//
// resolve(s);
// }
// );
// }
// );
// }
//
//
// getProfile(name: string): Promise<ProfileRecord | undefined> {
// return new Promise(
// async(resolve, reject) => {
// (await this.stmtGetProfile).get(
// this.toProfileId(name),
// (err: any, data: any) => {
// if (err) {
// reject(err);
// return;
// }
//
// if (!data) {
// resolve();
// return;
// }
//
// // tslint:disable-next-line: no-unsafe-any
// data.profileData = JSON.parse(data.profileData) as ComplexCharacter;
//
// resolve(data as ProfileRecord);
// }
// );
// }
// );
// }
//
//
// protected run(stmtName: 'stmtStoreProfile' | 'stmtUpdateCounts', data: any[]): Promise<void> {
// return new Promise(
// async(resolve, reject) => {
// if (!(stmtName in this)) {
// throw new Error(`Unknown statement: ${stmtName}`);
// }
//
// (await this[stmtName]).run(
// ...data,
// (err: any) => {
// if (err) {
// reject(err);
// return;
// }
//
// resolve();
// }
// );
// }
// );
// }
//
//
// async start(): Promise<void> {
// await this.stop();
//
// await (await this.db).run(SQL.DatabaseMigration);
// }
//
//
// async stop(): Promise<void> {
// if (this.checkpointTimer) {
// clearInterval(this.checkpointTimer);
//
// this.checkpointTimer = null;
// }
// }
//
// }

1403
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

7
package.json
View File

@ -10,13 +10,13 @@
"@fortawesome/fontawesome-free": "^5.6.1",
"@types/lodash": "^4.14.119",
"@types/sortablejs": "^1.7.0",
"@types/better-sqlite3": "^5.4.0",
"@types/sqlite3": "^3.1.5",
"@vue/devtools": "^5.1.0",
"axios": "^0.18.0",
"bootstrap": "^4.1.3",
"css-loader": "^2.0.1",
"date-fns": "^1.30.1",
"electron": "3.0.13",
"electron": "3.0.16",
"electron-log": "^2.2.17",
"electron-packager": "^13.0.1",
"electron-rebuild": "^1.8.5",
@ -40,7 +40,6 @@
"webpack": "^4.27.1"
},
"dependencies": {
"better-sqlite3": "^5.4.0",
"keytar": "^4.3.0",
"spellchecker": "^3.5.0"
},
@ -50,6 +49,6 @@
"electron-winstaller": "^2.7.0"
},
"scripts": {
"postinstall": "electron-rebuild -o spellchecker,keytar,better-sqlite3,integer"
"postinstall": "electron-rebuild -o spellchecker,keytar"
}
}

10
readme.md
View File

@ -34,6 +34,7 @@ This repository contains a modified version of the mainline F-Chat 3.0 client.
* Fix broken BBCode, such as `[big]` in character profiles
* Ad cache, so you can find your chat partners ads easily (channel names too)
* Which channels my chart partner is on?
* Profile doesn't scroll back up
# F-List Exported
@ -52,6 +53,15 @@ All necessary files to build F-Chat 3.0 as an Electron, mobile or web applicatio
- Run `yarn build`/`yarn watch` to build assets. They are placed into the `app` directory.
- Run `yarn start` to start the app in debug mode. Use `Ctrl+Shift+I` to open the Chromium debugger.
### Building on Windows
```
npm install --global --production windows-build-tools
npm install --global --production --vs2015 --add-python-to-path windows-build-tools
npm install --global --production --add-python-to-path windows-build-tools node-gyp
```
### Packaging
See https://electron.atom.io/docs/tutorial/application-distribution/
- Run `yarn build:dist` to create a minified production build.

96
site/character_page/character_page.vue
View File

@ -1,5 +1,5 @@
<template>
<div class="row character-page" id="pageBody">
<div class="row character-page" id="pageBody" ref="pageBody">
<div class="col-12" style="min-height:0">
<div class="alert alert-info" v-show="loading">Loading character information.</div>
<div class="alert alert-danger" v-show="error">{{error}}</div>
@ -72,6 +72,7 @@
import {Component, Hook, Prop, Watch} from '@f-list/vue-ts';
import Vue from 'vue';
import {standardParser} from '../../bbcode/standard';
import { CharacterCacheRecord } from '../../learn/profile-cache';
import * as Utils from '../utils';
import {methods, Store} from './data_store';
import {Character, SharedStore} from './interfaces';
@ -89,8 +90,8 @@
import { Matcher, MatchReport } from '../../learn/matcher';
import MatchReportView from './match-report.vue';
const CHARACTER_CACHE_EXPIRE = 4 * 60 * 60 * 1000;
const CHARACTER_CACHE_EXPIRE = 7 * 24 * 60 * 60 * 1000;
const CHARACTER_COUNT_CACHE_EXPIRE = 10 * 24 * 60 * 60 * 1000;
interface ShowableVueTab extends Vue {
show?(): void
@ -126,9 +127,9 @@
error = '';
tab = '0';
guestbookPostCount: string | null = null;
friendCount: string | null = null;
groupCount: string | null = null;
guestbookPostCount: number | null = null;
friendCount: number | null = null;
groupCount: number | null = null;
selfCharacter: Character | undefined;
@ -139,11 +140,15 @@
@Hook('beforeMount')
beforeMount(): void {
this.shared.authenticated = this.authenticated;
console.log('Beforemount');
}
@Hook('mounted')
async mounted(): Promise<void> {
await this.load(false);
console.log('mounted');
}
@Watch('tab')
@ -156,7 +161,21 @@
@Watch('name')
async onCharacterSet(): Promise<void> {
this.tab = '0';
return this.load();
await this.load();
// Kludge kluge
this.$nextTick(
() => {
const el = document.querySelector('.modal .profile-viewer .modal-body');
if (!el) {
console.error('Could not find modal body for profile view');
return;
}
el.scrollTo(0, 0);
}
);
}
async load(mustLoad: boolean = true): Promise<void> {
@ -198,7 +217,8 @@
const guestbookState = await methods.guestbookPageGet(this.character.character.id, 1, false);
this.guestbookPostCount = `${guestbookState.posts.length}${guestbookState.nextPage ? '+' : ''}`;
this.guestbookPostCount = guestbookState.posts.length;
// `${guestbookState.posts.length}${guestbookState.nextPage ? '+' : ''}`;
} catch (err) {
console.error(err);
this.guestbookPostCount = null;
@ -215,7 +235,7 @@
const groups = await methods.groupsGet(this.character.character.id);
this.groupCount = `${groups.length}`;
this.groupCount = groups.length; // `${groups.length}`;
} catch (err) {
console.error(err);
this.groupCount = null;
@ -235,7 +255,7 @@
const friends = await methods.friendsGet(this.character.character.id);
this.friendCount = `${friends.length}`;
this.friendCount = friends.length; // `${friends.length}`;
} catch (err) {
console.error(err);
this.friendCount = null;
@ -243,6 +263,23 @@
}
async updateCounts(name: string): Promise<void> {
await this.countGuestbookPosts();
await this.countFriends();
await this.countGroups();
await core.cache.profileCache.registerCount(
name,
{
lastCounted: Date.now() / 1000,
groupCount: this.groupCount,
friendCount: this.friendCount,
guestbookCount: this.guestbookPostCount
}
);
}
memo(memo: {id: number, memo: string}): void {
Vue.set(this.character!, 'memo', memo);
}
@ -264,7 +301,7 @@
this.updateMatches();
}
private async fetchCharacter(): Promise<Character> {
private async fetchCharacterCache(): Promise<CharacterCacheRecord | null> {
if (!this.name) {
throw new Error('A man must have a name');
}
@ -274,11 +311,11 @@
if (cachedCharacter) {
if (Date.now() - cachedCharacter.lastFetched.getTime() <= CHARACTER_CACHE_EXPIRE) {
return cachedCharacter.character;
return cachedCharacter;
}
}
return methods.characterData(this.name, this.characterid, false);
return null;
}
private async _getCharacter(): Promise<void> {
@ -291,25 +328,32 @@
return;
}
this.character = await this.fetchCharacter();
const cache = await this.fetchCharacterCache();
this.character = (cache)
? cache.character
: await methods.characterData(this.name, this.characterid, false);
standardParser.allowInlines = true;
standardParser.inlines = this.character.character.inlines;
if (
(cache)
&& (cache.counts)
&& (cache.counts.lastCounted)
&& ((Date.now() / 1000) - cache.counts.lastCounted < CHARACTER_COUNT_CACHE_EXPIRE)
) {
this.guestbookPostCount = cache.counts.guestbookCount;
this.friendCount = cache.counts.friendCount;
this.groupCount = cache.counts.groupCount;
} else {
// No awaits on these on purpose:
// tslint:disable-next-line no-floating-promises
this.updateCounts(this.name);
}
// console.log('LoadChar', this.name, this.character);
this.updateMatches();
// No awaits on these on purpose:
// tslint:disable-next-line no-floating-promises
this.countGuestbookPosts();
// tslint:disable-next-line no-floating-promises
this.countGroups();
// tslint:disable-next-line no-floating-promises
this.countFriends();
}

55
site/character_page/kinks.vue
View File

@ -70,13 +70,15 @@
</template>
<script lang="ts">
import * as _ from 'lodash';
import {Component, Prop, Watch, Hook} from '@f-list/vue-ts';
import Vue from 'vue';
import core from '../../chat/core';
import {Kink, KinkChoice} from '../../interfaces';
import * as Utils from '../utils';
import CopyCustomMenu from './copy_custom_menu.vue';
import {methods, Store} from './data_store';
import {Character, DisplayKink, KinkGroup} from './interfaces';
import { Character, CharacterKink, DisplayKink, KinkGroup } from './interfaces';
import KinkView from './kink.vue';
@Component({
@ -103,7 +105,47 @@
this.expandedCustoms = !this.expandedCustoms;
}
async compareKinks(): Promise<void> {
// iterateThroughAllKinks(c: Character, cb: (
resolveKinkChoice(c: Character, kinkValue: string | number | undefined): string | null {
if (typeof kinkValue === 'string') {
return kinkValue;
}
if (typeof kinkValue === 'number') {
const custom = c.character.customs[kinkValue];
if (custom) {
return custom.choice;
}
}
return null;
}
convertCharacterKinks(c: Character): CharacterKink[] {
return _.filter(
_.map(
c.character.kinks,
(kinkValue: string | number | undefined, kinkId: string) => {
const resolvedChoice = this.resolveKinkChoice(c, kinkValue);
if (!resolvedChoice)
return null;
return {
id: parseInt(kinkId, 10),
choice: resolvedChoice as KinkChoice
};
}
),
(v) => (v !== null)
) as CharacterKink[];
}
async compareKinks(overridingCharacter?: Character): Promise<void> {
if(this.comparing) {
this.comparison = {};
this.comparing = false;
@ -114,7 +156,10 @@
try {
this.loading = true;
this.comparing = true;
const kinks = await methods.kinksGet(this.characterToCompare);
const kinks = overridingCharacter
? this.convertCharacterKinks(overridingCharacter)
: await methods.kinksGet(this.characterToCompare);
const toAssign: {[key: number]: KinkChoice} = {};
for(const kink of kinks)
@ -146,7 +191,7 @@
if ((this.character) && (this.character.is_self))
return;
await this.compareKinks();
await this.compareKinks(core.characters.ownProfile);
}
@Watch('character')
@ -154,7 +199,7 @@
if ((this.character) && (this.character.is_self))
return;
await this.compareKinks();
await this.compareKinks(core.characters.ownProfile);
}
get kinkGroups(): {[key: string]: KinkGroup | undefined} {

1748
yarn.lock
View File

File diff suppressed because it is too large Load Diff