From 228352bedbc8081a5a4c0675cf88bc4072250f7e Mon Sep 17 00:00:00 2001 From: Michael Peters Date: Thu, 13 Jan 2022 20:31:18 -0600 Subject: [PATCH] support for unverifying during cache update --- src/client/webapp/auto-verifier-with-args.ts | 15 ++++-- src/client/webapp/auto-verifier.ts | 4 +- src/client/webapp/fetchable-pair-verifier.ts | 18 +++---- src/client/webapp/guild-combined.ts | 34 +++++++++--- src/client/webapp/guild-personal-db.ts | 25 +++++++-- src/client/webapp/guild-ram.ts | 28 +++++++--- src/client/webapp/guild-types.ts | 56 +++++++++++++++----- src/client/webapp/message-ram-cache.ts | 20 +++++-- 8 files changed, 152 insertions(+), 48 deletions(-) diff --git a/src/client/webapp/auto-verifier-with-args.ts b/src/client/webapp/auto-verifier-with-args.ts index 9ac2cb7..cebea0a 100644 --- a/src/client/webapp/auto-verifier-with-args.ts +++ b/src/client/webapp/auto-verifier-with-args.ts @@ -80,10 +80,17 @@ export class AutoVerifierWithArg { this.tokenQueries.clear(); } - unverifySome(toUnverifyFilter: (query: K) => boolean) { - const tokensToUnverify = Array.from(this.tokenQueries.values()) - .filter(toUnverifyFilter) - .map(query => this.tokenizer(query)); + getCachedQueries(): K[] { + return Array.from(this.tokenQueries.values()); + } + + unverifyByFilter(filter: (query: K) => boolean) { + this.unverifySome(this.getCachedQueries().filter(filter)); + } + + // use getUnverifySomeFunc for a more functional style + unverifySome(toUnverify: K[]) { + const tokensToUnverify = new Set(toUnverify.map(query => this.tokenizer(query))); for (const token of tokensToUnverify) { this.tokenAutoVerifiers.get(token)?.unverify(); this.tokenAutoVerifiers.delete(token); diff --git a/src/client/webapp/auto-verifier.ts b/src/client/webapp/auto-verifier.ts index a9ff803..3137700 100644 --- a/src/client/webapp/auto-verifier.ts +++ b/src/client/webapp/auto-verifier.ts @@ -24,7 +24,7 @@ export enum AutoVerifierChangesType { export class AutoVerifier { public primaryPromise: Promise | null = null; public trustedPromise: Promise | null = null; - public trustedStatus: 'none' | 'fetching' | 'verifying' | 'verified' | 'unverified' = 'none'; + public trustedStatus: 'none' | 'fetching' | 'verifying' | 'verified' = 'none'; /** * Allows a trusted function to verify the primary function @@ -226,7 +226,7 @@ export class AutoVerifier { // keep in mind that this will resolve 'unverified' promises with their resulting origTrustedPromise (which should be a good thing) } - if (!resolved && trustedResult) { + if (!resolved) { // Removed 01/09/2021 pretty sure should not be here... && trustedResult resolve(trustedResult); resolved = true; } diff --git a/src/client/webapp/fetchable-pair-verifier.ts b/src/client/webapp/fetchable-pair-verifier.ts index e9a4217..bfb9d35 100644 --- a/src/client/webapp/fetchable-pair-verifier.ts +++ b/src/client/webapp/fetchable-pair-verifier.ts @@ -18,9 +18,9 @@ export default class PairVerifierFetchable extends EventEmitter im private readonly fetchResourceVerifier: AutoVerifierWithArg - private readonly fetchMessagesRecentVerifier: AutoVerifierWithArg; - private readonly fetchMessagesBeforeVerifier: AutoVerifierWithArg; - private readonly fetchMessagesAfterVerifier: AutoVerifierWithArg; + public readonly fetchMessagesRecentVerifier: AutoVerifierWithArg; + public readonly fetchMessagesBeforeVerifier: AutoVerifierWithArg; + public readonly fetchMessagesAfterVerifier: AutoVerifierWithArg; constructor( private primary: Fetchable & Lackable, @@ -146,11 +146,11 @@ export default class PairVerifierFetchable extends EventEmitter im async handleResourceConflict(query: IDQuery, changesType: AutoVerifierChangesType, primaryResource: Resource | null, trustedResource: Resource | null): Promise { let success = true; if (changesType === AutoVerifierChangesType.PRIMARY_ONLY) { - success = success && await this.primary.handleResourceDeleted(trustedResource as Resource); + success = success && await this.primary.handleResourceDeleted(trustedResource as Resource, this.fetchResourceVerifier); } else if (changesType === AutoVerifierChangesType.TRUSTED_ONLY) { - success = success && await this.primary.handleResourceAdded(trustedResource as Resource); + success = success && await this.primary.handleResourceAdded(trustedResource as Resource, this.fetchResourceVerifier); } else if (changesType === AutoVerifierChangesType.CONFLICT) { - success = success && await this.primary.handleResourceChanged(trustedResource as Resource); + success = success && await this.primary.handleResourceChanged(trustedResource as Resource, this.fetchResourceVerifier); this.emit('conflict-resource', query, changesType, primaryResource as Resource, trustedResource as Resource); } return success; @@ -158,9 +158,9 @@ export default class PairVerifierFetchable extends EventEmitter im async handleMessagesConflict(query: PartialMessageListQuery, changesType: AutoVerifierChangesType, changes: Changes): Promise { let success = true; - if (changes.added.length > 0) success = success && await this.primary.handleMessagesAdded(changes.added); - if (changes.updated.length > 0) success = success && await this.primary.handleMessagesChanged(changes.updated.map(change => change.newDataPoint)); - if (changes.deleted.length > 0) success = success && await this.primary.handleMessagesDeleted(changes.deleted); + if (changes.added.length > 0) success = success && await this.primary.handleMessagesAdded(changes.added, this.fetchMessagesRecentVerifier, this.fetchMessagesBeforeVerifier, this.fetchMessagesAfterVerifier); + if (changes.updated.length > 0) success = success && await this.primary.handleMessagesChanged(changes.updated.map(change => change.newDataPoint), this.fetchMessagesRecentVerifier, this.fetchMessagesBeforeVerifier, this.fetchMessagesAfterVerifier); + if (changes.deleted.length > 0) success = success && await this.primary.handleMessagesDeleted(changes.deleted, this.fetchMessagesRecentVerifier, this.fetchMessagesBeforeVerifier, this.fetchMessagesAfterVerifier); if (changesType === AutoVerifierChangesType.CONFLICT) { this.emit('conflict-messages', query, changesType, changes); diff --git a/src/client/webapp/guild-combined.ts b/src/client/webapp/guild-combined.ts index ad1a2e1..988c1b2 100644 --- a/src/client/webapp/guild-combined.ts +++ b/src/client/webapp/guild-combined.ts @@ -43,6 +43,9 @@ export default class CombinedGuild extends EventEmitter { LOG.info(`g#${this.id} connected`); @@ -104,8 +107,18 @@ export default class CombinedGuild extends EventEmitter { LOG.info(`g#${this.id} metadata conflict`, { oldGuildMeta, newGuildMeta }); diff --git a/src/client/webapp/guild-personal-db.ts b/src/client/webapp/guild-personal-db.ts index 94a9219..88337be 100644 --- a/src/client/webapp/guild-personal-db.ts +++ b/src/client/webapp/guild-personal-db.ts @@ -7,6 +7,7 @@ import { AsyncFetchable, AsyncLackable } from "./guild-types"; import { Channel, GuildMetadata, GuildMetadataLocal, Member, Message, Resource, SocketConfig, Token } from "./data-types"; import PersonalDB from "./personal-db"; +import { AutoVerifierWithArg, PartialMessageListQuery } from './auto-verifier-with-args'; export default class PersonalDBGuild implements AsyncFetchable, AsyncLackable { constructor( @@ -94,17 +95,35 @@ export default class PersonalDBGuild implements AsyncFetchable, AsyncLackable { return true; } - async handleMessagesAdded(addedMessages: Message[]): Promise { + async handleMessagesAdded( + addedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): Promise { + // The PersonalDB does not remove old messages (yet) so nothing here would unverify //LOG.debug(addedMessages.length + ' personal messages added'); await this.db.addMessages(this.guildId, addedMessages); return true; } - async handleMessagesChanged(changedMessages: Message[]): Promise { + async handleMessagesChanged( + changedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): Promise { + // The PersonalDB does not remove old messages (yet) so nothing here would unverify //LOG.debug(changedMessages.length + ' personal messages changed'); await this.db.updateMessages(this.guildId, changedMessages); return true; } - async handleMessagesDeleted(deletedMessages: Message[]): Promise { + async handleMessagesDeleted( + deletedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): Promise { + // The PersonalDB does not remove old messages (yet) so nothing here would unverify //LOG.debug(deletedMessages.length + ' personal messages deleted'); await this.db.deleteMessages(this.guildId, deletedMessages); return true; diff --git a/src/client/webapp/guild-ram.ts b/src/client/webapp/guild-ram.ts index 68294fb..5001b5b 100644 --- a/src/client/webapp/guild-ram.ts +++ b/src/client/webapp/guild-ram.ts @@ -7,6 +7,7 @@ import { SyncFetchable, SyncLackable } from "./guild-types"; import { GuildMetadata, Member, Channel, Message, Resource, Token } from "./data-types"; import MessageRAMCache from "./message-ram-cache"; import ResourceRAMCache from "./resource-ram-cache"; +import { AutoVerifierWithArg, PartialMessageListQuery } from './auto-verifier-with-args'; export default class RAMGuild implements SyncFetchable, SyncLackable { private metadata: GuildMetadata | null = null; @@ -97,7 +98,12 @@ export default class RAMGuild implements SyncFetchable, SyncLackable { return true; } - handleMessagesAdded(addedMessages: Message[]): boolean { + handleMessagesAdded( + addedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): boolean { const byChannel = new Map(); for (const message of addedMessages) { if (!byChannel.has(message.channel.id)) { @@ -107,11 +113,16 @@ export default class RAMGuild implements SyncFetchable, SyncLackable { } } for (const [ channelId, messages ] of byChannel.entries()) { - this.recentMessages.upsertMessages(this.guildId, channelId, messages); + this.recentMessages.upsertMessages(this.guildId, channelId, messages, recentAutoVerifier); } return true; } - handleMessagesChanged(changedMessages: Message[]): boolean { + handleMessagesChanged( + changedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): boolean { const byChannel = new Map(); for (const message of changedMessages) { if (!byChannel.has(message.channel.id)) { @@ -121,11 +132,16 @@ export default class RAMGuild implements SyncFetchable, SyncLackable { } } for (const [ channelId, messages ] of byChannel.entries()) { - this.recentMessages.upsertMessages(this.guildId, channelId, messages); + this.recentMessages.upsertMessages(this.guildId, channelId, messages, recentAutoVerifier); } return true; } - handleMessagesDeleted(deletedMessages: Message[]): boolean { + handleMessagesDeleted( + deletedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): boolean { const byChannel = new Map(); for (const message of deletedMessages) { if (!byChannel.has(message.channel.id)) { @@ -135,7 +151,7 @@ export default class RAMGuild implements SyncFetchable, SyncLackable { } } for (const [ channelId, messages ] of byChannel.entries()) { - this.recentMessages.deleteMessages(this.guildId, channelId, messages); + this.recentMessages.deleteMessages(this.guildId, channelId, messages, recentAutoVerifier); } return true; } diff --git a/src/client/webapp/guild-types.ts b/src/client/webapp/guild-types.ts index 126dd51..f8d007c 100644 --- a/src/client/webapp/guild-types.ts +++ b/src/client/webapp/guild-types.ts @@ -1,6 +1,6 @@ import { Changes, Channel, GuildMetadata, Member, Message, Resource, Token } from './data-types'; import { AutoVerifierChangesType } from './auto-verifier'; -import { IDQuery, PartialMessageListQuery } from './auto-verifier-with-args'; +import { AutoVerifierWithArg, IDQuery, PartialMessageListQuery } from './auto-verifier-with-args'; // Fetchable @@ -72,13 +72,28 @@ export interface AsyncLackable { handleChannelsChanged(changedChannels: Channel[]): Promise; handleChannelsDeleted(deletedChannels: Channel[]): Promise; - handleMessagesAdded(addedMessages: Message[]): Promise; - handleMessagesChanged(changedMessages: Message[]): Promise; - handleMessagesDeleted(deletedMessages: Message[]): Promise; + handleMessagesAdded( + addedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): Promise; + handleMessagesChanged( + changedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): Promise; + handleMessagesDeleted( + deletedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): Promise; - handleResourceAdded(addedResource: Resource): Promise; - handleResourceChanged(changedResource: Resource): Promise; - handleResourceDeleted(deletedResource: Resource): Promise; + handleResourceAdded(addedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg): Promise; + handleResourceChanged(changedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg): Promise; + handleResourceDeleted(deletedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg): Promise; handleTokensAdded(addedTokens: Token[]): Promise; handleTokensChanged(changedTokens: Token[]): Promise; @@ -96,13 +111,28 @@ export interface SyncLackable { handleChannelsChanged(changedChannels: Channel[]): boolean; handleChannelsDeleted(deletedChannels: Channel[]): boolean; - handleMessagesAdded(addedMessages: Message[]): boolean; - handleMessagesChanged(changedMessages: Message[]): boolean; - handleMessagesDeleted(deletedMessages: Message[]): boolean; + handleMessagesAdded( + addedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): boolean; + handleMessagesChanged( + changedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): boolean; + handleMessagesDeleted( + deletedMessages: Message[], + recentAutoVerifier: AutoVerifierWithArg, + beforeAutoVerifier: AutoVerifierWithArg, + afterAutoVerifier: AutoVerifierWithArg + ): boolean; - handleResourceAdded(addedResource: Resource): boolean; - handleResourceChanged(changedResource: Resource): boolean; - handleResourceDeleted(deletedResource: Resource): boolean; + handleResourceAdded(addedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg): boolean; + handleResourceChanged(changedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg): boolean; + handleResourceDeleted(deletedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg): boolean; handleTokensAdded(addedTokens: Token[]): boolean; handleTokensChanged(changedTokens: Token[]): boolean; diff --git a/src/client/webapp/message-ram-cache.ts b/src/client/webapp/message-ram-cache.ts index bcfd476..5b8d4d7 100644 --- a/src/client/webapp/message-ram-cache.ts +++ b/src/client/webapp/message-ram-cache.ts @@ -4,6 +4,7 @@ import Globals from "./globals"; import * as electronRemote from '@electron/remote'; const electronConsole = electronRemote.getGlobal('console') as Console; import Logger from '../../logger/logger'; +import { AutoVerifierWithArg, PartialMessageListQuery } from "./auto-verifier-with-args"; const LOG = Logger.create(__filename, electronConsole); interface MessagesWithMetadata { @@ -33,13 +34,15 @@ export default class MessageRAMCache { } // Removes the oldest messages from the channel until the channel is under the max cached character limit - private trimOldChannelMessagesIfNeeded(guildId: number, channelId: string) { + // Returns true if messages were removed + private trimOldChannelMessagesIfNeeded(guildId: number, channelId: string, recentAutoVerifier: AutoVerifierWithArg): void { const id = `g#${guildId}/c#${channelId}`; const value = this.data.get(id); if (!value) return; if (value.totalCharacters > Globals.MAX_RAM_CACHED_MESSAGES_CHANNEL_CHARACTERS) { const messages = Array.from(value.messages.values()).sort(Message.sortOrder); + const beforeNumMessages = value.messages.size; while (value.totalCharacters > Globals.MAX_RAM_CACHED_MESSAGES_CHANNEL_CHARACTERS) { const message = messages.shift(); if (!message) throw new ShouldNeverHappenError('could not find a message to clear'); @@ -47,10 +50,13 @@ export default class MessageRAMCache { value.totalCharacters -= message.text?.length ?? 0; this.totalCharacters -= message.text?.length ?? 0; } + const afterNumMessages = value.messages.size; + // TODO: Test this (it's going to be a massive pain) + recentAutoVerifier.unverifyByFilter(query => query.channelId === channelId && beforeNumMessages >= query.number && afterNumMessages < query.number); } } - upsertMessages(guildId: number, channelId: string, messages: Message[]) { + upsertMessages(guildId: number, channelId: string, messages: Message[], recentAutoVerifier: AutoVerifierWithArg) { const id = `g#${guildId}/c#${channelId}`; if (!this.data.has(id)) { @@ -66,21 +72,27 @@ export default class MessageRAMCache { } this.cullLeastUsedChannelsIfNeeded(); - this.trimOldChannelMessagesIfNeeded(guildId, channelId); + this.trimOldChannelMessagesIfNeeded(guildId, channelId, recentAutoVerifier); } - deleteMessages(guildId: number, channelId: string, messages: Message[]) { + deleteMessages(guildId: number, channelId: string, messages: Message[], recentAutoVerifier: AutoVerifierWithArg) { const id = `g#${guildId}/c#${channelId}`; if (!this.data.has(id)) { return; } const value = this.data.get(id); if (!value) throw new ShouldNeverHappenError('unable to get message map'); + const beforeNumMessages = value.messages.size; for (const message of messages) { const deletedMessage = value.messages.get(message.id); if (!deletedMessage) continue; value.messages.delete(message.id); } + const afterNumMessages = value.messages.size; + // There could be new messages before. + // The query would have been fulfilled to the number of messages requested before the delete but now it is not. + // TODO: Test this (it's going to be a massive pain) + recentAutoVerifier.unverifyByFilter(query => query.channelId === channelId && beforeNumMessages >= query.number && afterNumMessages < query.number); } fetchRecentMessages(guildId: number, channelId: string, number: number): Message[] | null {