michael/cordis
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

support for unverifying during cache update

Browse Source
This commit is contained in:
Michael Peters 2022-01-13 20:31:18 -06:00
parent 0bb0e79939
commit 228352bedb
8 changed files with 152 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/client/webapp/auto-verifier-with-args.ts
View File

@ -80,10 +80,17 @@ export class AutoVerifierWithArg<T, K> {
this.tokenQueries.clear();
}
unverifySome(toUnverifyFilter: (query: K) => boolean) {
const tokensToUnverify = Array.from(this.tokenQueries.values())
.filter(toUnverifyFilter)
.map(query => this.tokenizer(query));
getCachedQueries(): K[] {
return Array.from(this.tokenQueries.values());
}
unverifyByFilter(filter: (query: K) => boolean) {
this.unverifySome(this.getCachedQueries().filter(filter));
}
// use getUnverifySomeFunc for a more functional style
unverifySome(toUnverify: K[]) {
const tokensToUnverify = new Set(toUnverify.map(query => this.tokenizer(query)));
for (const token of tokensToUnverify) {
this.tokenAutoVerifiers.get(token)?.unverify();
this.tokenAutoVerifiers.delete(token);

4
src/client/webapp/auto-verifier.ts
View File

@ -24,7 +24,7 @@ export enum AutoVerifierChangesType {
export class AutoVerifier<T> {
public primaryPromise: Promise<T | null> | null = null;
public trustedPromise: Promise<T | null> | null = null;
public trustedStatus: 'none' | 'fetching' | 'verifying' | 'verified' | 'unverified' = 'none';
public trustedStatus: 'none' | 'fetching' | 'verifying' | 'verified' = 'none';
/**
* Allows a trusted function to verify the primary function
@ -226,7 +226,7 @@ export class AutoVerifier<T> {
// keep in mind that this will resolve 'unverified' promises with their resulting origTrustedPromise (which should be a good thing)
}
if (!resolved && trustedResult) {
if (!resolved) { // Removed 01/09/2021 pretty sure should not be here... && trustedResult
resolve(trustedResult);
resolved = true;
}

18
src/client/webapp/fetchable-pair-verifier.ts
View File

@ -18,9 +18,9 @@ export default class PairVerifierFetchable extends EventEmitter<Conflictable> im
private readonly fetchResourceVerifier: AutoVerifierWithArg<Resource, IDQuery>
private readonly fetchMessagesRecentVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>;
private readonly fetchMessagesBeforeVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>;
private readonly fetchMessagesAfterVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>;
public readonly fetchMessagesRecentVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>;
public readonly fetchMessagesBeforeVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>;
public readonly fetchMessagesAfterVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>;
constructor(
private primary: Fetchable & Lackable,
@ -146,11 +146,11 @@ export default class PairVerifierFetchable extends EventEmitter<Conflictable> im
async handleResourceConflict(query: IDQuery, changesType: AutoVerifierChangesType, primaryResource: Resource | null, trustedResource: Resource | null): Promise<boolean> {
let success = true;
if (changesType === AutoVerifierChangesType.PRIMARY_ONLY) {
success = success && await this.primary.handleResourceDeleted(trustedResource as Resource);
success = success && await this.primary.handleResourceDeleted(trustedResource as Resource, this.fetchResourceVerifier);
} else if (changesType === AutoVerifierChangesType.TRUSTED_ONLY) {
success = success && await this.primary.handleResourceAdded(trustedResource as Resource);
success = success && await this.primary.handleResourceAdded(trustedResource as Resource, this.fetchResourceVerifier);
} else if (changesType === AutoVerifierChangesType.CONFLICT) {
success = success && await this.primary.handleResourceChanged(trustedResource as Resource);
success = success && await this.primary.handleResourceChanged(trustedResource as Resource, this.fetchResourceVerifier);
this.emit('conflict-resource', query, changesType, primaryResource as Resource, trustedResource as Resource);
}
return success;
@ -158,9 +158,9 @@ export default class PairVerifierFetchable extends EventEmitter<Conflictable> im
async handleMessagesConflict(query: PartialMessageListQuery, changesType: AutoVerifierChangesType, changes: Changes<Message>): Promise<boolean> {
let success = true;
if (changes.added.length > 0) success = success && await this.primary.handleMessagesAdded(changes.added);
if (changes.updated.length > 0) success = success && await this.primary.handleMessagesChanged(changes.updated.map(change => change.newDataPoint));
if (changes.deleted.length > 0) success = success && await this.primary.handleMessagesDeleted(changes.deleted);
if (changes.added.length > 0) success = success && await this.primary.handleMessagesAdded(changes.added, this.fetchMessagesRecentVerifier, this.fetchMessagesBeforeVerifier, this.fetchMessagesAfterVerifier);
if (changes.updated.length > 0) success = success && await this.primary.handleMessagesChanged(changes.updated.map(change => change.newDataPoint), this.fetchMessagesRecentVerifier, this.fetchMessagesBeforeVerifier, this.fetchMessagesAfterVerifier);
if (changes.deleted.length > 0) success = success && await this.primary.handleMessagesDeleted(changes.deleted, this.fetchMessagesRecentVerifier, this.fetchMessagesBeforeVerifier, this.fetchMessagesAfterVerifier);
if (changesType === AutoVerifierChangesType.CONFLICT) {
this.emit('conflict-messages', query, changesType, changes);

34
src/client/webapp/guild-combined.ts
View File

@ -43,6 +43,9 @@ export default class CombinedGuild extends EventEmitter<Connectable & Conflictab
this.personalDBGuild = new PersonalDBGuild(personalDB, this.id, this.memberId);
this.socketGuild = new SocketGuild(socket, socketVerifier);
const diskSocket = new PairVerifierFetchable(this.personalDBGuild, this.socketGuild);
const ramDiskSocket = new PairVerifierFetchable(this.ramGuild, diskSocket);
// Connect/Disconnect
this.socketGuild.on('connect', () => {
LOG.info(`g#${this.id} connected`);
@ -104,8 +107,18 @@ export default class CombinedGuild extends EventEmitter<Connectable & Conflictab
for (const message of messages) {
LOG.info(`g#${this.id} ${message}`);
}
this.ramGuild.handleMessagesAdded(messages);
await this.personalDBGuild.handleMessagesAdded(messages);
this.ramGuild.handleMessagesAdded(
messages,
diskSocket.fetchMessagesRecentVerifier,
diskSocket.fetchMessagesBeforeVerifier,
diskSocket.fetchMessagesAfterVerifier
);
await this.personalDBGuild.handleMessagesAdded(
messages,
diskSocket.fetchMessagesRecentVerifier,
diskSocket.fetchMessagesBeforeVerifier,
diskSocket.fetchMessagesAfterVerifier
);
const members = await this.grabRAMMembersMap();
const channels = await this.grabRAMChannelsMap();
for (const message of messages) message.fill(members, channels);
@ -115,8 +128,18 @@ export default class CombinedGuild extends EventEmitter<Connectable & Conflictab
for (const message of messages) {
LOG.info(`g#${this.id} updated ${message}`);
}
this.ramGuild.handleMessagesChanged(messages);
await this.personalDBGuild.handleMessagesChanged(messages);
this.ramGuild.handleMessagesChanged(
messages,
ramDiskSocket.fetchMessagesRecentVerifier,
ramDiskSocket.fetchMessagesBeforeVerifier,
ramDiskSocket.fetchMessagesAfterVerifier
);
await this.personalDBGuild.handleMessagesChanged(
messages,
diskSocket.fetchMessagesRecentVerifier,
diskSocket.fetchMessagesBeforeVerifier,
diskSocket.fetchMessagesAfterVerifier
);
const members = await this.grabRAMMembersMap();
const channels = await this.grabRAMChannelsMap();
for (const message of messages) message.fill(members, channels);
@ -157,9 +180,6 @@ export default class CombinedGuild extends EventEmitter<Connectable & Conflictab
this.emit('remove-tokens', removedTokens);
});
const diskSocket = new PairVerifierFetchable(this.personalDBGuild, this.socketGuild);
const ramDiskSocket = new PairVerifierFetchable(this.ramGuild, diskSocket);
// Forward the conflict events from the last verifier in the chain
ramDiskSocket.on('conflict-metadata', (changesType: AutoVerifierChangesType, oldGuildMeta: GuildMetadata, newGuildMeta: GuildMetadata) => {
LOG.info(`g#${this.id} metadata conflict`, { oldGuildMeta, newGuildMeta });

25
src/client/webapp/guild-personal-db.ts
View File

@ -7,6 +7,7 @@ import { AsyncFetchable, AsyncLackable } from "./guild-types";
import { Channel, GuildMetadata, GuildMetadataLocal, Member, Message, Resource, SocketConfig, Token } from "./data-types";
import PersonalDB from "./personal-db";
import { AutoVerifierWithArg, PartialMessageListQuery } from './auto-verifier-with-args';
export default class PersonalDBGuild implements AsyncFetchable, AsyncLackable {
constructor(
@ -94,17 +95,35 @@ export default class PersonalDBGuild implements AsyncFetchable, AsyncLackable {
return true;
}
async handleMessagesAdded(addedMessages: Message[]): Promise<boolean> {
async handleMessagesAdded(
addedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): Promise<boolean> {
// The PersonalDB does not remove old messages (yet) so nothing here would unverify
//LOG.debug(addedMessages.length + ' personal messages added');
await this.db.addMessages(this.guildId, addedMessages);
return true;
}
async handleMessagesChanged(changedMessages: Message[]): Promise<boolean> {
async handleMessagesChanged(
changedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): Promise<boolean> {
// The PersonalDB does not remove old messages (yet) so nothing here would unverify
//LOG.debug(changedMessages.length + ' personal messages changed');
await this.db.updateMessages(this.guildId, changedMessages);
return true;
}
async handleMessagesDeleted(deletedMessages: Message[]): Promise<boolean> {
async handleMessagesDeleted(
deletedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): Promise<boolean> {
// The PersonalDB does not remove old messages (yet) so nothing here would unverify
//LOG.debug(deletedMessages.length + ' personal messages deleted');
await this.db.deleteMessages(this.guildId, deletedMessages);
return true;

28
src/client/webapp/guild-ram.ts
View File

@ -7,6 +7,7 @@ import { SyncFetchable, SyncLackable } from "./guild-types";
import { GuildMetadata, Member, Channel, Message, Resource, Token } from "./data-types";
import MessageRAMCache from "./message-ram-cache";
import ResourceRAMCache from "./resource-ram-cache";
import { AutoVerifierWithArg, PartialMessageListQuery } from './auto-verifier-with-args';
export default class RAMGuild implements SyncFetchable, SyncLackable {
private metadata: GuildMetadata | null = null;
@ -97,7 +98,12 @@ export default class RAMGuild implements SyncFetchable, SyncLackable {
return true;
}
handleMessagesAdded(addedMessages: Message[]): boolean {
handleMessagesAdded(
addedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): boolean {
const byChannel = new Map<string, Message[]>();
for (const message of addedMessages) {
if (!byChannel.has(message.channel.id)) {
@ -107,11 +113,16 @@ export default class RAMGuild implements SyncFetchable, SyncLackable {
}
}
for (const [ channelId, messages ] of byChannel.entries()) {
this.recentMessages.upsertMessages(this.guildId, channelId, messages);
this.recentMessages.upsertMessages(this.guildId, channelId, messages, recentAutoVerifier);
}
return true;
}
handleMessagesChanged(changedMessages: Message[]): boolean {
handleMessagesChanged(
changedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): boolean {
const byChannel = new Map<string, Message[]>();
for (const message of changedMessages) {
if (!byChannel.has(message.channel.id)) {
@ -121,11 +132,16 @@ export default class RAMGuild implements SyncFetchable, SyncLackable {
}
}
for (const [ channelId, messages ] of byChannel.entries()) {
this.recentMessages.upsertMessages(this.guildId, channelId, messages);
this.recentMessages.upsertMessages(this.guildId, channelId, messages, recentAutoVerifier);
}
return true;
}
handleMessagesDeleted(deletedMessages: Message[]): boolean {
handleMessagesDeleted(
deletedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): boolean {
const byChannel = new Map<string, Message[]>();
for (const message of deletedMessages) {
if (!byChannel.has(message.channel.id)) {
@ -135,7 +151,7 @@ export default class RAMGuild implements SyncFetchable, SyncLackable {
}
}
for (const [ channelId, messages ] of byChannel.entries()) {
this.recentMessages.deleteMessages(this.guildId, channelId, messages);
this.recentMessages.deleteMessages(this.guildId, channelId, messages, recentAutoVerifier);
}
return true;
}

56
src/client/webapp/guild-types.ts
View File

@ -1,6 +1,6 @@
import { Changes, Channel, GuildMetadata, Member, Message, Resource, Token } from './data-types';
import { AutoVerifierChangesType } from './auto-verifier';
import { IDQuery, PartialMessageListQuery } from './auto-verifier-with-args';
import { AutoVerifierWithArg, IDQuery, PartialMessageListQuery } from './auto-verifier-with-args';
// Fetchable
@ -72,13 +72,28 @@ export interface AsyncLackable {
handleChannelsChanged(changedChannels: Channel[]): Promise<boolean>;
handleChannelsDeleted(deletedChannels: Channel[]): Promise<boolean>;
handleMessagesAdded(addedMessages: Message[]): Promise<boolean>;
handleMessagesChanged(changedMessages: Message[]): Promise<boolean>;
handleMessagesDeleted(deletedMessages: Message[]): Promise<boolean>;
handleMessagesAdded(
addedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): Promise<boolean>;
handleMessagesChanged(
changedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): Promise<boolean>;
handleMessagesDeleted(
deletedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): Promise<boolean>;
handleResourceAdded(addedResource: Resource): Promise<boolean>;
handleResourceChanged(changedResource: Resource): Promise<boolean>;
handleResourceDeleted(deletedResource: Resource): Promise<boolean>;
handleResourceAdded(addedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg<Resource, IDQuery>): Promise<boolean>;
handleResourceChanged(changedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg<Resource, IDQuery>): Promise<boolean>;
handleResourceDeleted(deletedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg<Resource, IDQuery>): Promise<boolean>;
handleTokensAdded(addedTokens: Token[]): Promise<boolean>;
handleTokensChanged(changedTokens: Token[]): Promise<boolean>;
@ -96,13 +111,28 @@ export interface SyncLackable {
handleChannelsChanged(changedChannels: Channel[]): boolean;
handleChannelsDeleted(deletedChannels: Channel[]): boolean;
handleMessagesAdded(addedMessages: Message[]): boolean;
handleMessagesChanged(changedMessages: Message[]): boolean;
handleMessagesDeleted(deletedMessages: Message[]): boolean;
handleMessagesAdded(
addedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): boolean;
handleMessagesChanged(
changedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): boolean;
handleMessagesDeleted(
deletedMessages: Message[],
recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
beforeAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>,
afterAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>
): boolean;
handleResourceAdded(addedResource: Resource): boolean;
handleResourceChanged(changedResource: Resource): boolean;
handleResourceDeleted(deletedResource: Resource): boolean;
handleResourceAdded(addedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg<Resource, IDQuery>): boolean;
handleResourceChanged(changedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg<Resource, IDQuery>): boolean;
handleResourceDeleted(deletedResource: Resource, resourceAutoVerifier: AutoVerifierWithArg<Resource, IDQuery>): boolean;
handleTokensAdded(addedTokens: Token[]): boolean;
handleTokensChanged(changedTokens: Token[]): boolean;

20
src/client/webapp/message-ram-cache.ts
View File

@ -4,6 +4,7 @@ import Globals from "./globals";
import * as electronRemote from '@electron/remote';
const electronConsole = electronRemote.getGlobal('console') as Console;
import Logger from '../../logger/logger';
import { AutoVerifierWithArg, PartialMessageListQuery } from "./auto-verifier-with-args";
const LOG = Logger.create(__filename, electronConsole);
interface MessagesWithMetadata {
@ -33,13 +34,15 @@ export default class MessageRAMCache {
}
// Removes the oldest messages from the channel until the channel is under the max cached character limit
private trimOldChannelMessagesIfNeeded(guildId: number, channelId: string) {
// Returns true if messages were removed
private trimOldChannelMessagesIfNeeded(guildId: number, channelId: string, recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>): void {
const id = `g#${guildId}/c#${channelId}`;
const value = this.data.get(id);
if (!value) return;
if (value.totalCharacters > Globals.MAX_RAM_CACHED_MESSAGES_CHANNEL_CHARACTERS) {
const messages = Array.from(value.messages.values()).sort(Message.sortOrder);
const beforeNumMessages = value.messages.size;
while (value.totalCharacters > Globals.MAX_RAM_CACHED_MESSAGES_CHANNEL_CHARACTERS) {
const message = messages.shift();
if (!message) throw new ShouldNeverHappenError('could not find a message to clear');
@ -47,10 +50,13 @@ export default class MessageRAMCache {
value.totalCharacters -= message.text?.length ?? 0;
this.totalCharacters -= message.text?.length ?? 0;
}
const afterNumMessages = value.messages.size;
// TODO: Test this (it's going to be a massive pain)
recentAutoVerifier.unverifyByFilter(query => query.channelId === channelId && beforeNumMessages >= query.number && afterNumMessages < query.number);
}
}
upsertMessages(guildId: number, channelId: string, messages: Message[]) {
upsertMessages(guildId: number, channelId: string, messages: Message[], recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>) {
const id = `g#${guildId}/c#${channelId}`;
if (!this.data.has(id)) {
@ -66,21 +72,27 @@ export default class MessageRAMCache {
}
this.cullLeastUsedChannelsIfNeeded();
this.trimOldChannelMessagesIfNeeded(guildId, channelId);
this.trimOldChannelMessagesIfNeeded(guildId, channelId, recentAutoVerifier);
}
deleteMessages(guildId: number, channelId: string, messages: Message[]) {
deleteMessages(guildId: number, channelId: string, messages: Message[], recentAutoVerifier: AutoVerifierWithArg<Message[], PartialMessageListQuery>) {
const id = `g#${guildId}/c#${channelId}`;
if (!this.data.has(id)) {
return;
}
const value = this.data.get(id);
if (!value) throw new ShouldNeverHappenError('unable to get message map');
const beforeNumMessages = value.messages.size;
for (const message of messages) {
const deletedMessage = value.messages.get(message.id);
if (!deletedMessage) continue;
value.messages.delete(message.id);
}
const afterNumMessages = value.messages.size;
// There could be new messages before.
// The query would have been fulfilled to the number of messages requested before the delete but now it is not.
// TODO: Test this (it's going to be a massive pain)
recentAutoVerifier.unverifyByFilter(query => query.channelId === channelId && beforeNumMessages >= query.number && afterNumMessages < query.number);
}
fetchRecentMessages(guildId: number, channelId: string, number: number): Message[] | null {