import { type Selectable, type Transaction } from "kysely";
import type { DB } from "../database/out/db";
import { log } from "../util";
import { sleep } from "bun";
import { db } from "../database/db";
import { AssertError, Value } from "@sinclair/typebox/value";
import type { TSchema, Static } from "@sinclair/typebox";
import { RateLimitError, secToMs } from "@blade-and-brawn/domain";
import { EventsService, type EventSource } from "./events";
type RetryType = "error" | "ratelimit";
type EventProcessor
=
(payload: Static
, setState: (state: Static) => Promise, id: string) => Promise;
export type EventTypeOptions = {
schemas: { payload: P, state: S },
processor: EventProcessor
,
dedupeKey: (payload: Static
) => string | undefined
};
type EventQueueCfg = {
error: {
limit: number,
initialRetryDelayMs: number,
},
concurrency: {
limit: number,
},
};
class AlreadyClaimedEventError extends Error { }
class ConcurrencyLimitReachedError extends Error { }
export function defineEventType
(opt: EventTypeOptions
): EventTypeOptions
{
return opt;
}
function addJitter(delay: number, jitterFactor: number): number {
const jitterMultiplier = 1 + (Math.random() - 0.5) * jitterFactor;
return delay * jitterMultiplier;
}
function exponentialDelay(initialDelay: number, tries: number, cap: number = Infinity) {
return Math.min(initialDelay * (2 ** tries), cap);
}
export class EventQueue>> {
private static readonly DEQUEUE_RETRY_DELAY_MS = secToMs(10);
private static readonly DEQUEUE_RETRY_LIMIT = 3;
private static readonly RATE_LIMIT_RETRY_LIMIT = 15;
private static readonly JITTER_FACTOR = 0.2;
readonly cfg: EventQueueCfg;
readonly group: G;
readonly types: T;
private readonly Events = new EventsService();
private _lastCleanDate: Date = new Date(0);
get lastCleanDate() { return this._lastCleanDate; };
constructor(opt: { group: G, cfg: EventQueueCfg, types: T }) {
this.cfg = opt.cfg;
this.group = opt.group;
this.types = opt.types;
}
async drain(): Promise {
log.trace("draining queue");
let consecutiveFailures = 0;
while (true) {
try {
const event = await this.dequeue();
if (!event) break;
consecutiveFailures = 0;
}
catch (err) {
if (err instanceof AlreadyClaimedEventError) {
log.warn("event processing already claimed by another worker, skipping");
continue;
}
if (err instanceof ConcurrencyLimitReachedError) {
await sleep(EventQueue.DEQUEUE_RETRY_DELAY_MS);
continue;
}
if (++consecutiveFailures <= EventQueue.DEQUEUE_RETRY_LIMIT) {
log.warn({ err, consecutiveFailures }, "dequeue failed, retrying")
await sleep(EventQueue.DEQUEUE_RETRY_DELAY_MS);
continue;
}
log.error({ err, consecutiveFailures }, "failed to drain queue");
throw err;
}
};
}
async dequeue() {
log.trace("dequeueing");
const front = await this.front();
if (!front) return;
const result = await db.updateTable("events")
.set({ started_at: new Date(), heartbeat_at: new Date() })
.where("id", "=", front.id)
.where("started_at", "is", null)
.where((eb) =>
eb(
eb.selectFrom("events")
.select(eb.fn.countAll().as("count"))
.where("group", "=", this.group)
.where("started_at", "is not", null)
.where("ended_at", "is", null),
"<",
this.cfg.concurrency.limit
)
)
.executeTakeFirstOrThrow();
// Guard against another worker/machine having already claimed this event and guard against max concurrency
if (result.numUpdatedRows === 0n) {
const frontFresh = await db.selectFrom("events")
.select("started_at")
.where("id", "=", front.id)
.executeTakeFirst();
if (!frontFresh) throw new AlreadyClaimedEventError();
if (frontFresh?.started_at) throw new AlreadyClaimedEventError();
throw new ConcurrencyLimitReachedError();
};
log.trace({ name: front.id }, "dequeuing event");
await this.process(front);
return front;
}
async enqueue({ source, type, payload }: {
source: EventSource,
type: K,
payload: Static
}): Promise {
if (!(type in this.types)) throw new Error(`event type not registered in this queue (${this.group}): ${type}`);
const cfg = this.types[type as keyof T]!;
const dedupeKey = cfg.dedupeKey(payload);
log.info({ payload, dedupeKey }, "enqueuing");
await db.insertInto("events")
.values({
group: this.group,
type,
source,
payload,
dedupe_key: dedupeKey ?? null,
})
.onConflict((oc) => oc
.columns(["group", "type", "dedupe_key"])
.where("dedupe_key", "is not", null)
.where("ended_at", "is", null)
.doNothing()
)
.returning(["id"])
.executeTakeFirst();
}
async front(trx?: Transaction) {
const result = await (trx ?? db).selectFrom("events")
.select(["id", "type", "payload", "available_at", "errors", "rate_limits"])
.where("group", "=", this.group)
.where("started_at", "is", null)
.where("available_at", "<=", new Date())
.orderBy("created_at", "asc")
.limit(1)
.executeTakeFirst();
return result;
}
async clean(opt: { filter?: { type?: string } } = {}) {
try {
await this.Events.clean({ filter: { group: this.group, ...opt.filter } });
}
finally {
this._lastCleanDate = new Date();
}
}
private async process(event: Pick, "id" | "type" | "errors" | "payload" | "rate_limits">) {
log.info({ id: event.id, type: event.type, payload: event.payload }, "processing event");
if (!(event.type in this.types)) throw new Error(`event type not registered in this queue (${this.group}): ${event.type}`);
const typeCfg = this.types[event.type as keyof T]!;
try {
Value.Assert(typeCfg.schemas.payload, event.payload);
await typeCfg.processor(event.payload, (s) => this.Events.setState(event.id, s), event.id);
await this.Events.resolve(event.id);
}
catch (err) {
await this.Events.setLastError(event.id, err);
if (err instanceof RateLimitError && event.rate_limits < EventQueue.RATE_LIMIT_RETRY_LIMIT) {
log.warn(
{ rateLimits: event.rate_limits, delay: err.retryDelayMs, source: err.source, payload: err.payload },
"encountered a rate limit error"
);
await this.scheduleRetry(event.id, err.retryDelayMs, "ratelimit");
return;
}
if (err instanceof AssertError) {
log.error({ errors: event.errors }, "could not process event: malformed payload");
await this.Events.fail(event.id);
return;
}
if (event.errors < this.cfg.error.limit) {
const retryDelayMs = addJitter(
exponentialDelay(this.cfg.error.initialRetryDelayMs, event.errors),
EventQueue.JITTER_FACTOR
);
log.warn({ err, errors: event.errors, retryDelayMs: retryDelayMs }, "encountered an error during event processing, retrying")
await this.scheduleRetry(event.id, retryDelayMs, "error");
return;
}
log.error({ err, errors: event.errors, rateLimits: event.rate_limits }, "event failed");
await this.Events.fail(event.id);
}
}
private async scheduleRetry(id: string, delayMs: number, type: RetryType) {
await db.updateTable("events")
.set((eb) => ({
errors: type === "error" ? eb("errors", "+", 1) : eb.ref("errors"),
rate_limits: type === "ratelimit" ? eb("rate_limits", "+", 1) : eb.ref("rate_limits"),
heartbeat_at: null,
started_at: null,
ended_at: null,
has_failed: false,
available_at: new Date(Date.now() + delayMs)
}))
.where("id", "=", id)
.execute();
}
}