Software/platform
1
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Finish up event queue abstraction

Browse Source
This commit is contained in:
Dominic Ferrando
2026-07-12 01:06:56 -04:00
parent 29a7a05a08
commit 80f8962a3b
4 changed files with 123 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/services/event-queue.ts
+40 -32
View File
@@ -6,8 +6,8 @@ import { db } from "../database/db";
import { Value } from "@sinclair/typebox/value";
import type { TSchema } from "@sinclair/typebox";
type EventType = "product_sync" | "product_order_created" /* TODO: more events... */;
type Source = "portal" | "printful" | "webflow";
export type EventType = "product_sync" | "product_order_created" /* TODO: more events... */;
export type EventSource = "portal" | "printful" | "webflow";
type EventHandler<Payload extends JsonValue, State extends JsonValue> =
(id: string, payload: Payload, setState: (state: State) => Promise<void>) => Promise<void>;
@@ -16,27 +16,33 @@ type EventStatus = "queued" | "active" | "failed" | "fulfilled";
class AlreadyClaimedEventError extends Error { }
class ConcurrencyLimitReachedError extends Error { }
type EventQueueServiceOptions = {
retry: {
limit: number,
delayMs: number,
type EventQueueServiceOptions<Payload extends JsonValue, State extends JsonValue> = {
type: EventType,
schemas: {
payload: TSchema & { static: Payload },
state: TSchema & { static: State },
},
concurrency: {
limit: number,
timeoutMs: number,
}
cfg: {
retry: {
limit: number,
delayMs: number,
},
concurrency: {
limit: number,
timeoutMs: number,
},
},
handler: EventHandler<Payload, State>,
}
export class EventQueueService<Payload extends JsonValue, State extends JsonValue> {
constructor(
readonly type: EventType,
private payloadSchema: TSchema & { static: Payload },
private stateSchema: TSchema & { static: State },
private handler: EventHandler<Payload, State>,
readonly opt: EventQueueServiceOptions
) { }
private _lastCleanDate: Date = new Date(0);
static status(event: Selectable<DB["events"]>): EventStatus {
constructor(private opt: EventQueueServiceOptions<Payload, State>) { }
get lastCleanDate() { return this._lastCleanDate; };
static status(event: Pick<Selectable<DB["events"]>, "started_at" | "ended_at" | "has_failed">): EventStatus {
if (!event.started_at) return "queued";
if (!event.ended_at) return "active";
return event.has_failed ? "failed" : "fulfilled";
@@ -57,17 +63,17 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
continue;
}
if (err instanceof ConcurrencyLimitReachedError) {
await sleep(this.opt.retry.delayMs);
await sleep(this.opt.cfg.retry.delayMs);
continue;
}
if (++consecutiveFailures <= this.opt.retry.limit) {
if (++consecutiveFailures <= this.opt.cfg.retry.limit) {
log.warn({ err, consecutiveFailures }, "Dequeue failed, retrying")
await sleep(this.opt.retry.delayMs);
await sleep(this.opt.cfg.retry.delayMs);
continue;
}
log.error({ err }, "failed to drain queue");
log.error({ err, consecutiveFailures }, "failed to drain queue");
throw err;
}
}
@@ -87,11 +93,11 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
eb(
eb.selectFrom("events")
.select(eb.fn.countAll().as("count"))
.where("type", "=", this.type)
.where("type", "=", this.opt.type)
.where("started_at", "is not", null)
.where("ended_at", "is", null),
"<",
this.opt.concurrency.limit
this.opt.cfg.concurrency.limit
)
)
.executeTakeFirstOrThrow();
@@ -113,10 +119,10 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
return front;
}
async enqueue(source: Source, payload: Payload): Promise<string> {
async enqueue({ source, payload }: { source: EventSource, payload: Payload }): Promise<string> {
log.info({ payload }, "enqueuing");
const result = await db.insertInto("events")
.values({ type: this.type, source, payload })
.values({ type: this.opt.type, source, payload })
.returning(["id"])
.executeTakeFirstOrThrow();
return result.id;
@@ -140,18 +146,20 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
// Find and fail any timed out active events
const activeEvents = await this.activeEvents();
for (const activeEvent of activeEvents) {
const activeEventTimedOut = (Date.now() - (activeEvent.started_at?.getTime() ?? 0) > this.opt.concurrency.timeoutMs);
const activeEventTimedOut = (Date.now() - (activeEvent.started_at?.getTime() ?? 0) > this.opt.cfg.concurrency.timeoutMs);
if (activeEventTimedOut) {
log.info({ name: activeEvent.id }, "active event has timed out. failing it and continuing...");
log.warn({ name: activeEvent.id }, "active event has timed out. failing it and continuing...");
await this.fail(activeEvent.id);
}
}
this._lastCleanDate = new Date();
}
async front(trx?: Transaction<DB>) {
const result = await (trx ?? db).selectFrom("events")
.selectAll()
.where("type", "=", this.type)
.where("type", "=", this.opt.type)
.where("started_at", "is", null)
.orderBy("created_at", "asc")
.limit(1)
@@ -162,7 +170,7 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
async activeEvents(trx?: Transaction<DB>) {
return await (trx ?? db).selectFrom("events")
.select(["started_at", "id"])
.where("type", "=", this.type)
.where("type", "=", this.opt.type)
.where("started_at", "is not", null)
.where("ended_at", "is", null)
.execute();
@@ -171,8 +179,8 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
private async handleEvent(id: string, payload: JsonValue) {
log.info({ id, payload }, "handling event");
try {
Value.Assert(this.payloadSchema, payload);
await this.handler(id, payload, async (state) => {
Value.Assert(this.opt.schemas.payload, payload);
await this.opt.handler(id, payload, async (state) => {
await db.updateTable("events")
.set({ state })
.where("id", "=", id)