Software/platform
1
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor event queue to allow multiple types of events

Browse Source
This commit is contained in:
Dominic Ferrando
2026-07-12 05:21:33 -04:00
parent 4d1b3bc3ef
commit 4c2acfaede
5 changed files with 88 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/services/commerce.ts
+30 -22
View File
@@ -1,13 +1,13 @@
import { PrintfulClient, ProductSyncer, WebflowClient } from "@blade-and-brawn/commerce";
import { env, log } from "../util";
import { db } from "../database/db";
import { EventQueueService, type EventType } from "./event-queue";
import { EventQueue, type EventSpec } from "./event-queue";
import { Type as t } from "@sinclair/typebox";
import type { Static } from "@sinclair/typebox";
import { Value } from "@sinclair/typebox/value";
import { sql } from "kysely";
const ProductSyncPayloadSchema = t.Object({
const ProductSyncUpdatePayloadSchema = t.Object({
session: t.Object({
name: t.String(),
id: t.String()
@@ -18,22 +18,27 @@ const ProductSyncPayloadSchema = t.Object({
})
)
});
type ProductSyncPayload = Static<typeof ProductSyncPayloadSchema>;
type ProductSyncUpdatePayload = Static<typeof ProductSyncUpdatePayloadSchema>;
const ProductSyncStateSchema = t.Union([
const ProductSyncUpdateStateSchema = t.Union([
t.Object({
pProductIds: t.Optional(t.Array(t.Number()))
}),
t.Null()
]);
type ProductSyncState = Static<typeof ProductSyncStateSchema>;
type ProductSyncUpdateState = Static<typeof ProductSyncUpdateStateSchema>;
export type ProductSyncEvents = {
product_sync_update: EventSpec<ProductSyncUpdatePayload, ProductSyncUpdateState>
// TODO: product_sync_delete, once its schema/processor are written
};
export class CommerceService {
public readonly Printful = new PrintfulClient({
readonly Printful = new PrintfulClient({
token: env.PRINTFUL_AUTH,
storeId: env.PRINTFUL_STORE_ID,
});
public readonly Webflow = new WebflowClient({
readonly Webflow = new WebflowClient({
siteId: env.WEBFLOW_SITE_ID,
collectionsId: env.WEBFLOW_COLLECTION_ID,
token: env.WEBFLOW_AUTH,
@@ -43,29 +48,32 @@ export class CommerceService {
}
class ProductSyncService {
static readonly EVENT_TYPE: EventType = "product_sync";
readonly Queue: EventQueueService<ProductSyncPayload, ProductSyncState>;
static readonly EVENT_GROUP = "product_sync";
readonly Queue: EventQueue<"product_sync", ProductSyncEvents>;
private readonly ProductSyncer: ProductSyncer;
// TODO: make each service have their own logger, so we can enforce .child({component: <ServiceName>})
constructor(private readonly Printful: PrintfulClient, private readonly Webflow: WebflowClient) {
this.ProductSyncer = new ProductSyncer(this.Printful, this.Webflow, log);
this.Queue = new EventQueueService({
type: ProductSyncService.EVENT_TYPE,
schemas: { payload: ProductSyncPayloadSchema, state: ProductSyncStateSchema },
this.Queue = new EventQueue<"product_sync", ProductSyncEvents>({
group: ProductSyncService.EVENT_GROUP,
cfg: {
retry: { limit: 3, delayMs: 10000 },
concurrency: { limit: 1, timeoutMs: 600000 }
},
processor: async (id, payload, setState) => {
await this.ProductSyncer.run({
filter: payload.filter,
beforeStep: async (pProductIds: number[]) => {
await setState({ pProductIds });
types: {
"product_sync_update": {
schemas: { payload: ProductSyncUpdatePayloadSchema, state: ProductSyncUpdateStateSchema },
processor: async (id, payload, setState) => {
await this.ProductSyncer.syncApparel({
filter: payload.filter,
beforeStep: async (pProductIds: number[]) => {
await setState({ pProductIds });
}
});
}
});
}
}
});
}
@@ -73,7 +81,7 @@ class ProductSyncService {
async getLatestSyncState(sessionId: string) {
const latestSync = await db.selectFrom("events")
.select(["started_at", "ended_at", "has_failed", "state"])
.where("type", "=", ProductSyncService.EVENT_TYPE)
.where("group", "=", ProductSyncService.EVENT_GROUP)
.where(sql<string>`payload->'session'->>'id'`, "=", sessionId)
.orderBy(sql`(started_at is not null and ended_at is null)`, "desc")
.orderBy("created_at", "desc")
@@ -81,11 +89,11 @@ class ProductSyncService {
.executeTakeFirst();
if (!latestSync) return;
Value.Assert(ProductSyncStateSchema, latestSync.state);
Value.Assert(ProductSyncUpdateStateSchema, latestSync.state);
return {
startDate: latestSync.started_at,
status: EventQueueService.status(latestSync),
status: EventQueue.status(latestSync),
syncingPProductIds: latestSync.state?.pProductIds ?? []
};
}
apps/api/src/services/event-queue.ts
+48 -31
View File
@@ -6,22 +6,26 @@ import { db } from "../database/db";
import { Value } from "@sinclair/typebox/value";
import type { TSchema } from "@sinclair/typebox";
export type EventType = "product_sync" | "product_order_created" /* TODO: more events... */;
export type EventSpec<P extends JsonValue = JsonValue, S extends JsonValue = JsonValue> = {
payload: P,
state: S
}
export type EventSource = "portal" | "printful" | "webflow";
type EventProcessor<Payload extends JsonValue, State extends JsonValue> =
(id: string, payload: Payload, setState: (state: State) => Promise<void>) => Promise<void>;
type EventProcessor<S extends EventSpec> =
(id: string, payload: S["payload"], setState: (state: S["state"]) => Promise<void>) => Promise<void>;
type EventStatus = "pending" | "processing" | "failed" | "fulfilled";
class AlreadyClaimedEventError extends Error { }
class ConcurrencyLimitReachedError extends Error { }
type EventQueueServiceOptions<Payload extends JsonValue, State extends JsonValue> = {
type: EventType,
type EventTypeOptions<S extends EventSpec> = {
schemas: {
payload: TSchema & { static: Payload },
state: TSchema & { static: State },
payload: TSchema & { static: S["payload"] },
state: TSchema & { static: S["state"] },
},
processor: EventProcessor<S>,
};
type EventQueueOptions<G extends string, M extends Record<string, EventSpec>> = {
group: G,
cfg: {
retry: {
limit: number,
@@ -32,17 +36,28 @@ type EventQueueServiceOptions<Payload extends JsonValue, State extends JsonValue
timeoutMs: number,
},
},
processor: EventProcessor<Payload, State>,
types: {
[T in keyof M]: EventTypeOptions<M[T]>
},
}
export class EventQueueService<Payload extends JsonValue, State extends JsonValue> {
private _lastCleanDate: Date = new Date(0);
readonly cfg: EventQueueServiceOptions<Payload, State>["cfg"];
readonly type: EventType
type EnqueueOptions<M extends Record<string, EventSpec>, T extends keyof M & string> = {
source: EventSource,
type: T,
payload: M[T]["payload"]
}
constructor(private readonly opt: EventQueueServiceOptions<Payload, State>) {
class AlreadyClaimedEventError extends Error { }
class ConcurrencyLimitReachedError extends Error { }
export class EventQueue<G extends string, M extends Record<string, EventSpec>> {
private _lastCleanDate: Date = new Date(0);
readonly cfg: EventQueueOptions<G, M>["cfg"];
readonly group: G;
constructor(private readonly opt: EventQueueOptions<G, M>) {
this.cfg = opt.cfg;
this.type = opt.type;
this.group = opt.group;
}
get lastCleanDate() { return this._lastCleanDate; };
@@ -68,13 +83,13 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
continue;
}
if (err instanceof ConcurrencyLimitReachedError) {
await sleep(this.opt.cfg.retry.delayMs);
await sleep(this.cfg.retry.delayMs);
continue;
}
if (++consecutiveFailures <= this.opt.cfg.retry.limit) {
if (++consecutiveFailures <= this.cfg.retry.limit) {
log.warn({ err, consecutiveFailures }, "Dequeue failed, retrying")
await sleep(this.opt.cfg.retry.delayMs);
await sleep(this.cfg.retry.delayMs);
continue;
}
@@ -98,11 +113,11 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
eb(
eb.selectFrom("events")
.select(eb.fn.countAll().as("count"))
.where("type", "=", this.opt.type)
.where("group", "=", this.group)
.where("started_at", "is not", null)
.where("ended_at", "is", null),
"<",
this.opt.cfg.concurrency.limit
this.cfg.concurrency.limit
)
)
.executeTakeFirstOrThrow();
@@ -120,14 +135,14 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
};
log.trace({ name: front.id }, "dequeuing event");
await this.handleEvent(front.id, front.payload);
await this.handleEvent(front.id, front.type, front.payload);
return front;
}
async enqueue({ source, payload }: { source: EventSource, payload: Payload }): Promise<string> {
async enqueue<T extends keyof M & string>({ source, type, payload }: EnqueueOptions<M, T>): Promise<string> {
log.info({ payload }, "enqueuing");
const result = await db.insertInto("events")
.values({ type: this.opt.type, source, payload })
.values({ group: this.group, type, source, payload })
.returning(["id"])
.executeTakeFirstOrThrow();
return result.id;
@@ -152,7 +167,7 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
// Find and fail any timed out processing events
const processingEvents = await this.processingEvents();
for (const processingEvent of processingEvents) {
const processingEventTimedOut = (Date.now() - (processingEvent.started_at?.getTime() ?? 0) > this.opt.cfg.concurrency.timeoutMs);
const processingEventTimedOut = (Date.now() - (processingEvent.started_at?.getTime() ?? 0) > this.cfg.concurrency.timeoutMs);
if (processingEventTimedOut) {
log.warn({ name: processingEvent.id }, "processing event has timed out. failing it and continuing...");
await this.fail(processingEvent.id);
@@ -167,7 +182,7 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
async front(trx?: Transaction<DB>) {
const result = await (trx ?? db).selectFrom("events")
.selectAll()
.where("type", "=", this.opt.type)
.where("group", "=", this.group)
.where("started_at", "is", null)
.orderBy("created_at", "asc")
.limit(1)
@@ -178,17 +193,19 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
async processingEvents(trx?: Transaction<DB>) {
return await (trx ?? db).selectFrom("events")
.select(["started_at", "id"])
.where("type", "=", this.opt.type)
.where("group", "=", this.group)
.where("started_at", "is not", null)
.where("ended_at", "is", null)
.execute();
}
private async handleEvent(id: string, payload: JsonValue) {
private async handleEvent(id: string, type: string, payload: JsonValue) {
log.info({ id, payload }, "handling event");
if (!(type in this.opt.types)) throw new Error(`Unknown event type: ${type}`);
const cfg = this.opt.types[type as keyof M];
try {
Value.Assert(this.opt.schemas.payload, payload);
await this.opt.processor(id, payload, async (state) => {
Value.Assert(cfg.schemas.payload, payload);
await cfg.processor(id, payload, async (state) => {
await db.updateTable("events")
.set({ state })
.where("id", "=", id)