Software/platform
1
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor event queue to allow multiple types of events

Browse Source
This commit is contained in:
Dominic Ferrando
2026-07-12 05:21:33 -04:00
parent 4d1b3bc3ef
commit 4c2acfaede
5 changed files with 88 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/database/migrations/2026-06-03.ts
+1
View File
@@ -5,6 +5,7 @@ export async function up(db: Kysely<any>): Promise<void> {
// TABLE: EVENTS // TABLE: EVENTS
await db.schema.createTable("events") await db.schema.createTable("events")
.$call(addDefaultColumns) .$call(addDefaultColumns)
.addColumn("group", "text", (cb) => cb.notNull())
.addColumn("type", "text", (cb) => cb.notNull()) .addColumn("type", "text", (cb) => cb.notNull())
.addColumn("source", "text", (cb) => cb.notNull()) .addColumn("source", "text", (cb) => cb.notNull())
.addColumn("payload", "jsonb", (cb) => cb.notNull()) .addColumn("payload", "jsonb", (cb) => cb.notNull())
apps/api/src/server.ts
+6 -4
View File
@@ -20,7 +20,7 @@ import { randomUUIDv7, sleep } from "bun";
import { CalculatorService, CalculatorUnavailableError } from "./services/calculator"; import { CalculatorService, CalculatorUnavailableError } from "./services/calculator";
import { StandardsParamsSchema } from "@blade-and-brawn/calculator"; import { StandardsParamsSchema } from "@blade-and-brawn/calculator";
import { StandardsService } from "./services/standards"; import { StandardsService } from "./services/standards";
import type { EventQueueService } from "./services/event-queue"; import type { EventQueue, EventSpec } from "./services/event-queue";
// CONSTANTS // CONSTANTS
// ----------------------- // -----------------------
@@ -201,6 +201,7 @@ export const app = new Elysia()
// Run sync // Run sync
.post("/:pProductId?", async ({ params: { pProductId }, sessionId }) => { .post("/:pProductId?", async ({ params: { pProductId }, sessionId }) => {
await s.Commerce.ProductSync.Queue.enqueue({ await s.Commerce.ProductSync.Queue.enqueue({
type: "product_sync_update",
source: "portal", source: "portal",
payload: { payload: {
session: { id: sessionId, name: "Portal" }, session: { id: sessionId, name: "Portal" },
@@ -237,6 +238,7 @@ export const app = new Elysia()
log.info({ productId: pProduct.id }, "printful webhook: product updated"); log.info({ productId: pProduct.id }, "printful webhook: product updated");
await s.Commerce.ProductSync.Queue.enqueue({ await s.Commerce.ProductSync.Queue.enqueue({
type: "product_sync_update",
source: "printful", source: "printful",
payload: { payload: {
session: { id: randomUUIDv7(), name: "Printful" }, session: { id: randomUUIDv7(), name: "Printful" },
@@ -317,19 +319,19 @@ app.listen(3000, async () => {
log.info({ port: 3000 }, "server started") log.info({ port: 3000 }, "server started")
// MANAGE QUEUES // MANAGE QUEUES
const queues: EventQueueService<any, any>[] = [s.Commerce.ProductSync.Queue]; const queues: EventQueue<string, Record<string, EventSpec<any, any>>>[] = [s.Commerce.ProductSync.Queue];
for (const queue of queues) { for (const queue of queues) {
(async () => { (async () => {
while (true) { while (true) {
try { try {
// clean, if ready // clean, if ready
if ((Date.now() - queue.lastCleanDate.getTime()) >= queue.cfg.concurrency.timeoutMs) if ((Date.now() - queue.lastCleanDate.getTime()) >= queue.cfg.concurrency.timeoutMs)
await queue.clean().catch((err) => log.error({ name: queue.type, err })); await queue.clean().catch((err) => log.error({ name: queue.group, err }));
// drain // drain
await queue.drain(); await queue.drain();
} }
catch (err) { catch (err) {
log.error({ name: queue.type, err }, "error occurred during queue management"); log.error({ name: queue.group, err }, "error occurred during queue management");
} }
await sleep(EVENT_QUEUE_MANAGE_DELAY_MS); await sleep(EVENT_QUEUE_MANAGE_DELAY_MS);
} }
apps/api/src/services/commerce.ts
+30 -22
View File
@@ -1,13 +1,13 @@
import { PrintfulClient, ProductSyncer, WebflowClient } from "@blade-and-brawn/commerce"; import { PrintfulClient, ProductSyncer, WebflowClient } from "@blade-and-brawn/commerce";
import { env, log } from "../util"; import { env, log } from "../util";
import { db } from "../database/db"; import { db } from "../database/db";
import { EventQueueService, type EventType } from "./event-queue"; import { EventQueue, type EventSpec } from "./event-queue";
import { Type as t } from "@sinclair/typebox"; import { Type as t } from "@sinclair/typebox";
import type { Static } from "@sinclair/typebox"; import type { Static } from "@sinclair/typebox";
import { Value } from "@sinclair/typebox/value"; import { Value } from "@sinclair/typebox/value";
import { sql } from "kysely"; import { sql } from "kysely";
const ProductSyncPayloadSchema = t.Object({ const ProductSyncUpdatePayloadSchema = t.Object({
session: t.Object({ session: t.Object({
name: t.String(), name: t.String(),
id: t.String() id: t.String()
@@ -18,22 +18,27 @@ const ProductSyncPayloadSchema = t.Object({
}) })
) )
}); });
type ProductSyncPayload = Static<typeof ProductSyncPayloadSchema>; type ProductSyncUpdatePayload = Static<typeof ProductSyncUpdatePayloadSchema>;
const ProductSyncStateSchema = t.Union([ const ProductSyncUpdateStateSchema = t.Union([
t.Object({ t.Object({
pProductIds: t.Optional(t.Array(t.Number())) pProductIds: t.Optional(t.Array(t.Number()))
}), }),
t.Null() t.Null()
]); ]);
type ProductSyncState = Static<typeof ProductSyncStateSchema>; type ProductSyncUpdateState = Static<typeof ProductSyncUpdateStateSchema>;
export type ProductSyncEvents = {
product_sync_update: EventSpec<ProductSyncUpdatePayload, ProductSyncUpdateState>
// TODO: product_sync_delete, once its schema/processor are written
};
export class CommerceService { export class CommerceService {
public readonly Printful = new PrintfulClient({ readonly Printful = new PrintfulClient({
token: env.PRINTFUL_AUTH, token: env.PRINTFUL_AUTH,
storeId: env.PRINTFUL_STORE_ID, storeId: env.PRINTFUL_STORE_ID,
}); });
public readonly Webflow = new WebflowClient({ readonly Webflow = new WebflowClient({
siteId: env.WEBFLOW_SITE_ID, siteId: env.WEBFLOW_SITE_ID,
collectionsId: env.WEBFLOW_COLLECTION_ID, collectionsId: env.WEBFLOW_COLLECTION_ID,
token: env.WEBFLOW_AUTH, token: env.WEBFLOW_AUTH,
@@ -43,29 +48,32 @@ export class CommerceService {
} }
class ProductSyncService { class ProductSyncService {
static readonly EVENT_TYPE: EventType = "product_sync"; static readonly EVENT_GROUP = "product_sync";
readonly Queue: EventQueue<"product_sync", ProductSyncEvents>;
readonly Queue: EventQueueService<ProductSyncPayload, ProductSyncState>;
private readonly ProductSyncer: ProductSyncer; private readonly ProductSyncer: ProductSyncer;
// TODO: make each service have their own logger, so we can enforce .child({component: <ServiceName>}) // TODO: make each service have their own logger, so we can enforce .child({component: <ServiceName>})
constructor(private readonly Printful: PrintfulClient, private readonly Webflow: WebflowClient) { constructor(private readonly Printful: PrintfulClient, private readonly Webflow: WebflowClient) {
this.ProductSyncer = new ProductSyncer(this.Printful, this.Webflow, log); this.ProductSyncer = new ProductSyncer(this.Printful, this.Webflow, log);
this.Queue = new EventQueueService({ this.Queue = new EventQueue<"product_sync", ProductSyncEvents>({
type: ProductSyncService.EVENT_TYPE, group: ProductSyncService.EVENT_GROUP,
schemas: { payload: ProductSyncPayloadSchema, state: ProductSyncStateSchema },
cfg: { cfg: {
retry: { limit: 3, delayMs: 10000 }, retry: { limit: 3, delayMs: 10000 },
concurrency: { limit: 1, timeoutMs: 600000 } concurrency: { limit: 1, timeoutMs: 600000 }
}, },
processor: async (id, payload, setState) => { types: {
await this.ProductSyncer.run({ "product_sync_update": {
filter: payload.filter, schemas: { payload: ProductSyncUpdatePayloadSchema, state: ProductSyncUpdateStateSchema },
beforeStep: async (pProductIds: number[]) => { processor: async (id, payload, setState) => {
await setState({ pProductIds }); await this.ProductSyncer.syncApparel({
filter: payload.filter,
beforeStep: async (pProductIds: number[]) => {
await setState({ pProductIds });
}
});
} }
}); }
} }
}); });
} }
@@ -73,7 +81,7 @@ class ProductSyncService {
async getLatestSyncState(sessionId: string) { async getLatestSyncState(sessionId: string) {
const latestSync = await db.selectFrom("events") const latestSync = await db.selectFrom("events")
.select(["started_at", "ended_at", "has_failed", "state"]) .select(["started_at", "ended_at", "has_failed", "state"])
.where("type", "=", ProductSyncService.EVENT_TYPE) .where("group", "=", ProductSyncService.EVENT_GROUP)
.where(sql<string>`payload->'session'->>'id'`, "=", sessionId) .where(sql<string>`payload->'session'->>'id'`, "=", sessionId)
.orderBy(sql`(started_at is not null and ended_at is null)`, "desc") .orderBy(sql`(started_at is not null and ended_at is null)`, "desc")
.orderBy("created_at", "desc") .orderBy("created_at", "desc")
@@ -81,11 +89,11 @@ class ProductSyncService {
.executeTakeFirst(); .executeTakeFirst();
if (!latestSync) return; if (!latestSync) return;
Value.Assert(ProductSyncStateSchema, latestSync.state); Value.Assert(ProductSyncUpdateStateSchema, latestSync.state);
return { return {
startDate: latestSync.started_at, startDate: latestSync.started_at,
status: EventQueueService.status(latestSync), status: EventQueue.status(latestSync),
syncingPProductIds: latestSync.state?.pProductIds ?? [] syncingPProductIds: latestSync.state?.pProductIds ?? []
}; };
} }
apps/api/src/services/event-queue.ts
+48 -31
View File
@@ -6,22 +6,26 @@ import { db } from "../database/db";
import { Value } from "@sinclair/typebox/value"; import { Value } from "@sinclair/typebox/value";
import type { TSchema } from "@sinclair/typebox"; import type { TSchema } from "@sinclair/typebox";
export type EventType = "product_sync" | "product_order_created" /* TODO: more events... */; export type EventSpec<P extends JsonValue = JsonValue, S extends JsonValue = JsonValue> = {
payload: P,
state: S
}
export type EventSource = "portal" | "printful" | "webflow"; export type EventSource = "portal" | "printful" | "webflow";
type EventProcessor<Payload extends JsonValue, State extends JsonValue> = type EventProcessor<S extends EventSpec> =
(id: string, payload: Payload, setState: (state: State) => Promise<void>) => Promise<void>; (id: string, payload: S["payload"], setState: (state: S["state"]) => Promise<void>) => Promise<void>;
type EventStatus = "pending" | "processing" | "failed" | "fulfilled"; type EventStatus = "pending" | "processing" | "failed" | "fulfilled";
class AlreadyClaimedEventError extends Error { } type EventTypeOptions<S extends EventSpec> = {
class ConcurrencyLimitReachedError extends Error { }
type EventQueueServiceOptions<Payload extends JsonValue, State extends JsonValue> = {
type: EventType,
schemas: { schemas: {
payload: TSchema & { static: Payload }, payload: TSchema & { static: S["payload"] },
state: TSchema & { static: State }, state: TSchema & { static: S["state"] },
}, },
processor: EventProcessor<S>,
};
type EventQueueOptions<G extends string, M extends Record<string, EventSpec>> = {
group: G,
cfg: { cfg: {
retry: { retry: {
limit: number, limit: number,
@@ -32,17 +36,28 @@ type EventQueueServiceOptions<Payload extends JsonValue, State extends JsonValue
timeoutMs: number, timeoutMs: number,
}, },
}, },
processor: EventProcessor<Payload, State>, types: {
[T in keyof M]: EventTypeOptions<M[T]>
},
} }
export class EventQueueService<Payload extends JsonValue, State extends JsonValue> { type EnqueueOptions<M extends Record<string, EventSpec>, T extends keyof M & string> = {
private _lastCleanDate: Date = new Date(0); source: EventSource,
readonly cfg: EventQueueServiceOptions<Payload, State>["cfg"]; type: T,
readonly type: EventType payload: M[T]["payload"]
}
constructor(private readonly opt: EventQueueServiceOptions<Payload, State>) { class AlreadyClaimedEventError extends Error { }
class ConcurrencyLimitReachedError extends Error { }
export class EventQueue<G extends string, M extends Record<string, EventSpec>> {
private _lastCleanDate: Date = new Date(0);
readonly cfg: EventQueueOptions<G, M>["cfg"];
readonly group: G;
constructor(private readonly opt: EventQueueOptions<G, M>) {
this.cfg = opt.cfg; this.cfg = opt.cfg;
this.type = opt.type; this.group = opt.group;
} }
get lastCleanDate() { return this._lastCleanDate; }; get lastCleanDate() { return this._lastCleanDate; };
@@ -68,13 +83,13 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
continue; continue;
} }
if (err instanceof ConcurrencyLimitReachedError) { if (err instanceof ConcurrencyLimitReachedError) {
await sleep(this.opt.cfg.retry.delayMs); await sleep(this.cfg.retry.delayMs);
continue; continue;
} }
if (++consecutiveFailures <= this.opt.cfg.retry.limit) { if (++consecutiveFailures <= this.cfg.retry.limit) {
log.warn({ err, consecutiveFailures }, "Dequeue failed, retrying") log.warn({ err, consecutiveFailures }, "Dequeue failed, retrying")
await sleep(this.opt.cfg.retry.delayMs); await sleep(this.cfg.retry.delayMs);
continue; continue;
} }
@@ -98,11 +113,11 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
eb( eb(
eb.selectFrom("events") eb.selectFrom("events")
.select(eb.fn.countAll().as("count")) .select(eb.fn.countAll().as("count"))
.where("type", "=", this.opt.type) .where("group", "=", this.group)
.where("started_at", "is not", null) .where("started_at", "is not", null)
.where("ended_at", "is", null), .where("ended_at", "is", null),
"<", "<",
this.opt.cfg.concurrency.limit this.cfg.concurrency.limit
) )
) )
.executeTakeFirstOrThrow(); .executeTakeFirstOrThrow();
@@ -120,14 +135,14 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
}; };
log.trace({ name: front.id }, "dequeuing event"); log.trace({ name: front.id }, "dequeuing event");
await this.handleEvent(front.id, front.payload); await this.handleEvent(front.id, front.type, front.payload);
return front; return front;
} }
async enqueue({ source, payload }: { source: EventSource, payload: Payload }): Promise<string> { async enqueue<T extends keyof M & string>({ source, type, payload }: EnqueueOptions<M, T>): Promise<string> {
log.info({ payload }, "enqueuing"); log.info({ payload }, "enqueuing");
const result = await db.insertInto("events") const result = await db.insertInto("events")
.values({ type: this.opt.type, source, payload }) .values({ group: this.group, type, source, payload })
.returning(["id"]) .returning(["id"])
.executeTakeFirstOrThrow(); .executeTakeFirstOrThrow();
return result.id; return result.id;
@@ -152,7 +167,7 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
// Find and fail any timed out processing events // Find and fail any timed out processing events
const processingEvents = await this.processingEvents(); const processingEvents = await this.processingEvents();
for (const processingEvent of processingEvents) { for (const processingEvent of processingEvents) {
const processingEventTimedOut = (Date.now() - (processingEvent.started_at?.getTime() ?? 0) > this.opt.cfg.concurrency.timeoutMs); const processingEventTimedOut = (Date.now() - (processingEvent.started_at?.getTime() ?? 0) > this.cfg.concurrency.timeoutMs);
if (processingEventTimedOut) { if (processingEventTimedOut) {
log.warn({ name: processingEvent.id }, "processing event has timed out. failing it and continuing..."); log.warn({ name: processingEvent.id }, "processing event has timed out. failing it and continuing...");
await this.fail(processingEvent.id); await this.fail(processingEvent.id);
@@ -167,7 +182,7 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
async front(trx?: Transaction<DB>) { async front(trx?: Transaction<DB>) {
const result = await (trx ?? db).selectFrom("events") const result = await (trx ?? db).selectFrom("events")
.selectAll() .selectAll()
.where("type", "=", this.opt.type) .where("group", "=", this.group)
.where("started_at", "is", null) .where("started_at", "is", null)
.orderBy("created_at", "asc") .orderBy("created_at", "asc")
.limit(1) .limit(1)
@@ -178,17 +193,19 @@ export class EventQueueService<Payload extends JsonValue, State extends JsonValu
async processingEvents(trx?: Transaction<DB>) { async processingEvents(trx?: Transaction<DB>) {
return await (trx ?? db).selectFrom("events") return await (trx ?? db).selectFrom("events")
.select(["started_at", "id"]) .select(["started_at", "id"])
.where("type", "=", this.opt.type) .where("group", "=", this.group)
.where("started_at", "is not", null) .where("started_at", "is not", null)
.where("ended_at", "is", null) .where("ended_at", "is", null)
.execute(); .execute();
} }
private async handleEvent(id: string, payload: JsonValue) { private async handleEvent(id: string, type: string, payload: JsonValue) {
log.info({ id, payload }, "handling event"); log.info({ id, payload }, "handling event");
if (!(type in this.opt.types)) throw new Error(`Unknown event type: ${type}`);
const cfg = this.opt.types[type as keyof M];
try { try {
Value.Assert(this.opt.schemas.payload, payload); Value.Assert(cfg.schemas.payload, payload);
await this.opt.processor(id, payload, async (state) => { await cfg.processor(id, payload, async (state) => {
await db.updateTable("events") await db.updateTable("events")
.set({ state }) .set({ state })
.where("id", "=", id) .where("id", "=", id)
packages/commerce/src/sync.ts
+3 -3
View File
@@ -37,14 +37,14 @@ export class ProductSyncer {
this.log = (log ?? pino({ level: "silent" })).child({ component: "ProductSyncer" }); this.log = (log ?? pino({ level: "silent" })).child({ component: "ProductSyncer" });
} }
async run(opt: ProductSyncerOptions = {}) { async syncApparel(opt: ProductSyncerOptions = {}) {
const startDate = new Date(); const startDate = new Date();
await opt.beforeStart?.(startDate); await opt.beforeStart?.(startDate);
if (opt.filter?.pProductIds) if (opt.filter?.pProductIds)
this.log.info({ pProductIds: opt.filter.pProductIds }, "sync started (filtered)"); this.log.info({ pProductIds: opt.filter.pProductIds }, "apparel sync started (filtered)");
else else
this.log.info("sync started (all)"); this.log.info("apparel sync started (all)");
this.log.debug("retrieving webflow products"); this.log.debug("retrieving webflow products");
const allWProducts = await this.Webflow.Products.list({ forceAll: true }); const allWProducts = await this.Webflow.Products.list({ forceAll: true });