diff --git a/apps/api/src/database/migrations/2026-06-03.ts b/apps/api/src/database/migrations/2026-06-03.ts index af5a188..ae40c02 100644 --- a/apps/api/src/database/migrations/2026-06-03.ts +++ b/apps/api/src/database/migrations/2026-06-03.ts @@ -5,6 +5,7 @@ export async function up(db: Kysely): Promise { // TABLE: EVENTS await db.schema.createTable("events") .$call(addDefaultColumns) + .addColumn("group", "text", (cb) => cb.notNull()) .addColumn("type", "text", (cb) => cb.notNull()) .addColumn("source", "text", (cb) => cb.notNull()) .addColumn("payload", "jsonb", (cb) => cb.notNull()) diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index e8d7065..5f298c6 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -20,7 +20,7 @@ import { randomUUIDv7, sleep } from "bun"; import { CalculatorService, CalculatorUnavailableError } from "./services/calculator"; import { StandardsParamsSchema } from "@blade-and-brawn/calculator"; import { StandardsService } from "./services/standards"; -import type { EventQueueService } from "./services/event-queue"; +import type { EventQueue, EventSpec } from "./services/event-queue"; // CONSTANTS // ----------------------- @@ -201,6 +201,7 @@ export const app = new Elysia() // Run sync .post("/:pProductId?", async ({ params: { pProductId }, sessionId }) => { await s.Commerce.ProductSync.Queue.enqueue({ + type: "product_sync_update", source: "portal", payload: { session: { id: sessionId, name: "Portal" }, @@ -237,6 +238,7 @@ export const app = new Elysia() log.info({ productId: pProduct.id }, "printful webhook: product updated"); await s.Commerce.ProductSync.Queue.enqueue({ + type: "product_sync_update", source: "printful", payload: { session: { id: randomUUIDv7(), name: "Printful" }, @@ -317,19 +319,19 @@ app.listen(3000, async () => { log.info({ port: 3000 }, "server started") // MANAGE QUEUES - const queues: EventQueueService[] = [s.Commerce.ProductSync.Queue]; + const queues: EventQueue>>[] = [s.Commerce.ProductSync.Queue]; for (const queue of queues) { (async () => { while (true) { try { // clean, if ready if ((Date.now() - queue.lastCleanDate.getTime()) >= queue.cfg.concurrency.timeoutMs) - await queue.clean().catch((err) => log.error({ name: queue.type, err })); + await queue.clean().catch((err) => log.error({ name: queue.group, err })); // drain await queue.drain(); } catch (err) { - log.error({ name: queue.type, err }, "error occurred during queue management"); + log.error({ name: queue.group, err }, "error occurred during queue management"); } await sleep(EVENT_QUEUE_MANAGE_DELAY_MS); } diff --git a/apps/api/src/services/commerce.ts b/apps/api/src/services/commerce.ts index a194a86..1de787e 100644 --- a/apps/api/src/services/commerce.ts +++ b/apps/api/src/services/commerce.ts @@ -1,13 +1,13 @@ import { PrintfulClient, ProductSyncer, WebflowClient } from "@blade-and-brawn/commerce"; import { env, log } from "../util"; import { db } from "../database/db"; -import { EventQueueService, type EventType } from "./event-queue"; +import { EventQueue, type EventSpec } from "./event-queue"; import { Type as t } from "@sinclair/typebox"; import type { Static } from "@sinclair/typebox"; import { Value } from "@sinclair/typebox/value"; import { sql } from "kysely"; -const ProductSyncPayloadSchema = t.Object({ +const ProductSyncUpdatePayloadSchema = t.Object({ session: t.Object({ name: t.String(), id: t.String() @@ -18,22 +18,27 @@ const ProductSyncPayloadSchema = t.Object({ }) ) }); -type ProductSyncPayload = Static; +type ProductSyncUpdatePayload = Static; -const ProductSyncStateSchema = t.Union([ +const ProductSyncUpdateStateSchema = t.Union([ t.Object({ pProductIds: t.Optional(t.Array(t.Number())) }), t.Null() ]); -type ProductSyncState = Static; +type ProductSyncUpdateState = Static; + +export type ProductSyncEvents = { + product_sync_update: EventSpec + // TODO: product_sync_delete, once its schema/processor are written +}; export class CommerceService { - public readonly Printful = new PrintfulClient({ + readonly Printful = new PrintfulClient({ token: env.PRINTFUL_AUTH, storeId: env.PRINTFUL_STORE_ID, }); - public readonly Webflow = new WebflowClient({ + readonly Webflow = new WebflowClient({ siteId: env.WEBFLOW_SITE_ID, collectionsId: env.WEBFLOW_COLLECTION_ID, token: env.WEBFLOW_AUTH, @@ -43,29 +48,32 @@ export class CommerceService { } class ProductSyncService { - static readonly EVENT_TYPE: EventType = "product_sync"; - - readonly Queue: EventQueueService; + static readonly EVENT_GROUP = "product_sync"; + readonly Queue: EventQueue<"product_sync", ProductSyncEvents>; private readonly ProductSyncer: ProductSyncer; // TODO: make each service have their own logger, so we can enforce .child({component: }) constructor(private readonly Printful: PrintfulClient, private readonly Webflow: WebflowClient) { this.ProductSyncer = new ProductSyncer(this.Printful, this.Webflow, log); - this.Queue = new EventQueueService({ - type: ProductSyncService.EVENT_TYPE, - schemas: { payload: ProductSyncPayloadSchema, state: ProductSyncStateSchema }, + this.Queue = new EventQueue<"product_sync", ProductSyncEvents>({ + group: ProductSyncService.EVENT_GROUP, cfg: { retry: { limit: 3, delayMs: 10000 }, concurrency: { limit: 1, timeoutMs: 600000 } }, - processor: async (id, payload, setState) => { - await this.ProductSyncer.run({ - filter: payload.filter, - beforeStep: async (pProductIds: number[]) => { - await setState({ pProductIds }); + types: { + "product_sync_update": { + schemas: { payload: ProductSyncUpdatePayloadSchema, state: ProductSyncUpdateStateSchema }, + processor: async (id, payload, setState) => { + await this.ProductSyncer.syncApparel({ + filter: payload.filter, + beforeStep: async (pProductIds: number[]) => { + await setState({ pProductIds }); + } + }); } - }); + } } }); } @@ -73,7 +81,7 @@ class ProductSyncService { async getLatestSyncState(sessionId: string) { const latestSync = await db.selectFrom("events") .select(["started_at", "ended_at", "has_failed", "state"]) - .where("type", "=", ProductSyncService.EVENT_TYPE) + .where("group", "=", ProductSyncService.EVENT_GROUP) .where(sql`payload->'session'->>'id'`, "=", sessionId) .orderBy(sql`(started_at is not null and ended_at is null)`, "desc") .orderBy("created_at", "desc") @@ -81,11 +89,11 @@ class ProductSyncService { .executeTakeFirst(); if (!latestSync) return; - Value.Assert(ProductSyncStateSchema, latestSync.state); + Value.Assert(ProductSyncUpdateStateSchema, latestSync.state); return { startDate: latestSync.started_at, - status: EventQueueService.status(latestSync), + status: EventQueue.status(latestSync), syncingPProductIds: latestSync.state?.pProductIds ?? [] }; } diff --git a/apps/api/src/services/event-queue.ts b/apps/api/src/services/event-queue.ts index bffcf6c..382cc1b 100644 --- a/apps/api/src/services/event-queue.ts +++ b/apps/api/src/services/event-queue.ts @@ -6,22 +6,26 @@ import { db } from "../database/db"; import { Value } from "@sinclair/typebox/value"; import type { TSchema } from "@sinclair/typebox"; -export type EventType = "product_sync" | "product_order_created" /* TODO: more events... */; +export type EventSpec

= { + payload: P, + state: S +} export type EventSource = "portal" | "printful" | "webflow"; -type EventProcessor = - (id: string, payload: Payload, setState: (state: State) => Promise) => Promise; +type EventProcessor = + (id: string, payload: S["payload"], setState: (state: S["state"]) => Promise) => Promise; type EventStatus = "pending" | "processing" | "failed" | "fulfilled"; -class AlreadyClaimedEventError extends Error { } -class ConcurrencyLimitReachedError extends Error { } - -type EventQueueServiceOptions = { - type: EventType, +type EventTypeOptions = { schemas: { - payload: TSchema & { static: Payload }, - state: TSchema & { static: State }, + payload: TSchema & { static: S["payload"] }, + state: TSchema & { static: S["state"] }, }, + processor: EventProcessor, +}; + +type EventQueueOptions> = { + group: G, cfg: { retry: { limit: number, @@ -32,17 +36,28 @@ type EventQueueServiceOptions, + types: { + [T in keyof M]: EventTypeOptions + }, } -export class EventQueueService { - private _lastCleanDate: Date = new Date(0); - readonly cfg: EventQueueServiceOptions["cfg"]; - readonly type: EventType +type EnqueueOptions, T extends keyof M & string> = { + source: EventSource, + type: T, + payload: M[T]["payload"] +} - constructor(private readonly opt: EventQueueServiceOptions) { +class AlreadyClaimedEventError extends Error { } +class ConcurrencyLimitReachedError extends Error { } + +export class EventQueue> { + private _lastCleanDate: Date = new Date(0); + readonly cfg: EventQueueOptions["cfg"]; + readonly group: G; + + constructor(private readonly opt: EventQueueOptions) { this.cfg = opt.cfg; - this.type = opt.type; + this.group = opt.group; } get lastCleanDate() { return this._lastCleanDate; }; @@ -68,13 +83,13 @@ export class EventQueueService { + async enqueue({ source, type, payload }: EnqueueOptions): Promise { log.info({ payload }, "enqueuing"); const result = await db.insertInto("events") - .values({ type: this.opt.type, source, payload }) + .values({ group: this.group, type, source, payload }) .returning(["id"]) .executeTakeFirstOrThrow(); return result.id; @@ -152,7 +167,7 @@ export class EventQueueService this.opt.cfg.concurrency.timeoutMs); + const processingEventTimedOut = (Date.now() - (processingEvent.started_at?.getTime() ?? 0) > this.cfg.concurrency.timeoutMs); if (processingEventTimedOut) { log.warn({ name: processingEvent.id }, "processing event has timed out. failing it and continuing..."); await this.fail(processingEvent.id); @@ -167,7 +182,7 @@ export class EventQueueService) { const result = await (trx ?? db).selectFrom("events") .selectAll() - .where("type", "=", this.opt.type) + .where("group", "=", this.group) .where("started_at", "is", null) .orderBy("created_at", "asc") .limit(1) @@ -178,17 +193,19 @@ export class EventQueueService) { return await (trx ?? db).selectFrom("events") .select(["started_at", "id"]) - .where("type", "=", this.opt.type) + .where("group", "=", this.group) .where("started_at", "is not", null) .where("ended_at", "is", null) .execute(); } - private async handleEvent(id: string, payload: JsonValue) { + private async handleEvent(id: string, type: string, payload: JsonValue) { log.info({ id, payload }, "handling event"); + if (!(type in this.opt.types)) throw new Error(`Unknown event type: ${type}`); + const cfg = this.opt.types[type as keyof M]; try { - Value.Assert(this.opt.schemas.payload, payload); - await this.opt.processor(id, payload, async (state) => { + Value.Assert(cfg.schemas.payload, payload); + await cfg.processor(id, payload, async (state) => { await db.updateTable("events") .set({ state }) .where("id", "=", id) diff --git a/packages/commerce/src/sync.ts b/packages/commerce/src/sync.ts index 17def48..91660ab 100644 --- a/packages/commerce/src/sync.ts +++ b/packages/commerce/src/sync.ts @@ -37,14 +37,14 @@ export class ProductSyncer { this.log = (log ?? pino({ level: "silent" })).child({ component: "ProductSyncer" }); } - async run(opt: ProductSyncerOptions = {}) { + async syncApparel(opt: ProductSyncerOptions = {}) { const startDate = new Date(); await opt.beforeStart?.(startDate); if (opt.filter?.pProductIds) - this.log.info({ pProductIds: opt.filter.pProductIds }, "sync started (filtered)"); + this.log.info({ pProductIds: opt.filter.pProductIds }, "apparel sync started (filtered)"); else - this.log.info("sync started (all)"); + this.log.info("apparel sync started (all)"); this.log.debug("retrieving webflow products"); const allWProducts = await this.Webflow.Products.list({ forceAll: true });