diff --git a/apps/api/src/database/migrations/2026-06-03.ts b/apps/api/src/database/migrations/2026-06-03.ts index 9b7db97..3c7b131 100644 --- a/apps/api/src/database/migrations/2026-06-03.ts +++ b/apps/api/src/database/migrations/2026-06-03.ts @@ -13,7 +13,7 @@ export async function up(db: Kysely): Promise { .addColumn("started_at", "timestamptz") .addColumn("ended_at", "timestamptz") .addColumn("available_at", "timestamptz", (cb) => cb.notNull().defaultTo(sql`now()`)) - .addColumn("tries", "integer", (cb) => cb.notNull().defaultTo(0)) + .addColumn("errors", "integer", (cb) => cb.notNull().defaultTo(0)) .addColumn("rate_limits", "integer", (cb) => cb.notNull().defaultTo(0)) .addColumn("has_failed", "boolean", (cb) => cb.notNull().defaultTo(false)) .execute() diff --git a/apps/api/src/services/commerce.ts b/apps/api/src/services/commerce.ts index 5aa59a5..f3b1a6c 100644 --- a/apps/api/src/services/commerce.ts +++ b/apps/api/src/services/commerce.ts @@ -6,6 +6,7 @@ import { Type as t } from "@sinclair/typebox"; import { Value } from "@sinclair/typebox/value"; import { sql } from "kysely"; import zipcodesUs from "zipcodes-us"; +import { minToMs, secToMs } from "@blade-and-brawn/domain"; export class CommerceService { readonly Printful = new PrintfulClient({ @@ -37,8 +38,8 @@ class ApparelOrdersService { this.Queue = new EventQueue({ group: "apparel_order", cfg: { - retry: { limit: 3, delayMs: 10000 }, - concurrency: { limit: 1, timeoutMs: 600000 } + error: { limit: 3, initialRetryDelayMs: secToMs(1) }, + concurrency: { limit: 1, timeoutMs: minToMs(10) } }, events: { "apparel_order_create": defineEventType({ @@ -105,8 +106,8 @@ class ApparelSyncService { this.Queue = new EventQueue({ group: "apparel_sync", cfg: { - retry: { limit: 3, delayMs: 10000 }, - concurrency: { limit: 1, timeoutMs: 600000 } + error: { limit: 3, initialRetryDelayMs: secToMs(1) }, + concurrency: { limit: 1, timeoutMs: minToMs(10) } }, events: { "apparel_sync_update": defineEventType({ diff --git a/apps/api/src/services/event-queue.ts b/apps/api/src/services/event-queue.ts index 83c0d7e..3e5a132 100644 --- a/apps/api/src/services/event-queue.ts +++ b/apps/api/src/services/event-queue.ts @@ -5,11 +5,13 @@ import { sleep } from "bun"; import { db } from "../database/db"; import { AssertError, Value } from "@sinclair/typebox/value"; import type { TSchema, Static } from "@sinclair/typebox"; -import { RateLimitError } from "@blade-and-brawn/domain"; +import { minToMs, RateLimitError, secToMs } from "@blade-and-brawn/domain"; export type EventSource = "portal" | "printful" | "webflow"; export type EventStatus = "waiting" | "available" | "processing" | "failed" | "fulfilled"; +type RetryType = "error" | "ratelimit"; + type EventProcessor

= (payload: Static

, setState: (state: Static) => Promise, id: string) => Promise; @@ -19,9 +21,9 @@ export type EventTypeOptions

= { }; type EventQueueCfg = { - retry: { + error: { limit: number, - delayMs: number, + initialRetryDelayMs: number, }, concurrency: { limit: number, @@ -36,10 +38,20 @@ export function defineEventType

(opt: Event return opt; } +function addJitter(delay: number, jitterFactor: number): number { + const jitterMultiplier = 1 + (Math.random() - 0.5) * jitterFactor; + return delay * jitterMultiplier; +} + +function exponentialDelay(initialDelay: number, tries: number, cap: number = Infinity) { + return Math.min(initialDelay * (2 ** tries), cap); +} + export class EventQueue>> { - private static readonly DEQUEUE_RETRY_DELAY_MS = 10000; // 10 seconds + private static readonly DEQUEUE_RETRY_DELAY_MS = secToMs(10); private static readonly DEQUEUE_RETRY_LIMIT = 3; private static readonly RATE_LIMIT_RETRY_LIMIT = 15; + private static readonly JITTER_FACTOR = 0.2; private _lastCleanDate: Date = new Date(0); readonly cfg: EventQueueCfg; @@ -181,7 +193,7 @@ export class EventQueue) { const result = await (trx ?? db).selectFrom("events") - .select(["id", "type", "payload", "available_at", "tries", "rate_limits"]) + .select(["id", "type", "payload", "available_at", "errors", "rate_limits"]) .where("group", "=", this.group) .where("started_at", "is", null) .where("available_at", "<=", new Date()) @@ -222,7 +234,7 @@ export class EventQueue, "id" | "type" | "tries" | "payload" | "rate_limits">) { + private async processEvent(event: Pick, "id" | "type" | "errors" | "payload" | "rate_limits">) { log.info({ id: event.id, type: event.type, payload: event.payload }, "processing event"); if (!(event.type in this.events)) throw new Error(`event type not registered in this queue (${this.group}): ${event.type}`); const cfg = this.events[event.type as keyof T]!; @@ -235,26 +247,30 @@ export class EventQueue ({ - tries: eb("tries", "+", 1), - rate_limits: opt.isRateLimit ? eb("rate_limits", "+", 1) : eb.ref("rate_limits"), + errors: type === "error" ? eb("errors", "+", 1) : eb.ref("errors"), + rate_limits: type === "ratelimit" ? eb("rate_limits", "+", 1) : eb.ref("rate_limits"), started_at: null, ended_at: null, has_failed: false,