Hook up syncing with postgres

apps/api/package.json

@@ -11,9 +11,7 @@
     "scripts": {
         "dev": "bun run --hot src/index.ts",
         "start": "bun run src/index.ts",
-        "build": "bun build --compile --minify-whitespace --minify-syntax --target bun-linux-x64 --outfile dist/server src/index.ts",
+        "build": "bun build --compile --minify-whitespace --minify-syntax --target bun-linux-x64 --outfile dist/server src/index.ts"
-        "db:codegen": "bunx kysely-codegen --env-file .env.development",
-        "db:seed": "bun run src/database/seed.ts"
     },
     "dependencies": {
         "@blade-and-brawn/calculator": "workspace:*",
@@ -25,13 +23,13 @@
         "@types/pg": "^8.20.0",
         "elysia": "^1.4.29",
         "kysely": "^0.29.2",
-        "kysely-codegen": "^0.20.0",
         "ml-levenberg-marquardt": "^5.0.1",
         "pg": "^8.22.0",
         "pino": "^10.3.1",
         "zipcodes-us": "^1.1.3"
     },
     "devDependencies": {
-        "pino-pretty": "^13.1.3"
+        "pino-pretty": "^13.1.3",
+        "kysely-codegen": "^0.20.0"
     }
 }

apps/api/src/database/seed.ts

@@ -6,12 +6,26 @@ import { log } from "../util";
 // SYNC RUNS
 // =====================
 {
-    log.info({ name: "sync_runs" }, "Creating table");
+    log.info({ name: "product_syncs" }, "Creating table");
-    await db.schema.dropTable("sync_runs").ifExists().execute()
+    await db.schema.dropTable("product_syncs").ifExists().execute()
-    await db.schema.createTable("sync_runs")
+    await db.schema.createTable("product_syncs")
         .$call(addDefaultColumns)
-        .addColumn("completed_at", "timestamptz")
+        .addColumn("printful_product_id_filter", sql`integer[]`)
+        .addColumn("syncing_printful_product_ids", sql`integer[]`, (cb) => cb.notNull())
+        .addColumn("started_at", "timestamptz", (cb) => cb.unique())
+        .addColumn("ended_at", "timestamptz")
+        .addColumn("has_failed", "boolean", (cb) => cb.notNull().defaultTo(false))
         .execute()
+
+    await db.schema.dropIndex("idx_one_active_product_sync").ifExists().execute();
+    await db.schema.createIndex("idx_one_active_product_sync")
+        .on("product_syncs")
+        .unique()
+        .column("ended_at")
+        .where(sql.ref("started_at"), "is not", null)
+        .where("ended_at", "is", null)
+        .nullsNotDistinct()
+        .execute();
 }
 
 function addDefaultColumns(ctb: CreateTableBuilder<any, any>) {
@@ -23,6 +37,5 @@ function addDefaultColumns(ctb: CreateTableBuilder<any, any>) {
         .addColumn("created_at", "timestamptz", (cb) => cb
             .notNull()
             .defaultTo(sql`now()`)
-            .unique()
         )
 };

apps/api/src/server.ts

@@ -13,7 +13,6 @@ import { Elysia, NotFoundError, status, t } from "elysia";
 import {
     PrintfulClient,
     PrintfulError,
-    ProductSyncer,
     WebflowClient,
     WebflowError,
     Printful,
@@ -23,6 +22,7 @@ import zipcodesUs from "zipcodes-us";
 import { env, log } from "./util";
 import serverTiming from "@elysia/server-timing";
 import jwt from "@elysia/jwt";
+import { SyncService } from "./services/sync";
 
 // SERVICES
 // -----------
@@ -42,7 +42,7 @@ const webflow = new WebflowClient({
     webhookSecret: env.WEBFLOW_WEBHOOK_SECRET,
 });
 
-const productSyncer = new ProductSyncer(printful, webflow, log);
+const syncService = new SyncService(printful, webflow);
 
 // ELYSIA
 // -----------
@@ -143,15 +143,22 @@ export const app = new Elysia()
             .group("/sync", (app) =>
                 app
                     // Sync status
-                    .get("/", async ({ }) => ({
+                    .get("/", async ({ }) => {
-                        state: await productSyncer.getState(),
+                        const activeRun = await syncService.getActiveRun();
-                        isRunning: await productSyncer.isRunning()
+                        if (!activeRun) throw new NotFoundError("No sync in progress");
-                    }))
+                        return {
+                            startDate: activeRun.started_at,
+                            syncingPrintfulProductIds: activeRun.syncing_printful_product_ids
+                        }
+                    })
                     // Run sync
                     .post("/:printfulProductId?", async ({ params: { printfulProductId } }) => {
                         log.debug({ printfulProductId });
-                        const canSync = await productSyncer.sync({ filter: { printfulProductIds: printfulProductId } });
+                        await syncService.start({
-                        if (!canSync) throw status(409, "Sync already in progress");
+                            printfulProductIds: printfulProductId ?
+                                [printfulProductId] :
+                                undefined
+                        });
                     }, { params: t.Object({ printfulProductId: t.Optional(t.Numeric()) }) }),
             )
             .get("/:printfulProductId", async ({ params }) => {
@@ -178,7 +185,7 @@ export const app = new Elysia()
             case Printful.Webhook.Event.ProductUpdated: {
                 const printfulProduct = payload.data.sync_product;
                 log.info({ productId: printfulProduct.id }, "printful webhook: product updated");
-                await productSyncer.sync({ filter: { printfulProductIds: printfulProduct.id } });
+                await syncService.start({ printfulProductIds: [printfulProduct.id] });
                 break;
             }
             case Printful.Webhook.Event.ProductDeleted: {

apps/api/src/services/sync.ts

@@ -0,0 +1,58 @@
+import { PrintfulClient, ProductSyncer, WebflowClient, type SyncOptions } from "@blade-and-brawn/commerce";
+import { log } from "../util";
+import { db } from "../database/db";
+
+export class SyncService {
+    readonly productSyncer: ProductSyncer
+
+    constructor(printful: PrintfulClient, webflow: WebflowClient) {
+        this.productSyncer = new ProductSyncer(printful, webflow, log);
+    }
+
+    async start(filter: SyncOptions["filter"]) {
+        let id: string | null = null;
+        try {
+            await this.productSyncer.sync({
+                filter,
+                onStart: async (startDate) => {
+                    const result = await db.insertInto("product_syncs")
+                        .values({
+                            started_at: startDate,
+                            printful_product_id_filter: filter?.printfulProductIds,
+                            syncing_printful_product_ids: []
+                        })
+                        .returning("id")
+                        .executeTakeFirstOrThrow();
+                    id = result.id;
+                },
+                onStep: async (printfulProductIds: number[]) => {
+                    if (!id) throw new Error("Expected sync run ID");
+                    await db.updateTable("product_syncs")
+                        .set({ syncing_printful_product_ids: printfulProductIds })
+                        .where("id", "=", id)
+                        .execute();
+                },
+                onCompletion: async (completionDate, duration) => {
+                    if (!id) throw new Error("Expected sync run ID");
+                    await db.updateTable("product_syncs")
+                        .set({ syncing_printful_product_ids: [], ended_at: completionDate, })
+                        .where("id", "=", id)
+                        .execute();
+                }
+            });
+        }
+        catch (err) {
+            if (id) {
+                await db.updateTable("product_syncs")
+                    .set({ syncing_printful_product_ids: [], ended_at: new Date(), has_failed: true })
+                    .where("id", "=", id)
+                    .execute();
+            }
+            throw err;
+        }
+    }
+
+    async getActiveRun() {
+        return await db.selectFrom("product_syncs").selectAll().where("ended_at", "is", null).executeTakeFirst();
+    }
+}

apps/portal/package.json

@@ -26,7 +26,7 @@
         "@blade-and-brawn/domain": "workspace:*",
         "@elysia/eden": "^1.4.10",
         "@tailwindcss/vite": "^4.3.2",
-        "daisyui": "^5.6.6",
+        "daisyui": "^5.6.7",
         "jose": "^6.2.3",
         "tailwindcss": "^4.3.2"
     }

apps/portal/src/routes/(app)/commerce/+page.svelte

@@ -7,13 +7,16 @@
 
     const synchronizer = $state({
         isRunning: false,
-        printfulProductIds: [] as number[],
+        syncingPrintfulProductIds: [] as number[],
         poll: async function () {
             const res = await api.products.sync.get();
-            if (res.data) {
+            if (res.error) {
-                synchronizer.isRunning = res.data.isRunning;
+                synchronizer.isRunning = false;
-                synchronizer.printfulProductIds =
+                synchronizer.syncingPrintfulProductIds = [];
-                    res.data.state.printfulProductIds;
+            } else {
+                synchronizer.isRunning = true;
+                synchronizer.syncingPrintfulProductIds =
+                    res.data.syncingPrintfulProductIds;
             }
         },
         startPolling: () => {
@@ -93,7 +96,7 @@
                         >
                         <td>
                             {#await isProductSynced(printfulProduct) then isSynced}
-                                {#if synchronizer.printfulProductIds.includes(printfulProduct.id)}
+                                {#if synchronizer.syncingPrintfulProductIds.includes(printfulProduct.id)}
                                     <div class="badge badge-warning">
                                         Syncing...
                                     </div>

bun.lock

@@ -21,13 +21,13 @@
         "@types/pg": "^8.20.0",
         "elysia": "^1.4.29",
         "kysely": "^0.29.2",
-        "kysely-codegen": "^0.20.0",
         "ml-levenberg-marquardt": "^5.0.1",
         "pg": "^8.22.0",
         "pino": "^10.3.1",
         "zipcodes-us": "^1.1.3",
       },
       "devDependencies": {
+        "kysely-codegen": "^0.20.0",
         "pino-pretty": "^13.1.3",
       },
     },
@@ -358,7 +358,7 @@
 
     "cosmiconfig": ["cosmiconfig@9.0.2", "", { "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", "js-yaml": "^4.1.0", "parse-json": "^5.2.0" }, "peerDependencies": { "typescript": ">=4.9.5" }, "optionalPeers": ["typescript"] }, "sha512-gtTZxTDau1wL7Y7zifc2dd8jHSK/k6BTx/2Xp/BpdlAdnlYWFVt7qhJqgwi7637yRwRQ3qL4ZidbB4I8tA5VOg=="],
 
-    "daisyui": ["daisyui@5.6.6", "", {}, "sha512-UFiuIZYucF7ylrnY3PT1SqwaEbVB10mqTZRyeRchZk0bud+HihYLLhXdxGDVmR/ftrM9G9YVr9FFPRVVIZJ5hA=="],
+    "daisyui": ["daisyui@5.6.7", "", {}, "sha512-FVpuUbesyAUBWNp5pXn9JDF1EWt3IZJzp5Staw1YqfP0MwL36sv3wtmfWeuEBkZcazaQbeZXA91oW5dSB+TUEg=="],
 
     "dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="],
 

package.json

@@ -9,7 +9,9 @@
         "dev:api": "bun run --filter @blade-and-brawn/api dev",
         "build:api": "bun run --filter @blade-and-brawn/api build",
         "dev:portal": "bun run --filter @blade-and-brawn/portal dev",
-        "build:portal": "bun run --filter @blade-and-brawn/portal build"
+        "build:portal": "bun run --filter @blade-and-brawn/portal build",
+        "db:codegen": "bunx kysely-codegen --env-file .env.development",
+        "db:seed": "bun run apps/api/src/database/seed.ts"
     },
     "devDependencies": {
         "@types/bun": "^1.3.14",

packages/commerce/src/sync.ts

@@ -2,7 +2,6 @@ import type { Printful, Webflow } from "./util/types";
 import { formatSlug, type DeepPartial } from "./util/misc";
 import { WebflowClient } from "./webflow";
 import { PrintfulClient } from "./printful";
-import { redis } from "bun";
 import pino from "pino";
 
 const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -18,14 +17,13 @@ type MetaPrintfulProduct = {
     entries: ProductEntry[];
 };
 
-type SyncState = {
+export type SyncOptions = {
-    printfulProductIds: number[];
-};
-
-type SyncOptions = {
     filter?: {
-        printfulProductIds?: number[] | number;
+        printfulProductIds?: number[];
-    }
+    },
+    onStart?: (startDate: Date) => Promise<void>
+    onCompletion?: (completionDate: Date, duration: number) => Promise<void>
+    onStep?: (printfulProductIds: number[]) => Promise<void>
 };
 
 export class ProductSyncer {
@@ -39,16 +37,11 @@ export class ProductSyncer {
         this.log = (log ?? pino({ level: "silent" })).child({ component: "ProductSyncer" });
     }
 
-    async sync(opt: SyncOptions = {}): Promise<boolean> {
+    async sync(opt: SyncOptions = {}) {
-        if (typeof opt.filter?.printfulProductIds === "number")
+        this.log.info("starting sync run");
-            opt.filter.printfulProductIds = [opt.filter.printfulProductIds];
+        const startDate = new Date();
+        await opt.onStart?.(startDate);
 
-        this.log.info("attempting sync run");
-        const canRun = await this.run();
-        if (!canRun) return false;
-
-        try {
-            const syncStart = Date.now();
         if (opt.filter?.printfulProductIds)
             this.log.info({ printfulProductIds: opt.filter.printfulProductIds }, "sync started (filtered)");
         else
@@ -64,11 +57,9 @@ export class ProductSyncer {
         const metaPrintfulProducts = await this.generateMetaPrintfulProducts(allWebflowProducts, allPrintfulProducts, opt);
 
         for (const metaPrintfulProduct of metaPrintfulProducts) {
-                this.log.info({ name: metaPrintfulProduct.name, externalId: metaPrintfulProduct.webflowProductId }, "syncing meta printful product");
+            const printfulProductIds = metaPrintfulProduct.entries.map((e) => e.product.sync_product.id);
-
+            this.log.info({ name: metaPrintfulProduct.name, externalId: metaPrintfulProduct.webflowProductId, printfulProductIds }, "syncing meta printful product");
-                await this.setState({
+            await opt.onStep?.(printfulProductIds);
-                    printfulProductIds: metaPrintfulProduct.entries.map((e) => e.product.sync_product.id),
-                });
 
             const newWebflowSkus: DeepPartial<Webflow.Products.Skus.Sku>[] = this.generateWebflowSkus(metaPrintfulProduct);
             const foundColors = new Set<string>(
@@ -206,16 +197,10 @@ export class ProductSyncer {
             }
         }
 
-            this.log.info({ durationMs: Date.now() - syncStart }, "sync complete");
+        const endDate = new Date();
-        } finally {
+        const duration = endDate.getTime() - startDate.getTime();
-            try {
+        await opt.onCompletion?.(endDate, duration);
-                await redis.del("commerce:sync:lock");
+        this.log.info({ durationMs: duration }, "sync complete");
-                await redis.del("commerce:sync:state");
-            } catch {
-                this.log.error("failed to reset sync state");
-            }
-        }
-        return true;
     }
 
     parseColorGroup(name: string): string | null {
@@ -273,9 +258,6 @@ export class ProductSyncer {
         allPrintfulProducts: Printful.Products.SyncProduct[],
         opt: SyncOptions,
     ) {
-        if (typeof opt.filter?.printfulProductIds === "number")
-            opt.filter.printfulProductIds = [opt.filter.printfulProductIds];
-
         const metaNameFilter = opt.filter?.printfulProductIds
             ?.map((id) => allPrintfulProducts.find((p) => p.id === id))
             ?.filter((p) => p !== undefined)
@@ -341,22 +323,4 @@ export class ProductSyncer {
     private isPrintfulProductSynced(printfulProduct: Printful.Products.SyncProduct, allWebflowProducts: Webflow.Products.ProductAndSkus[]) {
         return allWebflowProducts.some((wp) => wp.product.id === printfulProduct.external_id.split("-")[0]);
     }
-
-    async isRunning(): Promise<boolean> {
-        return !!await redis.get("commerce:sync:lock");
-    }
-
-    async getState(): Promise<SyncState> {
-        const val = await redis.get("commerce:sync:state");
-        if (val) return JSON.parse(val);
-        return { printfulProductIds: [] };
-    }
-
-    private async setState(state: SyncState) {
-        await redis.set("commerce:sync:state", JSON.stringify(state), "EX", 1800);
-    }
-
-    private async run(): Promise<boolean> {
-        return !!await redis.set("commerce:sync:lock", "1", "NX", "EX", "1800");
-    }
 }