Hook up syncing with postgres
This commit is contained in:
Dominic Ferrando
+151
-187
@@ -2,7 +2,6 @@ import type { Printful, Webflow } from "./util/types";
|
||||
import { formatSlug, type DeepPartial } from "./util/misc";
|
||||
import { WebflowClient } from "./webflow";
|
||||
import { PrintfulClient } from "./printful";
|
||||
import { redis } from "bun";
|
||||
import pino from "pino";
|
||||
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
@@ -18,14 +17,13 @@ type MetaPrintfulProduct = {
|
||||
entries: ProductEntry[];
|
||||
};
|
||||
|
||||
type SyncState = {
|
||||
printfulProductIds: number[];
|
||||
};
|
||||
|
||||
type SyncOptions = {
|
||||
export type SyncOptions = {
|
||||
filter?: {
|
||||
printfulProductIds?: number[] | number;
|
||||
}
|
||||
printfulProductIds?: number[];
|
||||
},
|
||||
onStart?: (startDate: Date) => Promise<void>
|
||||
onCompletion?: (completionDate: Date, duration: number) => Promise<void>
|
||||
onStep?: (printfulProductIds: number[]) => Promise<void>
|
||||
};
|
||||
|
||||
export class ProductSyncer {
|
||||
@@ -39,183 +37,170 @@ export class ProductSyncer {
|
||||
this.log = (log ?? pino({ level: "silent" })).child({ component: "ProductSyncer" });
|
||||
}
|
||||
|
||||
async sync(opt: SyncOptions = {}): Promise<boolean> {
|
||||
if (typeof opt.filter?.printfulProductIds === "number")
|
||||
opt.filter.printfulProductIds = [opt.filter.printfulProductIds];
|
||||
async sync(opt: SyncOptions = {}) {
|
||||
this.log.info("starting sync run");
|
||||
const startDate = new Date();
|
||||
await opt.onStart?.(startDate);
|
||||
|
||||
this.log.info("attempting sync run");
|
||||
const canRun = await this.run();
|
||||
if (!canRun) return false;
|
||||
if (opt.filter?.printfulProductIds)
|
||||
this.log.info({ printfulProductIds: opt.filter.printfulProductIds }, "sync started (filtered)");
|
||||
else
|
||||
this.log.info("sync started (all)");
|
||||
|
||||
try {
|
||||
const syncStart = Date.now();
|
||||
if (opt.filter?.printfulProductIds)
|
||||
this.log.info({ printfulProductIds: opt.filter.printfulProductIds }, "sync started (filtered)");
|
||||
else
|
||||
this.log.info("sync started (all)");
|
||||
this.log.debug("retrieving webflow products");
|
||||
const allWebflowProducts = await this.webflow.Products.list({ forceAll: true });
|
||||
|
||||
this.log.debug("retrieving webflow products");
|
||||
const allWebflowProducts = await this.webflow.Products.list({ forceAll: true });
|
||||
this.log.debug("retrieving printful products");
|
||||
const allPrintfulProducts = await this.printful.Products.list({ forceAll: true });
|
||||
|
||||
this.log.debug("retrieving printful products");
|
||||
const allPrintfulProducts = await this.printful.Products.list({ forceAll: true });
|
||||
this.log.info({ printfulProductCount: allPrintfulProducts.length, filter: opt.filter ?? "N/A" }, "generating meta printful products");
|
||||
const metaPrintfulProducts = await this.generateMetaPrintfulProducts(allWebflowProducts, allPrintfulProducts, opt);
|
||||
|
||||
this.log.info({ printfulProductCount: allPrintfulProducts.length, filter: opt.filter ?? "N/A" }, "generating meta printful products");
|
||||
const metaPrintfulProducts = await this.generateMetaPrintfulProducts(allWebflowProducts, allPrintfulProducts, opt);
|
||||
for (const metaPrintfulProduct of metaPrintfulProducts) {
|
||||
const printfulProductIds = metaPrintfulProduct.entries.map((e) => e.product.sync_product.id);
|
||||
this.log.info({ name: metaPrintfulProduct.name, externalId: metaPrintfulProduct.webflowProductId, printfulProductIds }, "syncing meta printful product");
|
||||
await opt.onStep?.(printfulProductIds);
|
||||
|
||||
for (const metaPrintfulProduct of metaPrintfulProducts) {
|
||||
this.log.info({ name: metaPrintfulProduct.name, externalId: metaPrintfulProduct.webflowProductId }, "syncing meta printful product");
|
||||
const newWebflowSkus: DeepPartial<Webflow.Products.Skus.Sku>[] = this.generateWebflowSkus(metaPrintfulProduct);
|
||||
const foundColors = new Set<string>(
|
||||
newWebflowSkus.map((sku) => sku.fieldData?.["sku-values"]?.["color"]).filter((v) => v !== undefined)
|
||||
);
|
||||
const foundSizes = new Set<string>(
|
||||
newWebflowSkus.map((sku) => sku.fieldData?.["sku-values"]?.["size"]).filter((v) => v !== undefined)
|
||||
);
|
||||
|
||||
await this.setState({
|
||||
printfulProductIds: metaPrintfulProduct.entries.map((e) => e.product.sync_product.id),
|
||||
if (!newWebflowSkus[0]) throw new Error("no webflow SKUs generated — all variants were filtered out");
|
||||
|
||||
const webflowProductFieldData = {
|
||||
name: metaPrintfulProduct.name,
|
||||
slug: formatSlug(metaPrintfulProduct.name),
|
||||
shippable: true,
|
||||
"tax-category": "standard-taxable",
|
||||
"sku-properties": [
|
||||
{
|
||||
id: "color",
|
||||
name: "Color",
|
||||
enum: Array.from(foundColors).map((color) => ({
|
||||
id: color,
|
||||
slug: formatSlug(color),
|
||||
name: color,
|
||||
})),
|
||||
},
|
||||
{
|
||||
id: "size",
|
||||
name: "Size",
|
||||
enum: Array.from(foundSizes).map((size) => ({
|
||||
id: size,
|
||||
slug: formatSlug(size),
|
||||
name: size,
|
||||
})),
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const webflowProduct = allWebflowProducts.find((wp) => wp.product.id === metaPrintfulProduct.webflowProductId);
|
||||
|
||||
if (webflowProduct) {
|
||||
this.log.info({ id: webflowProduct.product.id }, "existing webflow product found, updating it");
|
||||
|
||||
// do not update images
|
||||
for (const sku of newWebflowSkus) delete sku.fieldData?.["main-image"];
|
||||
|
||||
// handle case where webflow SKU created by associated printful variant external_id update failed
|
||||
if (!newWebflowSkus[0].id) {
|
||||
this.log.warn({ webflowSku: newWebflowSkus[0] }, "missing webflow SKU id");
|
||||
const webflowSkuMatch = this.resolveWebflowSku(
|
||||
webflowProduct.skus,
|
||||
newWebflowSkus[0].fieldData?.["sku-values"]?.["color"],
|
||||
newWebflowSkus[0].fieldData?.["sku-values"]?.["size"],
|
||||
);
|
||||
|
||||
if (webflowSkuMatch)
|
||||
newWebflowSkus[0].id = webflowSkuMatch.id;
|
||||
else
|
||||
this.log.warn({ webflowSku: newWebflowSkus[0] }, "could not find existing webflow SKU that matches");
|
||||
}
|
||||
|
||||
await this.webflow.Products.update(webflowProduct.product.id, {
|
||||
product: { fieldData: webflowProductFieldData },
|
||||
sku: newWebflowSkus[0],
|
||||
});
|
||||
|
||||
const newWebflowSkus: DeepPartial<Webflow.Products.Skus.Sku>[] = this.generateWebflowSkus(metaPrintfulProduct);
|
||||
const foundColors = new Set<string>(
|
||||
newWebflowSkus.map((sku) => sku.fieldData?.["sku-values"]?.["color"]).filter((v) => v !== undefined)
|
||||
);
|
||||
const foundSizes = new Set<string>(
|
||||
newWebflowSkus.map((sku) => sku.fieldData?.["sku-values"]?.["size"]).filter((v) => v !== undefined)
|
||||
);
|
||||
|
||||
if (!newWebflowSkus[0]) throw new Error("no webflow SKUs generated — all variants were filtered out");
|
||||
|
||||
const webflowProductFieldData = {
|
||||
name: metaPrintfulProduct.name,
|
||||
slug: formatSlug(metaPrintfulProduct.name),
|
||||
shippable: true,
|
||||
"tax-category": "standard-taxable",
|
||||
"sku-properties": [
|
||||
{
|
||||
id: "color",
|
||||
name: "Color",
|
||||
enum: Array.from(foundColors).map((color) => ({
|
||||
id: color,
|
||||
slug: formatSlug(color),
|
||||
name: color,
|
||||
})),
|
||||
},
|
||||
{
|
||||
id: "size",
|
||||
name: "Size",
|
||||
enum: Array.from(foundSizes).map((size) => ({
|
||||
id: size,
|
||||
slug: formatSlug(size),
|
||||
name: size,
|
||||
})),
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const webflowProduct = allWebflowProducts.find((wp) => wp.product.id === metaPrintfulProduct.webflowProductId);
|
||||
|
||||
if (webflowProduct) {
|
||||
this.log.info({ id: webflowProduct.product.id }, "existing webflow product found, updating it");
|
||||
|
||||
// do not update images
|
||||
for (const sku of newWebflowSkus) delete sku.fieldData?.["main-image"];
|
||||
|
||||
// handle case where webflow SKU created by associated printful variant external_id update failed
|
||||
if (!newWebflowSkus[0].id) {
|
||||
this.log.warn({ webflowSku: newWebflowSkus[0] }, "missing webflow SKU id");
|
||||
const webflowSkuMatch = this.resolveWebflowSku(
|
||||
webflowProduct.skus,
|
||||
newWebflowSkus[0].fieldData?.["sku-values"]?.["color"],
|
||||
newWebflowSkus[0].fieldData?.["sku-values"]?.["size"],
|
||||
for (const newWebflowSku of newWebflowSkus) {
|
||||
const existingWebflowSku = this.resolveWebflowSku(
|
||||
webflowProduct.skus,
|
||||
newWebflowSku.fieldData?.["sku-values"]?.["color"],
|
||||
newWebflowSku.fieldData?.["sku-values"]?.["size"],
|
||||
newWebflowSku.id,
|
||||
);
|
||||
if (existingWebflowSku) {
|
||||
await this.webflow.Products.Skus.update(
|
||||
metaPrintfulProduct.webflowProductId,
|
||||
existingWebflowSku.id,
|
||||
newWebflowSku,
|
||||
);
|
||||
|
||||
if (webflowSkuMatch)
|
||||
newWebflowSkus[0].id = webflowSkuMatch.id;
|
||||
else
|
||||
this.log.warn({ webflowSku: newWebflowSkus[0] }, "could not find existing webflow SKU that matches");
|
||||
}
|
||||
|
||||
await this.webflow.Products.update(webflowProduct.product.id, {
|
||||
product: { fieldData: webflowProductFieldData },
|
||||
sku: newWebflowSkus[0],
|
||||
});
|
||||
|
||||
for (const newWebflowSku of newWebflowSkus) {
|
||||
const existingWebflowSku = this.resolveWebflowSku(
|
||||
webflowProduct.skus,
|
||||
newWebflowSku.fieldData?.["sku-values"]?.["color"],
|
||||
newWebflowSku.fieldData?.["sku-values"]?.["size"],
|
||||
newWebflowSku.id,
|
||||
} else {
|
||||
await this.webflow.Products.Skus.create(
|
||||
metaPrintfulProduct.webflowProductId,
|
||||
[newWebflowSku],
|
||||
);
|
||||
if (existingWebflowSku) {
|
||||
await this.webflow.Products.Skus.update(
|
||||
metaPrintfulProduct.webflowProductId,
|
||||
existingWebflowSku.id,
|
||||
newWebflowSku,
|
||||
);
|
||||
} else {
|
||||
await this.webflow.Products.Skus.create(
|
||||
metaPrintfulProduct.webflowProductId,
|
||||
[newWebflowSku],
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
this.log.info("existing webflow product not found, creating it");
|
||||
|
||||
const webflowProductId = await this.webflow.Products.create({
|
||||
product: { fieldData: webflowProductFieldData },
|
||||
sku: newWebflowSkus[0],
|
||||
});
|
||||
metaPrintfulProduct.webflowProductId = webflowProductId;
|
||||
|
||||
if (newWebflowSkus.length > 1)
|
||||
await this.webflow.Products.Skus.create(webflowProductId, newWebflowSkus.slice(1));
|
||||
}
|
||||
} else {
|
||||
this.log.info("existing webflow product not found, creating it");
|
||||
|
||||
// Re-fetch after SKU mutations so newly created SKUs have their real IDs.
|
||||
const freshWebflowProduct = await this.webflow.Products.get(metaPrintfulProduct.webflowProductId);
|
||||
if (!freshWebflowProduct) throw new Error("webflow product missing");
|
||||
const webflowProductId = await this.webflow.Products.create({
|
||||
product: { fieldData: webflowProductFieldData },
|
||||
sku: newWebflowSkus[0],
|
||||
});
|
||||
metaPrintfulProduct.webflowProductId = webflowProductId;
|
||||
|
||||
// Update printful external IDs
|
||||
for (const { colorGroup, product: printfulProduct } of metaPrintfulProduct.entries) {
|
||||
const expectedExternalId = colorGroup ?
|
||||
`${metaPrintfulProduct.webflowProductId}-${colorGroup}` :
|
||||
metaPrintfulProduct.webflowProductId;
|
||||
if (printfulProduct.sync_product.external_id === expectedExternalId) continue;
|
||||
|
||||
const printfulVariantPatches: DeepPartial<Printful.Products.SyncVariant>[] = [];
|
||||
for (const printfulVariant of printfulProduct.sync_variants) {
|
||||
const webflowSku = this.resolveWebflowSku(
|
||||
freshWebflowProduct.skus,
|
||||
colorGroup ?? printfulVariant.color,
|
||||
printfulVariant.size,
|
||||
);
|
||||
if (webflowSku) {
|
||||
printfulVariantPatches.push({
|
||||
id: printfulVariant.id,
|
||||
external_id: String(webflowSku.id),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(10000);
|
||||
this.log.info({ printfulProduct: printfulProduct.sync_product.id, externalId: expectedExternalId }, "updating printful product's external ID");
|
||||
await this.printful.Products.update(printfulProduct.sync_product.id, {
|
||||
sync_product: {
|
||||
id: printfulProduct.sync_product.id,
|
||||
external_id: expectedExternalId,
|
||||
},
|
||||
sync_variants: printfulVariantPatches,
|
||||
});
|
||||
}
|
||||
if (newWebflowSkus.length > 1)
|
||||
await this.webflow.Products.Skus.create(webflowProductId, newWebflowSkus.slice(1));
|
||||
}
|
||||
|
||||
this.log.info({ durationMs: Date.now() - syncStart }, "sync complete");
|
||||
} finally {
|
||||
try {
|
||||
await redis.del("commerce:sync:lock");
|
||||
await redis.del("commerce:sync:state");
|
||||
} catch {
|
||||
this.log.error("failed to reset sync state");
|
||||
// Re-fetch after SKU mutations so newly created SKUs have their real IDs.
|
||||
const freshWebflowProduct = await this.webflow.Products.get(metaPrintfulProduct.webflowProductId);
|
||||
if (!freshWebflowProduct) throw new Error("webflow product missing");
|
||||
|
||||
// Update printful external IDs
|
||||
for (const { colorGroup, product: printfulProduct } of metaPrintfulProduct.entries) {
|
||||
const expectedExternalId = colorGroup ?
|
||||
`${metaPrintfulProduct.webflowProductId}-${colorGroup}` :
|
||||
metaPrintfulProduct.webflowProductId;
|
||||
if (printfulProduct.sync_product.external_id === expectedExternalId) continue;
|
||||
|
||||
const printfulVariantPatches: DeepPartial<Printful.Products.SyncVariant>[] = [];
|
||||
for (const printfulVariant of printfulProduct.sync_variants) {
|
||||
const webflowSku = this.resolveWebflowSku(
|
||||
freshWebflowProduct.skus,
|
||||
colorGroup ?? printfulVariant.color,
|
||||
printfulVariant.size,
|
||||
);
|
||||
if (webflowSku) {
|
||||
printfulVariantPatches.push({
|
||||
id: printfulVariant.id,
|
||||
external_id: String(webflowSku.id),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(10000);
|
||||
this.log.info({ printfulProduct: printfulProduct.sync_product.id, externalId: expectedExternalId }, "updating printful product's external ID");
|
||||
await this.printful.Products.update(printfulProduct.sync_product.id, {
|
||||
sync_product: {
|
||||
id: printfulProduct.sync_product.id,
|
||||
external_id: expectedExternalId,
|
||||
},
|
||||
sync_variants: printfulVariantPatches,
|
||||
});
|
||||
}
|
||||
}
|
||||
return true;
|
||||
|
||||
const endDate = new Date();
|
||||
const duration = endDate.getTime() - startDate.getTime();
|
||||
await opt.onCompletion?.(endDate, duration);
|
||||
this.log.info({ durationMs: duration }, "sync complete");
|
||||
}
|
||||
|
||||
parseColorGroup(name: string): string | null {
|
||||
@@ -273,9 +258,6 @@ export class ProductSyncer {
|
||||
allPrintfulProducts: Printful.Products.SyncProduct[],
|
||||
opt: SyncOptions,
|
||||
) {
|
||||
if (typeof opt.filter?.printfulProductIds === "number")
|
||||
opt.filter.printfulProductIds = [opt.filter.printfulProductIds];
|
||||
|
||||
const metaNameFilter = opt.filter?.printfulProductIds
|
||||
?.map((id) => allPrintfulProducts.find((p) => p.id === id))
|
||||
?.filter((p) => p !== undefined)
|
||||
@@ -341,22 +323,4 @@ export class ProductSyncer {
|
||||
private isPrintfulProductSynced(printfulProduct: Printful.Products.SyncProduct, allWebflowProducts: Webflow.Products.ProductAndSkus[]) {
|
||||
return allWebflowProducts.some((wp) => wp.product.id === printfulProduct.external_id.split("-")[0]);
|
||||
}
|
||||
|
||||
async isRunning(): Promise<boolean> {
|
||||
return !!await redis.get("commerce:sync:lock");
|
||||
}
|
||||
|
||||
async getState(): Promise<SyncState> {
|
||||
const val = await redis.get("commerce:sync:state");
|
||||
if (val) return JSON.parse(val);
|
||||
return { printfulProductIds: [] };
|
||||
}
|
||||
|
||||
private async setState(state: SyncState) {
|
||||
await redis.set("commerce:sync:state", JSON.stringify(state), "EX", 1800);
|
||||
}
|
||||
|
||||
private async run(): Promise<boolean> {
|
||||
return !!await redis.set("commerce:sync:lock", "1", "NX", "EX", "1800");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user