Improve getAll webflow/printful client api

packages/commerce/src/printful.ts

@@ -1,5 +1,5 @@
 import type { Printful } from "./util/types";
-import { type DeepPartial } from "./util/misc";
+import { type DeepPartial, type PagingOptions } from "./util/misc";
 
 const parsePayload = (res: Response): Promise<unknown> =>
     res.json().catch(() => res.text().catch(() => undefined));
@@ -35,7 +35,7 @@ class VariantsClient {
         });
         if (res.status === 404 || res.status === 400) return;
         if (!res.ok) throw new PrintfulError("Failed to get Printful variant", res.status, await parsePayload(res));
-        const payload = (await res.json()) as Printful.Products.MetaDataSingle<Printful.Products.SyncVariant>;
+        const payload = (await res.json()) as Printful.MetaDataSingle<Printful.Products.SyncVariant>;
         return payload.result;
     }
 }
@@ -47,19 +47,20 @@ class ProductsClient {
         this.Variants = new VariantsClient(creds);
     }
 
-    async getAll(offset = 0): Promise<Printful.Products.SyncProduct[]> {
+    async getAll(opt: PagingOptions): Promise<Printful.Products.SyncProduct[]> {
         const allSyncProducts: Printful.Products.SyncProduct[] = [];
-        while (true) {
+        let offset = opt.offset ?? 0;
-            const res = await fetch(`${this.creds.apiUrl}/store/products?offset=${offset}`, {
+        do {
+            const res = await fetch(`${this.creds.apiUrl}/store/products?offset=${offset}&limit=${opt.limit ?? 100}`, {
                 method: "GET",
                 headers: this.creds.authHeaders,
             });
             if (!res.ok) throw new PrintfulError("Failed to get all Printful products", res.status, await parsePayload(res));
-            const payload = (await res.json()) as Printful.Products.MetaDataMulti<Printful.Products.SyncProduct>;
+            const payload = (await res.json()) as Printful.MetaDataMulti<Printful.Products.SyncProduct>;
             allSyncProducts.push(...payload.result);
             offset = allSyncProducts.length;
             if (allSyncProducts.length >= payload.paging.total) break;
-        }
+        } while (opt.forceAll);
         return allSyncProducts;
     }
 
@@ -70,7 +71,7 @@ class ProductsClient {
         });
         if (res.status === 404 || res.status === 400) return;
         if (!res.ok) throw new PrintfulError("Failed to get Printful product", res.status, await parsePayload(res));
-        const payload = (await res.json()) as Printful.Products.MetaDataSingle<Printful.Products.Product>;
+        const payload = (await res.json()) as Printful.MetaDataSingle<Printful.Products.Product>;
         return payload.result;
     }
 
@@ -102,19 +103,20 @@ class OrdersClient {
         if (!res.ok) throw new PrintfulError("Failed to create Printful order", res.status, await parsePayload(res));
     }
 
-    async getAll(offset = 0): Promise<Printful.Orders.Order[]> {
+    async getAll(opt: PagingOptions): Promise<Printful.Orders.Order[]> {
         const allPrintfulOrders: Printful.Orders.Order[] = [];
-        while (true) {
+        let offset = opt.offset ?? 0;
-            const res = await fetch(`${this.creds.apiUrl}/orders?offset=${offset}`, {
+        do {
+            const res = await fetch(`${this.creds.apiUrl}/orders?offset=${offset}&limit=${opt.limit ?? 100}`, {
                 method: "GET",
                 headers: this.creds.authHeaders,
             });
             if (!res.ok) throw new PrintfulError("Failed to get all Printful orders", res.status, await parsePayload(res));
-            const payload = (await res.json()) as Printful.Products.MetaDataMulti<Printful.Orders.Order>;
+            const payload = (await res.json()) as Printful.MetaDataMulti<Printful.Orders.Order>;
             allPrintfulOrders.push(...payload.result);
             offset = allPrintfulOrders.length;
             if (allPrintfulOrders.length >= payload.paging.total) break;
-        }
+        } while (opt.forceAll);
         return allPrintfulOrders;
     }
 }

packages/commerce/src/sync.ts

@@ -42,10 +42,10 @@ export class ProductSyncer {
             await this.setState({ isSyncing: true, syncingIds: [] });
 
             this.log.debug("populating webflow products");
-            const webflowProducts: Webflow.Products.ProductAndSkus[] = await this.webflow.Products.getAll();
+            const webflowProducts: Webflow.Products.ProductAndSkus[] = await this.webflow.Products.getAll({ forceAll: true });
 
             this.log.debug("populating printful products");
-            let printfulProducts = await this.printful.Products.getAll();
+            let printfulProducts = await this.printful.Products.getAll({ forceAll: true });
             if (printfulProductId) {
                 const printfulProduct = printfulProducts.find((p) => p.id === printfulProductId);
                 if (printfulProduct) {

packages/commerce/src/util/misc.ts

@@ -13,3 +13,9 @@ export const formatSlug = (name: string): string => {
         .replace(/--+/g, "-") // collapse multiple dashes
         .replace(/^([^a-z0-9_])/, "_$1"); // if it starts with an invalid char, prefix underscore
 };
+
+export type PagingOptions = {
+    offset?: number,
+    limit?: number,
+    forceAll?: boolean
+};

packages/commerce/src/util/types.ts

@@ -1,5 +1,4 @@
 export namespace Printful {
-    export namespace Products {
     export interface MetaDataSingle<T = any> {
         code: number;
         result: T;
@@ -15,6 +14,7 @@ export namespace Printful {
         };
     }
 
+    export namespace Products {
         type Option = {
             id: string;
             value: string;
@@ -85,6 +85,16 @@ export namespace Printful {
     }
 
     export namespace Orders {
+        export interface MetaDataMulti<T = any> {
+            code: number;
+            result: T[];
+            paging: {
+                total: number;
+                offset: number;
+                limit: number;
+            };
+        }
+
         export type Order = {
             external_id: string;
             shipping: string;
@@ -179,6 +189,16 @@ export namespace Printful {
 }
 
 export namespace Webflow {
+    export type MetaDataMulti<K extends string, T = any> = {
+        [P in K]: T[];
+    } & {
+        pagination: {
+            limit: number;
+            offset: number;
+            total: number;
+        };
+    };
+
     export namespace Products {
         export namespace Skus {
             export type Sku = {

packages/commerce/src/webflow.ts

@@ -1,5 +1,5 @@
 import type { Webflow } from "./util/types";
-import { type DeepPartial } from "./util/misc";
+import { type DeepPartial, type PagingOptions } from "./util/misc";
 import crypto from "node:crypto";
 
 const parsePayload = (res: Response): Promise<unknown> =>
@@ -95,14 +95,21 @@ class ProductsClient {
         return payload.product.id;
     }
 
-    async getAll(): Promise<Webflow.Products.ProductAndSkus[]> {
+    async getAll(opt: PagingOptions): Promise<Webflow.Products.ProductAndSkus[]> {
-        const res = await fetch(`${this.creds.apiSitesUrl}/products`, {
+        const allWebflowProducts: Webflow.Products.ProductAndSkus[] = [];
+        let offset = opt.offset ?? 0;
+        do {
+            const res = await fetch(`${this.creds.apiSitesUrl}/products?offset=${offset}&limit=${opt.limit ?? 100}`, {
                 method: "GET",
                 headers: this.creds.authHeader,
             });
             if (!res.ok) throw new WebflowError("Failed to get all Webflow products", res.status, await parsePayload(res));
-        const payload = (await res.json()) as { items: Webflow.Products.ProductAndSkus[] };
+            const payload = (await res.json()) as Webflow.MetaDataMulti<"items", Webflow.Products.ProductAndSkus>;
-        return payload.items;
+            allWebflowProducts.push(...payload.items);
+            offset = allWebflowProducts.length;
+            if (allWebflowProducts.length >= payload.pagination.total) break;
+        } while (opt.forceAll);
+        return allWebflowProducts;
     }
 
     async get(webflowProductId: string): Promise<Webflow.Products.ProductAndSkus | undefined> {
@@ -149,17 +156,25 @@ class ProductsClient {
 class OrdersClient {
     constructor(private creds: Credentials) { }
 
-    async getAll(opt: { status?: Webflow.Orders.Order["status"] } = {}): Promise<Webflow.Orders.Order[]> {
+    async getAll(opt: PagingOptions & { status?: Webflow.Orders.Order["status"] }): Promise<Webflow.Orders.Order[]> {
-        // TODO: pagination
+        const allOrders: Webflow.Orders.Order[] = [];
+        let offset = opt.offset ?? 0;
+        do {
             const params = new URLSearchParams();
             if (opt.status !== undefined) params.set("status", opt.status);
+            params.set("offset", String(offset));
+            params.set("limit", String(opt.limit ?? 100));
             const res = await fetch(`${this.creds.apiSitesUrl}/orders?${params.toString()}`, {
                 method: "GET",
                 headers: this.creds.authHeader,
             });
             if (!res.ok) throw new WebflowError("Failed to get all Webflow orders", res.status, await parsePayload(res));
-        const payload = (await res.json()) as { orders: Webflow.Orders.Order[] };
+            const payload = (await res.json()) as Webflow.MetaDataMulti<"orders", Webflow.Orders.Order>;
-        return payload.orders;
+            allOrders.push(...payload.orders);
+            offset = allOrders.length;
+            if (allOrders.length >= payload.pagination.total) break;
+        } while (opt.forceAll);
+        return allOrders;
     }
 
     async update(