Software/platform
1
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Move queue management onto all workers and make clean single atomic operation

Browse Source
This commit is contained in:
Dominic Ferrando
2026-07-17 10:24:37 -04:00
parent 903a60fec5
commit 5829021658
2 changed files with 36 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/server.ts
+19 -19
View File
@@ -24,6 +24,7 @@ import { EventsService, EventStatusSchema } from "./services/events";
// CONSTANTS // CONSTANTS
// ----------------------- // -----------------------
const EVENT_QUEUE_MANAGE_DELAY_MS = 1000 // 1 sec const EVENT_QUEUE_MANAGE_DELAY_MS = 1000 // 1 sec
const JWT_EXP = "7d";
// SERVICES // SERVICES
// ----------------------- // -----------------------
@@ -124,7 +125,7 @@ export const app = new Elysia()
if (body.password !== env.ADMIN_PASSWORD) throw status(401, "Invalid credentials"); if (body.password !== env.ADMIN_PASSWORD) throw status(401, "Invalid credentials");
auth.set({ auth.set({
value: await jwt.sign({ sessionId: randomUUIDv7(), exp: "7d" }), value: await jwt.sign({ sessionId: randomUUIDv7(), exp: JWT_EXP }),
path: "/", path: "/",
maxAge: 60 * 60 * 24 * 7, maxAge: 60 * 60 * 24 * 7,
sameSite: "lax", sameSite: "lax",
@@ -345,27 +346,26 @@ export const app = new Elysia()
}); });
app.listen(3000, async () => { app.listen(3000, async () => {
if (cluster.worker?.id === 1) { if (cluster.worker?.id === 1)
log.info({ port: 3000 }, "server started") log.info({ port: 3000 }, "server started")
// MANAGE QUEUES // MANAGE QUEUES
for (const queue of queues) { for (const queue of queues) {
(async () => { (async () => {
while (true) { while (true) {
try { try {
// clean, if ready // clean, if ready
if ((Date.now() - queue.lastCleanDate.getTime()) >= EventsService.CONCURRENCY_TIMEOUT_MS) if ((Date.now() - queue.lastCleanDate.getTime()) >= EventsService.CONCURRENCY_TIMEOUT_MS)
await queue.clean().catch((err) => log.error({ name: queue.group, err })); await queue.clean().catch((err) => log.error({ name: queue.group, err }));
// drain // drain
await queue.drain(); await queue.drain();
}
catch (err) {
log.error({ name: queue.group, err }, "error occurred during queue management");
}
await sleep(EVENT_QUEUE_MANAGE_DELAY_MS);
} }
})(); catch (err) {
} log.error({ name: queue.group, err }, "error occurred during queue management");
}
await sleep(EVENT_QUEUE_MANAGE_DELAY_MS);
}
})();
} }
}); });
apps/api/src/services/events.ts
+17 -8
View File
@@ -138,14 +138,23 @@ export class EventsService {
async clean(opt: { filter?: { type?: string, group?: string } } = {}) { async clean(opt: { filter?: { type?: string, group?: string } } = {}) {
try { try {
// Find and fail any timed out processing events const timedOutEvents = await db.updateTable("events")
const processingEvents = await this.list({ filter: { status: "processing", ...opt.filter } }); .set((eb) => ({
for (const processingEvent of processingEvents) { state: null,
const processingEventTimedOut = (Date.now() - (processingEvent.started_at?.getTime() ?? 0) > EventsService.CONCURRENCY_TIMEOUT_MS); ended_at: new Date(),
if (processingEventTimedOut) { has_failed: true,
log.warn({ name: processingEvent.id }, "processing event has timed out. failing it and continuing..."); errors: eb("errors", "+", 1),
await this.fail(processingEvent.id); }))
} .where("started_at", "is not", null)
.where("ended_at", "is", null)
.where("started_at", "<", new Date(Date.now() - EventsService.CONCURRENCY_TIMEOUT_MS))
.$if(opt.filter?.type !== undefined, (qb) => qb.where("type", "=", opt.filter!.type!))
.$if(opt.filter?.group !== undefined, (qb) => qb.where("group", "=", opt.filter!.group!))
.returning(["id"])
.execute();
for (const event of timedOutEvents) {
log.warn({ name: event.id }, "processing event timed out, failed it");
} }
} }
finally { finally {