Software/platform
1
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Move queue management onto all workers and make clean single atomic operation

Browse Source
This commit is contained in:
Dominic Ferrando
2026-07-17 10:24:37 -04:00
parent 903a60fec5
commit 5829021658
2 changed files with 36 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/server.ts
+19 -19
View File
@@ -24,6 +24,7 @@ import { EventsService, EventStatusSchema } from "./services/events";
// CONSTANTS
// -----------------------
const EVENT_QUEUE_MANAGE_DELAY_MS = 1000 // 1 sec
const JWT_EXP = "7d";
// SERVICES
// -----------------------
@@ -124,7 +125,7 @@ export const app = new Elysia()
if (body.password !== env.ADMIN_PASSWORD) throw status(401, "Invalid credentials");
auth.set({
value: await jwt.sign({ sessionId: randomUUIDv7(), exp: "7d" }),
value: await jwt.sign({ sessionId: randomUUIDv7(), exp: JWT_EXP }),
path: "/",
maxAge: 60 * 60 * 24 * 7,
sameSite: "lax",
@@ -345,27 +346,26 @@ export const app = new Elysia()
});
app.listen(3000, async () => {
if (cluster.worker?.id === 1) {
if (cluster.worker?.id === 1)
log.info({ port: 3000 }, "server started")
// MANAGE QUEUES
for (const queue of queues) {
(async () => {
while (true) {
try {
// clean, if ready
if ((Date.now() - queue.lastCleanDate.getTime()) >= EventsService.CONCURRENCY_TIMEOUT_MS)
await queue.clean().catch((err) => log.error({ name: queue.group, err }));
// drain
await queue.drain();
}
catch (err) {
log.error({ name: queue.group, err }, "error occurred during queue management");
}
await sleep(EVENT_QUEUE_MANAGE_DELAY_MS);
// MANAGE QUEUES
for (const queue of queues) {
(async () => {
while (true) {
try {
// clean, if ready
if ((Date.now() - queue.lastCleanDate.getTime()) >= EventsService.CONCURRENCY_TIMEOUT_MS)
await queue.clean().catch((err) => log.error({ name: queue.group, err }));
// drain
await queue.drain();
}
})();
}
catch (err) {
log.error({ name: queue.group, err }, "error occurred during queue management");
}
await sleep(EVENT_QUEUE_MANAGE_DELAY_MS);
}
})();
}
});
apps/api/src/services/events.ts
+17 -8
View File
@@ -138,14 +138,23 @@ export class EventsService {
async clean(opt: { filter?: { type?: string, group?: string } } = {}) {
try {
// Find and fail any timed out processing events
const processingEvents = await this.list({ filter: { status: "processing", ...opt.filter } });
for (const processingEvent of processingEvents) {
const processingEventTimedOut = (Date.now() - (processingEvent.started_at?.getTime() ?? 0) > EventsService.CONCURRENCY_TIMEOUT_MS);
if (processingEventTimedOut) {
log.warn({ name: processingEvent.id }, "processing event has timed out. failing it and continuing...");
await this.fail(processingEvent.id);
}
const timedOutEvents = await db.updateTable("events")
.set((eb) => ({
state: null,
ended_at: new Date(),
has_failed: true,
errors: eb("errors", "+", 1),
}))
.where("started_at", "is not", null)
.where("ended_at", "is", null)
.where("started_at", "<", new Date(Date.now() - EventsService.CONCURRENCY_TIMEOUT_MS))
.$if(opt.filter?.type !== undefined, (qb) => qb.where("type", "=", opt.filter!.type!))
.$if(opt.filter?.group !== undefined, (qb) => qb.where("group", "=", opt.filter!.group!))
.returning(["id"])
.execute();
for (const event of timedOutEvents) {
log.warn({ name: event.id }, "processing event timed out, failed it");
}
}
finally {