-
+
+
+
({
label: monthDateFormatter.format(date),
@@ -153,47 +153,93 @@ export default function Page() {
))
}
- {promoCredits && (
-
-
-
-
Promo credits
-
- {formatCurrency(promoCredits.remainingCents / 100, false)}
-
-
-
-
-
0
- ? Math.min(
- 100,
- Math.max(
- 0,
- (promoCredits.remainingCents / promoCredits.grantedCents) * 100
- )
+
+
+
+
+ {promoCredits && (
+
+
+
+
Promo credits
+
+ {formatCurrency(promoCredits.remainingCents / 100, false)}
+
+
+
+
+
0
+ ? Math.min(
+ 100,
+ Math.max(
+ 0,
+ (promoCredits.remainingCents / promoCredits.grantedCents) * 100
)
- : 0
- }%`,
- }}
- />
-
-
- {formatCurrency(promoCredits.remainingCents / 100, false)} of{" "}
- {formatCurrency(promoCredits.grantedCents / 100, false)} remaining
- {promoCredits.expiresAt
- ? ` · expires ${creditExpiryFormatter.format(new Date(promoCredits.expiresAt))}`
- : ""}
-
+ )
+ : 0
+ }%`,
+ }}
+ />
+
+ {formatCurrency(promoCredits.remainingCents / 100, false)} of{" "}
+ {formatCurrency(promoCredits.grantedCents / 100, false)} remaining
+ {promoCredits.expiresAt
+ ? ` · expires ${creditExpiryFormatter.format(new Date(promoCredits.expiresAt))}`
+ : ""}
+
- )}
-
-
+ )}
+
+
}>
+
+ Failed to load graph.
+
+ }
+ >
+ {(usage) => (
+
+
+
+ {isCurrentMonth ? "Month-to-date" : "Usage"}
+
+
+ {formatCurrency(usage.overall.current, false)}
+
+
+
+
+ )}
+
+
+
+
+
+
+ Usage by day
+
+
+
+ }
+ >
}
>
- {(usage) => (
-
-
-
- {isCurrentMonth ? "Month-to-date" : "Usage"}
-
-
- {formatCurrency(usage.overall.current, false)}
-
-
-
-
-
-
-
-
- }
- >
-
- Failed to load graph.
-
- }
- >
- {(u) =>
}
-
-
-
-
-
-
-
- Tasks
-
-
}>
-
- Failed to load.
-
-
+
+
+
+
+ Tasks
+
+ Failed to load.
+
+ }
+ >
+ {(tasks) => {
+ return (
+ <>
+
+
+
+ Task
+ Runs
+ Average duration
+ Average cost
+ Total duration
+ Total cost
+
+
+
+ {tasks.length === 0 ? (
- Task
- Runs
- Average duration
- Average cost
- Total duration
- Total cost
+
+
+
No runs for this period
+
-
-
- {tasks.length === 0 ? (
-
-
-
-
- No runs for this period
-
-
+ {task.taskIdentifier}
+
+ {formatNumber(task.runCount)}
+
+
+ {formatDurationMilliseconds(task.averageDuration, {
+ style: "short",
+ })}
+
+
+ {formatCurrencyAccurate(task.averageCost)}
+
+
+ {formatDurationMilliseconds(task.totalDuration, {
+ style: "short",
+ })}
+
+
+ {formatCurrencyAccurate(task.totalCost)}
- ) : (
- tasks.map((task) => (
-
- {task.taskIdentifier}
-
- {formatNumber(task.runCount)}
-
-
- {formatDurationMilliseconds(task.averageDuration, {
- style: "short",
- })}
-
-
- {formatCurrencyAccurate(task.averageCost)}
-
-
- {formatDurationMilliseconds(task.totalDuration, {
- style: "short",
- })}
-
-
- {formatCurrencyAccurate(task.totalCost)}
-
-
- ))
- )}
-
-
-
- Dev environment runs are excluded from the usage data above, since they do
- not have an associated compute cost.
-
-
- );
- }}
-
-
-
-
-
+ ))
+ )}
+
+
+
+ Dev environment runs are excluded from the usage data above, since they do not
+ have an associated compute cost.
+
+
+ );
+ }}
+
+
+
+
);
}
diff --git a/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts b/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts
index 908e8f449a0..9717aa2b942 100644
--- a/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts
+++ b/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts
@@ -6,6 +6,7 @@ import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
import { engine } from "~/v3/runEngine.server";
import { updateEnvConcurrencyLimits } from "~/v3/runQueue.server";
+import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server";
const ParamsSchema = z.object({
environmentId: z.string(),
@@ -46,6 +47,9 @@ export async function action({ request, params }: ActionFunctionArgs) {
await updateEnvConcurrencyLimits(environment);
+ // Percent-based queue overrides follow the environment limit automatically.
+ await concurrencySystem.queues.recalculatePercentLimits(environment);
+
// Org max-concurrency changed too, which is embedded in every env of the org; invalidating
// the org drops the env/authEnv rows for all of them (including this env).
controlPlaneResolver.invalidateOrganization(environment.organizationId);
diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts
index c99637a0d10..403df19fe37 100644
--- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts
+++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts
@@ -5,6 +5,7 @@ import { prisma } from "~/db.server";
import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
import { updateEnvConcurrencyLimits } from "~/v3/runQueue.server";
+import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server";
const ParamsSchema = z.object({
organizationId: z.string(),
@@ -82,6 +83,12 @@ export async function action({ request, params }: ActionFunctionArgs) {
});
await updateEnvConcurrencyLimits({ ...modifiedEnvironment, organization });
+
+ // Percent-based queue overrides follow the environment limit automatically.
+ await concurrencySystem.queues.recalculatePercentLimits({
+ ...modifiedEnvironment,
+ organization,
+ });
}
// Org + every affected env's concurrency changed; one org invalidation covers them all.
diff --git a/apps/webapp/app/routes/admin.api.v1.queue-metrics.ts b/apps/webapp/app/routes/admin.api.v1.queue-metrics.ts
new file mode 100644
index 00000000000..69e4e8c1fac
--- /dev/null
+++ b/apps/webapp/app/routes/admin.api.v1.queue-metrics.ts
@@ -0,0 +1,45 @@
+import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
+import {
+ probeQueueMetricsStreams,
+ readQueueMetricsControls,
+ writeQueueMetricsControls,
+} from "~/v3/queueMetrics.server";
+
+export async function loader({ request }: LoaderFunctionArgs) {
+ await requireAdminApiRequest(request);
+ const [controls, streams] = await Promise.all([
+ readQueueMetricsControls(),
+ probeQueueMetricsStreams(),
+ ]);
+ return json({ controls, streams });
+}
+
+const BodySchema = z.object({
+ enabled: z.boolean().optional(),
+ sampleRate: z.number().min(0).max(1).optional(),
+});
+
+export async function action({ request }: ActionFunctionArgs) {
+ await requireAdminApiRequest(request);
+
+ if (request.method !== "POST") {
+ return json({ error: "Method not allowed" }, { status: 405 });
+ }
+
+ let body: unknown;
+ try {
+ body = await request.json();
+ } catch {
+ return json({ error: "Invalid JSON body" }, { status: 400 });
+ }
+
+ const parsed = BodySchema.safeParse(body);
+ if (!parsed.success) {
+ return json({ error: "Invalid payload", details: parsed.error.issues }, { status: 400 });
+ }
+
+ await writeQueueMetricsControls(parsed.data);
+ return json({ ok: true, controls: await readQueueMetricsControls() });
+}
diff --git a/apps/webapp/app/routes/admin.queue-metrics.tsx b/apps/webapp/app/routes/admin.queue-metrics.tsx
new file mode 100644
index 00000000000..6deaedce66e
--- /dev/null
+++ b/apps/webapp/app/routes/admin.queue-metrics.tsx
@@ -0,0 +1,190 @@
+import { useFetcher, useRevalidator } from "@remix-run/react";
+import { json } from "@remix-run/server-runtime";
+import { useEffect, useState } from "react";
+import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { z } from "zod";
+import { Button } from "~/components/primitives/Buttons";
+import { Callout } from "~/components/primitives/Callout";
+import { Header1, Header2 } from "~/components/primitives/Headers";
+import { Input } from "~/components/primitives/Input";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import {
+ Table,
+ TableBody,
+ TableCell,
+ TableHeader,
+ TableHeaderCell,
+ TableRow,
+} from "~/components/primitives/Table";
+import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder";
+import {
+ probeQueueMetricsStreams,
+ readQueueMetricsControls,
+ writeQueueMetricsControls,
+} from "~/v3/queueMetrics.server";
+
+export const loader = dashboardLoader({ authorization: { requireSuper: true } }, async () => {
+ const [controls, streams] = await Promise.all([
+ readQueueMetricsControls(),
+ probeQueueMetricsStreams(),
+ ]);
+ return typedjson({ controls, streams });
+});
+
+const BodySchema = z.object({
+ enabled: z.boolean().optional(),
+ sampleRate: z.number().min(0).max(1).optional(),
+});
+
+export const action = dashboardAction(
+ { authorization: { requireSuper: true } },
+ async ({ request }) => {
+ let body: unknown;
+ try {
+ body = await request.json();
+ } catch {
+ return json({ error: "Invalid JSON body" }, { status: 400 });
+ }
+ const parsed = BodySchema.safeParse(body);
+ if (!parsed.success) {
+ return json({ error: "Invalid payload" }, { status: 400 });
+ }
+ await writeQueueMetricsControls(parsed.data);
+ return json({ success: true });
+ }
+);
+
+export default function AdminQueueMetricsRoute() {
+ const { controls, streams } = useTypedLoaderData
();
+ const saveFetcher = useFetcher<{ success?: boolean; error?: string }>();
+ const revalidator = useRevalidator();
+
+ const [enabled, setEnabled] = useState(controls.enabled);
+ const [sampleRate, setSampleRate] = useState(String(controls.sampleRate));
+ const [error, setError] = useState(null);
+
+ useEffect(() => {
+ setEnabled(controls.enabled);
+ setSampleRate(String(controls.sampleRate));
+ }, [controls.enabled, controls.sampleRate]);
+
+ useEffect(() => {
+ if (saveFetcher.data?.success) {
+ setError(null);
+ revalidator.revalidate();
+ } else if (saveFetcher.data?.error) {
+ setError(saveFetcher.data.error);
+ }
+ }, [saveFetcher.data]);
+
+ const isSaving = saveFetcher.state === "submitting";
+
+ const handleSave = () => {
+ const rate = Number(sampleRate);
+ if (!Number.isFinite(rate) || rate < 0 || rate > 1) {
+ setError("Sample rate must be a number between 0 and 1");
+ return;
+ }
+ saveFetcher.submit(JSON.stringify({ enabled, sampleRate: rate }), {
+ method: "POST",
+ encType: "application/json",
+ });
+ };
+
+ const totalLag = streams.reduce((sum, s) => sum + (s.lag ?? 0), 0);
+ const lagUnknownCount = streams.filter((s) => s.lag === null).length;
+
+ return (
+
+
+
Queue metrics ingest
+
+ Live controls for the queue-metrics ingest pipeline on the run-queue Redis. Changes take
+ effect within ~10s across all instances (no redeploy). Watch EngineCPU on the run-queue
+ Redis when enabling or raising the sample rate.
+
+
+
+
Controls
+
+ (queue_metrics:enabled)
+
+
+
+ Gauge sample rate 0–1 (queue_metrics:gauge_sample_rate); default{" "}
+ {controls.sampleRateDefault}
+
+
+ {error &&
{error} }
+
+
+ {isSaving ? "Saving..." : "Save controls"}
+
+
+
+
+
+
+ Stream health{totalLag > 0 ? ` (lag ${totalLag})` : ""}
+ revalidator.revalidate()}
+ disabled={revalidator.state === "loading"}
+ >
+ Refresh
+
+
+
+ Depth = entries buffered in the shard stream; Lag = entries not yet delivered to the
+ consumer group (rising = consumer falling behind; "unknown" = entries were trimmed past
+ the group, i.e. data was lost); Pending = unacked entries. Gauges and counters share one
+ stream family on the metrics Redis.
+
+ {lagUnknownCount > 0 && (
+
+ Lag is unknown on {lagUnknownCount} shard{lagUnknownCount === 1 ? "" : "s"}: entries
+ were trimmed past the consumer group's read position, so stream data was lost. Check
+ consumer health.
+
+ )}
+
+
+
+ Stream
+ Shard
+ Depth
+ Lag
+ Pending
+
+
+
+ {streams.map((s) => (
+
+ {s.stream}
+ {s.shard}
+ {s.depth}
+ {s.lag ?? "unknown"}
+ {s.pending}
+
+ ))}
+
+
+
+
+ );
+}
diff --git a/apps/webapp/app/routes/admin.tsx b/apps/webapp/app/routes/admin.tsx
index 34ba6c62ca1..2cd7cbfd1ec 100644
--- a/apps/webapp/app/routes/admin.tsx
+++ b/apps/webapp/app/routes/admin.tsx
@@ -34,6 +34,10 @@ export default function Page() {
label: "Global Feature Flags",
to: "/admin/feature-flags",
},
+ {
+ label: "Queue Metrics",
+ to: "/admin/queue-metrics",
+ },
{
label: "Notifications",
to: "/admin/notifications",
diff --git a/apps/webapp/app/routes/api.v1.query.schema.ts b/apps/webapp/app/routes/api.v1.query.schema.ts
index 3e95d16818d..976fa72b267 100644
--- a/apps/webapp/app/routes/api.v1.query.schema.ts
+++ b/apps/webapp/app/routes/api.v1.query.schema.ts
@@ -1,7 +1,7 @@
import { json } from "@remix-run/server-runtime";
import type { ColumnSchema, TableSchema } from "@internal/tsql";
import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-import { querySchemas } from "~/v3/querySchemas";
+import { visibleQuerySchemas } from "~/v3/querySchemas";
function serializeColumn(col: ColumnSchema) {
const result: Record = {
@@ -51,7 +51,7 @@ export const loader = createLoaderApiRoute(
},
},
async () => {
- const tables = querySchemas.map(serializeTable);
+ const tables = visibleQuerySchemas.map(serializeTable);
return json({ tables });
}
);
diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts
index 3223a2a6062..acc723d61cb 100644
--- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts
+++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts
@@ -4,11 +4,23 @@ import { z } from "zod";
import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server";
import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server";
+import {
+ MAX_QUEUE_OVERRIDE_PERCENT,
+ MIN_QUEUE_OVERRIDE_PERCENT,
+} from "~/v3/services/concurrencySystem.server";
-const BodySchema = z.object({
- type: RetrieveQueueType.default("id"),
- concurrencyLimit: z.number().int().min(0).max(100000),
-});
+const BodySchema = z
+ .object({
+ type: RetrieveQueueType.default("id"),
+ // Absolute concurrency limit. Backwards compatible with existing callers.
+ concurrencyLimit: z.number().int().min(0).max(100000).optional(),
+ // Percentage of the environment's maximum concurrency limit (0 < percent <= 100).
+ // Stored as the source of truth; the absolute limit is materialized from it.
+ percent: z.number().gt(MIN_QUEUE_OVERRIDE_PERCENT).max(MAX_QUEUE_OVERRIDE_PERCENT).optional(),
+ })
+ .refine((body) => (body.concurrencyLimit === undefined) !== (body.percent === undefined), {
+ message: "Provide exactly one of `concurrencyLimit` or `percent`",
+ });
export const { action } = createActionApiRoute(
{
@@ -26,8 +38,11 @@ export const { action } = createActionApiRoute(
name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"),
};
+ const override =
+ body.percent !== undefined ? { percent: body.percent } : { limit: body.concurrencyLimit! };
+
return concurrencySystem.queues
- .overrideQueueConcurrencyLimit(authentication.environment, input, body.concurrencyLimit)
+ .overrideQueueConcurrencyLimit(authentication.environment, input, override)
.match(
(queue) => {
return json(
@@ -51,6 +66,10 @@ export const { action } = createActionApiRoute(
case "queue_not_found": {
return json({ error: "Queue not found" }, { status: 404 });
}
+ case "invalid_override":
+ case "concurrency_limit_exceeds_maximum": {
+ return json({ error: error.message }, { status: 400 });
+ }
case "queue_update_failed": {
return json({ error: "Failed to update queue concurrency limit" }, { status: 500 });
}
diff --git a/apps/webapp/app/routes/resources.metric.tsx b/apps/webapp/app/routes/resources.metric.tsx
index d456ba1ce1b..5bf0ed693ad 100644
--- a/apps/webapp/app/routes/resources.metric.tsx
+++ b/apps/webapp/app/routes/resources.metric.tsx
@@ -50,6 +50,8 @@ const MetricWidgetQuery = z.object({
operations: z.array(z.string()).optional(),
providers: z.array(z.string()).optional(),
tags: z.array(z.string()).optional(),
+ // Opt into server-side gap fill (carry-forward for gauges, zero-fill for counters).
+ fillGaps: z.boolean().optional(),
});
export const action = async ({ request }: ActionFunctionArgs) => {
@@ -85,6 +87,7 @@ export const action = async ({ request }: ActionFunctionArgs) => {
operations,
providers,
tags: _tags,
+ fillGaps,
} = submission.data;
// Check they should be able to access it
@@ -122,6 +125,7 @@ export const action = async ({ request }: ActionFunctionArgs) => {
promptVersions,
operations,
providers,
+ fillGaps,
// Set higher concurrency if many widgets are on screen at once
customOrgConcurrencyLimit: env.METRIC_WIDGET_DEFAULT_ORG_CONCURRENCY_LIMIT,
});
diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-generate.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-generate.tsx
index c1626b966d2..4a9ab462dcf 100644
--- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-generate.tsx
+++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-generate.tsx
@@ -8,7 +8,7 @@ import type { AITimeFilter } from "~/routes/_app.orgs.$organizationSlug.projects
import { requireUserId } from "~/services/session.server";
import { EnvironmentParamSchema } from "~/utils/pathBuilder";
import { AIQueryService } from "~/v3/services/aiQueryService.server";
-import { querySchemas } from "~/v3/querySchemas";
+import { visibleQuerySchemas } from "~/v3/querySchemas";
const RequestSchema = z.object({
prompt: z.string().min(1, "Prompt is required"),
@@ -85,7 +85,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
const { prompt, mode, currentQuery } = submission.data;
const service = new AIQueryService(
- querySchemas,
+ visibleQuerySchemas,
openai(env.AI_RUN_FILTER_MODEL ?? "gpt-4o-mini")
);
diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx
index 4c8f77a582a..047a6ab4342 100644
--- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx
+++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx
@@ -82,6 +82,12 @@ import { useProject } from "~/hooks/useProject";
import { useSearchParams } from "~/hooks/useSearchParam";
import { useHasAdminAccess } from "~/hooks/useUser";
import { redirectWithErrorMessage } from "~/models/message.server";
+import { formatWaitMs, QueueSparklineStat } from "~/components/queues/QueueMetricCards";
+import {
+ resolveRunQueueMetrics,
+ type RunQueueMetrics,
+ type RunQueueWaiting,
+} from "~/presenters/v3/RunQueueMetricsPresenter.server";
import { type Span, SpanPresenter, type SpanRun } from "~/presenters/v3/SpanPresenter.server";
import { logger } from "~/services/logger.server";
import { requireUserId } from "~/services/session.server";
@@ -91,6 +97,7 @@ import {
docsPath,
v3BatchPath,
v3DeploymentVersionPath,
+ v3QueuePath,
v3RunDownloadLogsPath,
v3RunIdempotencyKeyResetPath,
v3RunPath,
@@ -144,7 +151,14 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
// `{ ...result }` collapses the union and loses the
// `type === "run" | "span"` discriminant downstream in `SpanView`.
if (result.type === "run") {
- return typedjson({ type: "run" as const, run: result.run });
+ const queueMetrics = await resolveRunQueueMetrics({
+ userId,
+ organizationSlug,
+ projectParam,
+ envParam,
+ run: result.run,
+ });
+ return typedjson({ type: "run" as const, run: result.run, queueMetrics });
}
return typedjson({ type: "span" as const, span: result.span });
} catch (error) {
@@ -236,6 +250,7 @@ export function SpanView({
return (
void;
@@ -377,6 +394,12 @@ function RunBody({
const tab = value("tab");
const resetFetcher = useTypedFetcher();
+ const queuePath = queueMetrics?.queueFriendlyId
+ ? v3QueuePath(organization, project, environment, {
+ friendlyId: queueMetrics.queueFriendlyId,
+ })
+ : undefined;
+
return (
@@ -920,9 +943,31 @@ function RunBody({
Queue
- Name: {run.queue.name}
- Concurrency key: {run.queue.concurrencyKey ? run.queue.concurrencyKey : "–"}
+ Name:{" "}
+ {queuePath ? (
+ {run.queue.name}
+ ) : (
+ run.queue.name
+ )}
+
+
+ Concurrency key:{" "}
+ {run.queue.concurrencyKey ? (
+ queuePath ? (
+
+ {run.queue.concurrencyKey}
+
+ ) : (
+ run.queue.concurrencyKey
+ )
+ ) : (
+ "–"
+ )}
@@ -1070,6 +1115,15 @@ function RunBody({
content={descriptionForTaskRunStatus(run.status)}
/>
+ {queueMetrics?.waiting ? (
+
+ ) : null}
{run.error &&
}
@@ -1129,6 +1183,80 @@ function RunBody({
);
}
+function WaitingInQueueBlock({
+ queueName,
+ queuePath,
+ paused,
+ waiting,
+ status,
+}: {
+ queueName: string;
+ queuePath: string | undefined;
+ paused: boolean;
+ waiting: RunQueueWaiting;
+ status: SpanRun["status"];
+}) {
+ const key = waiting.concurrencyKey;
+ const limit = waiting.concurrencyLimit;
+ // The concurrency limit applies per key on keyed queues. PENDING renders as "Queued".
+ const atLimit = limit !== null && (key ? key.running >= limit : waiting.running >= limit);
+ const showAtLimit = status === "PENDING" && atLimit && !paused;
+
+ const runningLabel = (running: number, withLimit: boolean) =>
+ withLimit && limit !== null
+ ? `${running.toLocaleString()} of ${limit.toLocaleString()} running`
+ : `${running.toLocaleString()} running`;
+
+ return (
+
+
+ Waiting in queue
+ {queuePath ? View queue : null}
+
+
+ = 60_000 ? "text-warning" : undefined
+ }
+ query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS v\nFROM queue_metrics\nGROUP BY t\nORDER BY t`}
+ color="#F59E0B"
+ ids={waiting.ids}
+ queueName={queueName}
+ formatPeak={(v) => formatWaitMs(v)}
+ />
+
+
+ {runningLabel(waiting.running, !key)}
+ {key
+ ? ` · key ${key.key}: ${key.queued.toLocaleString()} queued · ${runningLabel(
+ key.running,
+ true
+ )}${key.oldestWaitMs !== null ? ` · oldest wait ${formatWaitMs(key.oldestWaitMs)}` : ""}`
+ : ""}
+
+ {paused ? (
+
+ Queue is paused. Runs will not start until it is resumed.
+
+ ) : showAtLimit ? (
+
+ At the concurrency limit. This run starts when a running one finishes.
+
+ ) : null}
+
+ );
+}
+
// Trace export menu items: copy the trace as Markdown (for pasting into an AI
// assistant) plus a download per format. The export route streams `?format=`
// server-side.
diff --git a/apps/webapp/app/routes/storybook.charts/route.tsx b/apps/webapp/app/routes/storybook.charts/route.tsx
index 8c7580cb613..040211437ad 100644
--- a/apps/webapp/app/routes/storybook.charts/route.tsx
+++ b/apps/webapp/app/routes/storybook.charts/route.tsx
@@ -13,6 +13,7 @@ import {
useDateRange,
} from "~/components/primitives/charts/DateRangeContext";
import type { ZoomRange } from "~/components/primitives/charts/hooks/useZoomSelection";
+import { MiniLineChart } from "~/components/metrics/MiniLineChart";
import { Paragraph } from "~/components/primitives/Paragraph";
import { RadioGroup, RadioGroupItem } from "~/components/primitives/RadioButton";
import SegmentedControl from "~/components/primitives/SegmentedControl";
@@ -262,6 +263,126 @@ function ChartsDashboard() {
/>
+
+ {/* Line with per-bucket warning overlay (queues redesign) */}
+
+
+
+ (per-bucket recolour)
+
+
+
+ {/* The base line stays the series colour; buckets strictly above `threshold` retrace
+ in the warning colour, so the same line reads blue -> yellow -> blue as it crosses. */}
+
+
+
+
+
+ {/* Line with gradient threshold split (queues redesign) */}
+
+
+
+ (gradient split)
+
+
+
+ {/* Gradient split above the threshold. Best when the threshold sits mid-domain — if it
+ sits far below the data max the split collapses onto the baseline; prefer
+ `warningOverlay` in that case (see the Warning overlay example). */}
+
+
+
+
+
+ {/* Line with outside-placed reference line label (queues redesign) */}
+
+
+
+ (label outside, in the gutter)
+
+
+
+ {/* labelPlacement: "outside" renders the label in the right gutter at the line's y;
+ the chart's right margin is widened automatically so it isn't clipped. */}
+
+
+
+
+
+ {/* MiniLineChart backlog sparklines (queues redesign) */}
+
+
+
+ (inline backlog sparkline)
+
+
+
+ {/* Fixed-size inline sparkline sized for a table cell, with a trailing peak label. The
+ second row passes aligned `throttled` buckets so throttled stretches retrace the same
+ line in the warning colour. */}
+
+
+
+ Queue
+ Backlog
+
+
+
+
+ emails
+
+
+
+
+ image-processing
+
+
+
+
+
+
+
);
@@ -681,6 +802,30 @@ const API_DATA = {
"analyze-document": 5678,
},
],
+ // Single-series queue depth that crosses the 5,000 threshold twice (blue -> yellow -> blue),
+ // used by the warning overlay, threshold stroke and outside reference-line examples.
+ queueDepthData: [
+ { day: "2023-11-01", queued: 1200 },
+ { day: "2023-11-02", queued: 2400 },
+ { day: "2023-11-03", queued: 3800 },
+ { day: "2023-11-04", queued: 4600 },
+ { day: "2023-11-05", queued: 6200 },
+ { day: "2023-11-06", queued: 7400 },
+ { day: "2023-11-07", queued: 6800 },
+ { day: "2023-11-08", queued: 5200 },
+ { day: "2023-11-09", queued: 3600 },
+ { day: "2023-11-10", queued: 2800 },
+ { day: "2023-11-11", queued: 3400 },
+ { day: "2023-11-12", queued: 5600 },
+ { day: "2023-11-13", queued: 7800 },
+ { day: "2023-11-14", queued: 6400 },
+ { day: "2023-11-15", queued: 4200 },
+ ],
+ // Inline sparkline buckets (oldest first). No throttling on this one.
+ miniLineData: [3, 5, 8, 12, 9, 14, 20, 18, 11, 7, 4, 6],
+ // A backlog that was throttled across a stretch in the middle; `throttled` aligns 1:1 with `data`.
+ miniLineThrottledData: [2, 4, 9, 16, 24, 30, 27, 19, 12, 8, 5, 3],
+ miniLineThrottledBuckets: [0, 0, 0, 6, 14, 18, 11, 0, 0, 0, 0, 0],
};
const lineChartConfig = {
@@ -694,6 +839,13 @@ const lineChartConfig = {
},
} satisfies ChartConfig;
+const queueDepthConfig = {
+ queued: {
+ label: "Queue depth",
+ color: "var(--color-tasks)",
+ },
+} satisfies ChartConfig;
+
const barChartBigDatasetConfig = {
"sync-data": {
label: (
diff --git a/apps/webapp/app/routes/storybook.layout/route.tsx b/apps/webapp/app/routes/storybook.layout/route.tsx
new file mode 100644
index 00000000000..08242fe51da
--- /dev/null
+++ b/apps/webapp/app/routes/storybook.layout/route.tsx
@@ -0,0 +1,373 @@
+import { useState } from "react";
+import { PageContainer } from "~/components/layout/AppLayout";
+import { MetricsLayout } from "~/components/layout/MetricsLayout";
+import { Badge } from "~/components/primitives/Badge";
+import { Header3 } from "~/components/primitives/Headers";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import SegmentedControl from "~/components/primitives/SegmentedControl";
+import {
+ Table,
+ TableBody,
+ TableCell,
+ TableHeader,
+ TableHeaderCell,
+ TableRow,
+} from "~/components/primitives/Table";
+import { TabButton, TabContainer } from "~/components/primitives/Tabs";
+import { cn } from "~/utils/cn";
+
+// A placeholder for a search input / TimeFilter / pagination — the real pages drop live controls
+// into the Filters slot; here we only show the row shape.
+function FilterChip({ children }: { children: React.ReactNode }) {
+ return (
+
+ {children}
+
+ );
+}
+
+// A stat BigNumber-style tile.
+function StatTile({ label, value }: { label: string; value: string }) {
+ return (
+
+
{label}
+
+ {value}
+
+
+ );
+}
+
+// A chart-card-style tile with a fake bar sparkline so the grid's height + column behaviour is
+// visible.
+function ChartTile({ label, className }: { label: string; className?: string }) {
+ return (
+
+
{label}
+
+ {Array.from({ length: 40 }).map((_, i) => (
+
+ ))}
+
+
+ );
+}
+
+function PlaceholderTable() {
+ return (
+
+
+
+ Name
+ Queued
+ Running
+ Limit
+
+
+
+ {[
+ { name: "background", queued: 0, running: 4, limit: 19 },
+ { name: "per-tenant", queued: 83, running: 11, limit: 10 },
+ { name: "emails", queued: 2, running: 1, limit: 50 },
+ ].map((row) => (
+
+ {row.name}
+ {row.queued}
+ {row.running}
+ {row.limit}
+
+ ))}
+
+
+ );
+}
+
+// A tall filler so a scroll region visibly overflows its container.
+function ScrollFiller({ label, rows }: { label: string; rows: number }) {
+ return (
+
+ {Array.from({ length: rows }).map((_, i) => (
+
+
+ {label} row {i + 1}
+
+ {Math.round(Math.abs(Math.sin(i)) * 1000)}
+
+ ))}
+
+ );
+}
+
+// The config panel dropped into MetricsLayout.Sidebar in the sidebar demos.
+function SidebarPanel({ resizable }: { resizable?: boolean }) {
+ return (
+
+
+ Sidebar slot
+
+
+
+ {resizable
+ ? "This sidebar is resizable — drag the handle on its left edge. Pass an autosaveId + a loader snapshot to persist the split across reloads."
+ : "This sidebar is fixed-width (width prop). The main column fills the rest and owns the page scroll."}
+
+
{resizable ? "resizable" : "fixed width"}
+ {Array.from({ length: 6 }).map((_, i) => (
+
Config option {i + 1}
+ ))}
+
+
+ );
+}
+
+// The overview demo. Every slot carries its baked chrome — no className is passed to Root,
+// Filters, Grid or Content. The pinned Filters bar spreads a left and right cluster; the Grids
+// bake the page gutter and adapt their columns (or take an explicit `columns` / `kind="charts"`);
+// Content toggles between a full-bleed table and an inset panel. The whole page scrolls as one.
+function OverviewDemo() {
+ const [tab, setTab] = useState<"panel" | "table">("panel");
+
+ return (
+
+ {/* Filters slot — the pinned bar directly under the NavBar. Left + right clusters as child
+ divs; the slot bakes the 40px height, border and insets. */}
+
+
+ Search…
+ Period: 7d
+ Filters slot
+
+ {"< 1 / 4 >"}
+
+
+ {/* Grid slot — 4 stat tiles. Columns are derived from the tile count: two-up, four-up
+ from lg. The gutter + gap are baked. */}
+
+ Grid — 4 tiles (auto: 2-up, 4-up from lg)
+
+
+
+
+ {/* Grid slot — kind="charts" bakes the fixed chart-row height (no wrapper needed). */}
+
+ Grid — kind="charts" (fixed row height, auto columns)
+
+
+
+
+ {/* Grid slot — 3 tiles. The same component lays out one-up, three-up from sm, proving the
+ grid adapts to the child count. */}
+
+ Grid — 3 tiles (auto: 1-up, 3-up from sm)
+
+
+
+
+ {/* Grid slot — explicit columns for a chart grid that should always be two-up regardless
+ of tile count. */}
+
+ Grid — explicit columns={{ base: 1, sm: 2 }} (5 tiles)
+
+
+
+
+ {/* Content slot — a doubled separation above it is baked in, so the tiles read as their own
+ band. `inset` toggles between a padded column (panel) and full-bleed (edge-to-edge
+ table). */}
+
+
+ setTab("panel")}
+ >
+ Panel (inset)
+
+ setTab("table")}
+ >
+ Table (full-bleed)
+
+
+ {tab === "table" ? (
+
+
+ With inset, Content becomes a padded column: this panel and the tabs
+ above it sit on the standard page gutter. Switch to the table to see Content go
+ full-bleed — the table spans edge to edge with its own top border. Either way the
+ whole page (filters aside) shares one vertical scroll.
+
+
+
+ );
+}
+
+// Fixed-width sidebar: a
child flips Root into a [main | sidebar] layout.
+// The main column keeps its normal top-to-bottom slots and owns the page scroll.
+function SidebarFixedDemo() {
+ return (
+
+
+
+ Search…
+ main column
+
+
+
+
+
+
+
+
+
+
+ );
+}
+
+// Resizable sidebar: same [main | sidebar] layout, but the split is draggable via the shared
+// Resizable primitives. autosaveId persists the split to a cookie (in a real page the loader
+// hydrates it back through a snapshot); here it persists live within the session.
+function SidebarResizableDemo() {
+ return (
+
+
+
+ Search…
+ drag the handle →
+
+
+
+
+
+
+
+
+
+
+ );
+}
+
+// scroll="regions": Root does NOT create the page scroll. It only bounds the height as a flex
+// column, so the page composes its own independently-scrolling areas — here a fixed toolbar over
+// two side-by-side lists that each scroll on their own.
+function RegionsDemo() {
+ return (
+
+
+ fixed toolbar (does not scroll)
+ Period: 7d
+
+
+
+
+ Left region — scrolls independently
+
+
+
+
+
+ Right region — scrolls independently
+
+
+
+
+ );
+}
+
+type Demo = "overview" | "sidebar-fixed" | "sidebar-resizable" | "regions";
+
+const DEMO_OPTIONS: { label: string; value: Demo }[] = [
+ { label: "Overview", value: "overview" },
+ { label: "Sidebar (fixed)", value: "sidebar-fixed" },
+ { label: "Sidebar (resizable)", value: "sidebar-resizable" },
+ { label: "Scroll: regions", value: "regions" },
+];
+
+/**
+ * Storybook for the MetricsLayout compound. A segmented control swaps between demos, each filling
+ * the page:
+ * - Overview — the base Filters / count-adaptive Grid / Content slots, page scroll.
+ * - Sidebar (fixed) — a fixed-width MetricsLayout.Sidebar beside the main column.
+ * - Sidebar (resizable) — the same, but with a draggable split (autosaveId persistence).
+ * - Scroll: regions — scroll="regions" with two independently-scrolling areas.
+ */
+export default function Story() {
+ const [demo, setDemo] = useState("overview");
+
+ return (
+
+
+
+ setDemo(value as Demo)}
+ />
+
+
+ {demo === "overview" ? (
+
+ );
+}
diff --git a/apps/webapp/app/routes/storybook/route.tsx b/apps/webapp/app/routes/storybook/route.tsx
index 012d47827de..8309fee5b2b 100644
--- a/apps/webapp/app/routes/storybook/route.tsx
+++ b/apps/webapp/app/routes/storybook/route.tsx
@@ -71,6 +71,10 @@ const stories: Story[] = [
name: "Inline code",
slug: "inline-code",
},
+ {
+ name: "Layout",
+ slug: "layout",
+ },
{
name: "Loading bar divider",
slug: "loading-bar-divider",
diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts
index 57b877ed876..70af40ec89f 100644
--- a/apps/webapp/app/services/queryService.server.ts
+++ b/apps/webapp/app/services/queryService.server.ts
@@ -7,7 +7,12 @@ import {
type TSQLQueryResult,
} from "@internal/clickhouse";
import type { CustomerQuerySource } from "@trigger.dev/database";
-import type { TableSchema, WhereClauseCondition } from "@internal/tsql";
+import {
+ calculateTimeBucketInterval,
+ type TableSchema,
+ type TimeBucketInterval,
+ type WhereClauseCondition,
+} from "@internal/tsql";
import { z } from "zod";
import { prisma } from "~/db.server";
import { env } from "~/env.server";
@@ -110,6 +115,41 @@ export type ExecuteQueryResult =
}
| { success: false; error: Error };
+const INTERVAL_UNIT_SECONDS: Record = {
+ SECOND: 1,
+ MINUTE: 60,
+ HOUR: 3_600,
+ DAY: 86_400,
+ WEEK: 604_800,
+ MONTH: 2_592_000,
+};
+
+function floorToSeconds(date: Date, alignSeconds: number): Date {
+ const ms = alignSeconds * 1000;
+ return new Date(Math.floor(date.getTime() / ms) * ms);
+}
+
+/**
+ * Swap a table for one of its rollups when the query's bucket interval is at least the
+ * rollup's granularity. The rollup has identical logical columns, so only the physical
+ * table (and therefore rows read) changes.
+ */
+function resolveRollup(schema: TableSchema, timeRange: { from: Date; to: Date }): TableSchema {
+ if (!schema.rollups || schema.rollups.length === 0) {
+ return schema;
+ }
+ const interval = calculateTimeBucketInterval(
+ timeRange.from,
+ timeRange.to,
+ schema.timeBucketThresholds
+ );
+ const intervalSeconds = interval.value * INTERVAL_UNIT_SECONDS[interval.unit];
+ const best = [...schema.rollups]
+ .sort((a, b) => b.minIntervalSeconds - a.minIntervalSeconds)
+ .find((r) => r.minIntervalSeconds <= intervalSeconds);
+ return best ? { ...schema, clickhouseName: best.clickhouseName } : schema;
+}
+
export async function getDefaultPeriod(organizationId: string): Promise {
const idealDefaultPeriodDays = 7;
const maxQueryPeriod = await getLimit(organizationId, "queryPeriodDays", 30);
@@ -183,6 +223,14 @@ export async function executeQuery(
defaultPeriod,
});
+ // Align the time bounds so repeated auto-refresh queries produce identical query
+ // params and can share ClickHouse query-cache entries (params are part of the key).
+ const alignSeconds = matchedSchema?.queryCache?.alignSeconds;
+ if (alignSeconds) {
+ if (timeFilter.from) timeFilter.from = floorToSeconds(timeFilter.from, alignSeconds);
+ if (timeFilter.to) timeFilter.to = floorToSeconds(timeFilter.to, alignSeconds);
+ }
+
// Calculate the effective "from" date the user is requesting (for period clipping check)
// This is null only when the user specifies just a "to" date (rare case)
let requestedFromDate: Date | null = null;
@@ -192,6 +240,9 @@ export async function executeQuery(
// Period specified (or default) - calculate from now
const periodMs = parse(timeFilter.period ?? defaultPeriod) ?? 7 * 24 * 60 * 60 * 1000;
requestedFromDate = new Date(Date.now() - periodMs);
+ if (alignSeconds) {
+ requestedFromDate = floorToSeconds(requestedFromDate, alignSeconds);
+ }
}
// Build the fallback WHERE condition based on what the user specified
@@ -207,7 +258,10 @@ export async function executeQuery(
}
const maxQueryPeriod = await getLimit(organizationId, "queryPeriodDays", 30);
- const maxQueryPeriodDate = new Date(Date.now() - maxQueryPeriod * 24 * 60 * 60 * 1000);
+ let maxQueryPeriodDate = new Date(Date.now() - maxQueryPeriod * 24 * 60 * 60 * 1000);
+ if (alignSeconds) {
+ maxQueryPeriodDate = floorToSeconds(maxQueryPeriodDate, alignSeconds);
+ }
// Check if the requested time period exceeds the plan limit
const periodClipped = requestedFromDate !== null && requestedFromDate < maxQueryPeriodDate;
@@ -255,6 +309,10 @@ export async function executeQuery(
to: to ?? undefined,
defaultPeriod,
});
+ if (alignSeconds) {
+ timeRange.from = floorToSeconds(timeRange.from, alignSeconds);
+ timeRange.to = floorToSeconds(timeRange.to, alignSeconds);
+ }
try {
// Build field mappings for project_ref → project_id and environment_id → slug translation
@@ -277,10 +335,19 @@ export async function executeQuery(
organizationId,
"query"
);
+ // Serve coarse-bucket queries from the table's rollup when one qualifies.
+ const effectiveSchemas = matchedSchema?.rollups
+ ? querySchemas.map((s) => (s === matchedSchema ? resolveRollup(s, timeRange) : s))
+ : querySchemas;
+
+ const queryCacheSettings: ClickHouseSettings = matchedSchema?.queryCache
+ ? { use_query_cache: 1, query_cache_ttl: matchedSchema.queryCache.ttlSeconds }
+ : {};
+
const result = await executeTSQL(queryClickhouse.reader, {
...baseOptions,
schema: z.record(z.any()),
- tableSchema: querySchemas,
+ tableSchema: effectiveSchemas,
transformValues: true,
enforcedWhereClause,
fieldMappings,
@@ -290,6 +357,7 @@ export async function executeQuery(
timeRange,
clickhouseSettings: {
...getDefaultClickhouseSettings(),
+ ...queryCacheSettings,
...baseOptions.clickhouseSettings, // Allow caller overrides if needed
},
querySettings: {
diff --git a/apps/webapp/app/utils/pathBuilder.ts b/apps/webapp/app/utils/pathBuilder.ts
index 187bc50b549..edd65f8bde4 100644
--- a/apps/webapp/app/utils/pathBuilder.ts
+++ b/apps/webapp/app/utils/pathBuilder.ts
@@ -522,6 +522,15 @@ export function v3QueuesPath(
return `${v3EnvironmentPath(organization, project, environment)}/queues`;
}
+export function v3QueuePath(
+ organization: OrgForPath,
+ project: ProjectForPath,
+ environment: EnvironmentForPath,
+ queue: { friendlyId: string }
+) {
+ return `${v3QueuesPath(organization, project, environment)}/${queue.friendlyId}`;
+}
+
export function v3WaitpointTokensPath(
organization: OrgForPath,
project: ProjectForPath,
diff --git a/apps/webapp/app/v3/canAccessQueueMetricsUi.server.ts b/apps/webapp/app/v3/canAccessQueueMetricsUi.server.ts
new file mode 100644
index 00000000000..0e3c142b272
--- /dev/null
+++ b/apps/webapp/app/v3/canAccessQueueMetricsUi.server.ts
@@ -0,0 +1,26 @@
+import { prisma } from "~/db.server";
+import { FEATURE_FLAG } from "~/v3/featureFlags";
+import { makeFlag } from "~/v3/featureFlags.server";
+
+// Per-org gate for the Queue Metrics dashboard UI. Org override wins over the global
+// FeatureFlag table value, which wins over the off-by-default. Ingestion/emission is a
+// separate global flag; this only decides whether an org sees the metrics view.
+export async function canAccessQueueMetricsUi(options: {
+ userId: string;
+ organizationSlug: string;
+}): Promise {
+ const org = await prisma.organization.findFirst({
+ where: {
+ slug: options.organizationSlug,
+ members: { some: { userId: options.userId } },
+ },
+ select: { featureFlags: true },
+ });
+
+ const flag = makeFlag();
+ return flag({
+ key: FEATURE_FLAG.queueMetricsUiEnabled,
+ defaultValue: false,
+ overrides: (org?.featureFlags as Record) ?? {},
+ });
+}
diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts
index 49a18eaf35f..eefb5d665ef 100644
--- a/apps/webapp/app/v3/featureFlags.ts
+++ b/apps/webapp/app/v3/featureFlags.ts
@@ -22,6 +22,7 @@ export const FEATURE_FLAG = {
// Grace-linger stamp carried alongside runOpsMintKind on flip. See mintFlipGrace.ts.
runOpsMintKindPrev: "runOpsMintKindPrev",
runOpsMintKindFlippedAt: "runOpsMintKindFlippedAt",
+ queueMetricsUiEnabled: "queueMetricsUiEnabled",
} as const;
export const FeatureFlagCatalog = {
@@ -61,6 +62,9 @@ export const FeatureFlagCatalog = {
// by stampMintKindFlip on a genuine flip. Display-only (see ORG_LOCKED_FLAGS).
[FEATURE_FLAG.runOpsMintKindPrev]: z.enum(["cuid", "runOpsId"]),
[FEATURE_FLAG.runOpsMintKindFlippedAt]: z.string().datetime(),
+ // Per-org access to the Queue Metrics dashboard UI (view only; emission is global and
+ // separate). Off unless enabled for the org.
+ [FEATURE_FLAG.queueMetricsUiEnabled]: z.coerce.boolean(),
};
export type FeatureFlagKey = keyof typeof FeatureFlagCatalog;
diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts
index 4784ad75629..540ae670091 100644
--- a/apps/webapp/app/v3/querySchemas.ts
+++ b/apps/webapp/app/v3/querySchemas.ts
@@ -614,8 +614,333 @@ export const metricsSchema: TableSchema = {
};
/**
- * All available schemas for the query editor
+ * Schema definition for the queue_metrics table (trigger_dev.queue_metrics_v1).
+ * Pre-aggregated into 10-second buckets. Counter columns re-aggregate with sum(),
+ * gauges with max(), and wait_quantiles with quantilesMerge() — never FINAL.
*/
+export const queueMetricsSchema: TableSchema = {
+ name: "queue_metrics",
+ clickhouseName: "trigger_dev.queue_metrics_v1",
+ description: "Per-queue depth, concurrency, throttling, and scheduling-delay metrics",
+ timeConstraint: "bucket_start",
+ tenantColumns: {
+ organizationId: "organization_id",
+ projectId: "project_id",
+ environmentId: "environment_id",
+ },
+ columns: {
+ environment: {
+ name: "environment",
+ clickhouseName: "environment_id",
+ ...column("String", { description: "The environment slug", example: "prod" }),
+ fieldMapping: "environment",
+ customRenderType: "environment",
+ },
+ project: {
+ name: "project",
+ clickhouseName: "project_id",
+ ...column("String", {
+ description: "The project reference, they always start with `proj_`.",
+ example: "proj_howcnaxbfxdmwmxazktx",
+ }),
+ fieldMapping: "project",
+ customRenderType: "project",
+ },
+ queue: {
+ name: "queue",
+ clickhouseName: "queue_name",
+ ...column("LowCardinality(String)", {
+ description: "The queue name",
+ example: "my-queue",
+ coreColumn: true,
+ }),
+ },
+ bucket_start: {
+ name: "bucket_start",
+ ...column("DateTime", {
+ description: "The start of the 10-second aggregation bucket",
+ example: "2024-01-15 09:30:00",
+ coreColumn: true,
+ }),
+ },
+ // Cumulative-counter delta states. Read with deltaSumTimestampMerge( {
+ const [enabledRaw, rateRaw] = (await adminRedis().mget(FLAG_KEY, SAMPLE_RATE_KEY)) as (
+ | string
+ | null
+ )[];
+ const sampleRateDefault = env.QUEUE_METRICS_GAUGE_SAMPLE_RATE;
+ const parsed = rateRaw == null ? Number.NaN : Number(rateRaw);
+ return {
+ enabled: enabledRaw != null && TRUTHY.has(enabledRaw.trim().toLowerCase()),
+ enabledKeySet: enabledRaw != null,
+ sampleRate: Number.isFinite(parsed) ? Math.min(1, Math.max(0, parsed)) : sampleRateDefault,
+ sampleRateKeySet: rateRaw != null,
+ sampleRateDefault,
+ };
+}
+
+export async function writeQueueMetricsControls(update: {
+ enabled?: boolean;
+ sampleRate?: number;
+}): Promise {
+ const client = adminRedis();
+ const ops: Promise[] = [];
+ if (update.enabled !== undefined) {
+ ops.push(client.set(FLAG_KEY, update.enabled ? "1" : "0"));
+ }
+ if (update.sampleRate !== undefined) {
+ ops.push(client.set(SAMPLE_RATE_KEY, String(Math.min(1, Math.max(0, update.sampleRate)))));
+ }
+ await Promise.all(ops);
+}
+
+export type LabeledShardState = ShardState & { stream: "queue_metrics" };
+
+export async function probeQueueMetricsStreams(): Promise {
+ const def = metricsDefinition();
+ const states = await probeShardStates(metricsAdminRedis(), allStreamKeys(def), def.consumerGroup);
+ return states.map((s) => ({ ...s, stream: "queue_metrics" as const }));
+}
+
+/** Injected into the RunQueue when QUEUE_METRICS_EMIT_ENABLED=1; emits only while the flag is on. */
+export function getQueueMetricsEmitter(): MetricsStreamEmitter {
+ return singleton("queueMetricsEmitter", () => {
+ // Control keys stay on the run-queue Redis (the admin surface + docs point there).
+ const controlRedis = runQueueRedisOptions();
+ const flag = new CachedRedisFlag({ redis: controlRedis, key: FLAG_KEY, cacheTtlMs: 10_000 });
+ // Live-tunable (Redis key, 10s cache); the env value is the default when the key is unset.
+ const gaugeSampleRate = new CachedRedisNumber({
+ redis: controlRedis,
+ key: SAMPLE_RATE_KEY,
+ defaultValue: env.QUEUE_METRICS_GAUGE_SAMPLE_RATE,
+ min: 0,
+ max: 1,
+ cacheTtlMs: 10_000,
+ });
+ return new MetricsStreamEmitter({
+ redis: metricsRedisOptions(),
+ definition: metricsDefinition(),
+ flag,
+ meter,
+ gaugeSampleRate,
+ counterOdometerTtlMs: env.QUEUE_METRICS_COUNTER_ODOMETER_TTL_SECONDS * 1000,
+ });
+ });
+}
+
+const queueNameLimiter = singleton(
+ "queueMetricsQueueNameLimiter",
+ () => new QueueNameLimiter(env.QUEUE_METRICS_MAX_QUEUE_NAMES_PER_ENV)
+);
+
+const concurrencyKeyLimiter = singleton(
+ "queueMetricsConcurrencyKeyLimiter",
+ () => new QueueNameLimiter(env.QUEUE_METRICS_MAX_CONCURRENCY_KEYS_PER_QUEUE, 50_000)
+);
+
+function mapEntry(entry: StreamEntry): QueueMetricsRawV1Input[] {
+ return mapEntryToRows(entry, {
+ queueNames: queueNameLimiter,
+ concurrencyKeys: concurrencyKeyLimiter,
+ });
+}
+
+function makeInsert(): (
+ rows: QueueMetricsRawV1Input[],
+ opts: { dedupToken: string }
+) => Promise {
+ const ch: ClickHouse = getDefaultClickhouseClient();
+ const insertRaw = ch.queueMetrics.insertRaw;
+ return async (rows, { dedupToken }) => {
+ const [error] = await insertRaw(rows, {
+ params: {
+ clickhouse_settings: {
+ insert_deduplication_token: dedupToken,
+ async_insert: 0,
+ // Propagate the token through the MV so a raw-deduped retry can't leave
+ // queue_metrics_v1 short when the MV insert failed on the first attempt.
+ deduplicate_blocks_in_dependent_materialized_views: 1,
+ },
+ },
+ });
+ if (error) throw error;
+ };
+}
+
+function getQueueMetricsConsumers(): MetricsStreamConsumer[] {
+ return singleton("queueMetricsConsumers", () => {
+ const insert = makeInsert();
+ return [
+ new MetricsStreamConsumer({
+ consumerName: `${os.hostname()}-${process.pid}`,
+ batchSize: env.QUEUE_METRICS_CONSUMER_BATCH_SIZE,
+ meter,
+ mapEntry,
+ insert,
+ redis: metricsRedisOptions(),
+ definition: metricsDefinition(),
+ }),
+ ];
+ });
+}
+
+// Construct the emitter at boot (not lazily on the first enqueue) so its flag has warmed
+// before any traffic — otherwise the first op after boot reads the default and is dropped.
+export function initQueueMetricsEmitter(): void {
+ if (env.QUEUE_METRICS_EMIT_ENABLED !== "1") return;
+ getQueueMetricsEmitter();
+}
+
+declare global {
+ // eslint-disable-next-line no-var
+ var __queueMetricsConsumerRegistered__: boolean | undefined;
+}
+
+export function initQueueMetricsConsumer(): void {
+ if (env.QUEUE_METRICS_CONSUMER_ENABLED !== "1") return;
+ if (global.__queueMetricsConsumerRegistered__) return;
+ global.__queueMetricsConsumerRegistered__ = true;
+
+ const consumers = getQueueMetricsConsumers();
+ const stop = () =>
+ Promise.all(consumers.map((c) => c.stop())).catch((error) =>
+ logger.error("queue metrics consumer stop failed", { error })
+ );
+ signalsEmitter.on("SIGTERM", stop);
+ signalsEmitter.on("SIGINT", stop);
+
+ Promise.all(consumers.map((c) => c.start()))
+ .then(() => logger.info("Queue metrics consumer started"))
+ .catch((error) => logger.error("queue metrics consumers failed to start", { error }));
+}
diff --git a/apps/webapp/app/v3/queueMetricsMapping.ts b/apps/webapp/app/v3/queueMetricsMapping.ts
new file mode 100644
index 00000000000..9433b361a88
--- /dev/null
+++ b/apps/webapp/app/v3/queueMetricsMapping.ts
@@ -0,0 +1,164 @@
+import { type QueueMetricsRawV1Input } from "@internal/clickhouse";
+import { entryOrderKey, entryTimeMs, type StreamEntry } from "@internal/metrics-pipeline";
+
+const OPS = new Set(["gauge", "enqueue", "started", "ack", "nack", "dlq"]);
+
+// {org:ORGID}:proj:PROJECTID:env:ENVID:queue:QUEUENAME[:ck:CK]. Anchored (not a
+// positional split) so a queue name containing ":" survives; the lazy name capture
+// stops before an optional ":ck:" suffix, which is captured (the ":ck:*" wildcard of
+// aggregate CK-dequeue gauges maps to no key).
+const DESCRIPTOR = /^\{org:([^}]+)\}:proj:([^:]+):env:([^:]+):queue:(.+?)(?::ck:(.+))?$/;
+
+export function descriptorFromQueue(q: string): {
+ organization_id: string;
+ project_id: string;
+ environment_id: string;
+ queue_name: string;
+ concurrency_key: string;
+} | null {
+ const match = DESCRIPTOR.exec(q);
+ if (!match) return null;
+ const ck = match[5];
+ return {
+ organization_id: match[1]!,
+ project_id: match[2]!,
+ environment_id: match[3]!,
+ queue_name: match[4]!,
+ concurrency_key: ck && ck !== "*" ? ck : "",
+ };
+}
+
+export const OVERFLOW_QUEUE_NAME = "__overflow__";
+
+/**
+ * Bounds per-scope name cardinality (both queue_name per env and concurrency_key per
+ * queue are user-controlled GROUP BY keys). Names beyond the cap map to OVERFLOW_QUEUE_NAME.
+ * Per-process and reset on restart, so the cap is approximate: a protective bound, not a quota.
+ */
+export class QueueNameLimiter {
+ private readonly byScope = new Map>();
+
+ constructor(
+ private readonly maxPerScope: number,
+ private readonly maxScopes = 10_000
+ ) {}
+
+ limit(scope: string, name: string): string {
+ if (this.maxPerScope <= 0) return name;
+ let names = this.byScope.get(scope);
+ if (!names) {
+ if (this.byScope.size >= this.maxScopes) {
+ const oldest = this.byScope.keys().next().value;
+ if (oldest !== undefined) this.byScope.delete(oldest);
+ }
+ names = new Set();
+ this.byScope.set(scope, names);
+ }
+ if (names.has(name)) return name;
+ if (names.size >= this.maxPerScope) return OVERFLOW_QUEUE_NAME;
+ names.add(name);
+ return name;
+ }
+}
+
+function num(value: string | undefined): number | undefined {
+ if (value == null) return undefined;
+ const n = Number(value);
+ return Number.isFinite(n) ? n : undefined;
+}
+
+export type QueueMetricsLimiters = {
+ queueNames?: QueueNameLimiter;
+ concurrencyKeys?: QueueNameLimiter;
+};
+
+/**
+ * One stream entry maps to 1..2 raw rows: gauges are single rows carrying their parsed
+ * concurrency_key; a counter entry yields a base row when `cum` is present plus a per-key
+ * row when `ck`/`ckcum` are present (the emitter's dual-odometer entry). Baseline entries
+ * carry only one of the two, by design.
+ */
+export function mapEntryToRows(
+ entry: StreamEntry,
+ limiters?: QueueMetricsLimiters
+): QueueMetricsRawV1Input[] {
+ const f = entry.fields;
+ const op = f.op;
+ if (!op || !OPS.has(op) || !f.q) return [];
+ const descriptor = descriptorFromQueue(f.q);
+ if (!descriptor || !descriptor.queue_name) return [];
+
+ let queueOverflowed = false;
+ if (limiters?.queueNames) {
+ descriptor.queue_name = limiters.queueNames.limit(
+ descriptor.environment_id,
+ descriptor.queue_name
+ );
+ queueOverflowed = descriptor.queue_name === OVERFLOW_QUEUE_NAME;
+ }
+
+ // Counter entries carry the key as a field (q is base-normalized); gauges carry it in q.
+ let ck = descriptor.concurrency_key || (typeof f.ck === "string" ? f.ck : "");
+ if (ck && limiters?.concurrencyKeys) {
+ const scope = `${descriptor.environment_id}:${descriptor.queue_name}`;
+ if (limiters.concurrencyKeys.limit(scope, ck) === OVERFLOW_QUEUE_NAME) ck = "";
+ }
+ // Overflowed queue names share one row; per-key attribution under them is meaningless.
+ if (queueOverflowed) ck = "";
+
+ const eventMs = entryTimeMs(entry.id) ?? Date.now();
+ const eventTime = new Date(eventMs).toISOString().slice(0, 19).replace("T", " ");
+ const base = {
+ organization_id: descriptor.organization_id,
+ project_id: descriptor.project_id,
+ environment_id: descriptor.environment_id,
+ queue_name: descriptor.queue_name,
+ event_time: eventTime,
+ op: op as QueueMetricsRawV1Input["op"],
+ };
+
+ if (op === "gauge") {
+ return [
+ {
+ ...base,
+ concurrency_key: ck,
+ queued: num(f.ql),
+ running: num(f.cc),
+ queue_limit: num(f.lim),
+ env_queued: num(f.eql),
+ env_running: num(f.ec),
+ env_limit: num(f.elim),
+ throttled: num(f.thr),
+ ck_backlogged: num(f.ckq),
+ ck_max_wait_ms: num(f.ckw),
+ },
+ ];
+ }
+
+ // Overflowed names drop counters entirely: merging distinct odometers under one shared
+ // name produces garbage deltas (gauges above stay, max across the overflow set is
+ // still meaningful).
+ if (queueOverflowed) return [];
+
+ const rows: QueueMetricsRawV1Input[] = [];
+ const orderKey = entryOrderKey(entry.id);
+ const waitMs = op === "started" && f.wait != null ? num(f.wait) : undefined;
+ if (f.cum != null) {
+ rows.push({
+ ...base,
+ cumulative: num(f.cum),
+ order_key: orderKey,
+ ...(waitMs !== undefined ? { wait_ms: waitMs } : {}),
+ });
+ }
+ if (ck && f.ckcum != null) {
+ rows.push({
+ ...base,
+ concurrency_key: ck,
+ cumulative: num(f.ckcum),
+ order_key: orderKey,
+ ...(waitMs !== undefined ? { wait_ms: waitMs } : {}),
+ });
+ }
+ return rows;
+}
diff --git a/apps/webapp/app/v3/runEngine.server.ts b/apps/webapp/app/v3/runEngine.server.ts
index 4d9e263d6be..85986933290 100644
--- a/apps/webapp/app/v3/runEngine.server.ts
+++ b/apps/webapp/app/v3/runEngine.server.ts
@@ -7,6 +7,7 @@ import { logger } from "~/services/logger.server";
import { defaultMachine, getCurrentPlan } from "~/services/platform.v3.server";
import { singleton } from "~/utils/singleton";
import { allMachines } from "./machinePresets.server";
+import { getQueueMetricsEmitter } from "./queueMetrics.server";
import { runEnginePendingVersionLookup } from "./runEnginePendingVersionLookup.server";
import { pickRunOpsStoreForCompletion } from "./runOpsMigration/crossSeamGuard.server";
import { runEngineControlPlaneResolver } from "./runOpsMigration/runEngineControlPlaneResolver.server";
@@ -83,6 +84,7 @@ function createRunEngine() {
tracer,
},
shardCount: env.RUN_ENGINE_RUN_QUEUE_SHARD_COUNT,
+ queueMetrics: env.QUEUE_METRICS_EMIT_ENABLED === "1" ? getQueueMetricsEmitter() : undefined,
processWorkerQueueDebounceMs: env.RUN_ENGINE_PROCESS_WORKER_QUEUE_DEBOUNCE_MS,
dequeueBlockingTimeoutSeconds: env.RUN_ENGINE_DEQUEUE_BLOCKING_TIMEOUT_SECONDS,
masterQueueConsumersIntervalMs: env.RUN_ENGINE_MASTER_QUEUE_CONSUMERS_INTERVAL_MS,
diff --git a/apps/webapp/app/v3/services/allocateConcurrency.server.ts b/apps/webapp/app/v3/services/allocateConcurrency.server.ts
index b9a28daba60..78c8b82915a 100644
--- a/apps/webapp/app/v3/services/allocateConcurrency.server.ts
+++ b/apps/webapp/app/v3/services/allocateConcurrency.server.ts
@@ -3,6 +3,7 @@ import { ManageConcurrencyPresenter } from "~/presenters/v3/ManageConcurrencyPre
import { BaseService } from "./baseService.server";
import { updateEnvConcurrencyLimits } from "../runQueue.server";
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
+import { concurrencySystem } from "./concurrencySystemInstance.server";
type Input = {
userId: string;
@@ -90,6 +91,14 @@ export class AllocateConcurrencyService extends BaseService {
await updateEnvConcurrencyLimits(updatedEnvironment);
}
+ // Percent-based queue overrides follow the environment limit automatically. Note the
+ // deliberate asymmetry with the env-level push above: `updateEnvConcurrencyLimits` is gated
+ // on `!paused`, but we recalculate queue limits even for paused environments. Queue-level
+ // pushes on a paused env are inert (the env-level gate stops dequeueing regardless), and
+ // keeping the queue limits synced means resume needs no extra reconciliation — skipping
+ // them here would instead leave stale engine limits after the env resumes.
+ await concurrencySystem.queues.recalculatePercentLimits(updatedEnvironment);
+
// maximumConcurrencyLimit changed in the control-plane; drop any cached copy.
controlPlaneResolver.invalidateEnvironment(environment.id);
}
diff --git a/apps/webapp/app/v3/services/concurrencySystem.server.ts b/apps/webapp/app/v3/services/concurrencySystem.server.ts
index 488f38ce279..f030cb72e5f 100644
--- a/apps/webapp/app/v3/services/concurrencySystem.server.ts
+++ b/apps/webapp/app/v3/services/concurrencySystem.server.ts
@@ -2,6 +2,7 @@ import type { TaskQueue, User } from "@trigger.dev/database";
import { errAsync, fromPromise, okAsync } from "neverthrow";
import type { PrismaClientOrTransaction } from "~/db.server";
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
import { removeQueueConcurrencyLimits, updateQueueConcurrencyLimits } from "../runQueue.server";
import { engine } from "../runEngine.server";
@@ -12,6 +13,42 @@ export type ConcurrencySystemOptions = {
export type QueueInput = string | { type: "task" | "custom"; name: string };
+/**
+ * The concurrency-limit override to apply to a queue. Either an absolute `limit` or a `percent`
+ * of the environment's maximum concurrency limit. A bare `number` is accepted for backwards
+ * compatibility and is treated as an absolute limit.
+ */
+export type ConcurrencyLimitOverride = number | { limit: number } | { percent: number };
+
+/**
+ * Materializes an absolute concurrency limit from a percentage of the environment limit.
+ * Reused by the recalculation task that runs when an environment limit changes.
+ *
+ * Clamped to `>= 1` so a percent-based override never produces a `0` (pause-like) limit, and
+ * to `<= envLimit` so it can never exceed the environment maximum.
+ */
+/**
+ * Valid range for a percent-based queue concurrency override: greater than 0 and up to 100% of
+ * the environment limit. Shared by every layer that validates the percent (the API zod schema,
+ * the dashboard mutation handler, and the override service) so the bound never drifts apart.
+ */
+export const MIN_QUEUE_OVERRIDE_PERCENT = 0;
+export const MAX_QUEUE_OVERRIDE_PERCENT = 100;
+
+/** Whether `percent` is a valid queue-override percentage (0 < percent <= 100). */
+export function isValidQueueOverridePercent(percent: number): boolean {
+ return (
+ Number.isFinite(percent) &&
+ percent > MIN_QUEUE_OVERRIDE_PERCENT &&
+ percent <= MAX_QUEUE_OVERRIDE_PERCENT
+ );
+}
+
+export function materializePercentLimit(envLimit: number, percent: number): number {
+ const materialized = Math.floor((envLimit * percent) / 100);
+ return Math.min(Math.max(materialized, 1), envLimit);
+}
+
export class ConcurrencySystem {
constructor(private readonly options: ConcurrencySystemOptions) {}
@@ -24,18 +61,12 @@ export class ConcurrencySystem {
overrideQueueConcurrencyLimit: (
environment: AuthenticatedEnvironment,
queue: QueueInput,
- concurrencyLimit: number,
+ override: ConcurrencyLimitOverride,
overriddenBy?: User
) => {
return findQueueFromInput(this.db, environment, queue)
.andThen((queue) =>
- overrideQueueConcurrencyLimit(
- this.db,
- environment,
- queue,
- concurrencyLimit,
- overriddenBy
- )
+ overrideQueueConcurrencyLimit(this.db, environment, queue, override, overriddenBy)
)
.andThen((queue) => syncQueueConcurrencyToEngine(environment, queue))
.andThen((queue) => getQueueStats(environment, queue));
@@ -46,6 +77,59 @@ export class ConcurrencySystem {
.andThen((queue) => syncQueueConcurrencyToEngine(environment, queue))
.andThen((queue) => getQueueStats(environment, queue));
},
+ /**
+ * Recalculates the materialized limit of every percent-based override in the environment
+ * against its CURRENT maximumConcurrencyLimit and syncs changed queues to the run engine.
+ * Call AFTER the environment-limit DB update has committed (engine syncs must not run
+ * inside an open transaction). Idempotent: unchanged queues are skipped. One failing queue
+ * is logged and skipped so the rest still converge.
+ */
+ recalculatePercentLimits: async (environment: AuthenticatedEnvironment) => {
+ const queues = await this.db.taskQueue.findMany({
+ where: {
+ runtimeEnvironmentId: environment.id,
+ concurrencyLimitOverridePercent: { not: null },
+ },
+ });
+
+ let updated = 0;
+ for (const queue of queues) {
+ try {
+ const percent = queue.concurrencyLimitOverridePercent;
+ if (percent === null) continue;
+ const newLimit = materializePercentLimit(
+ environment.maximumConcurrencyLimit,
+ percent.toNumber()
+ );
+
+ // Only write the DB when the materialized value actually changed.
+ if (newLimit !== queue.concurrencyLimit) {
+ await this.db.taskQueue.update({
+ where: { id: queue.id },
+ data: { concurrencyLimit: newLimit },
+ });
+ updated++;
+ }
+
+ // Always attempt the engine push (it's idempotent) for active queues — even when the
+ // DB value was unchanged — so a previously-failed sync self-heals on the next recalc
+ // instead of leaving the DB and engine diverged forever. Paused queues keep their
+ // engine limit at 0 (the pause/resume flow re-syncs from the stored value on resume);
+ // push nothing for them so a percent recalc never effectively un-pauses a queue.
+ if (!queue.paused) {
+ await updateQueueConcurrencyLimits(environment, queue.name, newLimit);
+ }
+ } catch (error) {
+ logger.error("Failed to recalculate percent queue limit", {
+ queueId: queue.id,
+ environmentId: environment.id,
+ error,
+ });
+ }
+ }
+
+ return { total: queues.length, updated };
+ },
};
}
}
@@ -117,13 +201,49 @@ function overrideQueueConcurrencyLimit(
db: PrismaClientOrTransaction,
environment: AuthenticatedEnvironment,
queue: TaskQueue,
- concurrencyLimit: number,
+ override: ConcurrencyLimitOverride,
overriddenBy?: User
) {
- const newConcurrencyLimit = Math.max(
- Math.min(concurrencyLimit, environment.maximumConcurrencyLimit),
- 0
- );
+ const maximum = environment.maximumConcurrencyLimit;
+
+ // Normalize the input into the absolute limit to persist and the percent source-of-truth
+ // (null for absolute overrides).
+ let newConcurrencyLimit: number;
+ let overridePercent: number | null;
+
+ if (typeof override === "object" && "percent" in override) {
+ const percent = override.percent;
+
+ if (!isValidQueueOverridePercent(percent)) {
+ return errAsync({
+ type: "invalid_override" as const,
+ message: `Percent must be greater than ${MIN_QUEUE_OVERRIDE_PERCENT} and less than or equal to ${MAX_QUEUE_OVERRIDE_PERCENT}`,
+ });
+ }
+
+ newConcurrencyLimit = materializePercentLimit(maximum, percent);
+ overridePercent = percent;
+ } else {
+ const limit = typeof override === "number" ? override : override.limit;
+
+ if (!Number.isFinite(limit) || limit < 0) {
+ return errAsync({
+ type: "invalid_override" as const,
+ message: "Concurrency limit must be a non-negative number",
+ });
+ }
+
+ // Cap: an absolute override may not exceed the environment limit. Reject rather than clamp.
+ if (limit > maximum) {
+ return errAsync({
+ type: "concurrency_limit_exceeds_maximum" as const,
+ message: `Concurrency limit (${limit}) cannot exceed the environment limit (${maximum})`,
+ });
+ }
+
+ newConcurrencyLimit = limit;
+ overridePercent = null;
+ }
const concurrencyLimitBase = queue.concurrencyLimitOverriddenAt
? queue.concurrencyLimitBase
@@ -137,6 +257,7 @@ function overrideQueueConcurrencyLimit(
data: {
concurrencyLimit: newConcurrencyLimit,
concurrencyLimitBase: concurrencyLimitBase ?? null,
+ concurrencyLimitOverridePercent: overridePercent,
concurrencyLimitOverriddenAt: new Date(),
concurrencyLimitOverriddenBy: overriddenBy?.id ?? null,
},
@@ -162,6 +283,7 @@ function resetQueueConcurrencyLimit(db: PrismaClientOrTransaction, queue: TaskQu
concurrencyLimitOverriddenAt: null,
concurrencyLimit: newConcurrencyLimit,
concurrencyLimitBase: null,
+ concurrencyLimitOverridePercent: null,
concurrencyLimitOverriddenBy: null,
},
}),
diff --git a/apps/webapp/package.json b/apps/webapp/package.json
index 31b2bb9f831..365946310db 100644
--- a/apps/webapp/package.json
+++ b/apps/webapp/package.json
@@ -19,6 +19,7 @@
"guard:runops-legacy": "cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsx ./scripts/runOpsLegacyGuard.ts",
"db:seed": "tsx seed.ts",
"db:seed:ai-spans": "tsx seed-ai-spans.mts",
+ "db:seed:queue-metrics": "tsx seed-queue-metrics.mts",
"upload:sourcemaps": "bash ./upload-sourcemaps.sh",
"test": "vitest --no-file-parallelism",
"eval:dev": "evalite watch"
@@ -55,6 +56,7 @@
"@internal/dashboard-agent": "workspace:*",
"@internal/dashboard-agent-db": "workspace:*",
"@internal/llm-model-catalog": "workspace:*",
+ "@internal/metrics-pipeline": "workspace:*",
"@internal/redis": "workspace:*",
"@internal/run-engine": "workspace:*",
"@internal/run-ops-database": "workspace:*",
diff --git a/apps/webapp/seed-queue-metrics.mts b/apps/webapp/seed-queue-metrics.mts
new file mode 100644
index 00000000000..911ce51d9c6
--- /dev/null
+++ b/apps/webapp/seed-queue-metrics.mts
@@ -0,0 +1,991 @@
+import { prisma } from "./app/db.server";
+import { createOrganization } from "./app/models/organization.server";
+import { createProject } from "./app/models/project.server";
+import { ClickHouse } from "@internal/clickhouse";
+import type { QueueMetricsRawV1Input } from "@internal/clickhouse";
+import { generateFriendlyId } from "./app/v3/friendlyIdentifiers";
+
+// Queue metrics simulator: writes realistic raw rows into a synthetic tenant's
+// queue_metrics_raw_v1 and lets the MV build queue_metrics_v1 (the same path the real
+// consumer uses), so the dashboard can be built without the run engine. See TRI-10407.
+
+const ORG_TITLE = "Queue Metrics Dev";
+const PROJECT_NAME = "queue-metrics-demo";
+
+type Rng = () => number;
+type QueueProfile = {
+ name: string;
+ limit: (bucket: number) => number;
+ arrivals: (bucket: number, rng: Rng) => number; // expected new runs enqueued this bucket
+ waitBaseMs: number;
+ sparse?: boolean; // emit no rows when the queue is fully idle (tests carry-forward gaps)
+ // Concurrency-key queue: adds CK-health gauge fields + live ckIndex staging (--usage)
+ ck?: {
+ backlogged: (bucket: number, rng: Rng) => number;
+ maxWaitMs: (bucket: number, rng: Rng) => number;
+ };
+};
+type Scenario = {
+ description: string;
+ envLimit: (bucket: number) => number;
+ queues: QueueProfile[];
+};
+
+// ---------------------------------------------------------------------------
+// CLI args
+// ---------------------------------------------------------------------------
+
+function parseArgs(argv: string[]) {
+ const flags: Record = {};
+ for (let i = 0; i < argv.length; i++) {
+ const t = argv[i];
+ if (t.startsWith("--")) {
+ const k = t.slice(2);
+ const n = argv[i + 1];
+ if (n && !n.startsWith("--")) {
+ flags[k] = n;
+ i++;
+ } else flags[k] = "true";
+ }
+ }
+ return flags;
+}
+
+function parseDuration(s: string): number {
+ const m = s.match(/^(\d+)\s*(s|m|h|d)?$/);
+ if (!m) throw new Error(`bad duration: ${s}`);
+ const n = Number(m[1]);
+ const unit = m[2] ?? "s";
+ return n * { s: 1, m: 60, h: 3600, d: 86400 }[unit]!;
+}
+
+// ---------------------------------------------------------------------------
+// Deterministic RNG + distributions
+// ---------------------------------------------------------------------------
+
+function mulberry32(seed: number): Rng {
+ let a = seed >>> 0;
+ return () => {
+ a |= 0;
+ a = (a + 0x6d2b79f5) | 0;
+ let t = Math.imul(a ^ (a >>> 15), 1 | a);
+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+ };
+}
+
+function standardNormal(rng: Rng): number {
+ let u = 0;
+ let v = 0;
+ while (u === 0) u = rng();
+ while (v === 0) v = rng();
+ return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
+}
+
+function lognormal(medianMs: number, sigma: number, rng: Rng): number {
+ return Math.exp(Math.log(Math.max(medianMs, 1)) + sigma * standardNormal(rng));
+}
+
+function poisson(lambda: number, rng: Rng): number {
+ if (lambda <= 0) return 0;
+ if (lambda > 30) return Math.max(0, Math.round(lambda + standardNormal(rng) * Math.sqrt(lambda)));
+ const L = Math.exp(-lambda);
+ let k = 0;
+ let p = 1;
+ do {
+ k++;
+ p *= rng();
+ } while (p > L);
+ return k - 1;
+}
+
+function formatChDateTime(date: Date): string {
+ return date.toISOString().slice(0, 19).replace("T", " ");
+}
+
+// ---------------------------------------------------------------------------
+// Scenarios
+// ---------------------------------------------------------------------------
+
+const steady = (): QueueProfile[] => [
+ { name: "emails", limit: () => 20, arrivals: (_b, r) => poisson(12, r), waitBaseMs: 40 },
+ { name: "webhooks", limit: () => 15, arrivals: (_b, r) => poisson(9, r), waitBaseMs: 40 },
+ { name: "reports", limit: () => 10, arrivals: (_b, r) => poisson(5, r), waitBaseMs: 60 },
+];
+
+// periodic bursts every ~30 buckets
+const bursty = (name: string, limit: number, base: number): QueueProfile => ({
+ name,
+ limit: () => limit,
+ arrivals: (b, r) => poisson(b % 30 < 4 ? base * 5 : base, r),
+ waitBaseMs: 50,
+});
+
+const scenarios: Record Scenario> = {
+ steady: () => ({
+ description: "all queues below capacity, no throttling",
+ envLimit: () => 60,
+ queues: steady(),
+ }),
+
+ burst: () => ({
+ description: "periodic arrival bursts -> backlog + wait spikes + throttling",
+ envLimit: () => 60,
+ queues: [bursty("ingest", 20, 6), bursty("transform", 20, 7)],
+ }),
+
+ // Tela case: sum of per-queue limits far exceeds the env limit, so queues compete.
+ "over-allocated-env": () => ({
+ description: "Sum(queue limits)=120 >> env limit=40; env saturates, queues env-limited",
+ envLimit: () => 40,
+ queues: Array.from({ length: 6 }, (_v, i) => ({
+ name: `worker-${i + 1}`,
+ limit: () => 20,
+ arrivals: (_b: number, r: Rng) => poisson(14, r),
+ waitBaseMs: 50,
+ })),
+ }),
+
+ "single-queue-starves-others": () => ({
+ description: "one greedy queue consumes most of a small env limit, starving the rest",
+ envLimit: () => 30,
+ queues: [
+ { name: "greedy", limit: () => 40, arrivals: (_b, r) => poisson(45, r), waitBaseMs: 60 },
+ { name: "polite-1", limit: () => 10, arrivals: (_b, r) => poisson(6, r), waitBaseMs: 50 },
+ { name: "polite-2", limit: () => 10, arrivals: (_b, r) => poisson(6, r), waitBaseMs: 50 },
+ ],
+ }),
+
+ "throttled-backlog": () => ({
+ description:
+ "arrival rate persistently above the queue limit -> permanent backlog + throttling",
+ envLimit: () => 50,
+ queues: [
+ { name: "overloaded", limit: () => 10, arrivals: (_b, r) => poisson(16, r), waitBaseMs: 80 },
+ ],
+ }),
+
+ "idle-sparse": () => ({
+ description: "sparse arrivals with many empty buckets (carry-forward gaps)",
+ envLimit: () => 50,
+ queues: Array.from({ length: 4 }, (_v, i) => ({
+ name: `sparse-${i + 1}`,
+ limit: () => 5,
+ arrivals: (_b: number, r: Rng) => (r() < 0.12 ? poisson(3, r) : 0),
+ waitBaseMs: 30,
+ sparse: true,
+ })),
+ }),
+
+ "spike-then-drain": (totalBuckets) => ({
+ description: "heavy arrivals for the first third, then zero; backlog builds then drains",
+ envLimit: () => 60,
+ queues: [
+ {
+ name: "batch-job",
+ limit: () => 15,
+ arrivals: (b, r) => (b < totalBuckets / 3 ? poisson(30, r) : 0),
+ waitBaseMs: 70,
+ },
+ ],
+ }),
+
+ // Pagination + relevance-ranking design surface: one runaway queue, a busy-but-healthy
+ // head, a bursty middle, and a long sparse tail across 61 queues (the list pages at 25).
+ "many-queues": () => ({
+ description:
+ "61 queues: one runaway, busy head, bursty middle, long sparse tail (pagination + ranking)",
+ envLimit: () => 150,
+ queues: [
+ { name: "imports", limit: () => 8, arrivals: (_b, r) => poisson(14, r), waitBaseMs: 80 },
+ ...["checkout", "notifications", "emails"].map((name, i) => ({
+ name,
+ limit: () => 15,
+ arrivals: (_b: number, r: Rng) => poisson(7 + i, r),
+ waitBaseMs: 60,
+ })),
+ ...Array.from({ length: 12 }, (_v, i) =>
+ bursty(`service-${String(i + 1).padStart(2, "0")}`, 10, 2)
+ ),
+ ...Array.from({ length: 20 }, (_v, i) => ({
+ name: `job-${String(i + 1).padStart(2, "0")}`,
+ limit: () => 5,
+ arrivals: (_b: number, r: Rng) => poisson(1, r),
+ waitBaseMs: 40,
+ })),
+ ...Array.from({ length: 25 }, (_v, i) => ({
+ name: `tenant-${String(i + 1).padStart(2, "0")}`,
+ limit: () => 3,
+ arrivals: (_b: number, r: Rng) => (r() < 0.05 ? poisson(2, r) : 0),
+ waitBaseMs: 30,
+ sparse: true,
+ })),
+ ],
+ }),
+
+ // Per-tenant concurrency keys: a hog tenant periodically floods the queue and starves
+ // the others, so the CK charts (keys with backlog, most-starved wait) and the live
+ // per-key table on the queue detail page have something to show. Use with --usage.
+ "tenant-hotspot": () => ({
+ description:
+ "CK queue where a hog tenant starves others: CK charts + live key table (use --usage)",
+ envLimit: () => 40,
+ queues: [
+ {
+ name: "per-tenant",
+ limit: () => 10,
+ arrivals: (b, r) => poisson(b % 60 < 20 ? 25 : 8, r),
+ waitBaseMs: 60,
+ ck: {
+ backlogged: (b, r) => (b % 60 < 20 ? 6 + Math.round(r() * 6) : Math.round(r() * 3)),
+ maxWaitMs: (b, r) =>
+ b % 60 < 20
+ ? Math.round(lognormal(90_000, 0.5, r))
+ : Math.round(lognormal(3_000, 0.6, r)),
+ },
+ },
+ { name: "background", limit: () => 10, arrivals: (_b, r) => poisson(5, r), waitBaseMs: 40 },
+ ],
+ }),
+
+ // Default: one env with a variety of queue behaviours + occasional env saturation.
+ mixed: (totalBuckets) => ({
+ description: "variety of queue profiles in one env, with occasional env saturation",
+ envLimit: (b) => (b % 40 < 12 ? 45 : 70), // dips low periodically to flip env saturation
+ queues: [
+ { name: "emails", limit: () => 20, arrivals: (_b, r) => poisson(12, r), waitBaseMs: 40 },
+ bursty("webhooks", 20, 6),
+ { name: "reports", limit: () => 10, arrivals: (_b, r) => poisson(8, r), waitBaseMs: 80 },
+ {
+ name: "cleanup",
+ limit: () => 5,
+ arrivals: (_b, r) => (r() < 0.12 ? poisson(3, r) : 0),
+ waitBaseMs: 30,
+ sparse: true,
+ },
+ {
+ name: "nightly-batch",
+ limit: () => 15,
+ arrivals: (b, r) => (b < totalBuckets / 5 ? poisson(18, r) : 0),
+ waitBaseMs: 70,
+ },
+ ],
+ }),
+};
+
+// ---------------------------------------------------------------------------
+// Simulation
+// ---------------------------------------------------------------------------
+
+type Ids = { organization_id: string; project_id: string; environment_id: string };
+const WAIT_SIGMA = 0.6;
+const NACK_RATE = 0.02;
+const DLQ_RATE = 0.004;
+
+type CounterOp = "enqueue" | "started" | "ack" | "nack" | "dlq";
+// Per-(queue, op) odometers, mirroring the production emitter: cumulative readings with a
+// cum=0 baseline on the first one, so deltaSumTimestamp captures the 0->1 delta.
+type CounterState = Record[];
+
+function counterRows(
+ counters: CounterState,
+ q: number,
+ ids: Ids,
+ queueName: string,
+ eventTime: string,
+ orderKey: () => number,
+ op: CounterOp,
+ wait_ms?: number
+): QueueMetricsRawV1Input[] {
+ const rows: QueueMetricsRawV1Input[] = [];
+ if (counters[q][op] === 0) {
+ rows.push({
+ ...ids,
+ queue_name: queueName,
+ event_time: eventTime,
+ op,
+ cumulative: 0,
+ order_key: orderKey(),
+ });
+ }
+ counters[q][op] += 1;
+ rows.push({
+ ...ids,
+ queue_name: queueName,
+ event_time: eventTime,
+ op,
+ cumulative: counters[q][op],
+ order_key: orderKey(),
+ ...(wait_ms !== undefined ? { wait_ms } : {}),
+ });
+ return rows;
+}
+
+function newCounterState(n: number): CounterState {
+ return Array.from({ length: n }, () => ({ enqueue: 0, started: 0, ack: 0, nack: 0, dlq: 0 }));
+}
+
+// Per-key simulation for CK profiles: 12 tenants (tenant-01 is the hog, matching
+// stageRedisUsage), per-tenant backlog drained round-robin, per-tenant odometers.
+const CK_TENANT_COUNT = 12;
+type CkSimState = { backlog: number[]; counters: Map> };
+const ckSim = new Map();
+
+function ckTenantName(t: number): string {
+ return `tenant-${String(t + 1).padStart(2, "0")}`;
+}
+
+function ckCounterRows(
+ state: CkSimState,
+ tenant: number,
+ ids: Ids,
+ queueName: string,
+ eventTime: string,
+ orderKey: () => number,
+ op: CounterOp,
+ wait_ms?: number
+): QueueMetricsRawV1Input[] {
+ let c = state.counters.get(tenant);
+ if (!c) {
+ c = { enqueue: 0, started: 0, ack: 0, nack: 0, dlq: 0 };
+ state.counters.set(tenant, c);
+ }
+ const common = {
+ ...ids,
+ queue_name: queueName,
+ concurrency_key: ckTenantName(tenant),
+ event_time: eventTime,
+ };
+ const rows: QueueMetricsRawV1Input[] = [];
+ if (c[op] === 0) rows.push({ ...common, op, cumulative: 0, order_key: orderKey() });
+ c[op] += 1;
+ rows.push({
+ ...common,
+ op,
+ cumulative: c[op],
+ order_key: orderKey(),
+ ...(wait_ms !== undefined ? { wait_ms } : {}),
+ });
+ return rows;
+}
+
+// Advance one bucket of the simulation for every queue, returning the raw rows to insert.
+// `backlog` and `counters` are mutated in place so state carries across buckets (and into
+// live mode).
+function simulateBucket(
+ scenario: Scenario,
+ bucket: number,
+ bucketSec: number,
+ eventTime: string,
+ bucketEpochSec: number,
+ ids: Ids,
+ backlog: number[],
+ counters: CounterState,
+ rng: Rng
+): QueueMetricsRawV1Input[] {
+ const envLimit = scenario.envLimit(bucket);
+ const n = scenario.queues.length;
+
+ const limit = new Array(n);
+ const desired = new Array(n);
+ for (let q = 0; q < n; q++) {
+ limit[q] = scenario.queues[q].limit(bucket);
+ const arrivals = Math.min(500, scenario.queues[q].arrivals(bucket, rng));
+ const prior = backlog[q]; // backlog carried from earlier buckets, before this bucket's arrivals
+ backlog[q] += arrivals; // arrivals join the backlog; recorded as enqueues below
+ (desired as any)[q] = { arrivals, prior, want: Math.min(limit[q], backlog[q]) };
+ }
+
+ // Env cap: if the queues collectively want more concurrency than the env allows, scale down.
+ const sumWant = desired.reduce((s: number, d: any) => s + d.want, 0);
+ const scale = sumWant > envLimit && sumWant > 0 ? envLimit / sumWant : 1;
+
+ const running = new Array(n);
+ const queued = new Array(n);
+ let envRunning = 0;
+ let envQueued = 0;
+ for (let q = 0; q < n; q++) {
+ const d = desired[q] as any;
+ running[q] = Math.floor(d.want * scale);
+ queued[q] = backlog[q] - running[q];
+ envRunning += running[q];
+ envQueued += queued[q];
+ }
+
+ // Order keys are time-based (like the production stream ids) so appended runs and live
+ // mode stay monotonic; the per-bucket sequence keeps them unique within a bucket.
+ let bucketSeq = 0;
+ const orderKey = () => bucketEpochSec * 1_000_000 + bucketSeq++;
+
+ const rows: QueueMetricsRawV1Input[] = [];
+ for (let q = 0; q < n; q++) {
+ const profile = scenario.queues[q];
+ const started = running[q];
+ const arrivals = (desired[q] as any).arrivals as number;
+ const prior = (desired[q] as any).prior as number; // depth a starting run actually queued behind
+ backlog[q] = queued[q]; // carry the unserved remainder forward
+
+ if (profile.sparse && arrivals === 0 && started === 0 && prior === 0) {
+ continue; // fully idle: leave a gap so carry-forward is exercised
+ }
+
+ // CK-health fields stay coherent with the depth: no queued runs means no backlogged keys.
+ const ckBacklogged = profile.ck
+ ? queued[q] > 0
+ ? Math.max(1, Math.min(profile.ck.backlogged(bucket, rng), queued[q]))
+ : 0
+ : undefined;
+ const ckMaxWaitMs =
+ profile.ck && ckBacklogged ? Math.round(profile.ck.maxWaitMs(bucket, rng)) : undefined;
+
+ const gauge: QueueMetricsRawV1Input = {
+ ...ids,
+ queue_name: profile.name,
+ event_time: eventTime,
+ op: "gauge",
+ running: running[q],
+ queued: queued[q],
+ queue_limit: limit[q],
+ env_running: envRunning,
+ env_queued: envQueued,
+ env_limit: envLimit,
+ throttled: queued[q] > 0 && (running[q] >= limit[q] || scale < 1) ? 1 : 0,
+ ...(ckBacklogged !== undefined
+ ? { ck_backlogged: ckBacklogged, ck_max_wait_ms: ckMaxWaitMs ?? 0 }
+ : {}),
+ };
+ rows.push(gauge);
+
+ for (let a = 0; a < arrivals; a++) {
+ rows.push(...counterRows(counters, q, ids, profile.name, eventTime, orderKey, "enqueue"));
+ }
+
+ // Per-key rows for CK profiles: assign arrivals hog-weighted, drain round-robin
+ // (fair share), then emit per-tenant odometers + a per-key gauge per active tenant.
+ if (profile.ck) {
+ let ckq = ckSim.get(q);
+ if (!ckq) {
+ ckq = { backlog: new Array(CK_TENANT_COUNT).fill(0), counters: new Map() };
+ ckSim.set(q, ckq);
+ }
+ const hogShare = bucket % 60 < 20 ? 0.6 : 0.15;
+ const arrivalsPerTenant = new Array(CK_TENANT_COUNT).fill(0);
+ for (let a = 0; a < arrivals; a++) {
+ const t = rng() < hogShare ? 0 : 1 + Math.floor(rng() * (CK_TENANT_COUNT - 1));
+ arrivalsPerTenant[t]++;
+ ckq.backlog[t]++;
+ }
+ const drainedPerTenant = new Array(CK_TENANT_COUNT).fill(0);
+ let remaining = started;
+ while (remaining > 0 && ckq.backlog.some((v) => v > 0)) {
+ for (let t = 0; t < CK_TENANT_COUNT && remaining > 0; t++) {
+ if (ckq.backlog[t] > 0) {
+ ckq.backlog[t]--;
+ drainedPerTenant[t]++;
+ remaining--;
+ }
+ }
+ }
+ for (let t = 0; t < CK_TENANT_COUNT; t++) {
+ const fairShare = Math.max(1, limit[q] / CK_TENANT_COUNT);
+ const ckMedianWait = profile.waitBaseMs + (ckq.backlog[t] / fairShare) * bucketSec * 1000;
+ for (let a = 0; a < arrivalsPerTenant[t]; a++) {
+ rows.push(...ckCounterRows(ckq, t, ids, profile.name, eventTime, orderKey, "enqueue"));
+ }
+ for (let d = 0; d < drainedPerTenant[t]; d++) {
+ rows.push(
+ ...ckCounterRows(
+ ckq,
+ t,
+ ids,
+ profile.name,
+ eventTime,
+ orderKey,
+ "started",
+ Math.round(lognormal(ckMedianWait, WAIT_SIGMA, rng))
+ )
+ );
+ rows.push(...ckCounterRows(ckq, t, ids, profile.name, eventTime, orderKey, "ack"));
+ }
+ if (ckq.backlog[t] > 0 || drainedPerTenant[t] > 0) {
+ rows.push({
+ ...ids,
+ queue_name: profile.name,
+ concurrency_key: ckTenantName(t),
+ event_time: eventTime,
+ op: "gauge",
+ queued: ckq.backlog[t],
+ running: drainedPerTenant[t],
+ });
+ }
+ }
+ }
+
+ const medianWait = profile.waitBaseMs + (prior / Math.max(limit[q], 1)) * bucketSec * 1000;
+ for (let s = 0; s < started; s++) {
+ rows.push(
+ ...counterRows(
+ counters,
+ q,
+ ids,
+ profile.name,
+ eventTime,
+ orderKey,
+ "started",
+ Math.round(lognormal(medianWait, WAIT_SIGMA, rng))
+ )
+ );
+ const roll = rng();
+ const op: CounterOp = roll < DLQ_RATE ? "dlq" : roll < DLQ_RATE + NACK_RATE ? "nack" : "ack";
+ rows.push(...counterRows(counters, q, ids, profile.name, eventTime, orderKey, op));
+ }
+ }
+ return rows;
+}
+
+// ---------------------------------------------------------------------------
+// ClickHouse
+// ---------------------------------------------------------------------------
+
+function clickhouse(): ClickHouse {
+ const clickhouseUrl = process.env.CLICKHOUSE_URL ?? process.env.EVENTS_CLICKHOUSE_URL;
+ if (!clickhouseUrl) {
+ console.error("CLICKHOUSE_URL not set");
+ process.exit(1);
+ }
+ const url = new URL(clickhouseUrl);
+ // Allowlist local hosts only (this script TRUNCATEs), and never echo the URL (it carries creds).
+ const localHosts = new Set(["localhost", "127.0.0.1", "::1", "0.0.0.0"]);
+ if (!localHosts.has(url.hostname)) {
+ console.error(`Refusing to run against a non-local ClickHouse host: ${url.hostname}`);
+ process.exit(1);
+ }
+ url.searchParams.delete("secure");
+ return new ClickHouse({ url: url.toString(), name: "queue-metrics-simulator" });
+}
+
+async function insertBatched(ch: ClickHouse, rows: QueueMetricsRawV1Input[], nonce: string) {
+ const BATCH = 25_000;
+ for (let i = 0; i < rows.length; i += BATCH) {
+ const slice = rows.slice(i, i + BATCH);
+ const [error] = await ch.queueMetrics.insertRaw(slice, {
+ params: { clickhouse_settings: { insert_deduplication_token: `${nonce}:${i}` } },
+ });
+ if (error) {
+ console.error("insert failed:", error.message);
+ process.exit(1);
+ }
+ }
+}
+
+async function resetEnv(ch: ClickHouse, environmentId: string) {
+ const raw = (
+ ch.writer as unknown as { client: { command: (a: { query: string }) => Promise } }
+ ).client;
+ for (const table of [
+ "queue_metrics_raw_v1",
+ "queue_metrics_v1",
+ "queue_metrics_5m_v1",
+ "env_metrics_v1",
+ "queue_metrics_ck_v1",
+ ]) {
+ await raw.command({
+ query: `DELETE FROM trigger_dev.${table} WHERE environment_id = '${environmentId}'`,
+ });
+ }
+ console.log(`Reset queue metrics for environment ${environmentId}`);
+}
+
+// Fake running counts in the run-queue Redis (Running column + allocation usage bars).
+// Reconciled every run: staged with --usage, cleared otherwise.
+async function stageRedisUsage(scenario: Scenario, ids: Ids, seed: number, clear: boolean) {
+ const host = process.env.RUN_ENGINE_RUN_QUEUE_REDIS_HOST ?? process.env.REDIS_HOST ?? "localhost";
+ const port = Number(
+ process.env.RUN_ENGINE_RUN_QUEUE_REDIS_PORT ?? process.env.REDIS_PORT ?? 6379
+ );
+ const localHosts = new Set(["localhost", "127.0.0.1", "::1", "0.0.0.0"]);
+ if (!localHosts.has(host)) {
+ console.warn(`Skipping Redis usage staging on a non-local host: ${host}`);
+ return;
+ }
+ try {
+ const { createRedisClient } = await import("@internal/redis");
+ const redis = createRedisClient({ host, port });
+ const rng = mulberry32(seed + 1);
+ const prefix = "engine:runqueue:";
+ const logicalBase = `{org:${ids.organization_id}}:proj:${ids.project_id}:env:${ids.environment_id}:queue:`;
+ const base = `${prefix}${logicalBase}`;
+
+ // Env-level structures the Queues list "Queued"/"Running" blocks read:
+ // lengthOfEnvQueue -> ZCARD(envQueueKey) (ZSET, no proj section)
+ // concurrencyOfEnvQueue -> SCARD(envCurrentDequeuedKey) (SET)
+ // We accumulate the per-queue staged counts below and stage these so the blocks
+ // equal the table's per-queue sums instead of showing 0/0.
+ const envQueueKey = `${prefix}{org:${ids.organization_id}}:env:${ids.environment_id}`;
+ const envCurrentDequeuedKey = `${prefix}{org:${ids.organization_id}}:proj:${ids.project_id}:env:${ids.environment_id}:currentDequeued`;
+ await redis.del(envQueueKey, envCurrentDequeuedKey);
+ // Table Queued = ZCARD(base) + lengthCounter (base zset unstaged -> only CK queues
+ // contribute their lengthCounter). Table Running = SCARD(currentDequeued) per queue.
+ let envQueuedTotal = 0;
+ let envRunningTotal = 0;
+
+ for (const [q, profile] of scenario.queues.entries()) {
+ const key = `${base}${profile.name}:currentDequeued`;
+ await redis.del(key);
+
+ // CK staging (ckIndex + per-key subqueues) feeds the live per-key table on the queue
+ // detail page. Members are stored unprefixed, exactly like the run-queue Lua does.
+ const ckIndexKey = `${base}${profile.name}:ckIndex`;
+ const lengthCounterKey = `${base}${profile.name}:lengthCounter`;
+ const staleMembers = await redis.zrange(ckIndexKey, 0, -1);
+ for (const member of staleMembers) {
+ await redis.del(`${prefix}${member}`, `${prefix}${member}:currentConcurrency`);
+ }
+ await redis.del(ckIndexKey, lengthCounterKey);
+
+ if (clear) continue;
+ const limit = profile.limit(0);
+ // First queue rides at/over its limit, the rest at 30-90%, sparse mostly idle.
+ const count = profile.sparse
+ ? rng() < 0.3
+ ? 1
+ : 0
+ : q === 0
+ ? limit + Math.round(rng() * 2)
+ : Math.round(limit * (0.3 + 0.6 * rng()));
+ if (count > 0) {
+ await redis.sadd(key, ...Array.from({ length: count }, (_v, i) => `sim_run_${i}`));
+ }
+ envRunningTotal += count;
+
+ if (profile.ck) {
+ const now = Date.now();
+ const tenants = 12;
+ let totalCkQueued = 0;
+ for (let t = 1; t <= tenants; t++) {
+ const tenant = `tenant-${String(t).padStart(2, "0")}`;
+ const member = `${logicalBase}${profile.name}:ck:${tenant}`;
+ const hog = t === 1;
+ const queuedCount = hog ? 40 : 1 + Math.round(rng() * 5);
+ const runningCount = hog ? limit : Math.round(rng() * 2);
+ const oldestAgeMs = hog ? 15 * 60_000 : 5_000 + Math.round(rng() * 55_000);
+ const zargs: Array = [];
+ for (let i = 0; i < queuedCount; i++) {
+ zargs.push(now - oldestAgeMs + i * 250, `sim_${tenant}_run_${i}`);
+ }
+ await redis.zadd(`${prefix}${member}`, ...zargs);
+ if (runningCount > 0) {
+ await redis.sadd(
+ `${prefix}${member}:currentConcurrency`,
+ ...Array.from({ length: runningCount }, (_v, i) => `sim_${tenant}_running_${i}`)
+ );
+ }
+ await redis.zadd(ckIndexKey, now - oldestAgeMs, member);
+ totalCkQueued += queuedCount;
+ }
+ // The aggregate "Queued now" reads ZCARD(base) + this counter; keep them coherent.
+ await redis.set(lengthCounterKey, totalCkQueued, "EX", 24 * 3600);
+ envQueuedTotal += totalCkQueued;
+ }
+ }
+
+ // Stage the env-level structures so the list-page blocks match the table sums.
+ // Members are unique across queues (each per-queue set uses its own key, but the
+ // env set/zset needs distinct members to reach the summed cardinality).
+ if (!clear) {
+ if (envRunningTotal > 0) {
+ await redis.sadd(
+ envCurrentDequeuedKey,
+ ...Array.from({ length: envRunningTotal }, (_v, i) => `sim_env_run_${i}`)
+ );
+ }
+ if (envQueuedTotal > 0) {
+ const now = Date.now();
+ const zargs: Array = [];
+ for (let i = 0; i < envQueuedTotal; i++) zargs.push(now + i, `sim_env_queued_${i}`);
+ await redis.zadd(envQueueKey, ...zargs);
+ }
+ }
+
+ await redis.quit();
+ console.log(
+ clear
+ ? "Cleared staged Redis usage."
+ : "Staged fake running counts in Redis (Running column + allocation usage bars)."
+ );
+ } catch (error) {
+ console.warn("Redis usage staging skipped:", error instanceof Error ? error.message : error);
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+// Make the synthetic project a V2 engine project with a current dev worker + a Postgres
+// TaskQueue per simulated queue, so the /queues list renders the V2 table (it pages from
+// Postgres and gates on engine version; ClickHouse only holds the metrics).
+async function ensureTaskQueues(
+ scenario: Scenario,
+ projectId: string,
+ runtimeEnvironmentId: string
+) {
+ await prisma.project.update({ where: { id: projectId }, data: { engine: "V2" } });
+
+ await prisma.backgroundWorker.upsert({
+ where: {
+ projectId_runtimeEnvironmentId_version: {
+ projectId,
+ runtimeEnvironmentId,
+ version: "queue-metrics-sim",
+ },
+ },
+ update: {},
+ create: {
+ friendlyId: generateFriendlyId("worker"),
+ engine: "V2",
+ contentHash: "queue-metrics-sim",
+ sdkVersion: "4.0.0",
+ cliVersion: "4.0.0",
+ projectId,
+ runtimeEnvironmentId,
+ version: "queue-metrics-sim",
+ metadata: {},
+ },
+ });
+
+ for (const profile of scenario.queues) {
+ const concurrencyLimit = profile.limit(0);
+ await prisma.taskQueue.upsert({
+ where: { runtimeEnvironmentId_name: { runtimeEnvironmentId, name: profile.name } },
+ create: {
+ friendlyId: generateFriendlyId("queue"),
+ version: "V2",
+ name: profile.name,
+ orderableName: profile.name,
+ concurrencyLimit,
+ runtimeEnvironmentId,
+ projectId,
+ type: "NAMED",
+ },
+ // Reset any dashboard override left from manual testing: re-seeding overwrites the
+ // materialized concurrencyLimit, so a surviving override percent/base would contradict it
+ // (e.g. "10 (77%)" with an env limit of 25).
+ update: {
+ concurrencyLimit,
+ concurrencyLimitBase: null,
+ concurrencyLimitOverridePercent: null,
+ concurrencyLimitOverriddenAt: null,
+ concurrencyLimitOverriddenBy: null,
+ },
+ });
+ }
+
+ // Drop queues left over from a previously seeded scenario so switching scenarios
+ // does not leave metric-less rows in the list.
+ const { count: pruned } = await prisma.taskQueue.deleteMany({
+ where: {
+ runtimeEnvironmentId,
+ name: { notIn: scenario.queues.map((q) => q.name) },
+ },
+ });
+ console.log(
+ `Ensured ${scenario.queues.length} task queues in Postgres${pruned > 0 ? `, pruned ${pruned} stale` : ""}.`
+ );
+}
+
+function printHelp() {
+ const lines = Object.entries(scenarios).map(
+ ([name, build]) => ` ${name.padEnd(28)}${build(720, 10).description}`
+ );
+ console.log(`Queue metrics simulator: seeds a synthetic tenant with realistic queue metrics.
+
+Usage: pnpm --filter webapp run db:seed:queue-metrics -- [flags]
+
+Flags:
+ --scenario which scenario to seed (default: mixed)
+ --project project to seed into (default: ${PROJECT_NAME}); use one
+ project per scenario to browse them side by side
+ --window how much history to backfill, e.g. 30m, 6h, 1d (default: 2h)
+ --bucket seconds per simulated bucket (default: 10)
+ --seed RNG seed for reproducible data (default: 1)
+ --usage stage fake running counts in Redis so the Running column and
+ the Allocation tab's usage bars have data (cleared when omitted)
+ --live after backfilling, keep appending one bucket per interval
+ --reset clear this environment's metrics before seeding
+ --reset-only clear and exit without seeding
+ --help this text
+
+Scenarios:
+${lines.join("\n")}
+
+Example designer setup (one project per scenario):
+ pnpm --filter webapp run db:seed:queue-metrics -- --scenario mixed --reset
+ pnpm --filter webapp run db:seed:queue-metrics -- --scenario many-queues --project qm-many-queues --reset
+ pnpm --filter webapp run db:seed:queue-metrics -- --scenario throttled-backlog --project qm-throttled --reset
+ pnpm --filter webapp run db:seed:queue-metrics -- --scenario tenant-hotspot --project qm-tenants --usage --reset`);
+}
+
+async function main() {
+ const flags = parseArgs(process.argv.slice(2));
+ if (flags.help === "true") {
+ printHelp();
+ process.exit(0);
+ }
+ const scenarioName = flags.scenario ?? "mixed";
+ const build = scenarios[scenarioName];
+ if (!build) {
+ console.error(
+ `Unknown scenario "${scenarioName}". Options: ${Object.keys(scenarios).join(", ")}`
+ );
+ process.exit(1);
+ }
+ const bucketSec = Number(flags.bucket ?? 10);
+ if (!Number.isFinite(bucketSec) || bucketSec <= 0) {
+ console.error(`--bucket must be a positive number of seconds, got: ${flags.bucket}`);
+ process.exit(1);
+ }
+ const windowSec = parseDuration(flags.window ?? "2h");
+ const totalBuckets = Math.floor(windowSec / bucketSec);
+ if (!Number.isFinite(totalBuckets) || totalBuckets <= 0) {
+ console.error(
+ `--window must be longer than --bucket (got ${windowSec}s window, ${bucketSec}s bucket)`
+ );
+ process.exit(1);
+ }
+ const seed = Number(flags.seed ?? 1);
+ const live = flags.live === "true";
+
+ const user = await prisma.user.findUnique({ where: { email: "local@trigger.dev" } });
+ if (!user) {
+ console.error("User local@trigger.dev not found. Run `pnpm run db:seed` first.");
+ process.exit(1);
+ }
+
+ let org = await prisma.organization.findFirst({
+ where: { title: ORG_TITLE, members: { some: { userId: user.id } } },
+ });
+ if (!org)
+ org = await createOrganization({ title: ORG_TITLE, userId: user.id, companySize: "1-10" });
+
+ const projectName = flags.project ?? PROJECT_NAME;
+ let project = await prisma.project.findFirst({
+ where: { name: projectName, organizationId: org.id },
+ });
+ if (!project) {
+ project = await createProject({
+ organizationSlug: org.slug,
+ name: projectName,
+ userId: user.id,
+ version: "v3",
+ });
+ }
+
+ const runtimeEnv = await prisma.runtimeEnvironment.findFirst({
+ where: { projectId: project.id, type: "DEVELOPMENT" },
+ });
+ if (!runtimeEnv) {
+ console.error("No DEVELOPMENT environment found for project.");
+ process.exit(1);
+ }
+
+ const ids: Ids = {
+ organization_id: org.id,
+ project_id: project.id,
+ environment_id: runtimeEnv.id,
+ };
+ const ch = clickhouse();
+ const nonce = `qmsim-${Date.now()}-${seed}`;
+
+ if (flags.reset === "true" || flags["reset-only"] === "true") {
+ await resetEnv(ch, runtimeEnv.id);
+ if (flags["reset-only"] === "true") {
+ await ch.close();
+ process.exit(0);
+ }
+ }
+
+ const scenario = build(totalBuckets, bucketSec);
+ await ensureTaskQueues(scenario, project.id, runtimeEnv.id);
+ await stageRedisUsage(scenario, ids, seed, flags.usage !== "true");
+ const rng = mulberry32(seed);
+ const backlog = new Array(scenario.queues.length).fill(0);
+
+ console.log(`Scenario "${scenarioName}": ${scenario.description}`);
+ console.log(
+ `Backfilling ${totalBuckets} x ${bucketSec}s buckets (${flags.window ?? "2h"}) for ${scenario.queues.length} queues...`
+ );
+
+ // Backfill: buckets from (now - window) up to now, aligned to the bucket grid.
+ const nowBucket = Math.floor(Date.now() / 1000 / bucketSec) * bucketSec;
+ const startBucket = nowBucket - totalBuckets * bucketSec;
+ const counters = newCounterState(scenario.queues.length);
+ const rows: QueueMetricsRawV1Input[] = [];
+ for (let b = 0; b < totalBuckets; b++) {
+ const bucketEpochSec = startBucket + b * bucketSec;
+ const eventTime = formatChDateTime(new Date(bucketEpochSec * 1000));
+ rows.push(
+ ...simulateBucket(
+ scenario,
+ b,
+ bucketSec,
+ eventTime,
+ bucketEpochSec,
+ ids,
+ backlog,
+ counters,
+ rng
+ )
+ );
+ }
+ await insertBatched(ch, rows, nonce);
+ console.log(`Inserted ${rows.length} raw rows.`);
+
+ // Merge the AggregatingMergeTree partials so argMax "current value" widgets read cleanly.
+ // The real pipeline relies on background merges; the simulator forces it for a tidy demo.
+ const raw = (
+ ch.writer as unknown as { client: { command: (a: { query: string }) => Promise } }
+ ).client;
+ await raw.command({ query: `OPTIMIZE TABLE trigger_dev.queue_metrics_v1 FINAL` });
+ await raw.command({ query: `OPTIMIZE TABLE trigger_dev.queue_metrics_5m_v1 FINAL` });
+ await raw.command({ query: `OPTIMIZE TABLE trigger_dev.env_metrics_v1 FINAL` });
+ await raw.command({ query: `OPTIMIZE TABLE trigger_dev.queue_metrics_ck_v1 FINAL` });
+
+ const origin = process.env.APP_ORIGIN ?? "http://localhost:3030";
+ console.log(
+ `\nQueues dashboard: ${origin}/orgs/${org.slug}/projects/${project.slug}/env/dev/dashboards/queues`
+ );
+
+ if (live) {
+ console.log(`\nLive mode: appending one bucket every ${bucketSec}s (Ctrl-C to stop)...`);
+ let b = totalBuckets;
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ await new Promise((r) => setTimeout(r, bucketSec * 1000));
+ const bucketEpochSec = Math.floor(Date.now() / 1000 / bucketSec) * bucketSec;
+ const eventTime = formatChDateTime(new Date(bucketEpochSec * 1000));
+ const liveRows = simulateBucket(
+ scenario,
+ b,
+ bucketSec,
+ eventTime,
+ bucketEpochSec,
+ ids,
+ backlog,
+ counters,
+ rng
+ );
+ await insertBatched(ch, liveRows, `${nonce}:live:${b}`);
+ console.log(`bucket ${b}: ${liveRows.length} rows @ ${eventTime}`);
+ b++;
+ }
+ }
+
+ await ch.close();
+ process.exit(0);
+}
+
+main().catch((e) => {
+ console.error(e);
+ process.exit(1);
+});
diff --git a/apps/webapp/test/concurrencySystemPercentOverride.test.ts b/apps/webapp/test/concurrencySystemPercentOverride.test.ts
new file mode 100644
index 00000000000..f81f1b1d10b
--- /dev/null
+++ b/apps/webapp/test/concurrencySystemPercentOverride.test.ts
@@ -0,0 +1,370 @@
+import { postgresTest } from "@internal/testcontainers";
+import type { PrismaClient } from "@trigger.dev/database";
+import { describe, expect, it, vi } from "vitest";
+import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { ConcurrencySystem, materializePercentLimit } from "~/v3/services/concurrencySystem.server";
+
+// The real run engine opens eager Redis connections and needs the whole engine
+// wired up. These tests exercise the DB-write + recalculation logic against a
+// real Postgres (postgresTest), so we replace the engine singleton with a thin
+// stub that satisfies the sync + stats calls the service makes. The engine sync
+// itself (RunQueue/Redis) is therefore NOT exercised here — see the note in the
+// verification report. Everything the service persists to Postgres IS real.
+// A controllable spy for the engine's queue-limit push so a test can simulate a push failure and
+// prove the next recalc self-heals the divergence.
+// This mocks the ENGINE method (`engine.runQueue.updateQueueConcurrencyLimits`). The service calls
+// the top-level `updateQueueConcurrencyLimits` from `~/v3/runQueue.server`, which forwards straight
+// to this engine method with the same `(environment, queueName, concurrency)` argument order (no
+// transformation) — so mocking the engine method genuinely exercises the recalc's sync path and the
+// `(env, queueName, 10)` assertion below is exact.
+const { updateQueueConcurrencyLimitsMock } = vi.hoisted(() => ({
+ updateQueueConcurrencyLimitsMock: vi.fn(async (..._args: unknown[]) => undefined),
+}));
+
+vi.mock("~/v3/runEngine.server", () => ({
+ engine: {
+ lengthOfQueues: async () => ({}),
+ currentConcurrencyOfQueues: async () => ({}),
+ runQueue: {
+ updateQueueConcurrencyLimits: updateQueueConcurrencyLimitsMock,
+ removeQueueConcurrencyLimits: async () => undefined,
+ updateEnvConcurrencyLimits: async () => undefined,
+ },
+ },
+}));
+
+vi.setConfig({ testTimeout: 30_000 });
+
+describe("materializePercentLimit", () => {
+ it("floors the materialized value", () => {
+ // 10 * 55 / 100 = 5.5 -> floor -> 5
+ expect(materializePercentLimit(10, 55)).toBe(5);
+ // 7 * 50 / 100 = 3.5 -> floor -> 3
+ expect(materializePercentLimit(7, 50)).toBe(3);
+ });
+
+ it("clamps to at least 1 so a percent override never produces a 0 (pause-like) limit", () => {
+ // 10 * 1 / 100 = 0.1 -> floor -> 0 -> clamp up to 1
+ expect(materializePercentLimit(10, 1)).toBe(1);
+ // tiny env, small percent still floors to 0 then clamps to 1
+ expect(materializePercentLimit(1, 1)).toBe(1);
+ expect(materializePercentLimit(3, 10)).toBe(1);
+ });
+
+ it("clamps to at most the environment limit at 100%", () => {
+ expect(materializePercentLimit(10, 100)).toBe(10);
+ expect(materializePercentLimit(1, 100)).toBe(1);
+ expect(materializePercentLimit(250, 100)).toBe(250);
+ });
+
+ it("handles a tiny environment limit at the 100% boundary", () => {
+ expect(materializePercentLimit(1, 50)).toBe(1); // 0.5 -> 0 -> clamp to 1
+ expect(materializePercentLimit(2, 50)).toBe(1); // 1.0 -> 1
+ expect(materializePercentLimit(2, 100)).toBe(2);
+ });
+
+ it("supports fractional percentages", () => {
+ // 100 * 12.5 / 100 = 12.5 -> floor -> 12
+ expect(materializePercentLimit(100, 12.5)).toBe(12);
+ });
+});
+
+async function seedEnvAndQueue(
+ prisma: PrismaClient,
+ opts: { maximumConcurrencyLimit: number; queueConcurrencyLimit?: number | null }
+) {
+ const slug = `s${Math.random().toString(36).slice(2, 10)}`;
+
+ const organization = await prisma.organization.create({
+ data: { title: slug, slug },
+ });
+
+ const project = await prisma.project.create({
+ data: { name: slug, slug, organizationId: organization.id, externalRef: slug },
+ });
+
+ const environment = await prisma.runtimeEnvironment.create({
+ data: {
+ slug,
+ type: "PRODUCTION",
+ projectId: project.id,
+ organizationId: organization.id,
+ apiKey: slug,
+ pkApiKey: slug,
+ shortcode: slug,
+ maximumConcurrencyLimit: opts.maximumConcurrencyLimit,
+ },
+ });
+
+ const queue = await prisma.taskQueue.create({
+ data: {
+ friendlyId: `queue_${slug}`,
+ name: `task/${slug}`,
+ projectId: project.id,
+ runtimeEnvironmentId: environment.id,
+ concurrencyLimit: opts.queueConcurrencyLimit ?? null,
+ },
+ });
+
+ const authEnv = {
+ id: environment.id,
+ maximumConcurrencyLimit: environment.maximumConcurrencyLimit,
+ } as unknown as AuthenticatedEnvironment;
+
+ return { organization, project, environment, queue, authEnv };
+}
+
+describe("ConcurrencySystem percent overrides", () => {
+ postgresTest(
+ "materializes a percent override and stores the percent as the source of truth",
+ async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const result = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, {
+ percent: 50,
+ });
+
+ expect(result.isOk()).toBe(true);
+
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ // floor(10 * 50 / 100) = 5
+ expect(row.concurrencyLimit).toBe(5);
+ expect(row.concurrencyLimitOverridePercent?.toNumber()).toBe(50);
+ // base captures the pre-override absolute limit
+ expect(row.concurrencyLimitBase).toBe(8);
+ expect(row.concurrencyLimitOverriddenAt).not.toBeNull();
+ }
+ );
+
+ postgresTest(
+ "rejects an absolute override that exceeds the environment limit and persists nothing",
+ async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const result = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, {
+ limit: 9999,
+ });
+
+ expect(result.isErr()).toBe(true);
+ if (result.isErr()) {
+ expect(result.error.type).toBe("concurrency_limit_exceeds_maximum");
+ }
+
+ // Nothing should have been written.
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ expect(row.concurrencyLimit).toBe(8);
+ expect(row.concurrencyLimitOverridePercent).toBeNull();
+ expect(row.concurrencyLimitOverriddenAt).toBeNull();
+ expect(row.concurrencyLimitBase).toBeNull();
+ }
+ );
+
+ postgresTest("rejects an out-of-range percent as invalid_override", async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const tooHigh = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, {
+ percent: 101,
+ });
+ expect(tooHigh.isErr()).toBe(true);
+ if (tooHigh.isErr()) expect(tooHigh.error.type).toBe("invalid_override");
+
+ const tooLow = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, {
+ percent: 0,
+ });
+ expect(tooLow.isErr()).toBe(true);
+ if (tooLow.isErr()) expect(tooLow.error.type).toBe("invalid_override");
+
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ expect(row.concurrencyLimitOverriddenAt).toBeNull();
+ });
+
+ postgresTest("reset clears both the absolute limit and the percent", async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const overridden = await system.queues.overrideQueueConcurrencyLimit(
+ authEnv,
+ queue.friendlyId,
+ { percent: 50 }
+ );
+ expect(overridden.isOk()).toBe(true);
+
+ const reset = await system.queues.resetConcurrencyLimit(authEnv, queue.friendlyId);
+ expect(reset.isOk()).toBe(true);
+
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ // restored to the base captured at override time
+ expect(row.concurrencyLimit).toBe(8);
+ expect(row.concurrencyLimitOverridePercent).toBeNull();
+ expect(row.concurrencyLimitBase).toBeNull();
+ expect(row.concurrencyLimitOverriddenAt).toBeNull();
+ expect(row.concurrencyLimitOverriddenBy).toBeNull();
+ });
+
+ postgresTest(
+ "recalculatePercentLimits recomputes percent queues and leaves absolute overrides untouched",
+ async ({ prisma }) => {
+ const { project, environment, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ // A percent-based queue: 50% of env(10) -> 5
+ const percentQueue = await prisma.taskQueue.create({
+ data: {
+ friendlyId: `queue_pct_${Math.random().toString(36).slice(2, 8)}`,
+ name: `task/pct-${Math.random().toString(36).slice(2, 8)}`,
+ projectId: project.id,
+ runtimeEnvironmentId: environment.id,
+ concurrencyLimit: 6,
+ },
+ });
+ const pctResult = await system.queues.overrideQueueConcurrencyLimit(
+ authEnv,
+ percentQueue.friendlyId,
+ { percent: 50 }
+ );
+ expect(pctResult.isOk()).toBe(true);
+ {
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: percentQueue.id } });
+ expect(row.concurrencyLimit).toBe(5); // floor(10 * 0.5)
+ }
+
+ // An absolute-override queue: limit 4, no percent
+ const absQueue = await prisma.taskQueue.create({
+ data: {
+ friendlyId: `queue_abs_${Math.random().toString(36).slice(2, 8)}`,
+ name: `task/abs-${Math.random().toString(36).slice(2, 8)}`,
+ projectId: project.id,
+ runtimeEnvironmentId: environment.id,
+ concurrencyLimit: 6,
+ },
+ });
+ const absResult = await system.queues.overrideQueueConcurrencyLimit(
+ authEnv,
+ absQueue.friendlyId,
+ { limit: 4 }
+ );
+ expect(absResult.isOk()).toBe(true);
+
+ // Simulate the environment limit changing from 10 -> 20 and recalculate.
+ const bumpedEnv = {
+ id: environment.id,
+ maximumConcurrencyLimit: 20,
+ } as unknown as AuthenticatedEnvironment;
+
+ const outcome = await system.queues.recalculatePercentLimits(bumpedEnv);
+ expect(outcome.total).toBe(1); // only the percent queue is considered
+ expect(outcome.updated).toBe(1);
+
+ const pctRow = await prisma.taskQueue.findUniqueOrThrow({ where: { id: percentQueue.id } });
+ // floor(20 * 50 / 100) = 10
+ expect(pctRow.concurrencyLimit).toBe(10);
+ expect(pctRow.concurrencyLimitOverridePercent?.toNumber()).toBe(50);
+
+ const absRow = await prisma.taskQueue.findUniqueOrThrow({ where: { id: absQueue.id } });
+ // absolute override must be untouched
+ expect(absRow.concurrencyLimit).toBe(4);
+ expect(absRow.concurrencyLimitOverridePercent).toBeNull();
+ }
+ );
+
+ postgresTest(
+ "recalculatePercentLimits is idempotent when the limit is unchanged",
+ async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const overridden = await system.queues.overrideQueueConcurrencyLimit(
+ authEnv,
+ queue.friendlyId,
+ { percent: 50 }
+ );
+ expect(overridden.isOk()).toBe(true);
+
+ // Recalculate against the SAME environment limit -> nothing to update.
+ const outcome = await system.queues.recalculatePercentLimits(authEnv);
+ expect(outcome.total).toBe(1);
+ expect(outcome.updated).toBe(0);
+
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ expect(row.concurrencyLimit).toBe(5);
+ }
+ );
+
+ postgresTest(
+ "recalculatePercentLimits re-syncs the engine on a later recalc after an engine push failed, even when the DB limit is unchanged",
+ async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const overridden = await system.queues.overrideQueueConcurrencyLimit(
+ authEnv,
+ queue.friendlyId,
+ { percent: 50 }
+ );
+ expect(overridden.isOk()).toBe(true);
+ {
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ expect(row.concurrencyLimit).toBe(5); // floor(10 * 0.5)
+ }
+
+ const bumpedEnv = {
+ id: authEnv.id,
+ maximumConcurrencyLimit: 20,
+ } as unknown as AuthenticatedEnvironment;
+
+ // The env-limit bump recalc writes the new DB limit (10) but the engine push fails and is
+ // swallowed by the per-queue catch: DB and engine are now diverged (DB=10, engine=5).
+ updateQueueConcurrencyLimitsMock.mockClear();
+ updateQueueConcurrencyLimitsMock.mockRejectedValueOnce(new Error("engine push failed"));
+
+ const first = await system.queues.recalculatePercentLimits(bumpedEnv);
+ expect(first.updated).toBe(1); // DB was written before the push threw
+ {
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ expect(row.concurrencyLimit).toBe(10);
+ }
+
+ // A later recalc at the SAME env limit makes no DB change, but MUST still push to the engine
+ // so the previously-failed sync self-heals. (Regression guard: the old code `continue`d when
+ // newLimit === concurrencyLimit and left the engine stuck at 5 forever.)
+ updateQueueConcurrencyLimitsMock.mockClear();
+ const second = await system.queues.recalculatePercentLimits(bumpedEnv);
+ expect(second.updated).toBe(0); // no DB write
+ expect(updateQueueConcurrencyLimitsMock).toHaveBeenCalledWith(
+ expect.anything(),
+ queue.name,
+ 10
+ );
+ }
+ );
+});
diff --git a/apps/webapp/test/queueMetricsMapping.test.ts b/apps/webapp/test/queueMetricsMapping.test.ts
new file mode 100644
index 00000000000..61e3893c7fb
--- /dev/null
+++ b/apps/webapp/test/queueMetricsMapping.test.ts
@@ -0,0 +1,239 @@
+import { describe, expect, it } from "vitest";
+import {
+ descriptorFromQueue,
+ mapEntryToRows,
+ OVERFLOW_QUEUE_NAME,
+ QueueNameLimiter,
+} from "~/v3/queueMetricsMapping";
+
+describe("descriptorFromQueue", () => {
+ it("parses a plain descriptor", () => {
+ expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1:queue:task/my-task")).toEqual({
+ organization_id: "o1",
+ project_id: "p1",
+ environment_id: "e1",
+ queue_name: "task/my-task",
+ concurrency_key: "",
+ });
+ });
+
+ it("captures a concurrency-key suffix", () => {
+ expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1:queue:task/t:ck:tenant-3")).toEqual(
+ expect.objectContaining({ queue_name: "task/t", concurrency_key: "tenant-3" })
+ );
+ });
+
+ it("maps the ck wildcard to no key", () => {
+ expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1:queue:task/t:ck:*")).toEqual(
+ expect.objectContaining({ queue_name: "task/t", concurrency_key: "" })
+ );
+ });
+
+ it("keeps colons inside the queue name", () => {
+ expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1:queue:my:odd:queue")).toEqual(
+ expect.objectContaining({ queue_name: "my:odd:queue", concurrency_key: "" })
+ );
+ });
+
+ it("keeps colons in the name while capturing a real ck suffix", () => {
+ expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1:queue:a:b:ck:t9")).toEqual(
+ expect.objectContaining({ queue_name: "a:b", concurrency_key: "t9" })
+ );
+ });
+
+ it("rejects malformed descriptors", () => {
+ expect(descriptorFromQueue("not-a-descriptor")).toBeNull();
+ expect(descriptorFromQueue("{org:o1}:proj:p1:env:e1")).toBeNull();
+ expect(descriptorFromQueue("")).toBeNull();
+ });
+});
+
+describe("QueueNameLimiter", () => {
+ it("passes names through under the cap and overflows past it, per scope", () => {
+ const limiter = new QueueNameLimiter(2);
+ expect(limiter.limit("env1", "a")).toBe("a");
+ expect(limiter.limit("env1", "b")).toBe("b");
+ expect(limiter.limit("env1", "c")).toBe(OVERFLOW_QUEUE_NAME);
+ expect(limiter.limit("env1", "a")).toBe("a");
+ expect(limiter.limit("env2", "c")).toBe("c");
+ });
+
+ it("is unlimited when the cap is 0", () => {
+ const limiter = new QueueNameLimiter(0);
+ for (let i = 0; i < 100; i++) {
+ expect(limiter.limit("env1", `q${i}`)).toBe(`q${i}`);
+ }
+ });
+
+ it("evicts the oldest scope when the scope map is full", () => {
+ const limiter = new QueueNameLimiter(1, 2);
+ expect(limiter.limit("env1", "a")).toBe("a");
+ expect(limiter.limit("env2", "a")).toBe("a");
+ expect(limiter.limit("env3", "a")).toBe("a");
+ expect(limiter.limit("env1", "b")).toBe("b");
+ });
+});
+
+describe("mapEntryToRows", () => {
+ const q = "{org:o1}:proj:p1:env:e1:queue:task/t";
+
+ it("maps a gauge entry with numeric fields", () => {
+ const rows = mapEntryToRows({
+ id: "1700000000000-0",
+ fields: {
+ op: "gauge",
+ q,
+ ql: "5",
+ cc: "2",
+ lim: "10",
+ eql: "7",
+ ec: "3",
+ elim: "20",
+ thr: "1",
+ },
+ });
+ expect(rows).toHaveLength(1);
+ expect(rows[0]).toEqual(
+ expect.objectContaining({
+ op: "gauge",
+ organization_id: "o1",
+ queue_name: "task/t",
+ concurrency_key: "",
+ queued: 5,
+ running: 2,
+ queue_limit: 10,
+ env_queued: 7,
+ env_running: 3,
+ env_limit: 20,
+ throttled: 1,
+ })
+ );
+ expect(rows[0]!.event_time).toBe("2023-11-14 22:13:20");
+ expect(rows[0]!.ck_backlogged).toBeUndefined();
+ expect(rows[0]!.ck_max_wait_ms).toBeUndefined();
+ });
+
+ it("keeps the key on per-subqueue gauges and maps the CK-health tail", () => {
+ const rows = mapEntryToRows({
+ id: "1700000000000-0",
+ fields: { op: "gauge", q: `${q}:ck:tenant-1`, ql: "4", ckq: "3", ckw: "2500" },
+ });
+ expect(rows).toHaveLength(1);
+ expect(rows[0]).toEqual(
+ expect.objectContaining({
+ op: "gauge",
+ queue_name: "task/t",
+ concurrency_key: "tenant-1",
+ queued: 4,
+ ck_backlogged: 3,
+ ck_max_wait_ms: 2500,
+ })
+ );
+ });
+
+ it("maps started with wait_ms + cumulative and drops unknown ops", () => {
+ const started = mapEntryToRows({
+ id: "1700000000000-0",
+ fields: { op: "started", q, wait: "48", cum: "512" },
+ });
+ expect(started).toHaveLength(1);
+ expect(started[0]).toEqual(
+ expect.objectContaining({
+ op: "started",
+ wait_ms: 48,
+ cumulative: 512,
+ order_key: (1700000000000n * 1000000n).toString(),
+ })
+ );
+ expect(mapEntryToRows({ id: "1-0", fields: { op: "ack", q, cum: "9" } })[0]).toEqual(
+ expect.objectContaining({ op: "ack", cumulative: 9 })
+ );
+ expect(mapEntryToRows({ id: "1-0", fields: { op: "bogus", q } })).toEqual([]);
+ expect(mapEntryToRows({ id: "1-0", fields: { op: "ack" } })).toEqual([]);
+ });
+
+ it("expands a dual-odometer counter entry into base + per-key rows", () => {
+ const rows = mapEntryToRows({
+ id: "1700000000000-3",
+ fields: { op: "started", q, ck: "tenant-9", wait: "80", cum: "41", ckcum: "7" },
+ });
+ expect(rows).toHaveLength(2);
+ expect(rows[0]).toEqual(
+ expect.objectContaining({ queue_name: "task/t", cumulative: 41, wait_ms: 80 })
+ );
+ expect(rows[0]!.concurrency_key).toBeUndefined();
+ expect(rows[1]).toEqual(
+ expect.objectContaining({
+ queue_name: "task/t",
+ concurrency_key: "tenant-9",
+ cumulative: 7,
+ wait_ms: 80,
+ })
+ );
+ expect(rows[0]!.order_key).toBe(rows[1]!.order_key);
+
+ // Baseline entries carry exactly one odometer each.
+ const baseBaseline = mapEntryToRows({ id: "1-0", fields: { op: "started", q, cum: "0" } });
+ expect(baseBaseline).toHaveLength(1);
+ expect(baseBaseline[0]!.concurrency_key).toBeUndefined();
+ const ckBaseline = mapEntryToRows({
+ id: "1-1",
+ fields: { op: "started", q, ck: "tenant-9", ckcum: "0" },
+ });
+ expect(ckBaseline).toHaveLength(1);
+ expect(ckBaseline[0]).toEqual(
+ expect.objectContaining({ concurrency_key: "tenant-9", cumulative: 0 })
+ );
+ });
+
+ it("applies the queue-name limiter: gauges overflow, counters drop", () => {
+ const limiters = { queueNames: new QueueNameLimiter(1) };
+ const first = mapEntryToRows({ id: "1-0", fields: { op: "ack", q, cum: "1" } }, limiters);
+ expect(first[0]!.queue_name).toBe("task/t");
+
+ // Overflowed gauges keep flowing under the shared name (max stays meaningful),
+ // with per-key attribution stripped.
+ const overflowGauge = mapEntryToRows(
+ {
+ id: "1-1",
+ fields: { op: "gauge", q: "{org:o1}:proj:p1:env:e1:queue:task/other:ck:t1", ql: "3" },
+ },
+ limiters
+ );
+ expect(overflowGauge[0]!.queue_name).toBe(OVERFLOW_QUEUE_NAME);
+ expect(overflowGauge[0]!.concurrency_key).toBe("");
+
+ // Overflowed counters are dropped: merging distinct odometers under one key
+ // produces garbage deltas.
+ const overflowCounter = mapEntryToRows(
+ { id: "1-2", fields: { op: "ack", q: "{org:o1}:proj:p1:env:e1:queue:task/other", cum: "4" } },
+ limiters
+ );
+ expect(overflowCounter).toEqual([]);
+ });
+
+ it("applies the concurrency-key limiter: overflow drops the per-key row, keeps base", () => {
+ const limiters = { concurrencyKeys: new QueueNameLimiter(1) };
+ const first = mapEntryToRows(
+ { id: "1-0", fields: { op: "ack", q, ck: "t1", cum: "5", ckcum: "2" } },
+ limiters
+ );
+ expect(first).toHaveLength(2);
+
+ const overflowed = mapEntryToRows(
+ { id: "1-1", fields: { op: "ack", q, ck: "t2", cum: "6", ckcum: "1" } },
+ limiters
+ );
+ expect(overflowed).toHaveLength(1);
+ expect(overflowed[0]!.cumulative).toBe(6);
+ expect(overflowed[0]!.concurrency_key).toBeUndefined();
+
+ // Gauge for an overflowed key keeps the row but loses the attribution.
+ const overflowGauge = mapEntryToRows(
+ { id: "1-2", fields: { op: "gauge", q: `${q}:ck:t3`, ql: "2" } },
+ limiters
+ );
+ expect(overflowGauge).toHaveLength(1);
+ expect(overflowGauge[0]!.concurrency_key).toBe("");
+ });
+});
diff --git a/apps/webapp/test/useTableSort.test.ts b/apps/webapp/test/useTableSort.test.ts
new file mode 100644
index 00000000000..4b6c83666d9
--- /dev/null
+++ b/apps/webapp/test/useTableSort.test.ts
@@ -0,0 +1,55 @@
+import { describe, expect, it } from "vitest";
+import { compareColumn, sortRows, type SortColumn } from "~/components/primitives/useTableSort";
+
+type Row = { id: number; name: string | null; queued: number | null };
+
+const rows: Row[] = [
+ { id: 0, name: "banana", queued: 3 },
+ { id: 1, name: "Apple", queued: null },
+ { id: 2, name: "cherry", queued: 3 },
+ { id: 3, name: null, queued: 1 },
+];
+
+describe("sortRows", () => {
+ it("sorts numbers ascending with nulls last", () => {
+ const column: SortColumn = { key: "queued", type: "number", value: (r) => r.queued };
+ expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([3, 0, 2, 1]);
+ });
+
+ it("keeps nulls last even when descending", () => {
+ const column: SortColumn = { key: "queued", type: "number", value: (r) => r.queued };
+ // 3 and 0 both have queued=3; stable order (0 before 2) is preserved, null (id 1) stays last.
+ expect(sortRows(rows, column, "desc").map((r) => r.id)).toEqual([0, 2, 3, 1]);
+ });
+
+ it("sorts alphabetically case-insensitively with empty/null last", () => {
+ const column: SortColumn = { key: "name", type: "alpha", value: (r) => r.name };
+ expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([1, 0, 2, 3]);
+ expect(sortRows(rows, column, "desc").map((r) => r.id)).toEqual([2, 0, 1, 3]);
+ });
+
+ it("is stable for equal values (preserves original order)", () => {
+ const column: SortColumn = { key: "queued", type: "number", value: () => 5 };
+ expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([0, 1, 2, 3]);
+ });
+
+ it("supports a custom comparator", () => {
+ // Sort by name length.
+ const column: SortColumn = {
+ key: "name",
+ type: "custom",
+ compare: (a, b) => (a.name?.length ?? 0) - (b.name?.length ?? 0),
+ };
+ expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([3, 1, 0, 2]);
+ });
+});
+
+describe("compareColumn", () => {
+ it("treats NaN as null (sorts last)", () => {
+ const column: SortColumn = { key: "queued", type: "number", value: (r) => r.queued };
+ const withNaN: Row = { id: 9, name: "x", queued: NaN };
+ const normal: Row = { id: 10, name: "y", queued: 1 };
+ expect(compareColumn(column, withNaN, normal, "asc")).toBe(1);
+ expect(compareColumn(column, withNaN, normal, "desc")).toBe(1);
+ });
+});
diff --git a/internal-packages/clickhouse/schema/036_create_queue_metrics_v1.sql b/internal-packages/clickhouse/schema/036_create_queue_metrics_v1.sql
new file mode 100644
index 00000000000..8ea1e65f09f
--- /dev/null
+++ b/internal-packages/clickhouse/schema/036_create_queue_metrics_v1.sql
@@ -0,0 +1,267 @@
+-- +goose Up
+
+-- Queue metrics: raw landing table -> MV -> aggregated read target (mirrors
+-- llm_model_aggregates_v1, migration 027). Raw rows feed an MV on insert, and
+-- reads hit the aggregated table.
+
+-- Short-TTL raw landing, one row per stream entry. non_replicated_deduplication_window
+-- makes consumer replays idempotent via insert_deduplication_token.
+CREATE TABLE IF NOT EXISTS trigger_dev.queue_metrics_raw_v1
+(
+ organization_id LowCardinality(String),
+ project_id LowCardinality(String),
+ environment_id String CODEC(ZSTD(1)),
+ queue_name String CODEC(ZSTD(1)),
+ concurrency_key String DEFAULT '' CODEC(ZSTD(1)), -- per-key attribution ('' = base/whole-queue row)
+ event_time DateTime CODEC(Delta(4), ZSTD(1)),
+ order_key UInt64 DEFAULT 0, -- stream-id composite (ms*1e6+seq), deltaSumTimestamp ordering key
+ op LowCardinality(String), -- gauge | enqueue | started | ack | nack | dlq
+ running UInt32 DEFAULT 0,
+ queued UInt32 DEFAULT 0,
+ queue_limit UInt32 DEFAULT 0,
+ env_running UInt32 DEFAULT 0,
+ env_queued UInt32 DEFAULT 0,
+ env_limit UInt32 DEFAULT 0,
+ throttled UInt8 DEFAULT 0, -- 1 on a gauge emission with running>=limit AND queued>0
+ ck_backlogged UInt32 DEFAULT 0, -- gauge on CK queues: distinct concurrency keys with queued work
+ ck_max_wait_ms UInt32 DEFAULT 0, -- gauge on CK queues: most-starved key's head-of-line wait
+ wait_ms UInt32 DEFAULT 0, -- set on op='started' (scheduling delay)
+ cumulative UInt64 DEFAULT 0 -- monotonic per-(queue,op) odometer on a counter op, diffed at read time
+)
+ENGINE = MergeTree()
+PARTITION BY toDate(event_time)
+ORDER BY (organization_id, project_id, environment_id, queue_name, event_time)
+TTL event_time + INTERVAL 6 HOUR
+SETTINGS non_replicated_deduplication_window = 1000, ttl_only_drop_parts = 1;
+
+-- (2) Aggregated read target (TRQL/dashboards query this).
+CREATE TABLE IF NOT EXISTS trigger_dev.queue_metrics_v1
+(
+ organization_id LowCardinality(String),
+ project_id LowCardinality(String),
+ environment_id String CODEC(ZSTD(1)),
+ queue_name String CODEC(ZSTD(1)),
+ bucket_start DateTime CODEC(Delta(4), ZSTD(1)),
+
+ -- Cumulative-counter deltas: each op maintains a monotonic odometer, and deltaSumTimestamp
+ -- sums positive consecutive deltas (ignoring resets) ordered by event_time, so a lost
+ -- reading self-heals (the next surviving reading restates the total). Read with
+ -- deltaSumTimestampMerge( {
* based on the span of the time range.
*/
timeRange?: TimeRange;
+ /**
+ * Opt-in: emit rows for empty time buckets in a top-level time-bucketed query
+ * (counters zero-fill, gauges carry forward). Off by default.
+ */
+ fillGaps?: boolean;
}
/**
@@ -192,6 +197,7 @@ export async function executeTSQL(
fieldMappings: options.fieldMappings,
whereClauseFallback: options.whereClauseFallback,
timeRange: options.timeRange,
+ fillGaps: options.fillGaps,
});
generatedSql = sql;
diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts
index bc8b6e662c8..1481be36536 100644
--- a/internal-packages/clickhouse/src/index.ts
+++ b/internal-packages/clickhouse/src/index.ts
@@ -33,6 +33,14 @@ import {
} from "./taskEvents.js";
import { insertMetrics } from "./metrics.js";
import { insertLlmMetrics } from "./llmMetrics.js";
+import {
+ insertQueueMetricsRaw,
+ getQueueListMetricsSummary,
+ getQueueDepthSparklines,
+ getQueueRanking,
+ getQueueRankingNames,
+ getQueueRankingCount,
+} from "./queueMetrics.js";
import {
getSessionTagsQueryBuilder,
getSessionsCountQueryBuilder,
@@ -66,6 +74,7 @@ export type * from "./taskRuns.js";
export type * from "./taskEvents.js";
export type * from "./metrics.js";
export type * from "./llmMetrics.js";
+export type * from "./queueMetrics.js";
export type * from "./llmModelAggregates.js";
export type * from "./errors.js";
export type * from "./sessions.js";
@@ -262,6 +271,17 @@ export class ClickHouse {
};
}
+ get queueMetrics() {
+ return {
+ insertRaw: insertQueueMetricsRaw(this.writer),
+ listSummary: getQueueListMetricsSummary(this.reader),
+ depthSparklines: getQueueDepthSparklines(this.reader),
+ ranking: getQueueRanking(this.reader),
+ rankingNames: getQueueRankingNames(this.reader),
+ rankingCount: getQueueRankingCount(this.reader),
+ };
+ }
+
get llmModelAggregates() {
return {
globalMetrics: getGlobalModelMetrics(this.reader),
diff --git a/internal-packages/clickhouse/src/queueMetrics.test.ts b/internal-packages/clickhouse/src/queueMetrics.test.ts
new file mode 100644
index 00000000000..00532041e44
--- /dev/null
+++ b/internal-packages/clickhouse/src/queueMetrics.test.ts
@@ -0,0 +1,525 @@
+import { clickhouseTest } from "@internal/testcontainers";
+import { z } from "zod";
+import { ClickHouse } from "./index.js";
+import type { QueueMetricsRawV1Input } from "./queueMetrics.js";
+
+const ORG = "org_qm";
+const PROJECT = "project_qm";
+const ENV = "env_qm";
+const EVENT_TIME = "2026-06-30 12:00:05"; // all rows land in the 10s bucket starting 12:00:00
+
+function base(op: QueueMetricsRawV1Input["op"], queue: string): QueueMetricsRawV1Input {
+ return {
+ organization_id: ORG,
+ project_id: PROJECT,
+ environment_id: ENV,
+ queue_name: queue,
+ event_time: EVENT_TIME,
+ op,
+ };
+}
+
+// Cumulative counters: each op keeps a monotonic per-(queue,op) odometer, so a counter row
+// carries the running total in `cumulative`. deltaSumTimestamp reconstructs the increase
+// (last - first) from a seeded cum=0 baseline; order_key orders readings within an op.
+let orderKey = 0;
+function counter(
+ op: QueueMetricsRawV1Input["op"],
+ queue: string,
+ total: number,
+ waits?: number[]
+): QueueMetricsRawV1Input[] {
+ const rows: QueueMetricsRawV1Input[] = [
+ { ...base(op, queue), cumulative: 0, order_key: orderKey++ },
+ ];
+ for (let cum = 1; cum <= total; cum++) {
+ rows.push({
+ ...base(op, queue),
+ cumulative: cum,
+ order_key: orderKey++,
+ ...(waits ? { wait_ms: waits[cum - 1] } : {}),
+ });
+ }
+ return rows;
+}
+
+const aggregatedRow = z.object({
+ enqueue_count: z.coerce.number(),
+ started_count: z.coerce.number(),
+ ack_count: z.coerce.number(),
+ nack_count: z.coerce.number(),
+ dlq_count: z.coerce.number(),
+ throttled_count: z.coerce.number(),
+ max_running: z.coerce.number(),
+ max_queued: z.coerce.number(),
+ max_limit: z.coerce.number(),
+ max_env_running: z.coerce.number(),
+ max_env_queued: z.coerce.number(),
+ max_env_limit: z.coerce.number(),
+ max_ck_backlogged: z.coerce.number(),
+ max_ck_wait_ms: z.coerce.number(),
+ wait_ms_sum: z.coerce.number(),
+ wait_ms_count: z.coerce.number(),
+ wait_p50: z.coerce.number(),
+ wait_p90: z.coerce.number(),
+ wait_p95: z.coerce.number(),
+ wait_p99: z.coerce.number(),
+});
+
+function readAggregated(ch: ClickHouse) {
+ return ch.reader.query({
+ name: "read-queue-metrics-aggregated",
+ query: `SELECT
+ deltaSumTimestampMerge(enqueue_delta) AS enqueue_count,
+ deltaSumTimestampMerge(started_delta) AS started_count,
+ deltaSumTimestampMerge(ack_delta) AS ack_count,
+ deltaSumTimestampMerge(nack_delta) AS nack_count,
+ deltaSumTimestampMerge(dlq_delta) AS dlq_count,
+ sum(throttled_count) AS throttled_count,
+ max(max_running) AS max_running,
+ max(max_queued) AS max_queued,
+ max(max_limit) AS max_limit,
+ max(max_env_running) AS max_env_running,
+ max(max_env_queued) AS max_env_queued,
+ max(max_env_limit) AS max_env_limit,
+ max(max_ck_backlogged) AS max_ck_backlogged,
+ max(max_ck_wait_ms) AS max_ck_wait_ms,
+ sum(wait_ms_sum) AS wait_ms_sum,
+ sum(wait_ms_count) AS wait_ms_count,
+ quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles) AS wait_arr,
+ wait_arr[1] AS wait_p50,
+ wait_arr[2] AS wait_p90,
+ wait_arr[3] AS wait_p95,
+ wait_arr[4] AS wait_p99
+ FROM trigger_dev.queue_metrics_v1
+ WHERE queue_name = {queueName: String}
+ GROUP BY organization_id, project_id, environment_id, queue_name, bucket_start`,
+ schema: aggregatedRow,
+ params: z.object({ queueName: z.string() }),
+ });
+}
+
+// Synchronous insert so the MV-populated rows are queryable immediately.
+const SYNC = { params: { clickhouse_settings: { async_insert: 0 as const } } };
+
+describe("queue_metrics_v1", () => {
+ clickhouseTest(
+ "buckets counters, gauges and wait percentiles via the MV",
+ async ({ clickhouseContainer }) => {
+ const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" });
+ const queue = "queue-a";
+
+ const rows: QueueMetricsRawV1Input[] = [
+ ...counter("enqueue", queue, 3),
+ ...counter("started", queue, 10, [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]),
+ ...counter("ack", queue, 2),
+ ...counter("nack", queue, 1),
+ ...counter("dlq", queue, 1),
+ {
+ ...base("gauge", queue),
+ running: 8,
+ queued: 4,
+ queue_limit: 10,
+ env_running: 40,
+ env_queued: 10,
+ env_limit: 50,
+ throttled: 0,
+ ck_backlogged: 3,
+ ck_max_wait_ms: 2500,
+ },
+ {
+ ...base("gauge", queue),
+ running: 10,
+ queued: 6,
+ queue_limit: 10,
+ env_running: 50,
+ env_queued: 20,
+ env_limit: 50,
+ throttled: 1, // running >= limit AND queued > 0
+ ck_backlogged: 2,
+ ck_max_wait_ms: 1500,
+ },
+ ];
+
+ const [insertError] = await ch.queueMetrics.insertRaw(rows, SYNC);
+ expect(insertError).toBeNull();
+
+ const [queryError, result] = await readAggregated(ch)({ queueName: queue });
+ expect(queryError).toBeNull();
+ expect(result).toHaveLength(1);
+ const row = result![0]!;
+
+ expect(row.enqueue_count).toBe(3);
+ expect(row.started_count).toBe(10);
+ expect(row.ack_count).toBe(2);
+ expect(row.nack_count).toBe(1);
+ expect(row.dlq_count).toBe(1);
+ expect(row.throttled_count).toBe(1);
+
+ expect(row.max_running).toBe(10);
+ expect(row.max_queued).toBe(6);
+ expect(row.max_limit).toBe(10);
+ expect(row.max_env_running).toBe(50);
+ expect(row.max_env_queued).toBe(20);
+ expect(row.max_env_limit).toBe(50);
+ expect(row.max_ck_backlogged).toBe(3);
+ expect(row.max_ck_wait_ms).toBe(2500);
+
+ expect(row.wait_ms_sum).toBe(5500);
+ expect(row.wait_ms_count).toBe(10);
+
+ // Percentiles over [100..1000]: monotonic and within the value range.
+ expect(row.wait_p50).toBeGreaterThanOrEqual(400);
+ expect(row.wait_p50).toBeLessThanOrEqual(650);
+ expect(row.wait_p90).toBeGreaterThanOrEqual(row.wait_p50);
+ expect(row.wait_p95).toBeGreaterThanOrEqual(row.wait_p90);
+ expect(row.wait_p99).toBeGreaterThanOrEqual(row.wait_p95);
+ expect(row.wait_p99).toBeLessThanOrEqual(1000);
+
+ await ch.close();
+ }
+ );
+
+ clickhouseTest(
+ "merges wait-quantile state across separate insert blocks",
+ async ({ clickhouseContainer }) => {
+ const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" });
+ const queue = "queue-b";
+
+ // Cumulative odometer continues across the two insert blocks (baseline 0, then 1..10);
+ // deltaSumTimestamp state and quantile state merge across the parts into one bucket.
+ const startedRow = (cum: number, wait_ms?: number): QueueMetricsRawV1Input => ({
+ ...base("started", queue),
+ cumulative: cum,
+ order_key: orderKey++,
+ ...(wait_ms !== undefined ? { wait_ms } : {}),
+ });
+
+ const [e1] = await ch.queueMetrics.insertRaw(
+ [startedRow(0), ...[100, 200, 300, 400, 500].map((w, i) => startedRow(i + 1, w))],
+ SYNC
+ );
+ expect(e1).toBeNull();
+ const [e2] = await ch.queueMetrics.insertRaw(
+ [600, 700, 800, 900, 1000].map((w, i) => startedRow(i + 6, w)),
+ SYNC
+ );
+ expect(e2).toBeNull();
+
+ const [queryError, result] = await readAggregated(ch)({ queueName: queue });
+ expect(queryError).toBeNull();
+ expect(result).toHaveLength(1);
+ const row = result![0]!;
+
+ // Both blocks contribute to one bucket: counts and sums add, quantile state merges.
+ expect(row.started_count).toBe(10);
+ expect(row.wait_ms_sum).toBe(5500);
+ expect(row.wait_ms_count).toBe(10);
+ expect(row.wait_p50).toBeGreaterThanOrEqual(400);
+ expect(row.wait_p50).toBeLessThanOrEqual(650);
+ expect(row.wait_p99).toBeGreaterThanOrEqual(row.wait_p50);
+ expect(row.wait_p99).toBeLessThanOrEqual(1000);
+
+ await ch.close();
+ }
+ );
+
+ clickhouseTest(
+ "5m and env rollups agree with the 10s tier, and env buckets are 10s",
+ async ({ clickhouseContainer }) => {
+ const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" });
+
+ // Own org so the env-level read (no queue filter) stays isolated from other tests.
+ const rollOrg = "org_qm_roll";
+ const rows: QueueMetricsRawV1Input[] = [
+ ...counter("started", "roll-a", 7, [100, 150, 200, 250, 300, 350, 400]),
+ ...counter("started", "roll-b", 3, [500, 600, 700]),
+ {
+ ...base("gauge", "roll-a"),
+ running: 4,
+ queued: 9,
+ env_running: 30,
+ env_limit: 50,
+ ck_backlogged: 5,
+ ck_max_wait_ms: 9000,
+ },
+ { ...base("gauge", "roll-b"), running: 2, queued: 1, env_running: 45, env_limit: 50 },
+ {
+ ...base("gauge", "roll-a"),
+ event_time: "2026-06-30 12:00:15",
+ running: 1,
+ queued: 2,
+ env_running: 20,
+ env_limit: 50,
+ ck_backlogged: 2,
+ ck_max_wait_ms: 3000,
+ },
+ ].map((row) => ({ ...row, organization_id: rollOrg }));
+ const [insertError] = await ch.queueMetrics.insertRaw(rows, SYNC);
+ expect(insertError).toBeNull();
+
+ const perQueue = (table: string) =>
+ ch.reader.query({
+ name: "per-queue-both-tiers",
+ query: `SELECT queue_name, deltaSumTimestampMerge(started_delta) AS started
+ FROM ${table}
+ WHERE queue_name IN ('roll-a', 'roll-b')
+ GROUP BY queue_name ORDER BY queue_name`,
+ schema: z.object({ queue_name: z.string(), started: z.coerce.number() }),
+ })({});
+ const [e10, rows10] = await perQueue("trigger_dev.queue_metrics_v1");
+ const [e5m, rows5m] = await perQueue("trigger_dev.queue_metrics_5m_v1");
+ expect(e10).toBeNull();
+ expect(e5m).toBeNull();
+ expect(rows10).toEqual([
+ { queue_name: "roll-a", started: 7 },
+ { queue_name: "roll-b", started: 3 },
+ ]);
+ expect(rows5m).toEqual(rows10);
+
+ // CK-health gauges roll into the 5m mirror too.
+ const [ckError, ckRows] = await ch.reader.query({
+ name: "ck-5m-read",
+ query: `SELECT max(max_ck_backlogged) AS ck_keys, max(max_ck_wait_ms) AS ck_wait
+ FROM trigger_dev.queue_metrics_5m_v1
+ WHERE queue_name = 'roll-a'`,
+ schema: z.object({ ck_keys: z.coerce.number(), ck_wait: z.coerce.number() }),
+ })({});
+ expect(ckError).toBeNull();
+ expect(ckRows![0]).toEqual({ ck_keys: 5, ck_wait: 9000 });
+
+ // Env-wide totals: sum of per-queue merges (a single merge across queues would mix
+ // odometers and double-count).
+ const [envTotalError, envTotal] = await ch.reader.query({
+ name: "env-total-per-queue-sum",
+ query: `SELECT sum(started) AS started FROM (
+ SELECT queue_name, deltaSumTimestampMerge(started_delta) AS started
+ FROM trigger_dev.queue_metrics_5m_v1
+ WHERE queue_name IN ('roll-a', 'roll-b')
+ GROUP BY queue_name
+ )`,
+ schema: z.object({ started: z.coerce.number() }),
+ })({});
+ expect(envTotalError).toBeNull();
+ expect(envTotal![0]!.started).toBe(10);
+
+ const [envError, envRows] = await ch.reader.query({
+ name: "env-rollup-read",
+ query: `SELECT
+ max(max_env_running) AS max_env_running,
+ max(max_env_limit) AS max_env_limit,
+ uniqExact(bucket_start) AS buckets,
+ round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS wait_p99
+ FROM trigger_dev.env_metrics_v1
+ WHERE organization_id = {org: String}`,
+ schema: z.object({
+ max_env_running: z.coerce.number(),
+ max_env_limit: z.coerce.number(),
+ buckets: z.coerce.number(),
+ wait_p99: z.coerce.number(),
+ }),
+ params: z.object({ org: z.string() }),
+ })({ org: rollOrg });
+ expect(envError).toBeNull();
+ expect(envRows![0]!.max_env_running).toBe(45);
+ expect(envRows![0]!.max_env_limit).toBe(50);
+ // 12:00:05 and 12:00:15 land in separate 10s env buckets (12:00:00 and 12:00:10).
+ expect(envRows![0]!.buckets).toBe(2);
+ expect(envRows![0]!.wait_p99).toBeGreaterThanOrEqual(600);
+ expect(envRows![0]!.wait_p99).toBeLessThanOrEqual(1000);
+
+ await ch.close();
+ }
+ );
+
+ clickhouseTest(
+ "merged ranking returns the page and the windowed total in one query",
+ async ({ clickhouseContainer }) => {
+ const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" });
+
+ const gauge = (queue: string, queued: number, running: number): QueueMetricsRawV1Input => ({
+ ...base("gauge", queue),
+ queued,
+ running,
+ });
+ const [insertError] = await ch.queueMetrics.insertRaw(
+ [gauge("rank-low", 1, 0), gauge("rank-high", 50, 3), gauge("rank-mid", 10, 2)],
+ SYNC
+ );
+ expect(insertError).toBeNull();
+
+ const args = {
+ organizationId: ORG,
+ projectId: PROJECT,
+ environmentId: ENV,
+ startTime: "2026-06-30 11:50:00",
+ nameContains: "rank-",
+ byQueuedOnly: 0,
+ };
+ const [pageError, page] = await ch.queueMetrics.ranking({ ...args, limit: 2, offset: 0 });
+ expect(pageError).toBeNull();
+ expect(page).toEqual([
+ { queue_name: "rank-high", ranked_total: 3 },
+ { queue_name: "rank-mid", ranked_total: 3 },
+ ]);
+
+ const [countError, count] = await ch.queueMetrics.rankingCount(args);
+ expect(countError).toBeNull();
+ expect(count![0]!.ranked).toBe(3);
+
+ const [namesError, names] = await ch.queueMetrics.rankingNames({ ...args, limit: 10 });
+ expect(namesError).toBeNull();
+ expect(names!.map((r) => r.queue_name)).toEqual(["rank-high", "rank-mid", "rank-low"]);
+
+ await ch.close();
+ }
+ );
+});
+
+describe("consumer retry idempotency", () => {
+ clickhouseTest(
+ "re-inserting a batch with the same dedup token does not inflate any tier",
+ async ({ clickhouseContainer }) => {
+ const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" });
+
+ const dedupOrg = "org_qm_dedup";
+ const rows: QueueMetricsRawV1Input[] = [
+ ...counter("started", "dedup-q", 3, [100, 200, 300]),
+ { ...base("gauge", "dedup-q"), running: 2, queued: 1, env_running: 5, env_limit: 10 },
+ ].map((row) => ({ ...row, organization_id: dedupOrg }));
+
+ const retrySettings = {
+ params: {
+ clickhouse_settings: {
+ async_insert: 0 as const,
+ insert_deduplication_token: "qm-test-retry-batch",
+ deduplicate_blocks_in_dependent_materialized_views: 1 as const,
+ },
+ },
+ };
+ for (let attempt = 0; attempt < 3; attempt++) {
+ const [error] = await ch.queueMetrics.insertRaw(rows, retrySettings);
+ expect(error).toBeNull();
+ }
+
+ const [tiersError, tiers] = await ch.reader.query({
+ name: "dedup-tier-counts",
+ query: `SELECT
+ (SELECT count() FROM trigger_dev.queue_metrics_v1 WHERE organization_id = {org: String}) AS rows_10s,
+ (SELECT count() FROM trigger_dev.queue_metrics_5m_v1 WHERE organization_id = {org: String}) AS rows_5m,
+ (SELECT count() FROM trigger_dev.env_metrics_v1 WHERE organization_id = {org: String}) AS rows_env,
+ (SELECT sum(wait_ms_count) FROM trigger_dev.env_metrics_v1 WHERE organization_id = {org: String}) AS wait_count,
+ (SELECT deltaSumTimestampMerge(started_delta) FROM trigger_dev.queue_metrics_v1 WHERE organization_id = {org: String}) AS started`,
+ schema: z.object({
+ rows_10s: z.coerce.number(),
+ rows_5m: z.coerce.number(),
+ rows_env: z.coerce.number(),
+ wait_count: z.coerce.number(),
+ started: z.coerce.number(),
+ }),
+ params: z.object({ org: z.string() }),
+ })({ org: dedupOrg });
+ expect(tiersError).toBeNull();
+ const t = tiers![0]!;
+ // Without dedup windows on the MV targets, retries append copies: rows and sums triple.
+ expect(t.rows_10s).toBe(1);
+ expect(t.rows_5m).toBe(1);
+ expect(t.rows_env).toBe(1);
+ expect(t.wait_count).toBe(3);
+ expect(t.started).toBe(3);
+
+ await ch.close();
+ }
+ );
+});
+
+describe("per-concurrency-key tier", () => {
+ clickhouseTest(
+ "per-key rows feed the ck tier without polluting per-queue counters or waits",
+ async ({ clickhouseContainer }) => {
+ const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" });
+ const ckOrg = "org_qm_ck";
+ const queue = "ck-tier-q";
+ const withCk = (row: QueueMetricsRawV1Input, ck: string): QueueMetricsRawV1Input => ({
+ ...row,
+ concurrency_key: ck,
+ });
+
+ // 5 started events on one queue across two keys (t1 x3, t2 x2). Each event lands as
+ // a base row (base odometer) + a per-key row (per-key odometer), both carrying wait,
+ // exactly like the consumer expansion. Baselines seed each odometer.
+ const rows: QueueMetricsRawV1Input[] = [];
+ let ok = 0;
+ const started = (cum: number, ck: string, ckcum: number, wait: number) => {
+ rows.push({ ...base("started", queue), cumulative: cum, order_key: ok, wait_ms: wait });
+ rows.push(
+ withCk({ ...base("started", queue), cumulative: ckcum, order_key: ok, wait_ms: wait }, ck)
+ );
+ ok++;
+ };
+ rows.push({ ...base("started", queue), cumulative: 0, order_key: ok++ });
+ rows.push(withCk({ ...base("started", queue), cumulative: 0, order_key: ok++ }, "t1"));
+ rows.push(withCk({ ...base("started", queue), cumulative: 0, order_key: ok++ }, "t2"));
+ started(1, "t1", 1, 100);
+ started(2, "t1", 2, 200);
+ started(3, "t2", 1, 300);
+ started(4, "t1", 3, 400);
+ started(5, "t2", 2, 500);
+ // Per-subqueue gauges carry the key.
+ rows.push(withCk({ ...base("gauge", queue), queued: 4, running: 1 }, "t1"));
+ rows.push(withCk({ ...base("gauge", queue), queued: 2, running: 0 }, "t2"));
+
+ const [insertError] = await ch.queueMetrics.insertRaw(
+ rows.map((r) => ({ ...r, organization_id: ckOrg })),
+ SYNC
+ );
+ expect(insertError).toBeNull();
+
+ const [perQueueError, perQueue] = await ch.reader.query({
+ name: "ck-per-queue-read",
+ query: `SELECT
+ deltaSumTimestampMerge(started_delta) AS started,
+ sum(wait_ms_sum) AS wait_sum,
+ sum(wait_ms_count) AS wait_count,
+ max(max_queued) AS peak_queued
+ FROM trigger_dev.queue_metrics_v1
+ WHERE organization_id = {org: String}`,
+ schema: z.object({
+ started: z.coerce.number(),
+ wait_sum: z.coerce.number(),
+ wait_count: z.coerce.number(),
+ peak_queued: z.coerce.number(),
+ }),
+ params: z.object({ org: z.string() }),
+ })({ org: ckOrg });
+ expect(perQueueError).toBeNull();
+ // Base rows only: 5 events (not 10), waits counted once, per-key gauges still max in.
+ expect(perQueue![0]).toEqual({ started: 5, wait_sum: 1500, wait_count: 5, peak_queued: 4 });
+
+ const [ckError, ckRows] = await ch.reader.query({
+ name: "ck-tier-read",
+ query: `SELECT concurrency_key,
+ deltaSumTimestampMerge(started_delta) AS started,
+ max(max_queued) AS peak_queued,
+ sum(wait_ms_sum) AS wait_sum
+ FROM trigger_dev.queue_metrics_ck_v1
+ WHERE organization_id = {org: String}
+ GROUP BY concurrency_key ORDER BY concurrency_key`,
+ schema: z.object({
+ concurrency_key: z.string(),
+ started: z.coerce.number(),
+ peak_queued: z.coerce.number(),
+ wait_sum: z.coerce.number(),
+ }),
+ params: z.object({ org: z.string() }),
+ })({ org: ckOrg });
+ expect(ckError).toBeNull();
+ expect(ckRows).toEqual([
+ { concurrency_key: "t1", started: 3, peak_queued: 4, wait_sum: 700 },
+ { concurrency_key: "t2", started: 2, peak_queued: 2, wait_sum: 800 },
+ ]);
+
+ await ch.close();
+ }
+ );
+});
diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts
new file mode 100644
index 00000000000..d86645e5153
--- /dev/null
+++ b/internal-packages/clickhouse/src/queueMetrics.ts
@@ -0,0 +1,222 @@
+import { z } from "zod";
+import type { ClickhouseReader, ClickhouseWriter } from "./client/types.js";
+
+export const QueueMetricsRawV1Input = z.object({
+ organization_id: z.string(),
+ project_id: z.string(),
+ environment_id: z.string(),
+ queue_name: z.string(),
+ concurrency_key: z.string().optional(),
+ event_time: z.string(),
+ // Exact UInt64 ordering key; a string preserves precision past JS safe-integer range
+ // (see entryOrderKey). A plain number is still accepted for small test values.
+ order_key: z.union([z.string(), z.number()]).optional(),
+ op: z.enum(["gauge", "enqueue", "started", "ack", "nack", "dlq"]),
+ running: z.number().optional(),
+ queued: z.number().optional(),
+ queue_limit: z.number().optional(),
+ env_running: z.number().optional(),
+ env_queued: z.number().optional(),
+ env_limit: z.number().optional(),
+ throttled: z.number().optional(),
+ ck_backlogged: z.number().optional(),
+ ck_max_wait_ms: z.number().optional(),
+ wait_ms: z.number().optional(),
+ cumulative: z.number().optional(),
+});
+
+export type QueueMetricsRawV1Input = z.input;
+
+export function insertQueueMetricsRaw(ch: ClickhouseWriter) {
+ return ch.insertUnsafe({
+ name: "insertQueueMetricsRaw",
+ table: "trigger_dev.queue_metrics_raw_v1",
+ });
+}
+
+// --- Reads (Queues list metrics + health) ---
+
+const QueueMetricsListParams = z.object({
+ organizationId: z.string(),
+ projectId: z.string(),
+ environmentId: z.string(),
+ queueNames: z.array(z.string()),
+ startTime: z.string(),
+ endTime: z.string(),
+});
+
+const QueueMetricsSummaryRow = z.object({
+ queue_name: z.string(),
+ p50_wait_ms: z.coerce.number(),
+ p95_wait_ms: z.coerce.number(),
+ peak_queued: z.coerce.number(),
+ started_count: z.coerce.number(),
+ throttled_count: z.coerce.number(),
+});
+
+// Callers align window bounds to the bucket grid so repeated loads share cache entries.
+const QUEUE_METRICS_CACHE_SETTINGS = {
+ use_query_cache: 1,
+ query_cache_ttl: 30,
+} as const;
+
+/** Per-queue rollups over a window, for a fixed set of queues (the visible page). */
+export function getQueueListMetricsSummary(reader: ClickhouseReader) {
+ return reader.query({
+ name: "getQueueListMetricsSummary",
+ query: `SELECT
+ queue_name,
+ round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50_wait_ms,
+ round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95_wait_ms,
+ max(max_queued) AS peak_queued,
+ deltaSumTimestampMerge(started_delta) AS started_count,
+ sum(throttled_count) AS throttled_count
+ FROM trigger_dev.queue_metrics_v1
+ WHERE organization_id = {organizationId: String}
+ AND project_id = {projectId: String}
+ AND environment_id = {environmentId: String}
+ AND queue_name IN {queueNames: Array(String)}
+ AND bucket_start >= {startTime: DateTime}
+ AND bucket_start < {endTime: DateTime}
+ GROUP BY queue_name`,
+ params: QueueMetricsListParams,
+ schema: QueueMetricsSummaryRow,
+ settings: QUEUE_METRICS_CACHE_SETTINGS,
+ });
+}
+
+const QueueDepthSparklineParams = QueueMetricsListParams.extend({
+ bucketSeconds: z.number(),
+});
+
+const QueueDepthSparklineRow = z.object({
+ queue_name: z.string(),
+ bucket: z.string(),
+ depth: z.coerce.number(),
+ throttled: z.coerce.number(),
+});
+
+/**
+ * Per-queue, per-bucket peak depth (carry-forward filled by the caller) plus the throttled
+ * count in each bucket, so the sparkline can tint the exact buckets where throttling occurred.
+ * The extra aggregate rides the same scan as the depth series — no additional round trip.
+ */
+export function getQueueDepthSparklines(reader: ClickhouseReader) {
+ return reader.query({
+ name: "getQueueDepthSparklines",
+ query: `SELECT
+ queue_name,
+ toStartOfInterval(bucket_start, toIntervalSecond({bucketSeconds: UInt32})) AS bucket,
+ max(max_queued) AS depth,
+ sum(throttled_count) AS throttled
+ FROM trigger_dev.queue_metrics_v1
+ WHERE organization_id = {organizationId: String}
+ AND project_id = {projectId: String}
+ AND environment_id = {environmentId: String}
+ AND queue_name IN {queueNames: Array(String)}
+ AND bucket_start >= {startTime: DateTime}
+ AND bucket_start < {endTime: DateTime}
+ GROUP BY queue_name, bucket
+ ORDER BY bucket`,
+ params: QueueDepthSparklineParams,
+ schema: QueueDepthSparklineRow,
+ settings: QUEUE_METRICS_CACHE_SETTINGS,
+ });
+}
+
+const QueueRankingParams = z.object({
+ organizationId: z.string(),
+ projectId: z.string(),
+ environmentId: z.string(),
+ startTime: z.string(),
+ /** 1 = rank by peak backlog only; 0 = backlog + running ("busiest"). */
+ byQueuedOnly: z.number(),
+ nameContains: z.string(),
+ limit: z.number(),
+ offset: z.number(),
+});
+
+const QueueRankingRow = z.object({
+ queue_name: z.string(),
+ ranked_total: z.coerce.number(),
+});
+
+// Ranking reads the 5m rollup: a 15-minute window there costs ~30x fewer rows than the
+// 10s table.
+const RANKING_WHERE = `organization_id = {organizationId: String}
+ AND project_id = {projectId: String}
+ AND environment_id = {environmentId: String}
+ AND bucket_start >= {startTime: DateTime}
+ AND queue_name != '__overflow__'
+ AND ({nameContains: String} = '' OR positionCaseInsensitive(queue_name, {nameContains: String}) > 0)`;
+
+/**
+ * One page of queue names ranked by recent activity, with the total ranked count on
+ * every row (window function), so page + count cost a single scan.
+ */
+export function getQueueRanking(reader: ClickhouseReader) {
+ return reader.query({
+ name: "getQueueRanking",
+ query: `SELECT queue_name, count() OVER () AS ranked_total
+ FROM (
+ SELECT queue_name
+ FROM trigger_dev.queue_metrics_5m_v1
+ WHERE ${RANKING_WHERE}
+ GROUP BY queue_name
+ ORDER BY
+ if({byQueuedOnly: UInt8} = 1, max(max_queued), max(max_queued) + max(max_running)) DESC,
+ queue_name ASC
+ )
+ LIMIT {limit: UInt32} OFFSET {offset: UInt32}`,
+ params: QueueRankingParams,
+ schema: QueueRankingRow,
+ settings: QUEUE_METRICS_CACHE_SETTINGS,
+ });
+}
+
+const QueueRankingNamesParams = QueueRankingParams.omit({ byQueuedOnly: true, offset: true });
+
+const QueueRankingNameRow = z.object({
+ queue_name: z.string(),
+});
+
+/** All ranked queue names (activity order), used to exclude them from the alphabetical tail. */
+export function getQueueRankingNames(reader: ClickhouseReader) {
+ return reader.query({
+ name: "getQueueRankingNames",
+ query: `SELECT queue_name
+ FROM trigger_dev.queue_metrics_5m_v1
+ WHERE ${RANKING_WHERE}
+ GROUP BY queue_name
+ ORDER BY max(max_queued) + max(max_running) DESC, queue_name ASC
+ LIMIT {limit: UInt32}`,
+ params: QueueRankingNamesParams,
+ schema: QueueRankingNameRow,
+ settings: QUEUE_METRICS_CACHE_SETTINGS,
+ });
+}
+
+const QueueRankingCountParams = QueueRankingParams.omit({
+ byQueuedOnly: true,
+ limit: true,
+ offset: true,
+});
+
+const QueueRankingCountRow = z.object({
+ ranked: z.coerce.number(),
+});
+
+/** Ranked-queue count alone, for pages past the ranked head (approximate uniq is fine). */
+export function getQueueRankingCount(reader: ClickhouseReader) {
+ return reader.query({
+ name: "getQueueRankingCount",
+ query: `SELECT uniq(queue_name) AS ranked
+ FROM trigger_dev.queue_metrics_5m_v1
+ WHERE ${RANKING_WHERE}`,
+ params: QueueRankingCountParams,
+ schema: QueueRankingCountRow,
+ settings: QUEUE_METRICS_CACHE_SETTINGS,
+ });
+}
+
+// (per-queue detail series is now fetched via TRQL + fillGaps from the metric resource route)
diff --git a/internal-packages/database/prisma/migrations/20260716120000_add_task_queue_concurrency_limit_override_percent/migration.sql b/internal-packages/database/prisma/migrations/20260716120000_add_task_queue_concurrency_limit_override_percent/migration.sql
new file mode 100644
index 00000000000..cf1aa0ed84e
--- /dev/null
+++ b/internal-packages/database/prisma/migrations/20260716120000_add_task_queue_concurrency_limit_override_percent/migration.sql
@@ -0,0 +1,5 @@
+-- Adds a nullable column to store a queue concurrency-limit override expressed as a percentage
+-- of the environment limit. This percentage is the source of truth; the absolute
+-- "concurrencyLimit" is materialized from it at save time. Additive and nullable, so existing
+-- absolute overrides are unaffected. Decimal(5,2) supports fractional percentages (0.01–100.00).
+ALTER TABLE "TaskQueue" ADD COLUMN IF NOT EXISTS "concurrencyLimitOverridePercent" DECIMAL(5,2);
diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma
index 1b8642e866f..b861b80d83a 100644
--- a/internal-packages/database/prisma/schema.prisma
+++ b/internal-packages/database/prisma/schema.prisma
@@ -1829,14 +1829,18 @@ model TaskQueue {
runtimeEnvironmentId String
/// Represents the current concurrency limit for the queue
- concurrencyLimit Int?
+ concurrencyLimit Int?
/// When the concurrency limit was overridden
- concurrencyLimitOverriddenAt DateTime?
+ concurrencyLimitOverriddenAt DateTime?
/// Who overrode the concurrency limit (will be null if overridden via the API)
- concurrencyLimitOverriddenBy String?
+ concurrencyLimitOverriddenBy String?
/// If concurrencyLimit is overridden, this is the overridden value
- concurrencyLimitBase Int?
- rateLimit Json?
+ concurrencyLimitBase Int?
+ /// If the override was expressed as a percentage of the environment limit, this stores that
+ /// percentage (the source of truth). The absolute concurrencyLimit is materialized from it.
+ /// Decimal(5,2) allows fractional percentages like 12.50% (0.01–100.00).
+ concurrencyLimitOverridePercent Decimal? @db.Decimal(5, 2)
+ rateLimit Json?
paused Boolean @default(false)
diff --git a/internal-packages/metrics-pipeline/package.json b/internal-packages/metrics-pipeline/package.json
new file mode 100644
index 00000000000..10a7c137a1f
--- /dev/null
+++ b/internal-packages/metrics-pipeline/package.json
@@ -0,0 +1,33 @@
+{
+ "name": "@internal/metrics-pipeline",
+ "private": true,
+ "version": "0.0.1",
+ "main": "./dist/src/index.js",
+ "types": "./dist/src/index.d.ts",
+ "type": "module",
+ "exports": {
+ ".": {
+ "@triggerdotdev/source": "./src/index.ts",
+ "import": "./dist/src/index.js",
+ "types": "./dist/src/index.d.ts",
+ "default": "./dist/src/index.js"
+ }
+ },
+ "dependencies": {
+ "@internal/redis": "workspace:*",
+ "@internal/tracing": "workspace:*",
+ "@trigger.dev/core": "workspace:*"
+ },
+ "devDependencies": {
+ "@internal/testcontainers": "workspace:*",
+ "rimraf": "6.0.1"
+ },
+ "scripts": {
+ "clean": "rimraf dist",
+ "typecheck": "tsc --noEmit -p tsconfig.build.json",
+ "test": "vitest --sequence.concurrent=false --no-file-parallelism",
+ "test:coverage": "vitest --sequence.concurrent=false --no-file-parallelism --coverage.enabled",
+ "build": "pnpm run clean && tsc -p tsconfig.build.json",
+ "dev": "tsc --watch -p tsconfig.build.json"
+ }
+}
diff --git a/internal-packages/metrics-pipeline/src/cachedValue.ts b/internal-packages/metrics-pipeline/src/cachedValue.ts
new file mode 100644
index 00000000000..7f7bbb07903
--- /dev/null
+++ b/internal-packages/metrics-pipeline/src/cachedValue.ts
@@ -0,0 +1,125 @@
+import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis";
+import { Logger } from "@trigger.dev/core/logger";
+
+export type CachedRedisValueOptions = {
+ redis: RedisOptions;
+ key: string;
+ parse: (raw: string | null) => T;
+ defaultValue: T;
+ cacheTtlMs?: number;
+ logger?: Logger;
+ loggerName?: string;
+};
+
+// Reads a Redis key with a short stale-while-revalidate cache and a synchronous getter for
+// hot paths. Warms eagerly on construction; concurrent refreshes dedupe onto one GET so an
+// awaited refresh always resolves to a completed read.
+export class CachedRedisValue {
+ private readonly redis: Redis;
+ private readonly key: string;
+ private readonly parse: (raw: string | null) => T;
+ private readonly cacheTtlMs: number;
+ private readonly logger: Logger;
+ private value: T;
+ private lastFetchedAt = 0;
+ private refreshPromise?: Promise;
+
+ constructor(options: CachedRedisValueOptions) {
+ this.logger = options.logger ?? new Logger(options.loggerName ?? "CachedRedisValue", "warn");
+ this.redis = createRedisClient(
+ { ...options.redis, keyPrefix: undefined },
+ {
+ onError: (error) =>
+ this.logger.error("cached value redis error", { error, key: options.key }),
+ }
+ );
+ this.key = options.key;
+ this.parse = options.parse;
+ this.cacheTtlMs = options.cacheTtlMs ?? 10_000;
+ this.value = options.defaultValue;
+ void this.refresh();
+ }
+
+ get(): T {
+ if (Date.now() - this.lastFetchedAt > this.cacheTtlMs) {
+ void this.refresh();
+ }
+ return this.value;
+ }
+
+ async refresh(): Promise {
+ if (this.refreshPromise) return this.refreshPromise;
+ this.refreshPromise = this.#doRefresh();
+ try {
+ return await this.refreshPromise;
+ } finally {
+ this.refreshPromise = undefined;
+ }
+ }
+
+ async #doRefresh(): Promise {
+ try {
+ this.value = this.parse(await this.redis.get(this.key));
+ } catch (error) {
+ this.logger.debug("cached value refresh failed, keeping cached value", {
+ error,
+ key: this.key,
+ });
+ } finally {
+ this.lastFetchedAt = Date.now();
+ }
+ return this.value;
+ }
+
+ async close(): Promise {
+ await this.redis.quit();
+ }
+}
+
+export type CachedRedisNumberOptions = {
+ redis: RedisOptions;
+ key: string;
+ defaultValue: number;
+ min?: number;
+ max?: number;
+ cacheTtlMs?: number;
+ logger?: Logger;
+};
+
+// Live-tunable numeric value, clamped to [min,max]; falls back to defaultValue on a
+// missing/unparseable key. Exposes a synchronous value() for hot paths.
+export class CachedRedisNumber {
+ private readonly inner: CachedRedisValue;
+
+ constructor(options: CachedRedisNumberOptions) {
+ const min = options.min ?? Number.NEGATIVE_INFINITY;
+ const max = options.max ?? Number.POSITIVE_INFINITY;
+ const clamp = (n: number) => Math.min(max, Math.max(min, n));
+ const fallback = clamp(options.defaultValue);
+ this.inner = new CachedRedisValue({
+ redis: options.redis,
+ key: options.key,
+ parse: (raw) => {
+ // Number("") is 0 (not NaN), so treat blank/whitespace as missing => fallback.
+ const n = raw == null || raw.trim() === "" ? Number.NaN : Number(raw);
+ return Number.isFinite(n) ? clamp(n) : fallback;
+ },
+ defaultValue: fallback,
+ cacheTtlMs: options.cacheTtlMs,
+ logger: options.logger,
+ loggerName: "CachedRedisNumber",
+ });
+ }
+
+ value(): number {
+ return this.inner.get();
+ }
+
+ refresh(): Promise {
+ return this.inner.refresh();
+ }
+
+ close(): Promise {
+ return this.inner.close();
+ }
+}
diff --git a/internal-packages/metrics-pipeline/src/consumer.test.ts b/internal-packages/metrics-pipeline/src/consumer.test.ts
new file mode 100644
index 00000000000..672fa426999
--- /dev/null
+++ b/internal-packages/metrics-pipeline/src/consumer.test.ts
@@ -0,0 +1,392 @@
+import { createRedisClient } from "@internal/redis";
+import { redisTest } from "@internal/testcontainers";
+import { expect } from "vitest";
+import { CachedRedisFlag } from "./flag.js";
+import { CachedRedisNumber } from "./cachedValue.js";
+import { MetricsStreamConsumer } from "./consumer.js";
+import { MetricsStreamEmitter } from "./emitter.js";
+import { shardFor } from "./hash.js";
+import { streamKey, type MetricDefinition } from "./types.js";
+
+async function waitFor(cond: () => boolean, timeoutMs = 5000): Promise {
+ const start = Date.now();
+ while (!cond()) {
+ if (Date.now() - start > timeoutMs) throw new Error("waitFor timed out");
+ await new Promise((r) => setTimeout(r, 50));
+ }
+}
+
+function definitionFor(suffix: string, shardCount = 2): MetricDefinition {
+ return { name: `qm_${Date.now()}_${suffix}`, shardCount, consumerGroup: "cg", maxLen: 1000 };
+}
+
+redisTest(
+ "emitter -> consumer round trip maps rows, dedups, and acks",
+ async ({ redisOptions }) => {
+ const definition = definitionFor("rt");
+ const emitter = new MetricsStreamEmitter({
+ redis: redisOptions,
+ definition,
+ flag: { enabled: () => true },
+ });
+ const inserted: Array<{ rows: Array>; dedupToken: string }> = [];
+
+ const consumer = new MetricsStreamConsumer>({
+ redis: redisOptions,
+ definition,
+ consumerName: "c1",
+ mapEntry: (e) => ({ id: e.id, ...e.fields }),
+ insert: async (rows, { dedupToken }) => {
+ inserted.push({ rows, dedupToken });
+ },
+ blockMs: 200,
+ });
+
+ await consumer.start();
+ emitter.emit("queueA", { op: "enqueue", q: "queueA" });
+ emitter.emit("queueB", { op: "started", q: "queueB", wait: 42 });
+
+ await waitFor(() => inserted.flatMap((i) => i.rows).length >= 2);
+ await consumer.stop();
+
+ const rows = inserted.flatMap((i) => i.rows);
+ expect(rows).toContainEqual(expect.objectContaining({ op: "enqueue", q: "queueA" }));
+ expect(rows).toContainEqual(
+ expect.objectContaining({ op: "started", q: "queueB", wait: "42" })
+ );
+ expect(inserted[0]!.dedupToken).toMatch(/^[0-9a-f]{40}$/);
+
+ const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined });
+ for (const key of consumer.streamKeys()) {
+ const pending = (await admin.xpending(key, definition.consumerGroup)) as [
+ number,
+ ...unknown[],
+ ];
+ expect(pending[0]).toBe(0);
+ }
+ await admin.quit();
+ await emitter.close();
+ }
+);
+
+redisTest("emit is a no-op when the flag is disabled", async ({ redisOptions }) => {
+ const definition = definitionFor("off");
+ const emitter = new MetricsStreamEmitter({
+ redis: redisOptions,
+ definition,
+ flag: { enabled: () => false },
+ });
+
+ emitter.emit("q", { op: "enqueue", q: "q" });
+ await new Promise((r) => setTimeout(r, 200));
+
+ const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined });
+ const len = await admin.xlen(streamKey(definition, shardFor("q", definition.shardCount)));
+ expect(len).toBe(0);
+ await admin.quit();
+ await emitter.close();
+});
+
+redisTest("reclaims stale pending entries from a dead consumer", async ({ redisOptions }) => {
+ const definition = definitionFor("claim", 1);
+ const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined });
+ const key = streamKey(definition, 0);
+
+ await admin.xgroup("CREATE", key, definition.consumerGroup, "$", "MKSTREAM");
+ await admin.xadd(key, "*", "op", "ack", "q", "qZ");
+ await admin.xadd(key, "*", "op", "nack", "q", "qZ");
+ await admin.xreadgroup(
+ "GROUP",
+ definition.consumerGroup,
+ "zombie",
+ "COUNT",
+ 10,
+ "STREAMS",
+ key,
+ ">"
+ );
+
+ const inserted: Array> = [];
+ const consumer = new MetricsStreamConsumer>({
+ redis: redisOptions,
+ definition,
+ consumerName: "live",
+ mapEntry: (e) => ({ id: e.id, ...e.fields }),
+ insert: async (rows) => {
+ inserted.push(...rows);
+ },
+ blockMs: 200,
+ claimIdleMs: 0,
+ });
+
+ await consumer.start();
+ await waitFor(() => inserted.length >= 2);
+ await consumer.stop();
+
+ expect(inserted.map((r) => r.op).sort()).toEqual(["ack", "nack"]);
+ const pending = (await admin.xpending(key, definition.consumerGroup)) as [number, ...unknown[]];
+ expect(pending[0]).toBe(0);
+ await admin.quit();
+});
+
+redisTest(
+ "per-stream batches: one insert + distinct dedup token per shard stream",
+ async ({ redisOptions }) => {
+ const definition = definitionFor("pershard", 2);
+ const emitter = new MetricsStreamEmitter({
+ redis: redisOptions,
+ definition,
+ flag: { enabled: () => true },
+ });
+ // Two shard keys that land on different shards.
+ const a = "shardkey-a";
+ let b = "shardkey-b0";
+ for (let i = 1; shardFor(b, 2) === shardFor(a, 2); i++) b = `shardkey-b${i}`;
+
+ const inserted: Array<{ rows: Array>; dedupToken: string }> = [];
+ const consumer = new MetricsStreamConsumer>({
+ redis: redisOptions,
+ definition,
+ consumerName: "c1",
+ mapEntry: (e) => ({ id: e.id, ...e.fields }),
+ insert: async (rows, { dedupToken }) => {
+ inserted.push({ rows, dedupToken });
+ },
+ blockMs: 200,
+ });
+
+ await consumer.start();
+ emitter.emit(a, { op: "enqueue", q: a });
+ emitter.emit(b, { op: "enqueue", q: b });
+ await waitFor(() => inserted.flatMap((i) => i.rows).length >= 2);
+ await consumer.stop();
+ await emitter.close();
+
+ // Each shard's batch is its own dedup block with its own (stream-scoped) token.
+ const batchesWithRows = inserted.filter((i) => i.rows.length > 0);
+ expect(batchesWithRows.length).toBe(2);
+ expect(new Set(batchesWithRows.map((i) => i.dedupToken)).size).toBe(2);
+ }
+);
+
+redisTest(
+ "probe reports lag as null (not 0) when Redis cannot compute it",
+ async ({ redisOptions }) => {
+ const definition = definitionFor("nillag", 1);
+ const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined });
+ const key = streamKey(definition, 0);
+
+ await admin.xgroup("CREATE", key, definition.consumerGroup, "0", "MKSTREAM");
+ const ids: string[] = [];
+ for (let i = 0; i < 5; i++) {
+ ids.push((await admin.xadd(key, "*", "op", "enqueue", "q", "qT")) as string);
+ }
+ // SETID to an arbitrary id makes the group's entries-read unknown => lag is nil
+ // (severe trimming can do the same in prod); the probe must NOT report that as 0.
+ await admin.xgroup("SETID", key, definition.consumerGroup, ids[2]!);
+
+ const consumer = new MetricsStreamConsumer>({
+ redis: redisOptions,
+ definition,
+ consumerName: "c1",
+ mapEntry: (e) => ({ id: e.id, ...e.fields }),
+ insert: async () => {},
+ });
+ try {
+ const states = await consumer.streamState();
+ expect(states[0]!.lag).toBeNull();
+ } finally {
+ await consumer.stop();
+ await admin.quit();
+ }
+ }
+);
+
+redisTest(
+ "emitGauge XADDs an op=gauge snapshot onto the shared metrics stream",
+ async ({ redisOptions }) => {
+ const definition = definitionFor("gauge", 2);
+ const emitter = new MetricsStreamEmitter({
+ redis: redisOptions,
+ definition,
+ flag: { enabled: () => true },
+ });
+
+ // Emits before the connection is ready are dropped by design (loss-tolerant).
+ await emitter.waitUntilReady();
+ emitter.emitGauge("q1", {
+ op: "gauge",
+ q: "q1",
+ ql: 5,
+ cc: 2,
+ lim: 10,
+ eql: 3,
+ ec: 1,
+ elim: 20,
+ thr: 0,
+ });
+
+ const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined });
+ const key = streamKey(definition, shardFor("q1", 2));
+ // Plain XADD (no odometer, no cum=0 seed) => exactly one entry, unlike counter emit().
+ await waitFor2(async () => (await admin.xlen(key)) === 1);
+ const raw = (await admin.xrange(key, "-", "+")) as Array<[string, string[]]>;
+ const flat = raw[0]![1];
+ const fields: Record = {};
+ for (let i = 0; i + 1 < flat.length; i += 2) fields[flat[i]!] = flat[i + 1]!;
+ expect(fields.op).toBe("gauge");
+ expect(fields.q).toBe("q1");
+ expect(fields.ql).toBe("5");
+ expect(fields.thr).toBe("0");
+ await admin.quit();
+ await emitter.close();
+ }
+);
+
+async function waitFor2(cond: () => Promise, timeoutMs = 5000): Promise {
+ const start = Date.now();
+ while (!(await cond())) {
+ if (Date.now() - start > timeoutMs) throw new Error("waitFor2 timed out");
+ await new Promise((r) => setTimeout(r, 50));
+ }
+}
+
+redisTest("sampledSync gates on both the flag and the sample rate", async ({ redisOptions }) => {
+ const definition = definitionFor("sample");
+ const off = new MetricsStreamEmitter({
+ redis: redisOptions,
+ definition,
+ flag: { enabled: () => true },
+ gaugeSampleRate: 0,
+ });
+ const on = new MetricsStreamEmitter({
+ redis: redisOptions,
+ definition,
+ flag: { enabled: () => true },
+ gaugeSampleRate: 1,
+ });
+ const disabled = new MetricsStreamEmitter({
+ redis: redisOptions,
+ definition,
+ flag: { enabled: () => false },
+ gaugeSampleRate: 1,
+ });
+
+ expect(off.sampledSync()).toBe(false); // rate 0 => never sampled in
+ expect(on.sampledSync()).toBe(true); // rate 1 + enabled => always
+ expect(disabled.sampledSync()).toBe(false); // disabled => never, regardless of rate
+ expect(on.enabledSync()).toBe(true); // enabledSync (counters) is unaffected by sampling
+
+ await Promise.all([off.close(), on.close(), disabled.close()]);
+});
+
+redisTest("sampledSync honors a live rate provider (no reconstruct)", async ({ redisOptions }) => {
+ const definition = definitionFor("live");
+ let rate = 1;
+ const emitter = new MetricsStreamEmitter({
+ redis: redisOptions,
+ definition,
+ flag: { enabled: () => true },
+ gaugeSampleRate: { value: () => rate },
+ });
+ expect(emitter.sampledSync()).toBe(true);
+ rate = 0;
+ expect(emitter.sampledSync()).toBe(false);
+ await emitter.close();
+});
+
+redisTest("CachedRedisNumber reads live, clamps, and falls back", async ({ redisOptions }) => {
+ const key = `rate_${Date.now()}`;
+ const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined });
+ const num = new CachedRedisNumber({ redis: redisOptions, key, defaultValue: 1, min: 0, max: 1 });
+
+ await num.refresh();
+ expect(num.value()).toBe(1); // missing key => default
+ await admin.set(key, "0.25");
+ await num.refresh();
+ expect(num.value()).toBe(0.25);
+ await admin.set(key, "5");
+ await num.refresh();
+ expect(num.value()).toBe(1); // out of range => clamped
+ await admin.set(key, "nonsense");
+ await num.refresh();
+ expect(num.value()).toBe(1); // unparseable => default
+
+ await num.close();
+ await admin.quit();
+});
+
+redisTest("streamState reports depth, lag, and pending per shard", async ({ redisOptions }) => {
+ const definition = definitionFor("state", 1);
+ const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined });
+ const key = streamKey(definition, 0);
+
+ await admin.xgroup("CREATE", key, definition.consumerGroup, "$", "MKSTREAM");
+ await admin.xadd(key, "*", "op", "enqueue", "q", "qX");
+ await admin.xadd(key, "*", "op", "ack", "q", "qX");
+ // Read one entry as some consumer and leave it unacked -> 1 pending, 1 still undelivered.
+ await admin.xreadgroup(
+ "GROUP",
+ definition.consumerGroup,
+ "reader",
+ "COUNT",
+ 1,
+ "STREAMS",
+ key,
+ ">"
+ );
+
+ const consumer = new MetricsStreamConsumer>({
+ redis: redisOptions,
+ definition,
+ consumerName: "c1",
+ mapEntry: (e) => ({ id: e.id, ...e.fields }),
+ insert: async () => {},
+ });
+
+ try {
+ const states = await consumer.streamState();
+ expect(states).toHaveLength(1);
+ expect(states[0]!.depth).toBe(2);
+ expect(states[0]!.pending).toBe(1);
+ expect(states[0]!.lag).toBe(1);
+ } finally {
+ await consumer.stop();
+ await admin.quit();
+ }
+});
+
+redisTest("CachedRedisFlag reads a redis key with caching", async ({ redisOptions }) => {
+ const key = `flag_${Date.now()}`;
+ const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined });
+ const flag = new CachedRedisFlag({ redis: redisOptions, key, cacheTtlMs: 10_000 });
+
+ expect(flag.enabled()).toBe(false);
+ await flag.refresh();
+ expect(flag.enabled()).toBe(false);
+
+ await admin.set(key, "1");
+ await flag.refresh();
+ expect(flag.enabled()).toBe(true);
+
+ await admin.set(key, "0");
+ await flag.refresh();
+ expect(flag.enabled()).toBe(false);
+
+ await flag.close();
+ await admin.quit();
+});
+
+redisTest("CachedRedisFlag warms eagerly on construction", async ({ redisOptions }) => {
+ const key = `flag_eager_${Date.now()}`;
+ const admin = createRedisClient({ ...redisOptions, keyPrefix: undefined });
+ await admin.set(key, "1");
+
+ const flag = new CachedRedisFlag({ redis: redisOptions, key });
+ // No manual refresh(): the constructor kicks one off so the first real read is warm.
+ await waitFor(() => flag.enabled() === true);
+ expect(flag.enabled()).toBe(true);
+
+ await flag.close();
+ await admin.quit();
+});
diff --git a/internal-packages/metrics-pipeline/src/consumer.ts b/internal-packages/metrics-pipeline/src/consumer.ts
new file mode 100644
index 00000000000..9e333e70ab1
--- /dev/null
+++ b/internal-packages/metrics-pipeline/src/consumer.ts
@@ -0,0 +1,336 @@
+import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis";
+import {
+ getMeter,
+ type Counter,
+ type Histogram,
+ type Meter,
+ type ObservableGauge,
+ ValueType,
+} from "@internal/tracing";
+import { Logger } from "@trigger.dev/core/logger";
+import { dedupTokenFromEntryIds } from "./idempotency.js";
+import { allStreamKeys, type MetricDefinition, type StreamEntry } from "./types.js";
+
+export type MetricsStreamConsumerOptions = {
+ redis: RedisOptions;
+ definition: MetricDefinition;
+ /** Unique per process; distinct replicas MUST use distinct names (PEL ownership). */
+ consumerName: string;
+ /** Map a stream entry to a row, or null to drop it (still acked). */
+ mapEntry: (entry: StreamEntry) => TRow | TRow[] | null;
+ /** Insert a batch. Must be idempotent w.r.t. dedupToken; throw to retry the batch. */
+ insert: (rows: TRow[], opts: { dedupToken: string }) => Promise;
+ batchSize?: number;
+ blockMs?: number;
+ claimIdleMs?: number;
+ /** How often to scan for stale pending entries (XAUTOCLAIM); not every poll. */
+ reclaimIntervalMs?: number;
+ errorBackoffMs?: number;
+ logger?: Logger;
+ meter?: Meter;
+};
+
+type RawEntry = [id: string, fields: string[]];
+type RawStream = [key: string, entries: RawEntry[]];
+
+/** Per-shard stream health, surfaced as observable gauges and usable directly in tests.
+ * `lag: null` means Redis could not compute it (entries trimmed past the group's read
+ * position) — treat as an alert, NOT as zero: it coincides with data loss. */
+export type ShardState = { shard: number; depth: number; lag: number | null; pending: number };
+
+function parseFields(flat: string[]): Record {
+ const out: Record = {};
+ for (let i = 0; i + 1 < flat.length; i += 2) {
+ out[flat[i]!] = flat[i + 1]!;
+ }
+ return out;
+}
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Reads a sharded metrics stream via a consumer group, inserting each stream's poll-batch
+ * as its own dedup block (so an XAUTOCLAIM-reclaimed batch re-forms the same id set and
+ * token), acking only after a successful insert. Sequential read/insert/ack per process.
+ */
+export class MetricsStreamConsumer {
+ private readonly redis: Redis;
+ private readonly probeRedis: Redis;
+ private readonly def: MetricDefinition;
+ private readonly keys: string[];
+ private readonly consumerName: string;
+ private readonly batchSize: number;
+ private readonly blockMs: number;
+ private readonly claimIdleMs: number;
+ private readonly reclaimIntervalMs: number;
+ private lastReclaimAt = 0;
+ private readonly errorBackoffMs: number;
+ private readonly logger: Logger;
+ private readonly mapEntry: (entry: StreamEntry) => TRow | TRow[] | null;
+ private readonly insert: (rows: TRow[], opts: { dedupToken: string }) => Promise;
+
+ private readonly meter: Meter;
+ private readonly entriesCounter: Counter;
+ private readonly rowsCounter: Counter;
+ private readonly insertErrorCounter: Counter;
+ private readonly insertDuration: Histogram;
+ private readonly observables: ObservableGauge[];
+ private readonly batchCallback: Parameters[0];
+
+ private running = false;
+ private loopPromise?: Promise;
+
+ constructor(options: MetricsStreamConsumerOptions) {
+ this.logger = options.logger ?? new Logger("MetricsStreamConsumer", "info");
+ const redisConfig = { ...options.redis, keyPrefix: undefined };
+ this.redis = createRedisClient(redisConfig, {
+ onError: (error) => this.logger.error("consumer redis error", { error }),
+ });
+ // Separate client so the observable-gauge probes never queue behind the blocking XREADGROUP.
+ this.probeRedis = createRedisClient(redisConfig, {
+ onError: (error) => this.logger.error("consumer probe redis error", { error }),
+ });
+ this.def = options.definition;
+ this.keys = allStreamKeys(options.definition);
+ this.consumerName = options.consumerName;
+ this.batchSize = options.batchSize ?? 1000;
+ this.blockMs = options.blockMs ?? 1000;
+ this.claimIdleMs = options.claimIdleMs ?? 60_000;
+ this.reclaimIntervalMs = options.reclaimIntervalMs ?? 15_000;
+ this.errorBackoffMs = options.errorBackoffMs ?? 1000;
+ this.mapEntry = options.mapEntry;
+ this.insert = options.insert;
+
+ this.meter = options.meter ?? getMeter("metrics-pipeline");
+ this.entriesCounter = this.meter.createCounter("queue_metrics.consumer.entries", {
+ description: "Stream entries read (attr source=new|reclaimed)",
+ valueType: ValueType.INT,
+ });
+ this.rowsCounter = this.meter.createCounter("queue_metrics.consumer.rows_inserted", {
+ description: "Rows inserted into the sink",
+ valueType: ValueType.INT,
+ });
+ this.insertErrorCounter = this.meter.createCounter("queue_metrics.consumer.insert_errors", {
+ description: "Failed inserts (batch left pending for retry)",
+ valueType: ValueType.INT,
+ });
+ this.insertDuration = this.meter.createHistogram("queue_metrics.consumer.insert_duration", {
+ description: "Sink insert latency",
+ unit: "ms",
+ valueType: ValueType.INT,
+ });
+
+ const depthGauge = this.meter.createObservableGauge("queue_metrics.consumer.stream_depth", {
+ description: "Entries currently in each shard stream (approaches MAXLEN => trimming)",
+ valueType: ValueType.INT,
+ });
+ const lagGauge = this.meter.createObservableGauge("queue_metrics.consumer.group_lag", {
+ description: "Entries not yet delivered to the consumer group (consumer falling behind)",
+ valueType: ValueType.INT,
+ });
+ const pendingGauge = this.meter.createObservableGauge("queue_metrics.consumer.pending", {
+ description: "Unacked (in-flight or stuck) entries in the group PEL",
+ valueType: ValueType.INT,
+ });
+ const lagUnknownGauge = this.meter.createObservableGauge("queue_metrics.consumer.lag_unknown", {
+ description:
+ "1 when Redis cannot compute group lag (entries trimmed => data loss); alert on this",
+ valueType: ValueType.INT,
+ });
+ this.observables = [depthGauge, lagGauge, pendingGauge, lagUnknownGauge];
+ this.batchCallback = async (result) => {
+ const states = await this.streamState();
+ for (const s of states) {
+ const attrs = { stream: this.def.name, shard: String(s.shard) };
+ result.observe(depthGauge, s.depth, attrs);
+ if (s.lag !== null) result.observe(lagGauge, s.lag, attrs);
+ result.observe(lagUnknownGauge, s.lag === null ? 1 : 0, attrs);
+ result.observe(pendingGauge, s.pending, attrs);
+ }
+ };
+ this.meter.addBatchObservableCallback(this.batchCallback, this.observables);
+ }
+
+ async start(): Promise {
+ if (this.running) return;
+ await this.ensureGroups();
+ this.running = true;
+ this.loopPromise = this.loop();
+ }
+
+ async stop(): Promise {
+ this.running = false;
+ this.meter.removeBatchObservableCallback(this.batchCallback, this.observables);
+ await this.loopPromise?.catch(() => {});
+ await Promise.all([this.redis.quit().catch(() => {}), this.probeRedis.quit().catch(() => {})]);
+ }
+
+ private async ensureGroups(): Promise {
+ for (const key of this.keys) {
+ try {
+ // "0" (not "$"): a brand-new stream's group must not skip entries emitted
+ // between emitter boot and the first consumer's group creation.
+ await this.redis.xgroup("CREATE", key, this.def.consumerGroup, "0", "MKSTREAM");
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ if (!message.includes("BUSYGROUP")) throw error;
+ }
+ }
+ }
+
+ private async loop(): Promise {
+ while (this.running) {
+ try {
+ if (Date.now() - this.lastReclaimAt >= this.reclaimIntervalMs) {
+ this.lastReclaimAt = Date.now();
+ await this.reclaimStale();
+ }
+ await this.readNew();
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ // Self-heal a missing group (stream trimmed to nothing / deleted / Redis flushed):
+ // recreate it rather than wedging the loop on NOGROUP forever.
+ if (message.includes("NOGROUP")) {
+ this.logger.warn("consumer group missing; recreating", { error });
+ await this.ensureGroups().catch(() => {});
+ } else {
+ this.logger.error("consumer loop iteration failed", { error });
+ }
+ await sleep(this.errorBackoffMs);
+ }
+ }
+ }
+
+ private async readNew(): Promise {
+ const ids = this.keys.map(() => ">");
+ const response = (await this.redis.xreadgroup(
+ "GROUP",
+ this.def.consumerGroup,
+ this.consumerName,
+ "COUNT",
+ this.batchSize,
+ "BLOCK",
+ this.blockMs,
+ "STREAMS",
+ ...this.keys,
+ ...ids
+ )) as RawStream[] | null;
+
+ if (!response) return 0;
+ return this.processStreams(response, "new");
+ }
+
+ private async reclaimStale(): Promise {
+ for (const key of this.keys) {
+ const result = (await this.redis.xautoclaim(
+ key,
+ this.def.consumerGroup,
+ this.consumerName,
+ this.claimIdleMs,
+ "0",
+ "COUNT",
+ this.batchSize
+ )) as [string, RawEntry[], string[]] | null;
+
+ const entries = result?.[1] ?? [];
+ if (entries.length === 0) continue;
+ await this.processStreams([[key, entries]], "reclaimed");
+ }
+ }
+
+ // One insert (dedup block) and XACK per stream, so a reclaimed batch re-forms the
+ // original per-stream id set and token. On insert failure that stream's entries stay
+ // pending for a later XAUTOCLAIM; other streams still progress.
+ private async processStreams(streams: RawStream[], source: "new" | "reclaimed"): Promise {
+ let processed = 0;
+ let firstError: unknown;
+
+ for (const [key, entries] of streams) {
+ if (entries.length === 0) continue;
+ const keyIds: string[] = [];
+ const rows: TRow[] = [];
+ for (const [id, flat] of entries) {
+ keyIds.push(id);
+ const mapped = this.mapEntry({ id, fields: parseFields(flat) });
+ if (Array.isArray(mapped)) rows.push(...mapped);
+ else if (mapped !== null) rows.push(mapped);
+ }
+ this.entriesCounter.add(keyIds.length, { source });
+
+ if (rows.length > 0) {
+ const startedAt = Date.now();
+ try {
+ await this.insert(rows, { dedupToken: dedupTokenFromEntryIds(keyIds, key) });
+ } catch (error) {
+ this.insertErrorCounter.add(1);
+ firstError ??= error;
+ continue;
+ } finally {
+ this.insertDuration.record(Date.now() - startedAt);
+ }
+ this.rowsCounter.add(rows.length);
+ }
+
+ await this.redis.xack(key, this.def.consumerGroup, ...keyIds);
+ processed += keyIds.length;
+ }
+
+ if (firstError !== undefined) throw firstError;
+ return processed;
+ }
+
+ /** Per-shard depth (XLEN), group lag, and pending — the consumer-health signals. */
+ async streamState(): Promise {
+ return probeShardStates(this.probeRedis, this.keys, this.def.consumerGroup);
+ }
+
+ /** All shard stream keys this consumer reads (for diagnostics/tests). */
+ streamKeys(): string[] {
+ return this.keys.slice();
+ }
+}
+
+/**
+ * Per-shard depth/lag/pending for a metric stream — usable without a running consumer
+ * (e.g. from an admin route). `redis` should have keyPrefix unset, matching the stream keys.
+ */
+export async function probeShardStates(
+ redis: Redis,
+ keys: string[],
+ consumerGroup: string
+): Promise {
+ const out: ShardState[] = [];
+ for (let shard = 0; shard < keys.length; shard++) {
+ const key = keys[shard]!;
+ const depth = Number(await redis.xlen(key)) || 0;
+ // lag defaults to null (unknown) and only becomes a number when the group is found and
+ // Redis reports one: a nil lag (or a missing group on an existing stream) means we can't
+ // compute it, e.g. entries were trimmed past the group's read position (data loss).
+ let lag: number | null = null;
+ let pending = 0;
+ try {
+ const groups = (await redis.call("XINFO", "GROUPS", key)) as unknown[];
+ for (const raw of groups) {
+ const info = flatToMap(raw as unknown[]);
+ if (info.name === consumerGroup) {
+ const rawLag = info.lag;
+ lag = rawLag == null ? null : Number(rawLag);
+ if (lag !== null && !Number.isFinite(lag)) lag = null;
+ pending = Number(info.pending) || 0;
+ }
+ }
+ } catch {
+ // Stream/group may not exist yet; treat as zero.
+ }
+ out.push({ shard, depth, lag, pending });
+ }
+ return out;
+}
+
+function flatToMap(flat: unknown[]): Record {
+ const out: Record = {};
+ for (let i = 0; i + 1 < flat.length; i += 2) {
+ out[String(flat[i])] = flat[i + 1];
+ }
+ return out;
+}
diff --git a/internal-packages/metrics-pipeline/src/emitter.ts b/internal-packages/metrics-pipeline/src/emitter.ts
new file mode 100644
index 00000000000..692956d98cb
--- /dev/null
+++ b/internal-packages/metrics-pipeline/src/emitter.ts
@@ -0,0 +1,242 @@
+import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis";
+import { getMeter, type Counter, type Meter, ValueType } from "@internal/tracing";
+import { Logger } from "@trigger.dev/core/logger";
+import { shardFor } from "./hash.js";
+import { streamKey, type MetricDefinition, type MetricFields } from "./types.js";
+
+export type MetricsStreamEmitterOptions = {
+ redis: RedisOptions;
+ definition: MetricDefinition;
+ /** Synchronous enabled check (e.g. CachedRedisFlag); emits are no-ops when false. */
+ flag: { enabled(): boolean };
+ /** Probability (0..1) that a sampled emission fires; applies to `sampledSync()`, not
+ * `emit()`. Pass a `{ value() }` provider (e.g. CachedRedisNumber) to tune it live
+ * without a redeploy. Default 1 (always). */
+ gaugeSampleRate?: number | { value(): number };
+ /** TTL (ms) refreshed on every counter write on the per-(queue,op) odometer key.
+ * Active queues never expire; idle-past-TTL queues purge and self-heal on return.
+ * Default 7 days. */
+ counterOdometerTtlMs?: number;
+ /** TTL (ms) for per-concurrency-key odometers; short because key cardinality is
+ * user-controlled and cumulative counters make idle-gap expiry loss-free. Default 24h. */
+ ckOdometerTtlMs?: number;
+ logger?: Logger;
+ meter?: Meter;
+};
+
+type CumulativeCommand = (
+ odometerKey: string,
+ streamKey: string,
+ ttlMs: string,
+ maxLen: string,
+ op: string,
+ q: string,
+ ...extraFields: string[]
+) => Promise;
+
+type CumulativeCkCommand = (
+ odometerKey: string,
+ ckOdometerKey: string,
+ streamKey: string,
+ ttlMs: string,
+ ckTtlMs: string,
+ maxLen: string,
+ op: string,
+ q: string,
+ ck: string,
+ ...extraFields: string[]
+) => Promise;
+
+// INCR the odometer, refresh its TTL, and XADD the reading (new value as `cum`) in one round
+// trip. Refresh-on-write is load-bearing: only genuinely idle queues expire. On first creation
+// (v==1) XADD a cum=0 baseline first (smaller stream id => sorts first) so deltaSum captures the
+// 0->1 transition and the total reconstructs exactly.
+// ARGV: [1]=ttlMs [2]=maxLen [3]=op [4]=q [5..]=extra field/value pairs (e.g. wait).
+const CUMULATIVE_LUA = `
+local v = redis.call('INCR', KEYS[1])
+redis.call('PEXPIRE', KEYS[1], ARGV[1])
+local maxlen = tonumber(ARGV[2]) or 0
+local function xadd(cum, withExtra)
+ local x = {'XADD', KEYS[2]}
+ if maxlen > 0 then x[#x+1]='MAXLEN'; x[#x+1]='~'; x[#x+1]=ARGV[2] end
+ x[#x+1]='*'
+ x[#x+1]='op'; x[#x+1]=ARGV[3]
+ x[#x+1]='q'; x[#x+1]=ARGV[4]
+ if withExtra then for i=5,#ARGV do x[#x+1]=ARGV[i] end end
+ x[#x+1]='cum'; x[#x+1]=cum
+ redis.call(unpack(x))
+end
+if v == 1 then xadd(0, false) end
+xadd(v, true)
+`;
+
+// CK variant: advances base + per-key odometers, ONE reading entry carries both (cum +
+// ck/ckcum), so per-key attribution adds no stream volume. Baselines seed independently:
+// cum-only entry = base row, ck+ckcum-only entry = per-key row, reading entry = both.
+// KEYS: [1]=baseOdometer [2]=ckOdometer [3]=stream. ARGV: [1]=baseTtlMs [2]=ckTtlMs
+// [3]=maxLen [4]=op [5]=q [6]=ck [7..]=extra field/value pairs.
+const CUMULATIVE_CK_LUA = `
+local v = redis.call('INCR', KEYS[1])
+redis.call('PEXPIRE', KEYS[1], ARGV[1])
+local ckv = redis.call('INCR', KEYS[2])
+redis.call('PEXPIRE', KEYS[2], ARGV[2])
+local maxlen = tonumber(ARGV[3]) or 0
+local function xadd(fields, withExtra)
+ local x = {'XADD', KEYS[3]}
+ if maxlen > 0 then x[#x+1]='MAXLEN'; x[#x+1]='~'; x[#x+1]=ARGV[3] end
+ x[#x+1]='*'
+ x[#x+1]='op'; x[#x+1]=ARGV[4]
+ x[#x+1]='q'; x[#x+1]=ARGV[5]
+ if withExtra then for i=7,#ARGV do x[#x+1]=ARGV[i] end end
+ for i=1,#fields do x[#x+1]=fields[i] end
+ redis.call(unpack(x))
+end
+if v == 1 then xadd({'cum', 0}, false) end
+if ckv == 1 then xadd({'ck', ARGV[6], 'ckcum', 0}, false) end
+xadd({'ck', ARGV[6], 'cum', v, 'ckcum', ckv}, true)
+`;
+
+/** Node-side producer: XADDs events to a sharded metrics stream, gated on a flag. */
+export class MetricsStreamEmitter {
+ private readonly redis: Redis;
+ private readonly def: MetricDefinition;
+ private readonly flag: { enabled(): boolean };
+ private readonly sampleRate: () => number;
+ private readonly odometerTtlMs: number;
+ private readonly ckOdometerTtlMs: number;
+ private readonly logger: Logger;
+ private readonly emittedCounter: Counter;
+ private readonly errorCounter: Counter;
+
+ constructor(options: MetricsStreamEmitterOptions) {
+ this.logger = options.logger ?? new Logger("MetricsStreamEmitter", "warn");
+ this.redis = createRedisClient(
+ { ...options.redis, keyPrefix: undefined },
+ { onError: (error) => this.logger.error("emitter redis error", { error }) }
+ );
+ this.redis.defineCommand("qmEmitCumulative", { numberOfKeys: 2, lua: CUMULATIVE_LUA });
+ this.redis.defineCommand("qmEmitCumulativeCk", { numberOfKeys: 3, lua: CUMULATIVE_CK_LUA });
+ this.odometerTtlMs = options.counterOdometerTtlMs ?? 7 * 24 * 60 * 60 * 1000;
+ this.ckOdometerTtlMs = options.ckOdometerTtlMs ?? 24 * 60 * 60 * 1000;
+ this.def = options.definition;
+ this.flag = options.flag;
+ const rate = options.gaugeSampleRate;
+ if (typeof rate === "object") {
+ this.sampleRate = () => rate.value();
+ } else {
+ const fixed = Math.min(1, Math.max(0, rate ?? 1));
+ this.sampleRate = () => fixed;
+ }
+
+ const meter = options.meter ?? getMeter("metrics-pipeline");
+ this.emittedCounter = meter.createCounter("queue_metrics.emitter.emitted", {
+ description: "Node-side metric events XADDed to the stream",
+ valueType: ValueType.INT,
+ });
+ this.errorCounter = meter.createCounter("queue_metrics.emitter.errors", {
+ description: "Failed metric-event XADDs (dropped)",
+ valueType: ValueType.INT,
+ });
+ }
+
+ enabledSync(): boolean {
+ return this.flag.enabled();
+ }
+
+ // Enabled AND (probabilistically) sampled-in. For high-frequency sampled emissions
+ // (e.g. Lua gauges); exact-count events use enabledSync()/emit() and are never sampled.
+ sampledSync(): boolean {
+ if (!this.flag.enabled()) return false;
+ const rate = this.sampleRate();
+ if (rate >= 1) return true;
+ if (rate <= 0) return false;
+ return Math.random() < rate;
+ }
+
+ // Fire-and-forget gauge emit: a plain XADD of an op=gauge snapshot (no odometer). The
+ // gauge value was read atomically inside the queue op's Lua and returned on the reply;
+ // this just lands it on the metrics stream. Loss-tolerant (sampled), never throws into
+ // the caller. Shares the counter stream (one stream family on the metrics Redis).
+ emitGauge(shardKey: string, fields: MetricFields): void {
+ if (!this.flag.enabled()) return;
+ // Drop rather than queue while the metrics Redis is unreachable: ioredis would hold
+ // every command in its offline queue until rejection, and metrics are loss-tolerant.
+ if (this.redis.status !== "ready") return;
+ const op = String(fields.op ?? "gauge");
+ const stream = streamKey(this.def, shardFor(shardKey, this.def.shardCount));
+ const args: string[] = [];
+ if (this.def.maxLen) args.push("MAXLEN", "~", String(this.def.maxLen));
+ args.push("*");
+ for (const [field, value] of Object.entries(fields)) {
+ args.push(field, String(value));
+ }
+ this.emittedCounter.add(1, { op });
+ this.redis.xadd(stream, ...(args as [string, ...string[]])).catch((error) => {
+ this.errorCounter.add(1);
+ this.logger.debug("metrics gauge emit failed", { error, stream });
+ });
+ }
+
+ // Fire-and-forget cumulative counter emit: advances the per-(queue,op) odometer and
+ // XADDs its new absolute value. No-op when disabled, never throws into the caller. A
+ // lost XADD self-heals (the next reading restates the total); the INCR is never sampled.
+ // A non-empty `fields.ck` also advances a per-concurrency-key odometer and rides the
+ // same entry as ck/ckcum (see CUMULATIVE_CK_LUA for the baseline/row mapping).
+ emit(shardKey: string, fields: MetricFields): void {
+ if (!this.flag.enabled()) return;
+ if (this.redis.status !== "ready") return;
+ const op = String(fields.op ?? "unknown");
+ const q = String(fields.q ?? "");
+ const ck = fields.ck != null && String(fields.ck) !== "" ? String(fields.ck) : null;
+ const shard = shardFor(shardKey, this.def.shardCount);
+ const stream = streamKey(this.def, shard);
+ // The odometer carries the stream's {shard} hash tag so INCR + XADD stay in one
+ // Cluster slot (the shard is derived from the queue, so the mapping is stable).
+ // The key format is part of the rolling-deploy data shape: concurrent old/new
+ // emitters with different formats split an odometer and corrupt its deltas.
+ const odometerKey = `${this.def.name}_cum:{${shard}}:${op}:${q}`;
+ const extra: string[] = [];
+ for (const [field, value] of Object.entries(fields)) {
+ if (field === "op" || field === "q" || field === "ck") continue;
+ extra.push(field, String(value));
+ }
+ this.emittedCounter.add(1, { op });
+ const maxLen = String(this.def.maxLen ?? 0);
+ const done = (error: unknown) => {
+ this.errorCounter.add(1);
+ this.logger.debug("metrics emit failed", { error, stream });
+ };
+ if (ck) {
+ const client = this.redis as unknown as { qmEmitCumulativeCk: CumulativeCkCommand };
+ client
+ .qmEmitCumulativeCk(
+ odometerKey,
+ `${odometerKey}:ck:${ck}`,
+ stream,
+ String(this.odometerTtlMs),
+ String(this.ckOdometerTtlMs),
+ maxLen,
+ op,
+ q,
+ ck,
+ ...extra
+ )
+ .catch(done);
+ return;
+ }
+ const client = this.redis as unknown as { qmEmitCumulative: CumulativeCommand };
+ client
+ .qmEmitCumulative(odometerKey, stream, String(this.odometerTtlMs), maxLen, op, q, ...extra)
+ .catch(done);
+ }
+
+ // Resolves once the metrics Redis connection is ready (emits before that are dropped).
+ waitUntilReady(): Promise {
+ if (this.redis.status === "ready") return Promise.resolve();
+ return new Promise((resolve) => this.redis.once("ready", () => resolve()));
+ }
+
+ async close(): Promise {
+ await this.redis.quit();
+ }
+}
diff --git a/internal-packages/metrics-pipeline/src/flag.ts b/internal-packages/metrics-pipeline/src/flag.ts
new file mode 100644
index 00000000000..5931e088939
--- /dev/null
+++ b/internal-packages/metrics-pipeline/src/flag.ts
@@ -0,0 +1,46 @@
+import type { RedisOptions } from "@internal/redis";
+import type { Logger } from "@trigger.dev/core/logger";
+import { CachedRedisValue } from "./cachedValue.js";
+
+export type CachedRedisFlagOptions = {
+ redis: RedisOptions;
+ /** Redis key holding the flag. A value of "1"/"true"/"on"/"enabled" is truthy. */
+ key: string;
+ cacheTtlMs?: number;
+ defaultValue?: boolean;
+ logger?: Logger;
+};
+
+const TRUTHY = new Set(["1", "true", "on", "enabled", "yes"]);
+
+/**
+ * Boolean feature flag from a Redis key with a short stale-while-revalidate cache,
+ * exposing a synchronous getter for hot paths (building Lua ARGV on every op).
+ */
+export class CachedRedisFlag {
+ private readonly inner: CachedRedisValue;
+
+ constructor(options: CachedRedisFlagOptions) {
+ this.inner = new CachedRedisValue({
+ redis: options.redis,
+ key: options.key,
+ parse: (raw) => raw != null && TRUTHY.has(raw.trim().toLowerCase()),
+ defaultValue: options.defaultValue ?? false,
+ cacheTtlMs: options.cacheTtlMs,
+ logger: options.logger,
+ loggerName: "CachedRedisFlag",
+ });
+ }
+
+ enabled(): boolean {
+ return this.inner.get();
+ }
+
+ refresh(): Promise {
+ return this.inner.refresh();
+ }
+
+ async close(): Promise {
+ await this.inner.close();
+ }
+}
diff --git a/internal-packages/metrics-pipeline/src/hash.ts b/internal-packages/metrics-pipeline/src/hash.ts
new file mode 100644
index 00000000000..b14324c138a
--- /dev/null
+++ b/internal-packages/metrics-pipeline/src/hash.ts
@@ -0,0 +1,15 @@
+/** FNV-1a 32-bit hash. Deterministic across processes; used only for sharding. */
+export function fnv1a32(str: string): number {
+ let hash = 0x811c9dc5;
+ for (let i = 0; i < str.length; i++) {
+ hash ^= str.charCodeAt(i);
+ hash = Math.imul(hash, 0x01000193);
+ }
+ return hash >>> 0;
+}
+
+/** Deterministic shard index in [0, shardCount) for a key. */
+export function shardFor(key: string, shardCount: number): number {
+ if (shardCount <= 1) return 0;
+ return fnv1a32(key) % shardCount;
+}
diff --git a/internal-packages/metrics-pipeline/src/idempotency.ts b/internal-packages/metrics-pipeline/src/idempotency.ts
new file mode 100644
index 00000000000..60cbd661f53
--- /dev/null
+++ b/internal-packages/metrics-pipeline/src/idempotency.ts
@@ -0,0 +1,11 @@
+import { createHash } from "node:crypto";
+
+// Deterministic, order-independent token over a batch of entry ids. A redelivered
+// batch yields the same token, so ClickHouse's raw-table dedup window drops the replay.
+// `scope` (the stream key) disambiguates id sets that could collide across streams.
+export function dedupTokenFromEntryIds(ids: string[], scope = ""): string {
+ const sorted = [...ids].sort();
+ return createHash("sha1")
+ .update(`${scope}|${sorted.join(",")}`)
+ .digest("hex");
+}
diff --git a/internal-packages/metrics-pipeline/src/index.ts b/internal-packages/metrics-pipeline/src/index.ts
new file mode 100644
index 00000000000..223c5feab17
--- /dev/null
+++ b/internal-packages/metrics-pipeline/src/index.ts
@@ -0,0 +1,26 @@
+export { CachedRedisFlag, type CachedRedisFlagOptions } from "./flag.js";
+export {
+ CachedRedisNumber,
+ type CachedRedisNumberOptions,
+ CachedRedisValue,
+ type CachedRedisValueOptions,
+} from "./cachedValue.js";
+export { MetricsStreamEmitter, type MetricsStreamEmitterOptions } from "./emitter.js";
+export {
+ MetricsStreamConsumer,
+ type MetricsStreamConsumerOptions,
+ type ShardState,
+ probeShardStates,
+} from "./consumer.js";
+export { createMetricsGaugeComputeLua, type GaugeComputeLuaParams } from "./lua.js";
+export { dedupTokenFromEntryIds } from "./idempotency.js";
+export { shardFor, fnv1a32 } from "./hash.js";
+export {
+ streamKey,
+ allStreamKeys,
+ entryTimeMs,
+ entryOrderKey,
+ type MetricDefinition,
+ type MetricFields,
+ type StreamEntry,
+} from "./types.js";
diff --git a/internal-packages/metrics-pipeline/src/lua.ts b/internal-packages/metrics-pipeline/src/lua.ts
new file mode 100644
index 00000000000..64f3b896c0d
--- /dev/null
+++ b/internal-packages/metrics-pipeline/src/lua.ts
@@ -0,0 +1,50 @@
+// Each field is a Lua expression evaluated inside the target script. queueLimit/
+// envLimit must be the EFFECTIVE enforced limit, else an unset limit reads as throttled.
+export type GaugeComputeLuaParams = {
+ // Lua boolean expression; when true the gauge is computed (else the extra reads are skipped).
+ enabledArg: string;
+ queued: string;
+ running: string;
+ queueLimit: string;
+ envQueued: string;
+ envRunning: string;
+ envLimit: string;
+ // Lua statements run first inside the pcall (e.g. to compute aggregate locals).
+ preamble?: string;
+ // Lua boolean expression (in __cc/__lim/__ql) for the throttled flag. Pass "false"
+ // where cc >= lim is not a valid throttle signal (e.g. summed CK aggregates).
+ throttledExpr?: string;
+ // CK-health extras (both or neither): appended as an optional gauge tail, gauge[8]/gauge[9].
+ ckBacklogged?: string;
+ ckMaxWaitMs?: string;
+};
+
+// Computes an op=gauge snapshot into the enclosing script's `__qm_g` local (a flat
+// {ql, cc, lim, eql, ec, elim, thr} array) so the script can RETURN it; Node then XADDs it
+// to the metrics Redis. No Redis write here (the run-queue Redis carries no metrics stream).
+// Gated on the sample flag and pcall-wrapped. The script MUST declare `local __qm_g` first.
+export function createMetricsGaugeComputeLua(params: GaugeComputeLuaParams): string {
+ const throttled = params.throttledExpr ?? "__cc >= __lim and __ql > 0";
+ const hasCk = params.ckBacklogged != null && params.ckMaxWaitMs != null;
+ const gauge = hasCk
+ ? ` local __ckq = tonumber(${params.ckBacklogged}) or 0
+ local __ckw = tonumber(${params.ckMaxWaitMs}) or 0
+ __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw}`
+ : ` __qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}`;
+
+ return `
+if ${params.enabledArg} then
+ pcall(function()
+ ${params.preamble ?? ""}
+ local __ql = tonumber(${params.queued}) or 0
+ local __cc = tonumber(${params.running}) or 0
+ local __lim = tonumber(${params.queueLimit}) or 0
+ local __eql = tonumber(${params.envQueued}) or 0
+ local __ec = tonumber(${params.envRunning}) or 0
+ local __elim = tonumber(${params.envLimit}) or 0
+ local __thr = 0
+ if ${throttled} then __thr = 1 end
+${gauge}
+ end)
+end`;
+}
diff --git a/internal-packages/metrics-pipeline/src/pipeline.test.ts b/internal-packages/metrics-pipeline/src/pipeline.test.ts
new file mode 100644
index 00000000000..73979310798
--- /dev/null
+++ b/internal-packages/metrics-pipeline/src/pipeline.test.ts
@@ -0,0 +1,116 @@
+import { describe, expect, it } from "vitest";
+import { createMetricsGaugeComputeLua } from "./lua.js";
+import { dedupTokenFromEntryIds } from "./idempotency.js";
+import { fnv1a32, shardFor } from "./hash.js";
+import { allStreamKeys, entryOrderKey, entryTimeMs, streamKey } from "./types.js";
+
+describe("shardFor", () => {
+ it("is deterministic and in range", () => {
+ expect(shardFor("queueA", 1)).toBe(0);
+ const s = shardFor("queueA", 4);
+ expect(s).toBeGreaterThanOrEqual(0);
+ expect(s).toBeLessThan(4);
+ expect(shardFor("queueA", 4)).toBe(s);
+ expect(fnv1a32("queueA")).toBe(fnv1a32("queueA"));
+ });
+});
+
+describe("dedupTokenFromEntryIds", () => {
+ it("is order-independent and set-sensitive", () => {
+ expect(dedupTokenFromEntryIds(["1-0", "2-0"])).toBe(dedupTokenFromEntryIds(["2-0", "1-0"]));
+ expect(dedupTokenFromEntryIds(["1-0"])).not.toBe(dedupTokenFromEntryIds(["2-0"]));
+ expect(dedupTokenFromEntryIds(["1-0"])).toMatch(/^[0-9a-f]{40}$/);
+ });
+});
+
+describe("stream keys", () => {
+ it("names and parses entry time", () => {
+ expect(streamKey({ name: "queue_metrics" }, 3)).toBe("queue_metrics:{3}");
+ expect(allStreamKeys({ name: "qm", shardCount: 2, consumerGroup: "cg" })).toEqual([
+ "qm:{0}",
+ "qm:{1}",
+ ]);
+ expect(entryTimeMs("1717000000000-5")).toBe(1717000000000);
+ expect(entryTimeMs("nope")).toBeNull();
+ });
+
+ it("entryOrderKey stays exact and strictly monotonic at real epoch magnitudes", () => {
+ const ms = 1783000000000; // ~2026: ms*1e6 is past JS safe-integer range, so a number key
+ const k = (seq: number) => BigInt(entryOrderKey(`${ms}-${seq}`));
+ // adjacent seq within one ms must not collapse to the same key (the float bug)
+ expect(k(0)).toBe(BigInt(ms) * 1000000n);
+ expect(k(1) - k(0)).toBe(1n);
+ expect(k(2) - k(1)).toBe(1n);
+ // a later ms always outranks any seq of an earlier ms (up to the 1M/ms factor)
+ expect(BigInt(entryOrderKey(`${ms + 1}-0`))).toBeGreaterThan(k(999999));
+ });
+});
+
+describe("createMetricsGaugeComputeLua", () => {
+ it("assigns __qm_g inside a gated, pcall-wrapped block and never XADDs", () => {
+ const lua = createMetricsGaugeComputeLua({
+ enabledArg: "ARGV[#ARGV] == '1'",
+ queued: "redis.call('ZCARD', KEYS[2])",
+ running: "queueCurrent",
+ queueLimit: "queueLimit",
+ envQueued: "redis.call('ZCARD', KEYS[8])",
+ envRunning: "envCurrent",
+ envLimit: "envLimit",
+ });
+
+ expect(lua).toContain("if ARGV[#ARGV] == '1' then");
+ expect(lua).toContain("pcall(function()");
+ expect(lua).toContain("__qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}");
+ expect(lua).toContain("if __cc >= __lim and __ql > 0 then __thr = 1 end");
+ // The whole point of the refactor: no Redis write happens in the run-queue script.
+ expect(lua).not.toContain("XADD");
+ });
+
+ it("honors a custom throttled expression and preamble", () => {
+ const lua = createMetricsGaugeComputeLua({
+ enabledArg: "true",
+ preamble: "local agg = 1",
+ queued: "0",
+ running: "0",
+ queueLimit: "0",
+ envQueued: "0",
+ envRunning: "0",
+ envLimit: "0",
+ throttledExpr: "false",
+ });
+ expect(lua).toContain("local agg = 1");
+ expect(lua).toContain("if false then __thr = 1 end");
+ expect(lua).not.toContain("XADD");
+ });
+
+ it("appends the CK-health tail only when both CK params are set", () => {
+ const withCk = createMetricsGaugeComputeLua({
+ enabledArg: "true",
+ queued: "0",
+ running: "0",
+ queueLimit: "0",
+ envQueued: "0",
+ envRunning: "0",
+ envLimit: "0",
+ ckBacklogged: "redis.call('ZCARD', ckIndexKey)",
+ ckMaxWaitMs: "__ckwait",
+ });
+ expect(withCk).toContain(
+ "__qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr, __ckq, __ckw}"
+ );
+ expect(withCk).toContain("local __ckq = tonumber(redis.call('ZCARD', ckIndexKey)) or 0");
+
+ const withoutCk = createMetricsGaugeComputeLua({
+ enabledArg: "true",
+ queued: "0",
+ running: "0",
+ queueLimit: "0",
+ envQueued: "0",
+ envRunning: "0",
+ envLimit: "0",
+ ckBacklogged: "0",
+ });
+ expect(withoutCk).toContain("__qm_g = {__ql, __cc, __lim, __eql, __ec, __elim, __thr}");
+ expect(withoutCk).not.toContain("__ckq");
+ });
+});
diff --git a/internal-packages/metrics-pipeline/src/types.ts b/internal-packages/metrics-pipeline/src/types.ts
new file mode 100644
index 00000000000..d9e9e43f554
--- /dev/null
+++ b/internal-packages/metrics-pipeline/src/types.ts
@@ -0,0 +1,42 @@
+export type MetricFields = Record;
+
+export type StreamEntry = {
+ id: string;
+ fields: Record;
+};
+
+export type MetricDefinition = {
+ /** Logical name, e.g. "queue_metrics". Used as the stream key prefix. */
+ name: string;
+ shardCount: number;
+ consumerGroup: string;
+ /** Approximate MAXLEN cap applied on XADD (`MAXLEN ~ N`). Omit for unbounded. */
+ maxLen?: number;
+};
+
+// Keys are used verbatim on every access path (Lua ARGV, emitter, consumer), so
+// they must NOT be subject to an ioredis keyPrefix. `{shard}` is a Cluster hash tag.
+export function streamKey(definition: Pick, shard: number): string {
+ return `${definition.name}:{${shard}}`;
+}
+
+export function allStreamKeys(definition: MetricDefinition): string[] {
+ return Array.from({ length: Math.max(1, definition.shardCount) }, (_, shard) =>
+ streamKey(definition, shard)
+ );
+}
+
+// The ms part of a stream entry id is its emission time.
+export function entryTimeMs(id: string): number | null {
+ const ms = Number(id.split("-")[0]);
+ return Number.isFinite(ms) ? ms : null;
+}
+
+// Ordering key from a stream id (`-`) = ms*1e6+seq, for deltaSumTimestamp. BigInt +
+// string because ms*1e6 exceeds JS safe-integer range at real epoch magnitudes (a number would
+// collapse nearby seq values); the ClickHouse order_key column is UInt64 and takes the string.
+// The 1e6 factor (1M entries/ms/shard, far above any single Redis stream) stays within UInt64.
+export function entryOrderKey(id: string): string {
+ const [ms, seq] = id.split("-");
+ return (BigInt(Number(ms) || 0) * 1000000n + BigInt(Number(seq) || 0)).toString();
+}
diff --git a/internal-packages/metrics-pipeline/test/setup.ts b/internal-packages/metrics-pipeline/test/setup.ts
new file mode 100644
index 00000000000..b2bacd6baf5
--- /dev/null
+++ b/internal-packages/metrics-pipeline/test/setup.ts
@@ -0,0 +1,4 @@
+import { vi } from "vitest";
+
+// Set extended timeout for container tests
+vi.setConfig({ testTimeout: 60_000 });
diff --git a/internal-packages/metrics-pipeline/tsconfig.build.json b/internal-packages/metrics-pipeline/tsconfig.build.json
new file mode 100644
index 00000000000..89c87a3dc67
--- /dev/null
+++ b/internal-packages/metrics-pipeline/tsconfig.build.json
@@ -0,0 +1,21 @@
+{
+ "include": ["src/**/*.ts"],
+ "exclude": ["src/**/*.test.ts"],
+ "compilerOptions": {
+ "composite": true,
+ "target": "ES2020",
+ "lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
+ "outDir": "dist",
+ "module": "Node16",
+ "moduleResolution": "Node16",
+ "moduleDetection": "force",
+ "verbatimModuleSyntax": false,
+ "esModuleInterop": true,
+ "forceConsistentCasingInFileNames": true,
+ "isolatedModules": true,
+ "preserveWatchOutput": true,
+ "skipLibCheck": true,
+ "strict": true,
+ "declaration": true
+ }
+}
diff --git a/internal-packages/metrics-pipeline/tsconfig.json b/internal-packages/metrics-pipeline/tsconfig.json
new file mode 100644
index 00000000000..af630abe1f1
--- /dev/null
+++ b/internal-packages/metrics-pipeline/tsconfig.json
@@ -0,0 +1,8 @@
+{
+ "references": [{ "path": "./tsconfig.src.json" }, { "path": "./tsconfig.test.json" }],
+ "compilerOptions": {
+ "moduleResolution": "Node16",
+ "module": "Node16",
+ "customConditions": ["@triggerdotdev/source"]
+ }
+}
diff --git a/internal-packages/metrics-pipeline/tsconfig.src.json b/internal-packages/metrics-pipeline/tsconfig.src.json
new file mode 100644
index 00000000000..0df3d2d222f
--- /dev/null
+++ b/internal-packages/metrics-pipeline/tsconfig.src.json
@@ -0,0 +1,20 @@
+{
+ "include": ["src/**/*.ts"],
+ "exclude": ["node_modules", "src/**/*.test.ts"],
+ "compilerOptions": {
+ "composite": true,
+ "target": "ES2020",
+ "lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
+ "module": "Node16",
+ "moduleResolution": "Node16",
+ "moduleDetection": "force",
+ "verbatimModuleSyntax": false,
+ "esModuleInterop": true,
+ "forceConsistentCasingInFileNames": true,
+ "isolatedModules": true,
+ "preserveWatchOutput": true,
+ "skipLibCheck": true,
+ "strict": true,
+ "customConditions": ["@triggerdotdev/source"]
+ }
+}
diff --git a/internal-packages/metrics-pipeline/tsconfig.test.json b/internal-packages/metrics-pipeline/tsconfig.test.json
new file mode 100644
index 00000000000..4c06c9f57bb
--- /dev/null
+++ b/internal-packages/metrics-pipeline/tsconfig.test.json
@@ -0,0 +1,21 @@
+{
+ "include": ["src/**/*.test.ts"],
+ "references": [{ "path": "./tsconfig.src.json" }],
+ "compilerOptions": {
+ "composite": true,
+ "target": "ES2020",
+ "lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
+ "module": "Node16",
+ "moduleResolution": "Node16",
+ "moduleDetection": "force",
+ "verbatimModuleSyntax": false,
+ "types": ["vitest/globals"],
+ "esModuleInterop": true,
+ "forceConsistentCasingInFileNames": true,
+ "isolatedModules": true,
+ "preserveWatchOutput": true,
+ "skipLibCheck": true,
+ "strict": true,
+ "customConditions": ["@triggerdotdev/source"]
+ }
+}
diff --git a/internal-packages/metrics-pipeline/vitest.config.ts b/internal-packages/metrics-pipeline/vitest.config.ts
new file mode 100644
index 00000000000..daafd294fa8
--- /dev/null
+++ b/internal-packages/metrics-pipeline/vitest.config.ts
@@ -0,0 +1,17 @@
+import { defineConfig } from "vitest/config";
+import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
+
+export default defineConfig({
+ test: {
+ sequence: { sequencer: DurationShardingSequencer },
+ globals: true,
+ retry: process.env.CI ? 2 : 0,
+ environment: "node",
+ setupFiles: ["./test/setup.ts"],
+ testTimeout: 30000,
+ hookTimeout: 30000,
+ },
+ esbuild: {
+ target: "node18",
+ },
+});
diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json
index 8d53974d10b..516e6a18696 100644
--- a/internal-packages/run-engine/package.json
+++ b/internal-packages/run-engine/package.json
@@ -21,6 +21,7 @@
},
"dependencies": {
"@internal/redis": "workspace:*",
+ "@internal/metrics-pipeline": "workspace:*",
"@internal/run-store": "workspace:*",
"@trigger.dev/redis-worker": "workspace:*",
"@internal/tracing": "workspace:*",
diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts
index 00edf64405b..93855ae4ba2 100644
--- a/internal-packages/run-engine/src/engine/index.ts
+++ b/internal-packages/run-engine/src/engine/index.ts
@@ -218,6 +218,7 @@ export class RunEngine {
callback: this.#concurrencySweeperCallback.bind(this),
},
shardCount: options.queue?.shardCount,
+ queueMetrics: options.queue?.queueMetrics,
masterQueueConsumersDisabled: options.queue?.masterQueueConsumersDisabled,
masterQueueConsumersIntervalMs: options.queue?.masterQueueConsumersIntervalMs,
processWorkerQueueDebounceMs: options.queue?.processWorkerQueueDebounceMs,
@@ -1605,9 +1606,26 @@ export class RunEngine {
async lengthOfQueue(
environment: MinimalAuthenticatedEnvironment,
- queue: string
+ queue: string,
+ concurrencyKey?: string
+ ): Promise {
+ return this.runQueue.lengthOfQueue(environment, queue, concurrencyKey);
+ }
+
+ async currentConcurrencyOfQueue(
+ environment: MinimalAuthenticatedEnvironment,
+ queue: string,
+ concurrencyKey?: string
): Promise {
- return this.runQueue.lengthOfQueue(environment, queue);
+ return this.runQueue.currentConcurrencyOfQueue(environment, queue, concurrencyKey);
+ }
+
+ async oldestMessageInQueue(
+ environment: MinimalAuthenticatedEnvironment,
+ queue: string,
+ concurrencyKey?: string
+ ): Promise {
+ return this.runQueue.oldestMessageInQueue(environment, queue, concurrencyKey);
}
async concurrencyOfEnvQueue(environment: MinimalAuthenticatedEnvironment): Promise {
@@ -1628,6 +1646,14 @@ export class RunEngine {
return this.runQueue.currentConcurrencyOfQueues(environment, queues);
}
+ async concurrencyKeyBreakdown(
+ environment: MinimalAuthenticatedEnvironment,
+ queue: string,
+ options?: { limit?: number }
+ ) {
+ return this.runQueue.concurrencyKeyBreakdown(environment, queue, options);
+ }
+
async removeEnvironmentQueuesFromMasterQueue({
runtimeEnvironmentId,
organizationId,
diff --git a/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts b/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts
index dc9d029c38c..4b236aefc16 100644
--- a/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts
+++ b/internal-packages/run-engine/src/engine/systems/enqueueSystem.ts
@@ -98,10 +98,16 @@ export class EnqueueSystem {
// Force development runs to use the environment id as the worker queue.
const workerQueue = env.type === "DEVELOPMENT" ? env.id : run.workerQueue;
- const timestamp = (run.queueTimestamp ?? run.createdAt).getTime() - run.priorityMs;
+ // Ordering keeps the run's original position; the scheduling-delay anchor is the
+ // trigger/delay time only on first enqueue (includeTtl). Re-enqueues anchor to now,
+ // else the wait metric absorbs the whole waitpoint/checkpoint duration.
+ const queuePositionMs = (run.queueTimestamp ?? run.createdAt).getTime();
+ const timestamp = queuePositionMs - run.priorityMs;
+ const eligibleAtMs = includeTtl ? queuePositionMs : Date.now();
- // Include TTL only when explicitly requested (first enqueue from trigger).
- // Re-enqueues (waitpoint, checkpoint, delayed, pending version) must not add TTL.
+ // Include TTL only when explicitly requested (first enqueue from trigger or the
+ // delayed-run system). Re-enqueues (waitpoint, checkpoint, pending version) must
+ // not add TTL.
let ttlExpiresAt: number | undefined;
if (includeTtl && run.ttl) {
const expireAt = parseNaturalLanguageDuration(run.ttl);
@@ -124,6 +130,7 @@ export class EnqueueSystem {
queue: run.queue,
concurrencyKey: run.concurrencyKey ?? undefined,
timestamp,
+ eligibleAtMs,
attempt: 0,
ttlExpiresAt,
},
diff --git a/internal-packages/run-engine/src/engine/tests/ttl.test.ts b/internal-packages/run-engine/src/engine/tests/ttl.test.ts
index 949e47f8574..e33b361abdb 100644
--- a/internal-packages/run-engine/src/engine/tests/ttl.test.ts
+++ b/internal-packages/run-engine/src/engine/tests/ttl.test.ts
@@ -293,7 +293,12 @@ describe("RunEngine ttl", () => {
);
assertNonNullable(messageAfterTrigger);
expect(messageAfterTrigger.ttlExpiresAt).toBeDefined();
+ // First enqueue anchors the scheduling-delay clock at the trigger time.
+ expect(messageAfterTrigger.eligibleAtMs).toBe(
+ (run.queueTimestamp ?? run.createdAt).getTime()
+ );
+ const beforeReenqueue = Date.now();
await engine.enqueueSystem.enqueueRun({
run,
env: authenticatedEnvironment,
@@ -308,6 +313,10 @@ describe("RunEngine ttl", () => {
);
assertNonNullable(messageAfterReenqueue);
expect(messageAfterReenqueue.ttlExpiresAt).toBeUndefined();
+ // Re-enqueues anchor to now so the wait metric measures only this queue stint,
+ // while the ordering timestamp keeps the run's original position.
+ expect(messageAfterReenqueue.eligibleAtMs).toBeGreaterThanOrEqual(beforeReenqueue);
+ expect(messageAfterReenqueue.timestamp).toBe(messageAfterTrigger.timestamp);
} finally {
await engine.quit();
}
diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts
index bb1d6eb2fa9..f37ec7df50a 100644
--- a/internal-packages/run-engine/src/engine/types.ts
+++ b/internal-packages/run-engine/src/engine/types.ts
@@ -16,6 +16,7 @@ import {
} from "@trigger.dev/redis-worker";
import type { ControlPlaneResolver } from "./controlPlaneResolver.js";
import type { FairQueueSelectionStrategyOptions } from "../run-queue/fairQueueSelectionStrategy.js";
+import type { RunQueueMetricsEmitter } from "../run-queue/index.js";
import type { MinimalAuthenticatedEnvironment } from "../shared/index.js";
import type { LockRetryConfig } from "./locking.js";
import type { workerCatalog } from "./workerCatalog.js";
@@ -90,6 +91,8 @@ export type RunEngineOptions = {
defaultEnvConcurrency?: number;
defaultEnvConcurrencyBurstFactor?: number;
logLevel?: LogLevel;
+ /** Optional queue-metrics emitter; enables gauge + counter emission from the RunQueue. */
+ queueMetrics?: RunQueueMetricsEmitter;
queueSelectionStrategyOptions?: Pick<
FairQueueSelectionStrategyOptions,
"parentQueueLimit" | "tracer" | "biases" | "reuseSnapshotCount" | "maximumEnvCount"
diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts
index a0571206538..4e6ca89d847 100644
--- a/internal-packages/run-engine/src/run-queue/index.ts
+++ b/internal-packages/run-engine/src/run-queue/index.ts
@@ -5,6 +5,7 @@ import {
type RedisOptions,
type Result,
} from "@internal/redis";
+import { createMetricsGaugeComputeLua } from "@internal/metrics-pipeline";
import type {
Attributes,
Meter,
@@ -57,6 +58,99 @@ const SemanticAttributes = {
ORG_ID: "runqueue.orgId",
};
+// Prelude spliced at the top of every gauge-carrying script: declares the gauge slot and
+// the return wrapper. A splice fills __qm_g; every return goes through __qmret so the reply
+// is always {original, gauge}. A nil original becomes false, else Lua drops it from the
+// multi-bulk reply (which would swallow the gauge on the dequeue throttle paths).
+const QUEUE_METRICS_GAUGE_PRELUDE = `
+local __qm_g = false
+local function __qmret(r) if r == nil then r = false end return {r, __qm_g} end`;
+
+// Fresh-read gauge for splice points with no reusable locals: enqueue slow-path (before
+// return 0) and the base dequeue top. Gated on the last ARGV so it is inert unless the
+// caller opts in. CK queues emit per-subqueue depth (queue_name aggregates via the MV).
+const QUEUE_METRICS_GAUGE_LUA = createMetricsGaugeComputeLua({
+ enabledArg: "ARGV[#ARGV] == '1'",
+ queued: "redis.call('ZCARD', queueKey)",
+ running: "redis.call('SCARD', queueCurrentConcurrencyKey)",
+ queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'",
+ envQueued: "redis.call('ZCARD', envQueueKey)",
+ envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)",
+ envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit",
+});
+
+// Enqueue fast-path gauge: the admission check already computed queueCurrent/envCurrent/
+// queueLimit/envLimit, so reuse them (only 2 ZCARDs stay fresh). Fast path was taken, so
+// cc < lim and thr is always 0 — reusing the effective queueLimit is fine (max() recovers raw).
+const QUEUE_METRICS_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua({
+ enabledArg: "ARGV[#ARGV] == '1'",
+ queued: "redis.call('ZCARD', queueKey)",
+ running: "queueCurrent",
+ queueLimit: "queueLimit",
+ envQueued: "redis.call('ZCARD', envQueueKey)",
+ envRunning: "envCurrent",
+ envLimit: "envLimit",
+});
+
+// CK-health extras: distinct backlogged keys + most-starved head-of-line wait (ckIndex scores
+// are per-subqueue oldest timestamps). Needs ckIndexKey/currentTime locals; clamps future scores.
+const QUEUE_METRICS_CK_GAUGE_EXTRAS = {
+ preamble: `local __ckhead = redis.call('ZRANGE', ckIndexKey, 0, 0, 'WITHSCORES')
+ local __ckwait = 0
+ if #__ckhead > 0 then __ckwait = math.floor(math.max(0, (tonumber(currentTime) or 0) - (tonumber(__ckhead[2]) or 0))) end`,
+ ckBacklogged: "redis.call('ZCARD', ckIndexKey)",
+ ckMaxWaitMs: "__ckwait",
+};
+
+// CK enqueue variants of the two gauges above, extended with the CK-health tail.
+const QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({
+ enabledArg: "ARGV[#ARGV] == '1'",
+ queued: "redis.call('ZCARD', queueKey)",
+ running: "redis.call('SCARD', queueCurrentConcurrencyKey)",
+ queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'",
+ envQueued: "redis.call('ZCARD', envQueueKey)",
+ envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)",
+ envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit",
+ ...QUEUE_METRICS_CK_GAUGE_EXTRAS,
+});
+
+const QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA = createMetricsGaugeComputeLua({
+ enabledArg: "ARGV[#ARGV] == '1'",
+ queued: "redis.call('ZCARD', queueKey)",
+ running: "queueCurrent",
+ queueLimit: "queueLimit",
+ envQueued: "redis.call('ZCARD', envQueueKey)",
+ envRunning: "envCurrent",
+ envLimit: "envLimit",
+ ...QUEUE_METRICS_CK_GAUGE_EXTRAS,
+});
+
+// CK dequeue: depth/running from the per-base-queue aggregate counters the run-queue already
+// maintains (two O(1) GETs, not a per-variant scan). thr suppressed — an aggregate cc >= per-CK
+// limit would over-report; per-CK throttle is caught by the per-subqueue enqueue gauges.
+const QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA = createMetricsGaugeComputeLua({
+ enabledArg: "ARGV[#ARGV] == '1'",
+ queued: "redis.call('GET', lengthCounterKey) or '0'",
+ running: "redis.call('GET', runningCounterKey) or '0'",
+ queueLimit: "redis.call('GET', queueConcurrencyLimitKey) or '1000000'",
+ envQueued: "redis.call('ZCARD', envQueueKey)",
+ envRunning: "redis.call('SCARD', envCurrentConcurrencyKey)",
+ envLimit: "redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit",
+ throttledExpr: "false",
+ ...QUEUE_METRICS_CK_GAUGE_EXTRAS,
+});
+
+/** Injected queue-metrics stream emitter; all calls are no-ops when metrics are disabled. */
+export interface RunQueueMetricsEmitter {
+ enabledSync(): boolean;
+ /** enabled AND sampled-in; gates high-frequency sampled emissions (the Lua gauge). */
+ sampledSync(): boolean;
+ /** Counter event (cumulative odometer). */
+ emit(shardKey: string, fields: Record): void;
+ /** Gauge snapshot read inside the queue-op Lua and returned on the reply. */
+ emitGauge(shardKey: string, fields: Record): void;
+}
+
export type RunQueueOptions = {
name: string;
tracer: Tracer;
@@ -93,6 +187,8 @@ export type RunQueueOptions = {
disabled?: boolean;
};
meter?: Meter;
+ /** When set, enqueue/dequeue/ack/nack/dlq emit queue-metrics events (gated on the emitter's flag). */
+ queueMetrics?: RunQueueMetricsEmitter;
dequeueBlockingTimeoutSeconds?: number;
concurrencySweeper?: {
scanSchedule?: string;
@@ -458,6 +554,65 @@ export class RunQueue {
);
}
+ /**
+ * Live per-concurrency-key breakdown of a queue's backlog, most-starved first.
+ * Reads the ckIndex zset (members = CK subqueue names, scores = oldest-message
+ * timestamps), so only keys with queued work appear; running-only keys do not.
+ */
+ public async concurrencyKeyBreakdown(
+ env: MinimalAuthenticatedEnvironment,
+ queue: string,
+ options?: { limit?: number }
+ ): Promise<{
+ totalBackloggedKeys: number;
+ keys: Array<{
+ concurrencyKey: string;
+ queued: number;
+ running: number;
+ oldestEnqueuedAt: number;
+ }>;
+ }> {
+ const limit = options?.limit ?? 50;
+ const ckIndexKey = this.keys.ckIndexKeyFromQueue(this.keys.queueKey(env, queue));
+
+ const indexPipeline = this.redis.pipeline();
+ indexPipeline.zcard(ckIndexKey);
+ indexPipeline.zrange(ckIndexKey, 0, limit - 1, "WITHSCORES");
+ const indexResults = await indexPipeline.exec();
+ if (!indexResults) return { totalBackloggedKeys: 0, keys: [] };
+
+ const [totalErr, totalVal] = indexResults[0];
+ const [rangeErr, rangeVal] = indexResults[1];
+ const totalBackloggedKeys = totalErr || totalVal == null ? 0 : (totalVal as number);
+ const flat = rangeErr || rangeVal == null ? [] : (rangeVal as string[]);
+
+ const members: Array<{ member: string; score: number }> = [];
+ for (let i = 0; i < flat.length; i += 2) {
+ members.push({ member: flat[i], score: Number(flat[i + 1]) });
+ }
+ if (members.length === 0) return { totalBackloggedKeys, keys: [] };
+
+ const statsPipeline = this.redis.pipeline();
+ for (const { member } of members) {
+ statsPipeline.zcard(member);
+ statsPipeline.scard(this.keys.queueCurrentConcurrencyKeyFromQueue(member));
+ }
+ const stats = await statsPipeline.exec();
+
+ const keys = members.map(({ member, score }, i) => {
+ const queuedResult = stats?.[i * 2];
+ const runningResult = stats?.[i * 2 + 1];
+ return {
+ concurrencyKey: this.#concurrencyKeyFromQueue(member) ?? "",
+ queued: queuedResult && !queuedResult[0] ? ((queuedResult[1] as number) ?? 0) : 0,
+ running: runningResult && !runningResult[0] ? ((runningResult[1] as number) ?? 0) : 0,
+ oldestEnqueuedAt: score,
+ };
+ });
+
+ return { totalBackloggedKeys, keys };
+ }
+
public async lengthOfEnvQueue(env: MinimalAuthenticatedEnvironment) {
return this.redis.zcard(this.keys.envQueueKey(env));
}
@@ -751,6 +906,8 @@ export class RunQueue {
span.setAttribute("fastPath", fastPathTaken);
+ this.#emitQueueMetric(queueKey, { op: "enqueue", q: queueKey });
+
if (!fastPathTaken && !skipDequeueProcessing) {
// Slow path: schedule the dequeue job to move the message from queue to worker queue
await this.worker.enqueueOnce({
@@ -810,6 +967,15 @@ export class RunQueue {
...flattenAttributes(dequeuedMessage.message, "message"),
});
+ const startedFields: Record = {
+ op: "started",
+ q: dequeuedMessage.message.queue,
+ };
+ if (typeof dequeuedMessage.message.eligibleAtMs === "number") {
+ startedFields.wait = Math.max(0, Date.now() - dequeuedMessage.message.eligibleAtMs);
+ }
+ this.#emitQueueMetric(dequeuedMessage.message.queue, startedFields);
+
return dequeuedMessage;
},
{
@@ -877,6 +1043,8 @@ export class RunQueue {
message,
removeFromWorkerQueue: options?.removeFromWorkerQueue,
});
+
+ this.#emitQueueMetric(message.queue, { op: "ack", q: message.queue });
},
{
kind: SpanKind.CONSUMER,
@@ -934,6 +1102,7 @@ export class RunQueue {
message.attempt = message.attempt + 1;
if (message.attempt >= maxAttempts) {
await this.#callMoveToDeadLetterQueue({ message });
+ this.#emitQueueMetric(message.queue, { op: "dlq", q: message.queue });
return false;
}
}
@@ -960,6 +1129,8 @@ export class RunQueue {
await this.#callNackMessage({ message, retryAt });
+ this.#emitQueueMetric(message.queue, { op: "nack", q: message.queue });
+
return true;
},
{
@@ -1831,6 +2002,57 @@ export class RunQueue {
*
* @returns true if the fast path was taken (message pushed directly to worker queue)
*/
+ #queueMetricsGaugeArg(): string {
+ // Gauge gate ARGV: enabled AND sampled-in (sampling applies to the gauge, not counters).
+ return this.options.queueMetrics?.sampledSync() ? "1" : "0";
+ }
+
+ // Gauge returned on a script reply as a flat [ql, cc, lim, eql, ec, elim, thr] array,
+ // plus an optional [ckq, ckw] tail on CK-path scripts.
+ // Unlike counters, gauges are NOT base-normalized: the q label keeps its :ck: suffix so
+ // the CK-aggregate and per-subqueue readings stay distinguishable; the consumer's mapEntry
+ // strips :ck: to the base queue_name and the MV maxes them into one row.
+ #emitGauge(queue: string, gauge: number[]): void {
+ if (!Array.isArray(gauge) || gauge.length < 7) return;
+ const [ql, cc, lim, eql, ec, elim, thr, ckq, ckw] = gauge;
+ const fields: Record = {
+ op: "gauge",
+ q: queue,
+ ql,
+ cc,
+ lim,
+ eql,
+ ec,
+ elim,
+ thr,
+ };
+ if (gauge.length >= 9) {
+ fields.ckq = ckq;
+ fields.ckw = ckw;
+ }
+ this.options.queueMetrics?.emitGauge(queue, fields);
+ }
+
+ #concurrencyKeyFromQueue(queue: string): string | undefined {
+ const idx = queue.indexOf(":ck:");
+ return idx === -1 || idx + 4 >= queue.length ? undefined : queue.slice(idx + 4);
+ }
+
+ #emitQueueMetric(shardKey: string, fields: Record): void {
+ // Counters roll up per BASE queue: normalize the CK-qualified queue to its base so all
+ // concurrency keys share one monotonic odometer (and one shard/order key), matching the
+ // base queue_name the consumer buckets on. A real concurrency key rides along as `ck`,
+ // driving a SEPARATE per-key odometer on the same entry (per-key history tier).
+ const baseQueue = this.keys.baseQueueKeyFromQueue(shardKey);
+ let baseFields = fields;
+ if (typeof fields.q === "string") {
+ baseFields = { ...fields, q: this.keys.baseQueueKeyFromQueue(fields.q) };
+ const ck = this.#concurrencyKeyFromQueue(fields.q);
+ if (ck && ck !== "*") baseFields.ck = ck;
+ }
+ this.options.queueMetrics?.emit(baseQueue, baseFields);
+ }
+
async #callEnqueueMessage(
message: OutputPayloadV2,
ttlInfo?: {
@@ -1869,6 +2091,7 @@ export class RunQueue {
const messageScore = String(message.timestamp);
const currentTime = String(Date.now());
const enableFastPathArg = enableFastPath ? "1" : "0";
+ const metricsGaugeArg = this.#queueMetricsGaugeArg();
const defaultEnvConcurrencyLimit = String(this.options.defaultEnvConcurrency);
const defaultEnvConcurrencyBurstFactor = String(
this.options.defaultEnvConcurrencyBurstFactor ?? 1.0
@@ -1892,7 +2115,8 @@ export class RunQueue {
service: this.name,
});
- let result: number;
+ // Every gauge-carrying script returns a 2-tuple [originalReturn, gauge|null].
+ let result: [number, number[] | null];
// Use CK-aware enqueue for messages with concurrency keys
if (message.concurrencyKey) {
@@ -1935,7 +2159,8 @@ export class RunQueue {
currentTime,
enableFastPathArg,
ckKeyPrefix,
- String(this.counterTtlSeconds)
+ String(this.counterTtlSeconds),
+ metricsGaugeArg
);
} else {
result = await this.redis.enqueueMessageCkTracked(
@@ -1967,7 +2192,8 @@ export class RunQueue {
currentTime,
enableFastPathArg,
ckKeyPrefix,
- String(this.counterTtlSeconds)
+ String(this.counterTtlSeconds),
+ metricsGaugeArg
);
}
} else if (ttlInfo) {
@@ -1998,7 +2224,8 @@ export class RunQueue {
defaultEnvConcurrencyLimit,
defaultEnvConcurrencyBurstFactor,
currentTime,
- enableFastPathArg
+ enableFastPathArg,
+ metricsGaugeArg
);
} else {
result = await this.redis.enqueueMessage(
@@ -2024,11 +2251,14 @@ export class RunQueue {
defaultEnvConcurrencyLimit,
defaultEnvConcurrencyBurstFactor,
currentTime,
- enableFastPathArg
+ enableFastPathArg,
+ metricsGaugeArg
);
}
- return result === 1;
+ const [enqueueResult, gauge] = result;
+ if (gauge) this.#emitGauge(queueName, gauge);
+ return enqueueResult === 1;
}
async #callDequeueMessagesFromQueue({
@@ -2081,7 +2311,9 @@ export class RunQueue {
maxCount,
});
- const result = await this.redis.dequeueMessagesFromQueue(
+ const metricsGaugeArg = this.#queueMetricsGaugeArg();
+
+ const reply = await this.redis.dequeueMessagesFromQueue(
//keys
messageQueue,
queueConcurrencyLimitKey,
@@ -2099,9 +2331,16 @@ export class RunQueue {
String(this.options.defaultEnvConcurrency),
String(this.options.defaultEnvConcurrencyBurstFactor ?? 1),
this.options.redis.keyPrefix ?? "",
- String(maxCount)
+ String(maxCount),
+ metricsGaugeArg
);
+ // Reply is [flatMessages|null, gauge|null]: emit the gauge (read atomically inside
+ // the script, present on the throttle/empty paths too) and keep element 0 as the array.
+ const gauge = reply?.[1] ?? null;
+ if (gauge) this.#emitGauge(messageQueue, gauge);
+ const result = reply?.[0] ?? null;
+
if (!result) {
span.setAttribute("message_count", 0);
@@ -2202,8 +2441,11 @@ export class RunQueue {
});
const lengthCounterKey = this.keys.queueLengthCounterKeyFromQueue(ckWildcardQueue);
+ const runningCounterKey = this.keys.queueRunningCounterKeyFromQueue(ckWildcardQueue);
+
+ const metricsGaugeArg = this.#queueMetricsGaugeArg();
- const result = await this.redis.dequeueMessagesFromCkQueueTracked(
+ const reply = await this.redis.dequeueMessagesFromCkQueueTracked(
//keys
ckIndexKey,
queueConcurrencyLimitKey,
@@ -2215,15 +2457,22 @@ export class RunQueue {
masterQueueKey,
ttlQueueKey,
lengthCounterKey,
+ runningCounterKey,
//args
ckWildcardQueue,
String(Date.now()),
String(this.options.defaultEnvConcurrency),
String(this.options.defaultEnvConcurrencyBurstFactor ?? 1),
this.options.redis.keyPrefix ?? "",
- String(maxCount)
+ String(maxCount),
+ metricsGaugeArg
);
+ // Reply is [flatMessages|null, gauge|null]; the CK aggregate gauge rides here.
+ const gauge = reply?.[1] ?? null;
+ if (gauge) this.#emitGauge(ckWildcardQueue, gauge);
+ const result = reply?.[0] ?? null;
+
if (!result) {
span.setAttribute("message_count", 0);
return [];
@@ -3062,6 +3311,8 @@ local defaultEnvConcurrencyBurstFactor = ARGV[7]
local currentTime = ARGV[8]
local enableFastPath = ARGV[9]
+${QUEUE_METRICS_GAUGE_PRELUDE}
+
-- Fast path: check if we can skip the queue and go directly to worker queue
if enableFastPath == '1' then
local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1)
@@ -3083,7 +3334,8 @@ if enableFastPath == '1' then
redis.call('SADD', queueCurrentConcurrencyKey, messageId)
redis.call('SADD', envCurrentConcurrencyKey, messageId)
redis.call('RPUSH', workerQueueKey, messageKeyValue)
- return 1
+${QUEUE_METRICS_ENQUEUE_FASTPATH_GAUGE_LUA}
+ return __qmret(1)
end
end
end
@@ -3113,8 +3365,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId)
redis.call('SREM', envCurrentConcurrencyKey, messageId)
redis.call('SREM', queueCurrentDequeuedKey, messageId)
redis.call('SREM', envCurrentDequeuedKey, messageId)
+${QUEUE_METRICS_GAUGE_LUA}
-return 0
+return __qmret(0)
`,
});
@@ -3153,6 +3406,8 @@ local defaultEnvConcurrencyBurstFactor = ARGV[9]
local currentTime = ARGV[10]
local enableFastPath = ARGV[11]
+${QUEUE_METRICS_GAUGE_PRELUDE}
+
-- Fast path: check if we can skip the queue and go directly to worker queue
if enableFastPath == '1' then
local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1)
@@ -3174,8 +3429,9 @@ if enableFastPath == '1' then
redis.call('SADD', queueCurrentConcurrencyKey, messageId)
redis.call('SADD', envCurrentConcurrencyKey, messageId)
redis.call('RPUSH', workerQueueKey, messageKeyValue)
+${QUEUE_METRICS_ENQUEUE_FASTPATH_GAUGE_LUA}
-- Skip TTL sorted set: the expireRun worker job handles TTL expiry independently
- return 1
+ return __qmret(1)
end
end
end
@@ -3208,8 +3464,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId)
redis.call('SREM', envCurrentConcurrencyKey, messageId)
redis.call('SREM', queueCurrentDequeuedKey, messageId)
redis.call('SREM', envCurrentDequeuedKey, messageId)
+${QUEUE_METRICS_GAUGE_LUA}
-return 0
+return __qmret(0)
`,
});
@@ -3246,6 +3503,8 @@ local defaultEnvConcurrencyBurstFactor = ARGV[8]
local currentTime = ARGV[9]
local enableFastPath = ARGV[10]
+${QUEUE_METRICS_GAUGE_PRELUDE}
+
-- Fast path: check if we can skip the queue and go directly to worker queue
if enableFastPath == '1' then
local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1)
@@ -3268,7 +3527,8 @@ if enableFastPath == '1' then
redis.call('SADD', queueCurrentConcurrencyKey, messageId)
redis.call('SADD', envCurrentConcurrencyKey, messageId)
redis.call('RPUSH', workerQueueKey, messageKeyValue)
- return 1
+${QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA}
+ return __qmret(1)
end
end
end
@@ -3304,8 +3564,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId)
redis.call('SREM', envCurrentConcurrencyKey, messageId)
redis.call('SREM', queueCurrentDequeuedKey, messageId)
redis.call('SREM', envCurrentDequeuedKey, messageId)
+${QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA}
-return 0
+return __qmret(0)
`,
});
@@ -3344,6 +3605,8 @@ local defaultEnvConcurrencyBurstFactor = ARGV[10]
local currentTime = ARGV[11]
local enableFastPath = ARGV[12]
+${QUEUE_METRICS_GAUGE_PRELUDE}
+
-- Fast path: check if we can skip the queue and go directly to worker queue
if enableFastPath == '1' then
local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1)
@@ -3365,8 +3628,9 @@ if enableFastPath == '1' then
redis.call('SADD', queueCurrentConcurrencyKey, messageId)
redis.call('SADD', envCurrentConcurrencyKey, messageId)
redis.call('RPUSH', workerQueueKey, messageKeyValue)
+${QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA}
-- Skip TTL sorted set: the expireRun worker job handles TTL expiry independently
- return 1
+ return __qmret(1)
end
end
end
@@ -3405,8 +3669,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId)
redis.call('SREM', envCurrentConcurrencyKey, messageId)
redis.call('SREM', queueCurrentDequeuedKey, messageId)
redis.call('SREM', envCurrentDequeuedKey, messageId)
+${QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA}
-return 0
+return __qmret(0)
`,
});
@@ -3455,6 +3720,8 @@ local keyPrefix = ARGV[11]
-- TTL (seconds) applied to counter lazy-init SETs
local counterTtl = ARGV[12]
+${QUEUE_METRICS_GAUGE_PRELUDE}
+
-- Fast path: check if we can skip the queue and go directly to worker queue
if enableFastPath == '1' then
local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1)
@@ -3476,10 +3743,11 @@ if enableFastPath == '1' then
redis.call('SADD', queueCurrentConcurrencyKey, messageId)
redis.call('SADD', envCurrentConcurrencyKey, messageId)
redis.call('RPUSH', workerQueueKey, messageKeyValue)
+${QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA}
-- Fast-path skips the CK variant zset entirely; lengthCounter is unchanged.
-- runningCounter is bumped later by dequeueMessageFromKeyTracked when the
-- worker pulls the message from the worker queue.
- return 1
+ return __qmret(1)
end
end
end
@@ -3531,8 +3799,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId)
redis.call('SREM', envCurrentConcurrencyKey, messageId)
redis.call('SREM', queueCurrentDequeuedKey, messageId)
redis.call('SREM', envCurrentDequeuedKey, messageId)
+${QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA}
-return 0
+return __qmret(0)
`,
});
@@ -3576,6 +3845,8 @@ local keyPrefix = ARGV[13]
-- TTL (seconds) applied to counter lazy-init SETs
local counterTtl = ARGV[14]
+${QUEUE_METRICS_GAUGE_PRELUDE}
+
-- Fast path: check if we can skip the queue and go directly to worker queue
if enableFastPath == '1' then
local available = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'LIMIT', 0, 1)
@@ -3597,7 +3868,8 @@ if enableFastPath == '1' then
redis.call('SADD', queueCurrentConcurrencyKey, messageId)
redis.call('SADD', envCurrentConcurrencyKey, messageId)
redis.call('RPUSH', workerQueueKey, messageKeyValue)
- return 1
+${QUEUE_METRICS_CK_ENQUEUE_FASTPATH_GAUGE_LUA}
+ return __qmret(1)
end
end
end
@@ -3645,8 +3917,9 @@ redis.call('SREM', queueCurrentConcurrencyKey, messageId)
redis.call('SREM', envCurrentConcurrencyKey, messageId)
redis.call('SREM', queueCurrentDequeuedKey, messageId)
redis.call('SREM', envCurrentDequeuedKey, messageId)
+${QUEUE_METRICS_CK_ENQUEUE_GAUGE_LUA}
-return 0
+return __qmret(0)
`,
});
@@ -3891,6 +4164,8 @@ local defaultEnvConcurrencyLimit = ARGV[3]
local defaultEnvConcurrencyBurstFactor = ARGV[4]
local keyPrefix = ARGV[5]
local maxCount = tonumber(ARGV[6] or '1')
+${QUEUE_METRICS_GAUGE_PRELUDE}
+${QUEUE_METRICS_GAUGE_LUA}
-- Check current env concurrency against the limit
local envCurrentConcurrency = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0')
@@ -3899,7 +4174,7 @@ local envConcurrencyLimitBurstFactor = tonumber(redis.call('GET', envConcurrency
local envConcurrencyLimitWithBurstFactor = math.floor(envConcurrencyLimit * envConcurrencyLimitBurstFactor)
if envCurrentConcurrency >= envConcurrencyLimitWithBurstFactor then
- return nil
+ return __qmret(nil)
end
-- Check current queue concurrency against the limit
@@ -3909,7 +4184,7 @@ local totalQueueConcurrencyLimit = queueConcurrencyLimit
-- Check condition only if concurrencyLimit exists
if queueCurrentConcurrency >= totalQueueConcurrencyLimit then
- return nil
+ return __qmret(nil)
end
-- Calculate how many messages we can actually dequeue based on concurrency limits
@@ -3918,14 +4193,14 @@ local queueAvailableCapacity = totalQueueConcurrencyLimit - queueCurrentConcurre
local actualMaxCount = math.min(maxCount, envAvailableCapacity, queueAvailableCapacity)
if actualMaxCount <= 0 then
- return nil
+ return __qmret(nil)
end
-- Attempt to dequeue messages up to actualMaxCount
local messages = redis.call('ZRANGEBYSCORE', queueKey, '-inf', currentTime, 'WITHSCORES', 'LIMIT', 0, actualMaxCount)
if #messages == 0 then
- return nil
+ return __qmret(nil)
end
local results = {}
@@ -3991,7 +4266,7 @@ else
end
-- Return results as a flat array: [messageId1, messageScore1, messagePayload1, messageId2, messageScore2, messagePayload2, ...]
-return results
+return __qmret(results)
`,
});
@@ -4145,7 +4420,7 @@ return results
// (normal dequeue, TTL-expired, or stale-orphan path — all of which were
// counted at enqueue time).
this.redis.defineCommand("dequeueMessagesFromCkQueueTracked", {
- numberOfKeys: 10,
+ numberOfKeys: 11,
lua: `
local ckIndexKey = KEYS[1]
local queueConcurrencyLimitKey = KEYS[2]
@@ -4157,6 +4432,7 @@ local envQueueKey = KEYS[7]
local masterQueueKey = KEYS[8]
local ttlQueueKey = KEYS[9]
local lengthCounterKey = KEYS[10]
+local runningCounterKey = KEYS[11]
local ckWildcardName = ARGV[1]
local currentTime = tonumber(ARGV[2])
@@ -4164,6 +4440,8 @@ local defaultEnvConcurrencyLimit = ARGV[3]
local defaultEnvConcurrencyBurstFactor = ARGV[4]
local keyPrefix = ARGV[5]
local maxCount = tonumber(ARGV[6] or '1')
+${QUEUE_METRICS_GAUGE_PRELUDE}
+${QUEUE_METRICS_CK_DEQUEUE_GAUGE_LUA}
local function decrLengthCounter()
if tonumber(redis.call('GET', lengthCounterKey) or '0') > 0 then
@@ -4178,7 +4456,7 @@ local envConcurrencyLimitBurstFactor = tonumber(redis.call('GET', envConcurrency
local envConcurrencyLimitWithBurstFactor = math.floor(envConcurrencyLimit * envConcurrencyLimitBurstFactor)
if envCurrentConcurrency >= envConcurrencyLimitWithBurstFactor then
- return nil
+ return __qmret(nil)
end
local queueConcurrencyLimit = math.min(tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envConcurrencyLimit)
@@ -4187,7 +4465,7 @@ local envAvailableCapacity = envConcurrencyLimitWithBurstFactor - envCurrentConc
local actualMaxCount = math.min(maxCount, envAvailableCapacity)
if actualMaxCount <= 0 then
- return nil
+ return __qmret(nil)
end
local ckQueues = redis.call('ZRANGEBYSCORE', ckIndexKey, '-inf', tostring(currentTime), 'LIMIT', 0, actualMaxCount * 3)
@@ -4199,7 +4477,7 @@ if #ckQueues == 0 then
else
redis.call('ZADD', masterQueueKey, anyIdx[2], ckWildcardName)
end
- return nil
+ return __qmret(nil)
end
local results = {}
@@ -4281,7 +4559,7 @@ else
redis.call('ZADD', masterQueueKey, earliestIdx[2], ckWildcardName)
end
-return results
+return __qmret(results)
`,
});
@@ -5199,8 +5477,9 @@ declare module "@internal/redis" {
defaultEnvConcurrencyBurstFactor: string,
currentTime: string,
enableFastPath: string,
- callback?: Callback
- ): Result;
+ metricsEnabled: string,
+ callback?: Callback<[number, number[] | null]>
+ ): Result<[number, number[] | null], Context>;
enqueueMessageWithTtl(
//keys
@@ -5229,8 +5508,9 @@ declare module "@internal/redis" {
defaultEnvConcurrencyBurstFactor: string,
currentTime: string,
enableFastPath: string,
- callback?: Callback
- ): Result;
+ metricsEnabled: string,
+ callback?: Callback<[number, number[] | null]>
+ ): Result<[number, number[] | null], Context>;
expireTtlRuns(
//keys
@@ -5265,8 +5545,9 @@ declare module "@internal/redis" {
defaultEnvConcurrencyBurstFactor: string,
keyPrefix: string,
maxCount: string,
- callback?: Callback
- ): Result;
+ metricsEnabled: string,
+ callback?: Callback<[string[] | null, number[] | null]>
+ ): Result<[string[] | null, number[] | null], Context>;
dequeueMessageFromWorkerQueueNonBlocking(
workerQueueKey: string,
@@ -5405,8 +5686,9 @@ declare module "@internal/redis" {
defaultEnvConcurrencyBurstFactor: string,
currentTime: string,
enableFastPath: string,
- callback?: Callback
- ): Result;
+ metricsEnabled: string,
+ callback?: Callback<[number, number[] | null]>
+ ): Result<[number, number[] | null], Context>;
enqueueMessageWithTtlCk(
//keys
@@ -5437,8 +5719,9 @@ declare module "@internal/redis" {
defaultEnvConcurrencyBurstFactor: string,
currentTime: string,
enableFastPath: string,
- callback?: Callback
- ): Result;
+ metricsEnabled: string,
+ callback?: Callback<[number, number[] | null]>
+ ): Result<[number, number[] | null], Context>;
dequeueMessagesFromCkQueue(
//keys
@@ -5551,8 +5834,9 @@ declare module "@internal/redis" {
enableFastPath: string,
keyPrefix: string,
counterTtl: string,
- callback?: Callback
- ): Result;
+ metricsEnabled: string,
+ callback?: Callback<[number, number[] | null]>
+ ): Result<[number, number[] | null], Context>;
enqueueMessageWithTtlCkTracked(
masterQueueKey: string,
@@ -5585,8 +5869,9 @@ declare module "@internal/redis" {
enableFastPath: string,
keyPrefix: string,
counterTtl: string,
- callback?: Callback
- ): Result;
+ metricsEnabled: string,
+ callback?: Callback<[number, number[] | null]>
+ ): Result<[number, number[] | null], Context>;
dequeueMessagesFromCkQueueTracked(
ckIndexKey: string,
@@ -5599,14 +5884,16 @@ declare module "@internal/redis" {
masterQueueKey: string,
ttlQueueKey: string,
lengthCounterKey: string,
+ runningCounterKey: string,
ckWildcardName: string,
currentTime: string,
defaultEnvConcurrencyLimit: string,
defaultEnvConcurrencyBurstFactor: string,
keyPrefix: string,
maxCount: string,
- callback?: Callback
- ): Result;
+ metricsEnabled: string,
+ callback?: Callback<[string[] | null, number[] | null]>
+ ): Result<[string[] | null, number[] | null], Context>;
dequeueMessageFromKeyTracked(
messageKey: string,
diff --git a/internal-packages/run-engine/src/run-queue/keyProducer.ts b/internal-packages/run-engine/src/run-queue/keyProducer.ts
index b185435f6f6..18a2727b7e4 100644
--- a/internal-packages/run-engine/src/run-queue/keyProducer.ts
+++ b/internal-packages/run-engine/src/run-queue/keyProducer.ts
@@ -141,8 +141,7 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer {
}
queueConcurrencyLimitKeyFromQueue(queue: string) {
- const concurrencyQueueName = queue.replace(/:ck:.+$/, "");
- return `${concurrencyQueueName}:${constants.CONCURRENCY_LIMIT_PART}`;
+ return `${this.baseQueueKeyFromQueue(queue)}:${constants.CONCURRENCY_LIMIT_PART}`;
}
queueCurrentConcurrencyKeyFromQueue(queue: string) {
@@ -313,12 +312,14 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer {
}
ckIndexKeyFromQueue(queue: string): string {
- const baseQueue = queue.replace(/:ck:.+$/, "");
- return `${baseQueue}:${constants.CK_INDEX_PART}`;
+ return `${this.baseQueueKeyFromQueue(queue)}:${constants.CK_INDEX_PART}`;
}
+ // indexOf instead of /:ck:.+$/ (queue names are user-controlled; polynomial regex).
+ // Only strips when at least one character follows ":ck:", matching the old semantics.
baseQueueKeyFromQueue(queue: string): string {
- return queue.replace(/:ck:.+$/, "");
+ const idx = queue.indexOf(":ck:");
+ return idx === -1 || idx + 4 >= queue.length ? queue : queue.slice(0, idx);
}
queueLengthCounterKey(env: RunQueueKeyProducerEnvironment, queue: string): string {
@@ -342,7 +343,8 @@ export class RunQueueFullKeyProducer implements RunQueueKeyProducer {
}
toCkWildcard(queue: string): string {
- return queue.replace(/:ck:.+$/, ":ck:*");
+ const base = this.baseQueueKeyFromQueue(queue);
+ return base === queue ? queue : `${base}:ck:*`;
}
descriptorFromQueue(queue: string): QueueDescriptor {
diff --git a/internal-packages/run-engine/src/run-queue/metrics.test.ts b/internal-packages/run-engine/src/run-queue/metrics.test.ts
new file mode 100644
index 00000000000..ebfc295470e
--- /dev/null
+++ b/internal-packages/run-engine/src/run-queue/metrics.test.ts
@@ -0,0 +1,397 @@
+import { createRedisClient } from "@internal/redis";
+import { redisTest } from "@internal/testcontainers";
+import { trace } from "@internal/tracing";
+import {
+ allStreamKeys,
+ MetricsStreamEmitter,
+ type MetricDefinition,
+} from "@internal/metrics-pipeline";
+import { Logger } from "@trigger.dev/core/logger";
+import { Decimal } from "@trigger.dev/database";
+import { setTimeout } from "node:timers/promises";
+import { describe, expect } from "vitest";
+import { FairQueueSelectionStrategy } from "./fairQueueSelectionStrategy.js";
+import { RunQueue } from "./index.js";
+import { RunQueueFullKeyProducer } from "./keyProducer.js";
+import type { InputPayload } from "./types.js";
+
+const authenticatedEnvDev = {
+ id: "e1234",
+ type: "DEVELOPMENT" as const,
+ maximumConcurrencyLimit: 10,
+ concurrencyLimitBurstFactor: new Decimal(1.0),
+ project: { id: "p1234" },
+ organization: { id: "o1234" },
+};
+
+async function readAllEntries(
+ redisOptions: {
+ host: string;
+ port: number;
+ },
+ definition: MetricDefinition
+) {
+ const client = createRedisClient({ ...redisOptions, keyPrefix: undefined });
+ const entries: Array<{ id: string; fields: Record }> = [];
+ for (const key of allStreamKeys(definition)) {
+ const raw = (await client.xrange(key, "-", "+")) as Array<[string, string[]]>;
+ for (const [id, flat] of raw) {
+ const fields: Record = {};
+ for (let i = 0; i + 1 < flat.length; i += 2) fields[flat[i]!] = flat[i + 1]!;
+ entries.push({ id, fields });
+ }
+ }
+ await client.quit();
+ return entries;
+}
+
+// Gauges now land via a fire-and-forget Node XADD after the script reply (not synchronously
+// inside the Lua), so reads must poll until the expected entries appear.
+async function waitForEntries(
+ redisOptions: { host: string; port: number },
+ definition: MetricDefinition,
+ predicate: (entries: Array<{ id: string; fields: Record }>) => boolean,
+ timeoutMs = 5000
+) {
+ const start = Date.now();
+ let entries = await readAllEntries(redisOptions, definition);
+ while (!predicate(entries)) {
+ if (Date.now() - start > timeoutMs) return entries;
+ await setTimeout(50);
+ entries = await readAllEntries(redisOptions, definition);
+ }
+ return entries;
+}
+
+describe("RunQueue queue-metrics emission", () => {
+ redisTest("emits gauge + enqueue/started/ack events when enabled", async ({ redisContainer }) => {
+ const redis = {
+ keyPrefix: "runqueue:test:",
+ host: redisContainer.getHost(),
+ port: redisContainer.getPort(),
+ };
+ const definition: MetricDefinition = {
+ name: `qm_test_${Date.now()}`,
+ shardCount: 2,
+ consumerGroup: "cg",
+ maxLen: 1000,
+ };
+ const emitter = new MetricsStreamEmitter({
+ redis,
+ definition,
+ flag: { enabled: () => true },
+ });
+
+ const queue = new RunQueue({
+ name: "rq",
+ tracer: trace.getTracer("rq"),
+ defaultEnvConcurrency: 25,
+ logger: new Logger("RunQueue", "error"),
+ keys: new RunQueueFullKeyProducer(),
+ queueSelectionStrategy: new FairQueueSelectionStrategy({
+ redis,
+ keys: new RunQueueFullKeyProducer(),
+ }),
+ redis,
+ queueMetrics: emitter,
+ });
+
+ const message: InputPayload = {
+ runId: "r-metrics",
+ taskIdentifier: "task/my-task",
+ orgId: "o1234",
+ projectId: "p1234",
+ environmentId: authenticatedEnvDev.id,
+ environmentType: "DEVELOPMENT",
+ queue: "task/my-task",
+ timestamp: Date.now(),
+ eligibleAtMs: Date.now() - 500,
+ attempt: 0,
+ };
+
+ try {
+ await queue.enqueueMessage({
+ env: authenticatedEnvDev,
+ message,
+ workerQueue: authenticatedEnvDev.id,
+ });
+ await setTimeout(1000);
+ const dequeued = await queue.dequeueMessageFromWorkerQueue("c1", authenticatedEnvDev.id);
+ expect(dequeued?.messageId).toBe(message.runId);
+ await queue.acknowledgeMessage(message.orgId, message.runId);
+ await setTimeout(100);
+
+ const entries = await waitForEntries(redis, definition, (es) => {
+ const seen = es.map((e) => e.fields.op);
+ return ["enqueue", "gauge", "started", "ack"].every((o) => seen.includes(o));
+ });
+ const ops = entries.map((e) => e.fields.op);
+ expect(ops).toContain("enqueue");
+ expect(ops).toContain("gauge");
+ expect(ops).toContain("started");
+ expect(ops).toContain("ack");
+
+ const gauge = entries.find((e) => e.fields.op === "gauge");
+ assertGauge(gauge);
+ expect(gauge!.fields.q).toContain("task/my-task");
+ for (const f of ["ql", "cc", "lim", "eql", "ec", "elim", "thr"]) {
+ expect(gauge!.fields[f]).toBeDefined();
+ }
+ // Non-CK scripts keep the 7-field gauge (no CK-health tail).
+ expect(gauge!.fields.ckq).toBeUndefined();
+ expect(gauge!.fields.ckw).toBeUndefined();
+
+ // The first counter emission also seeds a cum=0 baseline (no wait); the real reading
+ // carries wait. Pick the reading (cum > 0).
+ const started = entries.find((e) => e.fields.op === "started" && Number(e.fields.cum) > 0);
+ expect(started!.fields.wait).toBeDefined();
+ expect(Number(started!.fields.wait)).toBeGreaterThanOrEqual(0);
+ expect(Number(started!.fields.cum)).toBeGreaterThan(0);
+ } finally {
+ await queue.quit();
+ await emitter.close();
+ }
+ });
+
+ redisTest(
+ "emits a fast-path gauge reusing the admission-check locals",
+ async ({ redisContainer }) => {
+ const redis = {
+ keyPrefix: "runqueue:test:",
+ host: redisContainer.getHost(),
+ port: redisContainer.getPort(),
+ };
+ const definition: MetricDefinition = {
+ name: `qm_fp_${Date.now()}`,
+ shardCount: 2,
+ consumerGroup: "cg",
+ maxLen: 1000,
+ };
+ const emitter = new MetricsStreamEmitter({
+ redis,
+ definition,
+ flag: { enabled: () => true },
+ });
+ const queue = new RunQueue({
+ name: "rq",
+ tracer: trace.getTracer("rq"),
+ defaultEnvConcurrency: 25,
+ logger: new Logger("RunQueue", "error"),
+ keys: new RunQueueFullKeyProducer(),
+ queueSelectionStrategy: new FairQueueSelectionStrategy({
+ redis,
+ keys: new RunQueueFullKeyProducer(),
+ }),
+ redis,
+ queueMetrics: emitter,
+ });
+
+ const message: InputPayload = {
+ runId: "r-fastpath",
+ taskIdentifier: "task/my-task",
+ orgId: "o1234",
+ projectId: "p1234",
+ environmentId: authenticatedEnvDev.id,
+ environmentType: "DEVELOPMENT",
+ queue: "task/my-task",
+ timestamp: Date.now(),
+ attempt: 0,
+ };
+
+ try {
+ // enableFastPath + empty queue + zero concurrency => the Lua takes the fast path,
+ // so the gauge runs the reuse snippet (queueCurrent/envCurrent/queueLimit/envLimit).
+ await queue.enqueueMessage({
+ env: authenticatedEnvDev,
+ message,
+ workerQueue: authenticatedEnvDev.id,
+ enableFastPath: true,
+ });
+ const dequeued = await queue.dequeueMessageFromWorkerQueue("c1", authenticatedEnvDev.id);
+ expect(dequeued?.messageId).toBe(message.runId);
+
+ const entries = await waitForEntries(
+ redis,
+ definition,
+ (es) =>
+ es.some((e) => e.fields.op === "gauge") && es.some((e) => e.fields.op === "enqueue")
+ );
+ const gauge = entries.find((e) => e.fields.op === "gauge");
+ assertGauge(gauge);
+ for (const f of ["ql", "cc", "lim", "eql", "ec", "elim", "thr"]) {
+ expect(gauge!.fields[f]).toBeDefined();
+ }
+ // Fast path was taken => capacity was available => not throttled.
+ expect(gauge!.fields.thr).toBe("0");
+ expect(entries.some((e) => e.fields.op === "enqueue")).toBe(true);
+ } finally {
+ await queue.quit();
+ await emitter.close();
+ }
+ }
+ );
+
+ redisTest("emits an aggregate gauge for CK queues at dequeue", async ({ redisContainer }) => {
+ const redis = {
+ keyPrefix: "runqueue:test:",
+ host: redisContainer.getHost(),
+ port: redisContainer.getPort(),
+ };
+ const definition: MetricDefinition = {
+ name: `qm_ck_${Date.now()}`,
+ shardCount: 2,
+ consumerGroup: "cg",
+ maxLen: 1000,
+ };
+ const emitter = new MetricsStreamEmitter({ redis, definition, flag: { enabled: () => true } });
+ const queue = new RunQueue({
+ name: "rq",
+ tracer: trace.getTracer("rq"),
+ defaultEnvConcurrency: 25,
+ logger: new Logger("RunQueue", "error"),
+ keys: new RunQueueFullKeyProducer(),
+ queueSelectionStrategy: new FairQueueSelectionStrategy({
+ redis,
+ keys: new RunQueueFullKeyProducer(),
+ }),
+ redis,
+ queueMetrics: emitter,
+ });
+
+ const message: InputPayload = {
+ runId: "r-ck",
+ taskIdentifier: "task/my-task",
+ orgId: "o1234",
+ projectId: "p1234",
+ environmentId: authenticatedEnvDev.id,
+ environmentType: "DEVELOPMENT",
+ queue: "task/my-task",
+ concurrencyKey: "tenant-1",
+ timestamp: Date.now(),
+ eligibleAtMs: Date.now() - 300,
+ attempt: 0,
+ };
+
+ try {
+ await queue.enqueueMessage({
+ env: authenticatedEnvDev,
+ message,
+ workerQueue: authenticatedEnvDev.id,
+ });
+ await setTimeout(1000);
+ const dequeued = await queue.dequeueMessageFromWorkerQueue("c1", authenticatedEnvDev.id);
+ expect(dequeued?.messageId).toBe(message.runId);
+
+ const entries = await waitForEntries(redis, definition, (es) =>
+ es.some(
+ (e) => e.fields.op === "gauge" && e.fields.q.includes(":ck:") && e.fields.thr === "0"
+ )
+ );
+ const gauges = entries.filter((e) => e.fields.op === "gauge");
+ expect(gauges.length).toBeGreaterThan(0);
+ // The aggregate CK dequeue gauge targets the CK wildcard and never sets thr.
+ const aggregate = gauges.find((e) => e.fields.q.includes(":ck:") && e.fields.thr === "0");
+ assertGauge(aggregate);
+ expect(Number(aggregate!.fields.ql)).toBeGreaterThanOrEqual(0);
+ expect(Number(aggregate!.fields.cc)).toBeGreaterThanOrEqual(0);
+
+ // Every CK-path gauge carries the CK-health tail; the enqueue-time reading (and the
+ // pre-dequeue aggregate reading) sees the backlogged key.
+ const ckGauges = gauges.filter((e) => e.fields.q.includes(":ck:"));
+ for (const g of ckGauges) {
+ expect(g.fields.ckq).toBeDefined();
+ expect(g.fields.ckw).toBeDefined();
+ expect(Number(g.fields.ckw)).toBeGreaterThanOrEqual(0);
+ }
+ expect(ckGauges.some((g) => Number(g.fields.ckq) >= 1)).toBe(true);
+
+ // CK counter entries carry both odometers: the reading has cum + ck/ckcum, and each
+ // odometer seeds its own baseline entry (cum-only vs ck+ckcum-only).
+ const enqueues = entries.filter((e) => e.fields.op === "enqueue");
+ const reading = enqueues.find((e) => e.fields.cum != null && e.fields.ckcum != null);
+ expect(reading).toBeDefined();
+ expect(reading!.fields.ck).toBe("tenant-1");
+ expect(reading!.fields.q).not.toContain(":ck:");
+ expect(Number(reading!.fields.cum)).toBe(1);
+ expect(Number(reading!.fields.ckcum)).toBe(1);
+ const baseBaseline = enqueues.find((e) => e.fields.cum === "0" && e.fields.ck == null);
+ expect(baseBaseline).toBeDefined();
+ const ckBaseline = enqueues.find((e) => e.fields.ckcum === "0" && e.fields.cum == null);
+ expect(ckBaseline).toBeDefined();
+ expect(ckBaseline!.fields.ck).toBe("tenant-1");
+ } finally {
+ await queue.quit();
+ await emitter.close();
+ }
+ });
+
+ redisTest("gauge sampling gates gauges but not counters", async ({ redisContainer }) => {
+ const redis = {
+ keyPrefix: "runqueue:test:",
+ host: redisContainer.getHost(),
+ port: redisContainer.getPort(),
+ };
+ const definition: MetricDefinition = {
+ name: `qm_sample_${Date.now()}`,
+ shardCount: 2,
+ consumerGroup: "cg",
+ maxLen: 1000,
+ };
+ // gaugeSampleRate 0 => sampledSync() always false => Lua gauge never fires; counters still do.
+ const emitter = new MetricsStreamEmitter({
+ redis,
+ definition,
+ flag: { enabled: () => true },
+ gaugeSampleRate: 0,
+ });
+ const queue = new RunQueue({
+ name: "rq",
+ tracer: trace.getTracer("rq"),
+ defaultEnvConcurrency: 25,
+ logger: new Logger("RunQueue", "error"),
+ keys: new RunQueueFullKeyProducer(),
+ queueSelectionStrategy: new FairQueueSelectionStrategy({
+ redis,
+ keys: new RunQueueFullKeyProducer(),
+ }),
+ redis,
+ queueMetrics: emitter,
+ });
+
+ const message: InputPayload = {
+ runId: "r-sample",
+ taskIdentifier: "task/my-task",
+ orgId: "o1234",
+ projectId: "p1234",
+ environmentId: authenticatedEnvDev.id,
+ environmentType: "DEVELOPMENT",
+ queue: "task/my-task",
+ timestamp: Date.now(),
+ attempt: 0,
+ };
+
+ try {
+ await queue.enqueueMessage({
+ env: authenticatedEnvDev,
+ message,
+ workerQueue: authenticatedEnvDev.id,
+ });
+ await setTimeout(1000);
+ await queue.dequeueMessageFromWorkerQueue("c1", authenticatedEnvDev.id);
+
+ // Poll until the counter (enqueue) lands; by then a gauge would have too, if sampled in.
+ const entries = await waitForEntries(redis, definition, (es) =>
+ es.some((e) => e.fields.op === "enqueue")
+ );
+ expect(entries.some((e) => e.fields.op === "gauge")).toBe(false);
+ expect(entries.some((e) => e.fields.op === "enqueue")).toBe(true);
+ } finally {
+ await queue.quit();
+ await emitter.close();
+ }
+ });
+});
+
+function assertGauge(gauge: unknown): asserts gauge {
+ if (!gauge) throw new Error("expected a gauge entry");
+}
diff --git a/internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts b/internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts
index 224540f4efb..4eb47d59bc0 100644
--- a/internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts
+++ b/internal-packages/run-engine/src/run-queue/tests/ckIndex.test.ts
@@ -471,4 +471,46 @@ describe("CK Index", () => {
await queue.quit();
}
});
+
+ redisTest(
+ "concurrencyKeyBreakdown lists backlogged keys most-starved first",
+ async ({ redisContainer }) => {
+ const queue = createQueue(redisContainer);
+ try {
+ const now = Date.now();
+ const enqueue = (runId: string, concurrencyKey: string, timestamp: number) =>
+ queue.enqueueMessage({
+ env: authenticatedEnvDev,
+ message: makeMessage({ runId, concurrencyKey, timestamp }),
+ workerQueue: authenticatedEnvDev.id,
+ skipDequeueProcessing: true,
+ });
+
+ // ck-a has the oldest head (most starved) and 2 queued; ck-b has 1.
+ await enqueue("r1", "ck-a", now - 10_000);
+ await enqueue("r2", "ck-a", now - 5_000);
+ await enqueue("r3", "ck-b", now - 2_000);
+
+ const breakdown = await queue.concurrencyKeyBreakdown(authenticatedEnvDev, "task/my-task");
+ expect(breakdown.totalBackloggedKeys).toBe(2);
+ expect(breakdown.keys).toEqual([
+ { concurrencyKey: "ck-a", queued: 2, running: 0, oldestEnqueuedAt: now - 10_000 },
+ { concurrencyKey: "ck-b", queued: 1, running: 0, oldestEnqueuedAt: now - 2_000 },
+ ]);
+
+ const limited = await queue.concurrencyKeyBreakdown(authenticatedEnvDev, "task/my-task", {
+ limit: 1,
+ });
+ expect(limited.totalBackloggedKeys).toBe(2);
+ expect(limited.keys).toHaveLength(1);
+ expect(limited.keys[0]!.concurrencyKey).toBe("ck-a");
+
+ // Queues with no CK backlog return an empty breakdown.
+ const empty = await queue.concurrencyKeyBreakdown(authenticatedEnvDev, "task/other-task");
+ expect(empty).toEqual({ totalBackloggedKeys: 0, keys: [] });
+ } finally {
+ await queue.quit();
+ }
+ }
+ );
});
diff --git a/internal-packages/run-engine/src/run-queue/types.ts b/internal-packages/run-engine/src/run-queue/types.ts
index 0905f3971de..8a7d3c93ec5 100644
--- a/internal-packages/run-engine/src/run-queue/types.ts
+++ b/internal-packages/run-engine/src/run-queue/types.ts
@@ -13,6 +13,9 @@ export const InputPayload = z.object({
queue: z.string(),
concurrencyKey: z.string().optional(),
timestamp: z.number(),
+ // Unix ms the run became eligible (delayUntil if set, else triggered-at), pre-priority.
+ // Dequeue scheduling delay = dequeueTime - eligibleAtMs. Optional for old-payload compat.
+ eligibleAtMs: z.number().optional(),
attempt: z.number(),
/** TTL expiration timestamp (unix ms). If set, run will be expired when this time is reached. */
ttlExpiresAt: z.number().optional(),
diff --git a/internal-packages/tsql/src/index.test.ts b/internal-packages/tsql/src/index.test.ts
index f9aca2f236d..ce358e6ac08 100644
--- a/internal-packages/tsql/src/index.test.ts
+++ b/internal-packages/tsql/src/index.test.ts
@@ -231,6 +231,26 @@ describe("injectFallbackConditions", () => {
expect(modified.where.expression_type).toBe("and");
}
});
+
+ it("should inject into a FROM subquery, where the fallback column's table lives", () => {
+ const ast = parseTSQLSelect(
+ "SELECT t, sum(total) AS total FROM (SELECT time AS t, status, count(*) AS total FROM task_runs GROUP BY t, status) GROUP BY t"
+ );
+ const fallbacks: Record = {
+ time: { op: "gte", value: "2024-01-01" },
+ };
+
+ const modified = injectFallbackConditions(ast, fallbacks);
+ expect(modified.expression_type).toBe("select_query");
+ if (modified.expression_type === "select_query") {
+ expect(modified.where).toBeUndefined();
+ const inner = modified.select_from?.table;
+ expect(inner?.expression_type).toBe("select_query");
+ if (inner?.expression_type === "select_query") {
+ expect(isColumnReferencedInExpression(inner.where, "time")).toBe(true);
+ }
+ }
+ });
});
describe("compileTSQL with whereClauseFallback", () => {
diff --git a/internal-packages/tsql/src/index.ts b/internal-packages/tsql/src/index.ts
index 1d8759c108c..1ebd1a60a5d 100644
--- a/internal-packages/tsql/src/index.ts
+++ b/internal-packages/tsql/src/index.ts
@@ -429,6 +429,24 @@ export function injectFallbackConditions(
// Handle SelectQuery
const selectQuery = ast as SelectQuery;
+
+ // When the FROM is a subquery, the fallback columns belong to the inner query's
+ // table, not this level; descend so e.g. a time fallback lands next to the table ref.
+ const fromTable = selectQuery.select_from?.table;
+ if (
+ fromTable &&
+ (fromTable.expression_type === "select_query" ||
+ fromTable.expression_type === "select_set_query")
+ ) {
+ return {
+ ...selectQuery,
+ select_from: {
+ ...selectQuery.select_from!,
+ table: injectFallbackConditions(fromTable, fallbacks) as SelectQuery | SelectSetQuery,
+ },
+ };
+ }
+
const existingWhere = selectQuery.where;
// Collect fallback expressions for columns not already in WHERE
@@ -541,6 +559,12 @@ export interface CompileTSQLOptions {
* ```
*/
timeRange?: TimeRange;
+ /**
+ * Opt-in: emit rows for empty time buckets in a top-level time-bucketed query.
+ * Counters zero-fill, gauges (columns with `fillMode: "carry"`) carry forward.
+ * Off by default; output is unchanged when not set.
+ */
+ fillGaps?: boolean;
}
/**
@@ -599,6 +623,7 @@ export function compileTSQL(query: string, options: CompileTSQLOptions): PrintRe
fieldMappings: options.fieldMappings,
enforcedWhereClause,
timeRange: options.timeRange,
+ fillGaps: options.fillGaps,
});
// 6. Print the AST to ClickHouse SQL (enforced conditions applied at printer level)
diff --git a/internal-packages/tsql/src/query/functions.ts b/internal-packages/tsql/src/query/functions.ts
index 2f2b9278454..a6dadf0f609 100644
--- a/internal-packages/tsql/src/query/functions.ts
+++ b/internal-packages/tsql/src/query/functions.ts
@@ -645,11 +645,24 @@ export const TSQL_AGGREGATIONS: Record = {
maxParams: 1,
aggregate: true,
},
+ quantilesTDigestMerge: {
+ clickhouseName: "quantilesTDigestMerge",
+ minArgs: 1,
+ maxArgs: 1,
+ minParams: 1,
+ aggregate: true,
+ },
sumMerge: { clickhouseName: "sumMerge", minArgs: 1, maxArgs: 1, aggregate: true },
avgMerge: { clickhouseName: "avgMerge", minArgs: 1, maxArgs: 1, aggregate: true },
countMerge: { clickhouseName: "countMerge", minArgs: 1, maxArgs: 1, aggregate: true },
minMerge: { clickhouseName: "minMerge", minArgs: 1, maxArgs: 1, aggregate: true },
maxMerge: { clickhouseName: "maxMerge", minArgs: 1, maxArgs: 1, aggregate: true },
+ deltaSumTimestampMerge: {
+ clickhouseName: "deltaSumTimestampMerge",
+ minArgs: 1,
+ maxArgs: 1,
+ aggregate: true,
+ },
// Statistical functions
simpleLinearRegression: {
diff --git a/internal-packages/tsql/src/query/printer.test.ts b/internal-packages/tsql/src/query/printer.test.ts
index 30d7b3ef82c..c3a95fde325 100644
--- a/internal-packages/tsql/src/query/printer.test.ts
+++ b/internal-packages/tsql/src/query/printer.test.ts
@@ -3914,3 +3914,388 @@ describe("timeBucket()", () => {
});
});
});
+
+// ============================================================
+// fillGaps Tests
+// ============================================================
+
+describe("timeBucket() fillGaps", () => {
+ // Schema with a gauge column (fillMode: "carry"), a counter, and a groupable dim.
+ const metricsSchema: TableSchema = {
+ name: "metrics",
+ clickhouseName: "trigger_dev.queue_metrics_v1",
+ timeConstraint: "bucket_at",
+ columns: {
+ bucket_at: { name: "bucket_at", clickhouseName: "created_at", ...column("DateTime64") },
+ queue_name: { name: "queue_name", ...column("String") },
+ max_running: { name: "max_running", ...column("UInt64"), fillMode: "carry" },
+ enqueued: { name: "enqueued", ...column("UInt64"), fillMode: "zero" },
+ organization_id: { name: "organization_id", ...column("String") },
+ project_id: { name: "project_id", ...column("String") },
+ environment_id: { name: "environment_id", ...column("String") },
+ },
+ tenantColumns: {
+ organizationId: "organization_id",
+ projectId: "project_id",
+ environmentId: "environment_id",
+ },
+ };
+
+ // 7-day range -> 6 HOUR buckets (same as the timeBucket() block).
+ const sevenDayRange = {
+ from: new Date("2024-01-01T00:00:00Z"),
+ to: new Date("2024-01-08T00:00:00Z"),
+ };
+
+ function ctx(fillGaps: boolean): PrinterContext {
+ return createPrinterContext({
+ schema: createSchemaRegistry([metricsSchema]),
+ enforcedWhereClause: {
+ organization_id: { op: "eq", value: "org_test123" },
+ project_id: { op: "eq", value: "proj_test456" },
+ environment_id: { op: "eq", value: "env_test789" },
+ },
+ timeRange: sevenDayRange,
+ fillGaps,
+ });
+ }
+
+ function run(query: string, fillGaps: boolean) {
+ const context = ctx(fillGaps);
+ const result = printToClickHouse(parseTSQLSelect(query), context);
+ return { ...result, warnings: context.warnings };
+ }
+
+ it("emits no WITH FILL when fillGaps is off (unchanged)", () => {
+ const query =
+ "SELECT timeBucket(), max(max_running), count() FROM metrics GROUP BY timeBucket ORDER BY timeBucket";
+ const { sql } = run(query, false);
+ expect(sql).not.toContain("WITH FILL");
+ expect(sql).not.toContain("INTERPOLATE");
+ });
+
+ it("single-series gauge + counter: WITH FILL plus INTERPOLATE for the gauge only", () => {
+ const query =
+ "SELECT timeBucket(), max(max_running) AS max_running, count() AS runs FROM metrics GROUP BY timeBucket ORDER BY timeBucket";
+ const { sql, params } = run(query, true);
+
+ // STEP matches the 6 HOUR bucket interval, FROM/TO snapped + parameterized.
+ expect(sql).toContain("WITH FILL FROM toStartOfInterval({");
+ expect(sql).toContain("STEP INTERVAL 6 HOUR");
+ expect(sql).toMatch(/TO toStartOfInterval\(\{[^}]+: DateTime64\(6\)\}, INTERVAL 6 HOUR\)/);
+
+ // Gauge carried forward; counter omitted (defaults to 0).
+ expect(sql).toContain("INTERPOLATE (max_running AS max_running)");
+ expect(sql).not.toContain("runs AS runs");
+
+ // FROM/TO bounds are real parameters carrying the time range.
+ const dateParams = Object.values(params).filter((v) => v instanceof Date);
+ expect(dateParams).toContainEqual(sevenDayRange.from);
+ expect(dateParams).toContainEqual(sevenDayRange.to);
+ });
+
+ it("single-series counter only: WITH FILL but no INTERPOLATE", () => {
+ const query =
+ "SELECT timeBucket(), count() AS runs FROM metrics GROUP BY timeBucket ORDER BY timeBucket";
+ const { sql } = run(query, true);
+ expect(sql).toContain("WITH FILL FROM toStartOfInterval({");
+ expect(sql).toContain("STEP INTERVAL 6 HOUR");
+ expect(sql).not.toContain("INTERPOLATE");
+ });
+
+ it("grouped counter only: group dim first, then WITH FILL, no INTERPOLATE", () => {
+ const query =
+ "SELECT timeBucket(), queue_name, count() AS runs FROM metrics GROUP BY timeBucket, queue_name ORDER BY timeBucket";
+ const { sql } = run(query, true);
+ expect(sql).toMatch(/ORDER BY queue_name, timebucket ASC WITH FILL/);
+ expect(sql).toContain("STEP INTERVAL 6 HOUR");
+ expect(sql).not.toContain("INTERPOLATE");
+ });
+
+ it("grouped + carry gauge: per-group LOCF via window functions, no INTERPOLATE", () => {
+ const query =
+ "SELECT timeBucket(), queue_name, max(max_running) AS max_running FROM metrics GROUP BY timeBucket, queue_name ORDER BY timeBucket";
+ const { sql, warnings } = run(query, true);
+
+ // Inner query densifies per group (dims first, then the bucket WITH FILL) + sentinel.
+ expect(sql).toMatch(/ORDER BY queue_name, timebucket ASC WITH FILL/);
+ expect(sql).toContain("STEP INTERVAL 6 HOUR");
+ expect(sql).toContain("1 AS __tsql_present");
+
+ // Block id increments at each real row, partitioned by the group dim.
+ expect(sql).toContain(
+ "sum(__tsql_present) OVER (PARTITION BY queue_name ORDER BY timebucket ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS __tsql_block"
+ );
+
+ // Gauge carried within each (group, block); never INTERPOLATE (which bleeds across groups).
+ expect(sql).toContain(
+ "max(if(__tsql_present = 1, max_running, NULL)) OVER (PARTITION BY queue_name, __tsql_block) AS max_running"
+ );
+ expect(sql).not.toContain("INTERPOLATE");
+
+ // Final result re-ordered by the user's ORDER BY, and not skipped.
+ expect(sql).toMatch(/\)\s*ORDER BY timebucket ASC$/);
+ expect(warnings.some((w) => w.code === "fill_skipped_grouped_gauge")).toBe(false);
+ });
+
+ it("grouped + carry gauge with a non-plain group dim: fill is skipped", () => {
+ const query =
+ "SELECT timeBucket(), upper(queue_name) AS q, max(max_running) AS max_running FROM metrics GROUP BY timeBucket, upper(queue_name) ORDER BY timeBucket";
+ const { sql, warnings } = run(query, true);
+ expect(sql).not.toContain("WITH FILL");
+ expect(sql).not.toContain("__tsql_block");
+ expect(warnings.some((w) => w.code === "fill_skipped_grouped_gauge")).toBe(true);
+ });
+
+ it("user ORDER BY not led by timeBucket: fill is skipped", () => {
+ const query =
+ "SELECT timeBucket(), count() AS runs FROM metrics GROUP BY timeBucket ORDER BY runs DESC";
+ const { sql } = run(query, true);
+ expect(sql).not.toContain("WITH FILL");
+ expect(sql).not.toContain("INTERPOLATE");
+ });
+
+ it("bucket-led ORDER BY DESC: fill is skipped (ascending fill would be invalid)", () => {
+ const query =
+ "SELECT timeBucket(), count() AS runs FROM metrics GROUP BY timeBucket ORDER BY timeBucket DESC";
+ const { sql } = run(query, true);
+ expect(sql).not.toContain("WITH FILL");
+ expect(sql).not.toContain("INTERPOLATE");
+ // The plain descending order still stands.
+ expect(sql).toContain("ORDER BY timebucket DESC");
+ });
+});
+
+describe("cross-queue counter totals via subquery (env-wide throughput shape)", () => {
+ // deltaSumTimestamp states must merge per queue, then sum outside; this is the
+ // supported shape for env-wide totals.
+ const metricsSchema: TableSchema = {
+ name: "metrics",
+ clickhouseName: "trigger_dev.queue_metrics_v1",
+ timeConstraint: "bucket_at",
+ columns: {
+ bucket_at: { name: "bucket_at", clickhouseName: "created_at", ...column("DateTime64") },
+ queue_name: { name: "queue_name", ...column("String") },
+ started_delta: {
+ name: "started_delta",
+ mergeGroupKey: "queue_name",
+ ...column("String"),
+ groupable: false,
+ sortable: false,
+ filterable: false,
+ },
+ organization_id: { name: "organization_id", ...column("String") },
+ project_id: { name: "project_id", ...column("String") },
+ environment_id: { name: "environment_id", ...column("String") },
+ },
+ tenantColumns: {
+ organizationId: "organization_id",
+ projectId: "project_id",
+ environmentId: "environment_id",
+ },
+ };
+
+ function runSubquery(query: string) {
+ const context = createPrinterContext({
+ schema: createSchemaRegistry([metricsSchema]),
+ enforcedWhereClause: {
+ organization_id: { op: "eq", value: "org_test123" },
+ },
+ timeRange: {
+ from: new Date("2024-01-01T00:00:00Z"),
+ to: new Date("2024-01-08T00:00:00Z"),
+ },
+ });
+ const result = printToClickHouse(parseTSQLSelect(query), context);
+ return { ...result, warnings: context.warnings };
+ }
+
+ it("compiles per-queue merge + outer sum, with tenant scoping inside the subquery", () => {
+ const { sql, params } = runSubquery(`
+ SELECT t, sum(started) AS started
+ FROM (
+ SELECT timeBucket() AS t, queue_name, deltaSumTimestampMerge(started_delta) AS started
+ FROM metrics
+ GROUP BY t, queue_name
+ )
+ GROUP BY t
+ ORDER BY t
+ `);
+
+ expect(sql).toContain("deltaSumTimestampMerge(started_delta)");
+ expect(sql).toContain("toStartOfInterval(created_at, INTERVAL 6 HOUR)");
+ const subqueryStart = sql.indexOf("FROM (");
+ const tenantFilter = sql.indexOf("organization_id");
+ expect(subqueryStart).toBeGreaterThan(-1);
+ expect(tenantFilter).toBeGreaterThan(subqueryStart);
+ expect(Object.values(params)).toContain("org_test123");
+ });
+});
+
+describe("mergeGroupKey validation", () => {
+ const metricsSchema: TableSchema = {
+ name: "metrics",
+ clickhouseName: "trigger_dev.queue_metrics_v1",
+ timeConstraint: "bucket_at",
+ columns: {
+ bucket_at: { name: "bucket_at", ...column("DateTime64") },
+ queue: { name: "queue", clickhouseName: "queue_name", ...column("String") },
+ started_delta: {
+ name: "started_delta",
+ mergeGroupKey: "queue",
+ ...column("String"),
+ groupable: false,
+ sortable: false,
+ filterable: false,
+ },
+ organization_id: { name: "organization_id", ...column("String") },
+ project_id: { name: "project_id", ...column("String") },
+ environment_id: { name: "environment_id", ...column("String") },
+ },
+ tenantColumns: {
+ organizationId: "organization_id",
+ projectId: "project_id",
+ environmentId: "environment_id",
+ },
+ };
+
+ function compile(
+ query: string,
+ enforced: Record = { organization_id: { op: "eq", value: "org_x" } }
+ ) {
+ const context = createPrinterContext({
+ schema: createSchemaRegistry([metricsSchema]),
+ enforcedWhereClause: enforced as never,
+ timeRange: {
+ from: new Date("2024-01-01T00:00:00Z"),
+ to: new Date("2024-01-08T00:00:00Z"),
+ },
+ });
+ return printToClickHouse(parseTSQLSelect(query), context);
+ }
+
+ it("rejects an ungrouped, unpinned merge with an actionable message", () => {
+ expect(() =>
+ compile(
+ "SELECT timeBucket() AS t, deltaSumTimestampMerge(started_delta) AS started FROM metrics GROUP BY t"
+ )
+ ).toThrowError(
+ /Merging 'started_delta' across every queue[\s\S]*GROUP BY queue\)[\s\S]*WHERE queue = 'my-queue'[\s\S]*inner GROUP BY t, queue and outer GROUP BY t/
+ );
+ });
+
+ it("allows the merge when queue is in the GROUP BY", () => {
+ const { sql } = compile(
+ "SELECT timeBucket() AS t, queue, deltaSumTimestampMerge(started_delta) AS started FROM metrics GROUP BY t, queue"
+ );
+ expect(sql).toContain("deltaSumTimestampMerge(started_delta)");
+ });
+
+ it("allows the merge when queue is pinned by an equality filter", () => {
+ const { sql } = compile(
+ "SELECT deltaSumTimestampMerge(started_delta) AS started FROM metrics WHERE queue = 'emails'"
+ );
+ expect(sql).toContain("deltaSumTimestampMerge(started_delta)");
+ });
+
+ it("allows the merge when the enforced clause pins queue to one value", () => {
+ const { sql } = compile(
+ "SELECT deltaSumTimestampMerge(started_delta) AS started FROM metrics",
+ { organization_id: { op: "eq", value: "org_x" }, queue: { op: "in", values: ["emails"] } }
+ );
+ expect(sql).toContain("deltaSumTimestampMerge(started_delta)");
+ });
+
+ it("rejects the merge when the enforced clause spans several queues", () => {
+ expect(() =>
+ compile("SELECT deltaSumTimestampMerge(started_delta) AS started FROM metrics", {
+ organization_id: { op: "eq", value: "org_x" },
+ queue: { op: "in", values: ["emails", "webhooks"] },
+ })
+ ).toThrowError(/only combine correctly within one queue/);
+ });
+
+ it("allows a grouped inner merge summed by the outer query", () => {
+ const { sql } = compile(
+ "SELECT t, sum(started) AS started FROM (SELECT timeBucket() AS t, queue, deltaSumTimestampMerge(started_delta) AS started FROM metrics GROUP BY t, queue) GROUP BY t ORDER BY t"
+ );
+ expect(sql).toContain("GROUP BY t, queue_name");
+ });
+
+ it("rejects an ungrouped merge inside a subquery", () => {
+ expect(() =>
+ compile(
+ "SELECT t, sum(started) AS started FROM (SELECT timeBucket() AS t, deltaSumTimestampMerge(started_delta) AS started FROM metrics GROUP BY t) GROUP BY t"
+ )
+ ).toThrowError(/only combine correctly within one queue/);
+ });
+});
+
+describe("compound mergeGroupKey validation", () => {
+ const byKeySchema: TableSchema = {
+ name: "metrics_by_key",
+ clickhouseName: "trigger_dev.queue_metrics_ck_v1",
+ timeConstraint: "bucket_at",
+ columns: {
+ bucket_at: { name: "bucket_at", ...column("DateTime64") },
+ queue: { name: "queue", clickhouseName: "queue_name", ...column("String") },
+ concurrency_key: { name: "concurrency_key", ...column("String") },
+ started_delta: {
+ name: "started_delta",
+ mergeGroupKey: ["queue", "concurrency_key"],
+ ...column("String"),
+ groupable: false,
+ sortable: false,
+ filterable: false,
+ },
+ organization_id: { name: "organization_id", ...column("String") },
+ project_id: { name: "project_id", ...column("String") },
+ environment_id: { name: "environment_id", ...column("String") },
+ },
+ tenantColumns: {
+ organizationId: "organization_id",
+ projectId: "project_id",
+ environmentId: "environment_id",
+ },
+ };
+
+ function compile(query: string) {
+ const context = createPrinterContext({
+ schema: createSchemaRegistry([byKeySchema]),
+ enforcedWhereClause: { organization_id: { op: "eq", value: "org_x" } } as never,
+ timeRange: {
+ from: new Date("2024-01-01T00:00:00Z"),
+ to: new Date("2024-01-08T00:00:00Z"),
+ },
+ });
+ return printToClickHouse(parseTSQLSelect(query), context);
+ }
+
+ it("requires EVERY listed key grouped or pinned", () => {
+ expect(() =>
+ compile(
+ "SELECT deltaSumTimestampMerge(started_delta) AS started FROM metrics_by_key WHERE queue = 'emails'"
+ )
+ ).toThrowError(/only combine correctly within one concurrency_key/);
+ expect(() =>
+ compile(
+ "SELECT concurrency_key, deltaSumTimestampMerge(started_delta) AS started FROM metrics_by_key GROUP BY concurrency_key"
+ )
+ ).toThrowError(/only combine correctly within one queue/);
+ });
+
+ it("allows pin + group combinations covering both keys", () => {
+ const grouped = compile(
+ "SELECT concurrency_key, deltaSumTimestampMerge(started_delta) AS started FROM metrics_by_key WHERE queue = 'emails' GROUP BY concurrency_key"
+ );
+ expect(grouped.sql).toContain("deltaSumTimestampMerge(started_delta)");
+ const pinned = compile(
+ "SELECT deltaSumTimestampMerge(started_delta) AS started FROM metrics_by_key WHERE queue = 'emails' AND concurrency_key = 't1'"
+ );
+ expect(pinned.sql).toContain("deltaSumTimestampMerge(started_delta)");
+ const bothGrouped = compile(
+ "SELECT queue, concurrency_key, deltaSumTimestampMerge(started_delta) AS started FROM metrics_by_key GROUP BY queue, concurrency_key"
+ );
+ expect(bothGrouped.sql).toContain("GROUP BY queue_name, concurrency_key");
+ });
+});
diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts
index 6f1af07d71c..fe8c4030110 100644
--- a/internal-packages/tsql/src/query/printer.ts
+++ b/internal-packages/tsql/src/query/printer.ts
@@ -385,6 +385,8 @@ export class ClickHousePrinter {
nextJoin = nextJoin.next_join;
}
+ this.validateMergeScopedColumns(node);
+
// Extract SELECT column aliases BEFORE visiting columns
// This allows ORDER BY/HAVING to reference aliased columns
const savedAliases = this.selectAliases;
@@ -459,6 +461,25 @@ export class ClickHousePrinter {
this.inProjectionContext = false;
}
+ // Opt-in gap-fill: emit rows for empty time buckets via WITH FILL / INTERPOLATE.
+ // No-op unless enabled, top-level, and the query is fill-eligible.
+ let interpolateClause: string | null = null;
+ let groupedFillWrap: ((inner: string) => string) | null = null;
+ if (this.context.fillGaps && isTopLevelQuery) {
+ const fill = this.buildGapFill(node, orderBy, groupBy);
+ if (fill) {
+ orderBy = fill.orderBy;
+ if (fill.kind === "inline") {
+ interpolateClause = fill.interpolate;
+ } else {
+ // Grouped per-group LOCF: add the `present` sentinel to this (now inner) query
+ // and wrap the rendered SQL in the block-id + carry window layers below.
+ columns.push(fill.presentColumn);
+ groupedFillWrap = fill.wrap;
+ }
+ }
+ }
+
// Process ARRAY JOIN
let arrayJoin = "";
if (node.array_join_op) {
@@ -487,6 +508,8 @@ export class ClickHousePrinter {
having ? `HAVING${space}${having}` : null,
windowClause ? `WINDOW${space}${windowClause}` : null,
orderBy && orderBy.length > 0 ? `ORDER BY${space}${orderBy.join(comma)}` : null,
+ // INTERPOLATE must follow the full ORDER BY (including WITH FILL)
+ interpolateClause,
];
// Process LIMIT
@@ -549,6 +572,11 @@ export class ClickHousePrinter {
response = this.pretty ? `(${response.trim()})` : `(${response})`;
}
+ // Grouped per-group gap fill wraps this query in the block-id + carry window layers.
+ if (groupedFillWrap) {
+ response = groupedFillWrap(response);
+ }
+
// Restore saved contexts (for nested queries)
this.selectAliases = savedAliases;
this.queryHasGroupBy = savedQueryHasGroupBy;
@@ -559,6 +587,183 @@ export class ClickHousePrinter {
return response;
}
+ /**
+ * Build the gap-fill transformation (WITH FILL + optional INTERPOLATE) for a
+ * top-level time-bucketed query. Returns null when the query is not
+ * fill-eligible (correct-by-construction: emit nothing extra rather than risk
+ * wrong values).
+ *
+ * Eligibility: exactly one timeBucket() column in SELECT, and ORDER BY led by
+ * that timeBucket column. Carry (gauge) columns are LOCF'd via INTERPOLATE;
+ * counters zero-fill via WITH FILL's default. Grouped gauge queries are unsafe
+ * (INTERPOLATE bleeds across groups) and are skipped with a warning.
+ */
+ private buildGapFill(
+ node: SelectQuery,
+ orderBy: string[] | null,
+ groupBy: string[] | null
+ ):
+ | { kind: "inline"; orderBy: string[]; interpolate: string | null }
+ | { kind: "wrap"; orderBy: string[]; presentColumn: string; wrap: (inner: string) => string }
+ | null {
+ if (!orderBy || orderBy.length === 0 || !node.select || node.select.length === 0) {
+ return null;
+ }
+
+ const timeRange = this.context.timeRange;
+ if (!timeRange) {
+ return null;
+ }
+
+ // Need a time-constraint table to derive the bucket column + interval.
+ const tableWithConstraint = this.findTimeConstraintTable();
+ if (!tableWithConstraint) {
+ return null;
+ }
+ const { tableSchema, clickhouseColumnName } = tableWithConstraint;
+ const interval = calculateTimeBucketInterval(
+ timeRange.from,
+ timeRange.to,
+ tableSchema.timeBucketThresholds
+ );
+ const bucketSql = `toStartOfInterval(${escapeClickHouseIdentifier(clickhouseColumnName)}, INTERVAL ${interval.value} ${interval.unit})`;
+
+ // Find exactly one timeBucket() column in SELECT and its output alias.
+ let bucketAlias: string | null = null;
+ let bucketCount = 0;
+ for (const col of node.select) {
+ const inner = (col as Alias).expression_type === "alias" ? (col as Alias).expr : col;
+ if (
+ (inner as Call).expression_type === "call" &&
+ (inner as Call).name.toLowerCase() === "timebucket"
+ ) {
+ bucketCount++;
+ bucketAlias =
+ (col as Alias).expression_type === "alias" ? (col as Alias).alias : "timebucket";
+ }
+ }
+ if (bucketCount !== 1 || !bucketAlias) {
+ return null;
+ }
+
+ // ORDER BY must be led by the timeBucket column (alias or full expression).
+ // Don't fight a user ordering like `ORDER BY count DESC`.
+ const leadTerm = orderBy[0];
+ // Strip a trailing ASC/DESC direction without a regex: an unanchored `\s+` before the
+ // keyword backtracks polynomially across start positions on whitespace runs (CodeQL
+ // js/polynomial-redos). endsWith + slice is linear.
+ const trimmedLead = leadTerm.trim();
+ const upperLead = trimmedLead.toUpperCase();
+ const isDescending = upperLead.endsWith(" DESC");
+ const leadExpr = upperLead.endsWith(" ASC")
+ ? trimmedLead.slice(0, -4).trimEnd()
+ : isDescending
+ ? trimmedLead.slice(0, -5).trimEnd()
+ : trimmedLead;
+ const matchesBucket = (expr: string): boolean =>
+ expr.toLowerCase() === bucketAlias!.toLowerCase() || expr === bucketSql;
+ if (!matchesBucket(leadExpr)) {
+ return null;
+ }
+ // WITH FILL is emitted with ascending bounds and a positive STEP, which is
+ // only valid for an ascending bucket order. A descending order would need
+ // swapped bounds and a negative step (newer ClickHouse only), so skip the
+ // gap-fill rewrite and let the plain descending ORDER BY stand.
+ if (isDescending) {
+ return null;
+ }
+
+ // Group dims = GROUP BY expressions that are NOT the timeBucket column.
+ const groupDims = (groupBy ?? []).filter((g) => !matchesBucket(g.trim()));
+
+ // Classify each SELECT output column. Carry (gauge) columns survive through
+ // aliases + value-preserving aggregates (see analyzeSelectColumn). A bare column
+ // that isn't the bucket is a GROUP BY dimension; everything else is a counter or
+ // derived value that zero-fills.
+ const carryAliases: string[] = [];
+ const dimNames: string[] = [];
+ const orderedOutputs: Array<{ name: string; carry: boolean }> = [];
+ for (const col of node.select) {
+ const { outputName, sourceColumn } = this.analyzeSelectColumn(col);
+ if (!outputName) continue;
+ const carry = sourceColumn?.fillMode === "carry";
+ orderedOutputs.push({ name: outputName, carry });
+ if (carry) carryAliases.push(outputName);
+ const inner = (col as Alias).expression_type === "alias" ? (col as Alias).expr : col;
+ if (!matchesBucket(outputName) && (inner as Field).expression_type === "field") {
+ dimNames.push(outputName);
+ }
+ }
+
+ // Snap FROM/TO to the bucket grid and parameterize the bounds.
+ const fromBound = this.context.addValue(timeRange.from);
+ const toBound = this.context.addValue(timeRange.to);
+ const withFill =
+ `WITH FILL FROM toStartOfInterval(${fromBound}, INTERVAL ${interval.value} ${interval.unit})` +
+ ` TO toStartOfInterval(${toBound}, INTERVAL ${interval.value} ${interval.unit})` +
+ ` STEP INTERVAL ${interval.value} ${interval.unit}`;
+
+ const esc = escapeClickHouseIdentifier;
+
+ // Single series: WITH FILL on the bucket + INTERPOLATE the carry columns (LOCF);
+ // counters omitted from INTERPOLATE so they zero-fill.
+ if (groupDims.length === 0) {
+ const newOrderBy = [...orderBy];
+ newOrderBy[0] = `${leadTerm} ${withFill}`;
+ const interpolate =
+ carryAliases.length > 0
+ ? `INTERPOLATE (${carryAliases.map((a) => `${esc(a)} AS ${esc(a)}`).join(", ")})`
+ : null;
+ return { kind: "inline", orderBy: newOrderBy, interpolate };
+ }
+
+ // Grouped, counters only: per-group zero-fill via WITH FILL ordered by the dims.
+ if (carryAliases.length === 0) {
+ return {
+ kind: "inline",
+ orderBy: [...groupDims, `${leadTerm} ${withFill}`],
+ interpolate: null,
+ };
+ }
+
+ // Grouped + gauge: per-group LOCF. INTERPOLATE bleeds across groups, so densify per
+ // group (WITH FILL + a `present` sentinel that is 0 on filled rows), assign a block id
+ // that increments at each real row, then carry the block's real value via window max.
+ // Only safe when every GROUP BY dim is a plain column we can PARTITION BY.
+ if (dimNames.length !== groupDims.length) {
+ this.context.addWarning(
+ "fill_skipped_grouped_gauge",
+ "fillGaps was skipped: per-group gap fill needs every GROUP BY dimension to be a plain column."
+ );
+ return null;
+ }
+
+ const userOrderBy = [...orderBy];
+ const presentCol = "__tsql_present";
+ const blockCol = "__tsql_block";
+ const partitionDims = dimNames.map(esc).join(", ");
+ const blockExpr =
+ `sum(${esc(presentCol)}) OVER (PARTITION BY ${partitionDims} ORDER BY ${esc(bucketAlias)}` +
+ ` ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS ${esc(blockCol)}`;
+ const finalColumns = orderedOutputs.map(({ name, carry }) =>
+ carry
+ ? `max(if(${esc(presentCol)} = 1, ${esc(name)}, NULL)) OVER (PARTITION BY ${partitionDims}, ${esc(
+ blockCol
+ )}) AS ${esc(name)}`
+ : esc(name)
+ );
+ const finalOrderBy = userOrderBy.length > 0 ? ` ORDER BY ${userOrderBy.join(", ")}` : "";
+ const wrap = (inner: string): string =>
+ `SELECT ${finalColumns.join(", ")} FROM (SELECT *, ${blockExpr} FROM (${inner.trim()}))${finalOrderBy}`;
+
+ return {
+ kind: "wrap",
+ orderBy: [...dimNames.map(esc), `${leadTerm} ${withFill}`],
+ presentColumn: `1 AS ${esc(presentCol)}`,
+ wrap,
+ };
+ }
+
/**
* Extract column aliases from a SELECT expression.
* Handles explicit aliases (AS name) and implicit names from aggregations/functions.
@@ -1014,11 +1219,12 @@ export class ClickHousePrinter {
if ((firstArg as Field).expression_type === "field") {
const field = firstArg as Field;
const columnInfo = this.resolveFieldToColumn(field.chain);
- // Only propagate customRenderType, not the full column schema
- if (columnInfo.column?.customRenderType) {
+ // Propagate customRenderType and fillMode (gauge-ness), not the full column schema
+ if (columnInfo.column?.customRenderType || columnInfo.column?.fillMode) {
sourceColumn = {
type: inferredType,
customRenderType: columnInfo.column.customRenderType,
+ fillMode: columnInfo.column.fillMode,
};
}
}
@@ -1679,6 +1885,138 @@ export class ClickHousePrinter {
// Note: projectId and environmentId are optional - no validation needed
}
+ /**
+ * Reject queries that merge a scope-keyed aggregate state column (`mergeGroupKey`)
+ * across values of its key: such merges silently return wrong numbers. Valid shapes
+ * group by the key column or pin it to a single value (in the query's WHERE or via
+ * the enforced clause). Runs per SELECT scope; subqueries validate themselves.
+ */
+ private validateMergeScopedColumns(node: SelectQuery): void {
+ for (const tableSchema of this.tableContexts.values()) {
+ for (const column of Object.values(tableSchema.columns)) {
+ if (!column.mergeGroupKey) continue;
+ const keys = Array.isArray(column.mergeGroupKey)
+ ? column.mergeGroupKey
+ : [column.mergeGroupKey];
+ if (!this.scopeReferencesColumn(node, column.name)) continue;
+ for (const key of keys) {
+ if (this.groupByIncludesColumn(node, key)) continue;
+ if (this.wherePinsColumn(node.where, key)) continue;
+ if (this.enforcedPinsColumn(tableSchema, key)) continue;
+ throw new QueryError(
+ `Merging '${column.name}' across every ${key} returns wrong totals: its aggregate ` +
+ `states are kept per ${key} and only combine correctly within one ${key}. Either ` +
+ `add '${key}' to the GROUP BY and sum the per-${key} results in an outer query, ` +
+ `for example: SELECT sum(v) AS total FROM (SELECT ${key}, ` +
+ `deltaSumTimestampMerge(${column.name}) AS v FROM ${tableSchema.name} ` +
+ `GROUP BY ${key}). Or filter to a single ${key}, for example: ` +
+ `WHERE ${key} = 'my-${key}'. For a time series, bucket both layers: ` +
+ `inner GROUP BY t, ${key} and outer GROUP BY t.`
+ );
+ }
+ }
+ }
+ }
+
+ private scopeReferencesColumn(node: SelectQuery, name: string): boolean {
+ const parts: unknown[] = [
+ node.select,
+ node.prewhere,
+ node.where,
+ node.group_by,
+ node.having,
+ node.order_by,
+ ];
+ return parts.some((part) => this.expressionReferencesColumn(part, name));
+ }
+
+ private expressionReferencesColumn(
+ expr: unknown,
+ name: string,
+ seen = new WeakSet()
+ ): boolean {
+ if (expr === null || typeof expr !== "object") return false;
+ if (seen.has(expr)) return false;
+ seen.add(expr);
+ if (Array.isArray(expr)) {
+ return expr.some((item) => this.expressionReferencesColumn(item, name, seen));
+ }
+ const candidate = expr as { expression_type?: string; chain?: unknown[] };
+ if (
+ candidate.expression_type === "select_query" ||
+ candidate.expression_type === "select_set_query"
+ ) {
+ return false;
+ }
+ if (
+ candidate.expression_type === "field" &&
+ Array.isArray(candidate.chain) &&
+ candidate.chain[candidate.chain.length - 1] === name
+ ) {
+ return true;
+ }
+ return Object.entries(expr).some(
+ ([property, value]) =>
+ property !== "type" &&
+ property !== "parent" &&
+ this.expressionReferencesColumn(value, name, seen)
+ );
+ }
+
+ private groupByIncludesColumn(node: SelectQuery, name: string): boolean {
+ return (node.group_by ?? []).some((expr) => {
+ const field = expr as Field;
+ return (
+ field.expression_type === "field" &&
+ Array.isArray(field.chain) &&
+ field.chain[field.chain.length - 1] === name
+ );
+ });
+ }
+
+ // Pins only count on the top-level AND chain: a pin inside an OR guarantees nothing.
+ private wherePinsColumn(where: Expression | undefined, name: string): boolean {
+ if (!where) return false;
+ if (where.expression_type === "and") {
+ return (where as And).exprs.some((expr) => this.wherePinsColumn(expr, name));
+ }
+ if (where.expression_type !== "compare_operation") return false;
+ const cmp = where as CompareOperation;
+ const isKeyField = (side: Expression) => {
+ const field = side as Field;
+ return (
+ field.expression_type === "field" &&
+ Array.isArray(field.chain) &&
+ field.chain[field.chain.length - 1] === name
+ );
+ };
+ const fieldSide = [cmp.left, cmp.right].find(isKeyField);
+ if (!fieldSide) return false;
+ if (cmp.op === CompareOperationOp.Eq) return true;
+ if (cmp.op === CompareOperationOp.In || cmp.op === CompareOperationOp.GlobalIn) {
+ const other = fieldSide === cmp.left ? cmp.right : cmp.left;
+ if ((other as Constant).expression_type === "constant") return true;
+ const tuple = other as Tuple;
+ return tuple.expression_type === "tuple" && tuple.exprs.length === 1;
+ }
+ return false;
+ }
+
+ private enforcedPinsColumn(tableSchema: TableSchema, key: string): boolean {
+ const names = [key];
+ const clickhouseName = tableSchema.columns[key]?.clickhouseName;
+ if (clickhouseName) names.push(clickhouseName);
+ for (const name of names) {
+ const condition = this.context.enforcedWhereClause[name] as
+ | { op?: string; values?: unknown[] }
+ | undefined;
+ if (!condition) continue;
+ if (condition.op === "eq") return true;
+ if (condition.op === "in" && condition.values?.length === 1) return true;
+ }
+ return false;
+ }
+
/**
* Format a Date as a ClickHouse-compatible DateTime64 string.
* ClickHouse expects format: 'YYYY-MM-DD HH:MM:SS.mmm' (in UTC)
diff --git a/internal-packages/tsql/src/query/printer_context.ts b/internal-packages/tsql/src/query/printer_context.ts
index d0fb41b5327..a964e2e04af 100644
--- a/internal-packages/tsql/src/query/printer_context.ts
+++ b/internal-packages/tsql/src/query/printer_context.ts
@@ -125,6 +125,9 @@ export class PrinterContext {
*/
readonly timeRange?: TimeRange;
+ /** When true, time-bucketed queries emit rows for empty buckets (opt-in). */
+ readonly fillGaps?: boolean;
+
constructor(
/** Schema registry containing allowed tables and columns */
public readonly schema: SchemaRegistry,
@@ -138,13 +141,16 @@ export class PrinterContext {
*/
enforcedWhereClause: Record = {},
/** Time range for timeBucket() interval calculation */
- timeRange?: TimeRange
+ timeRange?: TimeRange,
+ /** Opt-in gap-fill for time-bucketed queries */
+ fillGaps?: boolean
) {
// Initialize with default settings
this.settings = { ...DEFAULT_QUERY_SETTINGS, ...settings };
this.fieldMappings = fieldMappings;
this.enforcedWhereClause = enforcedWhereClause;
this.timeRange = timeRange;
+ this.fillGaps = fillGaps;
}
/**
@@ -225,7 +231,8 @@ export class PrinterContext {
this.settings,
this.fieldMappings,
this.enforcedWhereClause,
- this.timeRange
+ this.timeRange,
+ this.fillGaps
);
// Share the same values map so parameters are unified
child.values = this.values;
@@ -277,6 +284,8 @@ export interface PrinterContextOptions {
* When provided, `timeBucket()` uses this to determine the appropriate bucket size.
*/
timeRange?: TimeRange;
+ /** When true, time-bucketed queries emit rows for empty buckets (opt-in). */
+ fillGaps?: boolean;
}
/**
@@ -288,6 +297,7 @@ export function createPrinterContext(options: PrinterContextOptions): PrinterCon
options.settings,
options.fieldMappings,
options.enforcedWhereClause,
- options.timeRange
+ options.timeRange,
+ options.fillGaps
);
}
diff --git a/internal-packages/tsql/src/query/schema.ts b/internal-packages/tsql/src/query/schema.ts
index 9a1e2d2ddfe..a32b8ea142c 100644
--- a/internal-packages/tsql/src/query/schema.ts
+++ b/internal-packages/tsql/src/query/schema.ts
@@ -122,6 +122,18 @@ export interface ColumnSchema {
* ```
*/
customRenderType?: string;
+ /**
+ * Gap-fill behavior when the opt-in `fillGaps` feature emits rows for empty
+ * time buckets: `"carry"` = gauge (LOCF via INTERPOLATE), `"zero"` (default)
+ * = counter (missing buckets get 0).
+ */
+ fillMode?: "zero" | "carry";
+ /**
+ * Aggregate-state column whose states only merge correctly within one value of the
+ * named column(s) (e.g. per-queue counter states). Queries referencing it must GROUP BY
+ * every listed column or pin each to a single value; other shapes fail to compile.
+ */
+ mergeGroupKey?: string | string[];
/**
* Example value for documentation purposes.
*
@@ -409,6 +421,21 @@ export interface TableSchema {
* is needed to get correct results. Not needed for plain MergeTree tables.
*/
useFinal?: boolean;
+ /**
+ * Coarser physical rollups with an identical logical schema, substituted by callers
+ * (not the printer) when the timeBucket() interval is at least minIntervalSeconds.
+ */
+ rollups?: Array<{ minIntervalSeconds: number; clickhouseName: string }>;
+ /**
+ * Opt into the ClickHouse query cache; callers align time bounds to alignSeconds
+ * so repeated auto-refresh queries share cache entries.
+ */
+ queryCache?: { ttlSeconds: number; alignSeconds: number };
+ /**
+ * Excluded from user-facing listings (query editor, schema docs, schema API) by
+ * callers; the engine still compiles queries against it.
+ */
+ hidden?: boolean;
}
/**
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 7e553d0d24a..4d9f8608dc8 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -288,6 +288,9 @@ importers:
'@internal/llm-model-catalog':
specifier: workspace:*
version: link:../../internal-packages/llm-model-catalog
+ '@internal/metrics-pipeline':
+ specifier: workspace:*
+ version: link:../../internal-packages/metrics-pipeline
'@internal/redis':
specifier: workspace:*
version: link:../../internal-packages/redis
@@ -1076,6 +1079,25 @@ importers:
specifier: 4.1.7
version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@22.20.0)(@vitest/coverage-v8@4.1.7)(vite@6.4.2(@types/node@22.20.0)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.22.4)(yaml@2.9.0))
+ internal-packages/metrics-pipeline:
+ dependencies:
+ '@internal/redis':
+ specifier: workspace:*
+ version: link:../redis
+ '@internal/tracing':
+ specifier: workspace:*
+ version: link:../tracing
+ '@trigger.dev/core':
+ specifier: workspace:*
+ version: link:../../packages/core
+ devDependencies:
+ '@internal/testcontainers':
+ specifier: workspace:*
+ version: link:../testcontainers
+ rimraf:
+ specifier: 6.0.1
+ version: 6.0.1
+
internal-packages/otlp-importer:
dependencies:
long:
@@ -1156,6 +1178,9 @@ importers:
'@internal/cache':
specifier: workspace:*
version: link:../cache
+ '@internal/metrics-pipeline':
+ specifier: workspace:*
+ version: link:../metrics-pipeline
'@internal/redis':
specifier: workspace:*
version: link:../redis