improvement(logs): object storage backed tracespans (#4787)

* improvement(logs): obj storage backed tracespans

* fix storage write context

* fix tests

* address comments

* address comments

* chore(db): remove migration 0219 to regenerate after staging merge

Drops the 0219_robust_shard SQL, its snapshot, and the journal entry so the
trace-spans/cost schema migration can be regenerated on top of the latest
staging migration chain (avoids a number collision with staging's migrations).

Co-authored-by: Cursor <cursoragent@cursor.com>

* improvement(billing): accurate per-member usage via shared ledger helper

Per-member/per-user usage in the org-member routes now adds the usage_log
ledger to the currentPeriodCost baseline (which is no longer incremented),
via a shared getOrgMemberLedgerByUser helper to avoid repeating the
subscription→period→ledger lookup across the admin and member-facing routes.

Co-authored-by: Cursor <cursoragent@cursor.com>

* regen migrations

* update migration

* address comments

* more code cleanup

* incorrect type cast

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Vikhyath Mondreti
2026-05-29 13:47:32 -07:00
committed by GitHub
co-authored by Cursor
parent c95aa879f7
commit 49673055be
61 changed files with 20003 additions and 932 deletions
-1
View File
@@ -140,7 +140,6 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
members: rawBillingData.members.map((m) => ({
...m,
joinedAt: m.joinedAt.toISOString(),
lastActive: m.lastActive?.toISOString() || null,
})),
}
+6 -1
View File
@@ -259,7 +259,12 @@ describe('Knowledge Utils', () => {
{}
)
expect(dbOps.order).toEqual(['insert', 'updateDoc'])
// Embeddings are inserted first, then the document counter update. A
// usage_log billing insert (recordUsage) may trail after updateDoc and is
// irrelevant to this ordering invariant, so assert position rather than
// exact array equality.
expect(dbOps.order[0]).toBe('insert')
expect(dbOps.order.indexOf('updateDoc')).toBeGreaterThan(0)
expect(dbOps.updatePayloads[0]).toMatchObject({
processingStatus: 'completed',
@@ -13,6 +13,7 @@ import { executionIdParamsSchema } from '@/lib/api/contracts/logs'
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
import type { TraceSpan, WorkflowExecutionLog } from '@/lib/logs/types'
const logger = createLogger('LogsByExecutionIdAPI')
@@ -39,13 +40,14 @@ export const GET = withRouteHandler(
.select({
id: workflowExecutionLogs.id,
workflowId: workflowExecutionLogs.workflowId,
workspaceId: workflowExecutionLogs.workspaceId,
executionId: workflowExecutionLogs.executionId,
stateSnapshotId: workflowExecutionLogs.stateSnapshotId,
trigger: workflowExecutionLogs.trigger,
startedAt: workflowExecutionLogs.startedAt,
endedAt: workflowExecutionLogs.endedAt,
totalDurationMs: workflowExecutionLogs.totalDurationMs,
cost: workflowExecutionLogs.cost,
costTotal: workflowExecutionLogs.costTotal,
executionData: workflowExecutionLogs.executionData,
})
.from(workflowExecutionLogs)
@@ -119,7 +121,14 @@ export const GET = withRouteHandler(
return NextResponse.json({ error: 'Workflow state snapshot not found' }, { status: 404 })
}
const executionData = workflowLog.executionData as WorkflowExecutionLog['executionData']
const executionData = (await materializeExecutionData(
workflowLog.executionData as Record<string, unknown> | null,
{
workspaceId: workflowLog.workspaceId,
workflowId: workflowLog.workflowId,
executionId: workflowLog.executionId,
}
)) as WorkflowExecutionLog['executionData']
const traceSpans = (executionData?.traceSpans as TraceSpan[]) || []
const childSnapshotIds = new Set<string>()
const collectSnapshotIds = (spans: TraceSpan[]) => {
@@ -163,7 +172,7 @@ export const GET = withRouteHandler(
startedAt: workflowLog.startedAt.toISOString(),
endedAt: workflowLog.endedAt?.toISOString(),
totalDurationMs: workflowLog.totalDurationMs,
cost: workflowLog.cost || null,
cost: workflowLog.costTotal != null ? { total: Number(workflowLog.costTotal) } : null,
},
}
+33 -8
View File
@@ -4,7 +4,9 @@ import { createLogger } from '@sim/logger'
import { and, desc, eq, sql } from 'drizzle-orm'
import { type NextRequest, NextResponse } from 'next/server'
import { getSession } from '@/lib/auth'
import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
import { buildFilterConditions, LogFilterParamsSchema } from '@/lib/logs/filters'
import { expandFolderIdsWithDescendants } from '@/lib/logs/folder-expansion'
@@ -41,7 +43,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
startedAt: workflowExecutionLogs.startedAt,
endedAt: workflowExecutionLogs.endedAt,
totalDurationMs: workflowExecutionLogs.totalDurationMs,
cost: workflowExecutionLogs.cost,
costTotal: workflowExecutionLogs.costTotal,
executionData: workflowExecutionLogs.executionData,
workflowName: sql<string>`COALESCE(${workflow.name}, 'Deleted Workflow')`,
}
@@ -96,11 +98,29 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
if (!rows.length) break
for (const r of rows as any[]) {
// Heavy execution data may live in object storage; materialize per
// row with bounded concurrency so a 1000-row page doesn't fan out
// into 1000 simultaneous reads.
const materialized = await mapWithConcurrency(
rows as any[],
MATERIALIZE_CONCURRENCY,
(r) =>
materializeExecutionData(r.executionData as Record<string, unknown> | null, {
workspaceId: params.workspaceId,
workflowId: r.workflowId,
executionId: r.executionId,
})
)
for (let j = 0; j < rows.length; j++) {
const r = rows[j] as any
const ed = materialized[j] as Record<string, any>
// A single malformed/unserializable row must not abort the whole CSV
// stream — derive the message/trace columns defensively and fall back
// to empty on error so the row's metadata still exports.
let message = ''
let traces: any = null
let tracesJson = ''
try {
const ed = (r as any).executionData
if (ed) {
if (ed.finalOutput)
message =
@@ -108,20 +128,25 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
? ed.finalOutput
: JSON.stringify(ed.finalOutput)
if (ed.message) message = ed.message
if (ed.traceSpans) traces = ed.traceSpans
if (ed.traceSpans) tracesJson = JSON.stringify(ed.traceSpans)
}
} catch {}
} catch (rowError) {
logger.warn('Skipping unserializable execution data for export row', {
executionId: r.executionId,
error: rowError instanceof Error ? rowError.message : String(rowError),
})
}
const line = [
escapeCsv(r.startedAt?.toISOString?.() || r.startedAt),
escapeCsv(r.level),
escapeCsv(r.workflowName),
escapeCsv(r.trigger),
escapeCsv(r.totalDurationMs ?? ''),
escapeCsv(r.cost?.total ?? r.cost?.value?.total ?? ''),
escapeCsv(r.costTotal ?? ''),
escapeCsv(r.workflowId ?? ''),
escapeCsv(r.executionId ?? ''),
escapeCsv(message),
escapeCsv(traces ? JSON.stringify(traces) : ''),
escapeCsv(tracesJson),
].join(',')
controller.enqueue(encoder.encode(`${line}\n`))
}
+8 -4
View File
@@ -31,6 +31,7 @@ import { listLogsContract, type WorkflowLogSummary } from '@/lib/api/contracts/l
import { parseRequest } from '@/lib/api/server'
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { jobCostTotal } from '@/lib/logs/fetch-log-detail'
import { buildFilterConditions } from '@/lib/logs/filters'
import { expandFolderIdsWithDescendants } from '@/lib/logs/folder-expansion'
@@ -81,7 +82,8 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
case 'duration':
return sql`${workflowExecutionLogs.totalDurationMs}`
case 'cost':
return sql`(${workflowExecutionLogs.cost}->>'total')::numeric`
// Indexed projection of the usage_log ledger (dollars); no live aggregation.
return sql`${workflowExecutionLogs.costTotal}`
case 'status':
return sql`${workflowExecutionLogs.status}`
default:
@@ -201,7 +203,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
startedAt: workflowExecutionLogs.startedAt,
endedAt: workflowExecutionLogs.endedAt,
totalDurationMs: workflowExecutionLogs.totalDurationMs,
cost: workflowExecutionLogs.cost,
costTotal: workflowExecutionLogs.costTotal,
createdAt: workflowExecutionLogs.createdAt,
workflowName: workflow.name,
workflowDescription: workflow.description,
@@ -379,7 +381,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
}
: null,
jobTitle: null,
cost: (log.cost as WorkflowLogSummary['cost']) ?? null,
// List cost is the cost_total projection (faithful ledger sum). Null until
// completion (running) or until the one-time legacy backfill populates it.
cost: log.costTotal != null ? { total: Number(log.costTotal) } : null,
pauseSummary: {
status: log.pausedStatus ?? null,
total: totalPauseCount,
@@ -405,7 +409,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
createdAt: log.startedAt.toISOString(),
workflow: null,
jobTitle: log.jobTitle ?? null,
cost: (log.cost as WorkflowLogSummary['cost']) ?? null,
cost: jobCostTotal(log.cost),
pauseSummary: { status: null, total: 0, resumed: 0 },
hasPendingPause: false,
}
@@ -11,6 +11,7 @@ import {
import { parseRequest } from '@/lib/api/server'
import { getSession } from '@/lib/auth'
import { setActiveOrganizationForCurrentSession } from '@/lib/auth/active-organization'
import { getOrgMemberLedgerByUser } from '@/lib/billing/core/organization'
import { getUserUsageData } from '@/lib/billing/core/usage'
import {
removeExternalUserFromOrganizationWorkspaces,
@@ -101,10 +102,25 @@ export const GET = withRouteHandler(
const computed = await getUserUsageData(memberId)
if (usageData.length > 0) {
// currentPeriodCost is only a baseline; add this member's attributed
// usage_log for the period. (getUserUsageData returns the org POOL for
// org-scoped members, so it can't supply the per-member figure.)
const memberLedger =
(
await getOrgMemberLedgerByUser(
organizationId,
computed.billingPeriodStart && computed.billingPeriodEnd
? { start: computed.billingPeriodStart, end: computed.billingPeriodEnd }
: null
)
).get(memberId) ?? 0
memberData = {
...memberData,
usage: {
...usageData[0],
currentPeriodCost: (
Number(usageData[0].currentPeriodCost ?? 0) + memberLedger
).toString(),
billingPeriodStart: computed.billingPeriodStart,
billingPeriodEnd: computed.billingPeriodEnd,
},
@@ -17,6 +17,7 @@ import {
} from '@/lib/api/contracts/organization'
import { getValidationErrorMessage, parseRequest } from '@/lib/api/server'
import { getSession } from '@/lib/auth'
import { getOrgMemberLedgerByUser } from '@/lib/billing/core/organization'
import { ENTITLED_SUBSCRIPTION_STATUSES } from '@/lib/billing/subscriptions/utils'
import { validateSeatAvailability } from '@/lib/billing/validation/seat-management'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
@@ -139,8 +140,21 @@ export const GET = withRouteHandler(
const billingPeriodStart = orgSub?.periodStart ?? null
const billingPeriodEnd = orgSub?.periodEnd ?? null
// currentPeriodCost is only a baseline; add each member's attributed
// usage_log for the period (batched, one query) so the roster shows real
// usage rather than the frozen baseline.
const usageByUser = await getOrgMemberLedgerByUser(
organizationId,
billingPeriodStart && billingPeriodEnd
? { start: billingPeriodStart, end: billingPeriodEnd }
: null
)
const membersWithUsage = base.map((row) => ({
...row,
currentPeriodCost: (
Number(row.currentPeriodCost ?? 0) + (usageByUser.get(row.userId) ?? 0)
).toString(),
billingPeriodStart,
billingPeriodEnd,
}))
@@ -35,6 +35,7 @@ import {
adminV1UpdateOrganizationMemberContract,
} from '@/lib/api/contracts/v1/admin'
import { parseRequest } from '@/lib/api/server'
import { getOrgMemberLedgerByUser } from '@/lib/billing/core/organization'
import { removeUserFromOrganization } from '@/lib/billing/organizations/membership'
import { isBillingEnabled } from '@/lib/core/config/feature-flags'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
@@ -86,7 +87,6 @@ export const GET = withRouteHandler(
userEmail: user.email,
currentPeriodCost: userStats.currentPeriodCost,
currentUsageLimit: userStats.currentUsageLimit,
lastActive: userStats.lastActive,
billingBlocked: userStats.billingBlocked,
})
.from(member)
@@ -99,6 +99,10 @@ export const GET = withRouteHandler(
return notFoundResponse('Member')
}
// currentPeriodCost is only a baseline; add this member's attributed
// usage_log for the org's period so admin shows real current usage.
const ledgerByUser = await getOrgMemberLedgerByUser(organizationId)
const data: AdminMemberDetail = {
id: memberData.id,
userId: memberData.userId,
@@ -107,9 +111,10 @@ export const GET = withRouteHandler(
createdAt: memberData.createdAt.toISOString(),
userName: memberData.userName,
userEmail: memberData.userEmail,
currentPeriodCost: memberData.currentPeriodCost ?? '0',
currentPeriodCost: (
Number(memberData.currentPeriodCost ?? 0) + (ledgerByUser.get(memberData.userId) ?? 0)
).toString(),
currentUsageLimit: memberData.currentUsageLimit,
lastActive: memberData.lastActive?.toISOString() ?? null,
billingBlocked: memberData.billingBlocked ?? false,
}
@@ -37,6 +37,7 @@ import {
adminV1ListOrganizationMembersContract,
} from '@/lib/api/contracts/v1/admin'
import { parseRequest } from '@/lib/api/server'
import { getOrgMemberLedgerByUser } from '@/lib/billing/core/organization'
import { addUserToOrganization } from '@/lib/billing/organizations/membership'
import { isBillingEnabled } from '@/lib/core/config/feature-flags'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
@@ -96,7 +97,6 @@ export const GET = withRouteHandler(
userEmail: user.email,
currentPeriodCost: userStats.currentPeriodCost,
currentUsageLimit: userStats.currentUsageLimit,
lastActive: userStats.lastActive,
billingBlocked: userStats.billingBlocked,
})
.from(member)
@@ -109,6 +109,11 @@ export const GET = withRouteHandler(
])
const total = countResult[0].count
// currentPeriodCost is only a baseline; add each member's attributed
// usage_log for the org's period so admin shows real current usage.
const usageByUser = await getOrgMemberLedgerByUser(organizationId)
const data: AdminMemberDetail[] = membersData.map((m) => ({
id: m.id,
userId: m.userId,
@@ -117,9 +122,10 @@ export const GET = withRouteHandler(
createdAt: m.createdAt.toISOString(),
userName: m.userName,
userEmail: m.userEmail,
currentPeriodCost: m.currentPeriodCost ?? '0',
currentPeriodCost: (
Number(m.currentPeriodCost ?? 0) + (usageByUser.get(m.userId) ?? 0)
).toString(),
currentUsageLimit: m.currentUsageLimit,
lastActive: m.lastActive?.toISOString() ?? null,
billingBlocked: m.billingBlocked ?? false,
}))
@@ -51,16 +51,6 @@ export const GET = withRouteHandler(
subscriptionPlan: analytics.subscriptionPlan,
canAddSeats: analytics.canAddSeats,
utilizationRate: analytics.utilizationRate,
activeMembers: analytics.activeMembers,
inactiveMembers: analytics.inactiveMembers,
memberActivity: analytics.memberActivity.map((m) => ({
userId: m.userId,
userName: m.userName,
userEmail: m.userEmail,
role: m.role,
joinedAt: m.joinedAt.toISOString(),
lastActive: m.lastActive?.toISOString() ?? null,
})),
}
logger.info(`Admin API: Retrieved seat analytics for organization ${organizationId}`)
+1 -25
View File
@@ -543,7 +543,6 @@ export interface AdminMemberDetail extends AdminMember {
// Billing/usage info from userStats
currentPeriodCost: string
currentUsageLimit: string | null
lastActive: string | null
billingBlocked: boolean
}
@@ -574,28 +573,15 @@ interface AdminUserBilling {
userEmail: string
stripeCustomerId: string | null
// Usage stats
totalManualExecutions: number
totalApiCalls: number
totalWebhookTriggers: number
totalScheduledExecutions: number
totalChatExecutions: number
totalMcpExecutions: number
totalA2aExecutions: number
totalTokensUsed: number
totalCost: string
currentUsageLimit: string | null
currentPeriodCost: string
lastPeriodCost: string | null
billedOverageThisPeriod: string
storageUsedBytes: number
lastActive: string | null
billingBlocked: boolean
// Copilot usage
totalCopilotCost: string
// Copilot usage (active per-period baselines)
currentPeriodCopilotCost: string
lastPeriodCopilotCost: string | null
totalCopilotTokens: number
totalCopilotCalls: number
}
export interface AdminUserBillingWithSubscription extends AdminUserBilling {
@@ -643,16 +629,6 @@ export interface AdminSeatAnalytics {
subscriptionPlan: string
canAddSeats: boolean
utilizationRate: number
activeMembers: number
inactiveMembers: number
memberActivity: Array<{
userId: string
userName: string
userEmail: string
role: string
joinedAt: string
lastActive: string | null
}>
}
export interface AdminDeploymentVersion {
@@ -29,6 +29,7 @@ import {
} from '@/lib/api/contracts/v1/admin'
import { parseRequest } from '@/lib/api/server'
import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
import { getUserUsageData } from '@/lib/billing/core/usage'
import { isOrgScopedSubscription } from '@/lib/billing/subscriptions/utils'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { withAdminAuthParams } from '@/app/api/v1/admin/middleware'
@@ -78,6 +79,11 @@ export const GET = withRouteHandler(
const [stats] = await db.select().from(userStats).where(eq(userStats.userId, userId)).limit(1)
// currentPeriodCost is now only a baseline; canonical current-period usage
// (baseline + attributed usage_log, refresh-adjusted) comes from the same
// helper users see, so admin reflects real usage instead of a stale 0.
const usage = await getUserUsageData(userId)
const memberOrgs = await db
.select({
organizationId: member.organizationId,
@@ -107,27 +113,14 @@ export const GET = withRouteHandler(
userName: userData.name,
userEmail: userData.email,
stripeCustomerId: userData.stripeCustomerId,
totalManualExecutions: stats?.totalManualExecutions ?? 0,
totalApiCalls: stats?.totalApiCalls ?? 0,
totalWebhookTriggers: stats?.totalWebhookTriggers ?? 0,
totalScheduledExecutions: stats?.totalScheduledExecutions ?? 0,
totalChatExecutions: stats?.totalChatExecutions ?? 0,
totalMcpExecutions: stats?.totalMcpExecutions ?? 0,
totalA2aExecutions: stats?.totalA2aExecutions ?? 0,
totalTokensUsed: stats?.totalTokensUsed ?? 0,
totalCost: stats?.totalCost ?? '0',
currentUsageLimit: stats?.currentUsageLimit ?? null,
currentPeriodCost: stats?.currentPeriodCost ?? '0',
currentPeriodCost: usage.currentUsage.toString(),
lastPeriodCost: stats?.lastPeriodCost ?? null,
billedOverageThisPeriod: stats?.billedOverageThisPeriod ?? '0',
storageUsedBytes: stats?.storageUsedBytes ?? 0,
lastActive: stats?.lastActive?.toISOString() ?? null,
billingBlocked: stats?.billingBlocked ?? false,
totalCopilotCost: stats?.totalCopilotCost ?? '0',
currentPeriodCopilotCost: stats?.currentPeriodCopilotCost ?? '0',
lastPeriodCopilotCost: stats?.lastPeriodCopilotCost ?? null,
totalCopilotTokens: stats?.totalCopilotTokens ?? 0,
totalCopilotCalls: stats?.totalCopilotCalls ?? 0,
subscriptions: subscriptions.map(toAdminSubscription),
organizationMemberships: memberOrgs.map((m) => ({
organizationId: m.organizationId,
+11 -3
View File
@@ -7,6 +7,7 @@ import { type NextRequest, NextResponse } from 'next/server'
import { v1GetLogContract } from '@/lib/api/contracts/v1/logs'
import { parseRequest } from '@/lib/api/server'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
import { createApiResponse, getUserLimits } from '@/app/api/v1/logs/meta'
import {
checkRateLimit,
@@ -50,7 +51,7 @@ export const GET = withRouteHandler(
endedAt: workflowExecutionLogs.endedAt,
totalDurationMs: workflowExecutionLogs.totalDurationMs,
executionData: workflowExecutionLogs.executionData,
cost: workflowExecutionLogs.cost,
costTotal: workflowExecutionLogs.costTotal,
files: workflowExecutionLogs.files,
createdAt: workflowExecutionLogs.createdAt,
workflowName: workflow.name,
@@ -101,8 +102,15 @@ export const GET = withRouteHandler(
totalDurationMs: log.totalDurationMs,
files: log.files || undefined,
workflow: workflowSummary,
executionData: log.executionData as any,
cost: log.cost as any,
executionData: (await materializeExecutionData(
log.executionData as Record<string, unknown> | null,
{
workspaceId: log.workspaceId,
workflowId: log.workflowId,
executionId: log.executionId,
}
)) as any,
cost: log.costTotal != null ? { total: Number(log.costTotal) } : null,
createdAt: log.createdAt.toISOString(),
}
@@ -70,7 +70,9 @@ export const GET = withRouteHandler(
startedAt: workflowLog.startedAt.toISOString(),
endedAt: workflowLog.endedAt?.toISOString(),
totalDurationMs: workflowLog.totalDurationMs,
cost: workflowLog.cost || null,
// Sourced from the cost_total projection of the usage_log ledger
// (the deprecated cost jsonb column was dropped).
cost: workflowLog.costTotal != null ? { total: Number(workflowLog.costTotal) } : null,
},
}
+6 -5
View File
@@ -84,18 +84,19 @@ export function buildLogFilters(filters: LogFilters): SQL<unknown> {
conditions.push(lte(workflowExecutionLogs.totalDurationMs, filters.maxDurationMs))
}
// Cost filters
// Cost filters — indexed projection of the usage_log ledger (dollars).
if (filters.minCost !== undefined) {
conditions.push(sql`(${workflowExecutionLogs.cost}->>'total')::numeric >= ${filters.minCost}`)
conditions.push(sql`${workflowExecutionLogs.costTotal} >= ${filters.minCost}`)
}
if (filters.maxCost !== undefined) {
conditions.push(sql`(${workflowExecutionLogs.cost}->>'total')::numeric <= ${filters.maxCost}`)
conditions.push(sql`${workflowExecutionLogs.costTotal} <= ${filters.maxCost}`)
}
// Model filter
// Model filter — uses the models_used projection (includes zero-cost/BYOK
// models, which the usage_log ledger drops), preserving prior behavior.
if (filters.model) {
conditions.push(sql`${workflowExecutionLogs.cost}->>'models' ? ${filters.model}`)
conditions.push(sql`${workflowExecutionLogs.modelsUsed} @> ARRAY[${filters.model}]::text[]`)
}
// Combine all conditions with AND
+38 -18
View File
@@ -6,7 +6,9 @@ import { and, eq, sql } from 'drizzle-orm'
import { type NextRequest, NextResponse } from 'next/server'
import { v1ListLogsContract } from '@/lib/api/contracts/v1/logs'
import { getValidationErrorMessage, parseRequest } from '@/lib/api/server'
import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
import { buildLogFilters, getOrderBy } from '@/app/api/v1/logs/filters'
import { createApiResponse, getUserLimits } from '@/app/api/v1/logs/meta'
import {
@@ -103,6 +105,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
.select({
id: workflowExecutionLogs.id,
workflowId: workflowExecutionLogs.workflowId,
workspaceId: workflowExecutionLogs.workspaceId,
executionId: workflowExecutionLogs.executionId,
deploymentVersionId: workflowExecutionLogs.deploymentVersionId,
level: workflowExecutionLogs.level,
@@ -110,7 +113,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
startedAt: workflowExecutionLogs.startedAt,
endedAt: workflowExecutionLogs.endedAt,
totalDurationMs: workflowExecutionLogs.totalDurationMs,
cost: workflowExecutionLogs.cost,
costTotal: workflowExecutionLogs.costTotal,
files: workflowExecutionLogs.files,
executionData: params.details === 'full' ? workflowExecutionLogs.executionData : sql`null`,
workflowName: workflow.name,
@@ -144,7 +147,12 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
})
}
const formattedLogs = data.map((log) => {
// Only materialize externalized execution data when the response actually
// needs it (details=full + finalOutput/traceSpans requested).
const needsMaterialize =
params.details === 'full' && (params.includeFinalOutput || params.includeTraceSpans)
const buildBase = (log: (typeof data)[number]) => {
const result: any = {
id: log.id,
workflowId: log.workflowId,
@@ -155,7 +163,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
startedAt: log.startedAt.toISOString(),
endedAt: log.endedAt?.toISOString() || null,
totalDurationMs: log.totalDurationMs,
cost: log.cost ? { total: (log.cost as any).total } : null,
cost: log.costTotal != null ? { total: Number(log.costTotal) } : null,
files: log.files || null,
}
@@ -166,24 +174,36 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
description: log.workflowDescription,
deleted: !log.workflowName,
}
if (log.cost) {
result.cost = log.cost
}
if (log.executionData) {
const execData = log.executionData as any
if (params.includeFinalOutput && execData.finalOutput) {
result.finalOutput = execData.finalOutput
}
if (params.includeTraceSpans && execData.traceSpans) {
result.traceSpans = execData.traceSpans
}
}
}
return result
})
}
// Only run the bounded-concurrency materialization when the response actually
// needs object-storage reads; otherwise a plain synchronous map avoids the
// per-row worker/promise overhead.
const formattedLogs = needsMaterialize
? await mapWithConcurrency(data, MATERIALIZE_CONCURRENCY, async (log) => {
const result = buildBase(log)
if (log.executionData) {
const execData = (await materializeExecutionData(
log.executionData as Record<string, unknown> | null,
{
workspaceId: log.workspaceId,
workflowId: log.workflowId,
executionId: log.executionId,
}
)) as any
if (params.includeFinalOutput && execData.finalOutput) {
result.finalOutput = execData.finalOutput
}
if (params.includeTraceSpans && execData.traceSpans) {
result.traceSpans = execData.traceSpans
}
}
return result
})
: data.map(buildBase)
const limits = await getUserLimits(userId)
@@ -9,6 +9,7 @@ import {
} from '@/lib/api/contracts/workflows'
import { parseRequest } from '@/lib/api/server'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
import { validateWorkflowAccess } from '@/app/api/workflows/middleware'
import type { PausePoint } from '@/executor/types'
@@ -117,6 +118,7 @@ export const GET = withRouteHandler(
.select({
executionId: workflowExecutionLogs.executionId,
workflowId: workflowExecutionLogs.workflowId,
workspaceId: workflowExecutionLogs.workspaceId,
status: workflowExecutionLogs.status,
level: workflowExecutionLogs.level,
trigger: workflowExecutionLogs.trigger,
@@ -124,7 +126,7 @@ export const GET = withRouteHandler(
endedAt: workflowExecutionLogs.endedAt,
totalDurationMs: workflowExecutionLogs.totalDurationMs,
executionData: workflowExecutionLogs.executionData,
cost: workflowExecutionLogs.cost,
costTotal: workflowExecutionLogs.costTotal,
})
.from(workflowExecutionLogs)
.where(
@@ -177,13 +179,20 @@ export const GET = withRouteHandler(
}
}
const cost = logRow.cost
? { total: Number((logRow.cost as { total?: number }).total ?? 0) }
: null
const cost = logRow.costTotal != null ? { total: Number(logRow.costTotal) } : null
const error = status === 'failed' ? extractError(logRow.executionData) : null
// Heavy execution data may live in object storage; resolve the pointer
// before reading error / finalOutput / traceSpans (no-op for inline rows).
const executionData = (await materializeExecutionData(
logRow.executionData as Record<string, unknown> | null,
{
workspaceId: logRow.workspaceId,
workflowId: logRow.workflowId,
executionId: logRow.executionId,
}
)) as ExecutionDataShape | undefined
const executionData = logRow.executionData as ExecutionDataShape | undefined
const error = status === 'failed' ? extractError(executionData) : null
const finalOutput =
includeOutput && status === 'completed' && executionData
@@ -55,6 +55,13 @@ const MIN_BAR_PCT = 0.5
interface TraceViewProps {
traceSpans: TraceSpan[]
/**
* Authoritative, multiplier-inclusive run cost (dollars) from the persisted
* execution log. When provided it drives the header credit chip so the Trace
* tab and the Overview cost breakdown can never show different totals. Falls
* back to the root span's own cost only when absent (e.g. live previews).
*/
runCostDollars?: number
}
interface FlatSpanEntry {
@@ -708,8 +715,10 @@ const TraceDetailPane = memo(function TraceDetailPane({ span }: { span: TraceSpa
if (cacheRead) metaEntries.push({ label: 'Cache read', value: cacheRead })
if (cacheWrite) metaEntries.push({ label: 'Cache write', value: cacheWrite })
if (reasoning) metaEntries.push({ label: 'Reasoning tokens', value: reasoning })
const costTotal = formatCostAmount(span.cost?.total)
if (costTotal) metaEntries.push({ label: 'Cost', value: costTotal })
// Per-span cost is intentionally not shown: cost lives only in the usage_log
// ledger (the authoritative, multiplier-inclusive run total drives the header
// chip). Persisted spans are cost-stripped, so a per-span row would render on
// live runs but vanish on reload — show one consistent total instead.
if (span.errorType) metaEntries.push({ label: 'Error type', value: span.errorType })
if (span.iterationIndex !== undefined)
metaEntries.push({ label: 'Iteration', value: String(span.iterationIndex + 1) })
@@ -812,7 +821,7 @@ const TraceDetailPane = memo(function TraceDetailPane({ span }: { span: TraceSpa
* in a way that mirrors the executor's internal structure so investigators can
* follow block-by-block and segment-by-segment what happened and why.
*/
export const TraceView = memo(function TraceView({ traceSpans }: TraceViewProps) {
export const TraceView = memo(function TraceView({ traceSpans, runCostDollars }: TraceViewProps) {
const treeRef = useRef<HTMLDivElement>(null)
const [searchQuery, setSearchQuery] = useState('')
const [treePaneWidth, setTreePaneWidth] = useState(DEFAULT_TREE_PANE_WIDTH)
@@ -1021,7 +1030,7 @@ export const TraceView = memo(function TraceView({ traceSpans }: TraceViewProps)
{blockCount} {blockCount === 1 ? 'span' : 'spans'}
</span>
{(() => {
const rootCost = formatCostAmount(normalizedSpans[0]?.cost?.total)
const rootCost = formatCostAmount(runCostDollars ?? normalizedSpans[0]?.cost?.total)
return rootCost ? (
<span className='flex-shrink-0 font-medium text-[var(--text-tertiary)] text-caption tabular-nums'>
{rootCost}
@@ -24,6 +24,7 @@ import {
} from '@/components/emcn'
import type { WorkflowLogRow } from '@/lib/api/contracts/logs'
import { BASE_EXECUTION_CHARGE } from '@/lib/billing/constants'
import { apportionCredits, dollarsToCredits } from '@/lib/billing/credits/conversion'
import { cn } from '@/lib/core/utils/cn'
import { handleKeyboardActivation } from '@/lib/core/utils/keyboard'
import { filterHiddenOutputKeys } from '@/lib/logs/execution/trace-spans/trace-spans'
@@ -49,6 +50,16 @@ import { formatCost } from '@/providers/utils'
import { useLogDetailsUIStore } from '@/stores/logs/store'
import { MAX_LOG_DETAILS_WIDTH_RATIO, MIN_LOG_DETAILS_WIDTH } from '@/stores/logs/utils'
/**
* Renders an already-apportioned integer credit value. `dollars` is only used
* to distinguish a genuine zero ("0 credits") from a sub-credit charge that
* rounded down to zero ("<1 credit"); the credit figure itself is authoritative.
*/
function creditLabel(credits: number, dollars: number): string {
if (credits <= 0) return dollars > 0 ? '<1 credit' : '0 credits'
return `${credits.toLocaleString()} ${credits === 1 ? 'credit' : 'credits'}`
}
export const WorkflowOutputSection = memo(
function WorkflowOutputSection({ output }: { output: Record<string, unknown> }) {
const contentRef = useRef<HTMLDivElement>(null)
@@ -326,6 +337,54 @@ export function LogDetailsContent({ log, onActiveTabChange }: LogDetailsContentP
return { input: raw } as Record<string, unknown>
}, [log.executionData])
// Cost breakdown, sourced solely from the usage_log ledger (single source of
// truth). Line items (Base Run / per-model / per-integration) get integer
// credits apportioned with a single round at the total so rows always
// reconcile (never round-then-sum, which drifts). Pre-ledger runs that only
// have the cost_total projection show the total alone — no itemization, no
// parallel jsonb reconstruction.
const costBreakdown = useMemo((): {
rows: Array<{ key: string; label: string; credits: number; dollars: number }>
totalCredits: number
totalDollars: number
tokens: { input: number; output: number }
} | null => {
const ledger = log.costLedger
if (ledger && ledger.items.length > 0) {
const credits = apportionCredits(
ledger.items.map((item, i) => ({ key: String(i), dollars: item.cost }))
)
const rows = ledger.items.map((item, i) => ({
key: String(i),
label:
item.category === 'fixed' && item.description === 'execution_fee'
? 'Base Run'
: item.description,
credits: credits[String(i)] ?? 0,
dollars: item.cost,
}))
return {
rows,
totalCredits: dollarsToCredits(ledger.total),
totalDollars: ledger.total,
tokens: {
input: ledger.items.reduce((s, it) => s + (it.inputTokens ?? 0), 0),
output: ledger.items.reduce((s, it) => s + (it.outputTokens ?? 0), 0),
},
}
}
// Total-only (pre-ledger runs with just the cost_total projection).
const total = log.cost?.total
if (total == null) return null
return {
rows: [],
totalCredits: dollarsToCredits(total),
totalDollars: total,
tokens: { input: 0, output: 0 },
}
}, [log.costLedger, log.cost])
const formattedTimestamp = formatDate(log.createdAt)
const logStatus = getDisplayStatus(log.status)
@@ -529,67 +588,39 @@ export function LogDetailsContent({ log, onActiveTabChange }: LogDetailsContentP
{log.files && log.files.length > 0 && <FileCards files={log.files} isExecutionFile />}
{/* Cost Breakdown */}
{hasCostInfo && (
{hasCostInfo && costBreakdown && (
<div className='divide-y divide-[var(--border)] overflow-hidden rounded-md border border-[var(--border)] bg-[var(--surface-2)] dark:bg-transparent'>
<div className='flex h-10 items-center justify-between px-3 transition-colors hover-hover:bg-[var(--surface-2)]'>
<span className='font-medium text-[var(--text-tertiary)] text-caption'>
Base Run
</span>
<span className='font-medium text-[var(--text-secondary)] text-caption tabular-nums'>
{formatCost(BASE_EXECUTION_CHARGE)}
</span>
</div>
<div className='flex h-10 items-center justify-between px-3 transition-colors hover-hover:bg-[var(--surface-2)]'>
<span className='font-medium text-[var(--text-tertiary)] text-caption'>
Model Input
</span>
<span className='font-medium text-[var(--text-secondary)] text-caption tabular-nums'>
{formatCost(log.cost?.input || 0)}
</span>
</div>
<div className='flex h-10 items-center justify-between px-3 transition-colors hover-hover:bg-[var(--surface-2)]'>
<span className='font-medium text-[var(--text-tertiary)] text-caption'>
Model Output
</span>
<span className='font-medium text-[var(--text-secondary)] text-caption tabular-nums'>
{formatCost(log.cost?.output || 0)}
</span>
</div>
{(() => {
const models = (log.cost as Record<string, unknown>)?.models as
| Record<string, { toolCost?: number }>
| undefined
const totalToolCost = models
? Object.values(models).reduce((sum, m) => sum + (m?.toolCost || 0), 0)
: 0
return totalToolCost > 0 ? (
<div className='flex h-10 items-center justify-between px-3 transition-colors hover-hover:bg-[var(--surface-2)]'>
<span className='font-medium text-[var(--text-tertiary)] text-caption'>
Tool Usage
</span>
<span className='font-medium text-[var(--text-secondary)] text-caption tabular-nums'>
{formatCost(totalToolCost)}
</span>
</div>
) : null
})()}
{costBreakdown.rows.map((row) => (
<div
key={row.key}
className='flex h-10 items-center justify-between px-3 transition-colors hover-hover:bg-[var(--surface-2)]'
>
<span className='min-w-0 truncate font-medium text-[var(--text-tertiary)] text-caption'>
{row.label}
</span>
<span className='flex-shrink-0 font-medium text-[var(--text-secondary)] text-caption tabular-nums'>
{creditLabel(row.credits, row.dollars)}
</span>
</div>
))}
<div className='flex h-10 items-center justify-between px-3 transition-colors hover-hover:bg-[var(--surface-2)]'>
<span className='font-medium text-[var(--text-secondary)] text-caption'>
Total
</span>
<span className='font-semibold text-[var(--text-primary)] text-caption tabular-nums'>
{formatCost(log.cost?.total || 0)}
</span>
</div>
<div className='flex h-10 items-center justify-between px-3 transition-colors hover-hover:bg-[var(--surface-2)]'>
<span className='font-medium text-[var(--text-tertiary)] text-caption'>
Tokens
</span>
<span className='font-medium text-[var(--text-secondary)] text-caption tabular-nums'>
{log.cost?.tokens?.input || log.cost?.tokens?.prompt || 0} in ·{' '}
{log.cost?.tokens?.output || log.cost?.tokens?.completion || 0} out
{creditLabel(costBreakdown.totalCredits, costBreakdown.totalDollars)}
</span>
</div>
{(costBreakdown.tokens.input > 0 || costBreakdown.tokens.output > 0) && (
<div className='flex h-10 items-center justify-between px-3 transition-colors hover-hover:bg-[var(--surface-2)]'>
<span className='font-medium text-[var(--text-tertiary)] text-caption'>
Tokens
</span>
<span className='font-medium text-[var(--text-secondary)] text-caption tabular-nums'>
{costBreakdown.tokens.input} in · {costBreakdown.tokens.output} out
</span>
</div>
)}
<div className='px-3 py-2'>
<p className='font-medium text-[var(--text-tertiary)] text-xs'>
Total includes a {formatCost(BASE_EXECUTION_CHARGE)} base charge plus model and
@@ -608,7 +639,7 @@ export function LogDetailsContent({ log, onActiveTabChange }: LogDetailsContentP
className='mt-3 min-h-0 flex-1 overflow-hidden focus-visible:outline-none'
>
{traceSpans?.length ? (
<TraceView traceSpans={traceSpans} />
<TraceView traceSpans={traceSpans} runCostDollars={log.cost?.total} />
) : log.executionData ? (
<div className='flex h-full items-center justify-center px-4 text-center'>
<span className='font-medium text-[var(--text-tertiary)] text-sm'>
@@ -1,6 +1,6 @@
import type React from 'react'
import { AgentSkillsIcon, WorkflowIcon } from '@/components/icons'
import { dollarsToCredits } from '@/lib/billing/credits/conversion'
import { formatCreditCost } from '@/lib/billing/credits/conversion'
import type { TraceSpan } from '@/lib/logs/types'
import { LoopTool } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/subflows/loop/loop-config'
import { ParallelTool } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/subflows/parallel/parallel-config'
@@ -117,10 +117,7 @@ export function getDisplayName(span: TraceSpan): string {
}
export function formatCostAmount(value: number | undefined): string | undefined {
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return undefined
const credits = dollarsToCredits(value)
if (credits <= 0) return '<1 credit'
return `${credits.toLocaleString('en-US')} ${credits === 1 ? 'credit' : 'credits'}`
return formatCreditCost(value, { emptyForZeroOrLess: true })
}
export function formatTokensSummary(tokens: TraceSpan['tokens']): string | undefined {
@@ -26,6 +26,7 @@ import { RateLimiter } from '@/lib/core/rate-limiter'
import { decryptSecret } from '@/lib/core/security/encryption'
import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
import { getBaseUrl } from '@/lib/core/utils/urls'
import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
import type { TraceSpan, WorkflowExecutionLog } from '@/lib/logs/types'
import { sendEmail } from '@/lib/messaging/email/mailer'
import type { AlertConfig } from '@/lib/notifications/alert-rules'
@@ -507,27 +508,6 @@ function formatLogDate(value: Date | string | null | undefined, fallback = ''):
return typeof value === 'string' ? value : fallback
}
function normalizeLogCost(value: unknown): WorkflowExecutionLog['cost'] {
if (!isRecord(value)) {
return undefined
}
const tokens = isRecord(value.tokens)
? {
input: typeof value.tokens.input === 'number' ? value.tokens.input : undefined,
output: typeof value.tokens.output === 'number' ? value.tokens.output : undefined,
total: typeof value.tokens.total === 'number' ? value.tokens.total : undefined,
}
: undefined
return {
input: typeof value.input === 'number' ? value.input : undefined,
output: typeof value.output === 'number' ? value.output : undefined,
total: typeof value.total === 'number' ? value.total : undefined,
tokens,
}
}
function normalizeLogFiles(value: unknown): WorkflowExecutionLog['files'] {
if (!Array.isArray(value)) {
return undefined
@@ -545,11 +525,18 @@ function normalizeLogFiles(value: unknown): WorkflowExecutionLog['files'] {
)
}
function normalizeWorkflowExecutionLog(
async function normalizeWorkflowExecutionLog(
row: typeof workflowExecutionLogs.$inferSelect
): WorkflowExecutionLog {
): Promise<WorkflowExecutionLog> {
const startedAt = formatLogDate(row.startedAt)
// Heavy execution data may live in object storage; resolve the pointer so
// retry deliveries get finalOutput/traceSpans (no-op for inline rows).
const executionData = await materializeExecutionData(
isRecord(row.executionData) ? row.executionData : {},
{ workspaceId: row.workspaceId, workflowId: row.workflowId, executionId: row.executionId }
)
return {
id: row.id,
workflowId: row.workflowId,
@@ -561,8 +548,9 @@ function normalizeWorkflowExecutionLog(
endedAt: formatLogDate(row.endedAt, startedAt),
totalDurationMs: row.totalDurationMs ?? 0,
files: normalizeLogFiles(row.files),
executionData: isRecord(row.executionData) ? row.executionData : {},
cost: normalizeLogCost(row.cost),
executionData: executionData as WorkflowExecutionLog['executionData'],
// cost_total projection of the usage_log ledger (not the deprecated jsonb).
cost: row.costTotal != null ? { total: Number(row.costTotal) } : undefined,
createdAt: formatLogDate(row.createdAt, startedAt),
}
}
@@ -580,7 +568,7 @@ async function buildRetryLog(params: NotificationDeliveryParams): Promise<Workfl
.limit(1)
if (storedLog) {
return normalizeWorkflowExecutionLog(storedLog)
return await normalizeWorkflowExecutionLog(storedLog)
}
const now = new Date().toISOString()
+29 -2
View File
@@ -104,6 +104,27 @@ const costSummarySchema = z
})
.partial()
/**
* Itemized cost breakdown derived from the usage_log ledger (the single source
* of truth) for the detail view. Each item is one billed line (base fee, a
* model, or a tool/integration); the items reconcile to `total`.
*/
const costLedgerItemSchema = z.object({
category: z.enum(['fixed', 'model', 'tool']),
description: z.string(),
cost: z.number(),
inputTokens: z.number().optional(),
outputTokens: z.number().optional(),
})
export const costLedgerSchema = z.object({
total: z.number(),
items: z.array(costLedgerItemSchema),
})
export type CostLedger = z.output<typeof costLedgerSchema>
export type CostLedgerItem = z.output<typeof costLedgerItemSchema>
const pauseSummarySchema = z.object({
status: z.string().nullable(),
total: z.number(),
@@ -218,7 +239,10 @@ export const workflowLogSummarySchema = z.object({
createdAt: z.string(),
workflow: workflowSummarySchema.nullable(),
jobTitle: z.string().nullable(),
cost: costSummarySchema.nullable(),
// Top-level run cost is the cost_total projection of the usage_log ledger,
// rendered as { total } (dollars). The itemized breakdown lives in costLedger
// (detail only); per-block costs use the richer costSummarySchema elsewhere.
cost: z.object({ total: z.number() }).nullable(),
pauseSummary: pauseSummarySchema,
hasPendingPause: z.boolean(),
})
@@ -226,6 +250,9 @@ export const workflowLogSummarySchema = z.object({
export const workflowLogDetailSchema = workflowLogSummarySchema.extend({
executionData: executionDataDetailSchema,
files: z.array(userFileSchema).nullable(),
// Itemized, ledger-sourced cost breakdown. Null for legacy/pre-ledger runs,
// where the UI falls back to the (reconciling) cost jsonb.
costLedger: costLedgerSchema.nullable().optional(),
})
export type WorkflowLogSummary = z.output<typeof workflowLogSummarySchema>
@@ -236,7 +263,7 @@ export type WorkflowLogDetail = z.output<typeof workflowLogDetailSchema>
* UI surfaces that render the same log before and after its detail query resolves.
*/
export type WorkflowLogRow = WorkflowLogSummary &
Partial<Pick<WorkflowLogDetail, 'executionData' | 'files'>>
Partial<Pick<WorkflowLogDetail, 'executionData' | 'files' | 'costLedger'>>
export const listLogsResponseSchema = z.object({
data: z.array(workflowLogSummarySchema),
@@ -103,7 +103,6 @@ export const organizationBillingMemberSchema = z
userName: z.string().nullable().optional(),
userEmail: z.string().nullable().optional(),
joinedAt: z.string().nullable().optional(),
lastActive: z.string().nullable().optional(),
})
.passthrough()
@@ -43,7 +43,6 @@ export const adminV1MemberSchema = z.object({
export const adminV1MemberDetailSchema = adminV1MemberSchema.extend({
currentPeriodCost: z.string(),
currentUsageLimit: z.string().nullable(),
lastActive: z.string().nullable(),
billingBlocked: z.boolean(),
})
@@ -75,18 +74,6 @@ export const adminV1SeatAnalyticsSchema = z.object({
subscriptionPlan: z.string(),
canAddSeats: z.boolean(),
utilizationRate: z.number(),
activeMembers: z.number(),
inactiveMembers: z.number(),
memberActivity: z.array(
z.object({
userId: z.string(),
userName: z.string(),
userEmail: z.string(),
role: z.string(),
joinedAt: z.string(),
lastActive: z.string().nullable(),
})
),
})
export const adminV1CreateOrganizationBodySchema = z.object({
@@ -23,27 +23,14 @@ export const adminV1UserBillingSchema = z.object({
userName: z.string(),
userEmail: z.string(),
stripeCustomerId: z.string().nullable(),
totalManualExecutions: z.number(),
totalApiCalls: z.number(),
totalWebhookTriggers: z.number(),
totalScheduledExecutions: z.number(),
totalChatExecutions: z.number(),
totalMcpExecutions: z.number(),
totalA2aExecutions: z.number(),
totalTokensUsed: z.number(),
totalCost: z.string(),
currentUsageLimit: z.string().nullable(),
currentPeriodCost: z.string(),
lastPeriodCost: z.string().nullable(),
billedOverageThisPeriod: z.string(),
storageUsedBytes: z.number(),
lastActive: z.string().nullable(),
billingBlocked: z.boolean(),
totalCopilotCost: z.string(),
currentPeriodCopilotCost: z.string(),
lastPeriodCopilotCost: z.string().nullable(),
totalCopilotTokens: z.number(),
totalCopilotCalls: z.number(),
})
export const adminV1UserBillingWithSubscriptionSchema = adminV1UserBillingSchema.extend({
+8 -1
View File
@@ -27,7 +27,14 @@ export const v1ListLogsQuerySchema = z.object({
details: z.enum(['basic', 'full']).optional().default('basic'),
includeTraceSpans: booleanQueryFlagSchema.optional().default(false),
includeFinalOutput: booleanQueryFlagSchema.optional().default(false),
limit: z.coerce.number().optional().default(100),
// Clamp rather than reject: this limit was previously unbounded, so a hard
// .max() would 400 existing consumers passing a larger value. Page size is
// still capped to bound response size and materialization reads.
limit: z.coerce
.number()
.optional()
.default(100)
.transform((v) => Math.min(Math.max(1, Math.trunc(v)), 1000)),
cursor: z.string().optional(),
order: z.enum(['desc', 'asc']).optional().default('desc'),
})
+42 -5
View File
@@ -7,7 +7,7 @@ import {
type SubscriptionMetadata,
} from '@/lib/billing/core/subscription'
import { getOrgUsageLimit, getUserUsageData } from '@/lib/billing/core/usage'
import { getBillingPeriodUsageCost } from '@/lib/billing/core/usage-log'
import { COPILOT_USAGE_SOURCES, getBillingPeriodUsageCost } from '@/lib/billing/core/usage-log'
import { getCreditBalance } from '@/lib/billing/credits/balance'
import {
computeDailyRefreshConsumed,
@@ -128,6 +128,8 @@ async function aggregateOrgMemberStats(organizationId: string): Promise<{
.where(eq(member.organizationId, organizationId))
let currentPeriodCost = new Decimal(0)
// Copilot baseline (copilot source). All copilot-family usage (incl. MCP) lives
// in usage_log and is added via the copilot ledger by callers — not a baseline.
let currentPeriodCopilotCost = new Decimal(0)
let lastPeriodCopilotCost = new Decimal(0)
const memberIds: string[] = []
@@ -454,20 +456,34 @@ export async function getSimplifiedBillingSummary(
const pooled = await aggregateOrgMemberStats(organizationId)
const rawCurrentUsage = pooled.currentPeriodCost
const totalCopilotCost = pooled.currentPeriodCopilotCost
const totalLastPeriodCopilotCost = pooled.lastPeriodCopilotCost
// Deduct daily-refresh credits against this specific org's pool.
// `usageData` is derived from the caller's priority subscription
// and may not match the requested org (multi-org admins, personal
// priority sub, etc.), so it cannot be reused here.
const ledgerUsage =
const orgBillingPeriod =
subscription.periodStart && subscription.periodEnd
? { start: subscription.periodStart, end: subscription.periodEnd }
: null
const ledgerUsage = orgBillingPeriod
? await getBillingPeriodUsageCost(
{ type: 'organization', id: organizationId },
orgBillingPeriod
)
: 0
// Copilot breakdown = member baselines (copilot + MCP) + the copilot-family
// ledger for the period (COPILOT_USAGE_SOURCES: copilot/workspace-chat/
// mcp_copilot/mothership_block); the baseline columns are no longer incremented.
const totalCopilotCost =
pooled.currentPeriodCopilotCost +
(orgBillingPeriod
? await getBillingPeriodUsageCost(
{ type: 'organization', id: organizationId },
{ start: subscription.periodStart, end: subscription.periodEnd }
orgBillingPeriod,
COPILOT_USAGE_SOURCES
)
: 0
: 0)
let refreshDeduction = 0
if (isPaid(plan) && subscription.periodStart) {
const planDollars = getPlanTierDollars(plan)
@@ -561,6 +577,8 @@ export async function getSimplifiedBillingSummary(
.where(eq(userStats.userId, userId))
.limit(1)
// Copilot baseline (copilot source). MCP copilot usage lives in usage_log and
// is added via the copilot ledger below, not a userStats baseline.
const copilotCost =
userStatsRows.length > 0 ? toNumber(toDecimal(userStatsRows[0].currentPeriodCopilotCost)) : 0
@@ -576,6 +594,25 @@ export async function getSimplifiedBillingSummary(
totalLastPeriodCopilotCost = pooled.lastPeriodCopilotCost
}
// Add the copilot-family ledger (COPILOT_USAGE_SOURCES: copilot/workspace-chat/
// mcp_copilot/mothership_block) on top of the baseline; those columns are no
// longer incremented per usage.
const copilotBillingPeriod =
usageData.billingPeriodStart && usageData.billingPeriodEnd
? { start: usageData.billingPeriodStart, end: usageData.billingPeriodEnd }
: null
if (copilotBillingPeriod) {
const copilotEntity =
orgScoped && subscription?.referenceId
? ({ type: 'organization', id: subscription.referenceId } as const)
: ({ type: 'user', id: userId } as const)
totalCopilotCost += await getBillingPeriodUsageCost(
copilotEntity,
copilotBillingPeriod,
COPILOT_USAGE_SOURCES
)
}
const percentUsed = usageData.limit > 0 ? (currentUsage / usageData.limit) * 100 : 0
const daysRemaining = usageData.billingPeriodEnd
+52 -10
View File
@@ -4,7 +4,10 @@ import { createLogger } from '@sim/logger'
import { and, count, eq, gt, ne } from 'drizzle-orm'
import { isOrganizationBillingBlocked } from '@/lib/billing/core/access'
import { getOrganizationSubscription, getPlanPricing } from '@/lib/billing/core/billing'
import { getBillingPeriodUsageCost } from '@/lib/billing/core/usage-log'
import {
getBillingPeriodUsageCost,
getBillingPeriodUsageCostByUser,
} from '@/lib/billing/core/usage-log'
import {
computeDailyRefreshConsumed,
getOrgMemberRefreshBounds,
@@ -50,7 +53,32 @@ interface MemberUsageData {
isOverLimit: boolean
role: string
joinedAt: Date
lastActive: Date | null
}
/**
* Per-member usage_log cost for an org's current billing period, keyed by userId.
* `currentPeriodCost` is only a baseline (no longer incremented on the hot path),
* so callers add this ledger component to it for each member's real current-period
* usage. Pass `period` to reuse an already-fetched subscription window; omit it to
* look up the org's subscription here. Returns an empty map when there's no period.
*/
export async function getOrgMemberLedgerByUser(
organizationId: string,
period?: { start: Date; end: Date } | null
): Promise<Map<string, number>> {
let billingPeriod = period ?? null
if (period === undefined) {
const subscription = await getOrganizationSubscription(organizationId)
billingPeriod =
subscription?.periodStart && subscription?.periodEnd
? { start: subscription.periodStart, end: subscription.periodEnd }
: null
}
if (!billingPeriod) return new Map<string, number>()
return getBillingPeriodUsageCostByUser(
{ type: 'organization', id: organizationId },
billingPeriod
)
}
/**
@@ -93,16 +121,25 @@ export async function getOrganizationBillingData(
// User stats fields
currentPeriodCost: userStats.currentPeriodCost,
currentUsageLimit: userStats.currentUsageLimit,
lastActive: userStats.lastActive,
})
.from(member)
.innerJoin(user, eq(member.userId, user.id))
.leftJoin(userStats, eq(member.userId, userStats.userId))
.where(eq(member.organizationId, organizationId))
// Per-member current-period usage = userStats baseline + attributed usage_log
// rows. currentPeriodCost is no longer incremented on the hot path, so the
// baseline alone under-reports; add each member's ledger sum for the period.
const billingPeriod =
subscription.periodStart && subscription.periodEnd
? { start: subscription.periodStart, end: subscription.periodEnd }
: null
const usageByUser = await getOrgMemberLedgerByUser(organizationId, billingPeriod)
// Process member data
const members: MemberUsageData[] = membersWithUsage.map((memberRecord) => {
const currentUsage = Number(memberRecord.currentPeriodCost || 0)
const currentUsage =
Number(memberRecord.currentPeriodCost || 0) + (usageByUser.get(memberRecord.userId) ?? 0)
const usageLimit = Number(memberRecord.currentUsageLimit || getFreeTierLimit())
const percentUsed = usageLimit > 0 ? (currentUsage / usageLimit) * 100 : 0
@@ -116,17 +153,22 @@ export async function getOrganizationBillingData(
isOverLimit: currentUsage > usageLimit,
role: memberRecord.role,
joinedAt: memberRecord.joinedAt,
lastActive: memberRecord.lastActive,
}
})
// Calculate aggregated statistics
let totalCurrentUsage = members.reduce((sum, m) => sum + m.currentUsage, 0)
if (subscription.periodStart && subscription.periodEnd) {
// Authoritative org total = member baselines + the org's full usage_log for
// the period (also captures rows from members no longer present). Computed
// from raw baselines, NOT members[].currentUsage — the latter already folds
// in per-member usage_log for display, so summing it AND adding the org
// ledger would double-count.
let totalCurrentUsage = membersWithUsage.reduce(
(sum, m) => sum + Number(m.currentPeriodCost || 0),
0
)
if (billingPeriod) {
totalCurrentUsage += await getBillingPeriodUsageCost(
{ type: 'organization', id: subscription.referenceId },
{ start: subscription.periodStart, end: subscription.periodEnd }
billingPeriod
)
}
+107 -47
View File
@@ -4,18 +4,18 @@ import { usageLog } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { generateId } from '@sim/utils/id'
import { and, desc, eq, gte, lte, sql } from 'drizzle-orm'
import { and, desc, eq, gte, inArray, lte, sql } from 'drizzle-orm'
import { defaultBillingPeriod } from '@/lib/billing/core/billing-period'
import { getHighestPrioritySubscription } from '@/lib/billing/core/plan'
import { isOrgScopedSubscription } from '@/lib/billing/subscriptions/utils'
import { isBillingEnabled } from '@/lib/core/config/feature-flags'
import type { DbOrTx } from '@/lib/db/types'
const logger = createLogger('UsageLog')
/**
* Usage log category types
*/
export type UsageLogCategory = 'model' | 'fixed'
export type UsageLogCategory = 'model' | 'fixed' | 'tool'
/**
* Usage log source types
@@ -31,6 +31,18 @@ export type UsageLogSource =
| 'voice-input'
| 'enrichment'
/**
* usage_log sources that make up the "copilot" cost breakdown shown in billing
* summaries: the copilot agent, mothership/workspace chat, MCP copilot, and
* mothership blocks. Mirrors the source set billed via /api/billing/update-cost.
*/
export const COPILOT_USAGE_SOURCES: UsageLogSource[] = [
'copilot',
'workspace-chat',
'mcp_copilot',
'mothership_block',
]
/**
* Metadata for 'model' category charges
*/
@@ -47,7 +59,7 @@ export type UsageLogMetadata = ModelUsageMetadata | Record<string, unknown> | nu
export type BillingEntityType = 'user' | 'organization'
interface BillingEntity {
export interface BillingEntity {
type: BillingEntityType
id: string
}
@@ -84,9 +96,16 @@ export interface RecordUsageParams {
billingEntity?: BillingEntity
/** Billing period bounds, resolved by caller when already known. */
billingPeriod?: { start: Date; end: Date }
/**
* Optional transaction to run the ledger INSERT in. Callers that reconcile a
* read-then-insert under a lock (e.g. the per-execution advisory lock in the
* workflow completion path) pass their tx so the insert participates in the
* same locked transaction. Defaults to the pooled db.
*/
tx?: DbOrTx
}
function stableEventKey(parts: Record<string, unknown>): string {
export function stableEventKey(parts: Record<string, unknown>): string {
const payload = Object.keys(parts)
.sort()
.map((key) => `${key}:${String(parts[key] ?? '')}`)
@@ -94,32 +113,53 @@ function stableEventKey(parts: Record<string, unknown>): string {
return createHash('sha256').update(payload).digest('hex')
}
type ResolvedSubscription = Awaited<ReturnType<typeof getHighestPrioritySubscription>>
export interface BillingContext {
billingEntity: BillingEntity
billingPeriod: { start: Date; end: Date }
}
/**
* Derive the billing entity + period from an ALREADY-resolved subscription.
* Callers that already hold the subscription (e.g. the workflow completion path,
* which fetches it for usage-threshold emails) can derive the context once and
* pass it into recordUsage so resolveBillingContext skips a redundant lookup.
* This is the single source of the entity/period derivation — keep it the only
* place that maps a subscription to a billing context.
*/
export function deriveBillingContext(
userId: string,
subscription: ResolvedSubscription
): BillingContext {
const billingEntity: BillingEntity =
subscription && isOrgScopedSubscription(subscription, userId)
? { type: 'organization', id: subscription.referenceId }
: { type: 'user', id: userId }
const billingPeriod =
subscription?.periodStart && subscription.periodEnd
? { start: subscription.periodStart, end: subscription.periodEnd }
: defaultBillingPeriod()
return { billingEntity, billingPeriod }
}
async function resolveBillingContext(
userId: string,
billingEntity?: BillingEntity,
billingPeriod?: { start: Date; end: Date }
): Promise<{
billingEntity: BillingEntity
billingPeriod: { start: Date; end: Date }
}> {
): Promise<BillingContext> {
if (billingEntity && billingPeriod) {
return { billingEntity, billingPeriod }
}
const subscription = await getHighestPrioritySubscription(userId)
const resolvedEntity =
billingEntity ??
(subscription && isOrgScopedSubscription(subscription, userId)
? { type: 'organization' as const, id: subscription.referenceId }
: { type: 'user' as const, id: userId })
const resolvedPeriod =
billingPeriod ??
(subscription?.periodStart && subscription.periodEnd
? { start: subscription.periodStart, end: subscription.periodEnd }
: defaultBillingPeriod())
return { billingEntity: resolvedEntity, billingPeriod: resolvedPeriod }
const derived = deriveBillingContext(userId, subscription)
return {
billingEntity: billingEntity ?? derived.billingEntity,
billingPeriod: billingPeriod ?? derived.billingPeriod,
}
}
/**
@@ -128,43 +168,55 @@ async function resolveBillingContext(
*/
export async function getBillingPeriodUsageCost(
billingEntity: BillingEntity,
billingPeriod: { start: Date; end: Date }
billingPeriod: { start: Date; end: Date },
source?: UsageLogSource | UsageLogSource[]
): Promise<number> {
const conditions = [
eq(usageLog.billingEntityType, billingEntity.type),
eq(usageLog.billingEntityId, billingEntity.id),
eq(usageLog.billingPeriodStart, billingPeriod.start),
eq(usageLog.billingPeriodEnd, billingPeriod.end),
]
if (source) {
conditions.push(
Array.isArray(source) ? inArray(usageLog.source, source) : eq(usageLog.source, source)
)
}
const [row] = await db
.select({
cost: sql<string>`COALESCE(SUM(${usageLog.cost}), 0)`,
})
.from(usageLog)
.where(
and(
eq(usageLog.billingEntityType, billingEntity.type),
eq(usageLog.billingEntityId, billingEntity.id),
eq(usageLog.billingPeriodStart, billingPeriod.start),
eq(usageLog.billingPeriodEnd, billingPeriod.end)
)
)
.where(and(...conditions))
return Number.parseFloat(row?.cost ?? '0')
}
export async function getBillingPeriodUsageCostByUser(
billingEntity: BillingEntity,
billingPeriod: { start: Date; end: Date }
billingPeriod: { start: Date; end: Date },
source?: UsageLogSource | UsageLogSource[]
): Promise<Map<string, number>> {
const conditions = [
eq(usageLog.billingEntityType, billingEntity.type),
eq(usageLog.billingEntityId, billingEntity.id),
eq(usageLog.billingPeriodStart, billingPeriod.start),
eq(usageLog.billingPeriodEnd, billingPeriod.end),
]
if (source) {
conditions.push(
Array.isArray(source) ? inArray(usageLog.source, source) : eq(usageLog.source, source)
)
}
const rows = await db
.select({
userId: usageLog.userId,
cost: sql<string>`COALESCE(SUM(${usageLog.cost}), 0)`,
})
.from(usageLog)
.where(
and(
eq(usageLog.billingEntityType, billingEntity.type),
eq(usageLog.billingEntityId, billingEntity.id),
eq(usageLog.billingPeriodStart, billingPeriod.start),
eq(usageLog.billingPeriodEnd, billingPeriod.end)
)
)
.where(and(...conditions))
.groupBy(usageLog.userId)
return new Map(rows.map((row) => [row.userId, Number.parseFloat(row.cost ?? '0')]))
@@ -178,12 +230,20 @@ export async function getBillingPeriodUsageCostByUser(
* but usage writes no longer contend on the user_stats row.
*/
export async function recordUsage(params: RecordUsageParams): Promise<void> {
if (!isBillingEnabled) {
return
}
const { userId, entries, workspaceId, workflowId, executionId, billingEntity, billingPeriod } =
params
// The usage ledger is written regardless of BILLING_ENABLED so it is the
// single, universal source of truth for cost (including self-hosted, where
// it powers the logs-page cost display). Billing *enforcement* (Stripe /
// overage) is gated separately by callers, not here.
const {
userId,
entries,
workspaceId,
workflowId,
executionId,
billingEntity,
billingPeriod,
tx,
} = params
const validEntries = entries.filter((e) => e.cost > 0)
@@ -193,7 +253,7 @@ export async function recordUsage(params: RecordUsageParams): Promise<void> {
const context = await resolveBillingContext(userId, billingEntity, billingPeriod)
const insertedRows = await db
const insertedRows = await (tx ?? db)
.insert(usageLog)
.values(
validEntries.map((entry, index) => {
-45
View File
@@ -607,51 +607,6 @@ export async function syncUsageLimitsFromSubscription(userId: string): Promise<v
// Keep higher custom limits unchanged
}
/**
* Get usage limit information for team members (for admin dashboard)
*/
export async function getTeamUsageLimits(organizationId: string): Promise<
Array<{
userId: string
userName: string
userEmail: string
currentLimit: number
currentUsage: number
totalCost: number
lastActive: Date | null
}>
> {
try {
const teamMembers = await db
.select({
userId: member.userId,
userName: user.name,
userEmail: user.email,
currentLimit: userStats.currentUsageLimit,
currentPeriodCost: userStats.currentPeriodCost,
totalCost: userStats.totalCost,
lastActive: userStats.lastActive,
})
.from(member)
.innerJoin(user, eq(member.userId, user.id))
.leftJoin(userStats, eq(member.userId, userStats.userId))
.where(eq(member.organizationId, organizationId))
return teamMembers.map((memberData) => ({
userId: memberData.userId,
userName: memberData.userName,
userEmail: memberData.userEmail,
currentLimit: toNumber(toDecimal(memberData.currentLimit || getFreeTierLimit().toString())),
currentUsage: toNumber(toDecimal(memberData.currentPeriodCost)),
totalCost: toNumber(toDecimal(memberData.totalCost)),
lastActive: memberData.lastActive,
}))
} catch (error) {
logger.error('Failed to get team usage limits', { organizationId, error })
return []
}
}
/**
* Returns the effective current period usage cost for a user, with daily
* refresh credits deducted. Org-scoped subs return the pooled sum across
@@ -0,0 +1,65 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import {
apportionCredits,
dollarsToCredits,
formatCreditCost,
} from '@/lib/billing/credits/conversion'
describe('formatCreditCost', () => {
it('renders multiplier-inclusive dollars as a single-rounded credit label', () => {
expect(formatCreditCost(0.005)).toBe('1 credit')
expect(formatCreditCost(0.03141848)).toBe('6 credits')
expect(formatCreditCost(1.234)).toBe('247 credits')
})
it('distinguishes sub-credit charges from zero', () => {
expect(formatCreditCost(0.001)).toBe('<1 credit')
expect(formatCreditCost(0)).toBe('0 credits')
})
it('honors emptyForZeroOrLess for the trace view contract', () => {
expect(formatCreditCost(0, { emptyForZeroOrLess: true })).toBeUndefined()
expect(formatCreditCost(undefined, { emptyForZeroOrLess: true })).toBeUndefined()
expect(formatCreditCost(undefined)).toBe('—')
})
})
describe('apportionCredits', () => {
it('keeps line items summing exactly to the rounded total (no round-then-sum drift)', () => {
// Real execution 43ef064d: base + 2x model, multiplier already applied.
// Round-then-sum would give 1 + 4 + 2 = 7; the true total is 6.
const credits = apportionCredits([
{ key: 'base', dollars: 0.005 },
{ key: 'input', dollars: 0.018798 },
{ key: 'output', dollars: 0.00762 },
{ key: 'tool', dollars: 0 },
])
const total = dollarsToCredits(0.005 + 0.018798 + 0.00762 + 0)
expect(total).toBe(6)
expect(credits.base + credits.input + credits.output + credits.tool).toBe(total)
expect(credits.base).toBe(1)
})
it('handles all-zero components', () => {
const credits = apportionCredits([
{ key: 'base', dollars: 0 },
{ key: 'model', dollars: 0 },
])
expect(credits.base + credits.model).toBe(0)
})
it('ignores negative/non-finite components without throwing', () => {
const credits = apportionCredits([
{ key: 'base', dollars: 0.005 },
{ key: 'model', dollars: Number.NaN },
{ key: 'tool', dollars: -1 },
])
expect(credits.base).toBe(1)
expect(credits.model).toBe(0)
expect(credits.tool).toBe(0)
})
})
@@ -12,6 +12,83 @@ export function dollarsToCredits(dollars: number): number {
return Math.round(dollars * CREDIT_MULTIPLIER)
}
/**
* Single source of truth for rendering a dollar cost as a credit label.
*
* Both the billing cost breakdown and the trace view derive their credit
* strings from here so the two surfaces can never diverge in rounding,
* thresholds, or pluralization. The dollar amount passed in is expected to
* already carry any cost multiplier (the value is converted with one — and
* only one — round via `dollarsToCredits`, i.e. multiply-then-round; never
* round per-line then multiply).
*
* `emptyForZeroOrLess` controls the zero/empty behavior so existing call sites
* keep their contracts: the breakdown wants a concrete "0 credits"/"—" string,
* the trace view wants `undefined` so it can hide the chip entirely.
*/
export function formatCreditCost(
dollars: number | null | undefined,
opts?: { emptyForZeroOrLess?: boolean }
): string | undefined {
if (dollars === undefined || dollars === null || !Number.isFinite(dollars)) {
return opts?.emptyForZeroOrLess ? undefined : '—'
}
const credits = dollarsToCredits(dollars)
if (credits <= 0) {
if (dollars > 0) return '<1 credit'
return opts?.emptyForZeroOrLess ? undefined : '0 credits'
}
return `${credits.toLocaleString()} ${credits === 1 ? 'credit' : 'credits'}`
}
/**
* Splits a set of cost components into integer credits that sum *exactly* to
* the credits of their combined total.
*
* This is the fix for the "line items don't add up to the total" class of bug:
* rounding each line independently (round-then-sum) drifts from the real charge
* (e.g. 1 + 2 + 1 = 4, or 1 + 4 + 2 = 7, when the true total is 6). Instead we
* convert the *summed* dollars to credits with a single round (multiply-then-
* round) and distribute that figure across the components via the largest-
* remainder method, so every component is multiplier-applied, internally
* consistent, and the rows always reconcile with the total.
*
* Each component's `dollars` is expected to already include any cost multiplier.
*/
export function apportionCredits<K extends string>(
components: { key: K; dollars: number }[]
): Record<K, number> {
const result = {} as Record<K, number>
const sanitized = components.map((c) => ({
key: c.key,
dollars: Number.isFinite(c.dollars) && c.dollars > 0 ? c.dollars : 0,
}))
const totalDollars = sanitized.reduce((sum, c) => sum + c.dollars, 0)
const targetCredits = dollarsToCredits(totalDollars)
const exact = sanitized.map((c) => ({
key: c.key,
floor: Math.floor(c.dollars * CREDIT_MULTIPLIER),
frac: c.dollars * CREDIT_MULTIPLIER - Math.floor(c.dollars * CREDIT_MULTIPLIER),
}))
for (const c of exact) result[c.key] = c.floor
let remainder = targetCredits - exact.reduce((sum, c) => sum + c.floor, 0)
const byFraction = [...exact].sort((a, b) => b.frac - a.frac)
for (let i = 0; i < byFraction.length && remainder > 0; i++) {
result[byFraction[i].key] += 1
remainder--
}
return result
}
/**
* Format a dollar amount as a comma-separated credit string.
* Values at or above the on-demand unlimited threshold display as ∞.
-1
View File
@@ -25,7 +25,6 @@ export {
export * from '@/lib/billing/core/usage'
export {
checkUsageStatus,
getTeamUsageLimits,
getUserUsageData as getUsageData,
getUserUsageLimit as getUsageLimit,
updateUserUsageLimit as updateUsageLimit,
-2
View File
@@ -137,8 +137,6 @@ interface TeamUsageLimit {
userEmail: string
currentLimit: number
currentUsage: number
totalCost: number
lastActive: Date | null
limitSetBy: string | null
limitUpdatedAt: Date | null
}
@@ -1,5 +1,5 @@
import { db } from '@sim/db'
import { invitation, member, organization, subscription, user, userStats } from '@sim/db/schema'
import { invitation, member, organization, subscription, user } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { and, count, eq, gt, ne } from 'drizzle-orm'
import { getOrganizationSubscription } from '@/lib/billing/core/billing'
@@ -307,35 +307,15 @@ export async function getOrganizationSeatAnalytics(organizationId: string) {
return null
}
const memberActivity = await db
.select({
userId: member.userId,
userName: user.name,
userEmail: user.email,
role: member.role,
joinedAt: member.createdAt,
lastActive: userStats.lastActive,
})
.from(member)
.innerJoin(user, eq(member.userId, user.id))
.leftJoin(userStats, eq(member.userId, userStats.userId))
.where(eq(member.organizationId, organizationId))
const utilizationRate =
seatInfo.maxSeats > 0 ? (seatInfo.currentSeats / seatInfo.maxSeats) * 100 : 0
const recentlyActive = memberActivity.filter((memberData) => {
if (!memberData.lastActive) return false
const daysSinceActive = (Date.now() - memberData.lastActive.getTime()) / (1000 * 60 * 60 * 24)
return daysSinceActive <= 30 // Active in last 30 days
}).length
// Member activity analytics (active/inactive counts, memberActivity) were
// derived from userStats.lastActive, which is no longer written. Dropped
// rather than report frozen data; reintroduce with a real activity source.
return {
...seatInfo,
utilizationRate: Math.round(utilizationRate * 100) / 100,
activeMembers: recentlyActive,
inactiveMembers: seatInfo.currentSeats - recentlyActive,
memberActivity,
}
} catch (error) {
logger.error('Failed to get organization seat analytics', { organizationId, error })
+44 -4
View File
@@ -13,7 +13,10 @@ import type Stripe from 'stripe'
import { getEmailSubject, PaymentFailedEmail, renderCreditPurchaseEmail } from '@/components/emails'
import { BILLING_LOCK_TIMEOUT_MS } from '@/lib/billing/constants'
import { calculateSubscriptionOverage, isSubscriptionOrgScoped } from '@/lib/billing/core/billing'
import { getBillingPeriodUsageCostByUser } from '@/lib/billing/core/usage-log'
import {
COPILOT_USAGE_SOURCES,
getBillingPeriodUsageCostByUser,
} from '@/lib/billing/core/usage-log'
import { addCredits, getCreditBalanceForEntity } from '@/lib/billing/credits/balance'
import { setUsageLimitForCredits } from '@/lib/billing/credits/purchase'
import { blockOrgMembers, unblockOrgMembers } from '@/lib/billing/organizations/membership'
@@ -424,6 +427,15 @@ export async function resetUsageForSubscription(sub: {
billingPeriod
)
: new Map<string, number>()
// Copilot-family ledger per user, so last-period copilot mirrors last-period
// cost (baseline + usage_log) instead of capturing the baseline alone.
const copilotLedgerByUser = billingPeriod
? await getBillingPeriodUsageCostByUser(
{ type: 'organization', id: sub.referenceId },
billingPeriod,
COPILOT_USAGE_SOURCES
)
: new Map<string, number>()
await db.transaction(async (tx) => {
await tx.execute(sql.raw(`SET LOCAL lock_timeout = '${BILLING_LOCK_TIMEOUT_MS}ms'`))
@@ -494,15 +506,27 @@ export async function resetUsageForSubscription(sub: {
memberStatsRows.map((row) => sql`WHEN ${row.userId} THEN ${row.currentCopilot ?? '0'}`),
sql` `
)
// Last-period copilot = baseline copilot + copilot-family ledger, mirroring
// lastPeriodCost. (The reset below still subtracts only the baseline, since
// the ledger is period-scoped and rolls over on its own.)
const lastCopilotCostByUser = sql.join(
memberStatsRows.map((row) => {
const baselineCopilot = toNumber(toDecimal(row.currentCopilot))
const copilotLedger = copilotLedgerByUser.get(row.userId) ?? 0
return sql`WHEN ${row.userId} THEN ${(baselineCopilot + copilotLedger).toString()}`
}),
sql` `
)
const capturedLastCost = sql`CASE ${userStats.userId} ${lastCostByUser} ELSE '0' END`
const capturedCurrentCost = sql`CASE ${userStats.userId} ${currentCostByUser} ELSE '0' END`
const capturedCurrentCopilotCost = sql`CASE ${userStats.userId} ${currentCopilotCostByUser} ELSE '0' END`
const capturedLastCopilotCost = sql`CASE ${userStats.userId} ${lastCopilotCostByUser} ELSE '0' END`
await tx
.update(userStats)
.set({
lastPeriodCost: capturedLastCost,
lastPeriodCopilotCost: capturedCurrentCopilotCost,
lastPeriodCopilotCost: capturedLastCopilotCost,
currentPeriodCost: sql`GREATEST(0, ${userStats.currentPeriodCost} - (${capturedCurrentCost})::decimal)`,
currentPeriodCopilotCost: sql`GREATEST(0, ${userStats.currentPeriodCopilotCost} - (${capturedCurrentCopilotCost})::decimal)`,
billedOverageThisPeriod: '0',
@@ -536,6 +560,15 @@ export async function resetUsageForSubscription(sub: {
)
: new Map<string, number>()
const userLedgerUsage = ledgerUsage.get(sub.referenceId) ?? 0
const copilotLedgerUsage = billingPeriod
? ((
await getBillingPeriodUsageCostByUser(
{ type: 'user', id: sub.referenceId },
billingPeriod,
COPILOT_USAGE_SOURCES
)
).get(sub.referenceId) ?? 0)
: 0
// Snapshot > 0: user joined a paid org mid-cycle. The pre-join
// portion was billed on this invoice (snapshot); `currentPeriodCost`
@@ -546,7 +579,12 @@ export async function resetUsageForSubscription(sub: {
.update(userStats)
.set({
lastPeriodCost: (snapshot + userLedgerUsage).toString(),
lastPeriodCopilotCost: '0',
// Pre-join personal copilot = the user-scoped copilot ledger only
// (post-join copilot usage is org-attributed, so this captures the
// pre-join portion). The copilot baseline stays with the org via the
// retained currentPeriodCopilotCost, so don't add it here (avoids a
// double count at the org's cycle-close).
lastPeriodCopilotCost: copilotLedgerUsage.toString(),
proPeriodCostSnapshot: '0',
proPeriodCostSnapshotAt: null,
billedOverageThisPeriod: '0',
@@ -563,7 +601,9 @@ export async function resetUsageForSubscription(sub: {
.update(userStats)
.set({
lastPeriodCost: totalLastPeriod,
lastPeriodCopilotCost: currentCopilot,
lastPeriodCopilotCost: (
toNumber(toDecimal(currentCopilot)) + copilotLedgerUsage
).toString(),
currentPeriodCost: sql`GREATEST(0, ${userStats.currentPeriodCost} - ${current}::decimal)`,
currentPeriodCopilotCost: sql`GREATEST(0, ${userStats.currentPeriodCopilotCost} - ${currentCopilot}::decimal)`,
proPeriodCostSnapshot: '0',
+13 -5
View File
@@ -639,6 +639,7 @@ async function processExecutionLogFromDb(
.select({
id: workflowExecutionLogs.id,
workflowId: workflowExecutionLogs.workflowId,
workspaceId: workflowExecutionLogs.workspaceId,
executionId: workflowExecutionLogs.executionId,
level: workflowExecutionLogs.level,
trigger: workflowExecutionLogs.trigger,
@@ -646,7 +647,7 @@ async function processExecutionLogFromDb(
endedAt: workflowExecutionLogs.endedAt,
totalDurationMs: workflowExecutionLogs.totalDurationMs,
executionData: workflowExecutionLogs.executionData,
cost: workflowExecutionLogs.cost,
costTotal: workflowExecutionLogs.costTotal,
workflowName: workflow.name,
})
.from(workflowExecutionLogs)
@@ -671,6 +672,13 @@ async function processExecutionLogFromDb(
}
}
// Heavy execution data may live in object storage; resolve the pointer.
const { materializeExecutionData } = await import('@/lib/logs/execution/trace-store')
const executionData = (await materializeExecutionData(
log.executionData as Record<string, unknown> | null,
{ workspaceId: log.workspaceId, workflowId: log.workflowId, executionId: log.executionId }
)) as any
const summary = {
id: log.id,
workflowId: log.workflowId,
@@ -681,13 +689,13 @@ async function processExecutionLogFromDb(
endedAt: log.endedAt?.toISOString?.() || (log.endedAt ? String(log.endedAt) : null),
totalDurationMs: log.totalDurationMs ?? null,
workflowName: log.workflowName || '',
executionData: log.executionData
executionData: executionData
? {
traceSpans: (log.executionData as any).traceSpans || undefined,
errorDetails: (log.executionData as any).errorDetails || undefined,
traceSpans: executionData.traceSpans || undefined,
errorDetails: executionData.errorDetails || undefined,
}
: undefined,
cost: log.cost || undefined,
cost: log.costTotal != null ? { total: Number(log.costTotal) } : undefined,
}
const content = JSON.stringify(summary)
@@ -4,6 +4,7 @@ import { createLogger } from '@sim/logger'
import { and, desc, eq, type SQL } from 'drizzle-orm'
import { GetExecutionSummary } from '@/lib/copilot/generated/tool-catalog-v1'
import type { BaseServerTool, ServerToolContext } from '@/lib/copilot/tools/server/base-tool'
import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils'
const logger = createLogger('GetExecutionSummaryServerTool')
@@ -86,13 +87,14 @@ export const getExecutionSummaryServerTool: BaseServerTool<
.select({
executionId: workflowExecutionLogs.executionId,
workflowId: workflowExecutionLogs.workflowId,
workspaceId: workflowExecutionLogs.workspaceId,
workflowName: workflow.name,
status: workflowExecutionLogs.status,
level: workflowExecutionLogs.level,
trigger: workflowExecutionLogs.trigger,
startedAt: workflowExecutionLogs.startedAt,
totalDurationMs: workflowExecutionLogs.totalDurationMs,
cost: workflowExecutionLogs.cost,
costTotal: workflowExecutionLogs.costTotal,
executionData: workflowExecutionLogs.executionData,
})
.from(workflowExecutionLogs)
@@ -101,26 +103,36 @@ export const getExecutionSummaryServerTool: BaseServerTool<
.orderBy(desc(workflowExecutionLogs.startedAt))
.limit(clampedLimit)
const summaries: ExecutionSummary[] = rows.map((row) => {
const costData = row.cost as any
const errorMsg = row.level === 'error' ? extractErrorMessage(row.executionData) : null
const summaries: ExecutionSummary[] = await Promise.all(
rows.map(async (row) => {
// Only externalized rows need a fetch; error fields live in the heavy data.
const executionData =
row.level === 'error'
? await materializeExecutionData(row.executionData as Record<string, unknown> | null, {
workspaceId: row.workspaceId,
workflowId: row.workflowId,
executionId: row.executionId,
})
: row.executionData
const errorMsg = row.level === 'error' ? extractErrorMessage(executionData) : null
return {
executionId: row.executionId,
workflowId: row.workflowId,
workflowName: row.workflowName,
status: row.status,
trigger: row.trigger,
startedAt: row.startedAt.toISOString(),
durationMs: row.totalDurationMs ?? null,
cost: costData?.total ? Number(costData.total) : null,
error: errorMsg
? typeof errorMsg === 'string'
? errorMsg
: JSON.stringify(errorMsg)
: null,
}
})
return {
executionId: row.executionId,
workflowId: row.workflowId,
workflowName: row.workflowName,
status: row.status,
trigger: row.trigger,
startedAt: row.startedAt.toISOString(),
durationMs: row.totalDurationMs ?? null,
cost: row.costTotal != null ? Number(row.costTotal) : null,
error: errorMsg
? typeof errorMsg === 'string'
? errorMsg
: JSON.stringify(errorMsg)
: null,
}
})
)
logger.info('Execution summary prepared', {
count: summaries.length,
@@ -5,6 +5,7 @@ import { authorizeWorkflowByWorkspacePermission } from '@sim/workflow-authz'
import { and, desc, eq } from 'drizzle-orm'
import { GetWorkflowLogs } from '@/lib/copilot/generated/tool-catalog-v1'
import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool'
import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
import type { TraceSpan } from '@/lib/logs/types'
const logger = createLogger('GetWorkflowLogsServerTool')
@@ -145,6 +146,8 @@ export const getWorkflowLogsServerTool: BaseServerTool<GetWorkflowLogsArgs, Simp
.select({
id: workflowExecutionLogs.id,
executionId: workflowExecutionLogs.executionId,
workflowId: workflowExecutionLogs.workflowId,
workspaceId: workflowExecutionLogs.workspaceId,
status: workflowExecutionLogs.status,
level: workflowExecutionLogs.level,
trigger: workflowExecutionLogs.trigger,
@@ -152,53 +155,62 @@ export const getWorkflowLogsServerTool: BaseServerTool<GetWorkflowLogsArgs, Simp
endedAt: workflowExecutionLogs.endedAt,
totalDurationMs: workflowExecutionLogs.totalDurationMs,
executionData: workflowExecutionLogs.executionData,
cost: workflowExecutionLogs.cost,
})
.from(workflowExecutionLogs)
.where(and(...conditions))
.orderBy(desc(workflowExecutionLogs.startedAt))
.limit(executionId ? 1 : limit)
// Enforce the documented hard limit (the materialization fans out per row).
.limit(executionId ? 1 : Math.min(Math.max(1, limit), 3))
const simplifiedExecutions: SimplifiedExecution[] = executionLogs.map((log) => {
const executionData = log.executionData as ExecutionData
const traceSpans = executionData?.traceSpans ?? []
const blockExecutions = includeDetails
? extractBlockExecutionsFromTraceSpans(traceSpans)
: []
const simplifiedExecutions: SimplifiedExecution[] = await Promise.all(
executionLogs.map(async (log) => {
const executionData = (await materializeExecutionData(
log.executionData as Record<string, unknown> | null,
{
workspaceId: log.workspaceId,
workflowId: log.workflowId,
executionId: log.executionId,
}
)) as ExecutionData
const traceSpans = executionData?.traceSpans ?? []
const blockExecutions = includeDetails
? extractBlockExecutionsFromTraceSpans(traceSpans)
: []
const simplifiedBlocks: SimplifiedBlock[] = blockExecutions.map((block) => ({
id: block.blockId,
name: block.blockName,
startedAt: block.startedAt,
endedAt: block.endedAt,
durationMs: block.durationMs,
output: block.outputData,
error: block.status === 'error' ? block.errorMessage : undefined,
}))
const simplifiedBlocks: SimplifiedBlock[] = blockExecutions.map((block) => ({
id: block.blockId,
name: block.blockName,
startedAt: block.startedAt,
endedAt: block.endedAt,
durationMs: block.durationMs,
output: block.outputData,
error: block.status === 'error' ? block.errorMessage : undefined,
}))
const rawError =
executionData?.errorDetails?.error ||
executionData?.errorDetails?.message ||
executionData?.finalOutput?.error ||
executionData?.error ||
null
const errorMessage = rawError
? typeof rawError === 'string'
? rawError
: JSON.stringify(rawError)
: undefined
const rawError =
executionData?.errorDetails?.error ||
executionData?.errorDetails?.message ||
executionData?.finalOutput?.error ||
executionData?.error ||
null
const errorMessage = rawError
? typeof rawError === 'string'
? rawError
: JSON.stringify(rawError)
: undefined
return {
id: log.id,
executionId: log.executionId,
status: log.status,
startedAt: log.startedAt.toISOString(),
endedAt: log.endedAt ? log.endedAt.toISOString() : null,
durationMs: log.totalDurationMs ?? null,
...(errorMessage ? { error: errorMessage } : {}),
...(simplifiedBlocks.length > 0 ? { blocks: simplifiedBlocks } : {}),
}
})
return {
id: log.id,
executionId: log.executionId,
status: log.status,
startedAt: log.startedAt.toISOString(),
endedAt: log.endedAt ? log.endedAt.toISOString() : null,
durationMs: log.totalDurationMs ?? null,
...(errorMessage ? { error: errorMessage } : {}),
...(simplifiedBlocks.length > 0 ? { blocks: simplifiedBlocks } : {}),
}
})
)
const resultSize = JSON.stringify(simplifiedExecutions).length
logger.info('Workflow logs result prepared', {
+35
View File
@@ -0,0 +1,35 @@
/**
* Maps over `items` with at most `limit` concurrent invocations of `fn`,
* preserving input order in the result. Use to bound fan-out (e.g. per-row
* object-storage reads) so a large batch doesn't issue every request at once.
*
* Contract: `fn` MUST NOT reject. Results are awaited via `Promise.all`, so a
* single rejection fails the entire batch (e.g. one bad row would break a whole
* logs export/list page). Callers that materialize per-row data pass a total
* mapper (one that catches and returns a degraded value rather than throwing);
* keep it that way. If a future caller needs per-item isolation, add an
* allSettled-style variant rather than letting a throwing mapper through here.
*/
export async function mapWithConcurrency<T, R>(
items: readonly T[],
limit: number,
fn: (item: T, index: number) => Promise<R>
): Promise<R[]> {
const results = new Array<R>(items.length)
const workerCount = Math.max(1, Math.min(limit, items.length))
let cursor = 0
const worker = async (): Promise<void> => {
while (true) {
const index = cursor++
if (index >= items.length) return
results[index] = await fn(items[index], index)
}
}
await Promise.all(Array.from({ length: workerCount }, worker))
return results
}
/** Default bound for per-row object-storage materialization fan-out. */
export const MATERIALIZE_CONCURRENCY = 20
@@ -1,6 +1,7 @@
import { db } from '@sim/db'
import { workflowExecutionLogs } from '@sim/db/schema'
import { and, inArray, isNotNull } from 'drizzle-orm'
import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency'
import {
decodeTimeCursor,
encodeTimeCursor,
@@ -9,6 +10,7 @@ import {
} from '@/lib/data-drains/sources/cursor'
import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers'
import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types'
import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
type WorkflowLogRow = typeof workflowExecutionLogs.$inferSelect
@@ -45,6 +47,22 @@ async function* pages(input: SourcePageInput): AsyncIterable<WorkflowLogRow[]> {
.limit(input.chunkSize)
if (rows.length === 0) return
// Heavy execution data may live in object storage; resolve pointers (bounded
// concurrency) so the drain exports full execution data, not the slim row.
// Use the order-preserving returned array (the util's documented contract)
// and write back, rather than mutating rows inside the mapper.
const materialized = await mapWithConcurrency(rows, MATERIALIZE_CONCURRENCY, (row) =>
materializeExecutionData(row.executionData as Record<string, unknown> | null, {
workspaceId: row.workspaceId,
workflowId: row.workflowId,
executionId: row.executionId,
})
)
for (let i = 0; i < rows.length; i++) {
rows[i].executionData = materialized[i] as WorkflowLogRow['executionData']
}
yield rows
const last = rows[rows.length - 1]
cursor = { ts: last.endedAt!.toISOString(), id: last.id }
@@ -71,7 +89,8 @@ export const workflowLogsSource: DrainSource<WorkflowLogRow> = {
endedAt: row.endedAt ? row.endedAt.toISOString() : null,
totalDurationMs: row.totalDurationMs,
executionData: row.executionData,
cost: row.cost,
// cost_total projection of the usage_log ledger (not the deprecated jsonb).
cost: row.costTotal != null ? { total: Number(row.costTotal) } : null,
files: row.files,
createdAt: row.createdAt.toISOString(),
}
+28 -10
View File
@@ -31,6 +31,13 @@ export interface LargeValueStoreContext {
userId?: string
requireDurable?: boolean
maxBytes?: number
/**
* When false, materialization does not register an execution_log reference for
* the key. Read-only consumers (e.g. viewing/exporting a completed log) set
* this: the value is already owned + referenced by its own execution, so
* re-registering on every read is wasteful and a needless failure point.
*/
trackReference?: boolean
}
function getKind(value: unknown): LargeValueKind {
@@ -163,6 +170,11 @@ export async function storeLargeValue(
const id = `lv_${generateShortId(12)}`
let key = await persistValue(id, json, context)
if (key) {
// Only clean up the uploaded object when registration definitively did NOT
// record ownership (returns false). If registration THROWS, the metadata
// state is uncertain (a row may have partially committed), so we propagate
// without deleting — deleting could orphan a metadata row pointing at a
// now-missing object.
const registered = await registerPersistedValueOwner(key, size, referencedKeys, context)
if (!registered) {
await deleteUntrackedPersistedValue(key)
@@ -204,16 +216,22 @@ export async function materializeLargeValueRef(
return materializeLargeValueRefSync(ref, context)
}
const { addLargeValueReference } = await import('@/lib/execution/payloads/large-value-metadata')
await addLargeValueReference(
{
workspaceId: context.workspaceId,
workflowId: context.workflowId,
executionId: context.executionId,
source: 'execution_log',
},
ref.key
)
if (context.trackReference !== false) {
const { addLargeValueReference } = await import('@/lib/execution/payloads/large-value-metadata')
// Reference tracking is GC-critical: if it fails, fail the read rather than
// return a value whose reference was never recorded (it could later be
// garbage-collected out from under a live consumer). Read-only consumers
// that don't need a reference set trackReference: false to skip this.
await addLargeValueReference(
{
workspaceId: context.workspaceId,
workflowId: context.workflowId,
executionId: context.executionId,
source: 'execution_log',
},
ref.key
)
}
try {
const cached = materializeLargeValueRefSync(ref, context)
+350
View File
@@ -1,7 +1,37 @@
import { featureFlagsMock } from '@sim/testing'
import { beforeEach, describe, expect, test, vi } from 'vitest'
import { recordUsage } from '@/lib/billing/core/usage-log'
import { ExecutionLogger } from '@/lib/logs/execution/logger'
const dbSelectMock = vi.hoisted(() => vi.fn())
const dbExecuteMock = vi.hoisted(() => vi.fn())
const txUpdateMock = vi.hoisted(() =>
vi.fn(() => ({ set: () => ({ where: () => Promise.resolve() }) }))
)
vi.mock('@sim/db', () => {
// The reconcile runs inside db.transaction with an advisory lock. The tx
// shares dbSelectMock so the existing call-order seeding (call 1 = workflow
// row via .limit, call 2 = already-billed via .groupBy) still applies;
// tx.execute (set_config + pg_advisory_xact_lock) is a no-op; tx.update backs
// the exact cost_total refine.
const tx = {
select: dbSelectMock,
insert: vi.fn(),
update: txUpdateMock,
execute: dbExecuteMock,
}
return {
db: {
select: dbSelectMock,
insert: vi.fn(),
update: vi.fn(),
execute: dbExecuteMock,
transaction: vi.fn(async (cb: (txArg: typeof tx) => Promise<unknown>) => cb(tx)),
},
}
})
// Mock billing modules
vi.mock('@/lib/billing/core/subscription', () => ({
getHighestPrioritySubscription: vi.fn(() => Promise.resolve(null)),
@@ -19,6 +49,7 @@ vi.mock('@/lib/billing/core/usage', () => ({
vi.mock('@/lib/billing/core/usage-log', () => ({
recordUsage: vi.fn(() => Promise.resolve()),
stableEventKey: vi.fn((parts: Record<string, unknown>) => JSON.stringify(parts)),
}))
vi.mock('@/lib/billing/threshold-billing', () => ({
@@ -439,3 +470,322 @@ describe('ExecutionLogger', () => {
})
})
})
describe('recordExecutionUsage boundary-delta reconciliation', () => {
let logger: any
beforeEach(() => {
logger = new ExecutionLogger() as any
vi.clearAllMocks()
})
const costSummary = (overrides: Record<string, unknown> = {}) => ({
totalCost: 0,
totalInputCost: 0,
totalOutputCost: 0,
totalTokens: 0,
totalPromptTokens: 0,
totalCompletionTokens: 0,
baseExecutionCharge: 0.005,
models: {},
charges: {},
...overrides,
})
// db.select() is called twice in recordExecutionUsage: first the workflow row
// (terminated by .limit), then the already-billed usage_log rows (terminated
// by .groupBy). Return each in order.
const mockDb = (billedRows: Array<Record<string, unknown>>) => {
let call = 0
dbSelectMock.mockImplementation(() => {
call += 1
const rows = call === 1 ? [{ id: 'workflow-1', workspaceId: 'ws-1' }] : billedRows
const chain: any = {
from: () => chain,
where: () => chain,
limit: () => Promise.resolve(rows),
groupBy: () => Promise.resolve(rows),
}
return chain
})
}
const run = (
summary: ReturnType<typeof costSummary>,
billedRows: Array<Record<string, unknown>>
) => {
mockDb(billedRows)
return logger.recordExecutionUsage('workflow-1', summary, 'api', 'exec-1', 'user-1')
}
const lastEntries = () => vi.mocked(recordUsage).mock.calls[0][0].entries
test('fresh completion records all targets (base fee + model) and returns the increment', async () => {
const recorded = await run(
costSummary({
models: {
'gpt-4o': {
total: 1,
input: 0.6,
output: 0.4,
tokens: { input: 10, output: 5, total: 15 },
},
},
}),
[]
)
expect(recordUsage).toHaveBeenCalledTimes(1)
expect(lastEntries()).toEqual([
expect.objectContaining({ category: 'fixed', description: 'execution_fee', cost: 0.005 }),
expect.objectContaining({ category: 'model', description: 'gpt-4o', cost: 1 }),
])
// Returns the amount recorded at this boundary (drives threshold-email math).
expect(recorded).toBeCloseTo(1.005, 8)
// cost_total is refined to the exact ledger sum inside the locked tx.
expect(txUpdateMock).toHaveBeenCalledTimes(1)
})
test('resume records only the increment over what is already billed', async () => {
await run(
costSummary({
models: {
'gpt-4o': {
total: 3,
input: 1.8,
output: 1.2,
tokens: { input: 30, output: 15, total: 45 },
},
},
}),
[
{ category: 'fixed', description: 'execution_fee', cost: '0.005' },
{ category: 'model', description: 'gpt-4o', cost: '1' },
]
)
expect(recordUsage).toHaveBeenCalledTimes(1)
const entries = lastEntries()
expect(entries).toHaveLength(1)
expect(entries[0]).toEqual(
expect.objectContaining({ category: 'model', description: 'gpt-4o', cost: 2 })
)
})
test('returns only the post-resume increment (not the cumulative total)', async () => {
const recorded = await run(
costSummary({
models: {
'gpt-4o': {
total: 3,
input: 0,
output: 0,
tokens: { input: 0, output: 0, total: 0 },
},
},
}),
[
{ category: 'fixed', description: 'execution_fee', cost: '0.005' },
{ category: 'model', description: 'gpt-4o', cost: '1' },
]
)
// Cumulative is 3.005, but only the $2 increment was recorded here — the
// threshold-email math must not re-count the pre-pause $1.005.
expect(recorded).toBe(2)
})
test('forwards a pre-resolved billing context to recordUsage (skips re-lookup)', async () => {
mockDb([])
const billingContext = {
billingEntity: { type: 'user' as const, id: 'user-1' },
billingPeriod: { start: new Date('2026-01-01'), end: new Date('2026-02-01') },
}
await (logger as any).recordExecutionUsage(
'workflow-1',
costSummary({
models: {
'gpt-4o': {
total: 1,
input: 0,
output: 0,
tokens: { input: 0, output: 0, total: 0 },
},
},
}),
'api',
'exec-1',
'user-1',
billingContext
)
expect(vi.mocked(recordUsage).mock.calls[0][0]).toMatchObject({
billingEntity: { type: 'user', id: 'user-1' },
billingPeriod: billingContext.billingPeriod,
})
})
test('retry with everything already billed records nothing (idempotent)', async () => {
await run(
costSummary({
models: {
'gpt-4o': {
total: 1,
input: 0.6,
output: 0.4,
tokens: { input: 10, output: 5, total: 15 },
},
},
}),
[
{ category: 'fixed', description: 'execution_fee', cost: '0.005' },
{ category: 'model', description: 'gpt-4o', cost: '1' },
]
)
expect(recordUsage).not.toHaveBeenCalled()
})
test('BYOK run records only the base fee (no zero-cost model rows)', async () => {
await run(costSummary({ models: {}, charges: {} }), [])
expect(recordUsage).toHaveBeenCalledTimes(1)
expect(lastEntries()).toEqual([
expect.objectContaining({ category: 'fixed', description: 'execution_fee', cost: 0.005 }),
])
})
test('standalone hosted-tool charge reconciles as a tool row', async () => {
await run(costSummary({ charges: { 'Exa Search': { total: 0.02 } } }), [
{ category: 'fixed', description: 'execution_fee', cost: '0.005' },
])
expect(lastEntries()).toEqual([
expect.objectContaining({ category: 'tool', description: 'Exa Search', cost: 0.02 }),
])
})
test('two boundaries (pause then resume) bill the full run exactly once', async () => {
const model = (total: number) => ({
'gpt-4o': {
input: total * 0.6,
output: total * 0.4,
total,
tokens: { input: 10, output: 5, total: 15 },
},
})
// Boundary 1 (pause): nothing billed yet, partial cost.
await run(costSummary({ models: model(1) }), [])
const firstEntries = vi.mocked(recordUsage).mock.calls[0][0].entries
// Feed boundary 1's rows back as already-billed for boundary 2.
const billedAfterFirst = firstEntries.map((e: any) => ({
category: e.category,
description: e.description,
cost: String(e.cost),
}))
// Boundary 2 (resume terminal): same model, higher cumulative cost.
await run(costSummary({ models: model(3) }), billedAfterFirst)
const secondEntries = vi.mocked(recordUsage).mock.calls[1][0].entries
const ledgerTotal = [...firstEntries, ...secondEntries].reduce(
(sum: number, e: any) => sum + e.cost,
0
)
expect(ledgerTotal).toBeCloseTo(3.005, 8) // base 0.005 once + gpt-4o 3 total
// Base fee billed once (boundary 1 only); model increment only at boundary 2.
expect(firstEntries.some((e: any) => e.category === 'fixed')).toBe(true)
expect(secondEntries.some((e: any) => e.category === 'fixed')).toBe(false)
expect(secondEntries).toEqual([
expect.objectContaining({ category: 'model', description: 'gpt-4o', cost: 2 }),
])
})
test('eventKey is scoped by billedBefore so cross-boundary increments do not collide', async () => {
const model = (total: number) => ({
'gpt-4o': { input: 0, output: 0, total, tokens: { input: 0, output: 0, total: 0 } },
})
await run(costSummary({ models: model(1) }), [])
const key0 = vi
.mocked(recordUsage)
.mock.calls[0][0].entries.find((e: any) => e.category === 'model')?.eventKey
await run(costSummary({ models: model(3) }), [
{ category: 'fixed', description: 'execution_fee', cost: '0.005' },
{ category: 'model', description: 'gpt-4o', cost: '1' },
])
const key1 = vi
.mocked(recordUsage)
.mock.calls[1][0].entries.find((e: any) => e.category === 'model')?.eventKey
expect(key0).toContain('"billedBefore":"0.00000000"')
expect(key1).toContain('"billedBefore":"1.00000000"')
expect(key0).not.toEqual(key1)
})
test('a decreased cumulative cost (negative delta) records nothing for that line', async () => {
await run(
costSummary({
models: {
'gpt-4o': { input: 0, output: 0, total: 3, tokens: { input: 0, output: 0, total: 0 } },
},
}),
[
{ category: 'fixed', description: 'execution_fee', cost: '0.005' },
{ category: 'model', description: 'gpt-4o', cost: '5' },
]
)
expect(recordUsage).not.toHaveBeenCalled()
})
test('a model introduced only post-resume is billed in full; the already-billed model is skipped', async () => {
await run(
costSummary({
models: {
'gpt-4o': { input: 0, output: 0, total: 1, tokens: { input: 0, output: 0, total: 0 } },
'claude-3': { input: 0, output: 0, total: 2, tokens: { input: 0, output: 0, total: 0 } },
},
}),
[
{ category: 'fixed', description: 'execution_fee', cost: '0.005' },
{ category: 'model', description: 'gpt-4o', cost: '1' },
]
)
expect(lastEntries()).toEqual([
expect.objectContaining({ category: 'model', description: 'claude-3', cost: 2 }),
])
})
test('zero-cost models and charges (BYOK) are filtered out, leaving only the base fee', async () => {
await run(
costSummary({
models: {
'gpt-4o': { input: 0, output: 0, total: 0, tokens: { input: 0, output: 0, total: 0 } },
},
charges: { Exa: { total: 0 } },
}),
[]
)
expect(lastEntries()).toEqual([
expect.objectContaining({ category: 'fixed', description: 'execution_fee', cost: 0.005 }),
])
})
test('reconciles inside a transaction holding a per-execution advisory lock', async () => {
await run(
costSummary({
models: {
'gpt-4o': { input: 0, output: 0, total: 1, tokens: { input: 0, output: 0, total: 0 } },
},
}),
[]
)
// set_config('lock_timeout') + pg_advisory_xact_lock both run on the tx.
expect(dbExecuteMock).toHaveBeenCalledTimes(2)
expect(recordUsage).toHaveBeenCalledTimes(1)
// The ledger INSERT participates in the locked transaction.
expect(vi.mocked(recordUsage).mock.calls[0][0]).toHaveProperty('tx')
})
})
+382 -157
View File
@@ -1,6 +1,7 @@
import { db } from '@sim/db'
import {
member,
usageLog,
userStats,
user as userTable,
workflow,
@@ -9,15 +10,20 @@ import {
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { generateId } from '@sim/utils/id'
import { eq, sql } from 'drizzle-orm'
import { BASE_EXECUTION_CHARGE } from '@/lib/billing/constants'
import { and, eq, sql } from 'drizzle-orm'
import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
import {
checkUsageStatus,
getOrgUsageLimit,
maybeSendUsageThresholdEmail,
} from '@/lib/billing/core/usage'
import { type ModelUsageMetadata, recordUsage } from '@/lib/billing/core/usage-log'
import {
type BillingContext,
deriveBillingContext,
type ModelUsageMetadata,
recordUsage,
stableEventKey,
} from '@/lib/billing/core/usage-log'
import { checkAndBillOverageThreshold } from '@/lib/billing/threshold-billing'
import { isBillingEnabled } from '@/lib/core/config/feature-flags'
import { redactApiKeys } from '@/lib/core/security/redaction'
@@ -28,6 +34,7 @@ import {
} from '@/lib/execution/payloads/large-value-metadata'
import { emitWorkflowExecutionCompleted } from '@/lib/logs/events'
import { snapshotService } from '@/lib/logs/execution/snapshot/service'
import { externalizeExecutionData, stripSpanCosts } from '@/lib/logs/execution/trace-store'
import type {
BlockOutputData,
ExecutionEnvironment,
@@ -48,6 +55,9 @@ const MAX_WORKFLOW_VALUE_BYTES = 512 * 1024
const EXECUTION_LOG_STATEMENT_TIMEOUT_MS = 30_000
const EXECUTION_LOG_LOCK_TIMEOUT_MS = 3_000
const EXECUTION_LOG_IDLE_TIMEOUT_MS = 5_000
// Bounds the wait for the per-execution usage-reconcile advisory lock. Generous
// (favor waiting over dropping a charge); only trips on a pathological lock hold.
const USAGE_RECONCILE_LOCK_TIMEOUT_MS = 10_000
type ExecutionData = WorkflowExecutionLog['executionData']
@@ -548,13 +558,6 @@ export class ExecutionLogger implements IExecutionLoggerService {
hasTraceSpans: false,
traceSpanCount: 0,
},
cost: {
total: BASE_EXECUTION_CHARGE,
input: 0,
output: 0,
tokens: { input: 0, output: 0, total: 0 },
models: {},
},
})
.returning()
@@ -590,7 +593,6 @@ export class ExecutionLogger implements IExecutionLoggerService {
totalPromptTokens: number
totalCompletionTokens: number
baseExecutionCharge: number
modelCost: number
models: Record<
string,
{
@@ -601,6 +603,7 @@ export class ExecutionLogger implements IExecutionLoggerService {
tokens: { input: number; output: number; total: number }
}
>
charges?: Record<string, { total: number }>
}
finalOutput: BlockOutputData
traceSpans?: TraceSpan[]
@@ -732,7 +735,26 @@ export class ExecutionLogger implements IExecutionLoggerService {
},
executionId
)
const completedExecutionLargeValueKeys = collectLargeValueReferenceKeys(completedExecutionData)
stripSpanCosts((completedExecutionData as Record<string, unknown>).traceSpans)
// Externalization requires the execution owner (workspace_files.user_id is
// NOT NULL). billingUserId comes from environment.userId and is effectively
// always present for a real run; if it's somehow absent, keep data inline.
let storedExecutionData = completedExecutionData as Record<string, unknown>
if (billingUserId) {
storedExecutionData = await externalizeExecutionData(storedExecutionData, {
workspaceId: existingLog?.workspaceId ?? null,
workflowId: existingLog?.workflowId ?? null,
executionId,
userId: billingUserId,
})
} else {
execLog.warn('Skipping execution-data externalization: missing owner userId', {
executionId,
})
}
const completedExecutionLargeValueKeys = collectLargeValueReferenceKeys(storedExecutionData)
const updatedLog = await db.transaction(async (tx) => {
await setExecutionLogWriteTimeouts(tx)
@@ -745,8 +767,17 @@ export class ExecutionLogger implements IExecutionLoggerService {
endedAt: new Date(endedAt),
totalDurationMs: totalDuration,
files: executionFiles.length > 0 ? executionFiles : null,
executionData: completedExecutionData,
cost: executionCost,
executionData: storedExecutionData,
// Faithful projection of the usage_log ledger. Neither cost_total nor
// models_used may regress below a prior boundary: a paused run that
// resumes into an empty-span error/cancel/cost-only fallback produces a
// base-only summary. GREATEST keeps the higher cumulative cost_total,
// and models_used is overwritten only when this boundary actually has
// models — so both stay == SUM(usage_log) on every monotonic path.
costTotal: sql`GREATEST(COALESCE(${workflowExecutionLogs.costTotal}, 0), ${costSummary.totalCost.toString()}::numeric)`,
...(Object.keys(costSummary.models).length > 0
? { modelsUsed: Object.keys(costSummary.models) }
: {}),
})
.where(eq(workflowExecutionLogs.executionId, executionId))
.returning()
@@ -770,116 +801,146 @@ export class ExecutionLogger implements IExecutionLoggerService {
})
try {
// Skip workflow lookup if workflow was deleted
// Skip workflow lookup if workflow was deleted.
const wf = updatedLog.workflowId
? (await db.select().from(workflow).where(eq(workflow.id, updatedLog.workflowId)))[0]
: undefined
if (wf && billingUserId) {
const [usr] = await db
.select({ id: userTable.id, email: userTable.email, name: userTable.name })
.from(userTable)
.where(eq(userTable.id, billingUserId))
.limit(1)
if (usr?.email) {
const sub = await getHighestPrioritySubscription(usr.id)
const usr =
wf && billingUserId
? (
await db
.select({ id: userTable.id, email: userTable.email, name: userTable.name })
.from(userTable)
.where(eq(userTable.id, billingUserId))
.limit(1)
)[0]
: undefined
const costDelta = costSummary.totalCost
// Resolve the billing context + the pre-increment usage snapshot for the
// threshold email BEFORE recording, so currentUsageAfter = before +
// costDelta doesn't double-count this boundary's own increment.
type EmailContext =
| {
scope: 'user'
userId: string
userEmail: string
userName: string | null
planName: string
before: Awaited<ReturnType<typeof checkUsageStatus>>
}
| {
scope: 'organization'
organizationId: string
planName: string
orgLimit: number
orgUsageBefore: number
}
let billingContext: BillingContext | undefined
let emailContext: EmailContext | undefined
const { getDisplayPlanName } = await import('@/lib/billing/plan-helpers')
const { isOrgScopedSubscription } = await import('@/lib/billing/subscriptions/utils')
const planName = getDisplayPlanName(sub?.plan)
const scope: 'user' | 'organization' = isOrgScopedSubscription(sub, usr.id)
? 'organization'
: 'user'
if (usr?.email) {
const sub = await getHighestPrioritySubscription(usr.id)
// Derive the billing context once from the subscription we just fetched
// and thread it into recordExecutionUsage so recordUsage doesn't
// re-resolve the subscription on the hot completion path.
billingContext = deriveBillingContext(usr.id, sub)
if (scope === 'user') {
const before = await checkUsageStatus(usr.id)
const { getDisplayPlanName } = await import('@/lib/billing/plan-helpers')
const { isOrgScopedSubscription } = await import('@/lib/billing/subscriptions/utils')
const planName = getDisplayPlanName(sub?.plan)
await this.updateUserStats(
updatedLog.workflowId,
costSummary,
updatedLog.trigger as ExecutionTrigger['type'],
executionId,
billingUserId
)
const limit = before.usageData.limit
const percentBefore = before.usageData.percentUsed
const percentAfter =
limit > 0 ? Math.min(100, percentBefore + (costDelta / limit) * 100) : percentBefore
const currentUsageAfter = before.usageData.currentUsage + costDelta
await maybeSendUsageThresholdEmail({
scope: 'user',
userId: usr.id,
userEmail: usr.email,
userName: usr.name || undefined,
planName,
percentBefore,
percentAfter,
currentUsageAfter,
limit,
})
} else if (sub?.referenceId) {
// Get org usage limit using shared helper
const { limit: orgLimit } = await getOrgUsageLimit(sub.referenceId, sub.plan, sub.seats)
const [{ sum: orgUsageBefore }] = await db
.select({ sum: sql`COALESCE(SUM(${userStats.currentPeriodCost}), 0)` })
.from(member)
.leftJoin(userStats, eq(member.userId, userStats.userId))
.where(eq(member.organizationId, sub.referenceId))
.limit(1)
const orgUsageBeforeNum = Number.parseFloat(String(orgUsageBefore ?? '0'))
await this.updateUserStats(
updatedLog.workflowId,
costSummary,
updatedLog.trigger as ExecutionTrigger['type'],
executionId,
billingUserId
)
const percentBefore =
orgLimit > 0 ? Math.min(100, (orgUsageBeforeNum / orgLimit) * 100) : 0
const percentAfter =
orgLimit > 0
? Math.min(100, percentBefore + (costDelta / orgLimit) * 100)
: percentBefore
const currentUsageAfter = orgUsageBeforeNum + costDelta
await maybeSendUsageThresholdEmail({
scope: 'organization',
organizationId: sub.referenceId,
planName,
percentBefore,
percentAfter,
currentUsageAfter,
limit: orgLimit,
})
if (isOrgScopedSubscription(sub, usr.id) && sub?.referenceId) {
const { limit: orgLimit } = await getOrgUsageLimit(sub.referenceId, sub.plan, sub.seats)
const [{ sum: orgBaselineSum }] = await db
.select({ sum: sql`COALESCE(SUM(${userStats.currentPeriodCost}), 0)` })
.from(member)
.leftJoin(userStats, eq(member.userId, userStats.userId))
.where(eq(member.organizationId, sub.referenceId))
.limit(1)
// currentPeriodCost is only a baseline; add the org's attributed
// usage_log for the period so the threshold email reflects real usage.
const { getBillingPeriodUsageCost } = await import('@/lib/billing/core/usage-log')
const orgLedger =
sub.periodStart && sub.periodEnd
? await getBillingPeriodUsageCost(
{ type: 'organization', id: sub.referenceId },
{ start: sub.periodStart, end: sub.periodEnd }
)
: 0
emailContext = {
scope: 'organization',
organizationId: sub.referenceId,
planName,
orgLimit,
orgUsageBefore: Number.parseFloat(String(orgBaselineSum ?? '0')) + orgLedger,
}
} else {
await this.updateUserStats(
updatedLog.workflowId,
costSummary,
updatedLog.trigger as ExecutionTrigger['type'],
executionId,
billingUserId
)
emailContext = {
scope: 'user',
userId: usr.id,
userEmail: usr.email,
userName: usr.name,
planName,
before: await checkUsageStatus(usr.id),
}
}
} else {
await this.updateUserStats(
updatedLog.workflowId,
costSummary,
updatedLog.trigger as ExecutionTrigger['type'],
executionId,
billingUserId
)
}
// Record usage exactly once for every path. costDelta is the amount
// actually recorded at this boundary (the increment), not the cumulative
// run total — so resumed runs don't double-count pre-pause cost below.
const costDelta = await this.recordExecutionUsage(
updatedLog.workflowId,
costSummary,
updatedLog.trigger as ExecutionTrigger['type'],
executionId,
billingUserId,
billingContext
)
// Best-effort usage-threshold email.
if (emailContext?.scope === 'user') {
const limit = emailContext.before.usageData.limit
const percentBefore = emailContext.before.usageData.percentUsed
const percentAfter =
limit > 0 ? Math.min(100, percentBefore + (costDelta / limit) * 100) : percentBefore
const currentUsageAfter = emailContext.before.usageData.currentUsage + costDelta
await maybeSendUsageThresholdEmail({
scope: 'user',
userId: emailContext.userId,
userEmail: emailContext.userEmail,
userName: emailContext.userName || undefined,
planName: emailContext.planName,
percentBefore,
percentAfter,
currentUsageAfter,
limit,
})
} else if (emailContext?.scope === 'organization') {
const { orgLimit, orgUsageBefore } = emailContext
const percentBefore = orgLimit > 0 ? Math.min(100, (orgUsageBefore / orgLimit) * 100) : 0
const percentAfter =
orgLimit > 0 ? Math.min(100, percentBefore + (costDelta / orgLimit) * 100) : percentBefore
const currentUsageAfter = orgUsageBefore + costDelta
await maybeSendUsageThresholdEmail({
scope: 'organization',
organizationId: emailContext.organizationId,
planName: emailContext.planName,
percentBefore,
percentAfter,
currentUsageAfter,
limit: orgLimit,
})
}
} catch (e) {
// Safety net: if a step above threw BEFORE the single record call, ensure
// the run is still billed. Reconciliation is idempotent, so re-recording
// after a successful call is a no-op.
try {
await this.updateUserStats(
await this.recordExecutionUsage(
updatedLog.workflowId,
costSummary,
updatedLog.trigger as ExecutionTrigger['type'],
@@ -902,8 +963,13 @@ export class ExecutionLogger implements IExecutionLoggerService {
startedAt: updatedLog.startedAt.toISOString(),
endedAt: updatedLog.endedAt?.toISOString() || endedAt,
totalDurationMs: updatedLog.totalDurationMs || totalDurationMs,
executionData: updatedLog.executionData as WorkflowExecutionLog['executionData'],
cost: updatedLog.cost as WorkflowExecutionLog['cost'],
// Return the full in-memory execution data (cost-stripped, with traceSpans
// and finalOutput), not the slim externalized row — downstream consumers
// (notification delivery, events) need the complete payload without an
// extra storage round-trip.
executionData: completedExecutionData as WorkflowExecutionLog['executionData'],
// From the in-memory cost summary (not the deprecated cost jsonb column).
cost: executionCost as WorkflowExecutionLog['cost'],
createdAt: updatedLog.createdAt.toISOString(),
}
@@ -934,7 +1000,10 @@ export class ExecutionLogger implements IExecutionLoggerService {
endedAt: workflowLog.endedAt?.toISOString() || workflowLog.startedAt.toISOString(),
totalDurationMs: workflowLog.totalDurationMs || 0,
executionData: workflowLog.executionData as WorkflowExecutionLog['executionData'],
cost: workflowLog.cost as WorkflowExecutionLog['cost'],
// cost_total projection of the usage_log ledger (not the deprecated jsonb).
cost: (workflowLog.costTotal != null
? { total: Number(workflowLog.costTotal) }
: null) as WorkflowExecutionLog['cost'],
createdAt: workflowLog.createdAt.toISOString(),
}
}
@@ -959,7 +1028,7 @@ export class ExecutionLogger implements IExecutionLoggerService {
return trimmedUserId.length > 0 ? trimmedUserId : null
}
private async updateUserStats(
private async recordExecutionUsage(
workflowId: string | null,
costSummary: {
totalCost: number
@@ -969,7 +1038,6 @@ export class ExecutionLogger implements IExecutionLoggerService {
totalPromptTokens: number
totalCompletionTokens: number
baseExecutionCharge: number
modelCost: number
models?: Record<
string,
{
@@ -980,28 +1048,31 @@ export class ExecutionLogger implements IExecutionLoggerService {
tokens: { input: number; output: number; total: number }
}
>
charges?: Record<string, { total: number }>
},
trigger: ExecutionTrigger['type'],
executionId?: string,
billingUserId?: string | null
): Promise<void> {
billingUserId?: string | null,
// Pre-resolved billing context. The completion path already fetches the
// subscription for usage-threshold emails; passing the derived context here
// lets recordUsage skip a redundant subscription lookup per completion.
billingContext?: BillingContext
): Promise<number> {
const statsLog = logger.withMetadata({ workflowId: workflowId ?? undefined, executionId })
if (!isBillingEnabled) {
statsLog.debug('Billing is disabled, skipping user stats cost update')
return
}
if (costSummary.totalCost <= 0) {
statsLog.debug('No cost to update in user stats')
return
}
// The usage ledger (recordUsage below) is written regardless of
// BILLING_ENABLED so cost is available everywhere (incl. self-hosted).
// Only enforcement (overage/Stripe) is gated on the flag.
// Returns the amount actually recorded at THIS boundary (the increment), so
// callers drive usage-threshold math off the delta rather than the
// cumulative run total (which would double-count pre-pause cost on resume).
if (!workflowId) {
statsLog.debug('Workflow was deleted, skipping user stats update')
return
statsLog.debug('Workflow was deleted, skipping usage recording')
return 0
}
let recordedIncrement = 0
try {
const [workflowRecord] = await db
.select()
@@ -1010,43 +1081,47 @@ export class ExecutionLogger implements IExecutionLoggerService {
.limit(1)
if (!workflowRecord) {
statsLog.error('Workflow not found for user stats update')
return
statsLog.error('Workflow not found for usage recording')
return 0
}
const userId = billingUserId?.trim() || null
if (!userId) {
statsLog.error('Missing billing actor in execution context; skipping stats update', {
statsLog.error('Missing billing actor in execution context; skipping usage recording', {
trigger,
})
return
return 0
}
const entries: Array<{
category: 'model' | 'fixed'
source: 'workflow'
// Build the run's *cumulative* target ledger lines from the cost summary.
// The usage_log is then reconciled to these targets: at each completion
// boundary (pause or terminal) we record only the increment versus what
// is already billed for this execution. This bills the full run exactly
// once across pause/resume without double-charging on resume, and keeps
// pre-pause work billed even if the run is later abandoned.
type TargetLine = {
category: 'model' | 'fixed' | 'tool'
description: string
cost: number
target: number
metadata?: ModelUsageMetadata | null
}> = []
}
const targets: TargetLine[] = []
if (costSummary.baseExecutionCharge > 0) {
entries.push({
targets.push({
category: 'fixed',
source: 'workflow',
description: 'execution_fee',
cost: costSummary.baseExecutionCharge,
target: costSummary.baseExecutionCharge,
})
}
if (costSummary.models) {
for (const [modelName, modelData] of Object.entries(costSummary.models)) {
if (modelData.total > 0) {
entries.push({
targets.push({
category: 'model',
source: 'workflow',
description: modelName,
cost: modelData.total,
target: modelData.total,
metadata: {
inputTokens: modelData.tokens.input,
outputTokens: modelData.tokens.output,
@@ -1058,23 +1133,173 @@ export class ExecutionLogger implements IExecutionLoggerService {
}
}
await recordUsage({
userId,
entries,
workspaceId: workflowRecord.workspaceId ?? undefined,
workflowId,
executionId,
})
// Non-model billable charges (standalone hosted-key tool/integration
// blocks). These derive from already-gated span costs in
// calculateCostSummary — BYOK'd tools produce no cost upstream, so they
// never create a row here. Recording them closes the standalone-tool gap
// so the ledger fully reconciles with the run total (no double charge:
// agent-embedded tool cost stays folded into its model row).
if (costSummary.charges) {
for (const [description, charge] of Object.entries(costSummary.charges)) {
if (charge.total > 0) {
targets.push({ category: 'tool', description, target: charge.total })
}
}
}
// Check if user has hit overage threshold and bill incrementally
await checkAndBillOverageThreshold(userId)
if (targets.length === 0) {
statsLog.debug('No cost to record')
return 0
}
// Matches the billedBefore key resolution (toFixed(8)): a delta below this
// is finer than the idempotency key can distinguish across boundaries, so
// ignoring it keeps the key and the gate consistent.
const COST_EPSILON = 1e-8
// Build the positive-increment ledger entries for a given already-billed
// snapshot. The eventKey is scoped by the already-billed-so-far amount so
// increments across boundaries never collide, while a retried boundary
// (same already-billed) dedups via onConflictDoNothing.
//
// Reconciliation keys on `description` (model name / tool name), which is
// the billing identity — DO NOT normalize or relabel it. Correctness
// across pause/resume relies on the same usage carrying the same
// description at every boundary; the paused snapshot retains each block's
// original model label, so pre-pause cost stays under its original key
// (delta 0 at terminal) and only genuinely new usage is charged. A future
// change that relabels historical spans would break this invariant.
const buildDeltaEntries = (alreadyBilled: Map<string, number>) => {
const entries: Array<{
category: 'model' | 'fixed' | 'tool'
source: 'workflow'
description: string
cost: number
eventKey: string
metadata?: ModelUsageMetadata | null
}> = []
for (const line of targets) {
const billed = alreadyBilled.get(`${line.category}::${line.description}`) ?? 0
const delta = line.target - billed
if (delta <= COST_EPSILON) continue
entries.push({
category: line.category,
source: 'workflow',
description: line.description,
cost: delta,
eventKey: stableEventKey({
executionId: executionId ?? '',
category: line.category,
description: line.description,
billedBefore: billed.toFixed(8),
}),
...(line.metadata !== undefined ? { metadata: line.metadata } : {}),
})
}
return entries
}
if (executionId) {
// Serialize concurrent completion boundaries for this execution so the
// read-then-insert reconciliation cannot race. pg_advisory_xact_lock is
// transaction-scoped (auto-released on commit/rollback, pool-safe) and
// bounded by lock_timeout. The critical section is one SELECT + one
// INSERT; the lock is uncontended in the normal (already-serialized)
// flow and only matters under a cross-process double-completion of the
// same execution, where it stops a stale already-billed read from
// dropping the larger delta.
await db.transaction(async (tx) => {
await tx.execute(
sql`select set_config('lock_timeout', ${`${USAGE_RECONCILE_LOCK_TIMEOUT_MS}ms`}, true)`
)
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${executionId}, 0))`)
// Already-billed for this execution, scoped to the rows this path owns
// (source='workflow') so a same-executionId row from another source
// can't suppress a charge.
const billedRows = await tx
.select({
category: usageLog.category,
description: usageLog.description,
cost: sql<string>`COALESCE(SUM(${usageLog.cost}), 0)`,
})
.from(usageLog)
.where(and(eq(usageLog.executionId, executionId), eq(usageLog.source, 'workflow')))
.groupBy(usageLog.category, usageLog.description)
const alreadyBilled = new Map<string, number>()
for (const row of billedRows) {
alreadyBilled.set(
`${row.category}::${row.description}`,
Number.parseFloat(row.cost ?? '0')
)
}
const entries = buildDeltaEntries(alreadyBilled)
if (entries.length > 0) {
await recordUsage({
userId,
entries,
workspaceId: workflowRecord.workspaceId ?? undefined,
workflowId,
executionId,
tx,
...(billingContext ?? {}),
})
recordedIncrement = entries.reduce((acc, e) => acc + e.cost, 0)
// Refine cost_total to the EXACT post-reconciliation ledger sum,
// inside the same advisory-locked tx so it is atomic with the inserts
// and can't be clobbered by a concurrent boundary. Exact by
// construction: under the lock no delta collides, so the new sum is
// the prior workflow-source sum plus the deltas just inserted. This
// supersedes the main-transaction GREATEST baseline (which remains for
// early-return / no-executionId / failed-reconcile paths).
const ledgerSum =
[...alreadyBilled.values()].reduce((acc, v) => acc + v, 0) + recordedIncrement
await tx
.update(workflowExecutionLogs)
.set({ costTotal: ledgerSum.toString() })
.where(eq(workflowExecutionLogs.executionId, executionId))
}
})
} else {
// No execution scope to reconcile/lock against (not expected at a
// workflow completion): record the full targets directly.
const entries = buildDeltaEntries(new Map())
if (entries.length > 0) {
await recordUsage({
userId,
entries,
workspaceId: workflowRecord.workspaceId ?? undefined,
workflowId,
...(billingContext ?? {}),
})
recordedIncrement = entries.reduce((acc, e) => acc + e.cost, 0)
}
}
// Enforcement only when billing is enabled: the ledger above is always
// written, but overage/Stripe billing is gated on BILLING_ENABLED.
if (isBillingEnabled) {
await checkAndBillOverageThreshold(userId)
}
} catch (error) {
statsLog.error('Error updating user stats with cost information', {
error,
costSummary,
})
// Don't throw - we want execution to continue even if user stats update fails
// Swallowed so a billing-write failure never fails the execution. The
// reconciliation self-heals on a later boundary; a TERMINAL-boundary
// failure leaves the run under-billed (and cost_total may then exceed
// SUM(usage_log)), so log loudly enough to alert / reconcile out of band.
statsLog.error(
'Failed to record execution usage to usage_log ledger; charge may be unbilled',
{
error,
billingUserId,
costSummary,
}
)
}
return recordedIncrement
}
/**
@@ -150,7 +150,6 @@ describe('calculateCostSummary', () => {
expect(result.totalCost).toBe(BASE_EXECUTION_CHARGE)
expect(result.baseExecutionCharge).toBe(BASE_EXECUTION_CHARGE)
expect(result.modelCost).toBe(0)
expect(result.totalInputCost).toBe(0)
expect(result.totalOutputCost).toBe(0)
expect(result.totalTokens).toBe(0)
@@ -188,7 +187,6 @@ describe('calculateCostSummary', () => {
const result = calculateCostSummary(traceSpans)
expect(result.totalCost).toBe(0.03 + BASE_EXECUTION_CHARGE)
expect(result.modelCost).toBe(0.03)
expect(result.totalInputCost).toBe(0.01)
expect(result.totalOutputCost).toBe(0.02)
expect(result.totalTokens).toBe(300)
@@ -221,7 +219,6 @@ describe('calculateCostSummary', () => {
const result = calculateCostSummary(traceSpans)
expect(result.totalCost).toBe(0.033 + BASE_EXECUTION_CHARGE)
expect(result.modelCost).toBe(0.033)
expect(result.totalInputCost).toBe(0.011)
expect(result.totalOutputCost).toBe(0.022)
expect(result.totalTokens).toBe(450)
@@ -280,7 +277,6 @@ describe('calculateCostSummary', () => {
const result = calculateCostSummary(traceSpans)
expect(result.modelCost).toBe(0.03)
expect(result.totalCost).toBe(0.03 + BASE_EXECUTION_CHARGE)
expect(result.models['claude-3']).toBeDefined()
expect(result.models['claude-3'].total).toBe(0.03)
@@ -308,7 +304,6 @@ describe('calculateCostSummary', () => {
const result = calculateCostSummary(traceSpans)
expect(result.modelCost).toBe(0.03)
expect(result.models['gpt-4']).toBeDefined()
})
@@ -345,7 +340,6 @@ describe('calculateCostSummary', () => {
const result = calculateCostSummary(traceSpans)
expect(result.modelCost).toBe(0.03)
expect(Object.keys(result.models)).toHaveLength(1)
})
@@ -361,7 +355,6 @@ describe('calculateCostSummary', () => {
const result = calculateCostSummary(traceSpans)
expect(result.modelCost).toBe(0.03)
expect(result.totalCost).toBe(0.03 + BASE_EXECUTION_CHARGE)
// Should not add to models if model is not specified
expect(Object.keys(result.models)).toHaveLength(0)
@@ -427,7 +420,6 @@ describe('calculateCostSummary', () => {
const result = calculateCostSummary(traceSpans)
expect(result.modelCost).toBe(0)
expect(result.totalCost).toBe(BASE_EXECUTION_CHARGE)
// Model is still tracked for token-usage display, but cost must be zero.
expect(result.models['claude-opus-4-6'].total).toBe(0)
@@ -462,7 +454,6 @@ describe('calculateCostSummary', () => {
const result = calculateCostSummary(traceSpans)
expect(result.modelCost).toBe(0.03)
expect(result.totalCost).toBe(0.03 + BASE_EXECUTION_CHARGE)
expect(result.models['gpt-4o'].total).toBe(0.03)
})
@@ -494,9 +485,95 @@ describe('calculateCostSummary', () => {
const result = calculateCostSummary(traceSpans)
expect(result.modelCost).toBe(0.045)
expect(result.totalCost).toBe(0.045 + BASE_EXECUTION_CHARGE)
expect(result.models['gpt-4o'].total).toBe(0.045)
expect(result.models['gpt-4o'].toolCost).toBe(0.015)
})
test('records a standalone non-model billable span as a charge (closes the tool gap)', () => {
const traceSpans = [
{
id: 'exa-block',
name: 'Exa Search',
type: 'tool',
cost: { input: 0, output: 0, total: 0.01 },
},
]
const result = calculateCostSummary(traceSpans)
expect(result.charges['Exa Search']).toBeDefined()
expect(result.charges['Exa Search'].total).toBe(0.01)
expect(Object.keys(result.models)).toHaveLength(0)
// Ledger partition reconciles with the run total.
const ledgerSum =
result.baseExecutionCharge +
Object.values(result.models).reduce((s, m) => s + m.total, 0) +
Object.values(result.charges).reduce((s, c) => s + c.total, 0)
expect(ledgerSum).toBeCloseTo(result.totalCost, 10)
})
test('does not double-count: agent-embedded tool stays in the model row, not charges', () => {
const traceSpans = [
{
id: 'agent-span',
name: 'Agent',
type: 'agent',
model: 'gpt-4o',
cost: { input: 0.01, output: 0.02, total: 0.045, toolCost: 0.015 },
tokens: { input: 1000, output: 2000, total: 3000 },
},
]
const result = calculateCostSummary(traceSpans)
expect(Object.keys(result.charges)).toHaveLength(0)
expect(result.models['gpt-4o'].total).toBe(0.045)
expect(result.models['gpt-4o'].toolCost).toBe(0.015)
})
test('mixed model + standalone tool run reconciles to total', () => {
const traceSpans = [
{
id: 'agent',
name: 'Agent',
type: 'agent',
model: 'gpt-4o',
cost: { input: 0.01, output: 0.02, total: 0.03 },
tokens: { input: 100, output: 200, total: 300 },
},
{
id: 'exa',
name: 'Exa Search',
type: 'tool',
cost: { input: 0, output: 0, total: 0.01 },
},
]
const result = calculateCostSummary(traceSpans)
expect(result.models['gpt-4o'].total).toBe(0.03)
expect(result.charges['Exa Search'].total).toBe(0.01)
const ledgerSum =
result.baseExecutionCharge +
Object.values(result.models).reduce((s, m) => s + m.total, 0) +
Object.values(result.charges).reduce((s, c) => s + c.total, 0)
expect(ledgerSum).toBeCloseTo(result.totalCost, 10)
})
test('BYOK tool (no cost generated upstream) produces no charge row', () => {
const traceSpans = [
{
id: 'exa-byok',
name: 'Exa Search',
type: 'tool',
cost: { input: 0, output: 0, total: 0 },
},
]
const result = calculateCostSummary(traceSpans)
expect(Object.keys(result.charges)).toHaveLength(0)
expect(result.totalCost).toBe(BASE_EXECUTION_CHARGE)
})
})
+47 -33
View File
@@ -87,9 +87,41 @@ export async function loadDeployedWorkflowStateForLogging(
type CostTraceSpan = Pick<TraceSpan, 'cost' | 'model' | 'tokens'> & {
type?: TraceSpan['type']
name?: TraceSpan['name']
children?: CostTraceSpan[]
}
export interface CostSummaryModel {
input: number
output: number
total: number
toolCost?: number
tokens: { input: number; output: number; total: number }
}
/**
* Non-model billable charge (e.g. a standalone hosted-key tool block such as
* Exa/Tavily/falai run outside an agent). These spans contribute to the run's
* total cost but carry no `model`, so they live here rather than in `models`.
* Summed per span name so the ledger has one row per integration.
*/
export interface CostSummaryCharge {
total: number
}
export interface CostSummary {
totalCost: number
totalInputCost: number
totalOutputCost: number
totalTokens: number
totalPromptTokens: number
totalCompletionTokens: number
baseExecutionCharge: number
models: Record<string, CostSummaryModel>
/** Non-model billable charges keyed by span name (tool/integration costs). */
charges: Record<string, CostSummaryCharge>
}
type BillableTraceSpan = CostTraceSpan & { cost: NonNullable<TraceSpan['cost']> }
function hasBillableCost(span: CostTraceSpan): span is BillableTraceSpan {
@@ -100,26 +132,7 @@ function isModelBreakdownSpan(span: CostTraceSpan): boolean {
return span.type === 'model'
}
export function calculateCostSummary(traceSpans: CostTraceSpan[] | undefined): {
totalCost: number
totalInputCost: number
totalOutputCost: number
totalTokens: number
totalPromptTokens: number
totalCompletionTokens: number
baseExecutionCharge: number
modelCost: number
models: Record<
string,
{
input: number
output: number
total: number
toolCost?: number
tokens: { input: number; output: number; total: number }
}
>
} {
export function calculateCostSummary(traceSpans: CostTraceSpan[] | undefined): CostSummary {
if (!traceSpans || traceSpans.length === 0) {
return {
totalCost: BASE_EXECUTION_CHARGE,
@@ -129,8 +142,8 @@ export function calculateCostSummary(traceSpans: CostTraceSpan[] | undefined): {
totalPromptTokens: 0,
totalCompletionTokens: 0,
baseExecutionCharge: BASE_EXECUTION_CHARGE,
modelCost: 0,
models: {},
charges: {},
}
}
@@ -181,16 +194,8 @@ export function calculateCostSummary(traceSpans: CostTraceSpan[] | undefined): {
let totalTokens = 0
let totalPromptTokens = 0
let totalCompletionTokens = 0
const models: Record<
string,
{
input: number
output: number
total: number
toolCost?: number
tokens: { input: number; output: number; total: number }
}
> = {}
const models: Record<string, CostSummaryModel> = {}
const charges: Record<string, CostSummaryCharge> = {}
for (const span of costSpans) {
totalCost += span.cost.total || 0
@@ -220,10 +225,19 @@ export function calculateCostSummary(traceSpans: CostTraceSpan[] | undefined): {
if (span.cost.toolCost) {
models[model].toolCost = (models[model].toolCost || 0) + span.cost.toolCost
}
} else if ((span.cost.total || 0) > 0) {
// Non-model billable span (e.g. a standalone hosted-key tool block).
// These previously contributed to the run total but were never itemized
// in the ledger (the "standalone tool gap"). Key by span name so each
// integration gets a single, reconciling charge row.
const description = span.name || span.type || 'tool'
if (!charges[description]) {
charges[description] = { total: 0 }
}
charges[description].total += span.cost.total || 0
}
}
const modelCost = totalCost
totalCost += BASE_EXECUTION_CHARGE
return {
@@ -234,7 +248,7 @@ export function calculateCostSummary(traceSpans: CostTraceSpan[] | undefined): {
totalPromptTokens,
totalCompletionTokens,
baseExecutionCharge: BASE_EXECUTION_CHARGE,
modelCost,
models,
charges,
}
}
@@ -75,7 +75,6 @@ vi.mock('@/lib/logs/execution/logging-factory', () => ({
totalPromptTokens: 0,
totalCompletionTokens: 0,
baseExecutionCharge: 0,
modelCost: 0,
models: {},
}),
createEnvironmentObject: vi.fn(),
@@ -84,6 +83,7 @@ vi.mock('@/lib/logs/execution/logging-factory', () => ({
loadWorkflowStateForExecution: loadWorkflowStateForExecutionMock,
}))
import { calculateCostSummary } from '@/lib/logs/execution/logging-factory'
import { LoggingSession } from './logging-session'
describe('LoggingSession start snapshots', () => {
@@ -238,34 +238,54 @@ describe('LoggingSession completion retries', () => {
)
})
it('preserves accumulated cost during fallback completion', async () => {
it('derives fallback cost from trace spans when the primary completion fails', async () => {
const session = new LoggingSession('workflow-1', 'execution-6', 'api', 'req-1') as any
session.accumulatedCost = {
total: 12,
input: 5,
output: 7,
tokens: { input: 11, output: 13, total: 24 },
models: {
'test-model': {
input: 5,
output: 7,
total: 12,
tokens: { input: 11, output: 13, total: 24 },
},
},
// Resume-accumulation is retired: the cost-only fallback now derives its
// cost summary from the in-memory trace spans (billing itself reconciles
// from the usage_log ledger in recordExecutionUsage). The primary complete()
// path consumes one calculateCostSummary call before it fails, so queue the
// same value twice (primary attempt + fallback).
const spanCostSummary = {
totalCost: 12,
totalInputCost: 5,
totalOutputCost: 7,
totalTokens: 24,
totalPromptTokens: 11,
totalCompletionTokens: 13,
baseExecutionCharge: 0,
models: {},
charges: {},
}
session.costFlushed = true
vi.mocked(calculateCostSummary)
.mockReturnValueOnce(spanCostSummary)
.mockReturnValueOnce(spanCostSummary)
completeWorkflowExecutionMock
.mockRejectedValueOnce(new Error('success finalize failed'))
.mockResolvedValueOnce({})
await expect(session.safeComplete({ finalOutput: { ok: true } })).resolves.toBeUndefined()
const traceSpans = [
{
id: 'span-1',
name: 'Block A',
type: 'tool',
duration: 25,
startTime: '2026-03-13T10:00:00.000Z',
endTime: '2026-03-13T10:00:00.025Z',
status: 'success',
},
] as any
await expect(
session.safeComplete({ finalOutput: { ok: true }, traceSpans })
).resolves.toBeUndefined()
expect(calculateCostSummary).toHaveBeenLastCalledWith(traceSpans)
expect(completeWorkflowExecutionMock).toHaveBeenLastCalledWith(
expect.objectContaining({
executionId: 'execution-6',
finalizationPath: 'fallback_completed',
costSummary: expect.objectContaining({
totalCost: 12,
totalInputCost: 5,
@@ -440,24 +460,21 @@ describe('LoggingSession completion retries', () => {
expect(session.complete).toHaveBeenCalledTimes(1)
})
it('drains fire-and-forget cost flushes before terminal completion', async () => {
let releaseFlush: (() => void) | undefined
const flushPromise = new Promise<void>((resolve) => {
releaseFlush = resolve
it('drains fire-and-forget block-complete marker writes before terminal completion', async () => {
let releasePersist: (() => void) | undefined
const persistPromise = new Promise<void>((resolve) => {
releasePersist = resolve
})
const session = new LoggingSession('workflow-1', 'execution-1', 'api', 'req-1') as any
session.flushAccumulatedCost = vi.fn(() => flushPromise)
session.persistLastCompletedBlock = vi.fn(() => persistPromise)
session.complete = vi.fn().mockResolvedValue(undefined)
await session.onBlockComplete('block-2', 'Transform', 'function', {
// onBlockComplete is now marker-only; its marker write is fire-and-forget
// but tracked, so terminal completion must drain it first.
void session.onBlockComplete('block-2', 'Transform', 'function', {
endedAt: '2025-01-01T00:00:01.000Z',
output: {
value: true,
cost: { total: 1, input: 1, output: 0 },
tokens: { input: 1, output: 0, total: 1 },
model: 'test-model',
},
output: { value: true },
})
const completionPromise = session.safeComplete({ finalOutput: { ok: true } })
@@ -466,11 +483,11 @@ describe('LoggingSession completion retries', () => {
expect(session.complete).not.toHaveBeenCalled()
releaseFlush?.()
releasePersist?.()
await completionPromise
expect(session.flushAccumulatedCost).toHaveBeenCalledTimes(1)
expect(session.persistLastCompletedBlock).toHaveBeenCalledTimes(1)
expect(session.complete).toHaveBeenCalledTimes(1)
})
+24 -208
View File
@@ -3,7 +3,6 @@ import { workflowExecutionLogs } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { and, eq, sql } from 'drizzle-orm'
import { BASE_EXECUTION_CHARGE } from '@/lib/billing/constants'
import { executionLogger } from '@/lib/logs/execution/logger'
import {
calculateCostSummary,
@@ -118,22 +117,6 @@ export interface SessionPausedParams {
workflowInput?: any
}
interface AccumulatedCost {
total: number
input: number
output: number
tokens: { input: number; output: number; total: number }
models: Record<
string,
{
input: number
output: number
total: number
tokens: { input: number; output: number; total: number }
}
>
}
export class LoggingSession {
private workflowId: string
private executionId: string
@@ -151,15 +134,7 @@ export class LoggingSession {
private completionPromise: Promise<void> | null = null
private completionAttempt: CompletionAttempt | null = null
private completionAttemptFailed = false
private accumulatedCost: AccumulatedCost = {
total: BASE_EXECUTION_CHARGE,
input: 0,
output: 0,
tokens: { input: 0, output: 0, total: 0 },
models: {},
}
private pendingProgressWrites = new Set<Promise<void>>()
private costFlushed = false
private postExecutionPromise: Promise<void> | null = null
constructor(
@@ -249,7 +224,6 @@ export class LoggingSession {
totalPromptTokens: number
totalCompletionTokens: number
baseExecutionCharge: number
modelCost: number
models: Record<
string,
{
@@ -259,6 +233,9 @@ export class LoggingSession {
tokens: { input: number; output: number; total: number }
}
>
// Non-model billable charges (standalone tool/integration costs). Carried
// through so the partition can't be silently dropped at this boundary.
charges?: Record<string, { total: number }>
}
finalOutput: Record<string, unknown>
traceSpans: TraceSpan[]
@@ -292,46 +269,9 @@ export class LoggingSession {
blockType: string,
output: any
): Promise<void> {
// Accumulate cost synchronously before any await so that fire-and-forget
// callers still capture the full cost even if DB writes are not awaited.
const blockOutput = output?.output
if (
blockOutput?.cost &&
typeof blockOutput.cost.total === 'number' &&
blockOutput.cost.total > 0
) {
const { cost, tokens, model } = blockOutput
this.accumulatedCost.total += cost.total || 0
this.accumulatedCost.input += cost.input || 0
this.accumulatedCost.output += cost.output || 0
if (tokens) {
this.accumulatedCost.tokens.input += tokens.input || 0
this.accumulatedCost.tokens.output += tokens.output || 0
this.accumulatedCost.tokens.total += tokens.total || 0
}
if (model) {
if (!this.accumulatedCost.models[model]) {
this.accumulatedCost.models[model] = {
input: 0,
output: 0,
total: 0,
tokens: { input: 0, output: 0, total: 0 },
}
}
this.accumulatedCost.models[model].input += cost.input || 0
this.accumulatedCost.models[model].output += cost.output || 0
this.accumulatedCost.models[model].total += cost.total || 0
if (tokens) {
this.accumulatedCost.models[model].tokens.input += tokens.input || 0
this.accumulatedCost.models[model].tokens.output += tokens.output || 0
this.accumulatedCost.models[model].tokens.total += tokens.total || 0
}
}
}
// Cost is recorded into the usage_log ledger and reconciled at completion
// boundaries (see recordExecutionUsage); onBlockComplete only persists the
// last-completed-block progress marker.
await this.trackProgressWrite(
this.persistLastCompletedBlock({
blockId,
@@ -341,76 +281,6 @@ export class LoggingSession {
success: !output?.output?.error,
})
)
if (
blockOutput?.cost &&
typeof blockOutput.cost.total === 'number' &&
blockOutput.cost.total > 0
) {
void this.trackProgressWrite(this.flushAccumulatedCost())
}
}
private async flushAccumulatedCost(): Promise<void> {
try {
await db
.update(workflowExecutionLogs)
.set({
cost: {
total: this.accumulatedCost.total,
input: this.accumulatedCost.input,
output: this.accumulatedCost.output,
tokens: this.accumulatedCost.tokens,
models: this.accumulatedCost.models,
},
})
.where(
and(
eq(workflowExecutionLogs.workflowId, this.workflowId),
eq(workflowExecutionLogs.executionId, this.executionId)
)
)
this.costFlushed = true
} catch (error) {
logger.error(`Failed to flush accumulated cost for execution ${this.executionId}:`, {
error: toError(error).message,
})
}
}
private async loadExistingCost(): Promise<void> {
try {
const [existing] = await db
.select({ cost: workflowExecutionLogs.cost })
.from(workflowExecutionLogs)
.where(
and(
eq(workflowExecutionLogs.workflowId, this.workflowId),
eq(workflowExecutionLogs.executionId, this.executionId)
)
)
.limit(1)
if (existing?.cost) {
const cost = existing.cost as AccumulatedCost
this.accumulatedCost = {
total: cost.total || BASE_EXECUTION_CHARGE,
input: cost.input || 0,
output: cost.output || 0,
tokens: {
input: cost.tokens?.input || 0,
output: cost.tokens?.output || 0,
total: cost.tokens?.total || 0,
},
models: cost.models || {},
}
}
} catch (error) {
logger.error(`Failed to load existing cost for execution ${this.executionId}:`, {
error: toError(error).message,
})
}
}
async start(params: SessionStartParams): Promise<void> {
@@ -451,8 +321,9 @@ export class LoggingSession {
deploymentVersionId,
})
} else {
// Resume: no cost reload needed. Billing reconciles from the usage_log
// ledger (pre-pause rows already exist) plus the live cost summary.
this.isResume = true
await this.loadExistingCost()
}
} catch (error) {
if (this.requestId) {
@@ -577,6 +448,8 @@ export class LoggingSession {
const hasProvidedSpans = Array.isArray(traceSpans) && traceSpans.length > 0
// calculateCostSummary([]) / (undefined) already returns the base-charge
// summary, so the no-spans branch needs no separate literal.
const costSummary = skipCost
? {
totalCost: 0,
@@ -586,22 +459,10 @@ export class LoggingSession {
totalPromptTokens: 0,
totalCompletionTokens: 0,
baseExecutionCharge: 0,
modelCost: 0,
models: {},
charges: {},
}
: hasProvidedSpans
? calculateCostSummary(traceSpans)
: {
totalCost: BASE_EXECUTION_CHARGE,
totalInputCost: 0,
totalOutputCost: 0,
totalTokens: 0,
totalPromptTokens: 0,
totalCompletionTokens: 0,
baseExecutionCharge: BASE_EXECUTION_CHARGE,
modelCost: 0,
models: {},
}
: calculateCostSummary(traceSpans)
const message = error?.message || 'Run failed before starting blocks'
@@ -710,19 +571,9 @@ export class LoggingSession {
return
}
const costSummary = traceSpans?.length
? calculateCostSummary(traceSpans)
: {
totalCost: BASE_EXECUTION_CHARGE,
totalInputCost: 0,
totalOutputCost: 0,
totalTokens: 0,
totalPromptTokens: 0,
totalCompletionTokens: 0,
baseExecutionCharge: BASE_EXECUTION_CHARGE,
modelCost: 0,
models: {},
}
// calculateCostSummary handles empty/undefined spans by returning the
// base-charge summary, so no separate no-spans literal is needed.
const costSummary = calculateCostSummary(traceSpans)
await this.completeExecutionWithFinalization({
endedAt: endTime.toISOString(),
@@ -814,19 +665,9 @@ export class LoggingSession {
return
}
const costSummary = traceSpans?.length
? calculateCostSummary(traceSpans)
: {
totalCost: BASE_EXECUTION_CHARGE,
totalInputCost: 0,
totalOutputCost: 0,
totalTokens: 0,
totalPromptTokens: 0,
totalCompletionTokens: 0,
baseExecutionCharge: BASE_EXECUTION_CHARGE,
modelCost: 0,
models: {},
}
// calculateCostSummary handles empty/undefined spans by returning the
// base-charge summary, so no separate no-spans literal is needed.
const costSummary = calculateCostSummary(traceSpans)
await this.completeExecutionWithFinalization({
endedAt: endTime.toISOString(),
@@ -1185,37 +1026,12 @@ export class LoggingSession {
)
try {
const hasAccumulatedCost =
this.costFlushed ||
this.accumulatedCost.total > BASE_EXECUTION_CHARGE ||
this.accumulatedCost.tokens.total > 0 ||
Object.keys(this.accumulatedCost.models).length > 0
const costSummary = hasAccumulatedCost
? {
totalCost: this.accumulatedCost.total,
totalInputCost: this.accumulatedCost.input,
totalOutputCost: this.accumulatedCost.output,
totalTokens: this.accumulatedCost.tokens.total,
totalPromptTokens: this.accumulatedCost.tokens.input,
totalCompletionTokens: this.accumulatedCost.tokens.output,
baseExecutionCharge: BASE_EXECUTION_CHARGE,
modelCost: Math.max(0, this.accumulatedCost.total - BASE_EXECUTION_CHARGE),
models: this.accumulatedCost.models,
}
: params.traceSpans?.length
? calculateCostSummary(params.traceSpans)
: {
totalCost: BASE_EXECUTION_CHARGE,
totalInputCost: 0,
totalOutputCost: 0,
totalTokens: 0,
totalPromptTokens: 0,
totalCompletionTokens: 0,
baseExecutionCharge: BASE_EXECUTION_CHARGE,
modelCost: 0,
models: {},
}
// Billing is reconciled from the usage_log ledger in recordExecutionUsage;
// here we only need a cost summary to compute the run total. Derive it
// from the in-memory trace spans when available (this fallback fires when
// persisting spans failed, not when computing them did), else just the
// base execution charge.
const costSummary = calculateCostSummary(params.traceSpans)
const finalOutput = params.finalOutput || { _fallback: true, error: params.errorMessage }
+179
View File
@@ -0,0 +1,179 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref'
import { materializeLargeValueRef, storeLargeValue } from '@/lib/execution/payloads/store'
const logger = createLogger('TraceStore')
/**
* Key under which the externalized-execution-data pointer (a `__simLargeValueRef`)
* is stored on the slim `execution_data` row.
*/
export const TRACE_STORE_REF_KEY = 'traceStoreRef'
/**
* The only metadata kept inline on the slim row (everything else lives in the
* externalized object). These two describe trace presence/count and uniquely
* survive object expiry — so a reader can still report "trace data expired (N
* spans)" after retention without an object fetch. All other fields
* (environment, trigger, tokens, models, truncation flags, and of course the
* heavy payloads) are in the stored object and recovered on materialize, so
* keeping them inline too would just be duplication.
*/
const INLINE_MARKER_KEYS = ['hasTraceSpans', 'traceSpanCount'] as const
/**
* Read-path context. Resolves an externalized payload by storage key, authorized
* via the (already-authorized) workspace — no owner needed.
*/
interface TraceStoreReadContext {
workspaceId: string | null
workflowId: string | null
executionId: string
}
/**
* Write-path context. Requires the execution owner's `userId`: the externalized
* object is tracked in `workspace_files`, whose `user_id` column is NOT NULL
* (FK -> user.id). Requiring it here makes "a write needs an owner" a
* compile-time invariant, so callers must resolve the owner before persisting.
*/
interface TraceStoreWriteContext extends TraceStoreReadContext {
userId: string
}
/**
* Recovers the workflowId embedded in a large-value storage key
* (`execution/{workspaceId}/{workflowId}/{executionId}/<file>`). Used when the
* log row's workflowId has been nulled by workflow deletion.
*/
function workflowIdFromStorageKey(key: string | undefined): string | undefined {
if (!key) return undefined
const parts = key.split('/')
return parts.length >= 5 && parts[0] === 'execution' ? parts[2] : undefined
}
/**
* Recursively removes `cost` from trace spans before persistence. Cost lives in
* exactly one place — the usage_log ledger — so persisted spans carry only
* structure, timing, and tokens (KTD7). Must run AFTER `calculateCostSummary`
* has consumed span costs in memory.
*/
export function stripSpanCosts(spans: unknown): void {
if (!Array.isArray(spans)) return
for (const span of spans) {
if (!span || typeof span !== 'object') continue
const record = span as { cost?: unknown; children?: unknown }
if ('cost' in record) record.cost = undefined
if (Array.isArray(record.children)) stripSpanCosts(record.children)
}
}
/**
* Externalizes heavy `execution_data` to object storage as a single large value
* (reusing the execution-context large-value store + its reference/dependency/GC
* machinery — KTD4/KTD8), returning a slim row payload that keeps inline markers
* plus the `__simLargeValueRef` pointer.
*
* On any failure (no scope, oversized, storage error) the original (already
* cost-stripped) execution data is returned unchanged so the log is never lost.
*/
export async function externalizeExecutionData(
executionData: Record<string, unknown>,
context: TraceStoreWriteContext
): Promise<Record<string, unknown>> {
const { workspaceId, workflowId, executionId, userId } = context
// workspaceId/workflowId build the storage key and can be null for
// deleted-workflow rows. userId is type-guaranteed by TraceStoreWriteContext;
// the falsy check is a defensive guard against an empty string. If any are
// missing the durable write can't succeed, so keep the data inline.
if (!workspaceId || !workflowId || !userId) return executionData
try {
const json = JSON.stringify(executionData)
const size = Buffer.byteLength(json, 'utf8')
// storeLargeValue persists to the execution bucket with a conforming key and
// registers owner + dependency closure (trace -> nested span large values),
// so GC keeps nested children alive while this run's log row exists.
const ref = await storeLargeValue(executionData, json, size, {
workspaceId,
workflowId,
executionId,
userId,
requireDurable: true,
})
const { preview: _preview, ...slimRef } = ref
const slim: Record<string, unknown> = { [TRACE_STORE_REF_KEY]: slimRef }
for (const key of INLINE_MARKER_KEYS) {
if (key in executionData) slim[key] = executionData[key]
}
return slim
} catch (error) {
logger.warn('Failed to externalize execution data; keeping inline', {
executionId,
error: toError(error).message,
})
return executionData
}
}
/**
* Resolves an `execution_data` row into its full form for reads. When the row
* carries a trace-store pointer, the payload is materialized from storage and
* merged with the inline markers; otherwise the row is returned unchanged
* (inline / pre-externalization runs). One level only — nested span
* `__simLargeValueRef` stubs remain as previews, matching prior behavior.
*
* Returns metadata-only (the slim row minus the pointer) if the object is
* missing/unreadable (e.g. post-retention) so reads degrade rather than crash.
*/
export async function materializeExecutionData(
executionData: Record<string, unknown> | null | undefined,
context: TraceStoreReadContext
): Promise<Record<string, unknown>> {
if (!executionData) return {}
const ref = executionData[TRACE_STORE_REF_KEY]
if (!isLargeValueRef(ref)) return executionData
const { [TRACE_STORE_REF_KEY]: _pointer, ...markers } = executionData
if (!context.workspaceId) return markers
// workflowId is `set null` on workflow delete, but the ref key embeds the
// original workflowId — recover it so deleted-workflow logs stay readable.
// Workspace authorization still comes from the (authorized) caller context.
const workflowId = context.workflowId ?? workflowIdFromStorageKey(ref.key)
if (!workflowId) return markers
try {
const materialized = await materializeLargeValueRef(ref, {
workspaceId: context.workspaceId,
workflowId,
executionId: context.executionId,
maxBytes: ref.size,
// Read-only: the value is already referenced by its own execution; don't
// re-register (or fail) on every view/export.
trackReference: false,
})
if (!materialized || typeof materialized !== 'object') {
logger.warn('Trace store object unavailable; returning metadata only', {
executionId: context.executionId,
key: ref.key,
})
return markers
}
return { ...(materialized as Record<string, unknown>), ...markers }
} catch (error) {
logger.warn('Failed to materialize execution data; returning metadata only', {
executionId: context.executionId,
error: toError(error).message,
})
return markers
}
}
+72 -4
View File
@@ -3,14 +3,69 @@ import {
jobExecutionLogs,
pausedExecutions,
permissions,
usageLog,
workflow,
workflowDeploymentVersion,
workflowExecutionLogs,
} from '@sim/db/schema'
import { and, eq, type SQL } from 'drizzle-orm'
import type { CostLedger } from '@/lib/api/contracts/logs'
import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
type LookupColumn = 'id' | 'executionId'
async function buildCostLedger(executionId: string): Promise<CostLedger | null> {
const rows = await db
.select({
category: usageLog.category,
description: usageLog.description,
cost: usageLog.cost,
metadata: usageLog.metadata,
})
.from(usageLog)
.where(and(eq(usageLog.executionId, executionId), eq(usageLog.source, 'workflow')))
if (rows.length === 0) return null
type LedgerItem = CostLedger['items'][number]
const byKey = new Map<string, LedgerItem>()
for (const row of rows) {
const metadata = (row.metadata ?? {}) as { inputTokens?: number; outputTokens?: number }
const category = row.category as LedgerItem['category']
const key = `${category}::${row.description}`
const existing = byKey.get(key)
if (existing) {
existing.cost += Number(row.cost)
if (typeof metadata.inputTokens === 'number') {
existing.inputTokens = Math.max(existing.inputTokens ?? 0, metadata.inputTokens)
}
if (typeof metadata.outputTokens === 'number') {
existing.outputTokens = Math.max(existing.outputTokens ?? 0, metadata.outputTokens)
}
} else {
byKey.set(key, {
category,
description: row.description,
cost: Number(row.cost),
...(typeof metadata.inputTokens === 'number' ? { inputTokens: metadata.inputTokens } : {}),
...(typeof metadata.outputTokens === 'number'
? { outputTokens: metadata.outputTokens }
: {}),
})
}
}
const items = [...byKey.values()]
const total = items.reduce((sum, item) => sum + item.cost, 0)
return { total, items }
}
export function jobCostTotal(raw: unknown): { total: number } | null {
const total = (raw as { total?: unknown } | null | undefined)?.total
const n = total == null ? Number.NaN : Number(total)
return Number.isFinite(n) ? { total: n } : null
}
interface FetchLogDetailArgs {
userId: string
workspaceId: string
@@ -47,7 +102,7 @@ export async function fetchLogDetail({
endedAt: workflowExecutionLogs.endedAt,
totalDurationMs: workflowExecutionLogs.totalDurationMs,
executionData: workflowExecutionLogs.executionData,
cost: workflowExecutionLogs.cost,
costTotal: workflowExecutionLogs.costTotal,
files: workflowExecutionLogs.files,
createdAt: workflowExecutionLogs.createdAt,
workflowName: workflow.name,
@@ -105,6 +160,18 @@ export async function fetchLogDetail({
(totalPauseCount > 0 && resumedCount < totalPauseCount) ||
(log.pausedStatus !== null && log.pausedStatus !== 'fully_resumed')
// Cost is sourced exclusively from the usage_log ledger (itemized breakdown)
// and its cost_total projection (run total). The cost jsonb is never read.
const costLedger = await buildCostLedger(log.executionId)
const totalDollars = costLedger?.total ?? (log.costTotal != null ? Number(log.costTotal) : null)
// Trace spans / heavy execution data may live in object storage; resolve the
// pointer here (no-op for inline / pre-externalization rows).
const executionData = await materializeExecutionData(
log.executionData as Record<string, unknown> | null,
{ workspaceId, workflowId: log.workflowId, executionId: log.executionId }
)
return {
id: log.id,
workflowId: log.workflowId,
@@ -119,7 +186,8 @@ export async function fetchLogDetail({
createdAt: log.startedAt.toISOString(),
workflow: workflowSummary,
jobTitle: null,
cost: log.cost ?? null,
cost: totalDollars != null ? { total: totalDollars } : null,
costLedger,
pauseSummary: {
status: log.pausedStatus ?? null,
total: totalPauseCount,
@@ -128,7 +196,7 @@ export async function fetchLogDetail({
hasPendingPause,
executionData: {
totalDuration: log.totalDurationMs,
...((log.executionData as Record<string, unknown> | null) ?? {}),
...executionData,
enhanced: true as const,
},
files: log.files ?? null,
@@ -184,7 +252,7 @@ export async function fetchLogDetail({
createdAt: jobLog.startedAt.toISOString(),
workflow: null,
jobTitle: ((execData.trigger as Record<string, unknown> | undefined)?.source as string) ?? null,
cost: jobLog.cost ?? null,
cost: jobCostTotal(jobLog.cost),
pauseSummary: { status: null, total: 0, resumed: 0 },
hasPendingPause: false,
executionData: {
+2 -1
View File
@@ -195,7 +195,8 @@ function buildSearchConditions(params: {
}
function buildCostCondition(operator: ComparisonOperator, value: number): SQL {
const costField = sql`(${workflowExecutionLogs.cost}->>'total')::numeric`
// Indexed projection of the usage_log ledger (dollars); no live aggregation.
const costField = sql`${workflowExecutionLogs.costTotal}`
switch (operator) {
case '=':
+13 -3
View File
@@ -18,8 +18,10 @@ import { createLogger } from '@sim/logger'
import { getPostgresErrorCode } from '@sim/utils/errors'
import { generateId } from '@sim/utils/id'
import { and, count, eq, gt, gte, inArray, isNull, type SQL, sql } from 'drizzle-orm'
import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency'
import { generateRestoreName } from '@/lib/core/utils/restore-name'
import type { DbOrTx } from '@/lib/db/types'
import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
import { COLUMN_TYPES, NAME_PATTERN, TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from './constants'
import { buildFilterClause, buildSortClause } from './sql'
import { fireTableTrigger } from './trigger'
@@ -3935,18 +3937,26 @@ async function backfillGroupOutputsFromLogs(opts: {
const logs = await db
.select({
executionId: workflowExecutionLogs.executionId,
workflowId: workflowExecutionLogs.workflowId,
workspaceId: workflowExecutionLogs.workspaceId,
executionData: workflowExecutionLogs.executionData,
})
.from(workflowExecutionLogs)
.where(inArray(workflowExecutionLogs.executionId, executionIds))
const logByExecutionId = new Map<string, { traceSpans?: BackfillTraceSpan[] }>()
for (const log of logs) {
// Heavy execution data may live in object storage; resolve pointers (bounded
// concurrency) so trace spans are available for table-column enrichment.
await mapWithConcurrency(logs, MATERIALIZE_CONCURRENCY, async (log) => {
const executionData = await materializeExecutionData(
log.executionData as Record<string, unknown> | null,
{ workspaceId: log.workspaceId, workflowId: log.workflowId, executionId: log.executionId }
)
logByExecutionId.set(
log.executionId,
(log.executionData as { traceSpans?: BackfillTraceSpan[] }) ?? {}
(executionData as { traceSpans?: BackfillTraceSpan[] }) ?? {}
)
}
})
const updates: Array<{ rowId: string; data: RowData }> = []
for (const r of rowRecords) {
@@ -71,7 +71,6 @@ interface MemberUsageData {
isOverLimit: boolean
role: string
joinedAt: string
lastActive: string | null
}
interface OrganizationBillingData {
+2 -6
View File
@@ -3,7 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors'
import type OpenAI from 'openai'
import type { ChatCompletionChunk } from 'openai/resources/chat/completions'
import type { CompletionUsage } from 'openai/resources/completions'
import { dollarsToCredits } from '@/lib/billing/credits/conversion'
import { formatCreditCost } from '@/lib/billing/credits/conversion'
import { env } from '@/lib/core/config/env'
import { getBlacklistedProvidersFromEnv, isHosted } from '@/lib/core/config/feature-flags'
import {
@@ -700,11 +700,7 @@ export function getModelPricing(modelId: string): any {
* @returns Formatted credit string (e.g. "200 credits", "<1 credit", "0 credits")
*/
export function formatCost(cost: number): string {
if (cost === undefined || cost === null) return '—'
const credits = dollarsToCredits(cost)
if (credits <= 0 && cost > 0) return '<1 credit'
if (credits <= 0) return '0 credits'
return `${credits.toLocaleString()} credits`
return formatCreditCost(cost) ?? '—'
}
/**
+188
View File
@@ -0,0 +1,188 @@
#!/usr/bin/env bun
/**
* One-shot, idempotent, resumable backfill that externalizes inline heavy
* `execution_data` (traceSpans, finalOutput, workflowInput, ...) into the
* execution-context large-value store, matching the completion path (cost-stripped
* spans, trace pointer + markers, owner/dependency + execution_log reference
* registration). Skips running rows and rows already carrying the pointer.
*
* Requires object storage to be configured; self-hosted deployments without it
* keep `execution_data` inline (reads resolve inline transparently) and can skip
* this script entirely.
*
* NOTE: the companion `cost_total` / `models_used` backfill is done in SQL by
* migration 0220 (batched, idempotent), so it runs for everyone — including
* self-hosted — and is intentionally NOT part of this script.
*
* Usage:
* DATABASE_URL=... bun apps/sim/scripts/backfill-trace-spans.ts [--max-batches=<n>]
*/
import { db } from '@sim/db'
import { workflowExecutionLogs } from '@sim/db/schema'
import { toError } from '@sim/utils/errors'
import { and, asc, eq, gt, sql } from 'drizzle-orm'
import {
collectLargeValueReferenceKeys,
replaceLargeValueReferenceKeysWithClient,
} from '@/lib/execution/payloads/large-value-metadata'
import {
externalizeExecutionData,
stripSpanCosts,
TRACE_STORE_REF_KEY,
} from '@/lib/logs/execution/trace-store'
const TRACE_BATCH_SIZE = 100
/**
* Recursively counts trace spans (matching the completion path). Legacy rows
* predate the inline hasTraceSpans/traceSpanCount markers, so we derive them
* before externalizing — otherwise a post-expiry degraded read can't report
* "trace data expired (N spans)".
*/
function countTraceSpans(spans: unknown): number {
if (!Array.isArray(spans)) return 0
return spans.reduce(
(count: number, span) =>
count + 1 + countTraceSpans((span as { children?: unknown } | null)?.children),
0
)
}
interface Options {
maxBatches: number
}
function parseArgs(argv: string[]): Options {
const maxBatchesArg = argv.find((a) => a.startsWith('--max-batches='))
const maxBatches = maxBatchesArg
? Number.parseInt(maxBatchesArg.slice('--max-batches='.length), 10)
: Number.POSITIVE_INFINITY
if (Number.isNaN(maxBatches) || maxBatches <= 0) {
throw new Error('--max-batches must be a positive integer')
}
return { maxBatches }
}
/** Externalize inline heavy execution_data into the large-value store. */
async function backfillTraceStorage(
maxBatches: number
): Promise<{ migrated: number; failed: number }> {
let migrated = 0
let failed = 0
// Keyset cursor by id: every row is visited at most once per run, so rows that
// can't be externalized (storage error, oversized) aren't re-selected into an
// infinite loop. A fresh re-run (cursor reset) retries any that failed.
let lastId = ''
for (let batch = 0; batch < maxBatches; batch++) {
const rows = await db
.select({
id: workflowExecutionLogs.id,
workspaceId: workflowExecutionLogs.workspaceId,
workflowId: workflowExecutionLogs.workflowId,
executionId: workflowExecutionLogs.executionId,
executionData: workflowExecutionLogs.executionData,
})
.from(workflowExecutionLogs)
.where(
and(
sql`${workflowExecutionLogs.endedAt} IS NOT NULL`,
// Skip deleted-workflow rows: externalization requires a workflowId.
sql`${workflowExecutionLogs.workflowId} IS NOT NULL`,
sql`${workflowExecutionLogs.executionData} ? 'traceSpans'`,
sql`NOT (${workflowExecutionLogs.executionData} ? ${TRACE_STORE_REF_KEY})`,
lastId ? gt(workflowExecutionLogs.id, lastId) : undefined
)
)
.orderBy(asc(workflowExecutionLogs.id))
.limit(TRACE_BATCH_SIZE)
if (rows.length === 0) break
for (const row of rows) {
try {
const executionData = (row.executionData ?? {}) as Record<string, unknown>
// Derive the inline markers legacy rows lack so externalizeExecutionData
// carries them onto the slim row (they survive object expiry).
const traceSpanCount = countTraceSpans(executionData.traceSpans)
executionData.hasTraceSpans = traceSpanCount > 0
executionData.traceSpanCount = traceSpanCount
stripSpanCosts(executionData.traceSpans)
// workspace_files.user_id (NOT NULL) needs the execution owner; legacy
// rows carry it under executionData.environment.userId. Rows without an
// owner can't be externalized — count them as failed and skip.
const environment = executionData.environment as { userId?: string } | undefined
const ownerUserId = environment?.userId
if (!ownerUserId) {
failed++
continue
}
const slim = await externalizeExecutionData(executionData, {
workspaceId: row.workspaceId,
workflowId: row.workflowId,
executionId: row.executionId,
userId: ownerUserId,
})
if (!(TRACE_STORE_REF_KEY in slim)) {
failed++
continue
}
await db.transaction(async (tx) => {
await tx
.update(workflowExecutionLogs)
.set({ executionData: slim })
.where(eq(workflowExecutionLogs.id, row.id))
await replaceLargeValueReferenceKeysWithClient(
tx,
{
workspaceId: row.workspaceId,
workflowId: row.workflowId,
executionId: row.executionId,
source: 'execution_log',
},
collectLargeValueReferenceKeys(slim)
)
})
migrated++
} catch (error) {
failed++
console.error(` [trace] row ${row.id} failed: ${toError(error).message}`)
}
}
// Advance the cursor past this batch so failed rows aren't re-selected.
lastId = rows[rows.length - 1].id
console.log(` [trace] batch ${batch + 1}: migrated ${migrated}, failed ${failed}`)
}
return { migrated, failed }
}
async function main(): Promise<void> {
const options = parseArgs(process.argv.slice(2))
const startedAt = Date.now()
console.log('Backfilling trace storage (externalizing execution_data)…')
const { migrated, failed } = await backfillTraceStorage(options.maxBatches)
console.log(`Trace storage done: ${migrated} migrated, ${failed} skipped/failed.`)
console.log(`Backfill complete in ${((Date.now() - startedAt) / 1000).toFixed(1)}s.`)
}
main()
.catch((err) => {
console.error('Backfill failed:', err)
process.exit(1)
})
.finally(() => {
process.exit(0)
})
-1
View File
@@ -1,6 +1,5 @@
{
"lockfileVersion": 1,
"configVersion": 0,
"workspaces": {
"": {
"name": "simstudio",
@@ -0,0 +1,36 @@
ALTER TABLE "workflow_execution_logs" ADD COLUMN IF NOT EXISTS "cost_total" numeric;--> statement-breakpoint
ALTER TABLE "workflow_execution_logs" ADD COLUMN IF NOT EXISTS "models_used" text[];--> statement-breakpoint
COMMIT;--> statement-breakpoint
ALTER TYPE "public"."usage_log_category" ADD VALUE IF NOT EXISTS 'tool';--> statement-breakpoint
CREATE OR REPLACE PROCEDURE backfill_wel_cost_total_0220() LANGUAGE plpgsql AS $$
DECLARE
updated integer;
BEGIN
LOOP
WITH candidates AS (
SELECT id FROM workflow_execution_logs
WHERE cost_total IS NULL
AND cost ? 'total'
AND (cost->>'total') ~ '^-?[0-9]+(\.[0-9]+)?$'
LIMIT 5000
)
UPDATE workflow_execution_logs wel
SET cost_total = NULLIF(wel.cost->>'total', '')::numeric,
models_used = CASE
WHEN jsonb_typeof(wel.cost->'models') = 'object'
THEN ARRAY(SELECT jsonb_object_keys(wel.cost->'models'))
ELSE wel.models_used
END
FROM candidates
WHERE wel.id = candidates.id;
GET DIAGNOSTICS updated = ROW_COUNT;
EXIT WHEN updated = 0;
COMMIT;
END LOOP;
END;
$$;--> statement-breakpoint
CALL backfill_wel_cost_total_0220();--> statement-breakpoint
DROP PROCEDURE backfill_wel_cost_total_0220();--> statement-breakpoint
CREATE INDEX CONCURRENTLY IF NOT EXISTS "usage_log_execution_id_idx" ON "usage_log" USING btree ("execution_id");--> statement-breakpoint
CREATE INDEX CONCURRENTLY IF NOT EXISTS "workflow_execution_logs_workspace_cost_total_idx" ON "workflow_execution_logs" USING btree ("workspace_id","cost_total");--> statement-breakpoint
CREATE INDEX CONCURRENTLY IF NOT EXISTS "workflow_execution_logs_models_used_idx" ON "workflow_execution_logs" USING gin ("models_used");
File diff suppressed because it is too large Load Diff
@@ -1534,6 +1534,13 @@
"when": 1780079220753,
"tag": "0219_amused_leo",
"breakpoints": true
},
{
"idx": 220,
"version": "7",
"when": 1780081787541,
"tag": "0220_early_hellion",
"breakpoints": true
}
]
}
+47 -40
View File
@@ -329,8 +329,25 @@ export const workflowExecutionLogs = pgTable(
endedAt: timestamp('ended_at'),
totalDurationMs: integer('total_duration_ms'),
/**
* Heavy trace data (traceSpans, finalOutput, workflowInput, executionState)
* is externalized to object storage; this column then holds a slim payload:
* a `traceStoreRef` (__simLargeValueRef) pointer to the stored object plus
* inline markers (hasTraceSpans, traceSpanCount, environment, trigger,
* truncation flags). It also still holds the FULL payload inline for legacy
* / not-yet-backfilled rows, for the storage-write-failure fallback, and for
* job_execution_logs. Required — not droppable. Read it via
* `materializeExecutionData`, which resolves the pointer.
*/
executionData: jsonb('execution_data').notNull().default('{}'),
/** @deprecated Not written/read; cost lives in usage_log + the `cost_total` projection. Drop in a follow-up PR after the `cost_total` backfill. */
cost: jsonb('cost'),
// Faithful, write-once projection of the run's usage_log ledger sum (dollars).
// Backs list cost display/filter/sort without live aggregation; never an
// independently-computed value (cost_total == SUM(usage_log) for the run).
costTotal: decimal('cost_total'),
// Model names used by the run (incl. zero-cost/BYOK), for the v1 model filter.
modelsUsed: text('models_used').array(),
files: jsonb('files'), // File metadata for execution files
createdAt: timestamp('created_at').notNull().defaultNow(),
},
@@ -356,6 +373,11 @@ export const workflowExecutionLogs = pgTable(
table.workspaceId,
table.startedAt
),
workspaceCostTotalIdx: index('workflow_execution_logs_workspace_cost_total_idx').on(
table.workspaceId,
table.costTotal
),
modelsUsedIdx: index('workflow_execution_logs_models_used_idx').using('gin', table.modelsUsed),
workspaceEndedAtIdIdx: index('workflow_execution_logs_workspace_ended_at_id_idx').on(
table.workspaceId,
sql`date_trunc('milliseconds', ${table.endedAt})`,
@@ -881,39 +903,33 @@ export const userStats = pgTable('user_stats', {
.notNull()
.references(() => user.id, { onDelete: 'cascade' })
.unique(), // One record per user
/**
* Deprecated former usage hot-path counters.
*
* These used to be incremented from execution/API/trigger/chat/MCP/A2A
* billing paths on every usage event. New usage reporting must derive
* these dimensions from `usage_log` instead of writing this row.
*/
// Retired usage hot-path counters: no writers/readers; derive from usage_log.
// Drop via DROP COLUMN in a follow-up migration.
/** @deprecated Retired usage counter; derive from usage_log. */
totalManualExecutions: integer('total_manual_executions').notNull().default(0),
/** @deprecated Retired usage counter; derive from usage_log. */
totalApiCalls: integer('total_api_calls').notNull().default(0),
/** @deprecated Retired usage counter; derive from usage_log. */
totalWebhookTriggers: integer('total_webhook_triggers').notNull().default(0),
/** @deprecated Retired usage counter; derive from usage_log. */
totalScheduledExecutions: integer('total_scheduled_executions').notNull().default(0),
/** @deprecated Retired usage counter; derive from usage_log. */
totalChatExecutions: integer('total_chat_executions').notNull().default(0),
/** @deprecated Retired usage counter; derive from usage_log. */
totalMcpExecutions: integer('total_mcp_executions').notNull().default(0),
/** @deprecated Retired usage counter; derive from usage_log. */
totalA2aExecutions: integer('total_a2a_executions').notNull().default(0),
/** @deprecated Retired usage counter; derive from usage_log. */
totalTokensUsed: bigint('total_tokens_used', { mode: 'number' }).notNull().default(0),
/**
* Deprecated former usage hot-path cost aggregate.
*
* `recordUsage` now appends attributed rows to `usage_log`; this column is
* retained only for legacy/admin reporting until consumers move to ledger
* aggregations.
*/
/** @deprecated Not written (recordUsage appends to usage_log); legacy/admin reads only. Move readers to ledger aggregation. */
totalCost: decimal('total_cost').notNull().default('0'),
currentUsageLimit: decimal('current_usage_limit').default(DEFAULT_FREE_CREDITS.toString()), // Default $5 (1,000 credits) for free plan, null for team/enterprise
usageLimitUpdatedAt: timestamp('usage_limit_updated_at').defaultNow(),
/**
* Deprecated former usage hot-path current-period aggregate.
*
* Keep only as the pre-shift baseline for the active billing period.
* Canonical current-period usage is `currentPeriodCost` baseline plus
* attributed `usage_log` rows for the same billing entity and period.
* Active per-period baseline (not a per-usage hot-path counter). Current usage
* = this baseline + attributed usage_log rows for the period; reset at rollover.
*/
currentPeriodCost: decimal('current_period_cost').notNull().default('0'), // Usage in current billing period
currentPeriodCost: decimal('current_period_cost').notNull().default('0'),
lastPeriodCost: decimal('last_period_cost').default('0'), // Usage from previous billing period
/**
* Threshold/final billing tracker.
@@ -933,25 +949,21 @@ export const userStats = pgTable('user_stats', {
* overage collection. It is not a per-usage aggregate counter.
*/
creditBalance: decimal('credit_balance').notNull().default('0'),
/**
* Deprecated former Copilot hot-path cost/counter aggregates.
*
* Copilot/MCP Copilot usage should be reported from `usage_log` going
* forward. Current/last period Copilot columns remain as legacy reset
* trackers until those consumers are migrated.
*/
/** @deprecated Not written; report Copilot cost from usage_log. Legacy/admin reads only. */
totalCopilotCost: decimal('total_copilot_cost').notNull().default('0'),
/** Active per-period Copilot baseline; reset at rollover (not a per-usage counter). */
currentPeriodCopilotCost: decimal('current_period_copilot_cost').notNull().default('0'),
/** Previous-period Copilot cost; set at rollover. */
lastPeriodCopilotCost: decimal('last_period_copilot_cost').default('0'),
/** @deprecated Not written; report Copilot tokens from usage_log. Legacy/admin reads only. */
totalCopilotTokens: bigint('total_copilot_tokens', { mode: 'number' }).notNull().default(0),
/** @deprecated Not written; report Copilot calls from usage_log. Legacy/admin reads only. */
totalCopilotCalls: integer('total_copilot_calls').notNull().default(0),
/**
* Deprecated former MCP Copilot hot-path aggregates.
*
* New MCP Copilot billing usage should be reported from `usage_log`.
*/
/** @deprecated Not written; report MCP Copilot calls from usage_log. Legacy/admin reads only. */
totalMcpCopilotCalls: integer('total_mcp_copilot_calls').notNull().default(0),
/** @deprecated Not written; report MCP Copilot cost from usage_log. Legacy/admin reads only. */
totalMcpCopilotCost: decimal('total_mcp_copilot_cost').notNull().default('0'),
/** @deprecated No writer (never incremented or reset). MCP copilot usage lives in usage_log (source 'mcp_copilot'); read it from there, not this column. */
currentPeriodMcpCopilotCost: decimal('current_period_mcp_copilot_cost').notNull().default('0'),
/**
* Storage upload/delete hot-path tracker for personal plans.
@@ -960,13 +972,7 @@ export const userStats = pgTable('user_stats', {
* org-scoped storage writes update `organization.storageUsedBytes`.
*/
storageUsedBytes: bigint('storage_used_bytes', { mode: 'number' }).notNull().default(0),
/**
* Deprecated former execution hot-path activity timestamp.
*
* Successful workflow execution no longer updates `user_stats`; this column
* is retained only for legacy/admin reporting until replaced by an activity
* source that does not contend on this row.
*/
/** @deprecated Not updated by execution (no user_stats write on completion); legacy/admin reads only. */
lastActive: timestamp('last_active').notNull().defaultNow(),
billingBlocked: boolean('billing_blocked').notNull().default(false),
billingBlockedReason: billingBlockedReasonEnum('billing_blocked_reason'),
@@ -2791,7 +2797,7 @@ export const auditLog = pgTable(
})
)
export const usageLogCategoryEnum = pgEnum('usage_log_category', ['model', 'fixed'])
export const usageLogCategoryEnum = pgEnum('usage_log_category', ['model', 'fixed', 'tool'])
export const usageLogSourceEnum = pgEnum('usage_log_source', [
'workflow',
'wand',
@@ -2861,6 +2867,7 @@ export const usageLog = pgTable(
table.workspaceId,
table.createdAt
),
executionIdIdx: index('usage_log_execution_id_idx').on(table.executionId),
})
)
+29
View File
@@ -2,6 +2,33 @@ import { drizzle } from 'drizzle-orm/postgres-js'
import { migrate } from 'drizzle-orm/postgres-js/migrator'
import postgres from 'postgres'
/**
* Concurrent-index convention (avoid write-blocking index builds on large tables)
* --------------------------------------------------------------------------------
* drizzle-kit emits plain `CREATE INDEX`, which takes a SHARE lock and blocks all
* writes for the build duration — on a big, write-hot table (e.g.
* workflow_execution_logs, usage_log) that stalls every in-flight workflow
* completion for minutes. drizzle wraps each migration in a transaction, and
* `CREATE INDEX CONCURRENTLY` cannot run inside a transaction block.
*
* So, after generating a migration that adds an index on a large/hot table, edit
* the generated SQL to end drizzle's transaction first, then build concurrently
* and idempotently:
*
* COMMIT;--> statement-breakpoint
* CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_name" ON "table" (...);
*
* Notes:
* - Put the `COMMIT` breakpoint AFTER all transactional DDL (ALTER TABLE/TYPE)
* in the file and only the concurrent CREATE INDEX statements below it.
* - Use `IF NOT EXISTS` (and make sibling DDL idempotent, e.g.
* `ADD COLUMN IF NOT EXISTS`, `ADD VALUE IF NOT EXISTS`) so a re-run after a
* failed CONCURRENTLY build is safe — fresh DBs and re-applies both work.
* - CONCURRENTLY only takes a SHARE UPDATE EXCLUSIVE lock (allows reads/writes).
* - Always validate on staging before prod; a failed CONCURRENTLY build can
* leave an INVALID index that must be dropped and rebuilt.
*/
const url = process.env.DATABASE_URL
if (!url) {
console.error('ERROR: Missing DATABASE_URL environment variable.')
@@ -12,6 +39,8 @@ if (!url) {
const client = postgres(url, { max: 1, connect_timeout: 10 })
try {
// statement_timeout=0: index builds (esp. CONCURRENTLY on large tables) can run
// far longer than the app default; a migration must never be killed mid-build.
await client`SET statement_timeout = 0`
await migrate(drizzle(client), { migrationsFolder: './migrations' })
console.log('Migrations applied successfully.')