improvement(cron): fire-and-forget for cron-invoked endpoints (#4764)

* improvement(cron): fire-and-forget for cron-invoked endpoints

* fix(cron): add staleness takeover to single-flight guard

* improvement(cron): drop single-flight guard, rely on DB row claiming
This commit is contained in:
Theodore Li
2026-05-27 19:47:23 -04:00
committed by GitHub
parent 28766ddaa9
commit 7ddd90be0b
11 changed files with 731 additions and 349 deletions
@@ -0,0 +1,90 @@
/**
* Tests for the Teams subscription renewal cron route.
*
* @vitest-environment node
*/
import {
authOAuthUtilsMock,
createMockRequest,
dbChainMock,
dbChainMockFns,
redisConfigMock,
redisConfigMockFns,
resetDbChainMock,
} from '@sim/testing'
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { mockVerifyCronAuth } = vi.hoisted(() => ({
mockVerifyCronAuth: vi.fn().mockReturnValue(null),
}))
vi.mock('@/lib/auth/internal', () => ({
verifyCronAuth: mockVerifyCronAuth,
}))
vi.mock('@/lib/core/config/redis', () => redisConfigMock)
vi.mock('@sim/db', () => dbChainMock)
vi.mock('@/app/api/auth/oauth/utils', () => authOAuthUtilsMock)
import { GET } from './route'
function createRequest() {
return createMockRequest(
'GET',
undefined,
{},
'http://localhost:3000/api/cron/renew-subscriptions'
)
}
const flushMicrotasks = () => new Promise((resolve) => setTimeout(resolve, 0))
describe('Teams subscription renewal route (fire-and-forget)', () => {
beforeEach(() => {
vi.clearAllMocks()
resetDbChainMock()
redisConfigMockFns.mockAcquireLock.mockResolvedValue(true)
redisConfigMockFns.mockReleaseLock.mockResolvedValue(true)
mockVerifyCronAuth.mockReturnValue(null)
})
it('returns the auth error when cron auth fails', async () => {
mockVerifyCronAuth.mockReturnValueOnce(new Response(null, { status: 401 }) as never)
const response = await GET(createRequest())
expect(response.status).toBe(401)
expect(redisConfigMockFns.mockAcquireLock).not.toHaveBeenCalled()
})
it('acknowledges with 202 and renews in the background after acquiring the lock', async () => {
const response = await GET(createRequest())
expect(response.status).toBe(202)
const data = await response.json()
expect(data).toMatchObject({ status: 'started' })
expect(redisConfigMockFns.mockAcquireLock).toHaveBeenCalledWith(
'teams-subscription-renewal-lock',
expect.any(String),
expect.any(Number)
)
await flushMicrotasks()
expect(dbChainMockFns.select).toHaveBeenCalled()
expect(redisConfigMockFns.mockReleaseLock).toHaveBeenCalledWith(
'teams-subscription-renewal-lock',
expect.any(String)
)
})
it('skips with 202 when the lock is already held', async () => {
redisConfigMockFns.mockAcquireLock.mockResolvedValueOnce(false)
const response = await GET(createRequest())
expect(response.status).toBe(202)
const data = await response.json()
expect(data).toMatchObject({ status: 'skip' })
expect(dbChainMockFns.select).not.toHaveBeenCalled()
})
})
+171 -134
View File
@@ -1,14 +1,21 @@
import { db } from '@sim/db'
import { account, webhook as webhookTable } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { generateShortId } from '@sim/utils/id'
import { and, eq, or } from 'drizzle-orm'
import { type NextRequest, NextResponse } from 'next/server'
import { verifyCronAuth } from '@/lib/auth/internal'
import { acquireLock, releaseLock } from '@/lib/core/config/redis'
import { runDetached } from '@/lib/core/utils/background'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/app/api/auth/oauth/utils'
const logger = createLogger('TeamsSubscriptionRenewal')
const LOCK_KEY = 'teams-subscription-renewal-lock'
/** Lock TTL in seconds — generous enough to cover the Graph API renewal loop. */
const LOCK_TTL_SECONDS = 300
async function getCredentialOwner(
credentialId: string
): Promise<{ userId: string; accountId: string } | null> {
@@ -29,159 +36,189 @@ async function getCredentialOwner(
}
/**
* Cron endpoint to renew Microsoft Teams chat subscriptions before they expire
* Renews Microsoft Teams chat subscriptions that are close to expiring.
*
* Teams subscriptions expire after ~3 days and must be renewed.
* Configured in helm/sim/values.yaml under cronjobs.jobs.renewSubscriptions
* Teams subscriptions expire after ~3 days and must be renewed. Runs detached
* from the HTTP response so the cron caller does not wait for the Graph API loop.
*/
export const GET = withRouteHandler(async (request: NextRequest) => {
try {
const authError = verifyCronAuth(request, 'Teams subscription renewal')
if (authError) {
return authError
}
async function renewExpiringSubscriptions(): Promise<{
checked: number
renewed: number
failed: number
total: number
}> {
logger.info('Starting Teams subscription renewal job')
logger.info('Starting Teams subscription renewal job')
let totalRenewed = 0
let totalFailed = 0
let totalChecked = 0
let totalRenewed = 0
let totalFailed = 0
let totalChecked = 0
// Get all active Microsoft Teams webhooks
const webhooksWithWorkflows = await db
.select({
webhook: webhookTable,
})
.from(webhookTable)
.where(
and(
eq(webhookTable.isActive, true),
or(
eq(webhookTable.provider, 'microsoft-teams'),
eq(webhookTable.provider, 'microsoftteams')
)
// Get all active Microsoft Teams webhooks
const webhooksWithWorkflows = await db
.select({
webhook: webhookTable,
})
.from(webhookTable)
.where(
and(
eq(webhookTable.isActive, true),
or(
eq(webhookTable.provider, 'microsoft-teams'),
eq(webhookTable.provider, 'microsoftteams')
)
)
logger.info(
`Found ${webhooksWithWorkflows.length} active Teams webhooks, checking for expiring subscriptions`
)
// Renewal threshold: 48 hours before expiration
const renewalThreshold = new Date(Date.now() + 48 * 60 * 60 * 1000)
logger.info(
`Found ${webhooksWithWorkflows.length} active Teams webhooks, checking for expiring subscriptions`
)
for (const { webhook } of webhooksWithWorkflows) {
const config = (webhook.providerConfig as Record<string, any>) || {}
// Renewal threshold: 48 hours before expiration
const renewalThreshold = new Date(Date.now() + 48 * 60 * 60 * 1000)
// Check if this is a Teams chat subscription that needs renewal
if (config.triggerId !== 'microsoftteams_chat_subscription') continue
for (const { webhook } of webhooksWithWorkflows) {
const config = (webhook.providerConfig as Record<string, any>) || {}
const expirationStr = config.subscriptionExpiration as string | undefined
if (!expirationStr) continue
// Check if this is a Teams chat subscription that needs renewal
if (config.triggerId !== 'microsoftteams_chat_subscription') continue
const expiresAt = new Date(expirationStr)
if (expiresAt > renewalThreshold) continue // Not expiring soon
const expirationStr = config.subscriptionExpiration as string | undefined
if (!expirationStr) continue
totalChecked++
const expiresAt = new Date(expirationStr)
if (expiresAt > renewalThreshold) continue // Not expiring soon
try {
logger.info(
`Renewing Teams subscription for webhook ${webhook.id} (expires: ${expiresAt.toISOString()})`
)
totalChecked++
const credentialId = config.credentialId as string | undefined
const externalSubscriptionId = config.externalSubscriptionId as string | undefined
try {
logger.info(
`Renewing Teams subscription for webhook ${webhook.id} (expires: ${expiresAt.toISOString()})`
)
if (!credentialId || !externalSubscriptionId) {
logger.error(`Missing credentialId or externalSubscriptionId for webhook ${webhook.id}`)
totalFailed++
continue
}
const credentialId = config.credentialId as string | undefined
const externalSubscriptionId = config.externalSubscriptionId as string | undefined
const credentialOwner = await getCredentialOwner(credentialId)
if (!credentialOwner) {
logger.error(`Credential owner not found for credential ${credentialId}`)
totalFailed++
continue
}
// Get fresh access token
const accessToken = await refreshAccessTokenIfNeeded(
credentialOwner.accountId,
credentialOwner.userId,
`renewal-${webhook.id}`
)
if (!accessToken) {
logger.error(`Failed to get access token for webhook ${webhook.id}`)
totalFailed++
continue
}
// Extend subscription to maximum lifetime (4230 minutes = ~3 days)
const maxLifetimeMinutes = 4230
const newExpirationDateTime = new Date(
Date.now() + maxLifetimeMinutes * 60 * 1000
).toISOString()
const res = await fetch(
`https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`,
{
method: 'PATCH',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({ expirationDateTime: newExpirationDateTime }),
}
)
if (!res.ok) {
const error = await res.json()
logger.error(
`Failed to renew Teams subscription ${externalSubscriptionId} for webhook ${webhook.id}`,
{ status: res.status, error: error.error }
)
totalFailed++
continue
}
const payload = await res.json()
// Update webhook config with new expiration
const updatedConfig = {
...config,
subscriptionExpiration: payload.expirationDateTime,
}
await db
.update(webhookTable)
.set({ providerConfig: updatedConfig, updatedAt: new Date() })
.where(eq(webhookTable.id, webhook.id))
logger.info(
`Successfully renewed Teams subscription for webhook ${webhook.id}. New expiration: ${payload.expirationDateTime}`
)
totalRenewed++
} catch (error) {
logger.error(`Error renewing subscription for webhook ${webhook.id}:`, error)
if (!credentialId || !externalSubscriptionId) {
logger.error(`Missing credentialId or externalSubscriptionId for webhook ${webhook.id}`)
totalFailed++
continue
}
const credentialOwner = await getCredentialOwner(credentialId)
if (!credentialOwner) {
logger.error(`Credential owner not found for credential ${credentialId}`)
totalFailed++
continue
}
// Get fresh access token
const accessToken = await refreshAccessTokenIfNeeded(
credentialOwner.accountId,
credentialOwner.userId,
`renewal-${webhook.id}`
)
if (!accessToken) {
logger.error(`Failed to get access token for webhook ${webhook.id}`)
totalFailed++
continue
}
// Extend subscription to maximum lifetime (4230 minutes = ~3 days)
const maxLifetimeMinutes = 4230
const newExpirationDateTime = new Date(
Date.now() + maxLifetimeMinutes * 60 * 1000
).toISOString()
const res = await fetch(
`https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`,
{
method: 'PATCH',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({ expirationDateTime: newExpirationDateTime }),
}
)
if (!res.ok) {
const error = await res.json()
logger.error(
`Failed to renew Teams subscription ${externalSubscriptionId} for webhook ${webhook.id}`,
{ status: res.status, error: error.error }
)
totalFailed++
continue
}
const payload = await res.json()
// Update webhook config with new expiration
const updatedConfig = {
...config,
subscriptionExpiration: payload.expirationDateTime,
}
await db
.update(webhookTable)
.set({ providerConfig: updatedConfig, updatedAt: new Date() })
.where(eq(webhookTable.id, webhook.id))
logger.info(
`Successfully renewed Teams subscription for webhook ${webhook.id}. New expiration: ${payload.expirationDateTime}`
)
totalRenewed++
} catch (error) {
logger.error(`Error renewing subscription for webhook ${webhook.id}:`, error)
totalFailed++
}
logger.info(
`Teams subscription renewal job completed. Checked: ${totalChecked}, Renewed: ${totalRenewed}, Failed: ${totalFailed}`
)
return NextResponse.json({
success: true,
checked: totalChecked,
renewed: totalRenewed,
failed: totalFailed,
total: webhooksWithWorkflows.length,
})
} catch (error) {
logger.error('Error in Teams subscription renewal job:', error)
return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
}
logger.info(
`Teams subscription renewal job completed. Checked: ${totalChecked}, Renewed: ${totalRenewed}, Failed: ${totalFailed}`
)
return {
checked: totalChecked,
renewed: totalRenewed,
failed: totalFailed,
total: webhooksWithWorkflows.length,
}
}
/**
* Cron endpoint to renew Microsoft Teams chat subscriptions before they expire.
* Configured in helm/sim/values.yaml under cronjobs.jobs.renewSubscriptions.
*
* Acknowledges the cron call immediately and renews subscriptions in the
* background; a Redis lock prevents overlapping runs.
*/
export const GET = withRouteHandler(async (request: NextRequest) => {
const authError = verifyCronAuth(request, 'Teams subscription renewal')
if (authError) {
return authError
}
const lockValue = generateShortId()
const locked = await acquireLock(LOCK_KEY, lockValue, LOCK_TTL_SECONDS)
if (!locked) {
return NextResponse.json(
{ success: true, message: 'Renewal already in progress – skipped', status: 'skip' },
{ status: 202 }
)
}
runDetached('teams-subscription-renewal', async () => {
try {
await renewExpiringSubscriptions()
} finally {
await releaseLock(LOCK_KEY, lockValue).catch(() => {})
}
})
return NextResponse.json(
{ success: true, message: 'Teams subscription renewal started', status: 'started' },
{ status: 202 }
)
})
@@ -0,0 +1,93 @@
/**
* Tests for the inactivity-alert polling cron route.
*
* @vitest-environment node
*/
import { createMockRequest, redisConfigMock, redisConfigMockFns } from '@sim/testing'
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { mockVerifyCronAuth, mockPollInactivityAlerts } = vi.hoisted(() => ({
mockVerifyCronAuth: vi.fn().mockReturnValue(null),
mockPollInactivityAlerts: vi.fn().mockResolvedValue({ checked: 0, delivered: 0 }),
}))
vi.mock('@/lib/auth/internal', () => ({
verifyCronAuth: mockVerifyCronAuth,
}))
vi.mock('@/lib/core/config/redis', () => redisConfigMock)
vi.mock('@/lib/notifications/inactivity-polling', () => ({
pollInactivityAlerts: mockPollInactivityAlerts,
}))
import { GET } from './route'
function createRequest() {
return createMockRequest('GET', undefined, {}, 'http://localhost:3000/api/notifications/poll')
}
const flushMicrotasks = () => new Promise((resolve) => setTimeout(resolve, 0))
describe('inactivity alert polling route (fire-and-forget)', () => {
beforeEach(() => {
vi.clearAllMocks()
redisConfigMockFns.mockAcquireLock.mockResolvedValue(true)
redisConfigMockFns.mockReleaseLock.mockResolvedValue(true)
mockVerifyCronAuth.mockReturnValue(null)
mockPollInactivityAlerts.mockResolvedValue({ checked: 0, delivered: 0 })
})
it('returns the auth error when cron auth fails', async () => {
mockVerifyCronAuth.mockReturnValueOnce(new Response(null, { status: 401 }) as never)
const response = await GET(createRequest())
expect(response.status).toBe(401)
expect(mockPollInactivityAlerts).not.toHaveBeenCalled()
})
it('acknowledges with 202 and polls in the background after acquiring the lock', async () => {
const response = await GET(createRequest())
expect(response.status).toBe(202)
const data = await response.json()
expect(data).toMatchObject({ status: 'started' })
expect(redisConfigMockFns.mockAcquireLock).toHaveBeenCalledWith(
'inactivity-alert-polling-lock',
expect.any(String),
expect.any(Number)
)
await flushMicrotasks()
expect(mockPollInactivityAlerts).toHaveBeenCalledTimes(1)
expect(redisConfigMockFns.mockReleaseLock).toHaveBeenCalledWith(
'inactivity-alert-polling-lock',
expect.any(String)
)
})
it('skips with 202 when the lock is already held', async () => {
redisConfigMockFns.mockAcquireLock.mockResolvedValueOnce(false)
const response = await GET(createRequest())
expect(response.status).toBe(202)
const data = await response.json()
expect(data).toMatchObject({ status: 'skip' })
expect(mockPollInactivityAlerts).not.toHaveBeenCalled()
})
it('releases the lock even when polling throws', async () => {
mockPollInactivityAlerts.mockRejectedValueOnce(new Error('poll failed'))
const response = await GET(createRequest())
expect(response.status).toBe(202)
await flushMicrotasks()
expect(redisConfigMockFns.mockReleaseLock).toHaveBeenCalledWith(
'inactivity-alert-polling-lock',
expect.any(String)
)
})
})
+18 -15
View File
@@ -6,6 +6,7 @@ import { noInputSchema } from '@/lib/api/contracts/primitives'
import { validationErrorResponse } from '@/lib/api/server'
import { verifyCronAuth } from '@/lib/auth/internal'
import { acquireLock, releaseLock } from '@/lib/core/config/redis'
import { runDetached } from '@/lib/core/utils/background'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { pollInactivityAlerts } from '@/lib/notifications/inactivity-polling'
@@ -24,15 +25,13 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
)
if (!queryValidation.success) return validationErrorResponse(queryValidation.error)
let lockAcquired = false
try {
const authError = verifyCronAuth(request, 'Inactivity alert polling')
if (authError) {
return authError
}
lockAcquired = await acquireLock(LOCK_KEY, requestId, LOCK_TTL_SECONDS)
const lockAcquired = await acquireLock(LOCK_KEY, requestId, LOCK_TTL_SECONDS)
if (!lockAcquired) {
return NextResponse.json(
@@ -46,15 +45,23 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
)
}
const results = await pollInactivityAlerts()
return NextResponse.json({
success: true,
message: 'Inactivity alert polling completed',
requestId,
status: 'completed',
...results,
runDetached('inactivity-alert-polling', async () => {
try {
await pollInactivityAlerts()
} finally {
await releaseLock(LOCK_KEY, requestId).catch(() => {})
}
})
return NextResponse.json(
{
success: true,
message: 'Inactivity alert polling started',
requestId,
status: 'started',
},
{ status: 202 }
)
} catch (error) {
logger.error(`Error during inactivity alert polling (${requestId}):`, error)
return NextResponse.json(
@@ -66,9 +73,5 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
},
{ status: 500 }
)
} finally {
if (lockAcquired) {
await releaseLock(LOCK_KEY, requestId).catch(() => {})
}
}
})
@@ -4,7 +4,7 @@
* @vitest-environment node
*/
import { dbChainMock, dbChainMockFns, requestUtilsMockFns, resetDbChainMock } from '@sim/testing'
import type { NextRequest } from 'next/server'
import { type NextRequest, NextResponse } from 'next/server'
import { beforeEach, describe, expect, it, vi } from 'vitest'
const orderByLimitMock = vi.fn()
@@ -131,7 +131,7 @@ vi.mock('@sim/utils/id', () => ({
),
}))
import { GET } from './route'
import { GET, runScheduleTick } from './route'
const SINGLE_SCHEDULE = [
{
@@ -284,13 +284,9 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
dbChainMockFns.returning.mockReturnValueOnce(SINGLE_SCHEDULE).mockReturnValueOnce([])
const response = await GET(createMockRequest())
const result = await runScheduleTick('test-request-id')
expect(response).toBeDefined()
expect(response.status).toBe(200)
const data = await response.json()
expect(data).toHaveProperty('message')
expect(data).toHaveProperty('processedCount', 1)
expect(result.processedCount).toBe(1)
})
it('should queue schedules to Trigger.dev when enabled', async () => {
@@ -300,23 +296,17 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
dbChainMockFns.returning.mockReturnValueOnce(SINGLE_SCHEDULE).mockReturnValueOnce([])
const response = await GET(createMockRequest())
const result = await runScheduleTick('test-request-id')
expect(response).toBeDefined()
expect(response.status).toBe(200)
const data = await response.json()
expect(data).toHaveProperty('processedCount', 1)
expect(result.processedCount).toBe(1)
})
it('should handle case with no due schedules', async () => {
dbChainMockFns.returning.mockReturnValueOnce([]).mockReturnValueOnce([])
const response = await GET(createMockRequest())
const result = await runScheduleTick('test-request-id')
expect(response.status).toBe(200)
const data = await response.json()
expect(data).toHaveProperty('message')
expect(data).toHaveProperty('processedCount', 0)
expect(result.processedCount).toBe(0)
})
it('should execute multiple schedules in parallel', async () => {
@@ -328,20 +318,16 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
dbChainMockFns.returning.mockReturnValueOnce(MULTIPLE_SCHEDULES).mockReturnValueOnce([])
const response = await GET(createMockRequest())
const result = await runScheduleTick('test-request-id')
expect(response.status).toBe(200)
const data = await response.json()
expect(data).toHaveProperty('processedCount', 2)
expect(result.processedCount).toBe(2)
})
it('should execute mothership jobs inline', async () => {
dbChainMockFns.limit.mockResolvedValueOnce([]).mockResolvedValueOnce([{ id: 'job-1' }])
dbChainMockFns.returning.mockReturnValueOnce(SINGLE_JOB)
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockExecuteJobInline).toHaveBeenCalledWith(
expect.objectContaining({
scheduleId: 'job-1',
@@ -358,9 +344,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
dbChainMockFns.returning.mockReturnValueOnce(SINGLE_SCHEDULE).mockReturnValueOnce([])
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockEnqueue).toHaveBeenCalledWith(
'schedule-execution',
expect.objectContaining({
@@ -398,9 +382,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([{ id: 'job-id-1' }])
try {
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockEnqueue).toHaveBeenCalledWith(
'schedule-execution',
expect.objectContaining({ scheduleId: 'schedule-1' }),
@@ -435,9 +417,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
try {
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockEnqueue).toHaveBeenCalled()
expect(mockExecuteScheduleJob).not.toHaveBeenCalled()
expect(mockCompleteJob).not.toHaveBeenCalled()
@@ -485,9 +465,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([{ id: 'job-id-1' }])
try {
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockExecuteScheduleJob).toHaveBeenCalledWith(
expect.objectContaining({ scheduleId: 'schedule-1' })
)
@@ -527,11 +505,9 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'pending-job-id' }])
const response = await GET(createMockRequest())
const result = await runScheduleTick('test-request-id')
expect(response.status).toBe(200)
const data = await response.json()
expect(data).toHaveProperty('processedCount', 1)
expect(result.processedCount).toBe(1)
expect(mockEnqueue).not.toHaveBeenCalled()
expect(mockExecuteScheduleJob).toHaveBeenCalledWith(
expect.objectContaining({
@@ -563,9 +539,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
dbChainMockFns.returning.mockReturnValueOnce([]).mockReturnValueOnce([])
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockExecuteScheduleJob).not.toHaveBeenCalled()
expect(mockCompleteJob).toHaveBeenCalledWith(
'stale-pending-job-id',
@@ -596,9 +570,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
.mockResolvedValueOnce([])
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockExecuteScheduleJob).not.toHaveBeenCalled()
expect(dbChainMockFns.set).toHaveBeenCalledWith(
expect.objectContaining({
@@ -629,9 +601,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
dbChainMockFns.returning.mockReturnValueOnce([schedule]).mockReturnValueOnce([])
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockEnqueue).not.toHaveBeenCalled()
expect(mockReleaseScheduleLock).not.toHaveBeenCalled()
expect(dbChainMockFns.set).toHaveBeenCalledWith(
@@ -655,9 +625,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
dbChainMockFns.returning.mockReturnValueOnce([schedule]).mockReturnValueOnce([])
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockEnqueue).not.toHaveBeenCalled()
expect(dbChainMockFns.set).toHaveBeenCalledWith(
expect.objectContaining({
@@ -685,9 +653,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([{ id: 'job-id-1' }])
try {
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockShouldExecuteInline).toHaveBeenCalledTimes(1)
expect(mockExecuteScheduleJob).toHaveBeenCalledWith(
expect.objectContaining({ scheduleId: 'schedule-1' })
@@ -718,9 +684,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
dbChainMockFns.returning.mockReturnValueOnce([schedule]).mockReturnValueOnce([])
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockEnqueue).not.toHaveBeenCalled()
expect(mockReleaseScheduleLock).not.toHaveBeenCalled()
expect(dbChainMockFns.set).toHaveBeenCalledWith(
@@ -769,9 +733,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockReturnValueOnce([schedule])
.mockReturnValueOnce([])
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockEnqueue).not.toHaveBeenCalled()
expect(dbChainMockFns.set).not.toHaveBeenCalledWith(
expect.objectContaining({
@@ -802,9 +764,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
dbChainMockFns.returning.mockReturnValueOnce([schedule]).mockReturnValueOnce([])
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockEnqueue).toHaveBeenCalled()
expect(dbChainMockFns.set).toHaveBeenCalledWith(
expect.objectContaining({
@@ -835,9 +795,7 @@ describe('Scheduled Workflow Execution API Route', () => {
.mockResolvedValueOnce([])
dbChainMockFns.returning.mockReturnValueOnce([schedule]).mockReturnValueOnce([])
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockCancelJob).toHaveBeenCalledWith('trigger-run-id')
expect(mockReleaseScheduleLock).toHaveBeenCalledWith(
'schedule-1',
@@ -871,11 +829,9 @@ describe('Scheduled Workflow Execution API Route', () => {
dbChainMockFns.limit.mockResolvedValueOnce(claimedIds).mockResolvedValueOnce([])
dbChainMockFns.returning.mockReturnValueOnce(claimedSchedules).mockReturnValueOnce([])
const response = await GET(createMockRequest())
const result = await runScheduleTick('test-request-id')
expect(response.status).toBe(200)
const data = await response.json()
expect(data).toHaveProperty('processedCount', 100)
expect(result.processedCount).toBe(100)
expect(dbChainMockFns.limit).toHaveBeenCalledWith(100)
expect(mockEnqueue).toHaveBeenCalledTimes(100)
})
@@ -892,9 +848,7 @@ describe('Scheduled Workflow Execution API Route', () => {
dbChainMockFns.returning.mockReturnValueOnce([schedule]).mockReturnValueOnce([])
mockGetJob.mockResolvedValueOnce({ id: 'job-id-1', status: 'completed' })
const response = await GET(createMockRequest())
expect(response.status).toBe(200)
await runScheduleTick('test-request-id')
expect(mockReleaseScheduleLock).toHaveBeenCalledWith(
'schedule-1',
'test-request-id',
@@ -904,4 +858,24 @@ describe('Scheduled Workflow Execution API Route', () => {
{ expectedLastQueuedAt: claimedAt }
)
})
describe('GET handler (fire-and-forget)', () => {
it('returns the auth error when cron auth fails', async () => {
mockVerifyCronAuth.mockReturnValueOnce(
NextResponse.json({ error: 'unauthorized' }, { status: 401 })
)
const response = await GET(createMockRequest())
expect(response.status).toBe(401)
})
it('acknowledges immediately with 202 and starts the tick in the background', async () => {
const response = await GET(createMockRequest())
expect(response.status).toBe(202)
const data = await response.json()
expect(data).toMatchObject({ status: 'started' })
})
})
})
+111 -96
View File
@@ -14,6 +14,7 @@ import { getJobQueue, shouldExecuteInline } from '@/lib/core/async-jobs'
import { JOB_STATUS, type Job } from '@/lib/core/async-jobs/types'
import { isRetryableInfrastructureError } from '@/lib/core/errors/retryable-infrastructure'
import { getMaxExecutionTimeout } from '@/lib/core/execution-limits'
import { runDetached } from '@/lib/core/utils/background'
import { generateRequestId } from '@/lib/core/utils/request'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import {
@@ -1071,9 +1072,112 @@ async function processJobItem(job: ClaimedJob, queuedAt: Date, requestId: string
}
}
interface ScheduleTickResult {
processedCount: number
totalSchedules: number
totalJobs: number
}
/**
* Drains due schedules and jobs, claiming and enqueuing work until the tick
* budget is exhausted or no more items are due. Runs detached from the HTTP
* response so the cron caller does not wait; cross-replica safety is provided by
* the `FOR UPDATE SKIP LOCKED` claim layer, not this function.
*/
export async function runScheduleTick(requestId: string): Promise<ScheduleTickResult> {
const tickStart = Date.now()
const jobQueue = await getJobQueue()
const useDatabaseFallback = shouldExecuteInline()
let totalSchedules = 0
let totalJobs = 0
let iterations = 0
let remainingWorkflowBudget = SCHEDULE_WORKFLOW_ENQUEUE_LIMIT
let schedulesExhausted = false
let jobsExhausted = false
while (Date.now() - tickStart < MAX_TICK_DURATION_MS) {
if (schedulesExhausted && jobsExhausted) break
const queuedAt = new Date()
let resumedPendingSchedules = 0
let databaseScheduleSlots = SCHEDULE_EXECUTION_CONCURRENCY_LIMIT
if (useDatabaseFallback) {
await recoverStaleDatabaseScheduleJobs(queuedAt)
databaseScheduleSlots = await getDatabaseScheduleExecutionSlots()
resumedPendingSchedules = await resumePendingDatabaseScheduleJobs(
jobQueue,
requestId,
databaseScheduleSlots
)
databaseScheduleSlots = await getDatabaseScheduleExecutionSlots()
}
const workflowClaimLimit = Math.min(
WORKFLOW_CHUNK_SIZE,
remainingWorkflowBudget,
useDatabaseFallback ? databaseScheduleSlots : WORKFLOW_CHUNK_SIZE
)
if (useDatabaseFallback && workflowClaimLimit <= 0) {
schedulesExhausted = true
}
const [dueSchedules, dueJobs] = await Promise.all([
schedulesExhausted ? [] : claimWorkflowSchedules(queuedAt, workflowClaimLimit),
jobsExhausted ? [] : claimJobSchedules(queuedAt, JOB_CHUNK_SIZE),
])
remainingWorkflowBudget -= dueSchedules.length
if (dueSchedules.length < workflowClaimLimit || remainingWorkflowBudget <= 0) {
schedulesExhausted = true
}
if (dueJobs.length < JOB_CHUNK_SIZE) jobsExhausted = true
if (dueSchedules.length === 0 && dueJobs.length === 0 && resumedPendingSchedules === 0) break
iterations += 1
totalSchedules += dueSchedules.length + resumedPendingSchedules
totalJobs += dueJobs.length
logger.info(
`[${requestId}] Iteration ${iterations}: claimed ${dueSchedules.length} schedules, resumed ${resumedPendingSchedules} pending schedule jobs, ${dueJobs.length} jobs`,
{
remainingWorkflowBudget,
scheduleConcurrencyLimit: SCHEDULE_EXECUTION_CONCURRENCY_LIMIT,
databaseScheduleSlots,
}
)
const schedulePromises =
dueSchedules.length > 0
? dueSchedules.map((schedule) =>
processScheduleItem(schedule, queuedAt, requestId, jobQueue, useDatabaseFallback)
)
: []
await Promise.allSettled([
...schedulePromises,
...dueJobs.map((job) => processJobItem(job, queuedAt, requestId)),
])
}
const totalCount = totalSchedules + totalJobs
const durationMs = Date.now() - tickStart
logger.info(
`[${requestId}] Processed ${totalCount} items across ${iterations} iteration(s) in ${durationMs}ms (${totalSchedules} schedules, ${totalJobs} jobs)`,
{
scheduleConcurrencyLimit: SCHEDULE_EXECUTION_CONCURRENCY_LIMIT,
scheduleEnqueueBudget: SCHEDULE_WORKFLOW_ENQUEUE_LIMIT,
remainingWorkflowBudget,
}
)
return { processedCount: totalCount, totalSchedules, totalJobs }
}
export const GET = withRouteHandler(async (request: NextRequest) => {
const requestId = generateRequestId()
const tickStart = Date.now()
logger.info(`[${requestId}] Scheduled execution triggered at ${new Date().toISOString()}`)
const authError = verifyCronAuth(request, 'Schedule execution')
@@ -1081,101 +1185,12 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
return authError
}
try {
const jobQueue = await getJobQueue()
const useDatabaseFallback = shouldExecuteInline()
let totalSchedules = 0
let totalJobs = 0
let iterations = 0
let remainingWorkflowBudget = SCHEDULE_WORKFLOW_ENQUEUE_LIMIT
let schedulesExhausted = false
let jobsExhausted = false
runDetached('schedule-execution-tick', () => runScheduleTick(requestId))
while (Date.now() - tickStart < MAX_TICK_DURATION_MS) {
if (schedulesExhausted && jobsExhausted) break
const queuedAt = new Date()
let resumedPendingSchedules = 0
let databaseScheduleSlots = SCHEDULE_EXECUTION_CONCURRENCY_LIMIT
const response = {
message: 'Scheduled execution started',
status: 'started',
} satisfies ExecuteSchedulesResponse
if (useDatabaseFallback) {
await recoverStaleDatabaseScheduleJobs(queuedAt)
databaseScheduleSlots = await getDatabaseScheduleExecutionSlots()
resumedPendingSchedules = await resumePendingDatabaseScheduleJobs(
jobQueue,
requestId,
databaseScheduleSlots
)
databaseScheduleSlots = await getDatabaseScheduleExecutionSlots()
}
const workflowClaimLimit = Math.min(
WORKFLOW_CHUNK_SIZE,
remainingWorkflowBudget,
useDatabaseFallback ? databaseScheduleSlots : WORKFLOW_CHUNK_SIZE
)
if (useDatabaseFallback && workflowClaimLimit <= 0) {
schedulesExhausted = true
}
const [dueSchedules, dueJobs] = await Promise.all([
schedulesExhausted ? [] : claimWorkflowSchedules(queuedAt, workflowClaimLimit),
jobsExhausted ? [] : claimJobSchedules(queuedAt, JOB_CHUNK_SIZE),
])
remainingWorkflowBudget -= dueSchedules.length
if (dueSchedules.length < workflowClaimLimit || remainingWorkflowBudget <= 0) {
schedulesExhausted = true
}
if (dueJobs.length < JOB_CHUNK_SIZE) jobsExhausted = true
if (dueSchedules.length === 0 && dueJobs.length === 0 && resumedPendingSchedules === 0) break
iterations += 1
totalSchedules += dueSchedules.length + resumedPendingSchedules
totalJobs += dueJobs.length
logger.info(
`[${requestId}] Iteration ${iterations}: claimed ${dueSchedules.length} schedules, resumed ${resumedPendingSchedules} pending schedule jobs, ${dueJobs.length} jobs`,
{
remainingWorkflowBudget,
scheduleConcurrencyLimit: SCHEDULE_EXECUTION_CONCURRENCY_LIMIT,
databaseScheduleSlots,
}
)
const schedulePromises =
dueSchedules.length > 0
? dueSchedules.map((schedule) =>
processScheduleItem(schedule, queuedAt, requestId, jobQueue, useDatabaseFallback)
)
: []
await Promise.allSettled([
...schedulePromises,
...dueJobs.map((job) => processJobItem(job, queuedAt, requestId)),
])
}
const totalCount = totalSchedules + totalJobs
const durationMs = Date.now() - tickStart
logger.info(
`[${requestId}] Processed ${totalCount} items across ${iterations} iteration(s) in ${durationMs}ms (${totalSchedules} schedules, ${totalJobs} jobs)`,
{
scheduleConcurrencyLimit: SCHEDULE_EXECUTION_CONCURRENCY_LIMIT,
scheduleEnqueueBudget: SCHEDULE_WORKFLOW_ENQUEUE_LIMIT,
remainingWorkflowBudget,
}
)
const response = {
message: 'Scheduled workflow executions processed',
processedCount: totalCount,
} satisfies ExecuteSchedulesResponse
return NextResponse.json(response)
} catch (error) {
logger.error(`[${requestId}] Error in scheduled execution handler`, error)
return NextResponse.json({ error: toError(error).message }, { status: 500 })
}
return NextResponse.json(response, { status: 202 })
})
@@ -0,0 +1,107 @@
/**
* Tests for the webhook polling cron route.
*
* @vitest-environment node
*/
import { createMockRequest, redisConfigMock, redisConfigMockFns } from '@sim/testing'
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { mockVerifyCronAuth, mockPollProvider } = vi.hoisted(() => ({
mockVerifyCronAuth: vi.fn().mockReturnValue(null),
mockPollProvider: vi.fn().mockResolvedValue({ processed: 0 }),
}))
vi.mock('@/lib/auth/internal', () => ({
verifyCronAuth: mockVerifyCronAuth,
}))
vi.mock('@/lib/core/config/redis', () => redisConfigMock)
vi.mock('@/lib/webhooks/polling', () => ({
pollProvider: mockPollProvider,
VALID_POLLING_PROVIDERS: new Set(['gmail', 'outlook', 'rss']),
}))
import { GET } from './route'
function createRequest() {
return createMockRequest('GET', undefined, {}, 'http://localhost:3000/api/webhooks/poll/gmail')
}
function createContext(provider: string) {
return { params: Promise.resolve({ provider }) }
}
const flushMicrotasks = () => new Promise((resolve) => setTimeout(resolve, 0))
describe('webhook polling route (fire-and-forget)', () => {
beforeEach(() => {
vi.clearAllMocks()
redisConfigMockFns.mockAcquireLock.mockResolvedValue(true)
redisConfigMockFns.mockReleaseLock.mockResolvedValue(true)
mockVerifyCronAuth.mockReturnValue(null)
mockPollProvider.mockResolvedValue({ processed: 0 })
})
it('returns the auth error when cron auth fails', async () => {
mockVerifyCronAuth.mockReturnValueOnce(
new Response(null, { status: 401 }) as unknown as Response
)
const response = await GET(createRequest(), createContext('gmail'))
expect(response.status).toBe(401)
expect(mockPollProvider).not.toHaveBeenCalled()
})
it('returns 404 for an unknown provider', async () => {
const response = await GET(createRequest(), createContext('unknown'))
expect(response.status).toBe(404)
expect(redisConfigMockFns.mockAcquireLock).not.toHaveBeenCalled()
})
it('acknowledges with 202 and polls in the background after acquiring the lock', async () => {
const response = await GET(createRequest(), createContext('gmail'))
expect(response.status).toBe(202)
const data = await response.json()
expect(data).toMatchObject({ status: 'started' })
expect(redisConfigMockFns.mockAcquireLock).toHaveBeenCalledWith(
'gmail-polling-lock',
expect.any(String),
expect.any(Number)
)
await flushMicrotasks()
expect(mockPollProvider).toHaveBeenCalledWith('gmail')
expect(redisConfigMockFns.mockReleaseLock).toHaveBeenCalledWith(
'gmail-polling-lock',
expect.any(String)
)
})
it('skips with 202 when the lock is already held', async () => {
redisConfigMockFns.mockAcquireLock.mockResolvedValueOnce(false)
const response = await GET(createRequest(), createContext('gmail'))
expect(response.status).toBe(202)
const data = await response.json()
expect(data).toMatchObject({ status: 'skip' })
expect(mockPollProvider).not.toHaveBeenCalled()
})
it('releases the lock even when polling throws', async () => {
mockPollProvider.mockRejectedValueOnce(new Error('poll failed'))
const response = await GET(createRequest(), createContext('gmail'))
expect(response.status).toBe(202)
await flushMicrotasks()
expect(redisConfigMockFns.mockReleaseLock).toHaveBeenCalledWith(
'gmail-polling-lock',
expect.any(String)
)
})
})
@@ -6,6 +6,7 @@ import { webhookPollingContract } from '@/lib/api/contracts/webhooks'
import { parseRequest } from '@/lib/api/server'
import { verifyCronAuth } from '@/lib/auth/internal'
import { acquireLock, releaseLock } from '@/lib/core/config/redis'
import { runDetached } from '@/lib/core/utils/background'
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
import { pollProvider, VALID_POLLING_PROVIDERS } from '@/lib/webhooks/polling'
@@ -38,37 +39,38 @@ export const GET = withRouteHandler(
}
const LOCK_KEY = `${provider}-polling-lock`
let lockValue: string | undefined
const lockValue = requestId
const locked = await acquireLock(LOCK_KEY, lockValue, LOCK_TTL_SECONDS)
if (!locked) {
return NextResponse.json(
{
success: true,
message: 'Polling already in progress – skipped',
requestId,
status: 'skip',
},
{ status: 202 }
)
}
try {
lockValue = requestId
const locked = await acquireLock(LOCK_KEY, lockValue, LOCK_TTL_SECONDS)
if (!locked) {
return NextResponse.json(
{
success: true,
message: 'Polling already in progress – skipped',
requestId,
status: 'skip',
},
{ status: 202 }
)
}
const results = await pollProvider(provider)
return NextResponse.json({
success: true,
message: `${provider} polling completed`,
requestId,
status: 'completed',
...results,
})
} finally {
if (lockValue) {
const pollingProvider = provider
runDetached(`${pollingProvider}-polling`, async () => {
try {
await pollProvider(pollingProvider)
} finally {
await releaseLock(LOCK_KEY, lockValue).catch(() => {})
}
}
})
return NextResponse.json(
{
success: true,
message: `${provider} polling started`,
requestId,
status: 'started',
},
{ status: 202 }
)
} catch (error) {
const providerLabel = provider ?? 'webhook'
logger.error(`Error during ${providerLabel} polling (${requestId}):`, error)
+1 -1
View File
@@ -134,7 +134,7 @@ const messageResponseSchema = z.object({
export const executeSchedulesResponseSchema = z.object({
message: z.string(),
processedCount: z.number().int().min(0),
status: z.literal('started'),
})
export type ExecuteSchedulesResponse = z.output<typeof executeSchedulesResponseSchema>
@@ -0,0 +1,35 @@
/**
* @vitest-environment node
*/
import { describe, expect, it, vi } from 'vitest'
import { runDetached } from '@/lib/core/utils/background'
const flushMicrotasks = () => new Promise((resolve) => setTimeout(resolve, 0))
describe('runDetached', () => {
it('runs the work without the caller awaiting it', async () => {
const work = vi.fn().mockResolvedValue(undefined)
runDetached('test', work)
await flushMicrotasks()
expect(work).toHaveBeenCalledTimes(1)
})
it('swallows rejections so they do not surface as unhandled', async () => {
const work = vi.fn().mockRejectedValue(new Error('boom'))
expect(() => runDetached('test', work)).not.toThrow()
await flushMicrotasks()
expect(work).toHaveBeenCalledTimes(1)
})
it('swallows synchronous throws from work', async () => {
const work = vi.fn(() => {
throw new Error('sync boom')
})
expect(() => runDetached('test', work)).not.toThrow()
await flushMicrotasks()
})
})
+26
View File
@@ -0,0 +1,26 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
const logger = createLogger('BackgroundTask')
/**
* Runs work detached from the HTTP response so a caller (e.g. a cron job with a
* short request timeout) receives an immediate response while processing
* continues on the long-lived server process.
*
* `withRouteHandler` only wraps awaited work in its try/catch, so a detached
* promise must catch its own rejection or it surfaces as an `unhandledRejection`.
* The request-scoped AsyncLocalStorage context (request ID) is captured when the
* work is scheduled and preserved across the detached continuation, so loggers
* inside `work` keep the originating request ID.
*
* @param label - Identifier used in the failure log line.
* @param work - The async work to run in the background.
*/
export function runDetached(label: string, work: () => Promise<unknown>): void {
void Promise.resolve()
.then(work)
.catch((error) => {
logger.error(`Background task failed: ${label}`, toError(error))
})
}