mirror of
https://github.com/simstudioai/sim.git
synced 2026-09-24 15:45:35 +08:00
fix(selectors): fetch all pages for paginated dropdown list routes (#4823)
* fix(selectors): fetch all pages for paginated dropdown list routes Dropdown selectors fetched only the first page of paginated provider APIs, silently hiding results past page one. Add bounded server-side draining to the list routes across Microsoft Graph, Google, Notion, Atlassian, Linear, AWS CloudWatch, and offset/token REST APIs, plus a shared client-side drain cap in the selector hook. Response shapes, stored values, and tool execution are unchanged; CloudWatch list tools still honor a caller-supplied limit. Also fixes the Word file picker that was searching for .xlsx files. * fix(selectors): harden JSM and Monday pagination draining - JSM service-desk/request-type drains advance `start` by the actual row count returned (not the fixed page size) and stop on an empty page, so a short non-final page can't skip items. - Monday boards drain now checks `response.ok` per page, surfacing a mid-drain HTTP failure instead of treating it as an empty final page and returning a partial 200. * docs(selectors): clarify JSM drain advances start by actual row count The offset-advancement fix (advance `start` by the rows returned, not the fixed page size) landed in 7b19788a8; update the TSDoc to match so it no longer reads as advancing by `limit`. * fix(selectors): drain fetchPage in direct fetchList callers Making `fetchList` optional left three direct callers (outside the useSelectorOptions hook) calling it unguarded, which broke the build's type check. Route them through a shared `loadAllSelectorOptions` helper that uses `fetchList` when present and otherwise drains `fetchPage`. This also prevents a regression: `confluence.spaces` / `knowledge.documents` now paginate via `fetchPage` only, and these callers (search/replace, value resolution) would otherwise have silently returned no options. * chore(selectors): rename MAX_PAGE_PAGES to MAX_NOTION_PAGES for readability
This commit is contained in:
Waleed
@@ -8,13 +8,57 @@ import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { getCredential, refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import { GRAPH_ID_PATTERN } from '@/tools/microsoft_excel/utils'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('MicrosoftFilesAPI')
|
||||
|
||||
/**
|
||||
* Get Excel files from Microsoft OneDrive
|
||||
* Microsoft Graph paginates `search()` results via the `@odata.nextLink`
|
||||
* absolute URL in the response body. Request the largest page (`$top` caps at
|
||||
* 999) and drain following nextLink, bounded by a page cap.
|
||||
* See https://learn.microsoft.com/en-us/graph/paging
|
||||
*/
|
||||
const MICROSOFT_FILES_PAGE_SIZE = 999
|
||||
const MAX_MICROSOFT_FILES_PAGES = 20
|
||||
|
||||
interface MicrosoftGraphFile {
|
||||
id: string
|
||||
name?: string
|
||||
mimeType?: string
|
||||
webUrl?: string
|
||||
size?: number
|
||||
createdDateTime?: string
|
||||
lastModifiedDateTime?: string
|
||||
thumbnails?: Array<{ small?: { url?: string }; medium?: { url?: string } }>
|
||||
createdBy?: { user?: { displayName?: string; email?: string } }
|
||||
}
|
||||
|
||||
/**
|
||||
* The shared `/api/auth/oauth/microsoft/files` route serves both the
|
||||
* `microsoft.excel` and `microsoft.word` selectors. The two are distinguished
|
||||
* by the `fileType` query parameter the selector forwards (defaulting to
|
||||
* `excel` for backward compatibility), which drives both the search-query
|
||||
* extension hint and the server-side result filter.
|
||||
*/
|
||||
const FILE_TYPE_CONFIG = {
|
||||
excel: {
|
||||
extension: '.xlsx',
|
||||
mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
},
|
||||
word: {
|
||||
extension: '.docx',
|
||||
mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
},
|
||||
} as const
|
||||
|
||||
type MicrosoftFileType = keyof typeof FILE_TYPE_CONFIG
|
||||
|
||||
/**
|
||||
* Get Excel or Word files from Microsoft OneDrive / SharePoint. The
|
||||
* `fileType` query parameter selects which Office document type to return
|
||||
* (defaults to `excel`).
|
||||
*/
|
||||
export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
@@ -27,6 +71,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
query: searchParams.get('query') ?? undefined,
|
||||
driveId: searchParams.get('driveId') ?? undefined,
|
||||
workflowId: searchParams.get('workflowId') ?? undefined,
|
||||
fileType: searchParams.get('fileType') ?? undefined,
|
||||
})
|
||||
|
||||
if (!parsedQuery.success) {
|
||||
@@ -40,6 +85,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
const { credentialId, driveId, workflowId } = parsedQuery.data
|
||||
const query = parsedQuery.data.query ?? ''
|
||||
|
||||
const fileType: MicrosoftFileType = parsedQuery.data.fileType ?? 'excel'
|
||||
const { extension, mimeType: targetMimeType } = FILE_TYPE_CONFIG[fileType]
|
||||
|
||||
const authz = await authorizeCredentialUse(request, {
|
||||
credentialId,
|
||||
workflowId,
|
||||
@@ -72,11 +120,8 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 })
|
||||
}
|
||||
|
||||
// Build search query for Excel files
|
||||
let searchQuery = '.xlsx'
|
||||
if (query) {
|
||||
searchQuery = `${query} .xlsx`
|
||||
}
|
||||
// Build search query for the requested Office document type
|
||||
const searchQuery = query ? `${query} ${extension}` : extension
|
||||
|
||||
// Build the query parameters for Microsoft Graph API
|
||||
const searchParams_new = new URLSearchParams()
|
||||
@@ -84,7 +129,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
'$select',
|
||||
'id,name,mimeType,webUrl,thumbnails,createdDateTime,lastModifiedDateTime,size,createdBy'
|
||||
)
|
||||
searchParams_new.append('$top', '50')
|
||||
searchParams_new.append('$top', String(MICROSOFT_FILES_PAGE_SIZE))
|
||||
|
||||
// When driveId is provided (SharePoint), search within that specific drive.
|
||||
// Otherwise, search the user's personal OneDrive.
|
||||
@@ -99,44 +144,57 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
}
|
||||
const drivePath = driveId ? `drives/${driveId}` : 'me/drive'
|
||||
|
||||
const response = await fetch(
|
||||
`https://graph.microsoft.com/v1.0/${drivePath}/root/search(q='${encodeURIComponent(searchQuery)}')?${searchParams_new.toString()}`,
|
||||
{
|
||||
const rawFiles: MicrosoftGraphFile[] = []
|
||||
let nextUrl: string | undefined =
|
||||
`https://graph.microsoft.com/v1.0/${drivePath}/root/search(q='${encodeURIComponent(searchQuery)}')?${searchParams_new.toString()}`
|
||||
|
||||
for (let page = 0; page < MAX_MICROSOFT_FILES_PAGES && nextUrl; page++) {
|
||||
const response = await fetch(nextUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
logger.error(`[${requestId}] Microsoft Graph API error`, {
|
||||
status: response.status,
|
||||
error: errorData.error?.message || 'Failed to fetch Excel files from Microsoft OneDrive',
|
||||
})
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: errorData.error?.message || 'Failed to fetch Excel files from Microsoft OneDrive',
|
||||
},
|
||||
{ status: response.status }
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response
|
||||
.json()
|
||||
.catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
logger.error(`[${requestId}] Microsoft Graph API error`, {
|
||||
status: response.status,
|
||||
error: errorData.error?.message || 'Failed to fetch files from Microsoft OneDrive',
|
||||
})
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: errorData.error?.message || 'Failed to fetch files from Microsoft OneDrive',
|
||||
},
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
rawFiles.push(...((data.value as MicrosoftGraphFile[]) || []))
|
||||
|
||||
const nextLink = getGraphNextPageUrl(data)
|
||||
nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined
|
||||
|
||||
if (nextUrl && page === MAX_MICROSOFT_FILES_PAGES - 1) {
|
||||
logger.warn(
|
||||
`[${requestId}] Microsoft files search hit pagination cap; list may be incomplete`,
|
||||
{ fileType, pages: MAX_MICROSOFT_FILES_PAGES, collected: rawFiles.length }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
let files = data.value || []
|
||||
|
||||
// Transform Microsoft Graph response to match expected format and filter for Excel files
|
||||
files = files
|
||||
// Transform Microsoft Graph response and filter to the requested file type
|
||||
const files = rawFiles
|
||||
.filter(
|
||||
(file: any) =>
|
||||
file.name?.toLowerCase().endsWith('.xlsx') ||
|
||||
file.mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
||||
(file: MicrosoftGraphFile) =>
|
||||
file.name?.toLowerCase().endsWith(extension) || file.mimeType === targetMimeType
|
||||
)
|
||||
.map((file: any) => ({
|
||||
.map((file: MicrosoftGraphFile) => ({
|
||||
id: file.id,
|
||||
name: file.name,
|
||||
mimeType:
|
||||
file.mimeType || 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
mimeType: file.mimeType || targetMimeType,
|
||||
iconLink: file.thumbnails?.[0]?.small?.url,
|
||||
webViewLink: file.webUrl,
|
||||
thumbnailLink: file.thumbnails?.[0]?.medium?.url,
|
||||
@@ -155,7 +213,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
|
||||
return NextResponse.json({ files }, { status: 200 })
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error fetching Excel files from Microsoft OneDrive`, error)
|
||||
logger.error(`[${requestId}] Error fetching files from Microsoft OneDrive`, error)
|
||||
return NextResponse.json({ error: 'Internal server error' }, { status: 500 })
|
||||
}
|
||||
})
|
||||
|
||||
@@ -11,6 +11,71 @@ const logger = createLogger('AirtableBasesAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const AIRTABLE_MAX_BASES_PAGES = 50
|
||||
|
||||
interface AirtableBase {
|
||||
id: string
|
||||
name: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists all Airtable bases, following the `offset` continuation token the Meta
|
||||
* API returns (an opaque string, passed back verbatim as `?offset=`) so the
|
||||
* full set is returned. Bounded by `AIRTABLE_MAX_BASES_PAGES`; logs a warning
|
||||
* rather than silently dropping bases when the cap is hit.
|
||||
*/
|
||||
async function fetchAllBases(accessToken: string): Promise<AirtableBase[]> {
|
||||
const bases: AirtableBase[] = []
|
||||
let offset: string | undefined
|
||||
|
||||
for (let page = 0; page < AIRTABLE_MAX_BASES_PAGES; page++) {
|
||||
const url = new URL('https://api.airtable.com/v0/meta/bases')
|
||||
if (offset) {
|
||||
url.searchParams.set('offset', offset)
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
throw new AirtableFetchError(response.status, errorData)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as { bases?: AirtableBase[]; offset?: string }
|
||||
if (Array.isArray(data.bases)) {
|
||||
bases.push(...data.bases)
|
||||
}
|
||||
|
||||
offset = data.offset || undefined
|
||||
if (!offset) {
|
||||
return bases
|
||||
}
|
||||
|
||||
if (page === AIRTABLE_MAX_BASES_PAGES - 1) {
|
||||
logger.warn('Airtable bases listing hit pagination cap; base list may be incomplete', {
|
||||
pages: AIRTABLE_MAX_BASES_PAGES,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return bases
|
||||
}
|
||||
|
||||
class AirtableFetchError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly details: unknown
|
||||
) {
|
||||
super('Failed to fetch Airtable bases')
|
||||
this.name = 'AirtableFetchError'
|
||||
}
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
@@ -45,27 +110,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch('https://api.airtable.com/v0/meta/bases', {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch Airtable bases', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Airtable bases', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
let allBases: AirtableBase[]
|
||||
try {
|
||||
allBases = await fetchAllBases(accessToken)
|
||||
} catch (error) {
|
||||
if (error instanceof AirtableFetchError) {
|
||||
logger.error('Failed to fetch Airtable bases', {
|
||||
status: error.status,
|
||||
error: error.details,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Airtable bases', details: error.details },
|
||||
{ status: error.status }
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const bases = (data.bases || []).map((base: { id: string; name: string }) => ({
|
||||
const bases = allBases.map((base) => ({
|
||||
id: base.id,
|
||||
name: base.name,
|
||||
}))
|
||||
|
||||
@@ -11,6 +11,81 @@ const logger = createLogger('AsanaWorkspacesAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const ASANA_PAGE_LIMIT = 100
|
||||
const ASANA_MAX_WORKSPACES_PAGES = 50
|
||||
|
||||
interface AsanaWorkspace {
|
||||
gid: string
|
||||
name: string
|
||||
}
|
||||
|
||||
interface AsanaWorkspacesPage {
|
||||
data?: AsanaWorkspace[]
|
||||
next_page?: {
|
||||
offset?: string
|
||||
} | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists all Asana workspaces using `limit`/`offset` pagination, following
|
||||
* `next_page.offset` (an opaque token, passed back verbatim as `?offset=`)
|
||||
* until `next_page` is null so the full set is returned. Bounded by
|
||||
* `ASANA_MAX_WORKSPACES_PAGES`; logs a warning rather than silently dropping
|
||||
* workspaces when the cap is hit.
|
||||
*/
|
||||
async function fetchAllWorkspaces(accessToken: string): Promise<AsanaWorkspace[]> {
|
||||
const workspaces: AsanaWorkspace[] = []
|
||||
let offset: string | undefined
|
||||
|
||||
for (let page = 0; page < ASANA_MAX_WORKSPACES_PAGES; page++) {
|
||||
const url = new URL('https://app.asana.com/api/1.0/workspaces')
|
||||
url.searchParams.set('limit', String(ASANA_PAGE_LIMIT))
|
||||
if (offset) {
|
||||
url.searchParams.set('offset', offset)
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
throw new AsanaFetchError(response.status, errorData)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as AsanaWorkspacesPage
|
||||
if (Array.isArray(data.data)) {
|
||||
workspaces.push(...data.data)
|
||||
}
|
||||
|
||||
offset = data.next_page?.offset || undefined
|
||||
if (!offset) {
|
||||
return workspaces
|
||||
}
|
||||
|
||||
if (page === ASANA_MAX_WORKSPACES_PAGES - 1) {
|
||||
logger.warn('Asana workspaces listing hit pagination cap; workspace list may be incomplete', {
|
||||
pages: ASANA_MAX_WORKSPACES_PAGES,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return workspaces
|
||||
}
|
||||
|
||||
class AsanaFetchError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly details: unknown
|
||||
) {
|
||||
super('Failed to fetch Asana workspaces')
|
||||
this.name = 'AsanaFetchError'
|
||||
}
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
@@ -42,27 +117,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch('https://app.asana.com/api/1.0/workspaces', {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch Asana workspaces', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Asana workspaces', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
let allWorkspaces: AsanaWorkspace[]
|
||||
try {
|
||||
allWorkspaces = await fetchAllWorkspaces(accessToken)
|
||||
} catch (error) {
|
||||
if (error instanceof AsanaFetchError) {
|
||||
logger.error('Failed to fetch Asana workspaces', {
|
||||
status: error.status,
|
||||
error: error.details,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Asana workspaces', details: error.details },
|
||||
{ status: error.status }
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const workspaces = (data.data || []).map((workspace: { gid: string; name: string }) => ({
|
||||
const workspaces = allWorkspaces.map((workspace) => ({
|
||||
id: workspace.gid,
|
||||
name: workspace.name,
|
||||
}))
|
||||
|
||||
@@ -10,6 +10,12 @@ import { createCloudWatchLogsClient } from '@/app/api/tools/cloudwatch/utils'
|
||||
|
||||
const logger = createLogger('CloudWatchDescribeLogGroups')
|
||||
|
||||
/** AWS DescribeLogGroups caps `limit` at 50 items per page. */
|
||||
const LOG_GROUPS_PAGE_SIZE = 50
|
||||
|
||||
/** Upper bound on pages drained to avoid unbounded loops on very large accounts. */
|
||||
const MAX_LOG_GROUPS_PAGES = 20
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
const auth = await checkSessionOrInternalAuth(request)
|
||||
@@ -33,26 +39,58 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
})
|
||||
|
||||
try {
|
||||
const command = new DescribeLogGroupsCommand({
|
||||
...(validatedData.prefix && { logGroupNamePrefix: validatedData.prefix }),
|
||||
...(validatedData.limit !== undefined && { limit: validatedData.limit }),
|
||||
})
|
||||
const totalLimit = validatedData.limit
|
||||
const logGroups: {
|
||||
logGroupName: string
|
||||
arn: string
|
||||
storedBytes: number
|
||||
retentionInDays: number | undefined
|
||||
creationTime: number | undefined
|
||||
}[] = []
|
||||
let nextToken: string | undefined
|
||||
|
||||
const response = await client.send(command)
|
||||
for (let page = 0; page < MAX_LOG_GROUPS_PAGES; page++) {
|
||||
const pageLimit =
|
||||
totalLimit !== undefined
|
||||
? Math.min(LOG_GROUPS_PAGE_SIZE, totalLimit - logGroups.length)
|
||||
: LOG_GROUPS_PAGE_SIZE
|
||||
|
||||
const logGroups = (response.logGroups ?? []).map((lg) => ({
|
||||
logGroupName: lg.logGroupName ?? '',
|
||||
arn: lg.arn ?? '',
|
||||
storedBytes: lg.storedBytes ?? 0,
|
||||
retentionInDays: lg.retentionInDays,
|
||||
creationTime: lg.creationTime,
|
||||
}))
|
||||
const command = new DescribeLogGroupsCommand({
|
||||
...(validatedData.prefix && { logGroupNamePrefix: validatedData.prefix }),
|
||||
limit: pageLimit,
|
||||
...(nextToken && { nextToken }),
|
||||
})
|
||||
|
||||
logger.info(`Successfully described ${logGroups.length} log groups`)
|
||||
const response = await client.send(command)
|
||||
|
||||
for (const lg of response.logGroups ?? []) {
|
||||
logGroups.push({
|
||||
logGroupName: lg.logGroupName ?? '',
|
||||
arn: lg.arn ?? '',
|
||||
storedBytes: lg.storedBytes ?? 0,
|
||||
retentionInDays: lg.retentionInDays,
|
||||
creationTime: lg.creationTime,
|
||||
})
|
||||
}
|
||||
|
||||
nextToken = response.nextToken
|
||||
if (!nextToken) break
|
||||
if (totalLimit !== undefined && logGroups.length >= totalLimit) break
|
||||
|
||||
if (page === MAX_LOG_GROUPS_PAGES - 1) {
|
||||
logger.warn(
|
||||
`DescribeLogGroups hit pagination cap of ${MAX_LOG_GROUPS_PAGES} pages; log group list may be incomplete`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const cappedLogGroups = totalLimit !== undefined ? logGroups.slice(0, totalLimit) : logGroups
|
||||
|
||||
logger.info(`Successfully described ${cappedLogGroups.length} log groups`)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
output: { logGroups },
|
||||
output: { logGroups: cappedLogGroups },
|
||||
})
|
||||
} finally {
|
||||
client.destroy()
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
GetQueryResultsCommand,
|
||||
type ResultField,
|
||||
} from '@aws-sdk/client-cloudwatch-logs'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { sleep } from '@sim/utils/helpers'
|
||||
import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
|
||||
|
||||
@@ -96,37 +97,82 @@ export async function pollQueryResults(
|
||||
}
|
||||
}
|
||||
|
||||
/** AWS DescribeLogStreams caps `limit` at 50 items per page. */
|
||||
const LOG_STREAMS_PAGE_SIZE = 50
|
||||
|
||||
/** Upper bound on pages drained to avoid unbounded loops on log groups with many streams. */
|
||||
const MAX_LOG_STREAMS_PAGES = 20
|
||||
|
||||
const logger = createLogger('CloudWatchUtils')
|
||||
|
||||
interface DescribedLogStream {
|
||||
logStreamName: string
|
||||
lastEventTimestamp: number | undefined
|
||||
firstEventTimestamp: number | undefined
|
||||
creationTime: number | undefined
|
||||
storedBytes: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists log streams for a log group, following `nextToken` so the complete set
|
||||
* is returned rather than just the first page. Bounded by
|
||||
* `MAX_LOG_STREAMS_PAGES`; logs a warning rather than silently dropping streams
|
||||
* when the cap is hit. Ordering/prefix inputs are preserved across all pages.
|
||||
*
|
||||
* When `limit` is provided it is treated as a total result cap: draining stops
|
||||
* once enough streams have been collected. When omitted, every page is drained.
|
||||
*/
|
||||
export async function describeLogStreams(
|
||||
client: CloudWatchLogsClient,
|
||||
logGroupName: string,
|
||||
options?: { prefix?: string; limit?: number }
|
||||
): Promise<{
|
||||
logStreams: {
|
||||
logStreamName: string
|
||||
lastEventTimestamp: number | undefined
|
||||
firstEventTimestamp: number | undefined
|
||||
creationTime: number | undefined
|
||||
storedBytes: number
|
||||
}[]
|
||||
}> {
|
||||
): Promise<{ logStreams: DescribedLogStream[] }> {
|
||||
const hasPrefix = Boolean(options?.prefix)
|
||||
const command = new DescribeLogStreamsCommand({
|
||||
logGroupName,
|
||||
...(hasPrefix
|
||||
? { orderBy: 'LogStreamName', logStreamNamePrefix: options!.prefix }
|
||||
: { orderBy: 'LastEventTime', descending: true }),
|
||||
...(options?.limit !== undefined && { limit: options.limit }),
|
||||
})
|
||||
const totalLimit = options?.limit
|
||||
const logStreams: DescribedLogStream[] = []
|
||||
let nextToken: string | undefined
|
||||
|
||||
for (let page = 0; page < MAX_LOG_STREAMS_PAGES; page++) {
|
||||
const pageLimit =
|
||||
totalLimit !== undefined
|
||||
? Math.min(LOG_STREAMS_PAGE_SIZE, totalLimit - logStreams.length)
|
||||
: LOG_STREAMS_PAGE_SIZE
|
||||
|
||||
const command = new DescribeLogStreamsCommand({
|
||||
logGroupName,
|
||||
...(hasPrefix
|
||||
? { orderBy: 'LogStreamName', logStreamNamePrefix: options!.prefix }
|
||||
: { orderBy: 'LastEventTime', descending: true }),
|
||||
limit: pageLimit,
|
||||
...(nextToken && { nextToken }),
|
||||
})
|
||||
|
||||
const response = await client.send(command)
|
||||
|
||||
for (const ls of response.logStreams ?? []) {
|
||||
logStreams.push({
|
||||
logStreamName: ls.logStreamName ?? '',
|
||||
lastEventTimestamp: ls.lastEventTimestamp,
|
||||
firstEventTimestamp: ls.firstEventTimestamp,
|
||||
creationTime: ls.creationTime,
|
||||
storedBytes: ls.storedBytes ?? 0,
|
||||
})
|
||||
}
|
||||
|
||||
nextToken = response.nextToken
|
||||
if (!nextToken) break
|
||||
if (totalLimit !== undefined && logStreams.length >= totalLimit) break
|
||||
|
||||
if (page === MAX_LOG_STREAMS_PAGES - 1) {
|
||||
logger.warn(
|
||||
`DescribeLogStreams hit pagination cap of ${MAX_LOG_STREAMS_PAGES} pages; log stream list may be incomplete`,
|
||||
{ logGroupName }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const response = await client.send(command)
|
||||
return {
|
||||
logStreams: (response.logStreams ?? []).map((ls) => ({
|
||||
logStreamName: ls.logStreamName ?? '',
|
||||
lastEventTimestamp: ls.lastEventTimestamp,
|
||||
firstEventTimestamp: ls.firstEventTimestamp,
|
||||
creationTime: ls.creationTime,
|
||||
storedBytes: ls.storedBytes ?? 0,
|
||||
})),
|
||||
logStreams: totalLimit !== undefined ? logStreams.slice(0, totalLimit) : logStreams,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,12 +7,21 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { validateAlphanumericId } from '@/lib/core/security/input-validation'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination'
|
||||
import { getScopesForService } from '@/lib/oauth/utils'
|
||||
import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils'
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('GoogleDriveFilesAPI')
|
||||
|
||||
const MAX_DRIVE_FILE_PAGES = 20
|
||||
const DRIVE_FILE_PAGE_SIZE = 100
|
||||
|
||||
interface DriveFilesResponse {
|
||||
files?: DriveFile[]
|
||||
nextPageToken?: string
|
||||
}
|
||||
|
||||
function escapeForDriveQuery(value: string): string {
|
||||
return value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")
|
||||
}
|
||||
@@ -138,34 +147,56 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
if (query) {
|
||||
qParts.push(`name contains '${escapeForDriveQuery(query)}'`)
|
||||
}
|
||||
const q = encodeURIComponent(qParts.join(' and '))
|
||||
const q = qParts.join(' and ')
|
||||
|
||||
const response = await fetch(
|
||||
`https://www.googleapis.com/drive/v3/files?q=${q}&corpora=allDrives&supportsAllDrives=true&includeItemsFromAllDrives=true&fields=files(id,name,mimeType,iconLink,webViewLink,thumbnailLink,createdTime,modifiedTime,size,owners,parents)`,
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
let files: DriveFile[]
|
||||
try {
|
||||
const drained = await drainGooglePagedList<DriveFile, DriveFilesResponse>({
|
||||
buildUrl: (pageToken) => {
|
||||
const url = new URL('https://www.googleapis.com/drive/v3/files')
|
||||
url.searchParams.set('q', q)
|
||||
url.searchParams.set('corpora', 'allDrives')
|
||||
url.searchParams.set('supportsAllDrives', 'true')
|
||||
url.searchParams.set('includeItemsFromAllDrives', 'true')
|
||||
url.searchParams.set('pageSize', String(DRIVE_FILE_PAGE_SIZE))
|
||||
url.searchParams.set(
|
||||
'fields',
|
||||
'nextPageToken,files(id,name,mimeType,iconLink,webViewLink,thumbnailLink,createdTime,modifiedTime,size,owners,parents)'
|
||||
)
|
||||
if (pageToken) url.searchParams.set('pageToken', pageToken)
|
||||
return url.toString()
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
logger.error(`[${requestId}] Google Drive API error`, {
|
||||
status: response.status,
|
||||
error: error.error?.message || 'Failed to fetch files from Google Drive',
|
||||
fetch: (url) =>
|
||||
fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
}),
|
||||
parseError: (response) =>
|
||||
response.json().catch(() => ({ error: { message: 'Unknown error' } })),
|
||||
getItems: (body) => body.files,
|
||||
getNextPageToken: (body) => body.nextPageToken,
|
||||
maxPages: MAX_DRIVE_FILE_PAGES,
|
||||
label: 'Google Drive files',
|
||||
})
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: error.error?.message || 'Failed to fetch files from Google Drive',
|
||||
},
|
||||
{ status: response.status }
|
||||
)
|
||||
files = drained.items
|
||||
} catch (error) {
|
||||
if (error instanceof GooglePageError) {
|
||||
const errorBody = error.body as { error?: { message?: string } }
|
||||
logger.error(`[${requestId}] Google Drive API error`, {
|
||||
status: error.status,
|
||||
error: errorBody?.error?.message || 'Failed to fetch files from Google Drive',
|
||||
})
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: errorBody?.error?.message || 'Failed to fetch files from Google Drive',
|
||||
},
|
||||
{ status: error.status }
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
let files: DriveFile[] = data.files || []
|
||||
|
||||
if (mimeType === 'application/vnd.google-apps.spreadsheet') {
|
||||
files = files.filter(
|
||||
(file: DriveFile) => file.mimeType === 'application/vnd.google-apps.spreadsheet'
|
||||
|
||||
@@ -5,6 +5,7 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server'
|
||||
import { authorizeCredentialUse } from '@/lib/auth/credential-access'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination'
|
||||
import { getScopesForService } from '@/lib/oauth/utils'
|
||||
import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils'
|
||||
|
||||
@@ -12,6 +13,19 @@ const logger = createLogger('GoogleBigQueryDatasetsAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const MAX_DATASET_PAGES = 20
|
||||
const DATASET_PAGE_SIZE = 200
|
||||
|
||||
interface BigQueryDataset {
|
||||
datasetReference: { datasetId: string; projectId: string }
|
||||
friendlyName?: string
|
||||
}
|
||||
|
||||
interface BigQueryDatasetsResponse {
|
||||
datasets?: BigQueryDataset[]
|
||||
nextPageToken?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/tools/google_bigquery/datasets
|
||||
*
|
||||
@@ -71,41 +85,46 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch(
|
||||
`https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets?maxResults=200`,
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}
|
||||
)
|
||||
const { items } = await drainGooglePagedList<BigQueryDataset, BigQueryDatasetsResponse>({
|
||||
buildUrl: (pageToken) => {
|
||||
const url = new URL(
|
||||
`https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets`
|
||||
)
|
||||
url.searchParams.set('maxResults', String(DATASET_PAGE_SIZE))
|
||||
if (pageToken) url.searchParams.set('pageToken', pageToken)
|
||||
return url.toString()
|
||||
},
|
||||
fetch: (url) =>
|
||||
fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}),
|
||||
parseError: (response) => response.json().catch(() => ({})),
|
||||
getItems: (body) => body.datasets,
|
||||
getNextPageToken: (body) => body.nextPageToken,
|
||||
maxPages: MAX_DATASET_PAGES,
|
||||
label: 'BigQuery datasets',
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch BigQuery datasets', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch BigQuery datasets', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const datasets = (data.datasets || []).map(
|
||||
(ds: {
|
||||
datasetReference: { datasetId: string; projectId: string }
|
||||
friendlyName?: string
|
||||
}) => ({
|
||||
datasetReference: ds.datasetReference,
|
||||
friendlyName: ds.friendlyName,
|
||||
})
|
||||
)
|
||||
const datasets = items.map((ds) => ({
|
||||
datasetReference: ds.datasetReference,
|
||||
friendlyName: ds.friendlyName,
|
||||
}))
|
||||
|
||||
return NextResponse.json({ datasets })
|
||||
} catch (error) {
|
||||
if (error instanceof GooglePageError) {
|
||||
logger.error('Failed to fetch BigQuery datasets', {
|
||||
status: error.status,
|
||||
error: error.body,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch BigQuery datasets', details: error.body },
|
||||
{ status: error.status }
|
||||
)
|
||||
}
|
||||
if (error instanceof ServiceAccountTokenError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 400 })
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server'
|
||||
import { authorizeCredentialUse } from '@/lib/auth/credential-access'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination'
|
||||
import { getScopesForService } from '@/lib/oauth/utils'
|
||||
import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils'
|
||||
|
||||
@@ -12,6 +13,19 @@ const logger = createLogger('GoogleBigQueryTablesAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const MAX_TABLE_PAGES = 20
|
||||
const TABLE_PAGE_SIZE = 200
|
||||
|
||||
interface BigQueryTable {
|
||||
tableReference: { tableId: string }
|
||||
friendlyName?: string
|
||||
}
|
||||
|
||||
interface BigQueryTablesResponse {
|
||||
tables?: BigQueryTable[]
|
||||
nextPageToken?: string
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
@@ -68,38 +82,46 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch(
|
||||
`https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets/${encodeURIComponent(datasetId)}/tables?maxResults=200`,
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}
|
||||
)
|
||||
const { items } = await drainGooglePagedList<BigQueryTable, BigQueryTablesResponse>({
|
||||
buildUrl: (pageToken) => {
|
||||
const url = new URL(
|
||||
`https://bigquery.googleapis.com/bigquery/v2/projects/${encodeURIComponent(projectId)}/datasets/${encodeURIComponent(datasetId)}/tables`
|
||||
)
|
||||
url.searchParams.set('maxResults', String(TABLE_PAGE_SIZE))
|
||||
if (pageToken) url.searchParams.set('pageToken', pageToken)
|
||||
return url.toString()
|
||||
},
|
||||
fetch: (url) =>
|
||||
fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}),
|
||||
parseError: (response) => response.json().catch(() => ({})),
|
||||
getItems: (body) => body.tables,
|
||||
getNextPageToken: (body) => body.nextPageToken,
|
||||
maxPages: MAX_TABLE_PAGES,
|
||||
label: 'BigQuery tables',
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch BigQuery tables', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch BigQuery tables', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const tables = (data.tables || []).map(
|
||||
(t: { tableReference: { tableId: string }; friendlyName?: string }) => ({
|
||||
tableReference: t.tableReference,
|
||||
friendlyName: t.friendlyName,
|
||||
})
|
||||
)
|
||||
const tables = items.map((t) => ({
|
||||
tableReference: t.tableReference,
|
||||
friendlyName: t.friendlyName,
|
||||
}))
|
||||
|
||||
return NextResponse.json({ tables })
|
||||
} catch (error) {
|
||||
if (error instanceof GooglePageError) {
|
||||
logger.error('Failed to fetch BigQuery tables', {
|
||||
status: error.status,
|
||||
error: error.body,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch BigQuery tables', details: error.body },
|
||||
{ status: error.status }
|
||||
)
|
||||
}
|
||||
if (error instanceof ServiceAccountTokenError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 400 })
|
||||
}
|
||||
|
||||
@@ -5,12 +5,16 @@ import { parseRequest } from '@/lib/api/server'
|
||||
import { authorizeCredentialUse } from '@/lib/auth/credential-access'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination'
|
||||
import { getScopesForService } from '@/lib/oauth/utils'
|
||||
import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils'
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('GoogleCalendarAPI')
|
||||
|
||||
const MAX_CALENDAR_PAGES = 20
|
||||
const CALENDAR_PAGE_SIZE = 250
|
||||
|
||||
interface CalendarListItem {
|
||||
id: string
|
||||
summary: string
|
||||
@@ -21,6 +25,11 @@ interface CalendarListItem {
|
||||
foregroundColor?: string
|
||||
}
|
||||
|
||||
interface CalendarListResponse {
|
||||
items?: CalendarListItem[]
|
||||
nextPageToken?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Get calendars from Google Calendar
|
||||
*/
|
||||
@@ -64,35 +73,50 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Fetching calendars from Google Calendar API`)
|
||||
const calendarResponse = await fetch(
|
||||
'https://www.googleapis.com/calendar/v3/users/me/calendarList',
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
|
||||
let calendars: CalendarListItem[]
|
||||
try {
|
||||
const drained = await drainGooglePagedList<CalendarListItem, CalendarListResponse>({
|
||||
buildUrl: (pageToken) => {
|
||||
const url = new URL('https://www.googleapis.com/calendar/v3/users/me/calendarList')
|
||||
url.searchParams.set('maxResults', String(CALENDAR_PAGE_SIZE))
|
||||
if (pageToken) url.searchParams.set('pageToken', pageToken)
|
||||
return url.toString()
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if (!calendarResponse.ok) {
|
||||
const errorData = await calendarResponse
|
||||
.text()
|
||||
.then((text) => JSON.parse(text))
|
||||
.catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
logger.error(`[${requestId}] Google Calendar API error`, {
|
||||
status: calendarResponse.status,
|
||||
error: errorData.error?.message || 'Failed to fetch calendars',
|
||||
fetch: (url) =>
|
||||
fetch(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}),
|
||||
parseError: (response) =>
|
||||
response
|
||||
.text()
|
||||
.then((text) => JSON.parse(text))
|
||||
.catch(() => ({ error: { message: 'Unknown error' } })),
|
||||
getItems: (body) => body.items,
|
||||
getNextPageToken: (body) => body.nextPageToken,
|
||||
maxPages: MAX_CALENDAR_PAGES,
|
||||
label: 'Google Calendar calendars',
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: errorData.error?.message || 'Failed to fetch calendars' },
|
||||
{ status: calendarResponse.status }
|
||||
)
|
||||
calendars = drained.items
|
||||
} catch (error) {
|
||||
if (error instanceof GooglePageError) {
|
||||
const errorData = error.body as { error?: { message?: string } }
|
||||
logger.error(`[${requestId}] Google Calendar API error`, {
|
||||
status: error.status,
|
||||
error: errorData?.error?.message || 'Failed to fetch calendars',
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: errorData?.error?.message || 'Failed to fetch calendars' },
|
||||
{ status: error.status }
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
const data = await calendarResponse.json()
|
||||
const calendars: CalendarListItem[] = data.items || []
|
||||
|
||||
calendars.sort((a, b) => {
|
||||
if (a.primary && !b.primary) return -1
|
||||
if (!a.primary && b.primary) return 1
|
||||
|
||||
@@ -5,6 +5,7 @@ import { parseRequest } from '@/lib/api/server'
|
||||
import { authorizeCredentialUse } from '@/lib/auth/credential-access'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { drainGooglePagedList, GooglePageError } from '@/lib/oauth/google-pagination'
|
||||
import { getScopesForService } from '@/lib/oauth/utils'
|
||||
import { refreshAccessTokenIfNeeded, ServiceAccountTokenError } from '@/app/api/auth/oauth/utils'
|
||||
|
||||
@@ -12,6 +13,19 @@ const logger = createLogger('GoogleTasksTaskListsAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const MAX_TASK_LIST_PAGES = 20
|
||||
const TASK_LIST_PAGE_SIZE = 1000
|
||||
|
||||
interface GoogleTaskList {
|
||||
id: string
|
||||
title: string
|
||||
}
|
||||
|
||||
interface GoogleTaskListsResponse {
|
||||
items?: GoogleTaskList[]
|
||||
nextPageToken?: string
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
@@ -56,33 +70,44 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch('https://tasks.googleapis.com/tasks/v1/users/@me/lists', {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
const { items } = await drainGooglePagedList<GoogleTaskList, GoogleTaskListsResponse>({
|
||||
buildUrl: (pageToken) => {
|
||||
const url = new URL('https://tasks.googleapis.com/tasks/v1/users/@me/lists')
|
||||
url.searchParams.set('maxResults', String(TASK_LIST_PAGE_SIZE))
|
||||
if (pageToken) url.searchParams.set('pageToken', pageToken)
|
||||
return url.toString()
|
||||
},
|
||||
fetch: (url) =>
|
||||
fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}),
|
||||
parseError: (response) => response.json().catch(() => ({})),
|
||||
getItems: (body) => body.items,
|
||||
getNextPageToken: (body) => body.nextPageToken,
|
||||
maxPages: MAX_TASK_LIST_PAGES,
|
||||
label: 'Google Tasks task lists',
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch Google Tasks task lists', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Google Tasks task lists', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const taskLists = (data.items || []).map((list: { id: string; title: string }) => ({
|
||||
const taskLists = items.map((list) => ({
|
||||
id: list.id,
|
||||
title: list.title,
|
||||
}))
|
||||
|
||||
return NextResponse.json({ taskLists })
|
||||
} catch (error) {
|
||||
if (error instanceof GooglePageError) {
|
||||
logger.error('Failed to fetch Google Tasks task lists', {
|
||||
status: error.status,
|
||||
error: error.body,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Google Tasks task lists', details: error.body },
|
||||
{ status: error.status }
|
||||
)
|
||||
}
|
||||
if (error instanceof ServiceAccountTokenError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 400 })
|
||||
}
|
||||
|
||||
@@ -14,6 +14,77 @@ export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('JiraProjectsAPI')
|
||||
|
||||
const JIRA_PROJECTS_PAGE_SIZE = 50
|
||||
const MAX_JIRA_PROJECTS_PAGES = 40
|
||||
|
||||
interface JiraProjectSearchPage {
|
||||
values?: unknown[]
|
||||
isLast?: boolean
|
||||
maxResults?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Drains the offset-paginated Jira `/project/search` endpoint, advancing
|
||||
* `startAt` by the server-returned page size until `isLast === true` (or a short
|
||||
* page is seen). Bounded by `MAX_JIRA_PROJECTS_PAGES`; emits a `logger.warn` and
|
||||
* returns the partial set rather than looping unbounded when the cap is hit.
|
||||
*/
|
||||
async function fetchAllJiraProjects(
|
||||
apiUrl: string,
|
||||
baseParams: URLSearchParams,
|
||||
accessToken: string
|
||||
): Promise<{ values: unknown[]; lastResponse: Response }> {
|
||||
const values: unknown[] = []
|
||||
let startAt = 0
|
||||
let lastResponse: Response
|
||||
|
||||
for (let page = 0; page < MAX_JIRA_PROJECTS_PAGES; page++) {
|
||||
const params = new URLSearchParams(baseParams)
|
||||
params.set('startAt', String(startAt))
|
||||
params.set('maxResults', String(JIRA_PROJECTS_PAGE_SIZE))
|
||||
|
||||
const finalUrl = `${apiUrl}?${params.toString()}`
|
||||
logger.info(`Fetching Jira projects from: ${finalUrl}`)
|
||||
|
||||
const response = await fetch(finalUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
logger.info(`Response status: ${response.status} ${response.statusText}`)
|
||||
|
||||
if (!response.ok) {
|
||||
return { values, lastResponse: response }
|
||||
}
|
||||
|
||||
const data = (await response.json()) as JiraProjectSearchPage
|
||||
lastResponse = response
|
||||
|
||||
const pageValues = data.values ?? []
|
||||
values.push(...pageValues)
|
||||
|
||||
const pageSize =
|
||||
data.maxResults && data.maxResults > 0 ? data.maxResults : JIRA_PROJECTS_PAGE_SIZE
|
||||
if (data.isLast === true || pageValues.length < pageSize) {
|
||||
return { values, lastResponse }
|
||||
}
|
||||
|
||||
startAt += pageValues.length
|
||||
|
||||
if (page === MAX_JIRA_PROJECTS_PAGES - 1) {
|
||||
logger.warn('Jira project search hit pagination cap; project list may be incomplete', {
|
||||
pages: MAX_JIRA_PROJECTS_PAGES,
|
||||
collected: values.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return { values, lastResponse: lastResponse! }
|
||||
}
|
||||
|
||||
export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
const auth = await checkSessionOrInternalAuth(request)
|
||||
@@ -51,35 +122,28 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
queryParams.append('orderBy', 'name')
|
||||
queryParams.append('expand', 'description,lead,url,projectKeys')
|
||||
|
||||
const finalUrl = `${apiUrl}?${queryParams.toString()}`
|
||||
logger.info(`Fetching Jira projects from: ${finalUrl}`)
|
||||
const { values, lastResponse } = await fetchAllJiraProjects(apiUrl, queryParams, accessToken)
|
||||
|
||||
const response = await fetch(finalUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
logger.info(`Response status: ${response.status} ${response.statusText}`)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Jira API error:', { status: response.status, error: errorText })
|
||||
if (!lastResponse.ok) {
|
||||
const errorText = await lastResponse.text()
|
||||
logger.error('Jira API error:', { status: lastResponse.status, error: errorText })
|
||||
return NextResponse.json(
|
||||
{ error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) },
|
||||
{ status: response.status }
|
||||
{
|
||||
error: parseAtlassianErrorMessage(
|
||||
lastResponse.status,
|
||||
lastResponse.statusText,
|
||||
errorText
|
||||
),
|
||||
},
|
||||
{ status: lastResponse.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
|
||||
logger.info(`Jira API Response Status: ${response.status}`)
|
||||
logger.info(`Found projects: ${data.values?.length || 0}`)
|
||||
logger.info(`Jira API Response Status: ${lastResponse.status}`)
|
||||
logger.info(`Found projects: ${values.length}`)
|
||||
|
||||
const projects =
|
||||
data.values?.map((project: any) => ({
|
||||
values.map((project: any) => ({
|
||||
id: project.id,
|
||||
key: project.key,
|
||||
name: project.name,
|
||||
|
||||
@@ -14,6 +14,72 @@ const logger = createLogger('JsmSelectorRequestTypesAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const JSM_REQUEST_TYPES_PAGE_SIZE = 100
|
||||
const MAX_JSM_REQUEST_TYPES_PAGES = 50
|
||||
|
||||
interface JsmPagedResponse<T> {
|
||||
values?: T[]
|
||||
isLastPage?: boolean
|
||||
_links?: { next?: string }
|
||||
}
|
||||
|
||||
interface JsmRequestTypeValue {
|
||||
id: string
|
||||
name: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Drains the offset-paginated JSM `/servicedesk/{id}/requesttype` endpoint,
|
||||
* advancing `start` by the number of rows actually returned until
|
||||
* `isLastPage === true` (or `_links.next` is absent, or a page comes back
|
||||
* empty). Advancing by the real row count — not the requested `limit` —
|
||||
* prevents skipping items if the server returns a short non-final page. Bounded
|
||||
* by `MAX_JSM_REQUEST_TYPES_PAGES`; emits a `logger.warn` and returns the
|
||||
* partial set rather than looping unbounded when the cap is hit.
|
||||
*/
|
||||
async function fetchAllJsmRequestTypes(
|
||||
requestTypeUrl: string,
|
||||
accessToken: string
|
||||
): Promise<{ values: JsmRequestTypeValue[]; lastResponse: Response }> {
|
||||
const values: JsmRequestTypeValue[] = []
|
||||
let start = 0
|
||||
let lastResponse: Response
|
||||
|
||||
for (let page = 0; page < MAX_JSM_REQUEST_TYPES_PAGES; page++) {
|
||||
const url = `${requestTypeUrl}?start=${start}&limit=${JSM_REQUEST_TYPES_PAGE_SIZE}`
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
headers: getJsmHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
return { values, lastResponse: response }
|
||||
}
|
||||
|
||||
const data = (await response.json()) as JsmPagedResponse<JsmRequestTypeValue>
|
||||
lastResponse = response
|
||||
|
||||
const pageValues = data.values ?? []
|
||||
values.push(...pageValues)
|
||||
|
||||
if (data.isLastPage === true || !data._links?.next || pageValues.length === 0) {
|
||||
return { values, lastResponse }
|
||||
}
|
||||
|
||||
start += pageValues.length
|
||||
|
||||
if (page === MAX_JSM_REQUEST_TYPES_PAGES - 1) {
|
||||
logger.warn('JSM request type list hit pagination cap; list may be incomplete', {
|
||||
pages: MAX_JSM_REQUEST_TYPES_PAGES,
|
||||
collected: values.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return { values, lastResponse: lastResponse! }
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
@@ -72,28 +138,30 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
}
|
||||
|
||||
const baseUrl = getJsmApiBaseUrl(cloudIdValidation.sanitized!)
|
||||
const url = `${baseUrl}/servicedesk/${serviceDeskIdValidation.sanitized}/requesttype?limit=100`
|
||||
const requestTypeUrl = `${baseUrl}/servicedesk/${serviceDeskIdValidation.sanitized}/requesttype`
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
headers: getJsmHeaders(accessToken),
|
||||
})
|
||||
const { values, lastResponse } = await fetchAllJsmRequestTypes(requestTypeUrl, accessToken)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
if (!lastResponse.ok) {
|
||||
const errorText = await lastResponse.text()
|
||||
logger.error('JSM API error:', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
status: lastResponse.status,
|
||||
statusText: lastResponse.statusText,
|
||||
error: errorText,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) },
|
||||
{ status: response.status }
|
||||
{
|
||||
error: parseAtlassianErrorMessage(
|
||||
lastResponse.status,
|
||||
lastResponse.statusText,
|
||||
errorText
|
||||
),
|
||||
},
|
||||
{ status: lastResponse.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const requestTypes = (data.values || []).map((rt: { id: string; name: string }) => ({
|
||||
const requestTypes = values.map((rt) => ({
|
||||
id: rt.id,
|
||||
name: rt.name,
|
||||
}))
|
||||
|
||||
@@ -14,6 +14,72 @@ const logger = createLogger('JsmSelectorServiceDesksAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const JSM_SERVICE_DESKS_PAGE_SIZE = 100
|
||||
const MAX_JSM_SERVICE_DESKS_PAGES = 50
|
||||
|
||||
interface JsmPagedResponse<T> {
|
||||
values?: T[]
|
||||
isLastPage?: boolean
|
||||
_links?: { next?: string }
|
||||
}
|
||||
|
||||
interface JsmServiceDeskValue {
|
||||
id: string
|
||||
projectName: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Drains the offset-paginated JSM `/servicedesk` endpoint, advancing `start` by
|
||||
* the number of rows actually returned until `isLastPage === true` (or
|
||||
* `_links.next` is absent, or a page comes back empty). Advancing by the real
|
||||
* row count — not the requested `limit` — prevents skipping items if the server
|
||||
* returns a short non-final page. Bounded by `MAX_JSM_SERVICE_DESKS_PAGES`;
|
||||
* emits a `logger.warn` and returns the partial set rather than looping
|
||||
* unbounded when the cap is hit.
|
||||
*/
|
||||
async function fetchAllJsmServiceDesks(
|
||||
baseUrl: string,
|
||||
accessToken: string
|
||||
): Promise<{ values: JsmServiceDeskValue[]; lastResponse: Response }> {
|
||||
const values: JsmServiceDeskValue[] = []
|
||||
let start = 0
|
||||
let lastResponse: Response
|
||||
|
||||
for (let page = 0; page < MAX_JSM_SERVICE_DESKS_PAGES; page++) {
|
||||
const url = `${baseUrl}/servicedesk?start=${start}&limit=${JSM_SERVICE_DESKS_PAGE_SIZE}`
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
headers: getJsmHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
return { values, lastResponse: response }
|
||||
}
|
||||
|
||||
const data = (await response.json()) as JsmPagedResponse<JsmServiceDeskValue>
|
||||
lastResponse = response
|
||||
|
||||
const pageValues = data.values ?? []
|
||||
values.push(...pageValues)
|
||||
|
||||
if (data.isLastPage === true || !data._links?.next || pageValues.length === 0) {
|
||||
return { values, lastResponse }
|
||||
}
|
||||
|
||||
start += pageValues.length
|
||||
|
||||
if (page === MAX_JSM_SERVICE_DESKS_PAGES - 1) {
|
||||
logger.warn('JSM service desk list hit pagination cap; list may be incomplete', {
|
||||
pages: MAX_JSM_SERVICE_DESKS_PAGES,
|
||||
collected: values.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return { values, lastResponse: lastResponse! }
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
@@ -63,28 +129,29 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
}
|
||||
|
||||
const baseUrl = getJsmApiBaseUrl(cloudIdValidation.sanitized!)
|
||||
const url = `${baseUrl}/servicedesk?limit=100`
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
headers: getJsmHeaders(accessToken),
|
||||
})
|
||||
const { values, lastResponse } = await fetchAllJsmServiceDesks(baseUrl, accessToken)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
if (!lastResponse.ok) {
|
||||
const errorText = await lastResponse.text()
|
||||
logger.error('JSM API error:', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
status: lastResponse.status,
|
||||
statusText: lastResponse.statusText,
|
||||
error: errorText,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) },
|
||||
{ status: response.status }
|
||||
{
|
||||
error: parseAtlassianErrorMessage(
|
||||
lastResponse.status,
|
||||
lastResponse.statusText,
|
||||
errorText
|
||||
),
|
||||
},
|
||||
{ status: lastResponse.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const serviceDesks = (data.values || []).map((sd: { id: string; projectName: string }) => ({
|
||||
const serviceDesks = values.map((sd) => ({
|
||||
id: sd.id,
|
||||
name: sd.projectName,
|
||||
}))
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { Project } from '@linear/sdk'
|
||||
import type { Project, Team } from '@linear/sdk'
|
||||
import { LinearClient } from '@linear/sdk'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
@@ -13,6 +13,50 @@ export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('LinearProjectsAPI')
|
||||
|
||||
/** Linear's maximum page size for a single connection request. */
|
||||
const LINEAR_PAGE_SIZE = 250
|
||||
|
||||
/**
|
||||
* Upper bound on pages to drain from a single team's projects connection. At
|
||||
* 250 projects/page this covers 2,500 projects per team; the cap guards
|
||||
* against runaway loops on a broken `hasNextPage` rather than a realistic
|
||||
* limit.
|
||||
*/
|
||||
const MAX_PROJECTS_PAGES = 10
|
||||
|
||||
/**
|
||||
* Drains a single team's projects connection by following
|
||||
* `pageInfo.endCursor` until `hasNextPage` is false. Bounded by
|
||||
* `MAX_PROJECTS_PAGES`; logs a warning if the cap is hit so a truncated list
|
||||
* is visible rather than silently dropped.
|
||||
*/
|
||||
async function fetchAllTeamProjects(team: Team): Promise<Project[]> {
|
||||
const projects: Project[] = []
|
||||
let after: string | undefined
|
||||
|
||||
for (let page = 0; page < MAX_PROJECTS_PAGES; page++) {
|
||||
const result = await team.projects({ first: LINEAR_PAGE_SIZE, after })
|
||||
projects.push(...result.nodes)
|
||||
|
||||
if (!result.pageInfo.hasNextPage) {
|
||||
return projects
|
||||
}
|
||||
after = result.pageInfo.endCursor ?? undefined
|
||||
if (!after) {
|
||||
return projects
|
||||
}
|
||||
if (page === MAX_PROJECTS_PAGES - 1) {
|
||||
logger.warn('Linear projects pagination hit cap; project list may be incomplete', {
|
||||
teamId: team.id,
|
||||
cap: MAX_PROJECTS_PAGES,
|
||||
fetched: projects.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return projects
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
const parsed = await parseRequest(linearProjectsSelectorContract, request, {})
|
||||
@@ -59,8 +103,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const perTeam = await Promise.all(
|
||||
teamIds.map(async (id) => {
|
||||
const team = await linearClient.team(id)
|
||||
const result = await team.projects()
|
||||
return result.nodes.map((project: Project) => ({
|
||||
const teamProjects = await fetchAllTeamProjects(team)
|
||||
return teamProjects.map((project: Project) => ({
|
||||
id: project.id,
|
||||
name: project.name,
|
||||
}))
|
||||
|
||||
@@ -13,6 +13,48 @@ export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('LinearTeamsAPI')
|
||||
|
||||
/** Linear's maximum page size for a single connection request. */
|
||||
const LINEAR_PAGE_SIZE = 250
|
||||
|
||||
/**
|
||||
* Upper bound on pages to drain from the teams connection. At 250 teams/page
|
||||
* this covers 2,500 teams; the cap guards against runaway loops on a broken
|
||||
* `hasNextPage` rather than a realistic limit.
|
||||
*/
|
||||
const MAX_TEAMS_PAGES = 10
|
||||
|
||||
/**
|
||||
* Drains the full Linear teams connection by following
|
||||
* `pageInfo.endCursor` until `hasNextPage` is false. Bounded by
|
||||
* `MAX_TEAMS_PAGES`; logs a warning if the cap is hit so a truncated list is
|
||||
* visible rather than silently dropped.
|
||||
*/
|
||||
async function fetchAllTeams(linearClient: LinearClient): Promise<Team[]> {
|
||||
const teams: Team[] = []
|
||||
let after: string | undefined
|
||||
|
||||
for (let page = 0; page < MAX_TEAMS_PAGES; page++) {
|
||||
const result = await linearClient.teams({ first: LINEAR_PAGE_SIZE, after })
|
||||
teams.push(...result.nodes)
|
||||
|
||||
if (!result.pageInfo.hasNextPage) {
|
||||
return teams
|
||||
}
|
||||
after = result.pageInfo.endCursor ?? undefined
|
||||
if (!after) {
|
||||
return teams
|
||||
}
|
||||
if (page === MAX_TEAMS_PAGES - 1) {
|
||||
logger.warn('Linear teams pagination hit cap; team list may be incomplete', {
|
||||
cap: MAX_TEAMS_PAGES,
|
||||
fetched: teams.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return teams
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
const requestId = generateRequestId()
|
||||
@@ -45,8 +87,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
}
|
||||
|
||||
const linearClient = new LinearClient({ accessToken })
|
||||
const teamsResult = await linearClient.teams()
|
||||
const teams = teamsResult.nodes.map((team: Team) => ({
|
||||
const allTeams = await fetchAllTeams(linearClient)
|
||||
const teams = allTeams.map((team: Team) => ({
|
||||
id: team.id,
|
||||
name: team.name,
|
||||
}))
|
||||
|
||||
@@ -7,11 +7,25 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access'
|
||||
import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('TeamsChannelsAPI')
|
||||
|
||||
/**
|
||||
* Upper bound on Microsoft Graph pages drained when listing a team's channels.
|
||||
* The `teams/{id}/channels` endpoint does not support `$top`, so paging is
|
||||
* driven entirely by the server via `@odata.nextLink`. The cap prevents an
|
||||
* unbounded loop; hitting it is logged as a warning.
|
||||
*/
|
||||
const MAX_CHANNELS_PAGES = 20
|
||||
|
||||
interface GraphChannel {
|
||||
id: string
|
||||
displayName?: string
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
const parsed = await parseRequest(microsoftChannelsSelectorContract, request, {})
|
||||
@@ -52,40 +66,59 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch(
|
||||
`https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(teamId)}/channels`,
|
||||
{
|
||||
const channels: GraphChannel[] = []
|
||||
let nextPageUrl: string | undefined =
|
||||
`https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(teamId)}/channels`
|
||||
|
||||
for (let page = 0; page < MAX_CHANNELS_PAGES; page++) {
|
||||
const response = await fetch(nextPageUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json()
|
||||
logger.error('Microsoft Graph API error getting channels', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
endpoint: `https://graph.microsoft.com/v1.0/teams/${teamId}/channels`,
|
||||
})
|
||||
|
||||
if (response.status === 401) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Authentication failed. Please reconnect your Microsoft Teams account.',
|
||||
authRequired: true,
|
||||
},
|
||||
{ status: 401 }
|
||||
)
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json()
|
||||
logger.error('Microsoft Graph API error getting channels', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
endpoint: nextPageUrl,
|
||||
})
|
||||
|
||||
if (response.status === 401) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Authentication failed. Please reconnect your Microsoft Teams account.',
|
||||
authRequired: true,
|
||||
},
|
||||
{ status: 401 }
|
||||
)
|
||||
}
|
||||
|
||||
throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`)
|
||||
}
|
||||
|
||||
throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`)
|
||||
}
|
||||
const data = await response.json()
|
||||
if (Array.isArray(data.value)) {
|
||||
channels.push(...(data.value as GraphChannel[]))
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const channels = data.value
|
||||
const rawNextLink = getGraphNextPageUrl(data)
|
||||
if (!rawNextLink) {
|
||||
nextPageUrl = undefined
|
||||
break
|
||||
}
|
||||
nextPageUrl = assertGraphNextPageUrl(rawNextLink)
|
||||
|
||||
if (page === MAX_CHANNELS_PAGES - 1) {
|
||||
logger.warn(
|
||||
'Hit Microsoft Graph channels pagination cap; channel list may be incomplete',
|
||||
{ maxPages: MAX_CHANNELS_PAGES, collected: channels.length }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
channels: channels,
|
||||
|
||||
@@ -7,11 +7,29 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access'
|
||||
import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('TeamsChatsAPI')
|
||||
|
||||
/**
|
||||
* Largest page size the `me/chats` Microsoft Graph endpoint permits via `$top`.
|
||||
*/
|
||||
const CHATS_PAGE_SIZE = 50
|
||||
|
||||
/**
|
||||
* Upper bound on Microsoft Graph pages drained when listing the user's chats.
|
||||
* Paging follows `@odata.nextLink`. The cap prevents an unbounded loop; hitting
|
||||
* it is logged as a warning.
|
||||
*/
|
||||
const MAX_CHATS_PAGES = 20
|
||||
|
||||
interface GraphChat {
|
||||
id: string
|
||||
topic?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to get chat members and create a meaningful name
|
||||
*
|
||||
@@ -153,39 +171,62 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
return NextResponse.json({ error: 'Could not retrieve access token' }, { status: 401 })
|
||||
}
|
||||
|
||||
const response = await fetch('https://graph.microsoft.com/v1.0/me/chats', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
const rawChats: GraphChat[] = []
|
||||
let nextPageUrl: string | undefined =
|
||||
`https://graph.microsoft.com/v1.0/me/chats?$top=${CHATS_PAGE_SIZE}`
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json()
|
||||
logger.error('Microsoft Graph API error getting chats', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
endpoint: 'https://graph.microsoft.com/v1.0/me/chats',
|
||||
for (let page = 0; page < MAX_CHATS_PAGES; page++) {
|
||||
const response = await fetch(nextPageUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (response.status === 401) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Authentication failed. Please reconnect your Microsoft Teams account.',
|
||||
authRequired: true,
|
||||
},
|
||||
{ status: 401 }
|
||||
)
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json()
|
||||
logger.error('Microsoft Graph API error getting chats', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
endpoint: nextPageUrl,
|
||||
})
|
||||
|
||||
if (response.status === 401) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Authentication failed. Please reconnect your Microsoft Teams account.',
|
||||
authRequired: true,
|
||||
},
|
||||
{ status: 401 }
|
||||
)
|
||||
}
|
||||
|
||||
throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`)
|
||||
}
|
||||
|
||||
throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`)
|
||||
const data = await response.json()
|
||||
if (Array.isArray(data.value)) {
|
||||
rawChats.push(...(data.value as GraphChat[]))
|
||||
}
|
||||
|
||||
const rawNextLink = getGraphNextPageUrl(data)
|
||||
if (!rawNextLink) {
|
||||
nextPageUrl = undefined
|
||||
break
|
||||
}
|
||||
nextPageUrl = assertGraphNextPageUrl(rawNextLink)
|
||||
|
||||
if (page === MAX_CHATS_PAGES - 1) {
|
||||
logger.warn('Hit Microsoft Graph chats pagination cap; chat list may be incomplete', {
|
||||
maxPages: MAX_CHATS_PAGES,
|
||||
collected: rawChats.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
|
||||
const chats = await Promise.all(
|
||||
data.value.map(async (chat: any) => ({
|
||||
rawChats.map(async (chat) => ({
|
||||
id: chat.id,
|
||||
displayName: await getChatDisplayName(chat.id, accessToken, chat.topic),
|
||||
}))
|
||||
|
||||
@@ -6,11 +6,25 @@ import { parseRequest } from '@/lib/api/server'
|
||||
import { authorizeCredentialUse } from '@/lib/auth/credential-access'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('TeamsTeamsAPI')
|
||||
|
||||
/**
|
||||
* Upper bound on Microsoft Graph pages drained when listing the user's joined
|
||||
* teams. The `me/joinedTeams` endpoint does not support `$top`, so paging is
|
||||
* driven entirely by the server via `@odata.nextLink`. The cap prevents an
|
||||
* unbounded loop; hitting it is logged as a warning.
|
||||
*/
|
||||
const MAX_TEAMS_PAGES = 20
|
||||
|
||||
interface GraphTeam {
|
||||
id: string
|
||||
displayName?: string
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
const parsed = await parseRequest(microsoftTeamsSelectorContract, request, {})
|
||||
@@ -46,38 +60,59 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch('https://graph.microsoft.com/v1.0/me/joinedTeams', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
const teams: GraphTeam[] = []
|
||||
let nextPageUrl: string | undefined = 'https://graph.microsoft.com/v1.0/me/joinedTeams'
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json()
|
||||
logger.error('Microsoft Graph API error getting teams', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
endpoint: 'https://graph.microsoft.com/v1.0/me/joinedTeams',
|
||||
for (let page = 0; page < MAX_TEAMS_PAGES; page++) {
|
||||
const response = await fetch(nextPageUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
// Check for auth errors specifically
|
||||
if (response.status === 401) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Authentication failed. Please reconnect your Microsoft Teams account.',
|
||||
authRequired: true,
|
||||
},
|
||||
{ status: 401 }
|
||||
)
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json()
|
||||
logger.error('Microsoft Graph API error getting teams', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
endpoint: nextPageUrl,
|
||||
})
|
||||
|
||||
// Check for auth errors specifically
|
||||
if (response.status === 401) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Authentication failed. Please reconnect your Microsoft Teams account.',
|
||||
authRequired: true,
|
||||
},
|
||||
{ status: 401 }
|
||||
)
|
||||
}
|
||||
|
||||
throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`)
|
||||
}
|
||||
|
||||
throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`)
|
||||
}
|
||||
const data = await response.json()
|
||||
if (Array.isArray(data.value)) {
|
||||
teams.push(...(data.value as GraphTeam[]))
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const teams = data.value
|
||||
const rawNextLink = getGraphNextPageUrl(data)
|
||||
if (!rawNextLink) {
|
||||
nextPageUrl = undefined
|
||||
break
|
||||
}
|
||||
nextPageUrl = assertGraphNextPageUrl(rawNextLink)
|
||||
|
||||
if (page === MAX_TEAMS_PAGES - 1) {
|
||||
logger.warn('Hit Microsoft Graph teams pagination cap; team list may be incomplete', {
|
||||
maxPages: MAX_TEAMS_PAGES,
|
||||
collected: teams.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
teams: teams,
|
||||
|
||||
@@ -8,11 +8,19 @@ import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import { extractGraphError, GRAPH_ID_PATTERN } from '@/tools/microsoft_excel/utils'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('MicrosoftExcelDrivesAPI')
|
||||
|
||||
/**
|
||||
* Upper bound on Microsoft Graph pages drained when listing site drives.
|
||||
* Each page returns up to `$top=999` drives, so this caps the result set at
|
||||
* roughly 10k drives while preventing an unbounded server-side loop.
|
||||
*/
|
||||
const MAX_DRIVES_PAGES = 10
|
||||
|
||||
interface GraphDrive {
|
||||
id: string
|
||||
name: string
|
||||
@@ -88,25 +96,41 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
}
|
||||
|
||||
// List all drives for the site
|
||||
const url = `https://graph.microsoft.com/v1.0/sites/${siteId}/drives?$select=id,name,driveType,webUrl`
|
||||
let nextUrl: string | undefined =
|
||||
`https://graph.microsoft.com/v1.0/sites/${siteId}/drives?$select=id,name,driveType,webUrl&$top=999`
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorMessage = await extractGraphError(response)
|
||||
logger.error(`[${requestId}] Microsoft Graph API error fetching drives`, {
|
||||
status: response.status,
|
||||
error: errorMessage,
|
||||
const rawDrives: GraphDrive[] = []
|
||||
for (let page = 0; page < MAX_DRIVES_PAGES && nextUrl; page++) {
|
||||
const response = await fetch(nextUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
return NextResponse.json({ error: errorMessage }, { status: response.status })
|
||||
|
||||
if (!response.ok) {
|
||||
const errorMessage = await extractGraphError(response)
|
||||
logger.error(`[${requestId}] Microsoft Graph API error fetching drives`, {
|
||||
status: response.status,
|
||||
error: errorMessage,
|
||||
})
|
||||
return NextResponse.json({ error: errorMessage }, { status: response.status })
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
if (Array.isArray(data.value)) {
|
||||
rawDrives.push(...data.value)
|
||||
}
|
||||
|
||||
const nextLink = getGraphNextPageUrl(data)
|
||||
nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined
|
||||
if (nextUrl && page === MAX_DRIVES_PAGES - 1) {
|
||||
logger.warn(
|
||||
`[${requestId}] Site drives pagination hit ${MAX_DRIVES_PAGES}-page cap; result may be incomplete`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const drives = (data.value || []).map((drive: GraphDrive) => ({
|
||||
const drives = rawDrives.map((drive: GraphDrive) => ({
|
||||
id: drive.id,
|
||||
name: drive.name,
|
||||
driveType: drive.driveType,
|
||||
|
||||
@@ -6,11 +6,19 @@ import { authorizeCredentialUse } from '@/lib/auth/credential-access'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
const logger = createLogger('MicrosoftPlannerPlansAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
/**
|
||||
* Upper bound on Microsoft Graph pages drained when listing Planner plans.
|
||||
* Planner uses server-side paging (`$top` is generally ignored), so this caps
|
||||
* the `@odata.nextLink` follow loop to prevent an unbounded drain.
|
||||
*/
|
||||
const MAX_PLANS_PAGES = 20
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
@@ -40,25 +48,40 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch('https://graph.microsoft.com/v1.0/me/planner/plans', {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
let nextUrl: string | undefined = 'https://graph.microsoft.com/v1.0/me/planner/plans'
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error(`[${requestId}] Microsoft Graph API error:`, errorText)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch plans from Microsoft Graph' },
|
||||
{ status: response.status }
|
||||
)
|
||||
const rawPlans: { id: string; title: string }[] = []
|
||||
for (let page = 0; page < MAX_PLANS_PAGES && nextUrl; page++) {
|
||||
const response = await fetch(nextUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error(`[${requestId}] Microsoft Graph API error:`, errorText)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch plans from Microsoft Graph' },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
if (Array.isArray(data.value)) {
|
||||
rawPlans.push(...data.value)
|
||||
}
|
||||
|
||||
const nextLink = getGraphNextPageUrl(data)
|
||||
nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined
|
||||
if (nextUrl && page === MAX_PLANS_PAGES - 1) {
|
||||
logger.warn(
|
||||
`[${requestId}] Planner plans pagination hit ${MAX_PLANS_PAGES}-page cap; result may be incomplete`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const plans = data.value || []
|
||||
|
||||
const filteredPlans = plans.map((plan: { id: string; title: string }) => ({
|
||||
const filteredPlans = rawPlans.map((plan: { id: string; title: string }) => ({
|
||||
id: plan.id,
|
||||
title: plan.title,
|
||||
}))
|
||||
|
||||
@@ -8,11 +8,19 @@ import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import type { PlannerTask } from '@/tools/microsoft_planner/types'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
const logger = createLogger('MicrosoftPlannerTasksAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
/**
|
||||
* Upper bound on Microsoft Graph pages drained when listing a plan's tasks.
|
||||
* Planner uses server-side paging (`$top` is generally ignored), so this caps
|
||||
* the `@odata.nextLink` follow loop to prevent an unbounded drain.
|
||||
*/
|
||||
const MAX_TASKS_PAGES = 20
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
@@ -48,28 +56,41 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch(
|
||||
`https://graph.microsoft.com/v1.0/planner/plans/${planIdValidation.sanitized}/tasks`,
|
||||
{
|
||||
let nextUrl: string | undefined =
|
||||
`https://graph.microsoft.com/v1.0/planner/plans/${planIdValidation.sanitized}/tasks`
|
||||
|
||||
const rawTasks: PlannerTask[] = []
|
||||
for (let page = 0; page < MAX_TASKS_PAGES && nextUrl; page++) {
|
||||
const response = await fetch(nextUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error(`[${requestId}] Microsoft Graph API error:`, errorText)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch tasks from Microsoft Graph' },
|
||||
{ status: response.status }
|
||||
)
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error(`[${requestId}] Microsoft Graph API error:`, errorText)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch tasks from Microsoft Graph' },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
if (Array.isArray(data.value)) {
|
||||
rawTasks.push(...data.value)
|
||||
}
|
||||
|
||||
const nextLink = getGraphNextPageUrl(data)
|
||||
nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined
|
||||
if (nextUrl && page === MAX_TASKS_PAGES - 1) {
|
||||
logger.warn(
|
||||
`[${requestId}] Planner tasks pagination hit ${MAX_TASKS_PAGES}-page cap; result may be incomplete`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const tasks = data.value || []
|
||||
|
||||
const filteredTasks = tasks.map((task: PlannerTask) => ({
|
||||
const filteredTasks = rawTasks.map((task: PlannerTask) => ({
|
||||
id: task.id,
|
||||
title: task.title,
|
||||
planId: task.planId,
|
||||
|
||||
@@ -11,18 +11,29 @@ export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('MondayBoardsAPI')
|
||||
|
||||
/**
|
||||
* Monday's GraphQL `boards(limit: N, page: P, state: active)` has no cursor:
|
||||
* `page` starts at 1 and you stop once a page returns fewer than `limit` items
|
||||
* (or an empty page). We request the largest page (`MONDAY_BOARDS_LIMIT`) and
|
||||
* bound the drain with `MAX_MONDAY_PAGES`.
|
||||
*/
|
||||
const MONDAY_BOARDS_LIMIT = 100
|
||||
const MAX_MONDAY_PAGES = 50
|
||||
|
||||
interface MondayGraphQLError {
|
||||
message?: string
|
||||
}
|
||||
|
||||
interface MondayBoard {
|
||||
id: string
|
||||
name: string
|
||||
}
|
||||
|
||||
interface MondayBoardsResponse {
|
||||
errors?: MondayGraphQLError[]
|
||||
error_message?: string
|
||||
data?: {
|
||||
boards?: Array<{
|
||||
id: string
|
||||
name: string
|
||||
}>
|
||||
boards?: MondayBoard[]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,34 +68,68 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch('https://api.monday.com/v2', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: accessToken,
|
||||
'API-Version': '2024-10',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query: '{ boards(limit: 100, state: active) { id name } }',
|
||||
}),
|
||||
})
|
||||
const allBoards: MondayBoard[] = []
|
||||
let page = 1
|
||||
|
||||
const data = (await response.json()) as MondayBoardsResponse
|
||||
for (; page <= MAX_MONDAY_PAGES; page++) {
|
||||
const response = await fetch('https://api.monday.com/v2', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: accessToken,
|
||||
'API-Version': '2024-10',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query: `{ boards(limit: ${MONDAY_BOARDS_LIMIT}, page: ${page}, state: active) { id name } }`,
|
||||
}),
|
||||
})
|
||||
|
||||
if (data.errors?.length) {
|
||||
logger.error('Monday.com API error', { errors: data.errors })
|
||||
return NextResponse.json(
|
||||
{ error: data.errors[0].message || 'Monday.com API error' },
|
||||
{ status: 500 }
|
||||
)
|
||||
if (!response.ok) {
|
||||
const details = await response.text().catch(() => '')
|
||||
logger.error('Monday.com API HTTP error', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
details,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: `Monday.com API error: ${response.status} ${response.statusText}` },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as MondayBoardsResponse
|
||||
|
||||
if (data.errors?.length) {
|
||||
logger.error('Monday.com API error', { errors: data.errors })
|
||||
return NextResponse.json(
|
||||
{ error: data.errors[0].message || 'Monday.com API error' },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
|
||||
if (data.error_message) {
|
||||
logger.error('Monday.com API error', { error_message: data.error_message })
|
||||
return NextResponse.json({ error: data.error_message }, { status: 500 })
|
||||
}
|
||||
|
||||
const pageBoards = data.data?.boards || []
|
||||
allBoards.push(...pageBoards)
|
||||
|
||||
if (pageBoards.length < MONDAY_BOARDS_LIMIT) {
|
||||
break
|
||||
}
|
||||
|
||||
if (page === MAX_MONDAY_PAGES) {
|
||||
logger.warn(
|
||||
'Monday boards pagination hit MAX_MONDAY_PAGES cap; board list may be incomplete',
|
||||
{
|
||||
maxPages: MAX_MONDAY_PAGES,
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (data.error_message) {
|
||||
logger.error('Monday.com API error', { error_message: data.error_message })
|
||||
return NextResponse.json({ error: data.error_message }, { status: 500 })
|
||||
}
|
||||
|
||||
const boards = (data.data?.boards || []).map((board) => ({
|
||||
const boards = allBoards.map((board) => ({
|
||||
id: board.id,
|
||||
name: board.name,
|
||||
}))
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { notionDatabasesSelectorContract } from '@/lib/api/contracts/selectors'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
@@ -12,6 +13,16 @@ const logger = createLogger('NotionDatabasesAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const NOTION_PAGE_SIZE = 100
|
||||
|
||||
/**
|
||||
* Notion's `POST /v1/search` returns at most `page_size` results per call and
|
||||
* exposes `has_more`/`next_cursor` for pagination. This caps the number of
|
||||
* pages drained so a tenant with a very large workspace cannot make this route
|
||||
* loop unbounded. With `NOTION_PAGE_SIZE` of 100 this covers up to 2,000 items.
|
||||
*/
|
||||
const MAX_DATABASE_PAGES = 20
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
@@ -43,33 +54,55 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch('https://api.notion.com/v1/search', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
'Notion-Version': '2022-06-28',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
filter: { value: 'database', property: 'object' },
|
||||
page_size: 100,
|
||||
}),
|
||||
})
|
||||
const results: Record<string, unknown>[] = []
|
||||
let startCursor: string | undefined
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch Notion databases', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
for (let page = 0; page < MAX_DATABASE_PAGES; page++) {
|
||||
const response = await fetch('https://api.notion.com/v1/search', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
'Notion-Version': '2022-06-28',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
filter: { value: 'database', property: 'object' },
|
||||
page_size: NOTION_PAGE_SIZE,
|
||||
...(startCursor ? { start_cursor: startCursor } : {}),
|
||||
}),
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Notion databases', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch Notion databases', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Notion databases', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
if (Array.isArray(data.results)) {
|
||||
results.push(...(data.results as Record<string, unknown>[]))
|
||||
}
|
||||
|
||||
if (!data.has_more || !data.next_cursor) {
|
||||
break
|
||||
}
|
||||
startCursor = data.next_cursor as string
|
||||
|
||||
if (page === MAX_DATABASE_PAGES - 1) {
|
||||
logger.warn('Notion databases search hit pagination cap; results may be incomplete', {
|
||||
maxPages: MAX_DATABASE_PAGES,
|
||||
fetched: results.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const databases = (data.results || []).map((db: Record<string, unknown>) => ({
|
||||
const databases = results.map((db) => ({
|
||||
id: db.id as string,
|
||||
name: extractTitleFromItem(db),
|
||||
}))
|
||||
@@ -78,7 +111,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
} catch (error) {
|
||||
logger.error('Error processing Notion databases request:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to retrieve Notion databases', details: (error as Error).message },
|
||||
{ error: 'Failed to retrieve Notion databases', details: getErrorMessage(error) },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { notionPagesSelectorContract } from '@/lib/api/contracts/selectors'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
@@ -12,6 +13,16 @@ const logger = createLogger('NotionPagesAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const NOTION_PAGE_SIZE = 100
|
||||
|
||||
/**
|
||||
* Notion's `POST /v1/search` returns at most `page_size` results per call and
|
||||
* exposes `has_more`/`next_cursor` for pagination. This caps the number of
|
||||
* pages drained so a tenant with a very large workspace cannot make this route
|
||||
* loop unbounded. With `NOTION_PAGE_SIZE` of 100 this covers up to 2,000 items.
|
||||
*/
|
||||
const MAX_NOTION_PAGES = 20
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
@@ -43,33 +54,55 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch('https://api.notion.com/v1/search', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
'Notion-Version': '2022-06-28',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
filter: { value: 'page', property: 'object' },
|
||||
page_size: 100,
|
||||
}),
|
||||
})
|
||||
const results: Record<string, unknown>[] = []
|
||||
let startCursor: string | undefined
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch Notion pages', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
for (let page = 0; page < MAX_NOTION_PAGES; page++) {
|
||||
const response = await fetch('https://api.notion.com/v1/search', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
'Notion-Version': '2022-06-28',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
filter: { value: 'page', property: 'object' },
|
||||
page_size: NOTION_PAGE_SIZE,
|
||||
...(startCursor ? { start_cursor: startCursor } : {}),
|
||||
}),
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Notion pages', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch Notion pages', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Notion pages', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
if (Array.isArray(data.results)) {
|
||||
results.push(...(data.results as Record<string, unknown>[]))
|
||||
}
|
||||
|
||||
if (!data.has_more || !data.next_cursor) {
|
||||
break
|
||||
}
|
||||
startCursor = data.next_cursor as string
|
||||
|
||||
if (page === MAX_NOTION_PAGES - 1) {
|
||||
logger.warn('Notion pages search hit pagination cap; results may be incomplete', {
|
||||
maxPages: MAX_NOTION_PAGES,
|
||||
fetched: results.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const pages = (data.results || []).map((page: Record<string, unknown>) => ({
|
||||
const pages = results.map((page) => ({
|
||||
id: page.id as string,
|
||||
name: extractTitleFromItem(page),
|
||||
}))
|
||||
@@ -78,7 +111,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
} catch (error) {
|
||||
logger.error('Error processing Notion pages request:', error)
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to retrieve Notion pages', details: (error as Error).message },
|
||||
{ error: 'Failed to retrieve Notion pages', details: getErrorMessage(error) },
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
|
||||
@@ -8,11 +8,21 @@ import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('OneDriveFilesAPI')
|
||||
|
||||
/**
|
||||
* Microsoft Graph paginates drive item collections via the `@odata.nextLink`
|
||||
* absolute URL in the response body. Request the largest page (`$top` caps at
|
||||
* 999) and drain following nextLink, bounded by a page cap.
|
||||
* See https://learn.microsoft.com/en-us/graph/paging
|
||||
*/
|
||||
const ONEDRIVE_FILES_PAGE_SIZE = 999
|
||||
const MAX_ONEDRIVE_FILES_PAGES = 20
|
||||
|
||||
/**
|
||||
* Get files (not folders) from Microsoft OneDrive
|
||||
*/
|
||||
@@ -71,7 +81,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
'$select',
|
||||
'id,name,file,webUrl,size,createdDateTime,lastModifiedDateTime,createdBy,thumbnails'
|
||||
)
|
||||
searchParams_new.append('$top', '50')
|
||||
searchParams_new.append('$top', String(ONEDRIVE_FILES_PAGE_SIZE))
|
||||
url = `https://graph.microsoft.com/v1.0/me/drive/root/search(q='${encodeURIComponent(query)}')?${searchParams_new.toString()}`
|
||||
} else {
|
||||
const searchParams_new = new URLSearchParams()
|
||||
@@ -79,34 +89,53 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
'$select',
|
||||
'id,name,file,folder,webUrl,size,createdDateTime,lastModifiedDateTime,createdBy,thumbnails'
|
||||
)
|
||||
searchParams_new.append('$top', '50')
|
||||
searchParams_new.append('$top', String(ONEDRIVE_FILES_PAGE_SIZE))
|
||||
url = `https://graph.microsoft.com/v1.0/me/drive/root/children?${searchParams_new.toString()}`
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Fetching files from Microsoft Graph`, { url })
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
const rawItems: MicrosoftGraphDriveItem[] = []
|
||||
let nextUrl: string | undefined = url
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
logger.error(`[${requestId}] Microsoft Graph API error`, {
|
||||
status: response.status,
|
||||
error: errorData.error?.message || 'Failed to fetch files from OneDrive',
|
||||
for (let page = 0; page < MAX_ONEDRIVE_FILES_PAGES && nextUrl; page++) {
|
||||
const response = await fetch(nextUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: errorData.error?.message || 'Failed to fetch files from OneDrive' },
|
||||
{ status: response.status }
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response
|
||||
.json()
|
||||
.catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
logger.error(`[${requestId}] Microsoft Graph API error`, {
|
||||
status: response.status,
|
||||
error: errorData.error?.message || 'Failed to fetch files from OneDrive',
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: errorData.error?.message || 'Failed to fetch files from OneDrive' },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
rawItems.push(...((data.value as MicrosoftGraphDriveItem[]) || []))
|
||||
|
||||
const nextLink = getGraphNextPageUrl(data)
|
||||
nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined
|
||||
|
||||
if (nextUrl && page === MAX_ONEDRIVE_FILES_PAGES - 1) {
|
||||
logger.warn(`[${requestId}] OneDrive files hit pagination cap; list may be incomplete`, {
|
||||
pages: MAX_ONEDRIVE_FILES_PAGES,
|
||||
collected: rawItems.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
logger.info(`[${requestId}] Received ${data.value?.length || 0} items from Microsoft Graph`)
|
||||
logger.info(`[${requestId}] Received ${rawItems.length} items from Microsoft Graph`)
|
||||
|
||||
const files = (data.value || [])
|
||||
const files = rawItems
|
||||
.filter((item: MicrosoftGraphDriveItem) => !!item.file && !item.folder)
|
||||
.map((file: MicrosoftGraphDriveItem) => ({
|
||||
id: file.id,
|
||||
@@ -129,7 +158,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
}))
|
||||
|
||||
logger.info(`[${requestId}] Returning ${files.length} files`, {
|
||||
totalItems: data.value?.length || 0,
|
||||
totalItems: rawItems.length,
|
||||
})
|
||||
|
||||
return NextResponse.json({ files }, { status: 200 })
|
||||
|
||||
@@ -8,11 +8,21 @@ import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('OneDriveFoldersAPI')
|
||||
|
||||
/**
|
||||
* Microsoft Graph paginates drive item collections via the `@odata.nextLink`
|
||||
* absolute URL in the response body. Request the largest page (`$top` caps at
|
||||
* 999) and drain following nextLink, bounded by a page cap.
|
||||
* See https://learn.microsoft.com/en-us/graph/paging
|
||||
*/
|
||||
const ONEDRIVE_FOLDERS_PAGE_SIZE = 999
|
||||
const MAX_ONEDRIVE_FOLDERS_PAGES = 20
|
||||
|
||||
/**
|
||||
* Get folders from Microsoft OneDrive
|
||||
*/
|
||||
@@ -60,28 +70,47 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 })
|
||||
}
|
||||
|
||||
let url = `https://graph.microsoft.com/v1.0/me/drive/root/children?$filter=folder ne null&$select=id,name,folder,webUrl,createdDateTime,lastModifiedDateTime&$top=50`
|
||||
let url = `https://graph.microsoft.com/v1.0/me/drive/root/children?$filter=folder ne null&$select=id,name,folder,webUrl,createdDateTime,lastModifiedDateTime&$top=${ONEDRIVE_FOLDERS_PAGE_SIZE}`
|
||||
|
||||
if (query) {
|
||||
url += `&$search="${encodeURIComponent(query)}"`
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
const rawItems: MicrosoftGraphDriveItem[] = []
|
||||
let nextUrl: string | undefined = url
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
return NextResponse.json(
|
||||
{ error: errorData.error?.message || 'Failed to fetch folders from OneDrive' },
|
||||
{ status: response.status }
|
||||
)
|
||||
for (let page = 0; page < MAX_ONEDRIVE_FOLDERS_PAGES && nextUrl; page++) {
|
||||
const response = await fetch(nextUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response
|
||||
.json()
|
||||
.catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
return NextResponse.json(
|
||||
{ error: errorData.error?.message || 'Failed to fetch folders from OneDrive' },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
rawItems.push(...((data.value as MicrosoftGraphDriveItem[]) || []))
|
||||
|
||||
const nextLink = getGraphNextPageUrl(data)
|
||||
nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined
|
||||
|
||||
if (nextUrl && page === MAX_ONEDRIVE_FOLDERS_PAGES - 1) {
|
||||
logger.warn(`[${requestId}] OneDrive folders hit pagination cap; list may be incomplete`, {
|
||||
pages: MAX_ONEDRIVE_FOLDERS_PAGES,
|
||||
collected: rawItems.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const folders = (data.value || [])
|
||||
const folders = rawItems
|
||||
.filter((item: MicrosoftGraphDriveItem) => item.folder)
|
||||
.map((folder: MicrosoftGraphDriveItem) => ({
|
||||
id: folder.id,
|
||||
|
||||
@@ -8,11 +8,21 @@ import { validateAlphanumericId } from '@/lib/core/security/input-validation'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('OutlookFoldersAPI')
|
||||
|
||||
/**
|
||||
* Microsoft Graph paginates `mailFolders` via the `@odata.nextLink` absolute
|
||||
* URL in the response body (default page size is ~10). Bound the drain so a
|
||||
* pathological account can't loop unbounded; `$top` is capped at 999 by Graph.
|
||||
* See https://learn.microsoft.com/en-us/graph/paging
|
||||
*/
|
||||
const OUTLOOK_FOLDERS_PAGE_SIZE = 999
|
||||
const MAX_OUTLOOK_FOLDERS_PAGES = 20
|
||||
|
||||
interface OutlookFolder {
|
||||
id: string
|
||||
displayName: string
|
||||
@@ -65,37 +75,53 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch('https://graph.microsoft.com/v1.0/me/mailFolders', {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
const folders: OutlookFolder[] = []
|
||||
let nextUrl: string | undefined =
|
||||
`https://graph.microsoft.com/v1.0/me/mailFolders?$top=${OUTLOOK_FOLDERS_PAGE_SIZE}`
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json()
|
||||
logger.error('Microsoft Graph API error getting folders', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
endpoint: 'https://graph.microsoft.com/v1.0/me/mailFolders',
|
||||
for (let page = 0; page < MAX_OUTLOOK_FOLDERS_PAGES && nextUrl; page++) {
|
||||
const response = await fetch(nextUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (response.status === 401) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Authentication failed. Please reconnect your Outlook account.',
|
||||
authRequired: true,
|
||||
},
|
||||
{ status: 401 }
|
||||
)
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json()
|
||||
logger.error('Microsoft Graph API error getting folders', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
endpoint: nextUrl,
|
||||
})
|
||||
|
||||
if (response.status === 401) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Authentication failed. Please reconnect your Outlook account.',
|
||||
authRequired: true,
|
||||
},
|
||||
{ status: 401 }
|
||||
)
|
||||
}
|
||||
|
||||
throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`)
|
||||
}
|
||||
|
||||
throw new Error(`Microsoft Graph API error: ${JSON.stringify(errorData)}`)
|
||||
}
|
||||
const data = await response.json()
|
||||
folders.push(...((data.value as OutlookFolder[]) || []))
|
||||
|
||||
const data = await response.json()
|
||||
const folders = data.value || []
|
||||
const nextLink = getGraphNextPageUrl(data)
|
||||
nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined
|
||||
|
||||
if (nextUrl && page === MAX_OUTLOOK_FOLDERS_PAGES - 1) {
|
||||
logger.warn('Outlook mailFolders hit pagination cap; folder list may be incomplete', {
|
||||
pages: MAX_OUTLOOK_FOLDERS_PAGES,
|
||||
collected: folders.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const transformedFolders = folders.map((folder: OutlookFolder) => ({
|
||||
id: folder.id,
|
||||
|
||||
@@ -11,6 +11,86 @@ const logger = createLogger('PipedrivePipelinesAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const PIPEDRIVE_PAGE_LIMIT = 500
|
||||
const PIPEDRIVE_MAX_PIPELINES_PAGES = 50
|
||||
|
||||
interface PipedrivePipeline {
|
||||
id: number
|
||||
name: string
|
||||
}
|
||||
|
||||
interface PipedrivePipelinesPage {
|
||||
data?: PipedrivePipeline[]
|
||||
additional_data?: {
|
||||
pagination?: {
|
||||
more_items_in_collection?: boolean
|
||||
next_start?: number
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists all Pipedrive pipelines using v1 offset pagination (`start`/`limit`),
|
||||
* following `additional_data.pagination.next_start` while
|
||||
* `more_items_in_collection` is true so the full set is returned. Bounded by
|
||||
* `PIPEDRIVE_MAX_PIPELINES_PAGES`; logs a warning rather than silently dropping
|
||||
* pipelines when the cap is hit.
|
||||
*/
|
||||
async function fetchAllPipelines(accessToken: string): Promise<PipedrivePipeline[]> {
|
||||
const pipelines: PipedrivePipeline[] = []
|
||||
let start = 0
|
||||
|
||||
for (let page = 0; page < PIPEDRIVE_MAX_PIPELINES_PAGES; page++) {
|
||||
const url = new URL('https://api.pipedrive.com/v1/pipelines')
|
||||
url.searchParams.set('start', String(start))
|
||||
url.searchParams.set('limit', String(PIPEDRIVE_PAGE_LIMIT))
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
throw new PipedriveFetchError(response.status, errorData)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as PipedrivePipelinesPage
|
||||
if (Array.isArray(data.data)) {
|
||||
pipelines.push(...data.data)
|
||||
}
|
||||
|
||||
const pagination = data.additional_data?.pagination
|
||||
if (!pagination?.more_items_in_collection || typeof pagination.next_start !== 'number') {
|
||||
return pipelines
|
||||
}
|
||||
start = pagination.next_start
|
||||
|
||||
if (page === PIPEDRIVE_MAX_PIPELINES_PAGES - 1) {
|
||||
logger.warn(
|
||||
'Pipedrive pipelines listing hit pagination cap; pipeline list may be incomplete',
|
||||
{
|
||||
pages: PIPEDRIVE_MAX_PIPELINES_PAGES,
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return pipelines
|
||||
}
|
||||
|
||||
class PipedriveFetchError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly details: unknown
|
||||
) {
|
||||
super('Failed to fetch Pipedrive pipelines')
|
||||
this.name = 'PipedriveFetchError'
|
||||
}
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
@@ -42,27 +122,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch('https://api.pipedrive.com/v1/pipelines', {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch Pipedrive pipelines', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Pipedrive pipelines', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
let allPipelines: PipedrivePipeline[]
|
||||
try {
|
||||
allPipelines = await fetchAllPipelines(accessToken)
|
||||
} catch (error) {
|
||||
if (error instanceof PipedriveFetchError) {
|
||||
logger.error('Failed to fetch Pipedrive pipelines', {
|
||||
status: error.status,
|
||||
error: error.details,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Pipedrive pipelines', details: error.details },
|
||||
{ status: error.status }
|
||||
)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const pipelines = (data.data || []).map((pipeline: { id: number; name: string }) => ({
|
||||
const pipelines = allPipelines.map((pipeline) => ({
|
||||
id: String(pipeline.id),
|
||||
name: pipeline.name,
|
||||
}))
|
||||
|
||||
@@ -7,11 +7,19 @@ import { validateSharePointSiteId } from '@/lib/core/security/input-validation'
|
||||
import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('SharePointListsAPI')
|
||||
|
||||
/**
|
||||
* Upper bound on Microsoft Graph pages drained when listing SharePoint lists.
|
||||
* Each page returns up to `$top=999` lists, so this caps the result set at
|
||||
* roughly 10k lists while preventing an unbounded server-side loop.
|
||||
*/
|
||||
const MAX_LISTS_PAGES = 10
|
||||
|
||||
interface SharePointList {
|
||||
id: string
|
||||
displayName: string
|
||||
@@ -60,24 +68,42 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const url = `https://graph.microsoft.com/v1.0/sites/${siteIdValidation.sanitized}/lists?$select=id,displayName,description,webUrl&$expand=list($select=hidden)&$top=100`
|
||||
let nextUrl: string | undefined =
|
||||
`https://graph.microsoft.com/v1.0/sites/${siteIdValidation.sanitized}/lists?$select=id,displayName,description,webUrl&$expand=list($select=hidden)&$top=999`
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
const rawLists: SharePointList[] = []
|
||||
for (let page = 0; page < MAX_LISTS_PAGES && nextUrl; page++) {
|
||||
const response = await fetch(nextUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
return NextResponse.json(
|
||||
{ error: errorData.error?.message || 'Failed to fetch lists from SharePoint' },
|
||||
{ status: response.status }
|
||||
)
|
||||
if (!response.ok) {
|
||||
const errorData = await response
|
||||
.json()
|
||||
.catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
return NextResponse.json(
|
||||
{ error: errorData.error?.message || 'Failed to fetch lists from SharePoint' },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
if (Array.isArray(data.value)) {
|
||||
rawLists.push(...data.value)
|
||||
}
|
||||
|
||||
const nextLink = getGraphNextPageUrl(data)
|
||||
nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined
|
||||
if (nextUrl && page === MAX_LISTS_PAGES - 1) {
|
||||
logger.warn(
|
||||
`[${requestId}] SharePoint lists pagination hit ${MAX_LISTS_PAGES}-page cap; result may be incomplete`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const lists = (data.value || [])
|
||||
const lists = rawLists
|
||||
.filter((list: SharePointList) => list.list?.hidden !== true)
|
||||
.map((list: SharePointList) => ({
|
||||
id: list.id,
|
||||
|
||||
@@ -7,11 +7,19 @@ import { generateRequestId } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
|
||||
import type { SharepointSite } from '@/tools/sharepoint/types'
|
||||
import { assertGraphNextPageUrl, getGraphNextPageUrl } from '@/tools/sharepoint/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('SharePointSitesAPI')
|
||||
|
||||
/**
|
||||
* Upper bound on Microsoft Graph pages drained when listing SharePoint sites.
|
||||
* Each page returns up to `$top=999` sites, so this caps the result set at
|
||||
* roughly 10k sites while preventing an unbounded server-side loop.
|
||||
*/
|
||||
const MAX_SITES_PAGES = 10
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
|
||||
@@ -45,24 +53,42 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
}
|
||||
|
||||
const searchQuery = query || '*'
|
||||
const url = `https://graph.microsoft.com/v1.0/sites?search=${encodeURIComponent(searchQuery)}&$select=id,name,displayName,webUrl,createdDateTime,lastModifiedDateTime&$top=50`
|
||||
let nextUrl: string | undefined =
|
||||
`https://graph.microsoft.com/v1.0/sites?search=${encodeURIComponent(searchQuery)}&$select=id,name,displayName,webUrl,createdDateTime,lastModifiedDateTime&$top=999`
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
const rawSites: SharepointSite[] = []
|
||||
for (let page = 0; page < MAX_SITES_PAGES && nextUrl; page++) {
|
||||
const response = await fetch(nextUrl, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
return NextResponse.json(
|
||||
{ error: errorData.error?.message || 'Failed to fetch sites from SharePoint' },
|
||||
{ status: response.status }
|
||||
)
|
||||
if (!response.ok) {
|
||||
const errorData = await response
|
||||
.json()
|
||||
.catch(() => ({ error: { message: 'Unknown error' } }))
|
||||
return NextResponse.json(
|
||||
{ error: errorData.error?.message || 'Failed to fetch sites from SharePoint' },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
if (Array.isArray(data.value)) {
|
||||
rawSites.push(...data.value)
|
||||
}
|
||||
|
||||
const nextLink = getGraphNextPageUrl(data)
|
||||
nextUrl = nextLink ? assertGraphNextPageUrl(nextLink) : undefined
|
||||
if (nextUrl && page === MAX_SITES_PAGES - 1) {
|
||||
logger.warn(
|
||||
`[${requestId}] SharePoint sites pagination hit ${MAX_SITES_PAGES}-page cap; result may be incomplete`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const sites = (data.value || []).map((site: SharepointSite) => ({
|
||||
const sites = rawSites.map((site: SharepointSite) => ({
|
||||
id: site.id,
|
||||
name: site.displayName || site.name,
|
||||
mimeType: 'application/vnd.microsoft.graph.site',
|
||||
|
||||
@@ -12,6 +12,9 @@ export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('SlackUsersAPI')
|
||||
|
||||
const SLACK_PAGE_LIMIT = 200
|
||||
const SLACK_MAX_USER_PAGES = 10
|
||||
|
||||
interface SlackUser {
|
||||
id: string
|
||||
name: string
|
||||
@@ -20,6 +23,11 @@ interface SlackUser {
|
||||
is_bot: boolean
|
||||
}
|
||||
|
||||
interface SlackUsersResult {
|
||||
members: SlackUser[]
|
||||
truncated: boolean
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
const requestId = generateRequestId()
|
||||
@@ -86,6 +94,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
}
|
||||
|
||||
const data = await fetchSlackUsers(accessToken)
|
||||
if (data.truncated) {
|
||||
logger.warn('users.list hit pagination cap; user list may be incomplete')
|
||||
}
|
||||
|
||||
const users = (data.members || [])
|
||||
.filter((user: SlackUser) => !user.deleted && !user.is_bot)
|
||||
@@ -134,27 +145,53 @@ async function fetchSlackUser(accessToken: string, userId: string) {
|
||||
return data
|
||||
}
|
||||
|
||||
async function fetchSlackUsers(accessToken: string) {
|
||||
const url = new URL('https://slack.com/api/users.list')
|
||||
url.searchParams.append('limit', '200')
|
||||
/**
|
||||
* Lists Slack workspace members, following `response_metadata.next_cursor` so
|
||||
* the full set is returned. Bounded by `SLACK_MAX_USER_PAGES`; sets `truncated`
|
||||
* rather than silently dropping members when the cap is hit.
|
||||
*/
|
||||
async function fetchSlackUsers(accessToken: string): Promise<SlackUsersResult> {
|
||||
const members: SlackUser[] = []
|
||||
let cursor: string | undefined
|
||||
let truncated = false
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
for (let page = 0; page < SLACK_MAX_USER_PAGES; page++) {
|
||||
const url = new URL('https://slack.com/api/users.list')
|
||||
url.searchParams.append('limit', String(SLACK_PAGE_LIMIT))
|
||||
if (cursor) {
|
||||
url.searchParams.append('cursor', cursor)
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Slack API error: ${response.status} ${response.statusText}`)
|
||||
const response = await fetch(url.toString(), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Slack API error: ${response.status} ${response.statusText}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
|
||||
if (!data.ok) {
|
||||
throw new Error(data.error || 'Failed to fetch users')
|
||||
}
|
||||
|
||||
if (Array.isArray(data.members)) {
|
||||
members.push(...data.members)
|
||||
}
|
||||
|
||||
cursor = data.response_metadata?.next_cursor?.trim() || undefined
|
||||
if (!cursor) {
|
||||
return { members, truncated }
|
||||
}
|
||||
if (page === SLACK_MAX_USER_PAGES - 1) {
|
||||
truncated = true
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
|
||||
if (!data.ok) {
|
||||
throw new Error(data.error || 'Failed to fetch users')
|
||||
}
|
||||
|
||||
return data
|
||||
return { members, truncated }
|
||||
}
|
||||
|
||||
@@ -12,6 +12,18 @@ export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('WealthboxItemsAPI')
|
||||
|
||||
/**
|
||||
* Wealthbox `GET /v1/contacts` paginates with `?page=` / `?per_page=`, starting
|
||||
* at page 1. Wealthbox documents no `per_page` maximum and its contacts response
|
||||
* carries no pagination `meta`, so termination relies on the short-page check:
|
||||
* we stop once a page returns fewer items than `WEALTHBOX_PAGE_SIZE` (the
|
||||
* `meta.total_pages` / `meta.current_page` check is a defensive fallback for if
|
||||
* Wealthbox ever adds that block). Bounded by `MAX_WEALTHBOX_PAGES` so a runaway
|
||||
* response can't loop forever.
|
||||
*/
|
||||
const WEALTHBOX_PAGE_SIZE = 50
|
||||
const MAX_WEALTHBOX_PAGES = 50
|
||||
|
||||
interface WealthboxItem {
|
||||
id: string
|
||||
name: string
|
||||
@@ -21,6 +33,15 @@ interface WealthboxItem {
|
||||
updatedAt: string
|
||||
}
|
||||
|
||||
interface WealthboxContactsPage {
|
||||
contacts?: Array<Record<string, unknown>>
|
||||
meta?: {
|
||||
total_count?: number
|
||||
total_pages?: number
|
||||
current_page?: number
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get items (notes, contacts, tasks) from Wealthbox
|
||||
*/
|
||||
@@ -69,63 +90,83 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
|
||||
}
|
||||
const endpoint = endpoints[type as keyof typeof endpoints]
|
||||
|
||||
const url = new URL(`https://api.crmworkspace.com/v1/${endpoint}`)
|
||||
|
||||
logger.info(`[${requestId}] Fetching ${type}s from Wealthbox`, {
|
||||
endpoint,
|
||||
url: url.toString(),
|
||||
hasQuery: !!query.trim(),
|
||||
})
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
const allContacts: Array<Record<string, unknown>> = []
|
||||
let page = 1
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error(
|
||||
`[${requestId}] Wealthbox API error: ${response.status} ${response.statusText}`,
|
||||
{
|
||||
error: errorText,
|
||||
endpoint,
|
||||
url: url.toString(),
|
||||
}
|
||||
)
|
||||
return NextResponse.json(
|
||||
{ error: `Failed to fetch ${type}s from Wealthbox` },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
for (; page <= MAX_WEALTHBOX_PAGES; page++) {
|
||||
const url = new URL(`https://api.crmworkspace.com/v1/${endpoint}`)
|
||||
url.searchParams.set('per_page', String(WEALTHBOX_PAGE_SIZE))
|
||||
url.searchParams.set('page', String(page))
|
||||
|
||||
const data = (await response.json()) as { contacts?: Array<Record<string, unknown>> } & Record<
|
||||
string,
|
||||
unknown
|
||||
>
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
logger.info(`[${requestId}] Wealthbox API raw response`, {
|
||||
type,
|
||||
status: response.status,
|
||||
dataKeys: Object.keys(data || {}),
|
||||
hasContacts: !!data.contacts,
|
||||
dataStructure: typeof data === 'object' ? Object.keys(data) : 'not an object',
|
||||
})
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error(
|
||||
`[${requestId}] Wealthbox API error: ${response.status} ${response.statusText}`,
|
||||
{
|
||||
error: errorText,
|
||||
endpoint,
|
||||
url: url.toString(),
|
||||
}
|
||||
)
|
||||
return NextResponse.json(
|
||||
{ error: `Failed to fetch ${type}s from Wealthbox` },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
let items: WealthboxItem[] = []
|
||||
const data = (await response.json()) as WealthboxContactsPage
|
||||
|
||||
if (type === 'contact') {
|
||||
const contacts = data.contacts || []
|
||||
if (!Array.isArray(contacts)) {
|
||||
logger.warn(`[${requestId}] Contacts is not an array`, {
|
||||
contacts,
|
||||
dataType: typeof contacts,
|
||||
})
|
||||
return NextResponse.json({ items: [] }, { status: 200 })
|
||||
break
|
||||
}
|
||||
|
||||
items = contacts.map((item) => {
|
||||
allContacts.push(...contacts)
|
||||
|
||||
const totalPages = data.meta?.total_pages
|
||||
const currentPage = data.meta?.current_page ?? page
|
||||
const reachedLastByMeta =
|
||||
typeof totalPages === 'number' && totalPages > 0 && currentPage >= totalPages
|
||||
const reachedLastByCount = contacts.length < WEALTHBOX_PAGE_SIZE
|
||||
|
||||
if (reachedLastByMeta || reachedLastByCount) {
|
||||
break
|
||||
}
|
||||
|
||||
if (page === MAX_WEALTHBOX_PAGES) {
|
||||
logger.warn(
|
||||
`[${requestId}] Wealthbox pagination hit MAX_WEALTHBOX_PAGES cap; contact list may be incomplete`,
|
||||
{ endpoint, maxPages: MAX_WEALTHBOX_PAGES }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`[${requestId}] Wealthbox API drained`, {
|
||||
type,
|
||||
pagesFetched: Math.min(page, MAX_WEALTHBOX_PAGES),
|
||||
totalContacts: allContacts.length,
|
||||
})
|
||||
|
||||
let items: WealthboxItem[] = []
|
||||
|
||||
if (type === 'contact') {
|
||||
items = allContacts.map((item) => {
|
||||
const firstName = typeof item.first_name === 'string' ? item.first_name : ''
|
||||
const lastName = typeof item.last_name === 'string' ? item.last_name : ''
|
||||
return {
|
||||
|
||||
@@ -12,6 +12,9 @@ const logger = createLogger('WebflowItemsAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
const WEBFLOW_PAGE_LIMIT = 100
|
||||
const WEBFLOW_MAX_ITEMS_PAGES = 50
|
||||
|
||||
interface WebflowItem {
|
||||
id: string
|
||||
fieldData?: {
|
||||
@@ -21,6 +24,74 @@ interface WebflowItem {
|
||||
}
|
||||
}
|
||||
|
||||
interface WebflowItemsPage {
|
||||
items?: WebflowItem[]
|
||||
pagination?: {
|
||||
total?: number
|
||||
limit?: number
|
||||
offset?: number
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists all items in a Webflow collection using `offset`/`limit` pagination
|
||||
* (limit capped at 100), advancing the numeric `offset` until the accumulated
|
||||
* count reaches `pagination.total` so the full set is returned. Bounded by
|
||||
* `WEBFLOW_MAX_ITEMS_PAGES`; logs a warning rather than silently dropping items
|
||||
* when the cap is hit.
|
||||
*/
|
||||
async function fetchAllItems(accessToken: string, collectionId: string): Promise<WebflowItem[]> {
|
||||
const items: WebflowItem[] = []
|
||||
let offset = 0
|
||||
|
||||
for (let page = 0; page < WEBFLOW_MAX_ITEMS_PAGES; page++) {
|
||||
const url = new URL(`https://api.webflow.com/v2/collections/${collectionId}/items`)
|
||||
url.searchParams.set('limit', String(WEBFLOW_PAGE_LIMIT))
|
||||
url.searchParams.set('offset', String(offset))
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
throw new WebflowFetchError(response.status, errorData)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as WebflowItemsPage
|
||||
const pageItems = data.items || []
|
||||
items.push(...pageItems)
|
||||
|
||||
const total = data.pagination?.total
|
||||
offset += pageItems.length
|
||||
if (pageItems.length === 0 || (typeof total === 'number' && items.length >= total)) {
|
||||
return items
|
||||
}
|
||||
|
||||
if (page === WEBFLOW_MAX_ITEMS_PAGES - 1) {
|
||||
logger.warn('Webflow items listing hit pagination cap; item list may be incomplete', {
|
||||
collectionId,
|
||||
pages: WEBFLOW_MAX_ITEMS_PAGES,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
class WebflowFetchError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly details: unknown
|
||||
) {
|
||||
super('Failed to fetch Webflow items')
|
||||
this.name = 'WebflowFetchError'
|
||||
}
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
const requestId = generateRequestId()
|
||||
@@ -61,32 +132,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch(
|
||||
`https://api.webflow.com/v2/collections/${collectionId}/items?limit=100`,
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
accept: 'application/json',
|
||||
},
|
||||
let items: WebflowItem[]
|
||||
try {
|
||||
items = await fetchAllItems(accessToken, collectionId)
|
||||
} catch (error) {
|
||||
if (error instanceof WebflowFetchError) {
|
||||
logger.error('Failed to fetch Webflow items', {
|
||||
status: error.status,
|
||||
error: error.details,
|
||||
collectionId,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Webflow items', details: error.details },
|
||||
{ status: error.status }
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch Webflow items', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
collectionId,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Webflow items', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
throw error
|
||||
}
|
||||
|
||||
const data = (await response.json()) as { items?: WebflowItem[] }
|
||||
const items = data.items || []
|
||||
|
||||
let formattedItems = items.map((item) => {
|
||||
const fieldData = item.fieldData || {}
|
||||
const name = fieldData.name || fieldData.title || fieldData.slug || item.id
|
||||
|
||||
@@ -11,6 +11,24 @@ const logger = createLogger('ZoomMeetingsAPI')
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
|
||||
/**
|
||||
* Zoom `GET /v2/users/me/meetings` returns `next_page_token`, which is passed
|
||||
* back as `?next_page_token=` until it comes back as an empty string. `page_size`
|
||||
* max is 300. Bounded by `MAX_ZOOM_PAGES` so a runaway response can't loop forever.
|
||||
*/
|
||||
const ZOOM_PAGE_SIZE = 300
|
||||
const MAX_ZOOM_PAGES = 50
|
||||
|
||||
interface ZoomMeeting {
|
||||
id: number
|
||||
topic: string
|
||||
}
|
||||
|
||||
interface ZoomMeetingsPage {
|
||||
meetings?: ZoomMeeting[]
|
||||
next_page_token?: string
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const requestId = generateRequestId()
|
||||
try {
|
||||
@@ -42,30 +60,57 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
}
|
||||
|
||||
const response = await fetch(
|
||||
'https://api.zoom.us/v2/users/me/meetings?page_size=300&type=scheduled',
|
||||
{
|
||||
const allMeetings: ZoomMeeting[] = []
|
||||
let nextPageToken = ''
|
||||
|
||||
for (let page = 0; page < MAX_ZOOM_PAGES; page++) {
|
||||
const url = new URL('https://api.zoom.us/v2/users/me/meetings')
|
||||
url.searchParams.set('page_size', String(ZOOM_PAGE_SIZE))
|
||||
url.searchParams.set('type', 'scheduled')
|
||||
if (nextPageToken) {
|
||||
url.searchParams.set('next_page_token', nextPageToken)
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch Zoom meetings', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Zoom meetings', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({}))
|
||||
logger.error('Failed to fetch Zoom meetings', {
|
||||
status: response.status,
|
||||
error: errorData,
|
||||
})
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to fetch Zoom meetings', details: errorData },
|
||||
{ status: response.status }
|
||||
)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as ZoomMeetingsPage
|
||||
if (Array.isArray(data.meetings)) {
|
||||
allMeetings.push(...data.meetings)
|
||||
}
|
||||
|
||||
nextPageToken = data.next_page_token?.trim() || ''
|
||||
if (!nextPageToken) {
|
||||
break
|
||||
}
|
||||
|
||||
if (page === MAX_ZOOM_PAGES - 1) {
|
||||
logger.warn(
|
||||
'Zoom meetings pagination hit MAX_ZOOM_PAGES cap; meeting list may be incomplete',
|
||||
{
|
||||
maxPages: MAX_ZOOM_PAGES,
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const meetings = (data.meetings || []).map((meeting: { id: number; topic: string }) => ({
|
||||
const meetings = allMeetings.map((meeting) => ({
|
||||
id: String(meeting.id),
|
||||
name: meeting.topic,
|
||||
}))
|
||||
|
||||
@@ -33,7 +33,7 @@ import {
|
||||
fetchOAuthCredentialDetail,
|
||||
fetchOAuthCredentials,
|
||||
} from '@/hooks/queries/oauth/oauth-credentials'
|
||||
import { getSelectorDefinition } from '@/hooks/selectors/registry'
|
||||
import { getSelectorDefinition, loadAllSelectorOptions } from '@/hooks/selectors/registry'
|
||||
import type { SelectorKey, SelectorOption } from '@/hooks/selectors/types'
|
||||
|
||||
export interface WorkflowSearchResolvedResource {
|
||||
@@ -374,7 +374,11 @@ export function useWorkflowSearchSelectorDetails(matches: WorkflowSearchMatch[])
|
||||
return definition.fetchById({ ...queryArgs, signal })
|
||||
}
|
||||
|
||||
const options = await definition.fetchList({ key: selectorKey, context, signal })
|
||||
const options = await loadAllSelectorOptions(definition, {
|
||||
key: selectorKey,
|
||||
context,
|
||||
signal,
|
||||
})
|
||||
return options.find((option) => option.id === match.rawValue) ?? null
|
||||
},
|
||||
enabled: Boolean(selectorKey && match.rawValue && baseEnabled),
|
||||
@@ -620,7 +624,7 @@ export function useWorkflowSearchSelectorReplacementOptions(matches: WorkflowSea
|
||||
return {
|
||||
queryKey: workflowSearchReplaceKeys.selectorReplacementOptions(selectorKey, contextKey),
|
||||
queryFn: ({ signal }: { signal: AbortSignal }) =>
|
||||
definition.fetchList({ ...queryArgs, signal }),
|
||||
loadAllSelectorOptions(definition, { ...queryArgs, signal }),
|
||||
enabled: Boolean(selectorKey && baseEnabled),
|
||||
staleTime: definition.staleTime ?? 60 * 1000,
|
||||
select: (options: SelectorOption[]): WorkflowSearchReplacementOption[] =>
|
||||
|
||||
@@ -9,6 +9,13 @@ function formatConfluenceSpaceLabel(space: { name: string; key: string; status?:
|
||||
return space.status === 'archived' ? `${base} — archived` : base
|
||||
}
|
||||
|
||||
function toSpaceOption(space: { name: string; key: string; status?: string }): {
|
||||
id: string
|
||||
label: string
|
||||
} {
|
||||
return { id: space.key, label: formatConfluenceSpaceLabel(space) }
|
||||
}
|
||||
|
||||
export const confluenceSelectors = {
|
||||
'confluence.spaces': {
|
||||
key: 'confluence.spaces',
|
||||
@@ -21,28 +28,10 @@ export const confluenceSelectors = {
|
||||
context.domain ?? 'none',
|
||||
],
|
||||
enabled: ({ context }) => Boolean(context.oauthCredential && context.domain),
|
||||
fetchList: async ({ context, signal }: SelectorQueryArgs) => {
|
||||
const credentialId = ensureCredential(context, 'confluence.spaces')
|
||||
const domain = ensureDomain(context, 'confluence.spaces')
|
||||
const collected: { id: string; label: string }[] = []
|
||||
let cursor: string | undefined
|
||||
do {
|
||||
const data = await requestJson(selectorContracts.confluenceSpacesSelectorContract, {
|
||||
body: {
|
||||
credential: credentialId,
|
||||
workflowId: context.workflowId,
|
||||
domain,
|
||||
cursor,
|
||||
},
|
||||
signal,
|
||||
})
|
||||
for (const space of data.spaces || []) {
|
||||
collected.push({ id: space.key, label: formatConfluenceSpaceLabel(space) })
|
||||
}
|
||||
cursor = data.nextCursor
|
||||
} while (cursor)
|
||||
return collected
|
||||
},
|
||||
/**
|
||||
* Drives pagination through {@link useSelectorOptions}, which drains every
|
||||
* page via this callback. No `fetchList` — the paged path supersedes it.
|
||||
*/
|
||||
fetchPage: async ({ context, cursor, signal }) => {
|
||||
const credentialId = ensureCredential(context, 'confluence.spaces')
|
||||
const domain = ensureDomain(context, 'confluence.spaces')
|
||||
@@ -56,10 +45,7 @@ export const confluenceSelectors = {
|
||||
signal,
|
||||
})
|
||||
return {
|
||||
items: (data.spaces || []).map((space) => ({
|
||||
id: space.key,
|
||||
label: formatConfluenceSpaceLabel(space),
|
||||
})),
|
||||
items: (data.spaces || []).map(toSpaceOption),
|
||||
nextCursor: data.nextCursor,
|
||||
}
|
||||
},
|
||||
@@ -83,7 +69,7 @@ export const confluenceSelectors = {
|
||||
})
|
||||
const space = (data.spaces || []).find((s) => s.key === detailId) ?? null
|
||||
if (!space) return null
|
||||
return { id: space.key, label: formatConfluenceSpaceLabel(space) }
|
||||
return toSpaceOption(space)
|
||||
},
|
||||
},
|
||||
'confluence.pages': {
|
||||
|
||||
@@ -3,6 +3,8 @@ import * as selectorContracts from '@/lib/api/contracts/selectors'
|
||||
import { ensureKnowledgeBase, SELECTOR_STALE } from '@/hooks/selectors/providers/shared'
|
||||
import type { SelectorDefinition, SelectorKey, SelectorQueryArgs } from '@/hooks/selectors/types'
|
||||
|
||||
const KNOWLEDGE_DOCUMENTS_PAGE_LIMIT = 100
|
||||
|
||||
export const knowledgeSelectors = {
|
||||
'knowledge.documents': {
|
||||
key: 'knowledge.documents',
|
||||
@@ -18,20 +20,32 @@ export const knowledgeSelectors = {
|
||||
search ?? '',
|
||||
],
|
||||
enabled: ({ context }) => Boolean(context.knowledgeBaseId),
|
||||
fetchList: async ({ context, search, signal }: SelectorQueryArgs) => {
|
||||
/**
|
||||
* Drives pagination through {@link useSelectorOptions}, which drains every
|
||||
* page via this callback. The `pagination.hasMore` flag from the route
|
||||
* decides when to stop; `nextCursor` encodes the next `offset`.
|
||||
*/
|
||||
fetchPage: async ({ context, search, cursor, signal }) => {
|
||||
const knowledgeBaseId = ensureKnowledgeBase(context)
|
||||
const offset = cursor ? Number(cursor) : 0
|
||||
const result = await requestJson(selectorContracts.listKnowledgeSelectorDocumentsContract, {
|
||||
params: { id: knowledgeBaseId },
|
||||
query: {
|
||||
limit: 100,
|
||||
limit: KNOWLEDGE_DOCUMENTS_PAGE_LIMIT,
|
||||
offset,
|
||||
search,
|
||||
},
|
||||
signal,
|
||||
})
|
||||
return result.data.documents.map((doc) => ({
|
||||
id: doc.id,
|
||||
label: doc.filename,
|
||||
}))
|
||||
const { pagination } = result.data
|
||||
const nextOffset = pagination.offset + pagination.limit
|
||||
return {
|
||||
items: result.data.documents.map((doc) => ({
|
||||
id: doc.id,
|
||||
label: doc.filename,
|
||||
})),
|
||||
nextCursor: pagination.hasMore ? String(nextOffset) : undefined,
|
||||
}
|
||||
},
|
||||
fetchById: async ({ context, detailId, signal }: SelectorQueryArgs) => {
|
||||
if (!detailId) return null
|
||||
|
||||
@@ -320,6 +320,7 @@ export const microsoftSelectors = {
|
||||
query: search,
|
||||
driveId: context.driveId,
|
||||
workflowId: context.workflowId,
|
||||
fileType: 'excel',
|
||||
},
|
||||
signal,
|
||||
})
|
||||
@@ -347,6 +348,7 @@ export const microsoftSelectors = {
|
||||
credentialId,
|
||||
query: search,
|
||||
workflowId: context.workflowId,
|
||||
fileType: 'word',
|
||||
},
|
||||
signal,
|
||||
})
|
||||
|
||||
@@ -61,7 +61,7 @@ describe('sim.workflows selector', () => {
|
||||
it('reads workflow options from the scoped workflow cache', async () => {
|
||||
const definition = getSelectorDefinition('sim.workflows')
|
||||
|
||||
const options = await definition.fetchList({
|
||||
const options = await definition.fetchList!({
|
||||
key: 'sim.workflows',
|
||||
context: { workspaceId: 'ws-1', excludeWorkflowId: 'wf-2' },
|
||||
})
|
||||
@@ -112,7 +112,7 @@ describe('sim.workflows selector', () => {
|
||||
})
|
||||
|
||||
const definition = getSelectorDefinition('sim.workflows')
|
||||
const options = await definition.fetchList({
|
||||
const options = await definition.fetchList!({
|
||||
key: 'sim.workflows',
|
||||
context: { workspaceId: 'ws-1' },
|
||||
})
|
||||
|
||||
@@ -21,7 +21,12 @@ import { trelloSelectors } from '@/hooks/selectors/providers/trello/selectors'
|
||||
import { wealthboxSelectors } from '@/hooks/selectors/providers/wealthbox/selectors'
|
||||
import { webflowSelectors } from '@/hooks/selectors/providers/webflow/selectors'
|
||||
import { zoomSelectors } from '@/hooks/selectors/providers/zoom/selectors'
|
||||
import type { SelectorDefinition, SelectorKey, SelectorOption } from '@/hooks/selectors/types'
|
||||
import type {
|
||||
SelectorDefinition,
|
||||
SelectorKey,
|
||||
SelectorOption,
|
||||
SelectorQueryArgs,
|
||||
} from '@/hooks/selectors/types'
|
||||
|
||||
export const selectorRegistry = {
|
||||
...airtableSelectors,
|
||||
@@ -57,6 +62,38 @@ export function getSelectorDefinition(key: SelectorKey): SelectorDefinition {
|
||||
return definition
|
||||
}
|
||||
|
||||
const MAX_LOAD_ALL_PAGES = 50
|
||||
|
||||
/**
|
||||
* Loads the complete option list for a selector outside the React Query hook —
|
||||
* for callers (search/replace, value resolution) that need every option in one
|
||||
* call. Uses `fetchList` when defined, otherwise drains `fetchPage` (bounded by
|
||||
* {@link MAX_LOAD_ALL_PAGES}). Returns an empty array for a selector that
|
||||
* provides neither.
|
||||
*/
|
||||
export async function loadAllSelectorOptions(
|
||||
definition: SelectorDefinition,
|
||||
args: SelectorQueryArgs
|
||||
): Promise<SelectorOption[]> {
|
||||
if (definition.fetchList) {
|
||||
return definition.fetchList(args)
|
||||
}
|
||||
|
||||
if (definition.fetchPage) {
|
||||
const items: SelectorOption[] = []
|
||||
let cursor: string | undefined
|
||||
for (let page = 0; page < MAX_LOAD_ALL_PAGES; page++) {
|
||||
const { items: pageItems, nextCursor } = await definition.fetchPage({ ...args, cursor })
|
||||
items.push(...pageItems)
|
||||
cursor = nextCursor
|
||||
if (!cursor) break
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
return []
|
||||
}
|
||||
|
||||
export function mergeOption(options: SelectorOption[], option?: SelectorOption | null) {
|
||||
if (!option) return options
|
||||
if (options.some((item) => item.id === option.id)) {
|
||||
|
||||
@@ -114,7 +114,12 @@ export interface SelectorDefinition {
|
||||
key: SelectorKey
|
||||
contracts?: readonly AnyApiRouteContract[]
|
||||
getQueryKey: (args: SelectorQueryArgs) => QueryKey
|
||||
fetchList: (args: SelectorQueryArgs) => Promise<SelectorOption[]>
|
||||
/**
|
||||
* Loads the full option list in a single call. Required unless `fetchPage` is
|
||||
* defined, in which case the hook drives pagination through `fetchPage` and
|
||||
* `fetchList` is never invoked — provide one or the other, not both.
|
||||
*/
|
||||
fetchList?: (args: SelectorQueryArgs) => Promise<SelectorOption[]>
|
||||
/**
|
||||
* Optional. When defined, the selector hook fetches one page at a time and
|
||||
* auto-drains remaining pages so the dropdown populates progressively.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useEffect, useMemo } from 'react'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { useInfiniteQuery, useQuery } from '@tanstack/react-query'
|
||||
import { extractEnvVarName, isEnvVarReference, isReference } from '@/executor/constants'
|
||||
import { usePersonalEnvironment } from '@/hooks/queries/environment'
|
||||
@@ -30,11 +31,26 @@ export interface SelectorOptionsResult {
|
||||
* for non-paginated selectors.
|
||||
*/
|
||||
hasMore: boolean
|
||||
/**
|
||||
* True when the paginated drain stopped at {@link MAX_AUTO_DRAIN_PAGES} with
|
||||
* pages still remaining, so the option list is a partial view. Always false
|
||||
* for non-paginated selectors.
|
||||
*/
|
||||
truncated: boolean
|
||||
error: Error | null
|
||||
}
|
||||
|
||||
const logger = createLogger('SelectorQuery')
|
||||
|
||||
const EMPTY_PAGE: SelectorPage = { items: [], nextCursor: undefined }
|
||||
|
||||
/**
|
||||
* Safety bound on the background auto-drain. Real dropdowns settle in a handful
|
||||
* of pages; this only trips for pathological result sets and prevents an
|
||||
* unbounded request loop when a provider keeps handing back cursors.
|
||||
*/
|
||||
const MAX_AUTO_DRAIN_PAGES = 50
|
||||
|
||||
export function useSelectorOptions(
|
||||
key: SelectorKey,
|
||||
args: SelectorHookArgs
|
||||
@@ -50,7 +66,8 @@ export function useSelectorOptions(
|
||||
|
||||
const flatQuery = useQuery<SelectorOption[]>({
|
||||
queryKey: definition.getQueryKey(queryArgs),
|
||||
queryFn: ({ signal }) => definition.fetchList({ ...queryArgs, signal }),
|
||||
queryFn: ({ signal }) =>
|
||||
definition.fetchList?.({ ...queryArgs, signal }) ?? Promise.resolve([]),
|
||||
enabled: !supportsPagination && isEnabled,
|
||||
staleTime: definition.staleTime ?? 30_000,
|
||||
})
|
||||
@@ -72,13 +89,33 @@ export function useSelectorOptions(
|
||||
})
|
||||
|
||||
const { hasNextPage, isFetchingNextPage, fetchNextPage, isError } = pagedQuery
|
||||
const pageCount = pagedQuery.data?.pages.length ?? 0
|
||||
const reachedDrainCap = pageCount >= MAX_AUTO_DRAIN_PAGES
|
||||
useEffect(() => {
|
||||
if (!supportsPagination) return
|
||||
if (isError) return
|
||||
if (reachedDrainCap) {
|
||||
if (hasNextPage) {
|
||||
logger.warn('Selector hit auto-drain cap; option list is truncated', {
|
||||
key,
|
||||
pages: pageCount,
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
if (hasNextPage && !isFetchingNextPage) {
|
||||
void fetchNextPage()
|
||||
}
|
||||
}, [supportsPagination, hasNextPage, isFetchingNextPage, isError, fetchNextPage])
|
||||
}, [
|
||||
supportsPagination,
|
||||
hasNextPage,
|
||||
isFetchingNextPage,
|
||||
isError,
|
||||
fetchNextPage,
|
||||
reachedDrainCap,
|
||||
pageCount,
|
||||
key,
|
||||
])
|
||||
|
||||
const pagedOptions = useMemo<SelectorOption[] | undefined>(() => {
|
||||
if (!supportsPagination) return undefined
|
||||
@@ -92,7 +129,8 @@ export function useSelectorOptions(
|
||||
isLoading: pagedQuery.isLoading,
|
||||
isFetching: pagedQuery.isFetching,
|
||||
isFetchingMore: pagedQuery.isFetchingNextPage,
|
||||
hasMore: pagedQuery.hasNextPage ?? false,
|
||||
hasMore: (pagedQuery.hasNextPage ?? false) && !reachedDrainCap,
|
||||
truncated: reachedDrainCap && (pagedQuery.hasNextPage ?? false),
|
||||
error: (pagedQuery.error as Error | null) ?? null,
|
||||
}
|
||||
}
|
||||
@@ -103,6 +141,7 @@ export function useSelectorOptions(
|
||||
isFetching: flatQuery.isFetching,
|
||||
isFetchingMore: false,
|
||||
hasMore: false,
|
||||
truncated: false,
|
||||
error: (flatQuery.error as Error | null) ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ const knowledgeDocumentParamsSchema = knowledgeDocumentsParamsSchema.extend({
|
||||
|
||||
const knowledgeDocumentsQuerySchema = z.object({
|
||||
limit: z.coerce.number().int().min(1).max(100).optional(),
|
||||
offset: z.coerce.number().int().min(0).optional(),
|
||||
search: optionalString,
|
||||
})
|
||||
|
||||
|
||||
@@ -38,10 +38,17 @@ export const microsoftExcelDrivesBodySchema = credentialWorkflowBodySchema.exten
|
||||
driveId: optionalString,
|
||||
})
|
||||
|
||||
/**
|
||||
* The `/api/auth/oauth/microsoft/files` route is shared by the
|
||||
* `microsoft.excel` and `microsoft.word` selectors. `fileType` lets the route
|
||||
* search for and filter to the correct Office document type; it is optional and
|
||||
* defaults to `excel` on the server for backward compatibility.
|
||||
*/
|
||||
export const microsoftFilesQuerySchema = credentialIdQuerySchema.extend({
|
||||
query: optionalString,
|
||||
driveId: optionalString,
|
||||
workflowId: optionalString,
|
||||
fileType: z.enum(['excel', 'word']).optional(),
|
||||
})
|
||||
|
||||
export const microsoftFileQuerySchema = credentialIdQuerySchema.extend({
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
|
||||
const logger = createLogger('GooglePagination')
|
||||
|
||||
/**
|
||||
* Thrown by {@link drainGooglePagedList} when a page request returns a non-OK
|
||||
* HTTP status. Carries the status and parsed error body so callers can shape
|
||||
* their existing error responses unchanged.
|
||||
*/
|
||||
export class GooglePageError extends Error {
|
||||
readonly status: number
|
||||
readonly body: unknown
|
||||
|
||||
constructor(status: number, body: unknown) {
|
||||
super(`Google API error: ${status}`)
|
||||
this.name = 'GooglePageError'
|
||||
this.status = status
|
||||
this.body = body
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of draining a token-paginated Google REST list endpoint.
|
||||
*/
|
||||
export interface GoogleDrainResult<T> {
|
||||
/** All items accumulated across every fetched page. */
|
||||
items: T[]
|
||||
/** True when the page cap was reached before the API stopped returning a token. */
|
||||
truncated: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for {@link drainGooglePagedList}.
|
||||
*/
|
||||
export interface DrainGooglePagedListOptions<T, R> {
|
||||
/**
|
||||
* Builds the request URL for a given page. `pageToken` is `undefined` for the
|
||||
* first page, then the value of the previous response's `nextPageToken`.
|
||||
*/
|
||||
buildUrl: (pageToken: string | undefined) => string
|
||||
/** Performs the HTTP request for a built URL. */
|
||||
fetch: (url: string) => Promise<Response>
|
||||
/** Parses an error body from a non-OK response (used to build {@link GooglePageError}). */
|
||||
parseError: (response: Response) => Promise<unknown>
|
||||
/** Extracts the array of items from a single page's JSON body. */
|
||||
getItems: (body: R) => T[] | undefined
|
||||
/** Reads the continuation token from a single page's JSON body. */
|
||||
getNextPageToken: (body: R) => string | undefined
|
||||
/** Maximum number of pages to fetch before stopping and flagging `truncated`. */
|
||||
maxPages: number
|
||||
/** Label used in the cap-reached warning log. */
|
||||
label: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Drains a token-paginated Google REST list endpoint, following each response's
|
||||
* `nextPageToken` until it is absent or the `maxPages` cap is hit.
|
||||
*
|
||||
* Mirrors the bounded-loop pattern used by the Slack channels selector: the loop
|
||||
* is bounded and emits a `logger.warn` (and sets `truncated`) when the cap is
|
||||
* reached rather than silently dropping items. A non-OK page response throws a
|
||||
* {@link GooglePageError} carrying the status and parsed body so callers preserve
|
||||
* their existing error-response shapes.
|
||||
*/
|
||||
export async function drainGooglePagedList<T, R>(
|
||||
options: DrainGooglePagedListOptions<T, R>
|
||||
): Promise<GoogleDrainResult<T>> {
|
||||
const {
|
||||
buildUrl,
|
||||
fetch: fetchPage,
|
||||
parseError,
|
||||
getItems,
|
||||
getNextPageToken,
|
||||
maxPages,
|
||||
label,
|
||||
} = options
|
||||
|
||||
const items: T[] = []
|
||||
let pageToken: string | undefined
|
||||
let truncated = false
|
||||
|
||||
for (let page = 0; page < maxPages; page++) {
|
||||
const response = await fetchPage(buildUrl(pageToken))
|
||||
|
||||
if (!response.ok) {
|
||||
throw new GooglePageError(response.status, await parseError(response))
|
||||
}
|
||||
|
||||
const body = (await response.json()) as R
|
||||
|
||||
const pageItems = getItems(body)
|
||||
if (pageItems?.length) {
|
||||
items.push(...pageItems)
|
||||
}
|
||||
|
||||
pageToken = getNextPageToken(body)?.trim() || undefined
|
||||
if (!pageToken) {
|
||||
return { items, truncated }
|
||||
}
|
||||
|
||||
if (page === maxPages - 1) {
|
||||
truncated = true
|
||||
logger.warn(`${label}: hit pagination cap of ${maxPages} pages; results may be incomplete`)
|
||||
}
|
||||
}
|
||||
|
||||
return { items, truncated }
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import { SELECTOR_TYPES_HYDRATION_REQUIRED, type SubBlockConfig } from '@/blocks
|
||||
import { CREDENTIAL_SET, isUuid } from '@/executor/constants'
|
||||
import { fetchCredentialSetById } from '@/hooks/queries/credential-sets'
|
||||
import { fetchOAuthCredentialDetail } from '@/hooks/queries/oauth/oauth-credentials'
|
||||
import { getSelectorDefinition } from '@/hooks/selectors/registry'
|
||||
import { getSelectorDefinition, loadAllSelectorOptions } from '@/hooks/selectors/registry'
|
||||
import { resolveSelectorForSubBlock } from '@/hooks/selectors/resolution'
|
||||
import type { SelectorContext, SelectorKey } from '@/hooks/selectors/types'
|
||||
import type { WorkflowState } from '@/stores/workflows/workflow/types'
|
||||
@@ -107,7 +107,7 @@ async function resolveSelectorValue(
|
||||
}
|
||||
}
|
||||
|
||||
const options = await definition.fetchList({
|
||||
const options = await loadAllSelectorOptions(definition, {
|
||||
key: selectorKey,
|
||||
context: selectorContext,
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user