improvement(confluence): expand scopes, persist canonical mode toggle (#4461)

* improvement(confluence): expand scopes, persist canonical mode toggle

* improvement(confluence): memoize persisted canonical modes parse

* fix(confluence): paginate space selector dropdown

Confluence v2 spaces endpoint caps at limit=250 per page. The selector
endpoint was making one request and silently dropping every space past
the first page, which is why some spaces only worked when entered as a
manual spaceKey. Now follows _links.next cursor up to 20 pages (5000
spaces).

* fix(confluence): include archived spaces in selector dropdown

Confluence v2 /spaces defaults to status=current and the status param
is a single-value enum, so archived spaces never surface. They synced
fine when entered manually as a spaceKey because the connector looks
up spaces via ?keys=<key> which ignores status. Now fetches current
and archived in parallel and tags archived ones in the dropdown label.

* improvement(confluence): stream paginated space selector results

Bake pagination support into the selector abstraction via an opt-in
fetchPage definition so dropdowns populate progressively instead of
blocking on a full page-walk. Confluence spaces now stream current
then archived in a single cursor sequence.
This commit is contained in:
Waleed
2026-05-05 20:11:28 -07:00
committed by GitHub
parent cef351fa25
commit 3a7928987e
11 changed files with 303 additions and 54 deletions
@@ -19,13 +19,33 @@ const logger = createLogger('ConfluenceSelectorSpacesAPI')
export const dynamic = 'force-dynamic'
const PAGE_LIMIT = 250
type SpaceStatus = 'current' | 'archived'
/**
* Cursor format: `<status>:<innerCursor>`. Empty inner cursor means "first page
* of that status". When current is exhausted we hand back `archived:` so the
* client transparently flips to the archived stream — listing both surfaces
* archived spaces in the dropdown, which would otherwise only be reachable by
* typing the space key manually even though sync works against archived spaces.
*/
function parseCursor(raw: string | undefined): { status: SpaceStatus; inner?: string } {
if (!raw) return { status: 'current' }
const idx = raw.indexOf(':')
if (idx === -1) return { status: 'current' }
const status = raw.slice(0, idx) === 'archived' ? 'archived' : 'current'
const inner = raw.slice(idx + 1)
return { status, inner: inner || undefined }
}
export const POST = withRouteHandler(async (request: NextRequest) => {
const requestId = generateRequestId()
try {
const parsed = await parseRequest(confluenceSpacesSelectorContract, request, {})
if (!parsed.success) return parsed.response
const { credential, workflowId, domain } = parsed.data.body
const { credential, workflowId, domain, cursor } = parsed.data.body
if (!credential) {
logger.error('Missing credential in request')
@@ -44,11 +64,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 })
}
// Resolve once so we know whether this is an Atlassian SA credential before
// doing any token / cloudId work. Atlassian SAs short-circuit the entire path:
// the API token IS the access token, and cloudId lives in the encrypted secret —
// so we skip refreshAccessTokenIfNeeded (avoids a redundant resolve+decrypt) and
// skip getConfluenceCloudId (which 401s for scoped SA tokens).
const resolved = await resolveOAuthAccountId(credential)
const isAtlassianServiceAccount =
resolved?.providerId === ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID && !!resolved.credentialId
@@ -83,27 +98,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 })
}
const url = `https://api.atlassian.com/ex/confluence/${cloudIdValidation.sanitized}/wiki/api/v2/spaces?limit=250`
const baseUrl = `https://api.atlassian.com/ex/confluence/${cloudIdValidation.sanitized}/wiki/api/v2/spaces`
const { status, inner } = parseCursor(cursor)
const params = new URLSearchParams({ limit: String(PAGE_LIMIT), status })
if (inner) params.set('cursor', inner)
const url = `${baseUrl}?${params.toString()}`
const response = await fetch(url, {
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` },
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Confluence API error response:', {
status: response.status,
statusText: response.statusText,
error: errorText,
})
return NextResponse.json(
{ error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) },
{ status: response.status }
)
const message = parseAtlassianErrorMessage(response.status, response.statusText, errorText)
logger.error('Confluence API error response', { error: message, status: response.status })
return NextResponse.json({ error: message }, { status: 502 })
}
const data = await response.json()
@@ -111,9 +122,27 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
id: space.id,
name: space.name,
key: space.key,
status,
}))
return NextResponse.json({ spaces })
let nextInner: string | undefined
const nextLink = data._links?.next as string | undefined
if (nextLink) {
try {
nextInner = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined
} catch {
nextInner = undefined
}
}
let nextCursor: string | undefined
if (nextInner) {
nextCursor = `${status}:${nextInner}`
} else if (status === 'current') {
nextCursor = 'archived:'
}
return NextResponse.json({ spaces, nextCursor })
} catch (error) {
logger.error('Error listing Confluence spaces:', error)
return NextResponse.json(
@@ -157,10 +157,13 @@ export function AddConnectorModal({
for (const [key, value] of Object.entries(resolveSourceConfig())) {
if (value) resolvedConfig[key] = value
}
const finalSourceConfig =
disabledTagIds.size > 0
? { ...resolvedConfig, disabledTagIds: Array.from(disabledTagIds) }
: resolvedConfig
if (disabledTagIds.size > 0) {
resolvedConfig.disabledTagIds = Array.from(disabledTagIds)
}
if (Object.keys(canonicalModes).length > 0) {
resolvedConfig._canonicalModes = canonicalModes
}
const finalSourceConfig = resolvedConfig
createConnector(
{
@@ -43,8 +43,33 @@ import type { SelectorKey } from '@/hooks/selectors/types'
const logger = createLogger('EditConnectorModal')
/** Keys injected by the sync engine — not user-editable */
const INTERNAL_CONFIG_KEYS = new Set(['tagSlotMapping', 'disabledTagIds'])
/** Keys injected by the sync engine or modal state — not user-editable */
const INTERNAL_CONFIG_KEYS = new Set(['tagSlotMapping', 'disabledTagIds', '_canonicalModes'])
const CANONICAL_MODES_KEY = '_canonicalModes'
function readPersistedCanonicalModes(
sourceConfig: Record<string, unknown>
): Record<string, 'basic' | 'advanced'> {
const raw = sourceConfig[CANONICAL_MODES_KEY]
if (!raw || typeof raw !== 'object') return {}
const result: Record<string, 'basic' | 'advanced'> = {}
for (const [key, value] of Object.entries(raw as Record<string, unknown>)) {
if (value === 'basic' || value === 'advanced') result[key] = value
}
return result
}
function didCanonicalModesChange(
current: Record<string, 'basic' | 'advanced'>,
persisted: Record<string, 'basic' | 'advanced'>
): boolean {
const keys = new Set([...Object.keys(persisted), ...Object.keys(current)])
for (const key of keys) {
if ((current[key] ?? 'basic') !== (persisted[key] ?? 'basic')) return true
}
return false
}
interface EditConnectorModalProps {
open: boolean
@@ -87,6 +112,10 @@ export function EditConnectorModal({
return config
})
const [initialCanonicalModes] = useState<Record<string, 'basic' | 'advanced'>>(() =>
readPersistedCanonicalModes(connector.sourceConfig)
)
const {
sourceConfig,
canonicalModes,
@@ -95,7 +124,11 @@ export function EditConnectorModal({
handleFieldChange,
toggleCanonicalMode,
resolveSourceConfig,
} = useConnectorConfigFields({ connectorConfig, initialSourceConfig })
} = useConnectorConfigFields({
connectorConfig,
initialSourceConfig,
initialCanonicalModes,
})
const { mutate: updateConnector, isPending: isSaving } = useUpdateConnector()
@@ -103,14 +136,27 @@ export function EditConnectorModal({
const subscriptionAccess = getSubscriptionAccessState(subscriptionResponse?.data)
const hasMaxAccess = !isBillingEnabled || subscriptionAccess.hasUsableMaxAccess
const persistedCanonicalModes = useMemo(
() => readPersistedCanonicalModes(connector.sourceConfig),
[connector.sourceConfig]
)
const hasChanges = useMemo(() => {
if (syncInterval !== connector.syncIntervalMinutes) return true
if (didCanonicalModesChange(canonicalModes, persistedCanonicalModes)) return true
const resolved = resolveSourceConfig()
for (const [key, value] of Object.entries(resolved)) {
if (String(connector.sourceConfig[key] ?? '') !== value) return true
}
return false
}, [resolveSourceConfig, syncInterval, connector.syncIntervalMinutes, connector.sourceConfig])
}, [
resolveSourceConfig,
syncInterval,
connector.syncIntervalMinutes,
connector.sourceConfig,
canonicalModes,
persistedCanonicalModes,
])
const handleSave = () => {
setError(null)
@@ -126,8 +172,17 @@ export function EditConnectorModal({
for (const [key, value] of Object.entries(resolved)) {
if (String(connector.sourceConfig[key] ?? '') !== value) changedEntries[key] = value
}
if (Object.keys(changedEntries).length > 0) {
updates.sourceConfig = { ...connector.sourceConfig, ...changedEntries }
const modesChanged = didCanonicalModesChange(canonicalModes, persistedCanonicalModes)
if (Object.keys(changedEntries).length > 0 || modesChanged) {
const next: Record<string, unknown> = { ...connector.sourceConfig, ...changedEntries }
if (Object.keys(canonicalModes).length > 0) {
next[CANONICAL_MODES_KEY] = canonicalModes
} else {
delete next[CANONICAL_MODES_KEY]
}
updates.sourceConfig = next
}
if (Object.keys(updates).length === 0) {
@@ -7,6 +7,7 @@ import type { ConnectorConfig, ConnectorConfigField } from '@/connectors/types'
export interface UseConnectorConfigFieldsOptions {
connectorConfig: ConnectorConfig | null
initialSourceConfig?: Record<string, string>
initialCanonicalModes?: Record<string, 'basic' | 'advanced'>
}
export interface UseConnectorConfigFieldsResult {
@@ -34,11 +35,14 @@ export interface UseConnectorConfigFieldsResult {
export function useConnectorConfigFields({
connectorConfig,
initialSourceConfig,
initialCanonicalModes,
}: UseConnectorConfigFieldsOptions): UseConnectorConfigFieldsResult {
const [sourceConfig, setSourceConfig] = useState<Record<string, string>>(
() => initialSourceConfig ?? {}
)
const [canonicalModes, setCanonicalModes] = useState<Record<string, 'basic' | 'advanced'>>({})
const [canonicalModes, setCanonicalModes] = useState<Record<string, 'basic' | 'advanced'>>(
() => initialCanonicalModes ?? {}
)
const canonicalGroups = useMemo(() => {
const groups = new Map<string, ConnectorConfigField[]>()
@@ -53,6 +53,7 @@ export function SelectorCombobox({
const {
data: options = [],
isLoading,
hasMore,
error,
} = useSelectorOptions(selectorKey, {
context: selectorContext,
@@ -67,6 +68,7 @@ export function SelectorCombobox({
Boolean(activeValue) &&
Boolean(missingOptionLabel) &&
!isLoading &&
!hasMore &&
!optionMap.get(activeValue!)
const selectedLabel = activeValue
? hasMissingOption
+12 -4
View File
@@ -4,7 +4,7 @@ import { ConfluenceIcon } from '@/components/icons'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils'
import { getConfluenceCloudId } from '@/tools/confluence/utils'
import { getConfluenceCloudId, normalizeConfluenceDomainHost } from '@/tools/confluence/utils'
const logger = createLogger('ConfluenceConnector')
@@ -141,7 +141,15 @@ export const confluenceConnector: ConnectorConfig = {
auth: {
mode: 'oauth',
provider: 'confluence',
requiredScopes: ['read:confluence-content.all', 'read:page:confluence', 'offline_access'],
requiredScopes: [
'read:confluence-content.all',
'read:page:confluence',
'read:blogpost:confluence',
'read:space:confluence',
'read:label:confluence',
'search:confluence',
'offline_access',
],
},
configFields: [
@@ -205,7 +213,7 @@ export const confluenceConnector: ConnectorConfig = {
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const domain = sourceConfig.domain as string
const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string)
const spaceKey = sourceConfig.spaceKey as string
const contentType = (sourceConfig.contentType as string) || 'page'
const labelFilter = (sourceConfig.labelFilter as string) || ''
@@ -269,7 +277,7 @@ export const confluenceConnector: ConnectorConfig = {
externalId: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocument | null> => {
const domain = sourceConfig.domain as string
const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string)
let cloudId = syncContext?.cloudId as string | undefined
if (!cloudId) {
cloudId = await getConfluenceCloudId(domain, accessToken)
@@ -4,6 +4,11 @@ import { fetchOAuthToken } from '@/hooks/selectors/helpers'
import { ensureCredential, ensureDomain, SELECTOR_STALE } from '@/hooks/selectors/providers/shared'
import type { SelectorDefinition, SelectorKey, SelectorQueryArgs } from '@/hooks/selectors/types'
function formatConfluenceSpaceLabel(space: { name: string; key: string; status?: string }): string {
const base = `${space.name} (${space.key})`
return space.status === 'archived' ? `${base} — archived` : base
}
export const confluenceSelectors = {
'confluence.spaces': {
key: 'confluence.spaces',
@@ -17,6 +22,28 @@ export const confluenceSelectors = {
],
enabled: ({ context }) => Boolean(context.oauthCredential && context.domain),
fetchList: async ({ context, signal }: SelectorQueryArgs) => {
const credentialId = ensureCredential(context, 'confluence.spaces')
const domain = ensureDomain(context, 'confluence.spaces')
const collected: { id: string; label: string }[] = []
let cursor: string | undefined
do {
const data = await requestJson(selectorContracts.confluenceSpacesSelectorContract, {
body: {
credential: credentialId,
workflowId: context.workflowId,
domain,
cursor,
},
signal,
})
for (const space of data.spaces || []) {
collected.push({ id: space.id, label: formatConfluenceSpaceLabel(space) })
}
cursor = data.nextCursor
} while (cursor)
return collected
},
fetchPage: async ({ context, cursor, signal }) => {
const credentialId = ensureCredential(context, 'confluence.spaces')
const domain = ensureDomain(context, 'confluence.spaces')
const data = await requestJson(selectorContracts.confluenceSpacesSelectorContract, {
@@ -24,14 +51,24 @@ export const confluenceSelectors = {
credential: credentialId,
workflowId: context.workflowId,
domain,
cursor,
},
signal,
})
return (data.spaces || []).map((space) => ({
id: space.id,
label: `${space.name} (${space.key})`,
}))
return {
items: (data.spaces || []).map((space) => ({
id: space.id,
label: formatConfluenceSpaceLabel(space),
})),
nextCursor: data.nextCursor,
}
},
/**
* Resolves a single space label. Hits only the first page — the dropdown's
* `fetchPage` stream populates the options cache for spaces beyond page 1,
* and `useSelectorOptionMap` merges them in. Walking all pages here would
* double API load since the stream is already running in parallel.
*/
fetchById: async ({ context, detailId, signal }: SelectorQueryArgs) => {
if (!detailId) return null
const credentialId = ensureCredential(context, 'confluence.spaces')
@@ -46,7 +83,7 @@ export const confluenceSelectors = {
})
const space = (data.spaces || []).find((s) => s.id === detailId) ?? null
if (!space) return null
return { id: space.id, label: `${space.name} (${space.key})` }
return { id: space.id, label: formatConfluenceSpaceLabel(space) }
},
},
'confluence.pages': {
+15
View File
@@ -100,11 +100,26 @@ export interface SelectorQueryArgs {
signal?: AbortSignal
}
export interface SelectorPage {
items: SelectorOption[]
nextCursor?: string
}
export interface SelectorPageArgs extends SelectorQueryArgs {
cursor?: string
}
export interface SelectorDefinition {
key: SelectorKey
contracts?: readonly AnyApiRouteContract[]
getQueryKey: (args: SelectorQueryArgs) => QueryKey
fetchList: (args: SelectorQueryArgs) => Promise<SelectorOption[]>
/**
* Optional. When defined, the selector hook fetches one page at a time and
* auto-drains remaining pages so the dropdown populates progressively.
* Returns `{ items, nextCursor }`; `nextCursor: undefined` ends the stream.
*/
fetchPage?: (args: SelectorPageArgs) => Promise<SelectorPage>
fetchById?: (args: SelectorQueryArgs) => Promise<SelectorOption | null>
enabled?: (args: SelectorQueryArgs) => boolean
staleTime?: number
+86 -6
View File
@@ -1,9 +1,14 @@
import { useMemo } from 'react'
import { useQuery } from '@tanstack/react-query'
import { useEffect, useMemo } from 'react'
import { useInfiniteQuery, useQuery } from '@tanstack/react-query'
import { extractEnvVarName, isEnvVarReference, isReference } from '@/executor/constants'
import { usePersonalEnvironment } from '@/hooks/queries/environment'
import { getSelectorDefinition, mergeOption } from '@/hooks/selectors/registry'
import type { SelectorKey, SelectorOption, SelectorQueryArgs } from '@/hooks/selectors/types'
import type {
SelectorKey,
SelectorOption,
SelectorPage,
SelectorQueryArgs,
} from '@/hooks/selectors/types'
interface SelectorHookArgs extends Omit<SelectorQueryArgs, 'key'> {
search?: string
@@ -11,7 +16,29 @@ interface SelectorHookArgs extends Omit<SelectorQueryArgs, 'key'> {
enabled?: boolean
}
export function useSelectorOptions(key: SelectorKey, args: SelectorHookArgs) {
export interface SelectorOptionsResult {
data: SelectorOption[] | undefined
isLoading: boolean
isFetching: boolean
/**
* True while paginated selectors are draining remaining pages in the
* background. Always false for non-paginated selectors.
*/
isFetchingMore: boolean
/**
* True when the paginated selector still has more pages queued. Always false
* for non-paginated selectors.
*/
hasMore: boolean
error: Error | null
}
const EMPTY_PAGE: SelectorPage = { items: [], nextCursor: undefined }
export function useSelectorOptions(
key: SelectorKey,
args: SelectorHookArgs
): SelectorOptionsResult {
const definition = getSelectorDefinition(key)
const queryArgs: SelectorQueryArgs = {
key,
@@ -19,12 +46,65 @@ export function useSelectorOptions(key: SelectorKey, args: SelectorHookArgs) {
search: args.search,
}
const isEnabled = args.enabled ?? (definition.enabled ? definition.enabled(queryArgs) : true)
return useQuery<SelectorOption[]>({
const supportsPagination = Boolean(definition.fetchPage)
const flatQuery = useQuery<SelectorOption[]>({
queryKey: definition.getQueryKey(queryArgs),
queryFn: ({ signal }) => definition.fetchList({ ...queryArgs, signal }),
enabled: isEnabled,
enabled: !supportsPagination && isEnabled,
staleTime: definition.staleTime ?? 30_000,
})
const pagedQuery = useInfiniteQuery<SelectorPage>({
queryKey: [...definition.getQueryKey(queryArgs), 'paged'],
queryFn: ({ pageParam, signal }) => {
if (!definition.fetchPage) return Promise.resolve(EMPTY_PAGE)
return definition.fetchPage({
...queryArgs,
cursor: pageParam as string | undefined,
signal,
})
},
getNextPageParam: (last) => last.nextCursor,
initialPageParam: undefined as string | undefined,
enabled: supportsPagination && isEnabled,
staleTime: definition.staleTime ?? 30_000,
})
const { hasNextPage, isFetchingNextPage, fetchNextPage, isError } = pagedQuery
useEffect(() => {
if (!supportsPagination) return
if (isError) return
if (hasNextPage && !isFetchingNextPage) {
void fetchNextPage()
}
}, [supportsPagination, hasNextPage, isFetchingNextPage, isError, fetchNextPage])
const pagedOptions = useMemo<SelectorOption[] | undefined>(() => {
if (!supportsPagination) return undefined
if (!pagedQuery.data) return undefined
return pagedQuery.data.pages.flatMap((page) => page.items)
}, [supportsPagination, pagedQuery.data])
if (supportsPagination) {
return {
data: pagedOptions,
isLoading: pagedQuery.isLoading,
isFetching: pagedQuery.isFetching,
isFetchingMore: pagedQuery.isFetchingNextPage,
hasMore: pagedQuery.hasNextPage ?? false,
error: (pagedQuery.error as Error | null) ?? null,
}
}
return {
data: flatQuery.data,
isLoading: flatQuery.isLoading,
isFetching: flatQuery.isFetching,
isFetchingMore: false,
hasMore: false,
error: (flatQuery.error as Error | null) ?? null,
}
}
export function useSelectorOptionDetail(
@@ -10,7 +10,12 @@ import { defineRouteContract } from '@/lib/api/contracts/types'
import { validateAlphanumericId } from '@/lib/core/security/input-validation'
const confluenceSpaceSchema = z
.object({ id: z.string(), name: z.string(), key: z.string() })
.object({
id: z.string(),
name: z.string(),
key: z.string(),
status: z.string().optional(),
})
.passthrough()
export const confluencePagesBodySchema = z.object({
@@ -354,10 +359,17 @@ const defineConfluenceGetContract = <TQuery extends z.ZodType>(path: string, que
},
})
export const confluenceSpacesSelectorBodySchema = credentialWorkflowDomainBodySchema.extend({
cursor: optionalString,
})
export const confluenceSpacesSelectorContract = definePostSelector(
'/api/tools/confluence/selector-spaces',
credentialWorkflowDomainBodySchema,
z.object({ spaces: z.array(confluenceSpaceSchema) })
confluenceSpacesSelectorBodySchema,
z.object({
spaces: z.array(confluenceSpaceSchema),
nextCursor: optionalString,
})
)
export const confluencePagesSelectorContract = definePostSelector(
+8 -4
View File
@@ -1,11 +1,15 @@
import type { RetryOptions } from '@/lib/knowledge/documents/utils'
import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
function normalizeDomain(domain: string): string {
return `https://${domain
/**
* Strips protocol and trailing slashes from a Confluence domain to produce
* a bare host (e.g. `yoursite.atlassian.net`).
*/
export function normalizeConfluenceDomainHost(domain: string): string {
return domain
.trim()
.replace(/^https?:\/\//i, '')
.replace(/\/+$/, '')}`.toLowerCase()
.replace(/\/+$/, '')
}
export async function getConfluenceCloudId(
@@ -31,7 +35,7 @@ export async function getConfluenceCloudId(
throw new Error('No Confluence resources found')
}
const normalized = normalizeDomain(domain)
const normalized = `https://${normalizeConfluenceDomainHost(domain)}`.toLowerCase()
const match = resources.find(
(r: { url: string }) => r.url.toLowerCase().replace(/\/+$/, '') === normalized
)